1//===--- CGExprScalar.cpp - Emit LLVM Code for Scalar Exprs ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit Expr nodes with scalar LLVM types as LLVM code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGCXXABI.h"
14#include "CGCleanup.h"
15#include "CGDebugInfo.h"
16#include "CGHLSLRuntime.h"
17#include "CGObjCRuntime.h"
18#include "CGOpenMPRuntime.h"
19#include "CGRecordLayout.h"
20#include "CodeGenFunction.h"
21#include "CodeGenModule.h"
22#include "ConstantEmitter.h"
23#include "TargetInfo.h"
24#include "TrapReasonBuilder.h"
25#include "clang/AST/ASTContext.h"
26#include "clang/AST/Attr.h"
27#include "clang/AST/DeclObjC.h"
28#include "clang/AST/Expr.h"
29#include "clang/AST/MatrixUtils.h"
30#include "clang/AST/ParentMapContext.h"
31#include "clang/AST/RecordLayout.h"
32#include "clang/AST/StmtVisitor.h"
33#include "clang/Basic/CodeGenOptions.h"
34#include "clang/Basic/DiagnosticTrap.h"
35#include "clang/Basic/TargetInfo.h"
36#include "llvm/ADT/APFixedPoint.h"
37#include "llvm/ADT/ScopeExit.h"
38#include "llvm/IR/Argument.h"
39#include "llvm/IR/CFG.h"
40#include "llvm/IR/Constants.h"
41#include "llvm/IR/DataLayout.h"
42#include "llvm/IR/DerivedTypes.h"
43#include "llvm/IR/FixedPointBuilder.h"
44#include "llvm/IR/Function.h"
45#include "llvm/IR/GEPNoWrapFlags.h"
46#include "llvm/IR/GetElementPtrTypeIterator.h"
47#include "llvm/IR/GlobalVariable.h"
48#include "llvm/IR/Intrinsics.h"
49#include "llvm/IR/IntrinsicsPowerPC.h"
50#include "llvm/IR/IntrinsicsWebAssembly.h"
51#include "llvm/IR/MatrixBuilder.h"
52#include "llvm/IR/Module.h"
53#include "llvm/Support/TypeSize.h"
54#include <cstdarg>
55#include <optional>
56
57using namespace clang;
58using namespace CodeGen;
59using llvm::Value;
60
61//===----------------------------------------------------------------------===//
62// Scalar Expression Emitter
63//===----------------------------------------------------------------------===//
64
65namespace llvm {
66extern cl::opt<bool> EnableSingleByteCoverage;
67} // namespace llvm
68
69namespace {
70
71/// Determine whether the given binary operation may overflow.
72/// Sets \p Result to the value of the operation for BO_Add, BO_Sub, BO_Mul,
73/// and signed BO_{Div,Rem}. For these opcodes, and for unsigned BO_{Div,Rem},
74/// the returned overflow check is precise. The returned value is 'true' for
75/// all other opcodes, to be conservative.
76bool mayHaveIntegerOverflow(llvm::ConstantInt *LHS, llvm::ConstantInt *RHS,
77 BinaryOperator::Opcode Opcode, bool Signed,
78 llvm::APInt &Result) {
79 // Assume overflow is possible, unless we can prove otherwise.
80 bool Overflow = true;
81 const auto &LHSAP = LHS->getValue();
82 const auto &RHSAP = RHS->getValue();
83 if (Opcode == BO_Add) {
84 Result = Signed ? LHSAP.sadd_ov(RHS: RHSAP, Overflow)
85 : LHSAP.uadd_ov(RHS: RHSAP, Overflow);
86 } else if (Opcode == BO_Sub) {
87 Result = Signed ? LHSAP.ssub_ov(RHS: RHSAP, Overflow)
88 : LHSAP.usub_ov(RHS: RHSAP, Overflow);
89 } else if (Opcode == BO_Mul) {
90 Result = Signed ? LHSAP.smul_ov(RHS: RHSAP, Overflow)
91 : LHSAP.umul_ov(RHS: RHSAP, Overflow);
92 } else if (Opcode == BO_Div || Opcode == BO_Rem) {
93 if (Signed && !RHS->isZero())
94 Result = LHSAP.sdiv_ov(RHS: RHSAP, Overflow);
95 else
96 return false;
97 }
98 return Overflow;
99}
100
101struct BinOpInfo {
102 Value *LHS;
103 Value *RHS;
104 QualType Ty; // Computation Type.
105 BinaryOperator::Opcode Opcode; // Opcode of BinOp to perform
106 FPOptions FPFeatures;
107 const Expr *E; // Entire expr, for error unsupported. May not be binop.
108
109 /// Check if the binop can result in integer overflow.
110 bool mayHaveIntegerOverflow() const {
111 // Without constant input, we can't rule out overflow.
112 auto *LHSCI = dyn_cast<llvm::ConstantInt>(Val: LHS);
113 auto *RHSCI = dyn_cast<llvm::ConstantInt>(Val: RHS);
114 if (!LHSCI || !RHSCI)
115 return true;
116
117 llvm::APInt Result;
118 return ::mayHaveIntegerOverflow(
119 LHS: LHSCI, RHS: RHSCI, Opcode, Signed: Ty->hasSignedIntegerRepresentation(), Result);
120 }
121
122 /// Check if the binop computes a division or a remainder.
123 bool isDivremOp() const {
124 return Opcode == BO_Div || Opcode == BO_Rem || Opcode == BO_DivAssign ||
125 Opcode == BO_RemAssign;
126 }
127
128 /// Check if the binop can result in an integer division by zero.
129 bool mayHaveIntegerDivisionByZero() const {
130 if (isDivremOp())
131 if (auto *CI = dyn_cast<llvm::ConstantInt>(Val: RHS))
132 return CI->isZero();
133 return true;
134 }
135
136 /// Check if the binop can result in a float division by zero.
137 bool mayHaveFloatDivisionByZero() const {
138 if (isDivremOp())
139 if (auto *CFP = dyn_cast<llvm::ConstantFP>(Val: RHS))
140 return CFP->isZero();
141 return true;
142 }
143
144 /// Check if at least one operand is a fixed point type. In such cases, this
145 /// operation did not follow usual arithmetic conversion and both operands
146 /// might not be of the same type.
147 bool isFixedPointOp() const {
148 // We cannot simply check the result type since comparison operations return
149 // an int.
150 if (const auto *BinOp = dyn_cast<BinaryOperator>(Val: E)) {
151 QualType LHSType = BinOp->getLHS()->getType();
152 QualType RHSType = BinOp->getRHS()->getType();
153 return LHSType->isFixedPointType() || RHSType->isFixedPointType();
154 }
155 if (const auto *UnOp = dyn_cast<UnaryOperator>(Val: E))
156 return UnOp->getSubExpr()->getType()->isFixedPointType();
157 return false;
158 }
159
160 /// Check if the RHS has a signed integer representation.
161 bool rhsHasSignedIntegerRepresentation() const {
162 if (const auto *BinOp = dyn_cast<BinaryOperator>(Val: E)) {
163 QualType RHSType = BinOp->getRHS()->getType();
164 return RHSType->hasSignedIntegerRepresentation();
165 }
166 return false;
167 }
168};
169
170static bool MustVisitNullValue(const Expr *E) {
171 // If a null pointer expression's type is the C++0x nullptr_t, then
172 // it's not necessarily a simple constant and it must be evaluated
173 // for its potential side effects.
174 return E->getType()->isNullPtrType();
175}
176
177/// If \p E is a widened promoted integer, get its base (unpromoted) type.
178static std::optional<QualType> getUnwidenedIntegerType(const ASTContext &Ctx,
179 const Expr *E) {
180 const Expr *Base = E->IgnoreImpCasts();
181 if (E == Base)
182 return std::nullopt;
183
184 QualType BaseTy = Base->getType();
185 if (!Ctx.isPromotableIntegerType(T: BaseTy) ||
186 Ctx.getTypeSize(T: BaseTy) >= Ctx.getTypeSize(T: E->getType()))
187 return std::nullopt;
188
189 return BaseTy;
190}
191
192/// Check if \p E is a widened promoted integer.
193static bool IsWidenedIntegerOp(const ASTContext &Ctx, const Expr *E) {
194 return getUnwidenedIntegerType(Ctx, E).has_value();
195}
196
197/// Consider OverflowBehaviorType and language options to calculate the final
198/// overflow behavior for an expression. There are no language options for
199/// unsigned overflow semantics so there is nothing to consider there.
200static LangOptions::OverflowBehaviorKind
201getOverflowBehaviorConsideringType(const CodeGenFunction &CGF,
202 const QualType Ty) {
203 const OverflowBehaviorType *OBT = Ty->getAs<OverflowBehaviorType>();
204 /// FIXME: Having two enums named `OverflowBehaviorKind` is not ideal, these
205 /// should be unified into one coherent enum that supports both unsigned and
206 /// signed overflow behavior semantics.
207 if (OBT) {
208 switch (OBT->getBehaviorKind()) {
209 case OverflowBehaviorType::OverflowBehaviorKind::Wrap:
210 return LangOptions::OverflowBehaviorKind::OB_Wrap;
211 case OverflowBehaviorType::OverflowBehaviorKind::Trap:
212 return LangOptions::OverflowBehaviorKind::OB_Trap;
213 }
214 llvm_unreachable("Unknown OverflowBehaviorKind");
215 }
216
217 if (Ty->isUnsignedIntegerType()) {
218 return LangOptions::OverflowBehaviorKind::OB_Unset;
219 }
220
221 switch (CGF.getLangOpts().getSignedOverflowBehavior()) {
222 case LangOptions::SignedOverflowBehaviorTy::SOB_Defined:
223 return LangOptions::OverflowBehaviorKind::OB_SignedAndDefined;
224 case LangOptions::SignedOverflowBehaviorTy::SOB_Undefined:
225 return LangOptions::OverflowBehaviorKind::OB_Unset;
226 case LangOptions::SignedOverflowBehaviorTy::SOB_Trapping:
227 return LangOptions::OverflowBehaviorKind::OB_Trap;
228 }
229 llvm_unreachable("Unknown SignedOverflowBehaviorTy");
230}
231
232/// Check if we can skip the overflow check for \p Op.
233static bool CanElideOverflowCheck(ASTContext &Ctx, const BinOpInfo &Op) {
234 assert((isa<UnaryOperator>(Op.E) || isa<BinaryOperator>(Op.E)) &&
235 "Expected a unary or binary operator");
236
237 // If the binop has constant inputs and we can prove there is no overflow,
238 // we can elide the overflow check.
239 if (!Op.mayHaveIntegerOverflow())
240 return true;
241
242 const UnaryOperator *UO = dyn_cast<UnaryOperator>(Val: Op.E);
243 if (UO && Ctx.isUnaryOverflowPatternExcluded(UO))
244 return true;
245
246 const auto *BO = dyn_cast<BinaryOperator>(Val: Op.E);
247 if (BO && BO->hasExcludedOverflowPattern())
248 return true;
249
250 if (Op.Ty.isWrapType())
251 return true;
252 if (Op.Ty.isTrapType())
253 return false;
254
255 if (Op.Ty->isSignedIntegerType() &&
256 Ctx.isTypeIgnoredBySanitizer(Mask: SanitizerKind::SignedIntegerOverflow,
257 Ty: Op.Ty)) {
258 return true;
259 }
260
261 if (Op.Ty->isUnsignedIntegerType() &&
262 Ctx.isTypeIgnoredBySanitizer(Mask: SanitizerKind::UnsignedIntegerOverflow,
263 Ty: Op.Ty)) {
264 return true;
265 }
266
267 // If a unary op has a widened operand, the op cannot overflow.
268 if (UO)
269 return !UO->canOverflow();
270
271 // We usually don't need overflow checks for binops with widened operands.
272 // Multiplication with promoted unsigned operands is a special case.
273 auto OptionalLHSTy = getUnwidenedIntegerType(Ctx, E: BO->getLHS());
274 if (!OptionalLHSTy)
275 return false;
276
277 auto OptionalRHSTy = getUnwidenedIntegerType(Ctx, E: BO->getRHS());
278 if (!OptionalRHSTy)
279 return false;
280
281 QualType LHSTy = *OptionalLHSTy;
282 QualType RHSTy = *OptionalRHSTy;
283
284 // This is the simple case: binops without unsigned multiplication, and with
285 // widened operands. No overflow check is needed here.
286 if ((Op.Opcode != BO_Mul && Op.Opcode != BO_MulAssign) ||
287 !LHSTy->isUnsignedIntegerType() || !RHSTy->isUnsignedIntegerType())
288 return true;
289
290 // For unsigned multiplication the overflow check can be elided if either one
291 // of the unpromoted types are less than half the size of the promoted type.
292 unsigned PromotedSize = Ctx.getTypeSize(T: Op.E->getType());
293 return (2 * Ctx.getTypeSize(T: LHSTy)) < PromotedSize ||
294 (2 * Ctx.getTypeSize(T: RHSTy)) < PromotedSize;
295}
296
297class ScalarExprEmitter
298 : public StmtVisitor<ScalarExprEmitter, Value*> {
299 CodeGenFunction &CGF;
300 CGBuilderTy &Builder;
301 bool IgnoreResultAssign;
302 llvm::LLVMContext &VMContext;
303public:
304
305 ScalarExprEmitter(CodeGenFunction &cgf, bool ira=false)
306 : CGF(cgf), Builder(CGF.Builder), IgnoreResultAssign(ira),
307 VMContext(cgf.getLLVMContext()) {
308 }
309
310 //===--------------------------------------------------------------------===//
311 // Utilities
312 //===--------------------------------------------------------------------===//
313
314 bool TestAndClearIgnoreResultAssign() {
315 bool I = IgnoreResultAssign;
316 IgnoreResultAssign = false;
317 return I;
318 }
319
320 llvm::Type *ConvertType(QualType T) { return CGF.ConvertType(T); }
321 LValue EmitLValue(const Expr *E) { return CGF.EmitLValue(E); }
322 LValue EmitCheckedLValue(const Expr *E, CodeGenFunction::TypeCheckKind TCK) {
323 return CGF.EmitCheckedLValue(E, TCK);
324 }
325
326 void EmitBinOpCheck(
327 ArrayRef<std::pair<Value *, SanitizerKind::SanitizerOrdinal>> Checks,
328 const BinOpInfo &Info);
329
330 Value *EmitLoadOfLValue(LValue LV, SourceLocation Loc) {
331 return CGF.EmitLoadOfLValue(V: LV, Loc).getScalarVal();
332 }
333
334 void EmitLValueAlignmentAssumption(const Expr *E, Value *V) {
335 const AlignValueAttr *AVAttr = nullptr;
336 if (const auto *DRE = dyn_cast<DeclRefExpr>(Val: E)) {
337 const ValueDecl *VD = DRE->getDecl();
338
339 if (VD->getType()->isReferenceType()) {
340 if (const auto *TTy =
341 VD->getType().getNonReferenceType()->getAs<TypedefType>())
342 AVAttr = TTy->getDecl()->getAttr<AlignValueAttr>();
343 } else {
344 // Assumptions for function parameters are emitted at the start of the
345 // function, so there is no need to repeat that here,
346 // unless the alignment-assumption sanitizer is enabled,
347 // then we prefer the assumption over alignment attribute
348 // on IR function param.
349 if (isa<ParmVarDecl>(Val: VD) && !CGF.SanOpts.has(K: SanitizerKind::Alignment))
350 return;
351
352 AVAttr = VD->getAttr<AlignValueAttr>();
353 }
354 }
355
356 if (!AVAttr)
357 if (const auto *TTy = E->getType()->getAs<TypedefType>())
358 AVAttr = TTy->getDecl()->getAttr<AlignValueAttr>();
359
360 if (!AVAttr)
361 return;
362
363 Value *AlignmentValue = CGF.EmitScalarExpr(E: AVAttr->getAlignment());
364 llvm::ConstantInt *AlignmentCI = cast<llvm::ConstantInt>(Val: AlignmentValue);
365 CGF.emitAlignmentAssumption(PtrValue: V, E, AssumptionLoc: AVAttr->getLocation(), Alignment: AlignmentCI);
366 }
367
368 /// EmitLoadOfLValue - Given an expression with complex type that represents a
369 /// value l-value, this method emits the address of the l-value, then loads
370 /// and returns the result.
371 Value *EmitLoadOfLValue(const Expr *E) {
372 Value *V = EmitLoadOfLValue(LV: EmitCheckedLValue(E, TCK: CodeGenFunction::TCK_Load),
373 Loc: E->getExprLoc());
374
375 EmitLValueAlignmentAssumption(E, V);
376 return V;
377 }
378
379 /// EmitConversionToBool - Convert the specified expression value to a
380 /// boolean (i1) truth value. This is equivalent to "Val != 0".
381 Value *EmitConversionToBool(Value *Src, QualType DstTy);
382
383 /// Emit a check that a conversion from a floating-point type does not
384 /// overflow.
385 void EmitFloatConversionCheck(Value *OrigSrc, QualType OrigSrcType,
386 Value *Src, QualType SrcType, QualType DstType,
387 llvm::Type *DstTy, SourceLocation Loc);
388
389 /// Known implicit conversion check kinds.
390 /// This is used for bitfield conversion checks as well.
391 /// Keep in sync with the enum of the same name in ubsan_handlers.h
392 enum ImplicitConversionCheckKind : unsigned char {
393 ICCK_IntegerTruncation = 0, // Legacy, was only used by clang 7.
394 ICCK_UnsignedIntegerTruncation = 1,
395 ICCK_SignedIntegerTruncation = 2,
396 ICCK_IntegerSignChange = 3,
397 ICCK_SignedIntegerTruncationOrSignChange = 4,
398 };
399
400 /// Emit a check that an [implicit] truncation of an integer does not
401 /// discard any bits. It is not UB, so we use the value after truncation.
402 void EmitIntegerTruncationCheck(Value *Src, QualType SrcType, Value *Dst,
403 QualType DstType, SourceLocation Loc,
404 bool OBTrapInvolved = false);
405
406 /// Emit a check that an [implicit] conversion of an integer does not change
407 /// the sign of the value. It is not UB, so we use the value after conversion.
408 /// NOTE: Src and Dst may be the exact same value! (point to the same thing)
409 void EmitIntegerSignChangeCheck(Value *Src, QualType SrcType, Value *Dst,
410 QualType DstType, SourceLocation Loc,
411 bool OBTrapInvolved = false);
412
413 /// Emit a conversion from the specified type to the specified destination
414 /// type, both of which are LLVM scalar types.
415 struct ScalarConversionOpts {
416 bool TreatBooleanAsSigned;
417 bool EmitImplicitIntegerTruncationChecks;
418 bool EmitImplicitIntegerSignChangeChecks;
419 /* Potential -fsanitize-undefined-ignore-overflow-pattern= */
420 bool PatternExcluded;
421
422 ScalarConversionOpts()
423 : TreatBooleanAsSigned(false),
424 EmitImplicitIntegerTruncationChecks(false),
425 EmitImplicitIntegerSignChangeChecks(false), PatternExcluded(false) {}
426
427 ScalarConversionOpts(clang::SanitizerSet SanOpts)
428 : TreatBooleanAsSigned(false),
429 EmitImplicitIntegerTruncationChecks(
430 SanOpts.hasOneOf(K: SanitizerKind::ImplicitIntegerTruncation)),
431 EmitImplicitIntegerSignChangeChecks(
432 SanOpts.has(K: SanitizerKind::ImplicitIntegerSignChange)),
433 PatternExcluded(false) {}
434 };
435 Value *EmitScalarCast(Value *Src, QualType SrcType, QualType DstType,
436 llvm::Type *SrcTy, llvm::Type *DstTy,
437 ScalarConversionOpts Opts);
438 Value *
439 EmitScalarConversion(Value *Src, QualType SrcTy, QualType DstTy,
440 SourceLocation Loc,
441 ScalarConversionOpts Opts = ScalarConversionOpts());
442
443 /// Convert between either a fixed point and other fixed point or fixed point
444 /// and an integer.
445 Value *EmitFixedPointConversion(Value *Src, QualType SrcTy, QualType DstTy,
446 SourceLocation Loc);
447
448 /// Emit a conversion from the specified complex type to the specified
449 /// destination type, where the destination type is an LLVM scalar type.
450 Value *EmitComplexToScalarConversion(CodeGenFunction::ComplexPairTy Src,
451 QualType SrcTy, QualType DstTy,
452 SourceLocation Loc);
453
454 /// EmitNullValue - Emit a value that corresponds to null for the given type.
455 Value *EmitNullValue(QualType Ty);
456
457 /// EmitFloatToBoolConversion - Perform an FP to boolean conversion.
458 Value *EmitFloatToBoolConversion(Value *V) {
459 // Compare against 0.0 for fp scalars.
460 llvm::Value *Zero = llvm::Constant::getNullValue(Ty: V->getType());
461 return Builder.CreateFCmpUNE(LHS: V, RHS: Zero, Name: "tobool");
462 }
463
464 /// EmitPointerToBoolConversion - Perform a pointer to boolean conversion.
465 Value *EmitPointerToBoolConversion(Value *V, QualType QT) {
466 Value *Zero = CGF.CGM.getNullPointer(T: cast<llvm::PointerType>(Val: V->getType()), QT);
467
468 return Builder.CreateICmpNE(LHS: V, RHS: Zero, Name: "tobool");
469 }
470
471 Value *EmitIntToBoolConversion(Value *V) {
472 // Because of the type rules of C, we often end up computing a
473 // logical value, then zero extending it to int, then wanting it
474 // as a logical value again. Optimize this common case.
475 if (llvm::ZExtInst *ZI = dyn_cast<llvm::ZExtInst>(Val: V)) {
476 if (ZI->getOperand(i_nocapture: 0)->getType() == Builder.getInt1Ty()) {
477 Value *Result = ZI->getOperand(i_nocapture: 0);
478 // If there aren't any more uses, zap the instruction to save space.
479 // Note that there can be more uses, for example if this
480 // is the result of an assignment.
481 if (ZI->use_empty())
482 ZI->eraseFromParent();
483 return Result;
484 }
485 }
486
487 return Builder.CreateIsNotNull(Arg: V, Name: "tobool");
488 }
489
490 //===--------------------------------------------------------------------===//
491 // Visitor Methods
492 //===--------------------------------------------------------------------===//
493
494 Value *Visit(Expr *E) {
495 ApplyDebugLocation DL(CGF, E);
496 return StmtVisitor<ScalarExprEmitter, Value*>::Visit(S: E);
497 }
498
499 Value *VisitStmt(Stmt *S) {
500 S->dump(OS&: llvm::errs(), Context: CGF.getContext());
501 llvm_unreachable("Stmt can't have complex result type!");
502 }
503 Value *VisitExpr(Expr *S);
504
505 Value *VisitConstantExpr(ConstantExpr *E) {
506 // A constant expression of type 'void' generates no code and produces no
507 // value.
508 if (E->getType()->isVoidType())
509 return nullptr;
510
511 if (Value *Result = ConstantEmitter(CGF).tryEmitConstantExpr(CE: E)) {
512 if (E->isGLValue()) {
513 // This was already converted to an rvalue when it was constant
514 // evaluated.
515 if (E->hasAPValueResult() && !E->getAPValueResult().isLValue())
516 return Result;
517 return CGF.EmitLoadOfScalar(
518 Addr: Address(Result, CGF.convertTypeForLoadStore(ASTTy: E->getType()),
519 CGF.getContext().getTypeAlignInChars(T: E->getType())),
520 /*Volatile*/ false, Ty: E->getType(), Loc: E->getExprLoc());
521 }
522 return Result;
523 }
524 return Visit(E: E->getSubExpr());
525 }
526 Value *VisitParenExpr(ParenExpr *PE) {
527 return Visit(E: PE->getSubExpr());
528 }
529 Value *VisitSubstNonTypeTemplateParmExpr(SubstNonTypeTemplateParmExpr *E) {
530 return Visit(E: E->getReplacement());
531 }
532 Value *VisitGenericSelectionExpr(GenericSelectionExpr *GE) {
533 return Visit(E: GE->getResultExpr());
534 }
535 Value *VisitCoawaitExpr(CoawaitExpr *S) {
536 return CGF.EmitCoawaitExpr(E: *S).getScalarVal();
537 }
538 Value *VisitCoyieldExpr(CoyieldExpr *S) {
539 return CGF.EmitCoyieldExpr(E: *S).getScalarVal();
540 }
541 Value *VisitUnaryCoawait(const UnaryOperator *E) {
542 return Visit(E: E->getSubExpr());
543 }
544
545 // Leaves.
546 Value *VisitIntegerLiteral(const IntegerLiteral *E) {
547 return Builder.getInt(AI: E->getValue());
548 }
549 Value *VisitFixedPointLiteral(const FixedPointLiteral *E) {
550 return Builder.getInt(AI: E->getValue());
551 }
552 Value *VisitFloatingLiteral(const FloatingLiteral *E) {
553 return llvm::ConstantFP::get(Context&: VMContext, V: E->getValue());
554 }
555 Value *VisitCharacterLiteral(const CharacterLiteral *E) {
556 // Character literals are always stored in an unsigned (even for signed
557 // char), so allow implicit truncation here.
558 return llvm::ConstantInt::get(Ty: ConvertType(T: E->getType()), V: E->getValue(),
559 /*IsSigned=*/false, /*ImplicitTrunc=*/true);
560 }
561 Value *VisitObjCBoolLiteralExpr(const ObjCBoolLiteralExpr *E) {
562 return llvm::ConstantInt::get(Ty: ConvertType(T: E->getType()), V: E->getValue());
563 }
564 Value *VisitCXXBoolLiteralExpr(const CXXBoolLiteralExpr *E) {
565 return llvm::ConstantInt::get(Ty: ConvertType(T: E->getType()), V: E->getValue());
566 }
567 Value *VisitCXXScalarValueInitExpr(const CXXScalarValueInitExpr *E) {
568 if (E->getType()->isVoidType())
569 return nullptr;
570
571 return EmitNullValue(Ty: E->getType());
572 }
573 Value *VisitGNUNullExpr(const GNUNullExpr *E) {
574 return EmitNullValue(Ty: E->getType());
575 }
576 Value *VisitOffsetOfExpr(OffsetOfExpr *E);
577 Value *VisitUnaryExprOrTypeTraitExpr(const UnaryExprOrTypeTraitExpr *E);
578 Value *VisitAddrLabelExpr(const AddrLabelExpr *E) {
579 llvm::Value *V = CGF.GetAddrOfLabel(L: E->getLabel());
580 return Builder.CreateBitCast(V, DestTy: ConvertType(T: E->getType()));
581 }
582
583 Value *VisitSizeOfPackExpr(SizeOfPackExpr *E) {
584 return llvm::ConstantInt::get(Ty: ConvertType(T: E->getType()),V: E->getPackLength());
585 }
586
587 Value *VisitPseudoObjectExpr(PseudoObjectExpr *E) {
588 return CGF.EmitPseudoObjectRValue(e: E).getScalarVal();
589 }
590
591 Value *VisitSYCLUniqueStableNameExpr(SYCLUniqueStableNameExpr *E);
592 Value *VisitEmbedExpr(EmbedExpr *E);
593
594 Value *VisitOpaqueValueExpr(OpaqueValueExpr *E) {
595 if (E->isGLValue())
596 return EmitLoadOfLValue(LV: CGF.getOrCreateOpaqueLValueMapping(e: E),
597 Loc: E->getExprLoc());
598
599 // Otherwise, assume the mapping is the scalar directly.
600 return CGF.getOrCreateOpaqueRValueMapping(e: E).getScalarVal();
601 }
602
603 Value *VisitOpenACCAsteriskSizeExpr(OpenACCAsteriskSizeExpr *E) {
604 llvm_unreachable("Codegen for this isn't defined/implemented");
605 }
606
607 // l-values.
608 Value *VisitDeclRefExpr(DeclRefExpr *E) {
609 if (CodeGenFunction::ConstantEmission Constant = CGF.tryEmitAsConstant(RefExpr: E))
610 return CGF.emitScalarConstant(Constant, E);
611 return EmitLoadOfLValue(E);
612 }
613
614 Value *VisitObjCSelectorExpr(ObjCSelectorExpr *E) {
615 return CGF.EmitObjCSelectorExpr(E);
616 }
617 Value *VisitObjCProtocolExpr(ObjCProtocolExpr *E) {
618 return CGF.EmitObjCProtocolExpr(E);
619 }
620 Value *VisitObjCIvarRefExpr(ObjCIvarRefExpr *E) {
621 return EmitLoadOfLValue(E);
622 }
623 Value *VisitObjCMessageExpr(ObjCMessageExpr *E) {
624 if (E->getMethodDecl() &&
625 E->getMethodDecl()->getReturnType()->isReferenceType())
626 return EmitLoadOfLValue(E);
627 return CGF.EmitObjCMessageExpr(E).getScalarVal();
628 }
629
630 Value *VisitObjCIsaExpr(ObjCIsaExpr *E) {
631 LValue LV = CGF.EmitObjCIsaExpr(E);
632 Value *V = CGF.EmitLoadOfLValue(V: LV, Loc: E->getExprLoc()).getScalarVal();
633 return V;
634 }
635
636 Value *VisitObjCAvailabilityCheckExpr(ObjCAvailabilityCheckExpr *E) {
637 VersionTuple Version = E->getVersion();
638
639 // If we're checking for a platform older than our minimum deployment
640 // target, we can fold the check away.
641 if (Version <= CGF.CGM.getTarget().getPlatformMinVersion())
642 return llvm::ConstantInt::get(Ty: Builder.getInt1Ty(), V: 1);
643
644 return CGF.EmitBuiltinAvailable(Version);
645 }
646
647 Value *VisitArraySubscriptExpr(ArraySubscriptExpr *E);
648 Value *VisitMatrixSingleSubscriptExpr(MatrixSingleSubscriptExpr *E);
649 Value *VisitMatrixSubscriptExpr(MatrixSubscriptExpr *E);
650 Value *VisitShuffleVectorExpr(ShuffleVectorExpr *E);
651 Value *VisitConvertVectorExpr(ConvertVectorExpr *E);
652 Value *VisitMemberExpr(MemberExpr *E);
653 Value *VisitExtVectorElementExpr(Expr *E) { return EmitLoadOfLValue(E); }
654 Value *VisitMatrixElementExpr(Expr *E) { return EmitLoadOfLValue(E); }
655 Value *VisitCompoundLiteralExpr(CompoundLiteralExpr *E) {
656 // Strictly speaking, we shouldn't be calling EmitLoadOfLValue, which
657 // transitively calls EmitCompoundLiteralLValue, here in C++ since compound
658 // literals aren't l-values in C++. We do so simply because that's the
659 // cleanest way to handle compound literals in C++.
660 // See the discussion here: https://reviews.llvm.org/D64464
661 return EmitLoadOfLValue(E);
662 }
663
664 Value *VisitInitListExpr(InitListExpr *E);
665
666 Value *VisitArrayInitIndexExpr(ArrayInitIndexExpr *E) {
667 assert(CGF.getArrayInitIndex() &&
668 "ArrayInitIndexExpr not inside an ArrayInitLoopExpr?");
669 return CGF.getArrayInitIndex();
670 }
671
672 Value *VisitImplicitValueInitExpr(const ImplicitValueInitExpr *E) {
673 return EmitNullValue(Ty: E->getType());
674 }
675 Value *VisitExplicitCastExpr(ExplicitCastExpr *E) {
676 CGF.CGM.EmitExplicitCastExprType(E, CGF: &CGF);
677 return VisitCastExpr(E);
678 }
679 Value *VisitCastExpr(CastExpr *E);
680
681 Value *VisitCallExpr(const CallExpr *E) {
682 if (E->getCallReturnType(Ctx: CGF.getContext())->isReferenceType())
683 return EmitLoadOfLValue(E);
684
685 Value *V = CGF.EmitCallExpr(E).getScalarVal();
686
687 EmitLValueAlignmentAssumption(E, V);
688 return V;
689 }
690
691 Value *VisitStmtExpr(const StmtExpr *E);
692
693 // Unary Operators.
694 Value *VisitUnaryPostDec(const UnaryOperator *E) {
695 LValue LV = EmitLValue(E: E->getSubExpr());
696 return EmitScalarPrePostIncDec(E, LV, isInc: false, isPre: false);
697 }
698 Value *VisitUnaryPostInc(const UnaryOperator *E) {
699 LValue LV = EmitLValue(E: E->getSubExpr());
700 return EmitScalarPrePostIncDec(E, LV, isInc: true, isPre: false);
701 }
702 Value *VisitUnaryPreDec(const UnaryOperator *E) {
703 LValue LV = EmitLValue(E: E->getSubExpr());
704 return EmitScalarPrePostIncDec(E, LV, isInc: false, isPre: true);
705 }
706 Value *VisitUnaryPreInc(const UnaryOperator *E) {
707 LValue LV = EmitLValue(E: E->getSubExpr());
708 return EmitScalarPrePostIncDec(E, LV, isInc: true, isPre: true);
709 }
710
711 llvm::Value *EmitIncDecConsiderOverflowBehavior(const UnaryOperator *E,
712 llvm::Value *InVal,
713 bool IsInc);
714
715 llvm::Value *EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV,
716 bool isInc, bool isPre);
717
718
719 Value *VisitUnaryAddrOf(const UnaryOperator *E) {
720 if (isa<MemberPointerType>(Val: E->getType())) // never sugared
721 return CGF.CGM.getMemberPointerConstant(e: E);
722
723 return EmitLValue(E: E->getSubExpr()).getPointer(CGF);
724 }
725 Value *VisitUnaryDeref(const UnaryOperator *E) {
726 if (E->getType()->isVoidType())
727 return Visit(E: E->getSubExpr()); // the actual value should be unused
728 return EmitLoadOfLValue(E);
729 }
730
731 Value *VisitUnaryPlus(const UnaryOperator *E,
732 QualType PromotionType = QualType());
733 Value *VisitPlus(const UnaryOperator *E, QualType PromotionType);
734 Value *VisitUnaryMinus(const UnaryOperator *E,
735 QualType PromotionType = QualType());
736 Value *VisitMinus(const UnaryOperator *E, QualType PromotionType);
737
738 Value *VisitUnaryNot (const UnaryOperator *E);
739 Value *VisitUnaryLNot (const UnaryOperator *E);
740 Value *VisitUnaryReal(const UnaryOperator *E,
741 QualType PromotionType = QualType());
742 Value *VisitReal(const UnaryOperator *E, QualType PromotionType);
743 Value *VisitUnaryImag(const UnaryOperator *E,
744 QualType PromotionType = QualType());
745 Value *VisitImag(const UnaryOperator *E, QualType PromotionType);
746 Value *VisitUnaryExtension(const UnaryOperator *E) {
747 return Visit(E: E->getSubExpr());
748 }
749
750 // C++
751 Value *VisitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *E) {
752 return EmitLoadOfLValue(E);
753 }
754 Value *VisitSourceLocExpr(SourceLocExpr *SLE) {
755 auto &Ctx = CGF.getContext();
756 APValue Evaluated =
757 SLE->EvaluateInContext(Ctx, DefaultExpr: CGF.CurSourceLocExprScope.getDefaultExpr());
758 return ConstantEmitter(CGF).emitAbstract(loc: SLE->getLocation(), value: Evaluated,
759 T: SLE->getType());
760 }
761
762 Value *VisitCXXDefaultArgExpr(CXXDefaultArgExpr *DAE) {
763 CodeGenFunction::CXXDefaultArgExprScope Scope(CGF, DAE);
764 return Visit(E: DAE->getExpr());
765 }
766 Value *VisitCXXDefaultInitExpr(CXXDefaultInitExpr *DIE) {
767 CodeGenFunction::CXXDefaultInitExprScope Scope(CGF, DIE);
768 return Visit(E: DIE->getExpr());
769 }
770 Value *VisitCXXThisExpr(CXXThisExpr *TE) {
771 return CGF.LoadCXXThis();
772 }
773
774 Value *VisitExprWithCleanups(ExprWithCleanups *E);
775 Value *VisitCXXNewExpr(const CXXNewExpr *E) {
776 return CGF.EmitCXXNewExpr(E);
777 }
778 Value *VisitCXXDeleteExpr(const CXXDeleteExpr *E) {
779 CGF.EmitCXXDeleteExpr(E);
780 return nullptr;
781 }
782
783 Value *VisitTypeTraitExpr(const TypeTraitExpr *E) {
784 if (E->isStoredAsBoolean())
785 return llvm::ConstantInt::get(Ty: ConvertType(T: E->getType()),
786 V: E->getBoolValue());
787 assert(E->getAPValue().isInt() && "APValue type not supported");
788 return llvm::ConstantInt::get(Ty: ConvertType(T: E->getType()),
789 V: E->getAPValue().getInt());
790 }
791
792 Value *VisitConceptSpecializationExpr(const ConceptSpecializationExpr *E) {
793 return Builder.getInt1(V: E->isSatisfied());
794 }
795
796 Value *VisitRequiresExpr(const RequiresExpr *E) {
797 return Builder.getInt1(V: E->isSatisfied());
798 }
799
800 Value *VisitArrayTypeTraitExpr(const ArrayTypeTraitExpr *E) {
801 return llvm::ConstantInt::get(Ty: ConvertType(T: E->getType()), V: E->getValue());
802 }
803
804 Value *VisitExpressionTraitExpr(const ExpressionTraitExpr *E) {
805 return llvm::ConstantInt::get(Ty: Builder.getInt1Ty(), V: E->getValue());
806 }
807
808 Value *VisitCXXPseudoDestructorExpr(const CXXPseudoDestructorExpr *E) {
809 // C++ [expr.pseudo]p1:
810 // The result shall only be used as the operand for the function call
811 // operator (), and the result of such a call has type void. The only
812 // effect is the evaluation of the postfix-expression before the dot or
813 // arrow.
814 CGF.EmitScalarExpr(E: E->getBase());
815 return nullptr;
816 }
817
818 Value *VisitCXXNullPtrLiteralExpr(const CXXNullPtrLiteralExpr *E) {
819 return EmitNullValue(Ty: E->getType());
820 }
821
822 Value *VisitCXXThrowExpr(const CXXThrowExpr *E) {
823 CGF.EmitCXXThrowExpr(E);
824 return nullptr;
825 }
826
827 Value *VisitCXXNoexceptExpr(const CXXNoexceptExpr *E) {
828 return Builder.getInt1(V: E->getValue());
829 }
830
831 // Binary Operators.
832 Value *EmitMul(const BinOpInfo &Ops) {
833 if (Ops.Ty->isSignedIntegerOrEnumerationType() ||
834 Ops.Ty->isUnsignedIntegerType()) {
835 const bool isSigned = Ops.Ty->isSignedIntegerOrEnumerationType();
836 const bool hasSan =
837 isSigned ? CGF.SanOpts.has(K: SanitizerKind::SignedIntegerOverflow)
838 : CGF.SanOpts.has(K: SanitizerKind::UnsignedIntegerOverflow);
839 switch (getOverflowBehaviorConsideringType(CGF, Ty: Ops.Ty)) {
840 case LangOptions::OB_Wrap:
841 return Builder.CreateMul(LHS: Ops.LHS, RHS: Ops.RHS, Name: "mul");
842 case LangOptions::OB_SignedAndDefined:
843 if (!hasSan)
844 return Builder.CreateMul(LHS: Ops.LHS, RHS: Ops.RHS, Name: "mul");
845 [[fallthrough]];
846 case LangOptions::OB_Unset:
847 if (!hasSan)
848 return isSigned ? Builder.CreateNSWMul(LHS: Ops.LHS, RHS: Ops.RHS, Name: "mul")
849 : Builder.CreateMul(LHS: Ops.LHS, RHS: Ops.RHS, Name: "mul");
850 [[fallthrough]];
851 case LangOptions::OB_Trap:
852 if (CanElideOverflowCheck(Ctx&: CGF.getContext(), Op: Ops))
853 return isSigned ? Builder.CreateNSWMul(LHS: Ops.LHS, RHS: Ops.RHS, Name: "mul")
854 : Builder.CreateMul(LHS: Ops.LHS, RHS: Ops.RHS, Name: "mul");
855 return EmitOverflowCheckedBinOp(Ops);
856 }
857 }
858
859 if (Ops.Ty->isConstantMatrixType()) {
860 llvm::MatrixBuilder MB(Builder);
861 // We need to check the types of the operands of the operator to get the
862 // correct matrix dimensions.
863 auto *BO = cast<BinaryOperator>(Val: Ops.E);
864 auto *LHSMatTy = dyn_cast<ConstantMatrixType>(
865 Val: BO->getLHS()->getType().getCanonicalType());
866 auto *RHSMatTy = dyn_cast<ConstantMatrixType>(
867 Val: BO->getRHS()->getType().getCanonicalType());
868 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, Ops.FPFeatures);
869 if (LHSMatTy && RHSMatTy)
870 return MB.CreateMatrixMultiply(LHS: Ops.LHS, RHS: Ops.RHS, LHSRows: LHSMatTy->getNumRows(),
871 LHSColumns: LHSMatTy->getNumColumns(),
872 RHSColumns: RHSMatTy->getNumColumns());
873 return MB.CreateScalarMultiply(LHS: Ops.LHS, RHS: Ops.RHS);
874 }
875
876 if (Ops.LHS->getType()->isFPOrFPVectorTy()) {
877 // Preserve the old values
878 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, Ops.FPFeatures);
879 return Builder.CreateFMul(L: Ops.LHS, R: Ops.RHS, Name: "mul");
880 }
881 if (Ops.isFixedPointOp())
882 return EmitFixedPointBinOp(Ops);
883 return Builder.CreateMul(LHS: Ops.LHS, RHS: Ops.RHS, Name: "mul");
884 }
885 /// Create a binary op that checks for overflow.
886 /// Currently only supports +, - and *.
887 Value *EmitOverflowCheckedBinOp(const BinOpInfo &Ops);
888
889 // Check for undefined division and modulus behaviors.
890 void EmitUndefinedBehaviorIntegerDivAndRemCheck(const BinOpInfo &Ops,
891 llvm::Value *Zero,bool isDiv);
892 // Common helper for getting how wide LHS of shift is.
893 static Value *GetMaximumShiftAmount(Value *LHS, Value *RHS, bool RHSIsSigned);
894
895 // Used for shifting constraints for OpenCL, do mask for powers of 2, URem for
896 // non powers of two.
897 Value *ConstrainShiftValue(Value *LHS, Value *RHS, const Twine &Name);
898
899 Value *EmitDiv(const BinOpInfo &Ops);
900 Value *EmitRem(const BinOpInfo &Ops);
901 Value *EmitAdd(const BinOpInfo &Ops);
902 Value *EmitSub(const BinOpInfo &Ops);
903 Value *EmitShl(const BinOpInfo &Ops);
904 Value *EmitShr(const BinOpInfo &Ops);
905 Value *EmitAnd(const BinOpInfo &Ops) {
906 return Builder.CreateAnd(LHS: Ops.LHS, RHS: Ops.RHS, Name: "and");
907 }
908 Value *EmitXor(const BinOpInfo &Ops) {
909 return Builder.CreateXor(LHS: Ops.LHS, RHS: Ops.RHS, Name: "xor");
910 }
911 Value *EmitOr (const BinOpInfo &Ops) {
912 return Builder.CreateOr(LHS: Ops.LHS, RHS: Ops.RHS, Name: "or");
913 }
914
915 // Helper functions for fixed point binary operations.
916 Value *EmitFixedPointBinOp(const BinOpInfo &Ops);
917
918 BinOpInfo EmitBinOps(const BinaryOperator *E,
919 QualType PromotionTy = QualType());
920
921 Value *EmitPromotedValue(Value *result, QualType PromotionType);
922 Value *EmitUnPromotedValue(Value *result, QualType ExprType);
923 Value *EmitPromoted(const Expr *E, QualType PromotionType);
924
925 LValue EmitCompoundAssignLValue(const CompoundAssignOperator *E,
926 Value *(ScalarExprEmitter::*F)(const BinOpInfo &),
927 Value *&Result);
928
929 Value *EmitCompoundAssign(const CompoundAssignOperator *E,
930 Value *(ScalarExprEmitter::*F)(const BinOpInfo &));
931
932 QualType getPromotionType(QualType Ty) {
933 const auto &Ctx = CGF.getContext();
934 if (auto *CT = Ty->getAs<ComplexType>()) {
935 QualType ElementType = CT->getElementType();
936 if (ElementType.UseExcessPrecision(Ctx))
937 return Ctx.getComplexType(T: Ctx.FloatTy);
938 }
939
940 if (Ty.UseExcessPrecision(Ctx)) {
941 if (auto *VT = Ty->getAs<VectorType>()) {
942 unsigned NumElements = VT->getNumElements();
943 return Ctx.getVectorType(VectorType: Ctx.FloatTy, NumElts: NumElements, VecKind: VT->getVectorKind());
944 }
945 return Ctx.FloatTy;
946 }
947
948 return QualType();
949 }
950
951 // Binary operators and binary compound assignment operators.
952#define HANDLEBINOP(OP) \
953 Value *VisitBin##OP(const BinaryOperator *E) { \
954 QualType promotionTy = getPromotionType(E->getType()); \
955 auto result = Emit##OP(EmitBinOps(E, promotionTy)); \
956 if (result && !promotionTy.isNull()) \
957 result = EmitUnPromotedValue(result, E->getType()); \
958 return result; \
959 } \
960 Value *VisitBin##OP##Assign(const CompoundAssignOperator *E) { \
961 ApplyAtomGroup Grp(CGF.getDebugInfo()); \
962 return EmitCompoundAssign(E, &ScalarExprEmitter::Emit##OP); \
963 }
964 HANDLEBINOP(Mul)
965 HANDLEBINOP(Div)
966 HANDLEBINOP(Rem)
967 HANDLEBINOP(Add)
968 HANDLEBINOP(Sub)
969 HANDLEBINOP(Shl)
970 HANDLEBINOP(Shr)
971 HANDLEBINOP(And)
972 HANDLEBINOP(Xor)
973 HANDLEBINOP(Or)
974#undef HANDLEBINOP
975
976 // Comparisons.
977 Value *EmitCompare(const BinaryOperator *E, llvm::CmpInst::Predicate UICmpOpc,
978 llvm::CmpInst::Predicate SICmpOpc,
979 llvm::CmpInst::Predicate FCmpOpc, bool IsSignaling);
980#define VISITCOMP(CODE, UI, SI, FP, SIG) \
981 Value *VisitBin##CODE(const BinaryOperator *E) { \
982 return EmitCompare(E, llvm::ICmpInst::UI, llvm::ICmpInst::SI, \
983 llvm::FCmpInst::FP, SIG); }
984 VISITCOMP(LT, ICMP_ULT, ICMP_SLT, FCMP_OLT, true)
985 VISITCOMP(GT, ICMP_UGT, ICMP_SGT, FCMP_OGT, true)
986 VISITCOMP(LE, ICMP_ULE, ICMP_SLE, FCMP_OLE, true)
987 VISITCOMP(GE, ICMP_UGE, ICMP_SGE, FCMP_OGE, true)
988 VISITCOMP(EQ, ICMP_EQ , ICMP_EQ , FCMP_OEQ, false)
989 VISITCOMP(NE, ICMP_NE , ICMP_NE , FCMP_UNE, false)
990#undef VISITCOMP
991
992 Value *VisitBinAssign (const BinaryOperator *E);
993
994 Value *VisitBinLAnd (const BinaryOperator *E);
995 Value *VisitBinLOr (const BinaryOperator *E);
996 Value *VisitBinComma (const BinaryOperator *E);
997
998 Value *VisitBinPtrMemD(const Expr *E) { return EmitLoadOfLValue(E); }
999 Value *VisitBinPtrMemI(const Expr *E) { return EmitLoadOfLValue(E); }
1000
1001 Value *VisitCXXRewrittenBinaryOperator(CXXRewrittenBinaryOperator *E) {
1002 return Visit(E: E->getSemanticForm());
1003 }
1004
1005 // Other Operators.
1006 Value *VisitBlockExpr(const BlockExpr *BE);
1007 Value *VisitAbstractConditionalOperator(const AbstractConditionalOperator *);
1008 Value *VisitChooseExpr(ChooseExpr *CE);
1009 Value *VisitVAArgExpr(VAArgExpr *VE);
1010 Value *VisitObjCStringLiteral(const ObjCStringLiteral *E) {
1011 return CGF.EmitObjCStringLiteral(E);
1012 }
1013 Value *VisitObjCBoxedExpr(ObjCBoxedExpr *E) {
1014 return CGF.EmitObjCBoxedExpr(E);
1015 }
1016 Value *VisitObjCArrayLiteral(ObjCArrayLiteral *E) {
1017 return CGF.EmitObjCArrayLiteral(E);
1018 }
1019 Value *VisitObjCDictionaryLiteral(ObjCDictionaryLiteral *E) {
1020 return CGF.EmitObjCDictionaryLiteral(E);
1021 }
1022 Value *VisitAsTypeExpr(AsTypeExpr *CE);
1023 Value *VisitAtomicExpr(AtomicExpr *AE);
1024 Value *VisitPackIndexingExpr(PackIndexingExpr *E) {
1025 return Visit(E: E->getSelectedExpr());
1026 }
1027};
1028} // end anonymous namespace.
1029
1030//===----------------------------------------------------------------------===//
1031// Utilities
1032//===----------------------------------------------------------------------===//
1033
1034/// EmitConversionToBool - Convert the specified expression value to a
1035/// boolean (i1) truth value. This is equivalent to "Val != 0".
1036Value *ScalarExprEmitter::EmitConversionToBool(Value *Src, QualType SrcType) {
1037 assert(SrcType.isCanonical() && "EmitScalarConversion strips typedefs");
1038
1039 if (SrcType->isRealFloatingType())
1040 return EmitFloatToBoolConversion(V: Src);
1041
1042 if (const MemberPointerType *MPT = dyn_cast<MemberPointerType>(Val&: SrcType))
1043 return CGF.CGM.getCXXABI().EmitMemberPointerIsNotNull(CGF, MemPtr: Src, MPT);
1044
1045 // The conversion is a NOP, and will be done when CodeGening the builtin.
1046 if (SrcType == CGF.getContext().AMDGPUFeaturePredicateTy)
1047 return Src;
1048
1049 assert((SrcType->isIntegerType() || isa<llvm::PointerType>(Src->getType())) &&
1050 "Unknown scalar type to convert");
1051
1052 if (isa<llvm::IntegerType>(Val: Src->getType()))
1053 return EmitIntToBoolConversion(V: Src);
1054
1055 assert(isa<llvm::PointerType>(Src->getType()));
1056 return EmitPointerToBoolConversion(V: Src, QT: SrcType);
1057}
1058
1059void ScalarExprEmitter::EmitFloatConversionCheck(
1060 Value *OrigSrc, QualType OrigSrcType, Value *Src, QualType SrcType,
1061 QualType DstType, llvm::Type *DstTy, SourceLocation Loc) {
1062 assert(SrcType->isFloatingType() && "not a conversion from floating point");
1063 if (!isa<llvm::IntegerType>(Val: DstTy))
1064 return;
1065
1066 auto CheckOrdinal = SanitizerKind::SO_FloatCastOverflow;
1067 auto CheckHandler = SanitizerHandler::FloatCastOverflow;
1068 SanitizerDebugLocation SanScope(&CGF, {CheckOrdinal}, CheckHandler);
1069 using llvm::APFloat;
1070 using llvm::APSInt;
1071
1072 llvm::Value *Check = nullptr;
1073 const llvm::fltSemantics &SrcSema =
1074 CGF.getContext().getFloatTypeSemantics(T: OrigSrcType);
1075
1076 // Floating-point to integer. This has undefined behavior if the source is
1077 // +-Inf, NaN, or doesn't fit into the destination type (after truncation
1078 // to an integer).
1079 unsigned Width = CGF.getContext().getIntWidth(T: DstType);
1080 bool Unsigned = DstType->isUnsignedIntegerOrEnumerationType();
1081
1082 APSInt Min = APSInt::getMinValue(numBits: Width, Unsigned);
1083 APFloat MinSrc(SrcSema, APFloat::uninitialized);
1084 if (MinSrc.convertFromAPInt(Input: Min, IsSigned: !Unsigned, RM: APFloat::rmTowardZero) &
1085 APFloat::opOverflow)
1086 // Don't need an overflow check for lower bound. Just check for
1087 // -Inf/NaN.
1088 MinSrc = APFloat::getInf(Sem: SrcSema, Negative: true);
1089 else
1090 // Find the largest value which is too small to represent (before
1091 // truncation toward zero).
1092 MinSrc.subtract(RHS: APFloat(SrcSema, 1), RM: APFloat::rmTowardNegative);
1093
1094 APSInt Max = APSInt::getMaxValue(numBits: Width, Unsigned);
1095 APFloat MaxSrc(SrcSema, APFloat::uninitialized);
1096 if (MaxSrc.convertFromAPInt(Input: Max, IsSigned: !Unsigned, RM: APFloat::rmTowardZero) &
1097 APFloat::opOverflow)
1098 // Don't need an overflow check for upper bound. Just check for
1099 // +Inf/NaN.
1100 MaxSrc = APFloat::getInf(Sem: SrcSema, Negative: false);
1101 else
1102 // Find the smallest value which is too large to represent (before
1103 // truncation toward zero).
1104 MaxSrc.add(RHS: APFloat(SrcSema, 1), RM: APFloat::rmTowardPositive);
1105
1106 // If we're converting from __half, convert the range to float to match
1107 // the type of src.
1108 if (OrigSrcType->isHalfType()) {
1109 const llvm::fltSemantics &Sema =
1110 CGF.getContext().getFloatTypeSemantics(T: SrcType);
1111 bool IsInexact;
1112 MinSrc.convert(ToSemantics: Sema, RM: APFloat::rmTowardZero, losesInfo: &IsInexact);
1113 MaxSrc.convert(ToSemantics: Sema, RM: APFloat::rmTowardZero, losesInfo: &IsInexact);
1114 }
1115
1116 llvm::Value *GE =
1117 Builder.CreateFCmpOGT(LHS: Src, RHS: llvm::ConstantFP::get(Context&: VMContext, V: MinSrc));
1118 llvm::Value *LE =
1119 Builder.CreateFCmpOLT(LHS: Src, RHS: llvm::ConstantFP::get(Context&: VMContext, V: MaxSrc));
1120 Check = Builder.CreateAnd(LHS: GE, RHS: LE);
1121
1122 llvm::Constant *StaticArgs[] = {CGF.EmitCheckSourceLocation(Loc),
1123 CGF.EmitCheckTypeDescriptor(T: OrigSrcType),
1124 CGF.EmitCheckTypeDescriptor(T: DstType)};
1125 CGF.EmitCheck(Checked: std::make_pair(x&: Check, y&: CheckOrdinal), Check: CheckHandler, StaticArgs,
1126 DynamicArgs: OrigSrc);
1127}
1128
1129// Should be called within CodeGenFunction::SanitizerScope RAII scope.
1130// Returns 'i1 false' when the truncation Src -> Dst was lossy.
1131static std::pair<ScalarExprEmitter::ImplicitConversionCheckKind,
1132 std::pair<llvm::Value *, SanitizerKind::SanitizerOrdinal>>
1133EmitIntegerTruncationCheckHelper(Value *Src, QualType SrcType, Value *Dst,
1134 QualType DstType, CGBuilderTy &Builder) {
1135 llvm::Type *SrcTy = Src->getType();
1136 llvm::Type *DstTy = Dst->getType();
1137 (void)DstTy; // Only used in assert()
1138
1139 // This should be truncation of integral types.
1140 assert(Src != Dst);
1141 assert(SrcTy->getScalarSizeInBits() > Dst->getType()->getScalarSizeInBits());
1142 assert(isa<llvm::IntegerType>(SrcTy) && isa<llvm::IntegerType>(DstTy) &&
1143 "non-integer llvm type");
1144
1145 bool SrcSigned = SrcType->isSignedIntegerOrEnumerationType();
1146 bool DstSigned = DstType->isSignedIntegerOrEnumerationType();
1147
1148 // If both (src and dst) types are unsigned, then it's an unsigned truncation.
1149 // Else, it is a signed truncation.
1150 ScalarExprEmitter::ImplicitConversionCheckKind Kind;
1151 SanitizerKind::SanitizerOrdinal Ordinal;
1152 if (!SrcSigned && !DstSigned) {
1153 Kind = ScalarExprEmitter::ICCK_UnsignedIntegerTruncation;
1154 Ordinal = SanitizerKind::SO_ImplicitUnsignedIntegerTruncation;
1155 } else {
1156 Kind = ScalarExprEmitter::ICCK_SignedIntegerTruncation;
1157 Ordinal = SanitizerKind::SO_ImplicitSignedIntegerTruncation;
1158 }
1159
1160 llvm::Value *Check = nullptr;
1161 // 1. Extend the truncated value back to the same width as the Src.
1162 Check = Builder.CreateIntCast(V: Dst, DestTy: SrcTy, isSigned: DstSigned, Name: "anyext");
1163 // 2. Equality-compare with the original source value
1164 Check = Builder.CreateICmpEQ(LHS: Check, RHS: Src, Name: "truncheck");
1165 // If the comparison result is 'i1 false', then the truncation was lossy.
1166 return std::make_pair(x&: Kind, y: std::make_pair(x&: Check, y&: Ordinal));
1167}
1168
1169static bool PromotionIsPotentiallyEligibleForImplicitIntegerConversionCheck(
1170 QualType SrcType, QualType DstType) {
1171 return SrcType->isIntegerType() && DstType->isIntegerType();
1172}
1173
1174void ScalarExprEmitter::EmitIntegerTruncationCheck(Value *Src, QualType SrcType,
1175 Value *Dst, QualType DstType,
1176 SourceLocation Loc,
1177 bool OBTrapInvolved) {
1178 if (!CGF.SanOpts.hasOneOf(K: SanitizerKind::ImplicitIntegerTruncation) &&
1179 !OBTrapInvolved)
1180 return;
1181
1182 // We only care about int->int conversions here.
1183 // We ignore conversions to/from pointer and/or bool.
1184 if (!PromotionIsPotentiallyEligibleForImplicitIntegerConversionCheck(SrcType,
1185 DstType))
1186 return;
1187
1188 unsigned SrcBits = Src->getType()->getScalarSizeInBits();
1189 unsigned DstBits = Dst->getType()->getScalarSizeInBits();
1190 // This must be truncation. Else we do not care.
1191 if (SrcBits <= DstBits)
1192 return;
1193
1194 assert(!DstType->isBooleanType() && "we should not get here with booleans.");
1195
1196 // If the integer sign change sanitizer is enabled,
1197 // and we are truncating from larger unsigned type to smaller signed type,
1198 // let that next sanitizer deal with it.
1199 bool SrcSigned = SrcType->isSignedIntegerOrEnumerationType();
1200 bool DstSigned = DstType->isSignedIntegerOrEnumerationType();
1201 if (CGF.SanOpts.has(K: SanitizerKind::ImplicitIntegerSignChange) &&
1202 (!SrcSigned && DstSigned))
1203 return;
1204
1205 std::pair<ScalarExprEmitter::ImplicitConversionCheckKind,
1206 std::pair<llvm::Value *, SanitizerKind::SanitizerOrdinal>>
1207 Check;
1208
1209 auto CheckHandler = SanitizerHandler::ImplicitConversion;
1210 {
1211 // We don't know the check kind until we call
1212 // EmitIntegerTruncationCheckHelper, but we want to annotate
1213 // EmitIntegerTruncationCheckHelper's instructions too.
1214 SanitizerDebugLocation SanScope(
1215 &CGF,
1216 {SanitizerKind::SO_ImplicitUnsignedIntegerTruncation,
1217 SanitizerKind::SO_ImplicitSignedIntegerTruncation},
1218 CheckHandler);
1219 Check =
1220 EmitIntegerTruncationCheckHelper(Src, SrcType, Dst, DstType, Builder);
1221 // If the comparison result is 'i1 false', then the truncation was lossy.
1222 }
1223
1224 // Do we care about this type of truncation?
1225 if (!CGF.SanOpts.has(O: Check.second.second)) {
1226 // Just emit a trap check if an __ob_trap was involved but appropriate
1227 // sanitizer isn't enabled.
1228 if (OBTrapInvolved)
1229 CGF.EmitTrapCheck(Checked: Check.second.first, CheckHandlerID: CheckHandler);
1230 return;
1231 }
1232
1233 SanitizerDebugLocation SanScope(&CGF, {Check.second.second}, CheckHandler);
1234
1235 // Does some SSCL ignore this type?
1236 const bool ignoredBySanitizer = CGF.getContext().isTypeIgnoredBySanitizer(
1237 Mask: SanitizerMask::bitPosToMask(Pos: Check.second.second), Ty: DstType);
1238
1239 // Consider OverflowBehaviorTypes which override SSCL type entries for
1240 // truncation sanitizers.
1241 if (const auto *OBT = DstType->getAs<OverflowBehaviorType>()) {
1242 if (OBT->isWrapKind())
1243 return;
1244 }
1245 if (ignoredBySanitizer && !OBTrapInvolved)
1246 return;
1247
1248 llvm::Constant *StaticArgs[] = {
1249 CGF.EmitCheckSourceLocation(Loc), CGF.EmitCheckTypeDescriptor(T: SrcType),
1250 CGF.EmitCheckTypeDescriptor(T: DstType),
1251 llvm::ConstantInt::get(Ty: Builder.getInt8Ty(), V: Check.first),
1252 llvm::ConstantInt::get(Ty: Builder.getInt32Ty(), V: 0)};
1253
1254 CGF.EmitCheck(Checked: Check.second, Check: CheckHandler, StaticArgs, DynamicArgs: {Src, Dst});
1255}
1256
1257static llvm::Value *EmitIsNegativeTestHelper(Value *V, QualType VType,
1258 const char *Name,
1259 CGBuilderTy &Builder) {
1260 bool VSigned = VType->isSignedIntegerOrEnumerationType();
1261 llvm::Type *VTy = V->getType();
1262 if (!VSigned) {
1263 // If the value is unsigned, then it is never negative.
1264 return llvm::ConstantInt::getFalse(Context&: VTy->getContext());
1265 }
1266 llvm::Constant *Zero = llvm::ConstantInt::get(Ty: VTy, V: 0);
1267 return Builder.CreateICmp(P: llvm::ICmpInst::ICMP_SLT, LHS: V, RHS: Zero,
1268 Name: llvm::Twine(Name) + "." + V->getName() +
1269 ".negativitycheck");
1270}
1271
1272// Should be called within CodeGenFunction::SanitizerScope RAII scope.
1273// Returns 'i1 false' when the conversion Src -> Dst changed the sign.
1274static std::pair<ScalarExprEmitter::ImplicitConversionCheckKind,
1275 std::pair<llvm::Value *, SanitizerKind::SanitizerOrdinal>>
1276EmitIntegerSignChangeCheckHelper(Value *Src, QualType SrcType, Value *Dst,
1277 QualType DstType, CGBuilderTy &Builder) {
1278 llvm::Type *SrcTy = Src->getType();
1279 llvm::Type *DstTy = Dst->getType();
1280
1281 assert(isa<llvm::IntegerType>(SrcTy) && isa<llvm::IntegerType>(DstTy) &&
1282 "non-integer llvm type");
1283
1284 bool SrcSigned = SrcType->isSignedIntegerOrEnumerationType();
1285 bool DstSigned = DstType->isSignedIntegerOrEnumerationType();
1286 (void)SrcSigned; // Only used in assert()
1287 (void)DstSigned; // Only used in assert()
1288 unsigned SrcBits = SrcTy->getScalarSizeInBits();
1289 unsigned DstBits = DstTy->getScalarSizeInBits();
1290 (void)SrcBits; // Only used in assert()
1291 (void)DstBits; // Only used in assert()
1292
1293 assert(((SrcBits != DstBits) || (SrcSigned != DstSigned)) &&
1294 "either the widths should be different, or the signednesses.");
1295
1296 // 1. Was the old Value negative?
1297 llvm::Value *SrcIsNegative =
1298 EmitIsNegativeTestHelper(V: Src, VType: SrcType, Name: "src", Builder);
1299 // 2. Is the new Value negative?
1300 llvm::Value *DstIsNegative =
1301 EmitIsNegativeTestHelper(V: Dst, VType: DstType, Name: "dst", Builder);
1302 // 3. Now, was the 'negativity status' preserved during the conversion?
1303 // NOTE: conversion from negative to zero is considered to change the sign.
1304 // (We want to get 'false' when the conversion changed the sign)
1305 // So we should just equality-compare the negativity statuses.
1306 llvm::Value *Check = nullptr;
1307 Check = Builder.CreateICmpEQ(LHS: SrcIsNegative, RHS: DstIsNegative, Name: "signchangecheck");
1308 // If the comparison result is 'false', then the conversion changed the sign.
1309 return std::make_pair(
1310 x: ScalarExprEmitter::ICCK_IntegerSignChange,
1311 y: std::make_pair(x&: Check, y: SanitizerKind::SO_ImplicitIntegerSignChange));
1312}
1313
1314void ScalarExprEmitter::EmitIntegerSignChangeCheck(Value *Src, QualType SrcType,
1315 Value *Dst, QualType DstType,
1316 SourceLocation Loc,
1317 bool OBTrapInvolved) {
1318 if (!CGF.SanOpts.has(O: SanitizerKind::SO_ImplicitIntegerSignChange) &&
1319 !OBTrapInvolved)
1320 return;
1321
1322 llvm::Type *SrcTy = Src->getType();
1323 llvm::Type *DstTy = Dst->getType();
1324
1325 // We only care about int->int conversions here.
1326 // We ignore conversions to/from pointer and/or bool.
1327 if (!PromotionIsPotentiallyEligibleForImplicitIntegerConversionCheck(SrcType,
1328 DstType))
1329 return;
1330
1331 bool SrcSigned = SrcType->isSignedIntegerOrEnumerationType();
1332 bool DstSigned = DstType->isSignedIntegerOrEnumerationType();
1333 unsigned SrcBits = SrcTy->getScalarSizeInBits();
1334 unsigned DstBits = DstTy->getScalarSizeInBits();
1335
1336 // Now, we do not need to emit the check in *all* of the cases.
1337 // We can avoid emitting it in some obvious cases where it would have been
1338 // dropped by the opt passes (instcombine) always anyways.
1339 // If it's a cast between effectively the same type, no check.
1340 // NOTE: this is *not* equivalent to checking the canonical types.
1341 if (SrcSigned == DstSigned && SrcBits == DstBits)
1342 return;
1343 // At least one of the values needs to have signed type.
1344 // If both are unsigned, then obviously, neither of them can be negative.
1345 if (!SrcSigned && !DstSigned)
1346 return;
1347 // If the conversion is to *larger* *signed* type, then no check is needed.
1348 // Because either sign-extension happens (so the sign will remain),
1349 // or zero-extension will happen (the sign bit will be zero.)
1350 if ((DstBits > SrcBits) && DstSigned)
1351 return;
1352 if (CGF.SanOpts.has(K: SanitizerKind::ImplicitSignedIntegerTruncation) &&
1353 (SrcBits > DstBits) && SrcSigned) {
1354 // If the signed integer truncation sanitizer is enabled,
1355 // and this is a truncation from signed type, then no check is needed.
1356 // Because here sign change check is interchangeable with truncation check.
1357 return;
1358 }
1359 // Does an SSCL have an entry for the DstType under its respective sanitizer
1360 // section? Don't check this if an __ob_trap type is involved as it has
1361 // priority to emit checks regardless of sanitizer case lists.
1362 if (!OBTrapInvolved) {
1363 if (DstSigned &&
1364 CGF.getContext().isTypeIgnoredBySanitizer(
1365 Mask: SanitizerKind::ImplicitSignedIntegerTruncation, Ty: DstType))
1366 return;
1367 if (!DstSigned &&
1368 CGF.getContext().isTypeIgnoredBySanitizer(
1369 Mask: SanitizerKind::ImplicitUnsignedIntegerTruncation, Ty: DstType))
1370 return;
1371 }
1372 // That's it. We can't rule out any more cases with the data we have.
1373
1374 auto CheckHandler = SanitizerHandler::ImplicitConversion;
1375 SanitizerDebugLocation SanScope(
1376 &CGF,
1377 {SanitizerKind::SO_ImplicitIntegerSignChange,
1378 SanitizerKind::SO_ImplicitUnsignedIntegerTruncation,
1379 SanitizerKind::SO_ImplicitSignedIntegerTruncation},
1380 CheckHandler);
1381
1382 std::pair<ScalarExprEmitter::ImplicitConversionCheckKind,
1383 std::pair<llvm::Value *, SanitizerKind::SanitizerOrdinal>>
1384 Check;
1385
1386 // Each of these checks needs to return 'false' when an issue was detected.
1387 ImplicitConversionCheckKind CheckKind;
1388 llvm::SmallVector<std::pair<llvm::Value *, SanitizerKind::SanitizerOrdinal>,
1389 2>
1390 Checks;
1391 // So we can 'and' all the checks together, and still get 'false',
1392 // if at least one of the checks detected an issue.
1393
1394 Check = EmitIntegerSignChangeCheckHelper(Src, SrcType, Dst, DstType, Builder);
1395 CheckKind = Check.first;
1396 Checks.emplace_back(Args&: Check.second);
1397
1398 if (CGF.SanOpts.has(K: SanitizerKind::ImplicitSignedIntegerTruncation) &&
1399 (SrcBits > DstBits) && !SrcSigned && DstSigned) {
1400 // If the signed integer truncation sanitizer was enabled,
1401 // and we are truncating from larger unsigned type to smaller signed type,
1402 // let's handle the case we skipped in that check.
1403 Check =
1404 EmitIntegerTruncationCheckHelper(Src, SrcType, Dst, DstType, Builder);
1405 CheckKind = ICCK_SignedIntegerTruncationOrSignChange;
1406 Checks.emplace_back(Args&: Check.second);
1407 // If the comparison result is 'i1 false', then the truncation was lossy.
1408 }
1409
1410 if (!CGF.SanOpts.has(O: SanitizerKind::SO_ImplicitIntegerSignChange)) {
1411 if (OBTrapInvolved) {
1412 llvm::Value *Combined = Check.second.first;
1413 for (const auto &C : Checks)
1414 Combined = Builder.CreateAnd(LHS: Combined, RHS: C.first);
1415 CGF.EmitTrapCheck(Checked: Combined, CheckHandlerID: CheckHandler);
1416 }
1417 return;
1418 }
1419
1420 llvm::Constant *StaticArgs[] = {
1421 CGF.EmitCheckSourceLocation(Loc), CGF.EmitCheckTypeDescriptor(T: SrcType),
1422 CGF.EmitCheckTypeDescriptor(T: DstType),
1423 llvm::ConstantInt::get(Ty: Builder.getInt8Ty(), V: CheckKind),
1424 llvm::ConstantInt::get(Ty: Builder.getInt32Ty(), V: 0)};
1425 // EmitCheck() will 'and' all the checks together.
1426 CGF.EmitCheck(Checked: Checks, Check: CheckHandler, StaticArgs, DynamicArgs: {Src, Dst});
1427}
1428
1429// Should be called within CodeGenFunction::SanitizerScope RAII scope.
1430// Returns 'i1 false' when the truncation Src -> Dst was lossy.
1431static std::pair<ScalarExprEmitter::ImplicitConversionCheckKind,
1432 std::pair<llvm::Value *, SanitizerKind::SanitizerOrdinal>>
1433EmitBitfieldTruncationCheckHelper(Value *Src, QualType SrcType, Value *Dst,
1434 QualType DstType, CGBuilderTy &Builder) {
1435 bool SrcSigned = SrcType->isSignedIntegerOrEnumerationType();
1436 bool DstSigned = DstType->isSignedIntegerOrEnumerationType();
1437
1438 ScalarExprEmitter::ImplicitConversionCheckKind Kind;
1439 if (!SrcSigned && !DstSigned)
1440 Kind = ScalarExprEmitter::ICCK_UnsignedIntegerTruncation;
1441 else
1442 Kind = ScalarExprEmitter::ICCK_SignedIntegerTruncation;
1443
1444 llvm::Value *Check = nullptr;
1445 // 1. Extend the truncated value back to the same width as the Src.
1446 Check = Builder.CreateIntCast(V: Dst, DestTy: Src->getType(), isSigned: DstSigned, Name: "bf.anyext");
1447 // 2. Equality-compare with the original source value
1448 Check = Builder.CreateICmpEQ(LHS: Check, RHS: Src, Name: "bf.truncheck");
1449 // If the comparison result is 'i1 false', then the truncation was lossy.
1450
1451 return std::make_pair(
1452 x&: Kind,
1453 y: std::make_pair(x&: Check, y: SanitizerKind::SO_ImplicitBitfieldConversion));
1454}
1455
1456// Should be called within CodeGenFunction::SanitizerScope RAII scope.
1457// Returns 'i1 false' when the conversion Src -> Dst changed the sign.
1458static std::pair<ScalarExprEmitter::ImplicitConversionCheckKind,
1459 std::pair<llvm::Value *, SanitizerKind::SanitizerOrdinal>>
1460EmitBitfieldSignChangeCheckHelper(Value *Src, QualType SrcType, Value *Dst,
1461 QualType DstType, CGBuilderTy &Builder) {
1462 // 1. Was the old Value negative?
1463 llvm::Value *SrcIsNegative =
1464 EmitIsNegativeTestHelper(V: Src, VType: SrcType, Name: "bf.src", Builder);
1465 // 2. Is the new Value negative?
1466 llvm::Value *DstIsNegative =
1467 EmitIsNegativeTestHelper(V: Dst, VType: DstType, Name: "bf.dst", Builder);
1468 // 3. Now, was the 'negativity status' preserved during the conversion?
1469 // NOTE: conversion from negative to zero is considered to change the sign.
1470 // (We want to get 'false' when the conversion changed the sign)
1471 // So we should just equality-compare the negativity statuses.
1472 llvm::Value *Check = nullptr;
1473 Check =
1474 Builder.CreateICmpEQ(LHS: SrcIsNegative, RHS: DstIsNegative, Name: "bf.signchangecheck");
1475 // If the comparison result is 'false', then the conversion changed the sign.
1476 return std::make_pair(
1477 x: ScalarExprEmitter::ICCK_IntegerSignChange,
1478 y: std::make_pair(x&: Check, y: SanitizerKind::SO_ImplicitBitfieldConversion));
1479}
1480
1481void CodeGenFunction::EmitBitfieldConversionCheck(Value *Src, QualType SrcType,
1482 Value *Dst, QualType DstType,
1483 const CGBitFieldInfo &Info,
1484 SourceLocation Loc) {
1485
1486 if (!SanOpts.has(K: SanitizerKind::ImplicitBitfieldConversion))
1487 return;
1488
1489 // We only care about int->int conversions here.
1490 // We ignore conversions to/from pointer and/or bool.
1491 if (!PromotionIsPotentiallyEligibleForImplicitIntegerConversionCheck(SrcType,
1492 DstType))
1493 return;
1494
1495 if (DstType->isBooleanType() || SrcType->isBooleanType())
1496 return;
1497
1498 // This should be truncation of integral types.
1499 assert(isa<llvm::IntegerType>(Src->getType()) &&
1500 isa<llvm::IntegerType>(Dst->getType()) && "non-integer llvm type");
1501
1502 // TODO: Calculate src width to avoid emitting code
1503 // for unecessary cases.
1504 unsigned SrcBits = ConvertType(T: SrcType)->getScalarSizeInBits();
1505 unsigned DstBits = Info.Size;
1506
1507 bool SrcSigned = SrcType->isSignedIntegerOrEnumerationType();
1508 bool DstSigned = DstType->isSignedIntegerOrEnumerationType();
1509
1510 auto CheckHandler = SanitizerHandler::ImplicitConversion;
1511 SanitizerDebugLocation SanScope(
1512 this, {SanitizerKind::SO_ImplicitBitfieldConversion}, CheckHandler);
1513
1514 std::pair<ScalarExprEmitter::ImplicitConversionCheckKind,
1515 std::pair<llvm::Value *, SanitizerKind::SanitizerOrdinal>>
1516 Check;
1517
1518 // Truncation
1519 bool EmitTruncation = DstBits < SrcBits;
1520 // If Dst is signed and Src unsigned, we want to be more specific
1521 // about the CheckKind we emit, in this case we want to emit
1522 // ICCK_SignedIntegerTruncationOrSignChange.
1523 bool EmitTruncationFromUnsignedToSigned =
1524 EmitTruncation && DstSigned && !SrcSigned;
1525 // Sign change
1526 bool SameTypeSameSize = SrcSigned == DstSigned && SrcBits == DstBits;
1527 bool BothUnsigned = !SrcSigned && !DstSigned;
1528 bool LargerSigned = (DstBits > SrcBits) && DstSigned;
1529 // We can avoid emitting sign change checks in some obvious cases
1530 // 1. If Src and Dst have the same signedness and size
1531 // 2. If both are unsigned sign check is unecessary!
1532 // 3. If Dst is signed and bigger than Src, either
1533 // sign-extension or zero-extension will make sure
1534 // the sign remains.
1535 bool EmitSignChange = !SameTypeSameSize && !BothUnsigned && !LargerSigned;
1536
1537 if (EmitTruncation)
1538 Check =
1539 EmitBitfieldTruncationCheckHelper(Src, SrcType, Dst, DstType, Builder);
1540 else if (EmitSignChange) {
1541 assert(((SrcBits != DstBits) || (SrcSigned != DstSigned)) &&
1542 "either the widths should be different, or the signednesses.");
1543 Check =
1544 EmitBitfieldSignChangeCheckHelper(Src, SrcType, Dst, DstType, Builder);
1545 } else
1546 return;
1547
1548 ScalarExprEmitter::ImplicitConversionCheckKind CheckKind = Check.first;
1549 if (EmitTruncationFromUnsignedToSigned)
1550 CheckKind = ScalarExprEmitter::ICCK_SignedIntegerTruncationOrSignChange;
1551
1552 llvm::Constant *StaticArgs[] = {
1553 EmitCheckSourceLocation(Loc), EmitCheckTypeDescriptor(T: SrcType),
1554 EmitCheckTypeDescriptor(T: DstType),
1555 llvm::ConstantInt::get(Ty: Builder.getInt8Ty(), V: CheckKind),
1556 llvm::ConstantInt::get(Ty: Builder.getInt32Ty(), V: Info.Size)};
1557
1558 EmitCheck(Checked: Check.second, Check: CheckHandler, StaticArgs, DynamicArgs: {Src, Dst});
1559}
1560
1561Value *ScalarExprEmitter::EmitScalarCast(Value *Src, QualType SrcType,
1562 QualType DstType, llvm::Type *SrcTy,
1563 llvm::Type *DstTy,
1564 ScalarConversionOpts Opts) {
1565 // The Element types determine the type of cast to perform.
1566 llvm::Type *SrcElementTy;
1567 llvm::Type *DstElementTy;
1568 QualType SrcElementType;
1569 QualType DstElementType;
1570 if (SrcType->isMatrixType() && DstType->isMatrixType()) {
1571 SrcElementTy = cast<llvm::VectorType>(Val: SrcTy)->getElementType();
1572 DstElementTy = cast<llvm::VectorType>(Val: DstTy)->getElementType();
1573 SrcElementType = SrcType->castAs<MatrixType>()->getElementType();
1574 DstElementType = DstType->castAs<MatrixType>()->getElementType();
1575 } else {
1576 assert(!SrcType->isMatrixType() && !DstType->isMatrixType() &&
1577 "cannot cast between matrix and non-matrix types");
1578 SrcElementTy = SrcTy;
1579 DstElementTy = DstTy;
1580 SrcElementType = SrcType;
1581 DstElementType = DstType;
1582 }
1583
1584 if (isa<llvm::IntegerType>(Val: SrcElementTy)) {
1585 bool InputSigned = SrcElementType->isSignedIntegerOrEnumerationType();
1586 if (SrcElementType->isBooleanType() && Opts.TreatBooleanAsSigned) {
1587 InputSigned = true;
1588 }
1589
1590 if (isa<llvm::IntegerType>(Val: DstElementTy))
1591 return Builder.CreateIntCast(V: Src, DestTy: DstTy, isSigned: InputSigned, Name: "conv");
1592 if (InputSigned)
1593 return Builder.CreateSIToFP(V: Src, DestTy: DstTy, Name: "conv");
1594 return Builder.CreateUIToFP(V: Src, DestTy: DstTy, Name: "conv");
1595 }
1596
1597 if (isa<llvm::IntegerType>(Val: DstElementTy)) {
1598 assert(SrcElementTy->isFloatingPointTy() && "Unknown real conversion");
1599 bool IsSigned = DstElementType->isSignedIntegerOrEnumerationType();
1600
1601 // If we can't recognize overflow as undefined behavior, assume that
1602 // overflow saturates. This protects against normal optimizations if we are
1603 // compiling with non-standard FP semantics.
1604 if (!CGF.CGM.getCodeGenOpts().StrictFloatCastOverflow) {
1605 llvm::Intrinsic::ID IID =
1606 IsSigned ? llvm::Intrinsic::fptosi_sat : llvm::Intrinsic::fptoui_sat;
1607 return Builder.CreateCall(Callee: CGF.CGM.getIntrinsic(IID, Tys: {DstTy, SrcTy}), Args: Src);
1608 }
1609
1610 if (IsSigned)
1611 return Builder.CreateFPToSI(V: Src, DestTy: DstTy, Name: "conv");
1612 return Builder.CreateFPToUI(V: Src, DestTy: DstTy, Name: "conv");
1613 }
1614
1615 if ((DstElementTy->is16bitFPTy() && SrcElementTy->is16bitFPTy())) {
1616 Value *FloatVal = Builder.CreateFPExt(V: Src, DestTy: Builder.getFloatTy(), Name: "fpext");
1617 return Builder.CreateFPTrunc(V: FloatVal, DestTy: DstTy, Name: "fptrunc");
1618 }
1619 if (DstElementTy->getTypeID() < SrcElementTy->getTypeID())
1620 return Builder.CreateFPTrunc(V: Src, DestTy: DstTy, Name: "conv");
1621 return Builder.CreateFPExt(V: Src, DestTy: DstTy, Name: "conv");
1622}
1623
1624/// Emit a conversion from the specified type to the specified destination type,
1625/// both of which are LLVM scalar types.
1626Value *ScalarExprEmitter::EmitScalarConversion(Value *Src, QualType SrcType,
1627 QualType DstType,
1628 SourceLocation Loc,
1629 ScalarConversionOpts Opts) {
1630 // All conversions involving fixed point types should be handled by the
1631 // EmitFixedPoint family functions. This is done to prevent bloating up this
1632 // function more, and although fixed point numbers are represented by
1633 // integers, we do not want to follow any logic that assumes they should be
1634 // treated as integers.
1635 // TODO(leonardchan): When necessary, add another if statement checking for
1636 // conversions to fixed point types from other types.
1637 if (SrcType->isFixedPointType()) {
1638 if (DstType->isBooleanType())
1639 // It is important that we check this before checking if the dest type is
1640 // an integer because booleans are technically integer types.
1641 // We do not need to check the padding bit on unsigned types if unsigned
1642 // padding is enabled because overflow into this bit is undefined
1643 // behavior.
1644 return Builder.CreateIsNotNull(Arg: Src, Name: "tobool");
1645 if (DstType->isFixedPointType() || DstType->isIntegerType() ||
1646 DstType->isRealFloatingType())
1647 return EmitFixedPointConversion(Src, SrcTy: SrcType, DstTy: DstType, Loc);
1648
1649 llvm_unreachable(
1650 "Unhandled scalar conversion from a fixed point type to another type.");
1651 } else if (DstType->isFixedPointType()) {
1652 if (SrcType->isIntegerType() || SrcType->isRealFloatingType())
1653 // This also includes converting booleans and enums to fixed point types.
1654 return EmitFixedPointConversion(Src, SrcTy: SrcType, DstTy: DstType, Loc);
1655
1656 llvm_unreachable(
1657 "Unhandled scalar conversion to a fixed point type from another type.");
1658 }
1659
1660 QualType NoncanonicalSrcType = SrcType;
1661 QualType NoncanonicalDstType = DstType;
1662
1663 SrcType = CGF.getContext().getCanonicalType(T: SrcType);
1664 DstType = CGF.getContext().getCanonicalType(T: DstType);
1665 if (SrcType == DstType) return Src;
1666
1667 if (DstType->isVoidType()) return nullptr;
1668
1669 llvm::Value *OrigSrc = Src;
1670 QualType OrigSrcType = SrcType;
1671 llvm::Type *SrcTy = Src->getType();
1672
1673 // Handle conversions to bool first, they are special: comparisons against 0.
1674 if (DstType->isBooleanType())
1675 return EmitConversionToBool(Src, SrcType);
1676
1677 llvm::Type *DstTy = ConvertType(T: DstType);
1678
1679 // Determine whether an overflow behavior of 'trap' has been specified for
1680 // either the destination or the source types. If so, we can elide sanitizer
1681 // capability checks as this overflow behavior kind is also capable of
1682 // emitting traps without runtime sanitizer support.
1683 // Also skip instrumentation if either source or destination has 'wrap'
1684 // behavior - the user has explicitly indicated they accept wrapping
1685 // semantics. Use non-canonical types to preserve OBT annotations.
1686 const auto *DstOBT = NoncanonicalDstType->getAs<OverflowBehaviorType>();
1687 const auto *SrcOBT = NoncanonicalSrcType->getAs<OverflowBehaviorType>();
1688 bool OBTrapInvolved =
1689 (DstOBT && DstOBT->isTrapKind()) || (SrcOBT && SrcOBT->isTrapKind());
1690 bool OBWrapInvolved =
1691 (DstOBT && DstOBT->isWrapKind()) || (SrcOBT && SrcOBT->isWrapKind());
1692
1693 // If half isn't a native type, cast to float for evaluation.
1694 if (SrcType->isHalfType() && !CGF.getContext().getLangOpts().NativeHalfType &&
1695 SrcTy == CGF.CGM.HalfTy && DstTy != CGF.CGM.HalfTy) {
1696 if (DstTy->isFloatingPointTy())
1697 return Builder.CreateFPExt(V: Src, DestTy: DstTy, Name: "conv");
1698
1699 // Cast to other types through float (as opposed to operations on half,
1700 // available with NativeHalfType).
1701 Src = Builder.CreateFPExt(V: Src, DestTy: CGF.CGM.FloatTy, Name: "conv");
1702 SrcType = CGF.getContext().FloatTy;
1703 SrcTy = CGF.FloatTy;
1704 }
1705
1706 // Ignore conversions like int -> uint.
1707 if (SrcTy == DstTy) {
1708 if (Opts.EmitImplicitIntegerSignChangeChecks ||
1709 (OBTrapInvolved && !OBWrapInvolved))
1710 EmitIntegerSignChangeCheck(Src, SrcType: NoncanonicalSrcType, Dst: Src,
1711 DstType: NoncanonicalDstType, Loc, OBTrapInvolved);
1712
1713 return Src;
1714 }
1715
1716 // Handle pointer conversions next: pointers can only be converted to/from
1717 // other pointers and integers. Check for pointer types in terms of LLVM, as
1718 // some native types (like Obj-C id) may map to a pointer type.
1719 if (auto DstPT = dyn_cast<llvm::PointerType>(Val: DstTy)) {
1720 // The source value may be an integer, or a pointer.
1721 if (isa<llvm::PointerType>(Val: SrcTy))
1722 return Src;
1723
1724 assert(SrcType->isIntegerType() && "Not ptr->ptr or int->ptr conversion?");
1725 // First, convert to the correct width so that we control the kind of
1726 // extension.
1727 llvm::Type *MiddleTy = CGF.CGM.getDataLayout().getIntPtrType(DstPT);
1728 bool InputSigned = SrcType->isSignedIntegerOrEnumerationType();
1729 llvm::Value* IntResult =
1730 Builder.CreateIntCast(V: Src, DestTy: MiddleTy, isSigned: InputSigned, Name: "conv");
1731 // Then, cast to pointer.
1732 return Builder.CreateIntToPtr(V: IntResult, DestTy: DstTy, Name: "conv");
1733 }
1734
1735 if (isa<llvm::PointerType>(Val: SrcTy)) {
1736 // Must be an ptr to int cast.
1737 assert(isa<llvm::IntegerType>(DstTy) && "not ptr->int?");
1738 return Builder.CreatePtrToInt(V: Src, DestTy: DstTy, Name: "conv");
1739 }
1740
1741 // A scalar can be splatted to an extended vector of the same element type
1742 if (DstType->isExtVectorType() && !SrcType->isVectorType()) {
1743 // Sema should add casts to make sure that the source expression's type is
1744 // the same as the vector's element type (sans qualifiers)
1745 assert(DstType->castAs<ExtVectorType>()->getElementType().getTypePtr() ==
1746 SrcType.getTypePtr() &&
1747 "Splatted expr doesn't match with vector element type?");
1748
1749 // Splat the element across to all elements
1750 unsigned NumElements = cast<llvm::FixedVectorType>(Val: DstTy)->getNumElements();
1751 return Builder.CreateVectorSplat(NumElts: NumElements, V: Src, Name: "splat");
1752 }
1753
1754 if (SrcType->isMatrixType() && DstType->isMatrixType())
1755 return EmitScalarCast(Src, SrcType, DstType, SrcTy, DstTy, Opts);
1756
1757 if (isa<llvm::VectorType>(Val: SrcTy) || isa<llvm::VectorType>(Val: DstTy)) {
1758 // Allow bitcast from vector to integer/fp of the same size.
1759 llvm::TypeSize SrcSize = SrcTy->getPrimitiveSizeInBits();
1760 llvm::TypeSize DstSize = DstTy->getPrimitiveSizeInBits();
1761 if (SrcSize == DstSize)
1762 return Builder.CreateBitCast(V: Src, DestTy: DstTy, Name: "conv");
1763
1764 // Conversions between vectors of different sizes are not allowed except
1765 // when vectors of half are involved. Operations on storage-only half
1766 // vectors require promoting half vector operands to float vectors and
1767 // truncating the result, which is either an int or float vector, to a
1768 // short or half vector.
1769
1770 // Source and destination are both expected to be vectors.
1771 llvm::Type *SrcElementTy = cast<llvm::VectorType>(Val: SrcTy)->getElementType();
1772 llvm::Type *DstElementTy = cast<llvm::VectorType>(Val: DstTy)->getElementType();
1773 (void)DstElementTy;
1774
1775 assert(((SrcElementTy->isIntegerTy() &&
1776 DstElementTy->isIntegerTy()) ||
1777 (SrcElementTy->isFloatingPointTy() &&
1778 DstElementTy->isFloatingPointTy())) &&
1779 "unexpected conversion between a floating-point vector and an "
1780 "integer vector");
1781
1782 // Truncate an i32 vector to an i16 vector.
1783 if (SrcElementTy->isIntegerTy())
1784 return Builder.CreateIntCast(V: Src, DestTy: DstTy, isSigned: false, Name: "conv");
1785
1786 // Truncate a float vector to a half vector.
1787 if (SrcSize > DstSize)
1788 return Builder.CreateFPTrunc(V: Src, DestTy: DstTy, Name: "conv");
1789
1790 // Promote a half vector to a float vector.
1791 return Builder.CreateFPExt(V: Src, DestTy: DstTy, Name: "conv");
1792 }
1793
1794 // Finally, we have the arithmetic types: real int/float.
1795 Value *Res = nullptr;
1796 llvm::Type *ResTy = DstTy;
1797
1798 // An overflowing conversion has undefined behavior if either the source type
1799 // or the destination type is a floating-point type. However, we consider the
1800 // range of representable values for all floating-point types to be
1801 // [-inf,+inf], so no overflow can ever happen when the destination type is a
1802 // floating-point type.
1803 if (CGF.SanOpts.has(K: SanitizerKind::FloatCastOverflow) &&
1804 OrigSrcType->isFloatingType())
1805 EmitFloatConversionCheck(OrigSrc, OrigSrcType, Src, SrcType, DstType, DstTy,
1806 Loc);
1807
1808 // Cast to half from float if half isn't a native type. When __fp16 isn't
1809 // native, arithmetic is evaluated as float.
1810 if (DstType->isHalfType() && !CGF.getContext().getLangOpts().NativeHalfType &&
1811 DstTy == CGF.CGM.HalfTy) {
1812 // Make sure we cast in a single step if from another FP type.
1813 if (SrcTy->isFloatingPointTy())
1814 return Builder.CreateFPTrunc(V: Src, DestTy: CGF.CGM.HalfTy, Name: "conv");
1815
1816 DstTy = CGF.FloatTy;
1817 }
1818
1819 Res = EmitScalarCast(Src, SrcType, DstType, SrcTy, DstTy, Opts);
1820
1821 if (DstTy != ResTy) {
1822 Res = Builder.CreateFPTrunc(V: Res, DestTy: CGF.CGM.HalfTy, Name: "conv");
1823
1824 if (ResTy != CGF.CGM.HalfTy) {
1825 assert(ResTy->isIntegerTy(16) &&
1826 "Only half FP requires extra conversion");
1827 Res = Builder.CreateBitCast(V: Res, DestTy: ResTy);
1828 }
1829 }
1830
1831 if ((Opts.EmitImplicitIntegerTruncationChecks || OBTrapInvolved) &&
1832 !OBWrapInvolved && !Opts.PatternExcluded)
1833 EmitIntegerTruncationCheck(Src, SrcType: NoncanonicalSrcType, Dst: Res,
1834 DstType: NoncanonicalDstType, Loc, OBTrapInvolved);
1835
1836 if (Opts.EmitImplicitIntegerSignChangeChecks ||
1837 (OBTrapInvolved && !OBWrapInvolved))
1838 EmitIntegerSignChangeCheck(Src, SrcType: NoncanonicalSrcType, Dst: Res,
1839 DstType: NoncanonicalDstType, Loc, OBTrapInvolved);
1840
1841 return Res;
1842}
1843
1844Value *ScalarExprEmitter::EmitFixedPointConversion(Value *Src, QualType SrcTy,
1845 QualType DstTy,
1846 SourceLocation Loc) {
1847 llvm::FixedPointBuilder<CGBuilderTy> FPBuilder(Builder);
1848 llvm::Value *Result;
1849 if (SrcTy->isRealFloatingType())
1850 Result = FPBuilder.CreateFloatingToFixed(Src,
1851 DstSema: CGF.getContext().getFixedPointSemantics(Ty: DstTy));
1852 else if (DstTy->isRealFloatingType())
1853 Result = FPBuilder.CreateFixedToFloating(Src,
1854 SrcSema: CGF.getContext().getFixedPointSemantics(Ty: SrcTy),
1855 DstTy: ConvertType(T: DstTy));
1856 else {
1857 auto SrcFPSema = CGF.getContext().getFixedPointSemantics(Ty: SrcTy);
1858 auto DstFPSema = CGF.getContext().getFixedPointSemantics(Ty: DstTy);
1859
1860 if (DstTy->isIntegerType())
1861 Result = FPBuilder.CreateFixedToInteger(Src, SrcSema: SrcFPSema,
1862 DstWidth: DstFPSema.getWidth(),
1863 DstIsSigned: DstFPSema.isSigned());
1864 else if (SrcTy->isIntegerType())
1865 Result = FPBuilder.CreateIntegerToFixed(Src, SrcIsSigned: SrcFPSema.isSigned(),
1866 DstSema: DstFPSema);
1867 else
1868 Result = FPBuilder.CreateFixedToFixed(Src, SrcSema: SrcFPSema, DstSema: DstFPSema);
1869 }
1870 return Result;
1871}
1872
1873/// Emit a conversion from the specified complex type to the specified
1874/// destination type, where the destination type is an LLVM scalar type.
1875Value *ScalarExprEmitter::EmitComplexToScalarConversion(
1876 CodeGenFunction::ComplexPairTy Src, QualType SrcTy, QualType DstTy,
1877 SourceLocation Loc) {
1878 // Get the source element type.
1879 SrcTy = SrcTy->castAs<ComplexType>()->getElementType();
1880
1881 // Handle conversions to bool first, they are special: comparisons against 0.
1882 if (DstTy->isBooleanType()) {
1883 // Complex != 0 -> (Real != 0) | (Imag != 0)
1884 Src.first = EmitScalarConversion(Src: Src.first, SrcType: SrcTy, DstType: DstTy, Loc);
1885 Src.second = EmitScalarConversion(Src: Src.second, SrcType: SrcTy, DstType: DstTy, Loc);
1886 return Builder.CreateOr(LHS: Src.first, RHS: Src.second, Name: "tobool");
1887 }
1888
1889 // C99 6.3.1.7p2: "When a value of complex type is converted to a real type,
1890 // the imaginary part of the complex value is discarded and the value of the
1891 // real part is converted according to the conversion rules for the
1892 // corresponding real type.
1893 return EmitScalarConversion(Src: Src.first, SrcType: SrcTy, DstType: DstTy, Loc);
1894}
1895
1896Value *ScalarExprEmitter::EmitNullValue(QualType Ty) {
1897 return CGF.EmitFromMemory(Value: CGF.CGM.EmitNullConstant(T: Ty), Ty);
1898}
1899
1900/// Emit a sanitization check for the given "binary" operation (which
1901/// might actually be a unary increment which has been lowered to a binary
1902/// operation). The check passes if all values in \p Checks (which are \c i1),
1903/// are \c true.
1904void ScalarExprEmitter::EmitBinOpCheck(
1905 ArrayRef<std::pair<Value *, SanitizerKind::SanitizerOrdinal>> Checks,
1906 const BinOpInfo &Info) {
1907 assert(CGF.IsSanitizerScope);
1908 SanitizerHandler Check;
1909 SmallVector<llvm::Constant *, 4> StaticData;
1910 SmallVector<llvm::Value *, 2> DynamicData;
1911 TrapReason TR;
1912
1913 BinaryOperatorKind Opcode = Info.Opcode;
1914 if (BinaryOperator::isCompoundAssignmentOp(Opc: Opcode))
1915 Opcode = BinaryOperator::getOpForCompoundAssignment(Opc: Opcode);
1916
1917 StaticData.push_back(Elt: CGF.EmitCheckSourceLocation(Loc: Info.E->getExprLoc()));
1918 const UnaryOperator *UO = dyn_cast<UnaryOperator>(Val: Info.E);
1919 if (UO && UO->getOpcode() == UO_Minus) {
1920 Check = SanitizerHandler::NegateOverflow;
1921 StaticData.push_back(Elt: CGF.EmitCheckTypeDescriptor(T: UO->getType()));
1922 DynamicData.push_back(Elt: Info.RHS);
1923 } else {
1924 if (BinaryOperator::isShiftOp(Opc: Opcode)) {
1925 // Shift LHS negative or too large, or RHS out of bounds.
1926 Check = SanitizerHandler::ShiftOutOfBounds;
1927 const BinaryOperator *BO = cast<BinaryOperator>(Val: Info.E);
1928 StaticData.push_back(
1929 Elt: CGF.EmitCheckTypeDescriptor(T: BO->getLHS()->getType()));
1930 StaticData.push_back(
1931 Elt: CGF.EmitCheckTypeDescriptor(T: BO->getRHS()->getType()));
1932 } else if (Opcode == BO_Div || Opcode == BO_Rem) {
1933 // Divide or modulo by zero, or signed overflow (eg INT_MAX / -1).
1934 Check = SanitizerHandler::DivremOverflow;
1935 StaticData.push_back(Elt: CGF.EmitCheckTypeDescriptor(T: Info.Ty));
1936 } else {
1937 // Arithmetic overflow (+, -, *).
1938 int ArithOverflowKind = 0;
1939 switch (Opcode) {
1940 case BO_Add: {
1941 Check = SanitizerHandler::AddOverflow;
1942 ArithOverflowKind = diag::UBSanArithKind::Add;
1943 break;
1944 }
1945 case BO_Sub: {
1946 Check = SanitizerHandler::SubOverflow;
1947 ArithOverflowKind = diag::UBSanArithKind::Sub;
1948 break;
1949 }
1950 case BO_Mul: {
1951 Check = SanitizerHandler::MulOverflow;
1952 ArithOverflowKind = diag::UBSanArithKind::Mul;
1953 break;
1954 }
1955 default:
1956 llvm_unreachable("unexpected opcode for bin op check");
1957 }
1958 StaticData.push_back(Elt: CGF.EmitCheckTypeDescriptor(T: Info.Ty));
1959 if (CGF.CGM.getCodeGenOpts().SanitizeTrap.has(
1960 K: SanitizerKind::UnsignedIntegerOverflow) ||
1961 CGF.CGM.getCodeGenOpts().SanitizeTrap.has(
1962 K: SanitizerKind::SignedIntegerOverflow)) {
1963 // Only pay the cost for constructing the trap diagnostic if they are
1964 // going to be used.
1965 CGF.CGM.BuildTrapReason(DiagID: diag::trap_ubsan_arith_overflow, TR)
1966 << Info.Ty->isSignedIntegerOrEnumerationType() << ArithOverflowKind
1967 << Info.E;
1968 }
1969 }
1970 DynamicData.push_back(Elt: Info.LHS);
1971 DynamicData.push_back(Elt: Info.RHS);
1972 }
1973
1974 CGF.EmitCheck(Checked: Checks, Check, StaticArgs: StaticData, DynamicArgs: DynamicData, TR: &TR);
1975}
1976
1977//===----------------------------------------------------------------------===//
1978// Visitor Methods
1979//===----------------------------------------------------------------------===//
1980
1981Value *ScalarExprEmitter::VisitExpr(Expr *E) {
1982 CGF.ErrorUnsupported(S: E, Type: "scalar expression");
1983 if (E->getType()->isVoidType())
1984 return nullptr;
1985 return llvm::PoisonValue::get(T: CGF.ConvertType(T: E->getType()));
1986}
1987
1988Value *
1989ScalarExprEmitter::VisitSYCLUniqueStableNameExpr(SYCLUniqueStableNameExpr *E) {
1990 ASTContext &Context = CGF.getContext();
1991 unsigned AddrSpace =
1992 Context.getTargetAddressSpace(AS: CGF.CGM.GetGlobalConstantAddressSpace());
1993 llvm::Constant *GlobalConstStr = Builder.CreateGlobalString(
1994 Str: E->ComputeName(Context), Name: "__usn_str", AddressSpace: AddrSpace);
1995
1996 llvm::Type *ExprTy = ConvertType(T: E->getType());
1997 return Builder.CreatePointerBitCastOrAddrSpaceCast(V: GlobalConstStr, DestTy: ExprTy,
1998 Name: "usn_addr_cast");
1999}
2000
2001Value *ScalarExprEmitter::VisitEmbedExpr(EmbedExpr *E) {
2002 assert(E->getDataElementCount() == 1);
2003 auto It = E->begin();
2004 return Builder.getInt(AI: (*It)->getValue());
2005}
2006
2007Value *ScalarExprEmitter::VisitShuffleVectorExpr(ShuffleVectorExpr *E) {
2008 // Vector Mask Case
2009 if (E->getNumSubExprs() == 2) {
2010 Value *LHS = CGF.EmitScalarExpr(E: E->getExpr(Index: 0));
2011 Value *RHS = CGF.EmitScalarExpr(E: E->getExpr(Index: 1));
2012 Value *Mask;
2013
2014 auto *LTy = cast<llvm::FixedVectorType>(Val: LHS->getType());
2015 unsigned LHSElts = LTy->getNumElements();
2016
2017 Mask = RHS;
2018
2019 auto *MTy = cast<llvm::FixedVectorType>(Val: Mask->getType());
2020
2021 // Mask off the high bits of each shuffle index.
2022 Value *MaskBits =
2023 llvm::ConstantInt::get(Ty: MTy, V: llvm::NextPowerOf2(A: LHSElts - 1) - 1);
2024 Mask = Builder.CreateAnd(LHS: Mask, RHS: MaskBits, Name: "mask");
2025
2026 // newv = undef
2027 // mask = mask & maskbits
2028 // for each elt
2029 // n = extract mask i
2030 // x = extract val n
2031 // newv = insert newv, x, i
2032 auto *RTy = llvm::FixedVectorType::get(ElementType: LTy->getElementType(),
2033 NumElts: MTy->getNumElements());
2034 Value* NewV = llvm::PoisonValue::get(T: RTy);
2035 for (unsigned i = 0, e = MTy->getNumElements(); i != e; ++i) {
2036 Value *IIndx = llvm::ConstantInt::get(Ty: CGF.SizeTy, V: i);
2037 Value *Indx = Builder.CreateExtractElement(Vec: Mask, Idx: IIndx, Name: "shuf_idx");
2038
2039 Value *VExt = Builder.CreateExtractElement(Vec: LHS, Idx: Indx, Name: "shuf_elt");
2040 NewV = Builder.CreateInsertElement(Vec: NewV, NewElt: VExt, Idx: IIndx, Name: "shuf_ins");
2041 }
2042 return NewV;
2043 }
2044
2045 Value* V1 = CGF.EmitScalarExpr(E: E->getExpr(Index: 0));
2046 Value* V2 = CGF.EmitScalarExpr(E: E->getExpr(Index: 1));
2047
2048 SmallVector<int, 32> Indices;
2049 for (unsigned i = 2; i < E->getNumSubExprs(); ++i) {
2050 llvm::APSInt Idx = E->getShuffleMaskIdx(N: i - 2);
2051 // Check for -1 and output it as undef in the IR.
2052 if (Idx.isSigned() && Idx.isAllOnes())
2053 Indices.push_back(Elt: -1);
2054 else
2055 Indices.push_back(Elt: Idx.getZExtValue());
2056 }
2057
2058 return Builder.CreateShuffleVector(V1, V2, Mask: Indices, Name: "shuffle");
2059}
2060
2061Value *ScalarExprEmitter::VisitConvertVectorExpr(ConvertVectorExpr *E) {
2062 QualType SrcType = E->getSrcExpr()->getType(),
2063 DstType = E->getType();
2064
2065 Value *Src = CGF.EmitScalarExpr(E: E->getSrcExpr());
2066
2067 SrcType = CGF.getContext().getCanonicalType(T: SrcType);
2068 DstType = CGF.getContext().getCanonicalType(T: DstType);
2069 if (SrcType == DstType) return Src;
2070
2071 assert(SrcType->isVectorType() &&
2072 "ConvertVector source type must be a vector");
2073 assert(DstType->isVectorType() &&
2074 "ConvertVector destination type must be a vector");
2075
2076 llvm::Type *SrcTy = Src->getType();
2077 llvm::Type *DstTy = ConvertType(T: DstType);
2078
2079 // Ignore conversions like int -> uint.
2080 if (SrcTy == DstTy)
2081 return Src;
2082
2083 QualType SrcEltType = SrcType->castAs<VectorType>()->getElementType(),
2084 DstEltType = DstType->castAs<VectorType>()->getElementType();
2085
2086 assert(SrcTy->isVectorTy() &&
2087 "ConvertVector source IR type must be a vector");
2088 assert(DstTy->isVectorTy() &&
2089 "ConvertVector destination IR type must be a vector");
2090
2091 llvm::Type *SrcEltTy = cast<llvm::VectorType>(Val: SrcTy)->getElementType(),
2092 *DstEltTy = cast<llvm::VectorType>(Val: DstTy)->getElementType();
2093
2094 if (DstEltType->isBooleanType()) {
2095 assert((SrcEltTy->isFloatingPointTy() ||
2096 isa<llvm::IntegerType>(SrcEltTy)) && "Unknown boolean conversion");
2097
2098 llvm::Value *Zero = llvm::Constant::getNullValue(Ty: SrcTy);
2099 if (SrcEltTy->isFloatingPointTy()) {
2100 CodeGenFunction::CGFPOptionsRAII FPOptions(CGF, E);
2101 return Builder.CreateFCmpUNE(LHS: Src, RHS: Zero, Name: "tobool");
2102 } else {
2103 return Builder.CreateICmpNE(LHS: Src, RHS: Zero, Name: "tobool");
2104 }
2105 }
2106
2107 // We have the arithmetic types: real int/float.
2108 Value *Res = nullptr;
2109
2110 if (isa<llvm::IntegerType>(Val: SrcEltTy)) {
2111 bool InputSigned = SrcEltType->isSignedIntegerOrEnumerationType();
2112 if (isa<llvm::IntegerType>(Val: DstEltTy))
2113 Res = Builder.CreateIntCast(V: Src, DestTy: DstTy, isSigned: InputSigned, Name: "conv");
2114 else {
2115 CodeGenFunction::CGFPOptionsRAII FPOptions(CGF, E);
2116 if (InputSigned)
2117 Res = Builder.CreateSIToFP(V: Src, DestTy: DstTy, Name: "conv");
2118 else
2119 Res = Builder.CreateUIToFP(V: Src, DestTy: DstTy, Name: "conv");
2120 }
2121 } else if (isa<llvm::IntegerType>(Val: DstEltTy)) {
2122 assert(SrcEltTy->isFloatingPointTy() && "Unknown real conversion");
2123 CodeGenFunction::CGFPOptionsRAII FPOptions(CGF, E);
2124 if (DstEltType->isSignedIntegerOrEnumerationType())
2125 Res = Builder.CreateFPToSI(V: Src, DestTy: DstTy, Name: "conv");
2126 else
2127 Res = Builder.CreateFPToUI(V: Src, DestTy: DstTy, Name: "conv");
2128 } else {
2129 assert(SrcEltTy->isFloatingPointTy() && DstEltTy->isFloatingPointTy() &&
2130 "Unknown real conversion");
2131 CodeGenFunction::CGFPOptionsRAII FPOptions(CGF, E);
2132 if (DstEltTy->getTypeID() < SrcEltTy->getTypeID())
2133 Res = Builder.CreateFPTrunc(V: Src, DestTy: DstTy, Name: "conv");
2134 else
2135 Res = Builder.CreateFPExt(V: Src, DestTy: DstTy, Name: "conv");
2136 }
2137
2138 return Res;
2139}
2140
2141Value *ScalarExprEmitter::VisitMemberExpr(MemberExpr *E) {
2142 if (CodeGenFunction::ConstantEmission Constant = CGF.tryEmitAsConstant(ME: E)) {
2143 CGF.EmitIgnoredExpr(E: E->getBase());
2144 return CGF.emitScalarConstant(Constant, E);
2145 } else {
2146 Expr::EvalResult Result;
2147 if (E->EvaluateAsInt(Result, Ctx: CGF.getContext(), AllowSideEffects: Expr::SE_AllowSideEffects)) {
2148 llvm::APSInt Value = Result.Val.getInt();
2149 CGF.EmitIgnoredExpr(E: E->getBase());
2150 return Builder.getInt(AI: Value);
2151 }
2152 }
2153
2154 llvm::Value *Result = EmitLoadOfLValue(E);
2155
2156 // If -fdebug-info-for-profiling is specified, emit a pseudo variable and its
2157 // debug info for the pointer, even if there is no variable associated with
2158 // the pointer's expression.
2159 if (CGF.CGM.getCodeGenOpts().DebugInfoForProfiling && CGF.getDebugInfo()) {
2160 if (llvm::LoadInst *Load = dyn_cast<llvm::LoadInst>(Val: Result)) {
2161 if (llvm::GetElementPtrInst *GEP =
2162 dyn_cast<llvm::GetElementPtrInst>(Val: Load->getPointerOperand())) {
2163 if (llvm::Instruction *Pointer =
2164 dyn_cast<llvm::Instruction>(Val: GEP->getPointerOperand())) {
2165 QualType Ty = E->getBase()->getType();
2166 if (!E->isArrow())
2167 Ty = CGF.getContext().getPointerType(T: Ty);
2168 CGF.getDebugInfo()->EmitPseudoVariable(Builder, Value: Pointer, Ty);
2169 }
2170 }
2171 }
2172 }
2173 return Result;
2174}
2175
2176Value *ScalarExprEmitter::VisitArraySubscriptExpr(ArraySubscriptExpr *E) {
2177 TestAndClearIgnoreResultAssign();
2178
2179 // Emit subscript expressions in rvalue context's. For most cases, this just
2180 // loads the lvalue formed by the subscript expr. However, we have to be
2181 // careful, because the base of a vector subscript is occasionally an rvalue,
2182 // so we can't get it as an lvalue.
2183 if (!E->getBase()->getType()->isVectorType() &&
2184 !E->getBase()->getType()->isSveVLSBuiltinType())
2185 return EmitLoadOfLValue(E);
2186
2187 // Handle the vector case. The base must be a vector, the index must be an
2188 // integer value.
2189 Value *Base = Visit(E: E->getBase());
2190 Value *Idx = Visit(E: E->getIdx());
2191 QualType IdxTy = E->getIdx()->getType();
2192
2193 if (CGF.SanOpts.has(K: SanitizerKind::ArrayBounds))
2194 CGF.EmitBoundsCheck(ArrayExpr: E, ArrayExprBase: E->getBase(), Index: Idx, IndexType: IdxTy, /*Accessed*/true);
2195
2196 Value *Ret = Builder.CreateExtractElement(Vec: Base, Idx, Name: "vecext");
2197
2198 // Even being a scalar the `__mfp8` type corresponds to `<1 x i8>` in LLVM IR.
2199 if (E->getType()->isMFloat8Type())
2200 Ret = Builder.CreateInsertElement(
2201 Vec: llvm::PoisonValue::get(T: llvm::FixedVectorType::get(ElementType: CGF.Int8Ty, NumElts: 1)), NewElt: Ret,
2202 Idx: uint64_t(0), Name: "mfp8ext");
2203
2204 return Ret;
2205}
2206
2207Value *ScalarExprEmitter::VisitMatrixSingleSubscriptExpr(
2208 MatrixSingleSubscriptExpr *E) {
2209 TestAndClearIgnoreResultAssign();
2210
2211 auto *MatrixTy = E->getBase()->getType()->castAs<ConstantMatrixType>();
2212 unsigned NumRows = MatrixTy->getNumRows();
2213 unsigned NumColumns = MatrixTy->getNumColumns();
2214
2215 // Row index
2216 Value *RowIdx = CGF.EmitMatrixIndexExpr(E: E->getRowIdx());
2217 llvm::MatrixBuilder MB(Builder);
2218
2219 // The row index must be in [0, NumRows)
2220 if (CGF.CGM.getCodeGenOpts().OptimizationLevel > 0)
2221 MB.CreateIndexAssumption(Idx: RowIdx, NumElements: NumRows);
2222
2223 Value *FlatMatrix = Visit(E: E->getBase());
2224 llvm::Type *ElemTy = CGF.ConvertTypeForMem(T: MatrixTy->getElementType());
2225 auto *ResultTy = llvm::FixedVectorType::get(ElementType: ElemTy, NumElts: NumColumns);
2226 Value *RowVec = llvm::PoisonValue::get(T: ResultTy);
2227
2228 bool IsMatrixRowMajor =
2229 isMatrixRowMajor(LangOpts: CGF.getLangOpts(), T: E->getBase()->getType());
2230
2231 for (unsigned Col = 0; Col != NumColumns; ++Col) {
2232 Value *ColVal = llvm::ConstantInt::get(Ty: RowIdx->getType(), V: Col);
2233 Value *EltIdx = MB.CreateIndex(RowIdx, ColumnIdx: ColVal, NumRows, NumCols: NumColumns,
2234 IsMatrixRowMajor, Name: "matrix_row_idx");
2235 Value *Elt =
2236 Builder.CreateExtractElement(Vec: FlatMatrix, Idx: EltIdx, Name: "matrix_elem");
2237 Value *Lane = llvm::ConstantInt::get(Ty: Builder.getInt32Ty(), V: Col);
2238 RowVec = Builder.CreateInsertElement(Vec: RowVec, NewElt: Elt, Idx: Lane, Name: "matrix_row_ins");
2239 }
2240
2241 return CGF.EmitFromMemory(Value: RowVec, Ty: E->getType());
2242}
2243
2244Value *ScalarExprEmitter::VisitMatrixSubscriptExpr(MatrixSubscriptExpr *E) {
2245 TestAndClearIgnoreResultAssign();
2246
2247 // Handle the vector case. The base must be a vector, the index must be an
2248 // integer value.
2249 Value *RowIdx = CGF.EmitMatrixIndexExpr(E: E->getRowIdx());
2250 Value *ColumnIdx = CGF.EmitMatrixIndexExpr(E: E->getColumnIdx());
2251
2252 const auto *MatrixTy = E->getBase()->getType()->castAs<ConstantMatrixType>();
2253 llvm::MatrixBuilder MB(Builder);
2254
2255 Value *Idx;
2256 unsigned NumCols = MatrixTy->getNumColumns();
2257 unsigned NumRows = MatrixTy->getNumRows();
2258 bool IsMatrixRowMajor =
2259 isMatrixRowMajor(LangOpts: CGF.getLangOpts(), T: E->getBase()->getType());
2260 Idx = MB.CreateIndex(RowIdx, ColumnIdx, NumRows, NumCols, IsMatrixRowMajor);
2261
2262 if (CGF.CGM.getCodeGenOpts().OptimizationLevel > 0)
2263 MB.CreateIndexAssumption(Idx, NumElements: MatrixTy->getNumElementsFlattened());
2264
2265 Value *Matrix = Visit(E: E->getBase());
2266
2267 // TODO: Should we emit bounds checks with SanitizerKind::ArrayBounds?
2268 return Builder.CreateExtractElement(Vec: Matrix, Idx, Name: "matrixext");
2269}
2270
2271static int getMaskElt(llvm::ShuffleVectorInst *SVI, unsigned Idx,
2272 unsigned Off) {
2273 int MV = SVI->getMaskValue(Elt: Idx);
2274 if (MV == -1)
2275 return -1;
2276 return Off + MV;
2277}
2278
2279static int getAsInt32(llvm::ConstantInt *C, llvm::Type *I32Ty) {
2280 assert(llvm::ConstantInt::isValueValidForType(I32Ty, C->getZExtValue()) &&
2281 "Index operand too large for shufflevector mask!");
2282 return C->getZExtValue();
2283}
2284
2285Value *ScalarExprEmitter::VisitInitListExpr(InitListExpr *E) {
2286 bool Ignore = TestAndClearIgnoreResultAssign();
2287 (void)Ignore;
2288 unsigned NumInitElements = E->getNumInits();
2289 assert((Ignore == false ||
2290 (NumInitElements == 0 && E->getType()->isVoidType())) &&
2291 "init list ignored");
2292
2293 // HLSL initialization lists in the AST are an expansion which can contain
2294 // side-effecting expressions wrapped in opaque value expressions. To properly
2295 // emit these we need to emit the opaque values before we emit the argument
2296 // expressions themselves. This is a little hacky, but it prevents us needing
2297 // to do a bigger AST-level change for a language feature that we need
2298 // deprecate in the near future. See related HLSL language proposals in the
2299 // proposals (https://github.com/microsoft/hlsl-specs/blob/main/proposals):
2300 // * 0005-strict-initializer-lists.md
2301 // * 0032-constructors.md
2302 if (CGF.getLangOpts().HLSL)
2303 CGF.CGM.getHLSLRuntime().emitInitListOpaqueValues(CGF, E);
2304
2305 if (E->hadArrayRangeDesignator())
2306 CGF.ErrorUnsupported(S: E, Type: "GNU array range designator extension");
2307
2308 llvm::VectorType *VType =
2309 dyn_cast<llvm::VectorType>(Val: ConvertType(T: E->getType()));
2310
2311 if (!VType) {
2312 if (NumInitElements == 0) {
2313 // C++11 value-initialization for the scalar.
2314 return EmitNullValue(Ty: E->getType());
2315 }
2316 // We have a scalar in braces. Just use the first element.
2317 return Visit(E: E->getInit(Init: 0));
2318 }
2319
2320 if (isa<llvm::ScalableVectorType>(Val: VType)) {
2321 if (NumInitElements == 0) {
2322 // C++11 value-initialization for the vector.
2323 return EmitNullValue(Ty: E->getType());
2324 }
2325
2326 if (NumInitElements == 1) {
2327 Expr *InitVector = E->getInit(Init: 0);
2328
2329 // Initialize from another scalable vector of the same type.
2330 if (InitVector->getType().getCanonicalType() ==
2331 E->getType().getCanonicalType())
2332 return Visit(E: InitVector);
2333 }
2334
2335 llvm_unreachable("Unexpected initialization of a scalable vector!");
2336 }
2337
2338 unsigned ResElts = cast<llvm::FixedVectorType>(Val: VType)->getNumElements();
2339
2340 // For column-major matrix types, we insert elements directly at their
2341 // column-major positions rather than inserting sequentially and shuffling.
2342 const ConstantMatrixType *ColMajorMT = nullptr;
2343 if (const auto *MT = E->getType()->getAs<ConstantMatrixType>();
2344 MT && !isMatrixRowMajor(LangOpts: CGF.getLangOpts(), T: E->getType()))
2345 ColMajorMT = MT;
2346
2347 // Loop over initializers collecting the Value for each, and remembering
2348 // whether the source was swizzle (ExtVectorElementExpr). This will allow
2349 // us to fold the shuffle for the swizzle into the shuffle for the vector
2350 // initializer, since LLVM optimizers generally do not want to touch
2351 // shuffles.
2352 unsigned CurIdx = 0;
2353 bool VIsPoisonShuffle = false;
2354 llvm::Value *V = llvm::PoisonValue::get(T: VType);
2355 for (unsigned i = 0; i != NumInitElements; ++i) {
2356 Expr *IE = E->getInit(Init: i);
2357 Value *Init = Visit(E: IE);
2358 SmallVector<int, 16> Args;
2359
2360 llvm::VectorType *VVT = dyn_cast<llvm::VectorType>(Val: Init->getType());
2361
2362 // Handle scalar elements. If the scalar initializer is actually one
2363 // element of a different vector of the same width, use shuffle instead of
2364 // extract+insert.
2365 if (!VVT) {
2366 if (isa<ExtVectorElementExpr>(Val: IE)) {
2367 llvm::ExtractElementInst *EI = cast<llvm::ExtractElementInst>(Val: Init);
2368
2369 if (cast<llvm::FixedVectorType>(Val: EI->getVectorOperandType())
2370 ->getNumElements() == ResElts) {
2371 llvm::ConstantInt *C = cast<llvm::ConstantInt>(Val: EI->getIndexOperand());
2372 Value *LHS = nullptr, *RHS = nullptr;
2373 if (CurIdx == 0) {
2374 // insert into poison -> shuffle (src, poison)
2375 // shufflemask must use an i32
2376 Args.push_back(Elt: getAsInt32(C, I32Ty: CGF.Int32Ty));
2377 Args.resize(N: ResElts, NV: -1);
2378
2379 LHS = EI->getVectorOperand();
2380 RHS = V;
2381 VIsPoisonShuffle = true;
2382 } else if (VIsPoisonShuffle) {
2383 // insert into poison shuffle && size match -> shuffle (v, src)
2384 llvm::ShuffleVectorInst *SVV = cast<llvm::ShuffleVectorInst>(Val: V);
2385 for (unsigned j = 0; j != CurIdx; ++j)
2386 Args.push_back(Elt: getMaskElt(SVI: SVV, Idx: j, Off: 0));
2387 Args.push_back(Elt: ResElts + C->getZExtValue());
2388 Args.resize(N: ResElts, NV: -1);
2389
2390 LHS = cast<llvm::ShuffleVectorInst>(Val: V)->getOperand(i_nocapture: 0);
2391 RHS = EI->getVectorOperand();
2392 VIsPoisonShuffle = false;
2393 }
2394 if (!Args.empty()) {
2395 V = Builder.CreateShuffleVector(V1: LHS, V2: RHS, Mask: Args);
2396 ++CurIdx;
2397 continue;
2398 }
2399 }
2400 }
2401 unsigned InsertIdx =
2402 ColMajorMT
2403 ? ColMajorMT->mapRowMajorToColumnMajorFlattenedIndex(RowMajorIdx: CurIdx)
2404 : CurIdx;
2405 V = Builder.CreateInsertElement(Vec: V, NewElt: Init, Idx: Builder.getInt32(C: InsertIdx),
2406 Name: "vecinit");
2407 VIsPoisonShuffle = false;
2408 ++CurIdx;
2409 continue;
2410 }
2411
2412 unsigned InitElts = cast<llvm::FixedVectorType>(Val: VVT)->getNumElements();
2413
2414 // If the initializer is an ExtVecEltExpr (a swizzle), and the swizzle's
2415 // input is the same width as the vector being constructed, generate an
2416 // optimized shuffle of the swizzle input into the result.
2417 unsigned Offset = (CurIdx == 0) ? 0 : ResElts;
2418 if (isa<ExtVectorElementExpr>(Val: IE)) {
2419 llvm::ShuffleVectorInst *SVI = cast<llvm::ShuffleVectorInst>(Val: Init);
2420 Value *SVOp = SVI->getOperand(i_nocapture: 0);
2421 auto *OpTy = cast<llvm::FixedVectorType>(Val: SVOp->getType());
2422
2423 if (OpTy->getNumElements() == ResElts) {
2424 for (unsigned j = 0; j != CurIdx; ++j) {
2425 // If the current vector initializer is a shuffle with poison, merge
2426 // this shuffle directly into it.
2427 if (VIsPoisonShuffle) {
2428 Args.push_back(Elt: getMaskElt(SVI: cast<llvm::ShuffleVectorInst>(Val: V), Idx: j, Off: 0));
2429 } else {
2430 Args.push_back(Elt: j);
2431 }
2432 }
2433 for (unsigned j = 0, je = InitElts; j != je; ++j)
2434 Args.push_back(Elt: getMaskElt(SVI, Idx: j, Off: Offset));
2435 Args.resize(N: ResElts, NV: -1);
2436
2437 if (VIsPoisonShuffle)
2438 V = cast<llvm::ShuffleVectorInst>(Val: V)->getOperand(i_nocapture: 0);
2439
2440 Init = SVOp;
2441 }
2442 }
2443
2444 // Extend init to result vector length, and then shuffle its contribution
2445 // to the vector initializer into V.
2446 if (Args.empty()) {
2447 for (unsigned j = 0; j != InitElts; ++j)
2448 Args.push_back(Elt: j);
2449 Args.resize(N: ResElts, NV: -1);
2450 Init = Builder.CreateShuffleVector(V: Init, Mask: Args, Name: "vext");
2451
2452 Args.clear();
2453 for (unsigned j = 0; j != CurIdx; ++j)
2454 Args.push_back(Elt: j);
2455 for (unsigned j = 0; j != InitElts; ++j)
2456 Args.push_back(Elt: j + Offset);
2457 Args.resize(N: ResElts, NV: -1);
2458 }
2459
2460 // If V is poison, make sure it ends up on the RHS of the shuffle to aid
2461 // merging subsequent shuffles into this one.
2462 if (CurIdx == 0)
2463 std::swap(a&: V, b&: Init);
2464 V = Builder.CreateShuffleVector(V1: V, V2: Init, Mask: Args, Name: "vecinit");
2465 VIsPoisonShuffle = isa<llvm::PoisonValue>(Val: Init);
2466 CurIdx += InitElts;
2467 }
2468
2469 // FIXME: evaluate codegen vs. shuffling against constant null vector.
2470 // Emit remaining default initializers.
2471 llvm::Type *EltTy = VType->getElementType();
2472
2473 // Emit remaining default initializers
2474 for (/* Do not initialize i*/; CurIdx < ResElts; ++CurIdx) {
2475 unsigned InsertIdx =
2476 ColMajorMT ? ColMajorMT->mapRowMajorToColumnMajorFlattenedIndex(RowMajorIdx: CurIdx)
2477 : CurIdx;
2478 Value *Idx = Builder.getInt32(C: InsertIdx);
2479 llvm::Value *Init = llvm::Constant::getNullValue(Ty: EltTy);
2480 V = Builder.CreateInsertElement(Vec: V, NewElt: Init, Idx, Name: "vecinit");
2481 }
2482
2483 return V;
2484}
2485
2486static bool isDeclRefKnownNonNull(CodeGenFunction &CGF, const ValueDecl *D) {
2487 return !D->isWeak();
2488}
2489
2490static bool isLValueKnownNonNull(CodeGenFunction &CGF, const Expr *E) {
2491 E = E->IgnoreParens();
2492
2493 if (const auto *UO = dyn_cast<UnaryOperator>(Val: E))
2494 if (UO->getOpcode() == UO_Deref)
2495 return CGF.isPointerKnownNonNull(E: UO->getSubExpr());
2496
2497 if (const auto *DRE = dyn_cast<DeclRefExpr>(Val: E))
2498 return isDeclRefKnownNonNull(CGF, D: DRE->getDecl());
2499
2500 if (const auto *ME = dyn_cast<MemberExpr>(Val: E)) {
2501 if (isa<FieldDecl>(Val: ME->getMemberDecl()))
2502 return true;
2503 return isDeclRefKnownNonNull(CGF, D: ME->getMemberDecl());
2504 }
2505
2506 // Array subscripts? Anything else?
2507
2508 return false;
2509}
2510
2511bool CodeGenFunction::isPointerKnownNonNull(const Expr *E) {
2512 assert(E->getType()->isSignableType(getContext()));
2513
2514 E = E->IgnoreParens();
2515
2516 if (isa<CXXThisExpr>(Val: E))
2517 return true;
2518
2519 if (const auto *UO = dyn_cast<UnaryOperator>(Val: E))
2520 if (UO->getOpcode() == UO_AddrOf)
2521 return isLValueKnownNonNull(CGF&: *this, E: UO->getSubExpr());
2522
2523 if (const auto *CE = dyn_cast<CastExpr>(Val: E))
2524 if (CE->getCastKind() == CK_FunctionToPointerDecay ||
2525 CE->getCastKind() == CK_ArrayToPointerDecay)
2526 return isLValueKnownNonNull(CGF&: *this, E: CE->getSubExpr());
2527
2528 // Maybe honor __nonnull?
2529
2530 return false;
2531}
2532
2533bool CodeGenFunction::ShouldNullCheckClassCastValue(const CastExpr *CE) {
2534 const Expr *E = CE->getSubExpr();
2535
2536 if (CE->getCastKind() == CK_UncheckedDerivedToBase)
2537 return false;
2538
2539 if (isa<CXXThisExpr>(Val: E->IgnoreParens())) {
2540 // We always assume that 'this' is never null.
2541 return false;
2542 }
2543
2544 if (const ImplicitCastExpr *ICE = dyn_cast<ImplicitCastExpr>(Val: CE)) {
2545 // And that glvalue casts are never null.
2546 if (ICE->isGLValue())
2547 return false;
2548 }
2549
2550 return true;
2551}
2552
2553// RHS is an aggregate type
2554static Value *EmitHLSLElementwiseCast(CodeGenFunction &CGF, LValue SrcVal,
2555 QualType DestTy, SourceLocation Loc) {
2556 SmallVector<LValue, 16> LoadList;
2557 CGF.FlattenAccessAndTypeLValue(LVal: SrcVal, AccessList&: LoadList);
2558 // Dest is either a vector, constant matrix, or a builtin
2559 // if its a vector create a temp alloca to store into and return that
2560 if (auto *VecTy = DestTy->getAs<VectorType>()) {
2561 assert(LoadList.size() >= VecTy->getNumElements() &&
2562 "Flattened type on RHS must have the same number or more elements "
2563 "than vector on LHS.");
2564 llvm::Value *V = CGF.Builder.CreateLoad(
2565 Addr: CGF.CreateIRTempWithoutCast(T: DestTy, Name: "flatcast.tmp"));
2566 // write to V.
2567 for (unsigned I = 0, E = VecTy->getNumElements(); I < E; I++) {
2568 RValue RVal = CGF.EmitLoadOfLValue(V: LoadList[I], Loc);
2569 assert(RVal.isScalar() &&
2570 "All flattened source values should be scalars.");
2571 llvm::Value *Cast =
2572 CGF.EmitScalarConversion(Src: RVal.getScalarVal(), SrcTy: LoadList[I].getType(),
2573 DstTy: VecTy->getElementType(), Loc);
2574 V = CGF.Builder.CreateInsertElement(Vec: V, NewElt: Cast, Idx: I);
2575 }
2576 return V;
2577 }
2578 if (auto *MatTy = DestTy->getAs<ConstantMatrixType>()) {
2579 assert(LoadList.size() >= MatTy->getNumElementsFlattened() &&
2580 "Flattened type on RHS must have the same number or more elements "
2581 "than vector on LHS.");
2582
2583 bool IsRowMajor = isMatrixRowMajor(LangOpts: CGF.getLangOpts(), T: DestTy);
2584
2585 llvm::Value *V = CGF.Builder.CreateLoad(
2586 Addr: CGF.CreateIRTempWithoutCast(T: DestTy, Name: "flatcast.tmp"));
2587 // V is an allocated temporary for constructing the matrix.
2588 for (unsigned Row = 0, RE = MatTy->getNumRows(); Row < RE; Row++) {
2589 for (unsigned Col = 0, CE = MatTy->getNumColumns(); Col < CE; Col++) {
2590 // When interpreted as a matrix, \p LoadList is *always* row-major order
2591 // regardless of the default matrix memory layout.
2592 unsigned LoadIdx = MatTy->getRowMajorFlattenedIndex(Row, Column: Col);
2593 RValue RVal = CGF.EmitLoadOfLValue(V: LoadList[LoadIdx], Loc);
2594 assert(RVal.isScalar() &&
2595 "All flattened source values should be scalars.");
2596 llvm::Value *Cast = CGF.EmitScalarConversion(
2597 Src: RVal.getScalarVal(), SrcTy: LoadList[LoadIdx].getType(),
2598 DstTy: MatTy->getElementType(), Loc);
2599 unsigned MatrixIdx = MatTy->getFlattenedIndex(Row, Column: Col, IsRowMajor);
2600 V = CGF.Builder.CreateInsertElement(Vec: V, NewElt: Cast, Idx: MatrixIdx);
2601 }
2602 }
2603 return V;
2604 }
2605 // if its a builtin just do an extract element or load.
2606 assert(DestTy->isBuiltinType() &&
2607 "Destination type must be a vector, matrix, or builtin type.");
2608 RValue RVal = CGF.EmitLoadOfLValue(V: LoadList[0], Loc);
2609 assert(RVal.isScalar() && "All flattened source values should be scalars.");
2610 return CGF.EmitScalarConversion(Src: RVal.getScalarVal(), SrcTy: LoadList[0].getType(),
2611 DstTy: DestTy, Loc);
2612}
2613
2614// VisitCastExpr - Emit code for an explicit or implicit cast. Implicit casts
2615// have to handle a more broad range of conversions than explicit casts, as they
2616// handle things like function to ptr-to-function decay etc.
2617Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) {
2618 llvm::scope_exit RestoreCurCast(
2619 [this, Prev = CGF.CurCast] { CGF.CurCast = Prev; });
2620 CGF.CurCast = CE;
2621
2622 Expr *E = CE->getSubExpr();
2623 QualType DestTy = CE->getType();
2624 CastKind Kind = CE->getCastKind();
2625 CodeGenFunction::CGFPOptionsRAII FPOptions(CGF, CE);
2626
2627 // These cases are generally not written to ignore the result of
2628 // evaluating their sub-expressions, so we clear this now.
2629 bool Ignored = TestAndClearIgnoreResultAssign();
2630
2631 // Since almost all cast kinds apply to scalars, this switch doesn't have
2632 // a default case, so the compiler will warn on a missing case. The cases
2633 // are in the same order as in the CastKind enum.
2634 switch (Kind) {
2635 case CK_Dependent: llvm_unreachable("dependent cast kind in IR gen!");
2636 case CK_BuiltinFnToFnPtr:
2637 llvm_unreachable("builtin functions are handled elsewhere");
2638
2639 case CK_LValueBitCast:
2640 case CK_ObjCObjectLValueCast: {
2641 Address Addr = EmitLValue(E).getAddress();
2642 Addr = Addr.withElementType(ElemTy: CGF.ConvertTypeForMem(T: DestTy));
2643 LValue LV = CGF.MakeAddrLValue(Addr, T: DestTy);
2644 return EmitLoadOfLValue(LV, Loc: CE->getExprLoc());
2645 }
2646
2647 case CK_LValueToRValueBitCast: {
2648 LValue SourceLVal = CGF.EmitLValue(E);
2649 Address Addr =
2650 SourceLVal.getAddress().withElementType(ElemTy: CGF.ConvertTypeForMem(T: DestTy));
2651 LValue DestLV = CGF.MakeAddrLValue(Addr, T: DestTy);
2652 DestLV.setTBAAInfo(TBAAAccessInfo::getMayAliasInfo());
2653 return EmitLoadOfLValue(LV: DestLV, Loc: CE->getExprLoc());
2654 }
2655
2656 case CK_CPointerToObjCPointerCast:
2657 case CK_BlockPointerToObjCPointerCast:
2658 case CK_AnyPointerToBlockPointerCast:
2659 case CK_BitCast: {
2660 Value *Src = Visit(E);
2661 llvm::Type *SrcTy = Src->getType();
2662 llvm::Type *DstTy = ConvertType(T: DestTy);
2663
2664 // FIXME: this is a gross but seemingly necessary workaround for an issue
2665 // manifesting when a target uses a non-default AS for indirect sret args,
2666 // but the source HLL is generic, wherein a valid C-cast or reinterpret_cast
2667 // on the address of a local struct that gets returned by value yields an
2668 // invalid bitcast from the a pointer to the IndirectAS to a pointer to the
2669 // DefaultAS. We can only do this subversive thing because sret args are
2670 // manufactured and them residing in the IndirectAS is a target specific
2671 // detail, and doing an AS cast here still retains the semantics the user
2672 // expects. It is desirable to remove this iff a better solution is found.
2673 if (auto A = dyn_cast<llvm::Argument>(Val: Src); A && A->hasStructRetAttr())
2674 return CGF.performAddrSpaceCast(Src, DestTy: DstTy);
2675
2676 // FIXME: Similarly to the sret case above, we need to handle BitCasts that
2677 // involve implicit address space conversions. This arises when the source
2678 // language lacks explicit address spaces, but the target's data layout
2679 // assigns different address spaces (e.g., program address space for
2680 // function pointers). Since Sema operates on Clang types (which don't carry
2681 // this information) and selects CK_BitCast, we must detect the address
2682 // space mismatch here in CodeGen when lowering to LLVM types. The most
2683 // common case is casting function pointers (which get the program AS from
2684 // the data layout) to/from object pointers (which use the default AS).
2685 // Ideally, this would be resolved at a higher level, but that would require
2686 // exposing data layout details to Sema.
2687 if (SrcTy->isPtrOrPtrVectorTy() && DstTy->isPtrOrPtrVectorTy() &&
2688 SrcTy->getPointerAddressSpace() != DstTy->getPointerAddressSpace()) {
2689 return CGF.performAddrSpaceCast(Src, DestTy: DstTy);
2690 }
2691
2692 assert(
2693 (!SrcTy->isPtrOrPtrVectorTy() || !DstTy->isPtrOrPtrVectorTy() ||
2694 SrcTy->getPointerAddressSpace() == DstTy->getPointerAddressSpace()) &&
2695 "Address-space cast must be used to convert address spaces");
2696
2697 if (CGF.SanOpts.has(K: SanitizerKind::CFIUnrelatedCast)) {
2698 if (auto *PT = DestTy->getAs<PointerType>()) {
2699 CGF.EmitVTablePtrCheckForCast(
2700 T: PT->getPointeeType(),
2701 Derived: Address(Src,
2702 CGF.ConvertTypeForMem(
2703 T: E->getType()->castAs<PointerType>()->getPointeeType()),
2704 CGF.getPointerAlign()),
2705 /*MayBeNull=*/true, TCK: CodeGenFunction::CFITCK_UnrelatedCast,
2706 Loc: CE->getBeginLoc());
2707 }
2708 }
2709
2710 if (CGF.CGM.getCodeGenOpts().StrictVTablePointers) {
2711 const QualType SrcType = E->getType();
2712
2713 if (SrcType.mayBeNotDynamicClass() && DestTy.mayBeDynamicClass()) {
2714 // Casting to pointer that could carry dynamic information (provided by
2715 // invariant.group) requires launder.
2716 Src = Builder.CreateLaunderInvariantGroup(Ptr: Src);
2717 } else if (SrcType.mayBeDynamicClass() && DestTy.mayBeNotDynamicClass()) {
2718 // Casting to pointer that does not carry dynamic information (provided
2719 // by invariant.group) requires stripping it. Note that we don't do it
2720 // if the source could not be dynamic type and destination could be
2721 // dynamic because dynamic information is already laundered. It is
2722 // because launder(strip(src)) == launder(src), so there is no need to
2723 // add extra strip before launder.
2724 Src = Builder.CreateStripInvariantGroup(Ptr: Src);
2725 }
2726 }
2727
2728 // Update heapallocsite metadata when there is an explicit pointer cast.
2729 if (auto *CI = dyn_cast<llvm::CallBase>(Val: Src)) {
2730 if (CI->getMetadata(Kind: "heapallocsite") && isa<ExplicitCastExpr>(Val: CE) &&
2731 !isa<CastExpr>(Val: E)) {
2732 QualType PointeeType = DestTy->getPointeeType();
2733 if (!PointeeType.isNull())
2734 CGF.getDebugInfo()->addHeapAllocSiteMetadata(CallSite: CI, AllocatedTy: PointeeType,
2735 Loc: CE->getExprLoc());
2736 }
2737 }
2738
2739 // If Src is a fixed vector and Dst is a scalable vector, and both have the
2740 // same element type, use the llvm.vector.insert intrinsic to perform the
2741 // bitcast.
2742 if (auto *FixedSrcTy = dyn_cast<llvm::FixedVectorType>(Val: SrcTy)) {
2743 if (auto *ScalableDstTy = dyn_cast<llvm::ScalableVectorType>(Val: DstTy)) {
2744 // If we are casting a fixed i8 vector to a scalable i1 predicate
2745 // vector, use a vector insert and bitcast the result.
2746 if (ScalableDstTy->getElementType()->isIntegerTy(BitWidth: 1) &&
2747 FixedSrcTy->getElementType()->isIntegerTy(BitWidth: 8)) {
2748 ScalableDstTy = llvm::ScalableVectorType::get(
2749 ElementType: FixedSrcTy->getElementType(),
2750 MinNumElts: llvm::divideCeil(
2751 Numerator: ScalableDstTy->getElementCount().getKnownMinValue(), Denominator: 8));
2752 }
2753 if (FixedSrcTy->getElementType() == ScalableDstTy->getElementType()) {
2754 llvm::Value *PoisonVec = llvm::PoisonValue::get(T: ScalableDstTy);
2755 llvm::Value *Result = Builder.CreateInsertVector(
2756 DstType: ScalableDstTy, SrcVec: PoisonVec, SubVec: Src, Idx: uint64_t(0), Name: "cast.scalable");
2757 ScalableDstTy = cast<llvm::ScalableVectorType>(
2758 Val: llvm::VectorType::getWithSizeAndScalar(SizeTy: ScalableDstTy, EltTy: DstTy));
2759 if (Result->getType() != ScalableDstTy)
2760 Result = Builder.CreateBitCast(V: Result, DestTy: ScalableDstTy);
2761 if (Result->getType() != DstTy)
2762 Result = Builder.CreateExtractVector(DstType: DstTy, SrcVec: Result, Idx: uint64_t(0));
2763 return Result;
2764 }
2765 }
2766 }
2767
2768 // If Src is a scalable vector and Dst is a fixed vector, and both have the
2769 // same element type, use the llvm.vector.extract intrinsic to perform the
2770 // bitcast.
2771 if (auto *ScalableSrcTy = dyn_cast<llvm::ScalableVectorType>(Val: SrcTy)) {
2772 if (auto *FixedDstTy = dyn_cast<llvm::FixedVectorType>(Val: DstTy)) {
2773 // If we are casting a scalable i1 predicate vector to a fixed i8
2774 // vector, bitcast the source and use a vector extract.
2775 if (ScalableSrcTy->getElementType()->isIntegerTy(BitWidth: 1) &&
2776 FixedDstTy->getElementType()->isIntegerTy(BitWidth: 8)) {
2777 if (!ScalableSrcTy->getElementCount().isKnownMultipleOf(RHS: 8)) {
2778 ScalableSrcTy = llvm::ScalableVectorType::get(
2779 ElementType: ScalableSrcTy->getElementType(),
2780 MinNumElts: llvm::alignTo<8>(
2781 Value: ScalableSrcTy->getElementCount().getKnownMinValue()));
2782 llvm::Value *ZeroVec = llvm::Constant::getNullValue(Ty: ScalableSrcTy);
2783 Src = Builder.CreateInsertVector(DstType: ScalableSrcTy, SrcVec: ZeroVec, SubVec: Src,
2784 Idx: uint64_t(0));
2785 }
2786
2787 ScalableSrcTy = llvm::ScalableVectorType::get(
2788 ElementType: FixedDstTy->getElementType(),
2789 MinNumElts: ScalableSrcTy->getElementCount().getKnownMinValue() / 8);
2790 Src = Builder.CreateBitCast(V: Src, DestTy: ScalableSrcTy);
2791 }
2792 if (ScalableSrcTy->getElementType() == FixedDstTy->getElementType())
2793 return Builder.CreateExtractVector(DstType: DstTy, SrcVec: Src, Idx: uint64_t(0),
2794 Name: "cast.fixed");
2795 }
2796 }
2797
2798 // Perform VLAT <-> VLST bitcast through memory.
2799 // TODO: since the llvm.vector.{insert,extract} intrinsics
2800 // require the element types of the vectors to be the same, we
2801 // need to keep this around for bitcasts between VLAT <-> VLST where
2802 // the element types of the vectors are not the same, until we figure
2803 // out a better way of doing these casts.
2804 if ((isa<llvm::FixedVectorType>(Val: SrcTy) &&
2805 isa<llvm::ScalableVectorType>(Val: DstTy)) ||
2806 (isa<llvm::ScalableVectorType>(Val: SrcTy) &&
2807 isa<llvm::FixedVectorType>(Val: DstTy))) {
2808 Address Addr = CGF.CreateDefaultAlignTempAlloca(Ty: SrcTy, Name: "saved-value");
2809 LValue LV = CGF.MakeAddrLValue(Addr, T: E->getType());
2810 CGF.EmitStoreOfScalar(value: Src, lvalue: LV);
2811 Addr = Addr.withElementType(ElemTy: CGF.ConvertTypeForMem(T: DestTy));
2812 LValue DestLV = CGF.MakeAddrLValue(Addr, T: DestTy);
2813 DestLV.setTBAAInfo(TBAAAccessInfo::getMayAliasInfo());
2814 return EmitLoadOfLValue(LV: DestLV, Loc: CE->getExprLoc());
2815 }
2816
2817 llvm::Value *Result = Builder.CreateBitCast(V: Src, DestTy: DstTy);
2818 return CGF.authPointerToPointerCast(ResultPtr: Result, SourceType: E->getType(), DestType: DestTy);
2819 }
2820 case CK_AddressSpaceConversion: {
2821 llvm::Type *DestLTy = ConvertType(T: DestTy);
2822 // WebAssembly reference types are opaque target extension types so an
2823 // "address space conversion" involving them is not a real pointer cast.
2824 auto IsWasmFuncref = [](llvm::Type *T) {
2825 auto *TET = dyn_cast<llvm::TargetExtType>(Val: T);
2826 return TET && TET->getName() == "wasm.funcref";
2827 };
2828 bool SrcIsFuncref = IsWasmFuncref(ConvertType(T: E->getType()));
2829 bool DestIsFuncref = IsWasmFuncref(DestLTy);
2830 if (SrcIsFuncref && DestIsFuncref) {
2831 // funcref -> funcref (e.g. between differently-typed funcrefs) is the
2832 // identity on the opaque reference value.
2833 return Visit(E);
2834 }
2835 if (SrcIsFuncref && !DestIsFuncref) {
2836 // funcref -> pointer: use wasm_funcref_to_ptr. This will probably crash
2837 // later in codegen since we haven't implemented a way to actually get a
2838 // function pointer from a funcref.
2839 llvm::Function *ToPtr =
2840 CGF.CGM.getIntrinsic(IID: llvm::Intrinsic::wasm_funcref_to_ptr);
2841 return CGF.Builder.CreateCall(Callee: ToPtr, Args: {Visit(E)});
2842 }
2843 if (!SrcIsFuncref && DestIsFuncref) {
2844 // A null function pointer converts to a null funcref (ref.null func),
2845 // rather than a table lookup at index 0.
2846 Expr::EvalResult NullResult;
2847 if (E->EvaluateAsRValue(Result&: NullResult, Ctx: CGF.getContext()) &&
2848 NullResult.Val.isNullPointer()) {
2849 if (NullResult.HasSideEffects)
2850 Visit(E);
2851 return llvm::Constant::getNullValue(Ty: DestLTy);
2852 }
2853 // pointer -> funcref: do a table.get from the indirect function table.
2854 llvm::Function *ToFuncref =
2855 CGF.CGM.getIntrinsic(IID: llvm::Intrinsic::wasm_ptr_to_funcref);
2856 return CGF.Builder.CreateCall(Callee: ToFuncref, Args: {Visit(E)});
2857 }
2858 Expr::EvalResult Result;
2859 if (E->EvaluateAsRValue(Result, Ctx: CGF.getContext()) &&
2860 Result.Val.isNullPointer()) {
2861 // If E has side effect, it is emitted even if its final result is a
2862 // null pointer. In that case, a DCE pass should be able to
2863 // eliminate the useless instructions emitted during translating E.
2864 if (Result.HasSideEffects)
2865 Visit(E);
2866 return CGF.CGM.getNullPointer(T: cast<llvm::PointerType>(Val: DestLTy), QT: DestTy);
2867 }
2868 // Since target may map different address spaces in AST to the same address
2869 // space, an address space conversion may end up as a bitcast.
2870 return CGF.performAddrSpaceCast(Src: Visit(E), DestTy: DestLTy);
2871 }
2872 case CK_AtomicToNonAtomic:
2873 case CK_NonAtomicToAtomic:
2874 case CK_UserDefinedConversion:
2875 return Visit(E);
2876
2877 case CK_NoOp: {
2878 return CE->changesVolatileQualification() ? EmitLoadOfLValue(E: CE) : Visit(E);
2879 }
2880
2881 case CK_BaseToDerived: {
2882 const CXXRecordDecl *DerivedClassDecl = DestTy->getPointeeCXXRecordDecl();
2883 assert(DerivedClassDecl && "BaseToDerived arg isn't a C++ object pointer!");
2884
2885 Address Base = CGF.EmitPointerWithAlignment(Addr: E);
2886 Address Derived =
2887 CGF.GetAddressOfDerivedClass(Value: Base, Derived: DerivedClassDecl,
2888 PathBegin: CE->path_begin(), PathEnd: CE->path_end(),
2889 NullCheckValue: CGF.ShouldNullCheckClassCastValue(CE));
2890
2891 // C++11 [expr.static.cast]p11: Behavior is undefined if a downcast is
2892 // performed and the object is not of the derived type.
2893 if (CGF.sanitizePerformTypeCheck())
2894 CGF.EmitTypeCheck(TCK: CodeGenFunction::TCK_DowncastPointer, Loc: CE->getExprLoc(),
2895 Addr: Derived, Type: DestTy->getPointeeType());
2896
2897 if (CGF.SanOpts.has(K: SanitizerKind::CFIDerivedCast))
2898 CGF.EmitVTablePtrCheckForCast(T: DestTy->getPointeeType(), Derived,
2899 /*MayBeNull=*/true,
2900 TCK: CodeGenFunction::CFITCK_DerivedCast,
2901 Loc: CE->getBeginLoc());
2902
2903 return CGF.getAsNaturalPointerTo(Addr: Derived, PointeeType: CE->getType()->getPointeeType());
2904 }
2905 case CK_UncheckedDerivedToBase:
2906 case CK_DerivedToBase: {
2907 // The EmitPointerWithAlignment path does this fine; just discard
2908 // the alignment.
2909 return CGF.getAsNaturalPointerTo(Addr: CGF.EmitPointerWithAlignment(Addr: CE),
2910 PointeeType: CE->getType()->getPointeeType());
2911 }
2912
2913 case CK_Dynamic: {
2914 Address V = CGF.EmitPointerWithAlignment(Addr: E);
2915 const CXXDynamicCastExpr *DCE = cast<CXXDynamicCastExpr>(Val: CE);
2916 return CGF.EmitDynamicCast(V, DCE);
2917 }
2918
2919 case CK_ArrayToPointerDecay:
2920 return CGF.getAsNaturalPointerTo(Addr: CGF.EmitArrayToPointerDecay(Array: E),
2921 PointeeType: CE->getType()->getPointeeType());
2922 case CK_FunctionToPointerDecay:
2923 return EmitLValue(E).getPointer(CGF);
2924
2925 case CK_NullToPointer:
2926 if (MustVisitNullValue(E))
2927 CGF.EmitIgnoredExpr(E);
2928
2929 return CGF.CGM.getNullPointer(T: cast<llvm::PointerType>(Val: ConvertType(T: DestTy)),
2930 QT: DestTy);
2931
2932 case CK_NullToMemberPointer: {
2933 if (MustVisitNullValue(E))
2934 CGF.EmitIgnoredExpr(E);
2935
2936 const MemberPointerType *MPT = CE->getType()->getAs<MemberPointerType>();
2937 return CGF.CGM.getCXXABI().EmitNullMemberPointer(MPT);
2938 }
2939
2940 case CK_ReinterpretMemberPointer:
2941 case CK_BaseToDerivedMemberPointer:
2942 case CK_DerivedToBaseMemberPointer: {
2943 Value *Src = Visit(E);
2944
2945 // Note that the AST doesn't distinguish between checked and
2946 // unchecked member pointer conversions, so we always have to
2947 // implement checked conversions here. This is inefficient when
2948 // actual control flow may be required in order to perform the
2949 // check, which it is for data member pointers (but not member
2950 // function pointers on Itanium and ARM).
2951 return CGF.CGM.getCXXABI().EmitMemberPointerConversion(CGF, E: CE, Src);
2952 }
2953
2954 case CK_ARCProduceObject:
2955 return CGF.EmitARCRetainScalarExpr(expr: E);
2956 case CK_ARCConsumeObject:
2957 return CGF.EmitObjCConsumeObject(T: E->getType(), Ptr: Visit(E));
2958 case CK_ARCReclaimReturnedObject:
2959 return CGF.EmitARCReclaimReturnedObject(e: E, /*allowUnsafe*/ allowUnsafeClaim: Ignored);
2960 case CK_ARCExtendBlockObject:
2961 return CGF.EmitARCExtendBlockObject(expr: E);
2962
2963 case CK_CopyAndAutoreleaseBlockObject:
2964 return CGF.EmitBlockCopyAndAutorelease(Block: Visit(E), Ty: E->getType());
2965
2966 case CK_FloatingRealToComplex:
2967 case CK_FloatingComplexCast:
2968 case CK_IntegralRealToComplex:
2969 case CK_IntegralComplexCast:
2970 case CK_IntegralComplexToFloatingComplex:
2971 case CK_FloatingComplexToIntegralComplex:
2972 case CK_ConstructorConversion:
2973 case CK_ToUnion:
2974 case CK_HLSLArrayRValue:
2975 llvm_unreachable("scalar cast to non-scalar value");
2976
2977 case CK_LValueToRValue:
2978 assert(CGF.getContext().hasSameUnqualifiedType(E->getType(), DestTy));
2979 assert(E->isGLValue() && "lvalue-to-rvalue applied to r-value!");
2980 return Visit(E);
2981
2982 case CK_IntegralToPointer: {
2983 Value *Src = Visit(E);
2984
2985 // First, convert to the correct width so that we control the kind of
2986 // extension.
2987 auto DestLLVMTy = ConvertType(T: DestTy);
2988 llvm::Type *MiddleTy = CGF.CGM.getDataLayout().getIntPtrType(DestLLVMTy);
2989 bool InputSigned = E->getType()->isSignedIntegerOrEnumerationType();
2990 llvm::Value* IntResult =
2991 Builder.CreateIntCast(V: Src, DestTy: MiddleTy, isSigned: InputSigned, Name: "conv");
2992
2993 auto *IntToPtr = Builder.CreateIntToPtr(V: IntResult, DestTy: DestLLVMTy);
2994
2995 if (CGF.CGM.getCodeGenOpts().StrictVTablePointers) {
2996 // Going from integer to pointer that could be dynamic requires reloading
2997 // dynamic information from invariant.group.
2998 if (DestTy.mayBeDynamicClass())
2999 IntToPtr = Builder.CreateLaunderInvariantGroup(Ptr: IntToPtr);
3000 }
3001
3002 IntToPtr = CGF.authPointerToPointerCast(ResultPtr: IntToPtr, SourceType: E->getType(), DestType: DestTy);
3003 return IntToPtr;
3004 }
3005 case CK_PointerToIntegral: {
3006 assert(!DestTy->isBooleanType() && "bool should use PointerToBool");
3007 auto *PtrExpr = Visit(E);
3008
3009 if (CGF.CGM.getCodeGenOpts().StrictVTablePointers) {
3010 const QualType SrcType = E->getType();
3011
3012 // Casting to integer requires stripping dynamic information as it does
3013 // not carries it.
3014 if (SrcType.mayBeDynamicClass())
3015 PtrExpr = Builder.CreateStripInvariantGroup(Ptr: PtrExpr);
3016 }
3017
3018 PtrExpr = CGF.authPointerToPointerCast(ResultPtr: PtrExpr, SourceType: E->getType(), DestType: DestTy);
3019 return Builder.CreatePtrToInt(V: PtrExpr, DestTy: ConvertType(T: DestTy));
3020 }
3021 case CK_ToVoid: {
3022 CGF.EmitIgnoredExpr(E);
3023 return nullptr;
3024 }
3025 case CK_MatrixCast: {
3026 return EmitScalarConversion(Src: Visit(E), SrcType: E->getType(), DstType: DestTy,
3027 Loc: CE->getExprLoc());
3028 }
3029 // CK_HLSLAggregateSplatCast only handles splatting to vectors from a vec1
3030 // Casts were inserted in Sema to Cast the Src Expr to a Scalar and
3031 // To perform any necessary Scalar Cast, so this Cast can be handled
3032 // by the regular Vector Splat cast code.
3033 case CK_HLSLAggregateSplatCast:
3034 case CK_VectorSplat: {
3035 llvm::Type *DstTy = ConvertType(T: DestTy);
3036 Value *Elt = Visit(E);
3037 // Splat the element across to all elements
3038 llvm::ElementCount NumElements =
3039 cast<llvm::VectorType>(Val: DstTy)->getElementCount();
3040 return Builder.CreateVectorSplat(EC: NumElements, V: Elt, Name: "splat");
3041 }
3042
3043 case CK_FixedPointCast:
3044 return EmitScalarConversion(Src: Visit(E), SrcType: E->getType(), DstType: DestTy,
3045 Loc: CE->getExprLoc());
3046
3047 case CK_FixedPointToBoolean:
3048 assert(E->getType()->isFixedPointType() &&
3049 "Expected src type to be fixed point type");
3050 assert(DestTy->isBooleanType() && "Expected dest type to be boolean type");
3051 return EmitScalarConversion(Src: Visit(E), SrcType: E->getType(), DstType: DestTy,
3052 Loc: CE->getExprLoc());
3053
3054 case CK_FixedPointToIntegral:
3055 assert(E->getType()->isFixedPointType() &&
3056 "Expected src type to be fixed point type");
3057 assert(DestTy->isIntegerType() && "Expected dest type to be an integer");
3058 return EmitScalarConversion(Src: Visit(E), SrcType: E->getType(), DstType: DestTy,
3059 Loc: CE->getExprLoc());
3060
3061 case CK_IntegralToFixedPoint:
3062 assert(E->getType()->isIntegerType() &&
3063 "Expected src type to be an integer");
3064 assert(DestTy->isFixedPointType() &&
3065 "Expected dest type to be fixed point type");
3066 return EmitScalarConversion(Src: Visit(E), SrcType: E->getType(), DstType: DestTy,
3067 Loc: CE->getExprLoc());
3068
3069 case CK_IntegralCast: {
3070 if (E->getType()->isExtVectorType() && DestTy->isExtVectorType()) {
3071 QualType SrcElTy = E->getType()->castAs<VectorType>()->getElementType();
3072 return Builder.CreateIntCast(V: Visit(E), DestTy: ConvertType(T: DestTy),
3073 isSigned: SrcElTy->isSignedIntegerOrEnumerationType(),
3074 Name: "conv");
3075 }
3076 ScalarConversionOpts Opts;
3077 if (auto *ICE = dyn_cast<ImplicitCastExpr>(Val: CE)) {
3078 if (!ICE->isPartOfExplicitCast())
3079 Opts = ScalarConversionOpts(CGF.SanOpts);
3080 }
3081 return EmitScalarConversion(Src: Visit(E), SrcType: E->getType(), DstType: DestTy,
3082 Loc: CE->getExprLoc(), Opts);
3083 }
3084 case CK_IntegralToFloating: {
3085 if (E->getType()->isVectorType() && DestTy->isVectorType()) {
3086 // TODO: Support constrained FP intrinsics.
3087 QualType SrcElTy = E->getType()->castAs<VectorType>()->getElementType();
3088 if (SrcElTy->isSignedIntegerOrEnumerationType())
3089 return Builder.CreateSIToFP(V: Visit(E), DestTy: ConvertType(T: DestTy), Name: "conv");
3090 return Builder.CreateUIToFP(V: Visit(E), DestTy: ConvertType(T: DestTy), Name: "conv");
3091 }
3092 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, CE);
3093 return EmitScalarConversion(Src: Visit(E), SrcType: E->getType(), DstType: DestTy,
3094 Loc: CE->getExprLoc());
3095 }
3096 case CK_FloatingToIntegral: {
3097 if (E->getType()->isVectorType() && DestTy->isVectorType()) {
3098 // TODO: Support constrained FP intrinsics.
3099 QualType DstElTy = DestTy->castAs<VectorType>()->getElementType();
3100 if (DstElTy->isSignedIntegerOrEnumerationType())
3101 return Builder.CreateFPToSI(V: Visit(E), DestTy: ConvertType(T: DestTy), Name: "conv");
3102 return Builder.CreateFPToUI(V: Visit(E), DestTy: ConvertType(T: DestTy), Name: "conv");
3103 }
3104 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, CE);
3105 return EmitScalarConversion(Src: Visit(E), SrcType: E->getType(), DstType: DestTy,
3106 Loc: CE->getExprLoc());
3107 }
3108 case CK_FloatingCast: {
3109 if (E->getType()->isVectorType() && DestTy->isVectorType()) {
3110 // TODO: Support constrained FP intrinsics.
3111 QualType SrcElTy = E->getType()->castAs<VectorType>()->getElementType();
3112 QualType DstElTy = DestTy->castAs<VectorType>()->getElementType();
3113 if (DstElTy->castAs<BuiltinType>()->getKind() <
3114 SrcElTy->castAs<BuiltinType>()->getKind())
3115 return Builder.CreateFPTrunc(V: Visit(E), DestTy: ConvertType(T: DestTy), Name: "conv");
3116 return Builder.CreateFPExt(V: Visit(E), DestTy: ConvertType(T: DestTy), Name: "conv");
3117 }
3118 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, CE);
3119 return EmitScalarConversion(Src: Visit(E), SrcType: E->getType(), DstType: DestTy,
3120 Loc: CE->getExprLoc());
3121 }
3122 case CK_FixedPointToFloating:
3123 case CK_FloatingToFixedPoint: {
3124 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, CE);
3125 return EmitScalarConversion(Src: Visit(E), SrcType: E->getType(), DstType: DestTy,
3126 Loc: CE->getExprLoc());
3127 }
3128 case CK_BooleanToSignedIntegral: {
3129 ScalarConversionOpts Opts;
3130 Opts.TreatBooleanAsSigned = true;
3131 return EmitScalarConversion(Src: Visit(E), SrcType: E->getType(), DstType: DestTy,
3132 Loc: CE->getExprLoc(), Opts);
3133 }
3134 case CK_IntegralToBoolean:
3135 return EmitIntToBoolConversion(V: Visit(E));
3136 case CK_PointerToBoolean:
3137 return EmitPointerToBoolConversion(V: Visit(E), QT: E->getType());
3138 case CK_FloatingToBoolean: {
3139 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, CE);
3140 return EmitFloatToBoolConversion(V: Visit(E));
3141 }
3142 case CK_MemberPointerToBoolean: {
3143 llvm::Value *MemPtr = Visit(E);
3144 const MemberPointerType *MPT = E->getType()->getAs<MemberPointerType>();
3145 return CGF.CGM.getCXXABI().EmitMemberPointerIsNotNull(CGF, MemPtr, MPT);
3146 }
3147
3148 case CK_FloatingComplexToReal:
3149 case CK_IntegralComplexToReal:
3150 return CGF.EmitComplexExpr(E, IgnoreReal: false, IgnoreImag: true).first;
3151
3152 case CK_FloatingComplexToBoolean:
3153 case CK_IntegralComplexToBoolean: {
3154 CodeGenFunction::ComplexPairTy V = CGF.EmitComplexExpr(E);
3155
3156 // TODO: kill this function off, inline appropriate case here
3157 return EmitComplexToScalarConversion(Src: V, SrcTy: E->getType(), DstTy: DestTy,
3158 Loc: CE->getExprLoc());
3159 }
3160
3161 case CK_ZeroToOCLOpaqueType: {
3162 assert((DestTy->isEventT() || DestTy->isQueueT() ||
3163 DestTy->isOCLIntelSubgroupAVCType()) &&
3164 "CK_ZeroToOCLEvent cast on non-event type");
3165 return llvm::Constant::getNullValue(Ty: ConvertType(T: DestTy));
3166 }
3167
3168 case CK_IntToOCLSampler:
3169 return CGF.CGM.createOpenCLIntToSamplerConversion(E, CGF);
3170
3171 case CK_HLSLVectorTruncation: {
3172 assert((DestTy->isVectorType() || DestTy->isBuiltinType()) &&
3173 "Destination type must be a vector or builtin type.");
3174 Value *Vec = Visit(E);
3175 if (auto *VecTy = DestTy->getAs<VectorType>()) {
3176 SmallVector<int> Mask;
3177 unsigned NumElts = VecTy->getNumElements();
3178 for (unsigned I = 0; I != NumElts; ++I)
3179 Mask.push_back(Elt: I);
3180
3181 return Builder.CreateShuffleVector(V: Vec, Mask, Name: "trunc");
3182 }
3183 llvm::Value *Zero = llvm::Constant::getNullValue(Ty: CGF.SizeTy);
3184 return Builder.CreateExtractElement(Vec, Idx: Zero, Name: "cast.vtrunc");
3185 }
3186 case CK_HLSLMatrixTruncation: {
3187 assert((DestTy->isMatrixType() || DestTy->isBuiltinType()) &&
3188 "Destination type must be a matrix or builtin type.");
3189 Value *Mat = Visit(E);
3190 if (auto *MatTy = DestTy->getAs<ConstantMatrixType>()) {
3191 SmallVector<int> Mask(MatTy->getNumElementsFlattened());
3192 unsigned NumCols = MatTy->getNumColumns();
3193 unsigned NumRows = MatTy->getNumRows();
3194 auto *SrcMatTy = E->getType()->getAs<ConstantMatrixType>();
3195 assert(SrcMatTy && "Source type must be a matrix type.");
3196 assert(NumRows <= SrcMatTy->getNumRows());
3197 assert(NumCols <= SrcMatTy->getNumColumns());
3198
3199 // isMatrix[Src|Dst]RowMajor needs the full sugared QualType to find
3200 // matrix layout attrs. So use E->getType() & DestTy rather than SrcMatTy
3201 // & MatTy b/c getAs<ConstantMatrixType>() strips the sugar.
3202 bool IsSrcRowMajor = isMatrixRowMajor(LangOpts: CGF.getLangOpts(), T: E->getType());
3203 bool IsDstRowMajor = isMatrixRowMajor(LangOpts: CGF.getLangOpts(), T: DestTy);
3204 for (unsigned R = 0; R < NumRows; R++)
3205 for (unsigned C = 0; C < NumCols; C++)
3206 Mask[MatTy->getFlattenedIndex(Row: R, Column: C, IsRowMajor: IsDstRowMajor)] =
3207 SrcMatTy->getFlattenedIndex(Row: R, Column: C, IsRowMajor: IsSrcRowMajor);
3208
3209 return Builder.CreateShuffleVector(V: Mat, Mask, Name: "trunc");
3210 }
3211 llvm::Value *Zero = llvm::Constant::getNullValue(Ty: CGF.SizeTy);
3212 return Builder.CreateExtractElement(Vec: Mat, Idx: Zero, Name: "cast.mtrunc");
3213 }
3214 case CK_HLSLElementwiseCast: {
3215 RValue RV = CGF.EmitAnyExpr(E);
3216 SourceLocation Loc = CE->getExprLoc();
3217
3218 Address SrcAddr = Address::invalid();
3219
3220 if (RV.isAggregate()) {
3221 SrcAddr = RV.getAggregateAddress();
3222 } else {
3223 SrcAddr = CGF.CreateMemTemp(T: E->getType(), Name: "hlsl.ewcast.src");
3224 LValue TmpLV = CGF.MakeAddrLValue(Addr: SrcAddr, T: E->getType());
3225 CGF.EmitStoreThroughLValue(Src: RV, Dst: TmpLV);
3226 }
3227
3228 LValue SrcVal = CGF.MakeAddrLValue(Addr: SrcAddr, T: E->getType());
3229 return EmitHLSLElementwiseCast(CGF, SrcVal, DestTy, Loc);
3230 }
3231
3232 } // end of switch
3233
3234 llvm_unreachable("unknown scalar cast");
3235}
3236
3237Value *ScalarExprEmitter::VisitStmtExpr(const StmtExpr *E) {
3238 CodeGenFunction::StmtExprEvaluation eval(CGF);
3239 Address RetAlloca = CGF.EmitCompoundStmt(S: *E->getSubStmt(),
3240 GetLast: !E->getType()->isVoidType());
3241 if (!RetAlloca.isValid())
3242 return nullptr;
3243 return CGF.EmitLoadOfScalar(lvalue: CGF.MakeAddrLValue(Addr: RetAlloca, T: E->getType()),
3244 Loc: E->getExprLoc());
3245}
3246
3247Value *ScalarExprEmitter::VisitExprWithCleanups(ExprWithCleanups *E) {
3248 CodeGenFunction::RunCleanupsScope Scope(CGF);
3249 Value *V = Visit(E: E->getSubExpr());
3250 // Defend against dominance problems caused by jumps out of expression
3251 // evaluation through the shared cleanup block.
3252 Scope.ForceCleanup(ValuesToReload: {&V});
3253 return V;
3254}
3255
3256//===----------------------------------------------------------------------===//
3257// Unary Operators
3258//===----------------------------------------------------------------------===//
3259
3260static BinOpInfo createBinOpInfoFromIncDec(const UnaryOperator *E,
3261 llvm::Value *InVal, bool IsInc,
3262 FPOptions FPFeatures) {
3263 BinOpInfo BinOp;
3264 BinOp.LHS = InVal;
3265 BinOp.RHS = llvm::ConstantInt::get(Ty: InVal->getType(), V: 1, IsSigned: false);
3266 BinOp.Ty = E->getType();
3267 BinOp.Opcode = IsInc ? BO_Add : BO_Sub;
3268 BinOp.FPFeatures = FPFeatures;
3269 BinOp.E = E;
3270 return BinOp;
3271}
3272
3273llvm::Value *ScalarExprEmitter::EmitIncDecConsiderOverflowBehavior(
3274 const UnaryOperator *E, llvm::Value *InVal, bool IsInc) {
3275 // Treat positive amount as unsigned to support inc of i1 (needed for
3276 // unsigned _BitInt(1)).
3277 llvm::Value *Amount =
3278 llvm::ConstantInt::get(Ty: InVal->getType(), V: IsInc ? 1 : -1, IsSigned: !IsInc);
3279 StringRef Name = IsInc ? "inc" : "dec";
3280 QualType Ty = E->getType();
3281 const bool isSigned = Ty->isSignedIntegerOrEnumerationType();
3282 const bool hasSan =
3283 isSigned ? CGF.SanOpts.has(K: SanitizerKind::SignedIntegerOverflow)
3284 : CGF.SanOpts.has(K: SanitizerKind::UnsignedIntegerOverflow);
3285
3286 switch (getOverflowBehaviorConsideringType(CGF, Ty)) {
3287 case LangOptions::OB_Wrap:
3288 return Builder.CreateAdd(LHS: InVal, RHS: Amount, Name);
3289 case LangOptions::OB_SignedAndDefined:
3290 if (!hasSan)
3291 return Builder.CreateAdd(LHS: InVal, RHS: Amount, Name);
3292 [[fallthrough]];
3293 case LangOptions::OB_Unset:
3294 if (!E->canOverflow())
3295 return Builder.CreateAdd(LHS: InVal, RHS: Amount, Name);
3296 if (!hasSan)
3297 return isSigned ? Builder.CreateNSWAdd(LHS: InVal, RHS: Amount, Name)
3298 : Builder.CreateAdd(LHS: InVal, RHS: Amount, Name);
3299 [[fallthrough]];
3300 case LangOptions::OB_Trap:
3301 if (!Ty->getAs<OverflowBehaviorType>() && !E->canOverflow())
3302 return Builder.CreateAdd(LHS: InVal, RHS: Amount, Name);
3303 BinOpInfo Info = createBinOpInfoFromIncDec(
3304 E, InVal, IsInc, FPFeatures: E->getFPFeaturesInEffect(LO: CGF.getLangOpts()));
3305 if (CanElideOverflowCheck(Ctx&: CGF.getContext(), Op: Info))
3306 return isSigned ? Builder.CreateNSWAdd(LHS: InVal, RHS: Amount, Name)
3307 : Builder.CreateAdd(LHS: InVal, RHS: Amount, Name);
3308 return EmitOverflowCheckedBinOp(Ops: Info);
3309 }
3310 llvm_unreachable("Unknown OverflowBehaviorKind");
3311}
3312
3313namespace {
3314/// Handles check and update for lastprivate conditional variables.
3315class OMPLastprivateConditionalUpdateRAII {
3316private:
3317 CodeGenFunction &CGF;
3318 const UnaryOperator *E;
3319
3320public:
3321 OMPLastprivateConditionalUpdateRAII(CodeGenFunction &CGF,
3322 const UnaryOperator *E)
3323 : CGF(CGF), E(E) {}
3324 ~OMPLastprivateConditionalUpdateRAII() {
3325 if (CGF.getLangOpts().OpenMP)
3326 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(
3327 CGF, LHS: E->getSubExpr());
3328 }
3329};
3330} // namespace
3331
3332llvm::Value *
3333ScalarExprEmitter::EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV,
3334 bool isInc, bool isPre) {
3335 ApplyAtomGroup Grp(CGF.getDebugInfo());
3336 OMPLastprivateConditionalUpdateRAII OMPRegion(CGF, E);
3337 QualType type = E->getSubExpr()->getType();
3338 llvm::PHINode *atomicPHI = nullptr;
3339 llvm::Value *value;
3340 llvm::Value *input;
3341 llvm::Value *Previous = nullptr;
3342 QualType SrcType = E->getType();
3343
3344 int amount = (isInc ? 1 : -1);
3345 bool isSubtraction = !isInc;
3346
3347 if (const AtomicType *atomicTy = type->getAs<AtomicType>()) {
3348 type = atomicTy->getValueType();
3349 if (isInc && type->isBooleanType()) {
3350 llvm::Value *True = CGF.EmitToMemory(Value: Builder.getTrue(), Ty: type);
3351 if (isPre) {
3352 Builder.CreateStore(Val: True, Addr: LV.getAddress(), IsVolatile: LV.isVolatileQualified())
3353 ->setAtomic(Ordering: llvm::AtomicOrdering::SequentiallyConsistent);
3354 return Builder.getTrue();
3355 }
3356 // For atomic bool increment, we just store true and return it for
3357 // preincrement, do an atomic swap with true for postincrement
3358 return Builder.CreateAtomicRMW(
3359 Op: llvm::AtomicRMWInst::Xchg, Addr: LV.getAddress(), Val: True,
3360 Ordering: llvm::AtomicOrdering::SequentiallyConsistent);
3361 }
3362 // Special case for atomic increment / decrement on integers, emit
3363 // atomicrmw instructions. We skip this if we want to be doing overflow
3364 // checking, and fall into the slow path with the atomic cmpxchg loop.
3365 if (!type->isBooleanType() && type->isIntegerType() &&
3366 !(type->isUnsignedIntegerType() &&
3367 CGF.SanOpts.has(K: SanitizerKind::UnsignedIntegerOverflow)) &&
3368 CGF.getLangOpts().getSignedOverflowBehavior() !=
3369 LangOptions::SOB_Trapping) {
3370 llvm::AtomicRMWInst::BinOp aop = isInc ? llvm::AtomicRMWInst::Add :
3371 llvm::AtomicRMWInst::Sub;
3372 llvm::Instruction::BinaryOps op = isInc ? llvm::Instruction::Add :
3373 llvm::Instruction::Sub;
3374 llvm::Value *amt = CGF.EmitToMemory(
3375 Value: llvm::ConstantInt::get(Ty: ConvertType(T: type), V: 1, IsSigned: true), Ty: type);
3376 llvm::Value *old =
3377 Builder.CreateAtomicRMW(Op: aop, Addr: LV.getAddress(), Val: amt,
3378 Ordering: llvm::AtomicOrdering::SequentiallyConsistent);
3379 return isPre ? Builder.CreateBinOp(Opc: op, LHS: old, RHS: amt) : old;
3380 }
3381 // Special case for atomic increment/decrement on floats.
3382 // Bail out non-power-of-2-sized floating point types (e.g., x86_fp80).
3383 if (type->isFloatingType()) {
3384 llvm::Type *Ty = ConvertType(T: type);
3385 if (llvm::has_single_bit(Value: Ty->getScalarSizeInBits())) {
3386 llvm::AtomicRMWInst::BinOp aop =
3387 isInc ? llvm::AtomicRMWInst::FAdd : llvm::AtomicRMWInst::FSub;
3388 llvm::Instruction::BinaryOps op =
3389 isInc ? llvm::Instruction::FAdd : llvm::Instruction::FSub;
3390 llvm::Value *amt = llvm::ConstantFP::get(Ty, V: 1.0);
3391 llvm::AtomicRMWInst *old =
3392 CGF.emitAtomicRMWInst(Op: aop, Addr: LV.getAddress(), Val: amt,
3393 Order: llvm::AtomicOrdering::SequentiallyConsistent);
3394
3395 return isPre ? Builder.CreateBinOp(Opc: op, LHS: old, RHS: amt) : old;
3396 }
3397 }
3398 value = EmitLoadOfLValue(LV, Loc: E->getExprLoc());
3399 input = value;
3400 // For every other atomic operation, we need to emit a load-op-cmpxchg loop
3401 llvm::BasicBlock *startBB = Builder.GetInsertBlock();
3402 llvm::BasicBlock *opBB = CGF.createBasicBlock(name: "atomic_op", parent: CGF.CurFn);
3403 value = CGF.EmitToMemory(Value: value, Ty: type);
3404 Builder.CreateBr(Dest: opBB);
3405 Builder.SetInsertPoint(opBB);
3406 atomicPHI = Builder.CreatePHI(Ty: value->getType(), NumReservedValues: 2);
3407 atomicPHI->addIncoming(V: value, BB: startBB);
3408 value = atomicPHI;
3409 } else {
3410 value = EmitLoadOfLValue(LV, Loc: E->getExprLoc());
3411 input = value;
3412 }
3413
3414 // Special case of integer increment that we have to check first: bool++.
3415 // Due to promotion rules, we get:
3416 // bool++ -> bool = bool + 1
3417 // -> bool = (int)bool + 1
3418 // -> bool = ((int)bool + 1 != 0)
3419 // An interesting aspect of this is that increment is always true.
3420 // Decrement does not have this property.
3421 if (isInc && type->isBooleanType()) {
3422 value = Builder.getTrue();
3423
3424 // Most common case by far: integer increment.
3425 } else if (type->isIntegerType()) {
3426 QualType promotedType;
3427 bool canPerformLossyDemotionCheck = false;
3428
3429 if (CGF.getContext().isPromotableIntegerType(T: type)) {
3430 promotedType = CGF.getContext().getPromotedIntegerType(PromotableType: type);
3431 assert(promotedType != type && "Shouldn't promote to the same type.");
3432 canPerformLossyDemotionCheck = true;
3433 canPerformLossyDemotionCheck &=
3434 CGF.getContext().getCanonicalType(T: type) !=
3435 CGF.getContext().getCanonicalType(T: promotedType);
3436 canPerformLossyDemotionCheck &=
3437 PromotionIsPotentiallyEligibleForImplicitIntegerConversionCheck(
3438 SrcType: type, DstType: promotedType);
3439 assert((!canPerformLossyDemotionCheck ||
3440 type->isSignedIntegerOrEnumerationType() ||
3441 promotedType->isSignedIntegerOrEnumerationType() ||
3442 ConvertType(type)->getScalarSizeInBits() ==
3443 ConvertType(promotedType)->getScalarSizeInBits()) &&
3444 "The following check expects that if we do promotion to different "
3445 "underlying canonical type, at least one of the types (either "
3446 "base or promoted) will be signed, or the bitwidths will match.");
3447 }
3448 if (CGF.SanOpts.hasOneOf(
3449 K: SanitizerKind::ImplicitIntegerArithmeticValueChange |
3450 SanitizerKind::ImplicitBitfieldConversion) &&
3451 canPerformLossyDemotionCheck) {
3452 // While `x += 1` (for `x` with width less than int) is modeled as
3453 // promotion+arithmetics+demotion, and we can catch lossy demotion with
3454 // ease; inc/dec with width less than int can't overflow because of
3455 // promotion rules, so we omit promotion+demotion, which means that we can
3456 // not catch lossy "demotion". Because we still want to catch these cases
3457 // when the sanitizer is enabled, we perform the promotion, then perform
3458 // the increment/decrement in the wider type, and finally
3459 // perform the demotion. This will catch lossy demotions.
3460
3461 // We have a special case for bitfields defined using all the bits of the
3462 // type. In this case we need to do the same trick as for the integer
3463 // sanitizer checks, i.e., promotion -> increment/decrement -> demotion.
3464
3465 value = EmitScalarConversion(Src: value, SrcType: type, DstType: promotedType, Loc: E->getExprLoc());
3466 Value *amt = llvm::ConstantInt::get(Ty: value->getType(), V: amount, IsSigned: true);
3467 value = Builder.CreateAdd(LHS: value, RHS: amt, Name: isInc ? "inc" : "dec");
3468 // Do pass non-default ScalarConversionOpts so that sanitizer check is
3469 // emitted if LV is not a bitfield, otherwise the bitfield sanitizer
3470 // checks will take care of the conversion.
3471 ScalarConversionOpts Opts;
3472 if (!LV.isBitField())
3473 Opts = ScalarConversionOpts(CGF.SanOpts);
3474 else if (CGF.SanOpts.has(K: SanitizerKind::ImplicitBitfieldConversion)) {
3475 Previous = value;
3476 SrcType = promotedType;
3477 }
3478
3479 Opts.PatternExcluded = CGF.getContext().isUnaryOverflowPatternExcluded(UO: E);
3480 value = EmitScalarConversion(Src: value, SrcType: promotedType, DstType: type, Loc: E->getExprLoc(),
3481 Opts);
3482
3483 // Note that signed integer inc/dec with width less than int can't
3484 // overflow because of promotion rules; we're just eliding a few steps
3485 // here.
3486 } else if (type->isSignedIntegerOrEnumerationType() ||
3487 type->isUnsignedIntegerType()) {
3488 value = EmitIncDecConsiderOverflowBehavior(E, InVal: value, IsInc: isInc);
3489 } else {
3490 // Treat positive amount as unsigned to support inc of i1 (needed for
3491 // unsigned _BitInt(1)).
3492 llvm::Value *amt =
3493 llvm::ConstantInt::get(Ty: value->getType(), V: amount, IsSigned: !isInc);
3494 value = Builder.CreateAdd(LHS: value, RHS: amt, Name: isInc ? "inc" : "dec");
3495 }
3496
3497 // Next most common: pointer increment.
3498 } else if (const PointerType *ptr = type->getAs<PointerType>()) {
3499 QualType type = ptr->getPointeeType();
3500
3501 // VLA types don't have constant size.
3502 if (const VariableArrayType *vla
3503 = CGF.getContext().getAsVariableArrayType(T: type)) {
3504 llvm::Value *numElts = CGF.getVLASize(vla).NumElts;
3505 if (!isInc) numElts = Builder.CreateNSWNeg(V: numElts, Name: "vla.negsize");
3506 llvm::Type *elemTy = CGF.ConvertTypeForMem(T: vla->getElementType());
3507 if (CGF.getLangOpts().PointerOverflowDefined)
3508 value = Builder.CreateGEP(Ty: elemTy, Ptr: value, IdxList: numElts, Name: "vla.inc");
3509 else
3510 value = CGF.EmitCheckedInBoundsGEP(
3511 ElemTy: elemTy, Ptr: value, IdxList: numElts, /*SignedIndices=*/false, IsSubtraction: isSubtraction,
3512 Loc: E->getExprLoc(), Name: "vla.inc");
3513
3514 // Arithmetic on function pointers (!) is just +-1.
3515 } else if (type->isFunctionType()) {
3516 llvm::Value *amt = Builder.getInt32(C: amount);
3517
3518 if (CGF.getLangOpts().PointerOverflowDefined)
3519 value = Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: value, IdxList: amt, Name: "incdec.funcptr");
3520 else
3521 value =
3522 CGF.EmitCheckedInBoundsGEP(ElemTy: CGF.Int8Ty, Ptr: value, IdxList: amt,
3523 /*SignedIndices=*/false, IsSubtraction: isSubtraction,
3524 Loc: E->getExprLoc(), Name: "incdec.funcptr");
3525
3526 // For everything else, we can just do a simple increment.
3527 } else {
3528 llvm::Value *amt = Builder.getInt32(C: amount);
3529 llvm::Type *elemTy = CGF.ConvertTypeForMem(T: type);
3530 if (CGF.getLangOpts().PointerOverflowDefined)
3531 value = Builder.CreateGEP(Ty: elemTy, Ptr: value, IdxList: amt, Name: "incdec.ptr");
3532 else
3533 value = CGF.EmitCheckedInBoundsGEP(
3534 ElemTy: elemTy, Ptr: value, IdxList: amt, /*SignedIndices=*/false, IsSubtraction: isSubtraction,
3535 Loc: E->getExprLoc(), Name: "incdec.ptr");
3536 }
3537
3538 // Vector increment/decrement.
3539 } else if (type->isVectorType()) {
3540 if (type->hasIntegerRepresentation()) {
3541 llvm::Value *amt = llvm::ConstantInt::getSigned(Ty: value->getType(), V: amount);
3542
3543 value = Builder.CreateAdd(LHS: value, RHS: amt, Name: isInc ? "inc" : "dec");
3544 } else {
3545 value = Builder.CreateFAdd(
3546 L: value,
3547 R: llvm::ConstantFP::get(Ty: value->getType(), V: amount),
3548 Name: isInc ? "inc" : "dec");
3549 }
3550
3551 // Floating point.
3552 } else if (type->isRealFloatingType()) {
3553 // Add the inc/dec to the real part.
3554 llvm::Value *amt;
3555 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
3556
3557 if (type->isHalfType() && !CGF.getContext().getLangOpts().NativeHalfType) {
3558 // Another special case: half FP increment should be done via float. If
3559 // the input isn't already half, it may be i16.
3560 Value *bitcast = Builder.CreateBitCast(V: input, DestTy: CGF.CGM.HalfTy);
3561 value = Builder.CreateFPExt(V: bitcast, DestTy: CGF.CGM.FloatTy, Name: "incdec.conv");
3562 }
3563
3564 if (value->getType()->isFloatTy())
3565 amt = llvm::ConstantFP::get(Context&: VMContext,
3566 V: llvm::APFloat(static_cast<float>(amount)));
3567 else if (value->getType()->isDoubleTy())
3568 amt = llvm::ConstantFP::get(Context&: VMContext,
3569 V: llvm::APFloat(static_cast<double>(amount)));
3570 else {
3571 // Remaining types are Half, Bfloat16, LongDouble, __ibm128 or __float128.
3572 // Convert from float.
3573 llvm::APFloat F(static_cast<float>(amount));
3574 bool ignored;
3575 const llvm::fltSemantics *FS;
3576 // Don't use getFloatTypeSemantics because Half isn't
3577 // necessarily represented using the "half" LLVM type.
3578 if (value->getType()->isFP128Ty())
3579 FS = &CGF.getTarget().getFloat128Format();
3580 else if (value->getType()->isHalfTy())
3581 FS = &CGF.getTarget().getHalfFormat();
3582 else if (value->getType()->isBFloatTy())
3583 FS = &CGF.getTarget().getBFloat16Format();
3584 else if (value->getType()->isPPC_FP128Ty())
3585 FS = &CGF.getTarget().getIbm128Format();
3586 else
3587 FS = &CGF.getTarget().getLongDoubleFormat();
3588 F.convert(ToSemantics: *FS, RM: llvm::APFloat::rmTowardZero, losesInfo: &ignored);
3589 amt = llvm::ConstantFP::get(Context&: VMContext, V: F);
3590 }
3591 value = Builder.CreateFAdd(L: value, R: amt, Name: isInc ? "inc" : "dec");
3592
3593 if (type->isHalfType() && !CGF.getContext().getLangOpts().NativeHalfType) {
3594 value = Builder.CreateFPTrunc(V: value, DestTy: CGF.CGM.HalfTy, Name: "incdec.conv");
3595 value = Builder.CreateBitCast(V: value, DestTy: input->getType());
3596 }
3597
3598 // Fixed-point types.
3599 } else if (type->isFixedPointType()) {
3600 // Fixed-point types are tricky. In some cases, it isn't possible to
3601 // represent a 1 or a -1 in the type at all. Piggyback off of
3602 // EmitFixedPointBinOp to avoid having to reimplement saturation.
3603 BinOpInfo Info;
3604 Info.E = E;
3605 Info.Ty = E->getType();
3606 Info.Opcode = isInc ? BO_Add : BO_Sub;
3607 Info.LHS = value;
3608 Info.RHS = llvm::ConstantInt::get(Ty: value->getType(), V: 1, IsSigned: false);
3609 // If the type is signed, it's better to represent this as +(-1) or -(-1),
3610 // since -1 is guaranteed to be representable.
3611 if (type->isSignedFixedPointType()) {
3612 Info.Opcode = isInc ? BO_Sub : BO_Add;
3613 Info.RHS = Builder.CreateNeg(V: Info.RHS);
3614 }
3615 // Now, convert from our invented integer literal to the type of the unary
3616 // op. This will upscale and saturate if necessary. This value can become
3617 // undef in some cases.
3618 llvm::FixedPointBuilder<CGBuilderTy> FPBuilder(Builder);
3619 auto DstSema = CGF.getContext().getFixedPointSemantics(Ty: Info.Ty);
3620 Info.RHS = FPBuilder.CreateIntegerToFixed(Src: Info.RHS, SrcIsSigned: true, DstSema);
3621 value = EmitFixedPointBinOp(Ops: Info);
3622
3623 // Objective-C pointer types.
3624 } else {
3625 const ObjCObjectPointerType *OPT = type->castAs<ObjCObjectPointerType>();
3626
3627 CharUnits size = CGF.getContext().getTypeSizeInChars(T: OPT->getObjectType());
3628 if (!isInc) size = -size;
3629 llvm::Value *sizeValue =
3630 llvm::ConstantInt::getSigned(Ty: CGF.SizeTy, V: size.getQuantity());
3631
3632 if (CGF.getLangOpts().PointerOverflowDefined)
3633 value = Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: value, IdxList: sizeValue, Name: "incdec.objptr");
3634 else
3635 value = CGF.EmitCheckedInBoundsGEP(
3636 ElemTy: CGF.Int8Ty, Ptr: value, IdxList: sizeValue, /*SignedIndices=*/false, IsSubtraction: isSubtraction,
3637 Loc: E->getExprLoc(), Name: "incdec.objptr");
3638 value = Builder.CreateBitCast(V: value, DestTy: input->getType());
3639 }
3640
3641 if (atomicPHI) {
3642 llvm::BasicBlock *curBlock = Builder.GetInsertBlock();
3643 llvm::BasicBlock *contBB = CGF.createBasicBlock(name: "atomic_cont", parent: CGF.CurFn);
3644 auto Pair = CGF.EmitAtomicCompareExchange(
3645 Obj: LV, Expected: RValue::get(V: atomicPHI), Desired: RValue::get(V: value), Loc: E->getExprLoc());
3646 llvm::Value *old = CGF.EmitToMemory(Value: Pair.first.getScalarVal(), Ty: type);
3647 llvm::Value *success = Pair.second;
3648 atomicPHI->addIncoming(V: old, BB: curBlock);
3649 Builder.CreateCondBr(Cond: success, True: contBB, False: atomicPHI->getParent());
3650 Builder.SetInsertPoint(contBB);
3651 return isPre ? value : input;
3652 }
3653
3654 // Store the updated result through the lvalue.
3655 if (LV.isBitField()) {
3656 Value *Src = Previous ? Previous : value;
3657 CGF.EmitStoreThroughBitfieldLValue(Src: RValue::get(V: value), Dst: LV, Result: &value);
3658 CGF.EmitBitfieldConversionCheck(Src, SrcType, Dst: value, DstType: E->getType(),
3659 Info: LV.getBitFieldInfo(), Loc: E->getExprLoc());
3660 } else
3661 CGF.EmitStoreThroughLValue(Src: RValue::get(V: value), Dst: LV);
3662
3663 // If this is a postinc, return the value read from memory, otherwise use the
3664 // updated value.
3665 return isPre ? value : input;
3666}
3667
3668
3669Value *ScalarExprEmitter::VisitUnaryPlus(const UnaryOperator *E,
3670 QualType PromotionType) {
3671 QualType promotionTy = PromotionType.isNull()
3672 ? getPromotionType(Ty: E->getSubExpr()->getType())
3673 : PromotionType;
3674 Value *result = VisitPlus(E, PromotionType: promotionTy);
3675 if (result && !promotionTy.isNull())
3676 result = EmitUnPromotedValue(result, ExprType: E->getType());
3677 return result;
3678}
3679
3680Value *ScalarExprEmitter::VisitPlus(const UnaryOperator *E,
3681 QualType PromotionType) {
3682 // This differs from gcc, though, most likely due to a bug in gcc.
3683 TestAndClearIgnoreResultAssign();
3684 if (!PromotionType.isNull())
3685 return CGF.EmitPromotedScalarExpr(E: E->getSubExpr(), PromotionType);
3686 return Visit(E: E->getSubExpr());
3687}
3688
3689Value *ScalarExprEmitter::VisitUnaryMinus(const UnaryOperator *E,
3690 QualType PromotionType) {
3691 QualType promotionTy = PromotionType.isNull()
3692 ? getPromotionType(Ty: E->getSubExpr()->getType())
3693 : PromotionType;
3694 Value *result = VisitMinus(E, PromotionType: promotionTy);
3695 if (result && !promotionTy.isNull())
3696 result = EmitUnPromotedValue(result, ExprType: E->getType());
3697 return result;
3698}
3699
3700Value *ScalarExprEmitter::VisitMinus(const UnaryOperator *E,
3701 QualType PromotionType) {
3702 TestAndClearIgnoreResultAssign();
3703 Value *Op;
3704 if (!PromotionType.isNull())
3705 Op = CGF.EmitPromotedScalarExpr(E: E->getSubExpr(), PromotionType);
3706 else
3707 Op = Visit(E: E->getSubExpr());
3708
3709 // Generate a unary FNeg for FP ops.
3710 if (Op->getType()->isFPOrFPVectorTy())
3711 return Builder.CreateFNeg(V: Op, Name: "fneg");
3712
3713 // Emit unary minus with EmitSub so we handle overflow cases etc.
3714 BinOpInfo BinOp;
3715 BinOp.RHS = Op;
3716 BinOp.LHS = llvm::Constant::getNullValue(Ty: BinOp.RHS->getType());
3717 BinOp.Ty = E->getType();
3718 BinOp.Opcode = BO_Sub;
3719 BinOp.FPFeatures = E->getFPFeaturesInEffect(LO: CGF.getLangOpts());
3720 BinOp.E = E;
3721 return EmitSub(Ops: BinOp);
3722}
3723
3724Value *ScalarExprEmitter::VisitUnaryNot(const UnaryOperator *E) {
3725 TestAndClearIgnoreResultAssign();
3726 Value *Op = Visit(E: E->getSubExpr());
3727 return Builder.CreateNot(V: Op, Name: "not");
3728}
3729
3730Value *ScalarExprEmitter::VisitUnaryLNot(const UnaryOperator *E) {
3731 // Perform vector logical not on comparison with zero vector.
3732 if (E->getType()->isVectorType() &&
3733 E->getType()->castAs<VectorType>()->getVectorKind() ==
3734 VectorKind::Generic) {
3735 Value *Oper = Visit(E: E->getSubExpr());
3736 Value *Zero = llvm::Constant::getNullValue(Ty: Oper->getType());
3737 Value *Result;
3738 if (Oper->getType()->isFPOrFPVectorTy()) {
3739 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(
3740 CGF, E->getFPFeaturesInEffect(LO: CGF.getLangOpts()));
3741 Result = Builder.CreateFCmp(P: llvm::CmpInst::FCMP_OEQ, LHS: Oper, RHS: Zero, Name: "cmp");
3742 } else
3743 Result = Builder.CreateICmp(P: llvm::CmpInst::ICMP_EQ, LHS: Oper, RHS: Zero, Name: "cmp");
3744 return Builder.CreateSExt(V: Result, DestTy: ConvertType(T: E->getType()), Name: "sext");
3745 }
3746
3747 // Compare operand to zero.
3748 Value *BoolVal = CGF.EvaluateExprAsBool(E: E->getSubExpr());
3749
3750 // Invert value.
3751 // TODO: Could dynamically modify easy computations here. For example, if
3752 // the operand is an icmp ne, turn into icmp eq.
3753 BoolVal = Builder.CreateNot(V: BoolVal, Name: "lnot");
3754
3755 // ZExt result to the expr type.
3756 return Builder.CreateZExt(V: BoolVal, DestTy: ConvertType(T: E->getType()), Name: "lnot.ext");
3757}
3758
3759Value *ScalarExprEmitter::VisitOffsetOfExpr(OffsetOfExpr *E) {
3760 // Try folding the offsetof to a constant.
3761 Expr::EvalResult EVResult;
3762 if (E->EvaluateAsInt(Result&: EVResult, Ctx: CGF.getContext())) {
3763 llvm::APSInt Value = EVResult.Val.getInt();
3764 return Builder.getInt(AI: Value);
3765 }
3766
3767 // Loop over the components of the offsetof to compute the value.
3768 unsigned n = E->getNumComponents();
3769 llvm::Type* ResultType = ConvertType(T: E->getType());
3770 llvm::Value* Result = llvm::Constant::getNullValue(Ty: ResultType);
3771 QualType CurrentType = E->getTypeSourceInfo()->getType();
3772 for (unsigned i = 0; i != n; ++i) {
3773 OffsetOfNode ON = E->getComponent(Idx: i);
3774 llvm::Value *Offset = nullptr;
3775 switch (ON.getKind()) {
3776 case OffsetOfNode::Array: {
3777 // Compute the index
3778 Expr *IdxExpr = E->getIndexExpr(Idx: ON.getArrayExprIndex());
3779 llvm::Value* Idx = CGF.EmitScalarExpr(E: IdxExpr);
3780 bool IdxSigned = IdxExpr->getType()->isSignedIntegerOrEnumerationType();
3781 Idx = Builder.CreateIntCast(V: Idx, DestTy: ResultType, isSigned: IdxSigned, Name: "conv");
3782
3783 // Save the element type
3784 CurrentType =
3785 CGF.getContext().getAsArrayType(T: CurrentType)->getElementType();
3786
3787 // Compute the element size
3788 llvm::Value* ElemSize = llvm::ConstantInt::get(Ty: ResultType,
3789 V: CGF.getContext().getTypeSizeInChars(T: CurrentType).getQuantity());
3790
3791 // Multiply out to compute the result
3792 Offset = Builder.CreateMul(LHS: Idx, RHS: ElemSize);
3793 break;
3794 }
3795
3796 case OffsetOfNode::Field: {
3797 FieldDecl *MemberDecl = ON.getField();
3798 auto *RD = CurrentType->castAsRecordDecl();
3799 const ASTRecordLayout &RL = CGF.getContext().getASTRecordLayout(D: RD);
3800
3801 // Get the index of the field in its parent.
3802 unsigned FieldIndex = MemberDecl->getFieldIndex();
3803
3804 // Compute the offset to the field
3805 int64_t OffsetInt =
3806 RL.getFieldOffset(FieldNo: FieldIndex) / CGF.getContext().getCharWidth();
3807 Offset = llvm::ConstantInt::get(Ty: ResultType, V: OffsetInt);
3808
3809 // Save the element type.
3810 CurrentType = MemberDecl->getType();
3811 break;
3812 }
3813
3814 case OffsetOfNode::Identifier:
3815 llvm_unreachable("dependent __builtin_offsetof");
3816
3817 case OffsetOfNode::Base: {
3818 if (ON.getBase()->isVirtual()) {
3819 CGF.ErrorUnsupported(S: E, Type: "virtual base in offsetof");
3820 continue;
3821 }
3822
3823 const ASTRecordLayout &RL = CGF.getContext().getASTRecordLayout(
3824 D: CurrentType->castAsCanonical<RecordType>()->getDecl());
3825
3826 // Save the element type.
3827 CurrentType = ON.getBase()->getType();
3828
3829 // Compute the offset to the base.
3830 auto *BaseRD = CurrentType->castAsCXXRecordDecl();
3831 CharUnits OffsetInt = RL.getBaseClassOffset(Base: BaseRD);
3832 Offset = llvm::ConstantInt::get(Ty: ResultType, V: OffsetInt.getQuantity());
3833 break;
3834 }
3835 }
3836 Result = Builder.CreateAdd(LHS: Result, RHS: Offset);
3837 }
3838 return Result;
3839}
3840
3841/// VisitUnaryExprOrTypeTraitExpr - Return the size or alignment of the type of
3842/// argument of the sizeof expression as an integer.
3843Value *
3844ScalarExprEmitter::VisitUnaryExprOrTypeTraitExpr(
3845 const UnaryExprOrTypeTraitExpr *E) {
3846 QualType TypeToSize = E->getTypeOfArgument();
3847 if (auto Kind = E->getKind();
3848 Kind == UETT_SizeOf || Kind == UETT_DataSizeOf || Kind == UETT_CountOf) {
3849 if (const VariableArrayType *VAT =
3850 CGF.getContext().getAsVariableArrayType(T: TypeToSize)) {
3851 // For _Countof, we only want to evaluate if the extent is actually
3852 // variable as opposed to a multi-dimensional array whose extent is
3853 // constant but whose element type is variable.
3854 bool EvaluateExtent = true;
3855 if (Kind == UETT_CountOf && VAT->getElementType()->isArrayType()) {
3856 EvaluateExtent =
3857 !VAT->getSizeExpr()->isIntegerConstantExpr(Ctx: CGF.getContext());
3858 }
3859 if (EvaluateExtent) {
3860 if (E->isArgumentType()) {
3861 // sizeof(type) - make sure to emit the VLA size.
3862 CGF.EmitVariablyModifiedType(Ty: TypeToSize);
3863 } else {
3864 // C99 6.5.3.4p2: If the argument is an expression of type
3865 // VLA, it is evaluated.
3866 CGF.EmitIgnoredExpr(E: E->getArgumentExpr());
3867 }
3868
3869 // For _Countof, we just want to return the size of a single dimension.
3870 if (Kind == UETT_CountOf)
3871 return CGF.getVLAElements1D(vla: VAT).NumElts;
3872
3873 // For sizeof and __datasizeof, we need to scale the number of elements
3874 // by the size of the array element type.
3875 auto VlaSize = CGF.getVLASize(vla: VAT);
3876
3877 // Scale the number of non-VLA elements by the non-VLA element size.
3878 CharUnits eltSize = CGF.getContext().getTypeSizeInChars(T: VlaSize.Type);
3879 if (!eltSize.isOne())
3880 return CGF.Builder.CreateNUWMul(LHS: CGF.CGM.getSize(numChars: eltSize),
3881 RHS: VlaSize.NumElts);
3882 return VlaSize.NumElts;
3883 }
3884 }
3885 } else if (E->getKind() == UETT_OpenMPRequiredSimdAlign) {
3886 auto Alignment =
3887 CGF.getContext()
3888 .toCharUnitsFromBits(BitSize: CGF.getContext().getOpenMPDefaultSimdAlign(
3889 T: E->getTypeOfArgument()->getPointeeType()))
3890 .getQuantity();
3891 return llvm::ConstantInt::get(Ty: CGF.SizeTy, V: Alignment);
3892 } else if (E->getKind() == UETT_VectorElements) {
3893 auto *VecTy = cast<llvm::VectorType>(Val: ConvertType(T: E->getTypeOfArgument()));
3894 return Builder.CreateElementCount(Ty: CGF.SizeTy, EC: VecTy->getElementCount());
3895 }
3896
3897 // If this isn't sizeof(vla), the result must be constant; use the constant
3898 // folding logic so we don't have to duplicate it here.
3899 return Builder.getInt(AI: E->EvaluateKnownConstInt(Ctx: CGF.getContext()));
3900}
3901
3902Value *ScalarExprEmitter::VisitUnaryReal(const UnaryOperator *E,
3903 QualType PromotionType) {
3904 QualType promotionTy = PromotionType.isNull()
3905 ? getPromotionType(Ty: E->getSubExpr()->getType())
3906 : PromotionType;
3907 Value *result = VisitReal(E, PromotionType: promotionTy);
3908 if (result && !promotionTy.isNull())
3909 result = EmitUnPromotedValue(result, ExprType: E->getType());
3910 return result;
3911}
3912
3913Value *ScalarExprEmitter::VisitReal(const UnaryOperator *E,
3914 QualType PromotionType) {
3915 Expr *Op = E->getSubExpr();
3916 if (Op->getType()->isAnyComplexType()) {
3917 // If it's an l-value, load through the appropriate subobject l-value.
3918 // Note that we have to ask E because Op might be an l-value that
3919 // this won't work for, e.g. an Obj-C property.
3920 if (E->isGLValue()) {
3921 if (!PromotionType.isNull()) {
3922 CodeGenFunction::ComplexPairTy result = CGF.EmitComplexExpr(
3923 E: Op, /*IgnoreReal*/ IgnoreResultAssign, /*IgnoreImag*/ true);
3924 PromotionType = PromotionType->isAnyComplexType()
3925 ? PromotionType
3926 : CGF.getContext().getComplexType(T: PromotionType);
3927 return result.first ? CGF.EmitPromotedValue(result, PromotionType).first
3928 : result.first;
3929 }
3930
3931 return CGF.EmitLoadOfLValue(V: CGF.EmitLValue(E), Loc: E->getExprLoc())
3932 .getScalarVal();
3933 }
3934 // Otherwise, calculate and project.
3935 return CGF.EmitComplexExpr(E: Op, IgnoreReal: false, IgnoreImag: true).first;
3936 }
3937
3938 if (!PromotionType.isNull())
3939 return CGF.EmitPromotedScalarExpr(E: Op, PromotionType);
3940 return Visit(E: Op);
3941}
3942
3943Value *ScalarExprEmitter::VisitUnaryImag(const UnaryOperator *E,
3944 QualType PromotionType) {
3945 QualType promotionTy = PromotionType.isNull()
3946 ? getPromotionType(Ty: E->getSubExpr()->getType())
3947 : PromotionType;
3948 Value *result = VisitImag(E, PromotionType: promotionTy);
3949 if (result && !promotionTy.isNull())
3950 result = EmitUnPromotedValue(result, ExprType: E->getType());
3951 return result;
3952}
3953
3954Value *ScalarExprEmitter::VisitImag(const UnaryOperator *E,
3955 QualType PromotionType) {
3956 Expr *Op = E->getSubExpr();
3957 if (Op->getType()->isAnyComplexType()) {
3958 // If it's an l-value, load through the appropriate subobject l-value.
3959 // Note that we have to ask E because Op might be an l-value that
3960 // this won't work for, e.g. an Obj-C property.
3961 if (Op->isGLValue()) {
3962 if (!PromotionType.isNull()) {
3963 CodeGenFunction::ComplexPairTy result = CGF.EmitComplexExpr(
3964 E: Op, /*IgnoreReal*/ true, /*IgnoreImag*/ IgnoreResultAssign);
3965 PromotionType = PromotionType->isAnyComplexType()
3966 ? PromotionType
3967 : CGF.getContext().getComplexType(T: PromotionType);
3968 return result.second
3969 ? CGF.EmitPromotedValue(result, PromotionType).second
3970 : result.second;
3971 }
3972
3973 return CGF.EmitLoadOfLValue(V: CGF.EmitLValue(E), Loc: E->getExprLoc())
3974 .getScalarVal();
3975 }
3976 // Otherwise, calculate and project.
3977 return CGF.EmitComplexExpr(E: Op, IgnoreReal: true, IgnoreImag: false).second;
3978 }
3979
3980 // __imag on a scalar returns zero. Emit the subexpr to ensure side
3981 // effects are evaluated, but not the actual value.
3982 if (Op->isGLValue())
3983 CGF.EmitLValue(E: Op);
3984 else if (!PromotionType.isNull())
3985 CGF.EmitPromotedScalarExpr(E: Op, PromotionType);
3986 else
3987 CGF.EmitScalarExpr(E: Op, IgnoreResultAssign: true);
3988 if (!PromotionType.isNull())
3989 return llvm::Constant::getNullValue(Ty: ConvertType(T: PromotionType));
3990 return llvm::Constant::getNullValue(Ty: ConvertType(T: E->getType()));
3991}
3992
3993//===----------------------------------------------------------------------===//
3994// Binary Operators
3995//===----------------------------------------------------------------------===//
3996
3997Value *ScalarExprEmitter::EmitPromotedValue(Value *result,
3998 QualType PromotionType) {
3999 return CGF.Builder.CreateFPExt(V: result, DestTy: ConvertType(T: PromotionType), Name: "ext");
4000}
4001
4002Value *ScalarExprEmitter::EmitUnPromotedValue(Value *result,
4003 QualType ExprType) {
4004 return CGF.Builder.CreateFPTrunc(V: result, DestTy: ConvertType(T: ExprType), Name: "unpromotion");
4005}
4006
4007Value *ScalarExprEmitter::EmitPromoted(const Expr *E, QualType PromotionType) {
4008 E = E->IgnoreParens();
4009 if (auto BO = dyn_cast<BinaryOperator>(Val: E)) {
4010 switch (BO->getOpcode()) {
4011#define HANDLE_BINOP(OP) \
4012 case BO_##OP: \
4013 return Emit##OP(EmitBinOps(BO, PromotionType));
4014 HANDLE_BINOP(Add)
4015 HANDLE_BINOP(Sub)
4016 HANDLE_BINOP(Mul)
4017 HANDLE_BINOP(Div)
4018#undef HANDLE_BINOP
4019 default:
4020 break;
4021 }
4022 } else if (auto UO = dyn_cast<UnaryOperator>(Val: E)) {
4023 switch (UO->getOpcode()) {
4024 case UO_Imag:
4025 return VisitImag(E: UO, PromotionType);
4026 case UO_Real:
4027 return VisitReal(E: UO, PromotionType);
4028 case UO_Minus:
4029 return VisitMinus(E: UO, PromotionType);
4030 case UO_Plus:
4031 return VisitPlus(E: UO, PromotionType);
4032 default:
4033 break;
4034 }
4035 }
4036 auto result = Visit(E: const_cast<Expr *>(E));
4037 if (result) {
4038 if (!PromotionType.isNull())
4039 return EmitPromotedValue(result, PromotionType);
4040 else
4041 return EmitUnPromotedValue(result, ExprType: E->getType());
4042 }
4043 return result;
4044}
4045
4046BinOpInfo ScalarExprEmitter::EmitBinOps(const BinaryOperator *E,
4047 QualType PromotionType) {
4048 TestAndClearIgnoreResultAssign();
4049 BinOpInfo Result;
4050 Result.LHS = CGF.EmitPromotedScalarExpr(E: E->getLHS(), PromotionType);
4051 Result.RHS = CGF.EmitPromotedScalarExpr(E: E->getRHS(), PromotionType);
4052 if (!PromotionType.isNull())
4053 Result.Ty = PromotionType;
4054 else
4055 Result.Ty = E->getType();
4056 Result.Opcode = E->getOpcode();
4057 Result.FPFeatures = E->getFPFeaturesInEffect(LO: CGF.getLangOpts());
4058 Result.E = E;
4059 return Result;
4060}
4061
4062LValue ScalarExprEmitter::EmitCompoundAssignLValue(
4063 const CompoundAssignOperator *E,
4064 Value *(ScalarExprEmitter::*Func)(const BinOpInfo &),
4065 Value *&Result) {
4066 QualType LHSTy = E->getLHS()->getType();
4067 BinOpInfo OpInfo;
4068
4069 if (E->getComputationResultType()->isAnyComplexType())
4070 return CGF.EmitScalarCompoundAssignWithComplex(E, Result);
4071
4072 // Emit the RHS first. __block variables need to have the rhs evaluated
4073 // first, plus this should improve codegen a little.
4074
4075 QualType PromotionTypeCR;
4076 PromotionTypeCR = getPromotionType(Ty: E->getComputationResultType());
4077 if (PromotionTypeCR.isNull())
4078 PromotionTypeCR = E->getComputationResultType();
4079 QualType PromotionTypeLHS = getPromotionType(Ty: E->getComputationLHSType());
4080 QualType PromotionTypeRHS = getPromotionType(Ty: E->getRHS()->getType());
4081 if (!PromotionTypeRHS.isNull())
4082 OpInfo.RHS = CGF.EmitPromotedScalarExpr(E: E->getRHS(), PromotionType: PromotionTypeRHS);
4083 else
4084 OpInfo.RHS = Visit(E: E->getRHS());
4085 OpInfo.Ty = PromotionTypeCR;
4086 OpInfo.Opcode = E->getOpcode();
4087 OpInfo.FPFeatures = E->getFPFeaturesInEffect(LO: CGF.getLangOpts());
4088 OpInfo.E = E;
4089 // Load/convert the LHS.
4090 LValue LHSLV = EmitCheckedLValue(E: E->getLHS(), TCK: CodeGenFunction::TCK_Store);
4091
4092 llvm::PHINode *atomicPHI = nullptr;
4093 if (const AtomicType *atomicTy = LHSTy->getAs<AtomicType>()) {
4094 // Type wrapped by _Atomic.
4095 QualType AtomicValueTy = atomicTy->getValueType();
4096 // Type resulting from FP conversion / integer promotion of the compound
4097 // assignment operands.
4098 QualType ResultTy = E->getComputationResultType();
4099 // Do not try the atomicrmw op fast-path when the compound assignment may
4100 // involve FP conversions, as the correct semantics would require promoting
4101 // the loaded integer to double, performing FP arithmetics, and truncation
4102 // back as a single atomic operation. Integer promotion is still
4103 // semantically safe.
4104 bool CanEmitAtomicRMW =
4105 !AtomicValueTy->isBooleanType() && AtomicValueTy->isIntegerType() &&
4106 ResultTy->isIntegerType() &&
4107 !(AtomicValueTy->isUnsignedIntegerType() &&
4108 CGF.SanOpts.has(K: SanitizerKind::UnsignedIntegerOverflow)) &&
4109 CGF.getLangOpts().getSignedOverflowBehavior() !=
4110 LangOptions::SOB_Trapping;
4111 if (CanEmitAtomicRMW) {
4112 llvm::AtomicRMWInst::BinOp AtomicOp = llvm::AtomicRMWInst::BAD_BINOP;
4113 llvm::Instruction::BinaryOps Op;
4114 switch (OpInfo.Opcode) {
4115 // We don't have atomicrmw operands for *, %, /, <<, >>
4116 case BO_MulAssign: case BO_DivAssign:
4117 case BO_RemAssign:
4118 case BO_ShlAssign:
4119 case BO_ShrAssign:
4120 break;
4121 case BO_AddAssign:
4122 AtomicOp = llvm::AtomicRMWInst::Add;
4123 Op = llvm::Instruction::Add;
4124 break;
4125 case BO_SubAssign:
4126 AtomicOp = llvm::AtomicRMWInst::Sub;
4127 Op = llvm::Instruction::Sub;
4128 break;
4129 case BO_AndAssign:
4130 AtomicOp = llvm::AtomicRMWInst::And;
4131 Op = llvm::Instruction::And;
4132 break;
4133 case BO_XorAssign:
4134 AtomicOp = llvm::AtomicRMWInst::Xor;
4135 Op = llvm::Instruction::Xor;
4136 break;
4137 case BO_OrAssign:
4138 AtomicOp = llvm::AtomicRMWInst::Or;
4139 Op = llvm::Instruction::Or;
4140 break;
4141 default:
4142 llvm_unreachable("Invalid compound assignment type");
4143 }
4144 if (AtomicOp != llvm::AtomicRMWInst::BAD_BINOP) {
4145 llvm::Value *Amt = CGF.EmitToMemory(
4146 Value: EmitScalarConversion(Src: OpInfo.RHS, SrcType: E->getRHS()->getType(), DstType: LHSTy,
4147 Loc: E->getExprLoc()),
4148 Ty: LHSTy);
4149
4150 llvm::AtomicRMWInst *OldVal =
4151 CGF.emitAtomicRMWInst(Op: AtomicOp, Addr: LHSLV.getAddress(), Val: Amt);
4152
4153 // Since operation is atomic, the result type is guaranteed to be the
4154 // same as the input in LLVM terms.
4155 Result = Builder.CreateBinOp(Opc: Op, LHS: OldVal, RHS: Amt);
4156 return LHSLV;
4157 }
4158 }
4159 // FIXME: For floating point types, we should be saving and restoring the
4160 // floating point environment in the loop.
4161 llvm::BasicBlock *startBB = Builder.GetInsertBlock();
4162 llvm::BasicBlock *opBB = CGF.createBasicBlock(name: "atomic_op", parent: CGF.CurFn);
4163 OpInfo.LHS = EmitLoadOfLValue(LV: LHSLV, Loc: E->getExprLoc());
4164 OpInfo.LHS = CGF.EmitToMemory(Value: OpInfo.LHS, Ty: AtomicValueTy);
4165 Builder.CreateBr(Dest: opBB);
4166 Builder.SetInsertPoint(opBB);
4167 atomicPHI = Builder.CreatePHI(Ty: OpInfo.LHS->getType(), NumReservedValues: 2);
4168 atomicPHI->addIncoming(V: OpInfo.LHS, BB: startBB);
4169 OpInfo.LHS = atomicPHI;
4170 }
4171 else
4172 OpInfo.LHS = EmitLoadOfLValue(LV: LHSLV, Loc: E->getExprLoc());
4173
4174 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, OpInfo.FPFeatures);
4175 SourceLocation Loc = E->getExprLoc();
4176 if (!PromotionTypeLHS.isNull())
4177 OpInfo.LHS = EmitScalarConversion(Src: OpInfo.LHS, SrcType: LHSTy, DstType: PromotionTypeLHS,
4178 Loc: E->getExprLoc());
4179 else
4180 OpInfo.LHS = EmitScalarConversion(Src: OpInfo.LHS, SrcType: LHSTy,
4181 DstType: E->getComputationLHSType(), Loc);
4182
4183 // Expand the binary operator.
4184 Result = (this->*Func)(OpInfo);
4185
4186 // Convert the result back to the LHS type,
4187 // potentially with Implicit Conversion sanitizer check.
4188 // If LHSLV is a bitfield, use default ScalarConversionOpts
4189 // to avoid emit any implicit integer checks.
4190 Value *Previous = nullptr;
4191 if (LHSLV.isBitField()) {
4192 Previous = Result;
4193 Result = EmitScalarConversion(Src: Result, SrcType: PromotionTypeCR, DstType: LHSTy, Loc);
4194 } else if (const auto *atomicTy = LHSTy->getAs<AtomicType>()) {
4195 Result =
4196 EmitScalarConversion(Src: Result, SrcType: PromotionTypeCR, DstType: atomicTy->getValueType(),
4197 Loc, Opts: ScalarConversionOpts(CGF.SanOpts));
4198 } else {
4199 Result = EmitScalarConversion(Src: Result, SrcType: PromotionTypeCR, DstType: LHSTy, Loc,
4200 Opts: ScalarConversionOpts(CGF.SanOpts));
4201 }
4202
4203 if (atomicPHI) {
4204 llvm::BasicBlock *curBlock = Builder.GetInsertBlock();
4205 llvm::BasicBlock *contBB = CGF.createBasicBlock(name: "atomic_cont", parent: CGF.CurFn);
4206 auto Pair = CGF.EmitAtomicCompareExchange(
4207 Obj: LHSLV, Expected: RValue::get(V: atomicPHI), Desired: RValue::get(V: Result), Loc: E->getExprLoc());
4208 llvm::Value *old = CGF.EmitToMemory(Value: Pair.first.getScalarVal(), Ty: LHSTy);
4209 llvm::Value *success = Pair.second;
4210 atomicPHI->addIncoming(V: old, BB: curBlock);
4211 Builder.CreateCondBr(Cond: success, True: contBB, False: atomicPHI->getParent());
4212 Builder.SetInsertPoint(contBB);
4213 return LHSLV;
4214 }
4215
4216 // Store the result value into the LHS lvalue. Bit-fields are handled
4217 // specially because the result is altered by the store, i.e., [C99 6.5.16p1]
4218 // 'An assignment expression has the value of the left operand after the
4219 // assignment...'.
4220 if (LHSLV.isBitField()) {
4221 Value *Src = Previous ? Previous : Result;
4222 QualType SrcType = E->getRHS()->getType();
4223 QualType DstType = E->getLHS()->getType();
4224 CGF.EmitStoreThroughBitfieldLValue(Src: RValue::get(V: Result), Dst: LHSLV, Result: &Result);
4225 CGF.EmitBitfieldConversionCheck(Src, SrcType, Dst: Result, DstType,
4226 Info: LHSLV.getBitFieldInfo(), Loc: E->getExprLoc());
4227 } else
4228 CGF.EmitStoreThroughLValue(Src: RValue::get(V: Result), Dst: LHSLV);
4229
4230 if (CGF.getLangOpts().OpenMP)
4231 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF,
4232 LHS: E->getLHS());
4233 return LHSLV;
4234}
4235
4236Value *ScalarExprEmitter::EmitCompoundAssign(const CompoundAssignOperator *E,
4237 Value *(ScalarExprEmitter::*Func)(const BinOpInfo &)) {
4238 bool Ignore = TestAndClearIgnoreResultAssign();
4239 Value *RHS = nullptr;
4240 LValue LHS = EmitCompoundAssignLValue(E, Func, Result&: RHS);
4241
4242 // If the result is clearly ignored, return now.
4243 if (Ignore)
4244 return nullptr;
4245
4246 // The result of an assignment in C is the assigned r-value.
4247 if (!CGF.getLangOpts().CPlusPlus)
4248 return RHS;
4249
4250 // If the lvalue is non-volatile, return the computed value of the assignment.
4251 if (!LHS.isVolatileQualified())
4252 return RHS;
4253
4254 // Otherwise, reload the value.
4255 return EmitLoadOfLValue(LV: LHS, Loc: E->getExprLoc());
4256}
4257
4258void ScalarExprEmitter::EmitUndefinedBehaviorIntegerDivAndRemCheck(
4259 const BinOpInfo &Ops, llvm::Value *Zero, bool isDiv) {
4260 SmallVector<std::pair<llvm::Value *, SanitizerKind::SanitizerOrdinal>, 2>
4261 Checks;
4262
4263 if (CGF.SanOpts.has(K: SanitizerKind::IntegerDivideByZero)) {
4264 Checks.push_back(Elt: std::make_pair(x: Builder.CreateICmpNE(LHS: Ops.RHS, RHS: Zero),
4265 y: SanitizerKind::SO_IntegerDivideByZero));
4266 }
4267
4268 const auto *BO = cast<BinaryOperator>(Val: Ops.E);
4269 if (CGF.SanOpts.has(K: SanitizerKind::SignedIntegerOverflow) &&
4270 Ops.Ty->hasSignedIntegerRepresentation() &&
4271 !IsWidenedIntegerOp(Ctx: CGF.getContext(), E: BO->getLHS()) &&
4272 Ops.mayHaveIntegerOverflow() &&
4273 !CGF.getContext().isTypeIgnoredBySanitizer(
4274 Mask: SanitizerKind::SignedIntegerOverflow, Ty: Ops.Ty)) {
4275 llvm::IntegerType *Ty = cast<llvm::IntegerType>(Val: Zero->getType());
4276
4277 llvm::Value *IntMin =
4278 Builder.getInt(AI: llvm::APInt::getSignedMinValue(numBits: Ty->getBitWidth()));
4279 llvm::Value *NegOne = llvm::Constant::getAllOnesValue(Ty);
4280
4281 llvm::Value *LHSCmp = Builder.CreateICmpNE(LHS: Ops.LHS, RHS: IntMin);
4282 llvm::Value *RHSCmp = Builder.CreateICmpNE(LHS: Ops.RHS, RHS: NegOne);
4283 llvm::Value *NotOverflow = Builder.CreateOr(LHS: LHSCmp, RHS: RHSCmp, Name: "or");
4284 Checks.push_back(
4285 Elt: std::make_pair(x&: NotOverflow, y: SanitizerKind::SO_SignedIntegerOverflow));
4286 }
4287
4288 if (Checks.size() > 0)
4289 EmitBinOpCheck(Checks, Info: Ops);
4290}
4291
4292Value *ScalarExprEmitter::EmitDiv(const BinOpInfo &Ops) {
4293 {
4294 SanitizerDebugLocation SanScope(&CGF,
4295 {SanitizerKind::SO_IntegerDivideByZero,
4296 SanitizerKind::SO_SignedIntegerOverflow,
4297 SanitizerKind::SO_FloatDivideByZero},
4298 SanitizerHandler::DivremOverflow);
4299 if ((CGF.SanOpts.has(K: SanitizerKind::IntegerDivideByZero) ||
4300 CGF.SanOpts.has(K: SanitizerKind::SignedIntegerOverflow)) &&
4301 Ops.Ty->isIntegerType() &&
4302 (Ops.mayHaveIntegerDivisionByZero() || Ops.mayHaveIntegerOverflow())) {
4303 llvm::Value *Zero = llvm::Constant::getNullValue(Ty: ConvertType(T: Ops.Ty));
4304 EmitUndefinedBehaviorIntegerDivAndRemCheck(Ops, Zero, isDiv: true);
4305 } else if (CGF.SanOpts.has(K: SanitizerKind::FloatDivideByZero) &&
4306 Ops.Ty->isRealFloatingType() &&
4307 Ops.mayHaveFloatDivisionByZero()) {
4308 llvm::Value *Zero = llvm::Constant::getNullValue(Ty: ConvertType(T: Ops.Ty));
4309 llvm::Value *NonZero = Builder.CreateFCmpUNE(LHS: Ops.RHS, RHS: Zero);
4310 EmitBinOpCheck(
4311 Checks: std::make_pair(x&: NonZero, y: SanitizerKind::SO_FloatDivideByZero), Info: Ops);
4312 }
4313 }
4314
4315 if (Ops.Ty->isConstantMatrixType()) {
4316 llvm::MatrixBuilder MB(Builder);
4317 // We need to check the types of the operands of the operator to get the
4318 // correct matrix dimensions.
4319 auto *BO = cast<BinaryOperator>(Val: Ops.E);
4320 (void)BO;
4321 assert(
4322 isa<ConstantMatrixType>(BO->getLHS()->getType().getCanonicalType()) &&
4323 "first operand must be a matrix");
4324 assert(BO->getRHS()->getType().getCanonicalType()->isArithmeticType() &&
4325 "second operand must be an arithmetic type");
4326 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, Ops.FPFeatures);
4327 return MB.CreateScalarDiv(LHS: Ops.LHS, RHS: Ops.RHS,
4328 IsUnsigned: Ops.Ty->hasUnsignedIntegerRepresentation());
4329 }
4330
4331 if (Ops.LHS->getType()->isFPOrFPVectorTy()) {
4332 llvm::Value *Val;
4333 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, Ops.FPFeatures);
4334 Val = Builder.CreateFDiv(L: Ops.LHS, R: Ops.RHS, Name: "div");
4335 CGF.SetDivFPAccuracy(Val);
4336 return Val;
4337 }
4338 else if (Ops.isFixedPointOp())
4339 return EmitFixedPointBinOp(Ops);
4340 else if (Ops.Ty->hasUnsignedIntegerRepresentation())
4341 return Builder.CreateUDiv(LHS: Ops.LHS, RHS: Ops.RHS, Name: "div");
4342 else
4343 return Builder.CreateSDiv(LHS: Ops.LHS, RHS: Ops.RHS, Name: "div");
4344}
4345
4346Value *ScalarExprEmitter::EmitRem(const BinOpInfo &Ops) {
4347 // Rem in C can't be a floating point type: C99 6.5.5p2.
4348 if ((CGF.SanOpts.has(K: SanitizerKind::IntegerDivideByZero) ||
4349 CGF.SanOpts.has(K: SanitizerKind::SignedIntegerOverflow)) &&
4350 Ops.Ty->isIntegerType() &&
4351 (Ops.mayHaveIntegerDivisionByZero() || Ops.mayHaveIntegerOverflow())) {
4352 SanitizerDebugLocation SanScope(&CGF,
4353 {SanitizerKind::SO_IntegerDivideByZero,
4354 SanitizerKind::SO_SignedIntegerOverflow},
4355 SanitizerHandler::DivremOverflow);
4356 llvm::Value *Zero = llvm::Constant::getNullValue(Ty: ConvertType(T: Ops.Ty));
4357 EmitUndefinedBehaviorIntegerDivAndRemCheck(Ops, Zero, isDiv: false);
4358 }
4359
4360 if (Ops.Ty->hasUnsignedIntegerRepresentation())
4361 return Builder.CreateURem(LHS: Ops.LHS, RHS: Ops.RHS, Name: "rem");
4362
4363 if (CGF.getLangOpts().HLSL && Ops.Ty->hasFloatingRepresentation())
4364 return Builder.CreateFRem(L: Ops.LHS, R: Ops.RHS, Name: "rem");
4365
4366 return Builder.CreateSRem(LHS: Ops.LHS, RHS: Ops.RHS, Name: "rem");
4367}
4368
4369Value *ScalarExprEmitter::EmitOverflowCheckedBinOp(const BinOpInfo &Ops) {
4370 unsigned IID;
4371 unsigned OpID = 0;
4372 SanitizerHandler OverflowKind;
4373
4374 bool isSigned = Ops.Ty->isSignedIntegerOrEnumerationType();
4375 switch (Ops.Opcode) {
4376 case BO_Add:
4377 case BO_AddAssign:
4378 OpID = 1;
4379 IID = isSigned ? llvm::Intrinsic::sadd_with_overflow :
4380 llvm::Intrinsic::uadd_with_overflow;
4381 OverflowKind = SanitizerHandler::AddOverflow;
4382 break;
4383 case BO_Sub:
4384 case BO_SubAssign:
4385 OpID = 2;
4386 IID = isSigned ? llvm::Intrinsic::ssub_with_overflow :
4387 llvm::Intrinsic::usub_with_overflow;
4388 OverflowKind = SanitizerHandler::SubOverflow;
4389 break;
4390 case BO_Mul:
4391 case BO_MulAssign:
4392 OpID = 3;
4393 IID = isSigned ? llvm::Intrinsic::smul_with_overflow :
4394 llvm::Intrinsic::umul_with_overflow;
4395 OverflowKind = SanitizerHandler::MulOverflow;
4396 break;
4397 default:
4398 llvm_unreachable("Unsupported operation for overflow detection");
4399 }
4400 OpID <<= 1;
4401 if (isSigned)
4402 OpID |= 1;
4403
4404 SanitizerDebugLocation SanScope(&CGF,
4405 {SanitizerKind::SO_SignedIntegerOverflow,
4406 SanitizerKind::SO_UnsignedIntegerOverflow},
4407 OverflowKind);
4408 llvm::Type *opTy = CGF.CGM.getTypes().ConvertType(T: Ops.Ty);
4409
4410 llvm::Function *intrinsic = CGF.CGM.getIntrinsic(IID, Tys: opTy);
4411
4412 Value *resultAndOverflow = Builder.CreateCall(Callee: intrinsic, Args: {Ops.LHS, Ops.RHS});
4413 Value *result = Builder.CreateExtractValue(Agg: resultAndOverflow, Idxs: 0);
4414 Value *overflow = Builder.CreateExtractValue(Agg: resultAndOverflow, Idxs: 1);
4415
4416 // Handle overflow with llvm.trap if no custom handler has been specified.
4417 const std::string *handlerName =
4418 &CGF.getLangOpts().OverflowHandler;
4419 if (handlerName->empty()) {
4420 // If no -ftrapv handler has been specified, try to use sanitizer runtimes
4421 // if available otherwise just emit a trap. It is possible for unsigned
4422 // arithmetic to result in a trap due to the OverflowBehaviorType attribute
4423 // which describes overflow behavior on a per-type basis.
4424 if (isSigned) {
4425 if (CGF.SanOpts.has(K: SanitizerKind::SignedIntegerOverflow)) {
4426 llvm::Value *NotOf = Builder.CreateNot(V: overflow);
4427 EmitBinOpCheck(
4428 Checks: std::make_pair(x&: NotOf, y: SanitizerKind::SO_SignedIntegerOverflow),
4429 Info: Ops);
4430 } else
4431 CGF.EmitTrapCheck(Checked: Builder.CreateNot(V: overflow), CheckHandlerID: OverflowKind);
4432 return result;
4433 }
4434 if (CGF.SanOpts.has(K: SanitizerKind::UnsignedIntegerOverflow)) {
4435 llvm::Value *NotOf = Builder.CreateNot(V: overflow);
4436 EmitBinOpCheck(
4437 Checks: std::make_pair(x&: NotOf, y: SanitizerKind::SO_UnsignedIntegerOverflow),
4438 Info: Ops);
4439 } else
4440 CGF.EmitTrapCheck(Checked: Builder.CreateNot(V: overflow), CheckHandlerID: OverflowKind);
4441 return result;
4442 }
4443
4444 // Branch in case of overflow.
4445 llvm::BasicBlock *initialBB = Builder.GetInsertBlock();
4446 llvm::BasicBlock *continueBB =
4447 CGF.createBasicBlock(name: "nooverflow", parent: CGF.CurFn, before: initialBB->getNextNode());
4448 llvm::BasicBlock *overflowBB = CGF.createBasicBlock(name: "overflow", parent: CGF.CurFn);
4449
4450 Builder.CreateCondBr(Cond: overflow, True: overflowBB, False: continueBB);
4451
4452 // If an overflow handler is set, then we want to call it and then use its
4453 // result, if it returns.
4454 Builder.SetInsertPoint(overflowBB);
4455
4456 // Get the overflow handler.
4457 llvm::Type *Int8Ty = CGF.Int8Ty;
4458 llvm::Type *argTypes[] = { CGF.Int64Ty, CGF.Int64Ty, Int8Ty, Int8Ty };
4459 llvm::FunctionType *handlerTy =
4460 llvm::FunctionType::get(Result: CGF.Int64Ty, Params: argTypes, isVarArg: true);
4461 llvm::FunctionCallee handler =
4462 CGF.CGM.CreateRuntimeFunction(Ty: handlerTy, Name: *handlerName);
4463
4464 // Sign extend the args to 64-bit, so that we can use the same handler for
4465 // all types of overflow.
4466 llvm::Value *lhs = Builder.CreateSExt(V: Ops.LHS, DestTy: CGF.Int64Ty);
4467 llvm::Value *rhs = Builder.CreateSExt(V: Ops.RHS, DestTy: CGF.Int64Ty);
4468
4469 // Call the handler with the two arguments, the operation, and the size of
4470 // the result.
4471 llvm::Value *handlerArgs[] = {
4472 lhs,
4473 rhs,
4474 Builder.getInt8(C: OpID),
4475 Builder.getInt8(C: cast<llvm::IntegerType>(Val: opTy)->getBitWidth())
4476 };
4477 llvm::Value *handlerResult =
4478 CGF.EmitNounwindRuntimeCall(callee: handler, args: handlerArgs);
4479
4480 // Truncate the result back to the desired size.
4481 handlerResult = Builder.CreateTrunc(V: handlerResult, DestTy: opTy);
4482 Builder.CreateBr(Dest: continueBB);
4483
4484 Builder.SetInsertPoint(continueBB);
4485 llvm::PHINode *phi = Builder.CreatePHI(Ty: opTy, NumReservedValues: 2);
4486 phi->addIncoming(V: result, BB: initialBB);
4487 phi->addIncoming(V: handlerResult, BB: overflowBB);
4488
4489 return phi;
4490}
4491
4492/// BO_Add/BO_Sub are handled by EmitPointerWithAlignment to preserve alignment
4493/// information.
4494/// This function is used for BO_AddAssign/BO_SubAssign.
4495static Value *emitPointerArithmetic(CodeGenFunction &CGF, const BinOpInfo &op,
4496 bool isSubtraction) {
4497 // Must have binary (not unary) expr here. Unary pointer
4498 // increment/decrement doesn't use this path.
4499 const BinaryOperator *expr = cast<BinaryOperator>(Val: op.E);
4500
4501 Value *pointer = op.LHS;
4502 Expr *pointerOperand = expr->getLHS();
4503 Value *index = op.RHS;
4504 Expr *indexOperand = expr->getRHS();
4505
4506 // In a subtraction, the LHS is always the pointer.
4507 if (!isSubtraction && !pointer->getType()->isPointerTy()) {
4508 std::swap(a&: pointer, b&: index);
4509 std::swap(a&: pointerOperand, b&: indexOperand);
4510 }
4511
4512 return CGF.EmitPointerArithmetic(BO: expr, pointerOperand, pointer, indexOperand,
4513 index, isSubtraction);
4514}
4515
4516/// Emit pointer + index arithmetic.
4517llvm::Value *CodeGenFunction::EmitPointerArithmetic(
4518 const BinaryOperator *BO, Expr *pointerOperand, llvm::Value *pointer,
4519 Expr *indexOperand, llvm::Value *index, bool isSubtraction) {
4520 bool isSigned = indexOperand->getType()->isSignedIntegerOrEnumerationType();
4521
4522 unsigned width = cast<llvm::IntegerType>(Val: index->getType())->getBitWidth();
4523 auto &DL = CGM.getDataLayout();
4524 auto *PtrTy = cast<llvm::PointerType>(Val: pointer->getType());
4525
4526 // Some versions of glibc and gcc use idioms (particularly in their malloc
4527 // routines) that add a pointer-sized integer (known to be a pointer value)
4528 // to a null pointer in order to cast the value back to an integer or as
4529 // part of a pointer alignment algorithm. This is undefined behavior, but
4530 // we'd like to be able to compile programs that use it.
4531 //
4532 // Normally, we'd generate a GEP with a null-pointer base here in response
4533 // to that code, but it's also UB to dereference a pointer created that
4534 // way. Instead (as an acknowledged hack to tolerate the idiom) we will
4535 // generate a direct cast of the integer value to a pointer.
4536 //
4537 // The idiom (p = nullptr + N) is not met if any of the following are true:
4538 //
4539 // The operation is subtraction.
4540 // The index is not pointer-sized.
4541 // The pointer type is not byte-sized.
4542 //
4543 // Note that we do not suppress the pointer overflow check in this case.
4544 if (BinaryOperator::isNullPointerArithmeticExtension(
4545 Ctx&: getContext(), Opc: BO->getOpcode(), LHS: pointerOperand, RHS: indexOperand)) {
4546 llvm::Value *Ptr = Builder.CreateIntToPtr(V: index, DestTy: pointer->getType());
4547 if (getLangOpts().PointerOverflowDefined ||
4548 !SanOpts.has(K: SanitizerKind::PointerOverflow) ||
4549 NullPointerIsDefined(F: Builder.GetInsertBlock()->getParent(),
4550 AS: PtrTy->getPointerAddressSpace()))
4551 return Ptr;
4552 // The inbounds GEP of null is valid iff the index is zero.
4553 auto CheckOrdinal = SanitizerKind::SO_PointerOverflow;
4554 auto CheckHandler = SanitizerHandler::PointerOverflow;
4555 SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);
4556 llvm::Value *IsZeroIndex = Builder.CreateIsNull(Arg: index);
4557 llvm::Constant *StaticArgs[] = {EmitCheckSourceLocation(Loc: BO->getExprLoc())};
4558 llvm::Type *IntPtrTy = DL.getIntPtrType(PtrTy);
4559 llvm::Value *IntPtr = llvm::Constant::getNullValue(Ty: IntPtrTy);
4560 llvm::Value *ComputedGEP = Builder.CreateZExtOrTrunc(V: index, DestTy: IntPtrTy);
4561 llvm::Value *DynamicArgs[] = {IntPtr, ComputedGEP};
4562 EmitCheck(Checked: {{IsZeroIndex, CheckOrdinal}}, Check: CheckHandler, StaticArgs,
4563 DynamicArgs);
4564 return Ptr;
4565 }
4566
4567 if (width != DL.getIndexTypeSizeInBits(Ty: PtrTy)) {
4568 // Zero-extend or sign-extend the pointer value according to
4569 // whether the index is signed or not.
4570 index = Builder.CreateIntCast(V: index, DestTy: DL.getIndexType(PtrTy), isSigned,
4571 Name: "idx.ext");
4572 }
4573
4574 // If this is subtraction, negate the index.
4575 if (isSubtraction)
4576 index = Builder.CreateNeg(V: index, Name: "idx.neg");
4577
4578 if (SanOpts.has(K: SanitizerKind::ArrayBounds))
4579 EmitBoundsCheck(ArrayExpr: BO, ArrayExprBase: pointerOperand, Index: index, IndexType: indexOperand->getType(),
4580 /*Accessed*/ false);
4581
4582 const PointerType *pointerType =
4583 pointerOperand->getType()->getAs<PointerType>();
4584 if (!pointerType) {
4585 QualType objectType = pointerOperand->getType()
4586 ->castAs<ObjCObjectPointerType>()
4587 ->getPointeeType();
4588 llvm::Value *objectSize =
4589 CGM.getSize(numChars: getContext().getTypeSizeInChars(T: objectType));
4590
4591 index = Builder.CreateMul(LHS: index, RHS: objectSize);
4592
4593 llvm::Value *result = Builder.CreateGEP(Ty: Int8Ty, Ptr: pointer, IdxList: index, Name: "add.ptr");
4594 return Builder.CreateBitCast(V: result, DestTy: pointer->getType());
4595 }
4596
4597 QualType elementType = pointerType->getPointeeType();
4598 if (const VariableArrayType *vla =
4599 getContext().getAsVariableArrayType(T: elementType)) {
4600 // The element count here is the total number of non-VLA elements.
4601 llvm::Value *numElements = getVLASize(vla).NumElts;
4602
4603 // Effectively, the multiply by the VLA size is part of the GEP.
4604 // GEP indexes are signed, and scaling an index isn't permitted to
4605 // signed-overflow, so we use the same semantics for our explicit
4606 // multiply. We suppress this if overflow is not undefined behavior.
4607 llvm::Type *elemTy = ConvertTypeForMem(T: vla->getElementType());
4608 if (getLangOpts().PointerOverflowDefined) {
4609 index = Builder.CreateMul(LHS: index, RHS: numElements, Name: "vla.index");
4610 pointer = Builder.CreateGEP(Ty: elemTy, Ptr: pointer, IdxList: index, Name: "add.ptr");
4611 } else {
4612 index = Builder.CreateNSWMul(LHS: index, RHS: numElements, Name: "vla.index");
4613 pointer =
4614 EmitCheckedInBoundsGEP(ElemTy: elemTy, Ptr: pointer, IdxList: index, SignedIndices: isSigned,
4615 IsSubtraction: isSubtraction, Loc: BO->getExprLoc(), Name: "add.ptr");
4616 }
4617 return pointer;
4618 }
4619
4620 // Explicitly handle GNU void* and function pointer arithmetic extensions. The
4621 // GNU void* casts amount to no-ops since our void* type is i8*, but this is
4622 // future proof.
4623 llvm::Type *elemTy;
4624 if (elementType->isVoidType() || elementType->isFunctionType())
4625 elemTy = Int8Ty;
4626 else
4627 elemTy = ConvertTypeForMem(T: elementType);
4628
4629 if (getLangOpts().PointerOverflowDefined)
4630 return Builder.CreateGEP(Ty: elemTy, Ptr: pointer, IdxList: index, Name: "add.ptr");
4631
4632 return EmitCheckedInBoundsGEP(ElemTy: elemTy, Ptr: pointer, IdxList: index, SignedIndices: isSigned, IsSubtraction: isSubtraction,
4633 Loc: BO->getExprLoc(), Name: "add.ptr");
4634}
4635
4636// Construct an fmuladd intrinsic to represent a fused mul-add of MulOp and
4637// Addend. Use negMul and negAdd to negate the first operand of the Mul or
4638// the add operand respectively. This allows fmuladd to represent a*b-c, or
4639// c-a*b. Patterns in LLVM should catch the negated forms and translate them to
4640// efficient operations.
4641static Value* buildFMulAdd(llvm::Instruction *MulOp, Value *Addend,
4642 const CodeGenFunction &CGF, CGBuilderTy &Builder,
4643 bool negMul, bool negAdd) {
4644 Value *MulOp0 = MulOp->getOperand(i: 0);
4645 Value *MulOp1 = MulOp->getOperand(i: 1);
4646 if (negMul)
4647 MulOp0 = Builder.CreateFNeg(V: MulOp0, Name: "neg");
4648 if (negAdd)
4649 Addend = Builder.CreateFNeg(V: Addend, Name: "neg");
4650
4651 Value *FMulAdd = nullptr;
4652 if (Builder.getIsFPConstrained()) {
4653 assert(isa<llvm::ConstrainedFPIntrinsic>(MulOp) &&
4654 "Only constrained operation should be created when Builder is in FP "
4655 "constrained mode");
4656 FMulAdd = Builder.CreateConstrainedFPCall(
4657 Callee: CGF.CGM.getIntrinsic(IID: llvm::Intrinsic::experimental_constrained_fmuladd,
4658 Tys: Addend->getType()),
4659 Args: {MulOp0, MulOp1, Addend});
4660 } else {
4661 FMulAdd = Builder.CreateCall(
4662 Callee: CGF.CGM.getIntrinsic(IID: llvm::Intrinsic::fmuladd, Tys: Addend->getType()),
4663 Args: {MulOp0, MulOp1, Addend});
4664 }
4665 MulOp->eraseFromParent();
4666
4667 return FMulAdd;
4668}
4669
4670// Check whether it would be legal to emit an fmuladd intrinsic call to
4671// represent op and if so, build the fmuladd.
4672//
4673// Checks that (a) the operation is fusable, and (b) -ffp-contract=on.
4674// Does NOT check the type of the operation - it's assumed that this function
4675// will be called from contexts where it's known that the type is contractable.
4676static Value* tryEmitFMulAdd(const BinOpInfo &op,
4677 const CodeGenFunction &CGF, CGBuilderTy &Builder,
4678 bool isSub=false) {
4679
4680 assert((op.Opcode == BO_Add || op.Opcode == BO_AddAssign ||
4681 op.Opcode == BO_Sub || op.Opcode == BO_SubAssign) &&
4682 "Only fadd/fsub can be the root of an fmuladd.");
4683
4684 // Check whether this op is marked as fusable.
4685 if (!op.FPFeatures.allowFPContractWithinStatement())
4686 return nullptr;
4687
4688 Value *LHS = op.LHS;
4689 Value *RHS = op.RHS;
4690
4691 // Peek through fneg to look for fmul. Make sure fneg has no users, and that
4692 // it is the only use of its operand.
4693 bool NegLHS = false;
4694 if (auto *LHSUnOp = dyn_cast<llvm::UnaryOperator>(Val: LHS)) {
4695 if (LHSUnOp->getOpcode() == llvm::Instruction::FNeg &&
4696 LHSUnOp->use_empty() && LHSUnOp->getOperand(i_nocapture: 0)->hasOneUse()) {
4697 LHS = LHSUnOp->getOperand(i_nocapture: 0);
4698 NegLHS = true;
4699 }
4700 }
4701
4702 bool NegRHS = false;
4703 if (auto *RHSUnOp = dyn_cast<llvm::UnaryOperator>(Val: RHS)) {
4704 if (RHSUnOp->getOpcode() == llvm::Instruction::FNeg &&
4705 RHSUnOp->use_empty() && RHSUnOp->getOperand(i_nocapture: 0)->hasOneUse()) {
4706 RHS = RHSUnOp->getOperand(i_nocapture: 0);
4707 NegRHS = true;
4708 }
4709 }
4710
4711 // We have a potentially fusable op. Look for a mul on one of the operands.
4712 // Also, make sure that the mul result isn't used directly. In that case,
4713 // there's no point creating a muladd operation.
4714 if (auto *LHSBinOp = dyn_cast<llvm::BinaryOperator>(Val: LHS)) {
4715 if (LHSBinOp->getOpcode() == llvm::Instruction::FMul &&
4716 (LHSBinOp->use_empty() || NegLHS)) {
4717 // If we looked through fneg, erase it.
4718 if (NegLHS)
4719 cast<llvm::Instruction>(Val: op.LHS)->eraseFromParent();
4720 return buildFMulAdd(MulOp: LHSBinOp, Addend: op.RHS, CGF, Builder, negMul: NegLHS, negAdd: isSub);
4721 }
4722 }
4723 if (auto *RHSBinOp = dyn_cast<llvm::BinaryOperator>(Val: RHS)) {
4724 if (RHSBinOp->getOpcode() == llvm::Instruction::FMul &&
4725 (RHSBinOp->use_empty() || NegRHS)) {
4726 // If we looked through fneg, erase it.
4727 if (NegRHS)
4728 cast<llvm::Instruction>(Val: op.RHS)->eraseFromParent();
4729 return buildFMulAdd(MulOp: RHSBinOp, Addend: op.LHS, CGF, Builder, negMul: isSub ^ NegRHS, negAdd: false);
4730 }
4731 }
4732
4733 if (auto *LHSBinOp = dyn_cast<llvm::CallBase>(Val: LHS)) {
4734 if (LHSBinOp->getIntrinsicID() ==
4735 llvm::Intrinsic::experimental_constrained_fmul &&
4736 (LHSBinOp->use_empty() || NegLHS)) {
4737 // If we looked through fneg, erase it.
4738 if (NegLHS)
4739 cast<llvm::Instruction>(Val: op.LHS)->eraseFromParent();
4740 return buildFMulAdd(MulOp: LHSBinOp, Addend: op.RHS, CGF, Builder, negMul: NegLHS, negAdd: isSub);
4741 }
4742 }
4743 if (auto *RHSBinOp = dyn_cast<llvm::CallBase>(Val: RHS)) {
4744 if (RHSBinOp->getIntrinsicID() ==
4745 llvm::Intrinsic::experimental_constrained_fmul &&
4746 (RHSBinOp->use_empty() || NegRHS)) {
4747 // If we looked through fneg, erase it.
4748 if (NegRHS)
4749 cast<llvm::Instruction>(Val: op.RHS)->eraseFromParent();
4750 return buildFMulAdd(MulOp: RHSBinOp, Addend: op.LHS, CGF, Builder, negMul: isSub ^ NegRHS, negAdd: false);
4751 }
4752 }
4753
4754 return nullptr;
4755}
4756
4757Value *ScalarExprEmitter::EmitAdd(const BinOpInfo &op) {
4758 if (op.LHS->getType()->isPointerTy() ||
4759 op.RHS->getType()->isPointerTy())
4760 return emitPointerArithmetic(CGF, op, isSubtraction: CodeGenFunction::NotSubtraction);
4761
4762 if (op.Ty->isSignedIntegerOrEnumerationType() ||
4763 op.Ty->isUnsignedIntegerType()) {
4764 const bool isSigned = op.Ty->isSignedIntegerOrEnumerationType();
4765 const bool hasSan =
4766 isSigned ? CGF.SanOpts.has(K: SanitizerKind::SignedIntegerOverflow)
4767 : CGF.SanOpts.has(K: SanitizerKind::UnsignedIntegerOverflow);
4768 switch (getOverflowBehaviorConsideringType(CGF, Ty: op.Ty)) {
4769 case LangOptions::OB_Wrap:
4770 return Builder.CreateAdd(LHS: op.LHS, RHS: op.RHS, Name: "add");
4771 case LangOptions::OB_SignedAndDefined:
4772 if (!hasSan)
4773 return Builder.CreateAdd(LHS: op.LHS, RHS: op.RHS, Name: "add");
4774 [[fallthrough]];
4775 case LangOptions::OB_Unset:
4776 if (!hasSan)
4777 return isSigned ? Builder.CreateNSWAdd(LHS: op.LHS, RHS: op.RHS, Name: "add")
4778 : Builder.CreateAdd(LHS: op.LHS, RHS: op.RHS, Name: "add");
4779 [[fallthrough]];
4780 case LangOptions::OB_Trap:
4781 if (CanElideOverflowCheck(Ctx&: CGF.getContext(), Op: op))
4782 return isSigned ? Builder.CreateNSWAdd(LHS: op.LHS, RHS: op.RHS, Name: "add")
4783 : Builder.CreateAdd(LHS: op.LHS, RHS: op.RHS, Name: "add");
4784 return EmitOverflowCheckedBinOp(Ops: op);
4785 }
4786 }
4787
4788 // For vector and matrix adds, try to fold into a fmuladd.
4789 if (op.LHS->getType()->isFPOrFPVectorTy()) {
4790 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, op.FPFeatures);
4791 // Try to form an fmuladd.
4792 if (Value *FMulAdd = tryEmitFMulAdd(op, CGF, Builder))
4793 return FMulAdd;
4794 }
4795
4796 if (op.Ty->isConstantMatrixType()) {
4797 llvm::MatrixBuilder MB(Builder);
4798 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, op.FPFeatures);
4799 return MB.CreateAdd(LHS: op.LHS, RHS: op.RHS);
4800 }
4801
4802 if (op.LHS->getType()->isFPOrFPVectorTy()) {
4803 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, op.FPFeatures);
4804 return Builder.CreateFAdd(L: op.LHS, R: op.RHS, Name: "add");
4805 }
4806
4807 if (op.isFixedPointOp())
4808 return EmitFixedPointBinOp(Ops: op);
4809
4810 return Builder.CreateAdd(LHS: op.LHS, RHS: op.RHS, Name: "add");
4811}
4812
4813/// The resulting value must be calculated with exact precision, so the operands
4814/// may not be the same type.
4815Value *ScalarExprEmitter::EmitFixedPointBinOp(const BinOpInfo &op) {
4816 using llvm::APSInt;
4817 using llvm::ConstantInt;
4818
4819 // This is either a binary operation where at least one of the operands is
4820 // a fixed-point type, or a unary operation where the operand is a fixed-point
4821 // type. The result type of a binary operation is determined by
4822 // Sema::handleFixedPointConversions().
4823 QualType ResultTy = op.Ty;
4824 QualType LHSTy, RHSTy;
4825 if (const auto *BinOp = dyn_cast<BinaryOperator>(Val: op.E)) {
4826 RHSTy = BinOp->getRHS()->getType();
4827 if (const auto *CAO = dyn_cast<CompoundAssignOperator>(Val: BinOp)) {
4828 // For compound assignment, the effective type of the LHS at this point
4829 // is the computation LHS type, not the actual LHS type, and the final
4830 // result type is not the type of the expression but rather the
4831 // computation result type.
4832 LHSTy = CAO->getComputationLHSType();
4833 ResultTy = CAO->getComputationResultType();
4834 } else
4835 LHSTy = BinOp->getLHS()->getType();
4836 } else if (const auto *UnOp = dyn_cast<UnaryOperator>(Val: op.E)) {
4837 LHSTy = UnOp->getSubExpr()->getType();
4838 RHSTy = UnOp->getSubExpr()->getType();
4839 }
4840 ASTContext &Ctx = CGF.getContext();
4841 Value *LHS = op.LHS;
4842 Value *RHS = op.RHS;
4843
4844 auto LHSFixedSema = Ctx.getFixedPointSemantics(Ty: LHSTy);
4845 auto RHSFixedSema = Ctx.getFixedPointSemantics(Ty: RHSTy);
4846 auto ResultFixedSema = Ctx.getFixedPointSemantics(Ty: ResultTy);
4847 auto CommonFixedSema = LHSFixedSema.getCommonSemantics(Other: RHSFixedSema);
4848
4849 // Perform the actual operation.
4850 Value *Result;
4851 llvm::FixedPointBuilder<CGBuilderTy> FPBuilder(Builder);
4852 switch (op.Opcode) {
4853 case BO_AddAssign:
4854 case BO_Add:
4855 Result = FPBuilder.CreateAdd(LHS, LHSSema: LHSFixedSema, RHS, RHSSema: RHSFixedSema);
4856 break;
4857 case BO_SubAssign:
4858 case BO_Sub:
4859 Result = FPBuilder.CreateSub(LHS, LHSSema: LHSFixedSema, RHS, RHSSema: RHSFixedSema);
4860 break;
4861 case BO_MulAssign:
4862 case BO_Mul:
4863 Result = FPBuilder.CreateMul(LHS, LHSSema: LHSFixedSema, RHS, RHSSema: RHSFixedSema);
4864 break;
4865 case BO_DivAssign:
4866 case BO_Div:
4867 Result = FPBuilder.CreateDiv(LHS, LHSSema: LHSFixedSema, RHS, RHSSema: RHSFixedSema);
4868 break;
4869 case BO_ShlAssign:
4870 case BO_Shl:
4871 Result = FPBuilder.CreateShl(LHS, LHSSema: LHSFixedSema, RHS);
4872 break;
4873 case BO_ShrAssign:
4874 case BO_Shr:
4875 Result = FPBuilder.CreateShr(LHS, LHSSema: LHSFixedSema, RHS);
4876 break;
4877 case BO_LT:
4878 return FPBuilder.CreateLT(LHS, LHSSema: LHSFixedSema, RHS, RHSSema: RHSFixedSema);
4879 case BO_GT:
4880 return FPBuilder.CreateGT(LHS, LHSSema: LHSFixedSema, RHS, RHSSema: RHSFixedSema);
4881 case BO_LE:
4882 return FPBuilder.CreateLE(LHS, LHSSema: LHSFixedSema, RHS, RHSSema: RHSFixedSema);
4883 case BO_GE:
4884 return FPBuilder.CreateGE(LHS, LHSSema: LHSFixedSema, RHS, RHSSema: RHSFixedSema);
4885 case BO_EQ:
4886 // For equality operations, we assume any padding bits on unsigned types are
4887 // zero'd out. They could be overwritten through non-saturating operations
4888 // that cause overflow, but this leads to undefined behavior.
4889 return FPBuilder.CreateEQ(LHS, LHSSema: LHSFixedSema, RHS, RHSSema: RHSFixedSema);
4890 case BO_NE:
4891 return FPBuilder.CreateNE(LHS, LHSSema: LHSFixedSema, RHS, RHSSema: RHSFixedSema);
4892 case BO_Cmp:
4893 case BO_LAnd:
4894 case BO_LOr:
4895 llvm_unreachable("Found unimplemented fixed point binary operation");
4896 case BO_PtrMemD:
4897 case BO_PtrMemI:
4898 case BO_Rem:
4899 case BO_Xor:
4900 case BO_And:
4901 case BO_Or:
4902 case BO_Assign:
4903 case BO_RemAssign:
4904 case BO_AndAssign:
4905 case BO_XorAssign:
4906 case BO_OrAssign:
4907 case BO_Comma:
4908 llvm_unreachable("Found unsupported binary operation for fixed point types.");
4909 }
4910
4911 bool IsShift = BinaryOperator::isShiftOp(Opc: op.Opcode) ||
4912 BinaryOperator::isShiftAssignOp(Opc: op.Opcode);
4913 // Convert to the result type.
4914 return FPBuilder.CreateFixedToFixed(Src: Result, SrcSema: IsShift ? LHSFixedSema
4915 : CommonFixedSema,
4916 DstSema: ResultFixedSema);
4917}
4918
4919Value *ScalarExprEmitter::EmitSub(const BinOpInfo &op) {
4920 // The LHS is always a pointer if either side is.
4921 if (!op.LHS->getType()->isPointerTy()) {
4922 if (op.Ty->isSignedIntegerOrEnumerationType() ||
4923 op.Ty->isUnsignedIntegerType()) {
4924 const bool isSigned = op.Ty->isSignedIntegerOrEnumerationType();
4925 const bool hasSan =
4926 isSigned ? CGF.SanOpts.has(K: SanitizerKind::SignedIntegerOverflow)
4927 : CGF.SanOpts.has(K: SanitizerKind::UnsignedIntegerOverflow);
4928 switch (getOverflowBehaviorConsideringType(CGF, Ty: op.Ty)) {
4929 case LangOptions::OB_Wrap:
4930 return Builder.CreateSub(LHS: op.LHS, RHS: op.RHS, Name: "sub");
4931 case LangOptions::OB_SignedAndDefined:
4932 if (!hasSan)
4933 return Builder.CreateSub(LHS: op.LHS, RHS: op.RHS, Name: "sub");
4934 [[fallthrough]];
4935 case LangOptions::OB_Unset:
4936 if (!hasSan)
4937 return isSigned ? Builder.CreateNSWSub(LHS: op.LHS, RHS: op.RHS, Name: "sub")
4938 : Builder.CreateSub(LHS: op.LHS, RHS: op.RHS, Name: "sub");
4939 [[fallthrough]];
4940 case LangOptions::OB_Trap:
4941 if (CanElideOverflowCheck(Ctx&: CGF.getContext(), Op: op))
4942 return isSigned ? Builder.CreateNSWSub(LHS: op.LHS, RHS: op.RHS, Name: "sub")
4943 : Builder.CreateSub(LHS: op.LHS, RHS: op.RHS, Name: "sub");
4944 return EmitOverflowCheckedBinOp(Ops: op);
4945 }
4946 }
4947
4948 // For vector and matrix subs, try to fold into a fmuladd.
4949 if (op.LHS->getType()->isFPOrFPVectorTy()) {
4950 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, op.FPFeatures);
4951 // Try to form an fmuladd.
4952 if (Value *FMulAdd = tryEmitFMulAdd(op, CGF, Builder, isSub: true))
4953 return FMulAdd;
4954 }
4955
4956 if (op.Ty->isConstantMatrixType()) {
4957 llvm::MatrixBuilder MB(Builder);
4958 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, op.FPFeatures);
4959 return MB.CreateSub(LHS: op.LHS, RHS: op.RHS);
4960 }
4961
4962 if (op.LHS->getType()->isFPOrFPVectorTy()) {
4963 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, op.FPFeatures);
4964 return Builder.CreateFSub(L: op.LHS, R: op.RHS, Name: "sub");
4965 }
4966
4967 if (op.isFixedPointOp())
4968 return EmitFixedPointBinOp(op);
4969
4970 return Builder.CreateSub(LHS: op.LHS, RHS: op.RHS, Name: "sub");
4971 }
4972
4973 // If the RHS is not a pointer, then we have normal pointer
4974 // arithmetic.
4975 if (!op.RHS->getType()->isPointerTy())
4976 return emitPointerArithmetic(CGF, op, isSubtraction: CodeGenFunction::IsSubtraction);
4977
4978 // Otherwise, this is a pointer subtraction.
4979
4980 // Do the raw subtraction part.
4981 llvm::Value *LHS
4982 = Builder.CreatePtrToInt(V: op.LHS, DestTy: CGF.PtrDiffTy, Name: "sub.ptr.lhs.cast");
4983 llvm::Value *RHS
4984 = Builder.CreatePtrToInt(V: op.RHS, DestTy: CGF.PtrDiffTy, Name: "sub.ptr.rhs.cast");
4985 Value *diffInChars = Builder.CreateSub(LHS, RHS, Name: "sub.ptr.sub");
4986
4987 // Okay, figure out the element size.
4988 const BinaryOperator *expr = cast<BinaryOperator>(Val: op.E);
4989 QualType elementType = expr->getLHS()->getType()->getPointeeType();
4990
4991 llvm::Value *divisor = nullptr;
4992
4993 // For a variable-length array, this is going to be non-constant.
4994 if (const VariableArrayType *vla
4995 = CGF.getContext().getAsVariableArrayType(T: elementType)) {
4996 auto VlaSize = CGF.getVLASize(vla);
4997 elementType = VlaSize.Type;
4998 divisor = VlaSize.NumElts;
4999
5000 // Scale the number of non-VLA elements by the non-VLA element size.
5001 CharUnits eltSize = CGF.getContext().getTypeSizeInChars(T: elementType);
5002 if (!eltSize.isOne())
5003 divisor = CGF.Builder.CreateNUWMul(LHS: CGF.CGM.getSize(numChars: eltSize), RHS: divisor);
5004
5005 // For everything elese, we can just compute it, safe in the
5006 // assumption that Sema won't let anything through that we can't
5007 // safely compute the size of.
5008 } else {
5009 CharUnits elementSize;
5010 // Handle GCC extension for pointer arithmetic on void* and
5011 // function pointer types.
5012 if (elementType->isVoidType() || elementType->isFunctionType())
5013 elementSize = CharUnits::One();
5014 else
5015 elementSize = CGF.getContext().getTypeSizeInChars(T: elementType);
5016
5017 // Don't even emit the divide for element size of 1.
5018 if (elementSize.isOne())
5019 return diffInChars;
5020
5021 divisor = CGF.CGM.getSize(numChars: elementSize);
5022 }
5023
5024 // Otherwise, do a full sdiv. This uses the "exact" form of sdiv, since
5025 // pointer difference in C is only defined in the case where both operands
5026 // are pointing to elements of an array.
5027 return Builder.CreateExactSDiv(LHS: diffInChars, RHS: divisor, Name: "sub.ptr.div");
5028}
5029
5030Value *ScalarExprEmitter::GetMaximumShiftAmount(Value *LHS, Value *RHS,
5031 bool RHSIsSigned) {
5032 llvm::IntegerType *Ty;
5033 if (llvm::VectorType *VT = dyn_cast<llvm::VectorType>(Val: LHS->getType()))
5034 Ty = cast<llvm::IntegerType>(Val: VT->getElementType());
5035 else
5036 Ty = cast<llvm::IntegerType>(Val: LHS->getType());
5037 // For a given type of LHS the maximum shift amount is width(LHS)-1, however
5038 // it can occur that width(LHS)-1 > range(RHS). Since there is no check for
5039 // this in ConstantInt::get, this results in the value getting truncated.
5040 // Constrain the return value to be max(RHS) in this case.
5041 llvm::Type *RHSTy = RHS->getType();
5042 llvm::APInt RHSMax =
5043 RHSIsSigned ? llvm::APInt::getSignedMaxValue(numBits: RHSTy->getScalarSizeInBits())
5044 : llvm::APInt::getMaxValue(numBits: RHSTy->getScalarSizeInBits());
5045 if (RHSMax.ult(RHS: Ty->getBitWidth()))
5046 return llvm::ConstantInt::get(Ty: RHSTy, V: RHSMax);
5047 return llvm::ConstantInt::get(Ty: RHSTy, V: Ty->getBitWidth() - 1);
5048}
5049
5050Value *ScalarExprEmitter::ConstrainShiftValue(Value *LHS, Value *RHS,
5051 const Twine &Name) {
5052 llvm::IntegerType *Ty;
5053 if (auto *VT = dyn_cast<llvm::VectorType>(Val: LHS->getType()))
5054 Ty = cast<llvm::IntegerType>(Val: VT->getElementType());
5055 else
5056 Ty = cast<llvm::IntegerType>(Val: LHS->getType());
5057
5058 if (llvm::isPowerOf2_64(Value: Ty->getBitWidth()))
5059 return Builder.CreateAnd(LHS: RHS, RHS: GetMaximumShiftAmount(LHS, RHS, RHSIsSigned: false), Name);
5060
5061 return Builder.CreateURem(
5062 LHS: RHS, RHS: llvm::ConstantInt::get(Ty: RHS->getType(), V: Ty->getBitWidth()), Name);
5063}
5064
5065Value *ScalarExprEmitter::EmitShl(const BinOpInfo &Ops) {
5066 // TODO: This misses out on the sanitizer check below.
5067 if (Ops.isFixedPointOp())
5068 return EmitFixedPointBinOp(op: Ops);
5069
5070 // LLVM requires the LHS and RHS to be the same type: promote or truncate the
5071 // RHS to the same size as the LHS.
5072 Value *RHS = Ops.RHS;
5073 if (Ops.LHS->getType() != RHS->getType())
5074 RHS = Builder.CreateIntCast(V: RHS, DestTy: Ops.LHS->getType(), isSigned: false, Name: "sh_prom");
5075
5076 bool SanitizeSignedBase = CGF.SanOpts.has(K: SanitizerKind::ShiftBase) &&
5077 Ops.Ty->hasSignedIntegerRepresentation() &&
5078 !CGF.getLangOpts().isSignedOverflowDefined() &&
5079 !CGF.getLangOpts().CPlusPlus20;
5080 bool SanitizeUnsignedBase =
5081 CGF.SanOpts.has(K: SanitizerKind::UnsignedShiftBase) &&
5082 Ops.Ty->hasUnsignedIntegerRepresentation();
5083 bool SanitizeBase = SanitizeSignedBase || SanitizeUnsignedBase;
5084 bool SanitizeExponent = CGF.SanOpts.has(K: SanitizerKind::ShiftExponent);
5085 // OpenCL 6.3j: shift values are effectively % word size of LHS.
5086 if (CGF.getLangOpts().OpenCL || CGF.getLangOpts().HLSL)
5087 RHS = ConstrainShiftValue(LHS: Ops.LHS, RHS, Name: "shl.mask");
5088 else if ((SanitizeBase || SanitizeExponent) &&
5089 isa<llvm::IntegerType>(Val: Ops.LHS->getType())) {
5090 SmallVector<SanitizerKind::SanitizerOrdinal, 3> Ordinals;
5091 if (SanitizeSignedBase)
5092 Ordinals.push_back(Elt: SanitizerKind::SO_ShiftBase);
5093 if (SanitizeUnsignedBase)
5094 Ordinals.push_back(Elt: SanitizerKind::SO_UnsignedShiftBase);
5095 if (SanitizeExponent)
5096 Ordinals.push_back(Elt: SanitizerKind::SO_ShiftExponent);
5097
5098 SanitizerDebugLocation SanScope(&CGF, Ordinals,
5099 SanitizerHandler::ShiftOutOfBounds);
5100 SmallVector<std::pair<Value *, SanitizerKind::SanitizerOrdinal>, 2> Checks;
5101 bool RHSIsSigned = Ops.rhsHasSignedIntegerRepresentation();
5102 llvm::Value *WidthMinusOne =
5103 GetMaximumShiftAmount(LHS: Ops.LHS, RHS: Ops.RHS, RHSIsSigned);
5104 llvm::Value *ValidExponent = Builder.CreateICmpULE(LHS: Ops.RHS, RHS: WidthMinusOne);
5105
5106 if (SanitizeExponent) {
5107 Checks.push_back(
5108 Elt: std::make_pair(x&: ValidExponent, y: SanitizerKind::SO_ShiftExponent));
5109 }
5110
5111 if (SanitizeBase) {
5112 // Check whether we are shifting any non-zero bits off the top of the
5113 // integer. We only emit this check if exponent is valid - otherwise
5114 // instructions below will have undefined behavior themselves.
5115 llvm::BasicBlock *Orig = Builder.GetInsertBlock();
5116 llvm::BasicBlock *Cont = CGF.createBasicBlock(name: "cont");
5117 llvm::BasicBlock *CheckShiftBase = CGF.createBasicBlock(name: "check");
5118 Builder.CreateCondBr(Cond: ValidExponent, True: CheckShiftBase, False: Cont);
5119 llvm::Value *PromotedWidthMinusOne =
5120 (RHS == Ops.RHS) ? WidthMinusOne
5121 : GetMaximumShiftAmount(LHS: Ops.LHS, RHS, RHSIsSigned);
5122 CGF.EmitBlock(BB: CheckShiftBase);
5123 llvm::Value *BitsShiftedOff = Builder.CreateLShr(
5124 LHS: Ops.LHS, RHS: Builder.CreateSub(LHS: PromotedWidthMinusOne, RHS, Name: "shl.zeros",
5125 /*NUW*/ HasNUW: true, /*NSW*/ HasNSW: true),
5126 Name: "shl.check");
5127 if (SanitizeUnsignedBase || CGF.getLangOpts().CPlusPlus) {
5128 // In C99, we are not permitted to shift a 1 bit into the sign bit.
5129 // Under C++11's rules, shifting a 1 bit into the sign bit is
5130 // OK, but shifting a 1 bit out of it is not. (C89 and C++03 don't
5131 // define signed left shifts, so we use the C99 and C++11 rules there).
5132 // Unsigned shifts can always shift into the top bit.
5133 llvm::Value *One = llvm::ConstantInt::get(Ty: BitsShiftedOff->getType(), V: 1);
5134 BitsShiftedOff = Builder.CreateLShr(LHS: BitsShiftedOff, RHS: One);
5135 }
5136 llvm::Value *Zero = llvm::ConstantInt::get(Ty: BitsShiftedOff->getType(), V: 0);
5137 llvm::Value *ValidBase = Builder.CreateICmpEQ(LHS: BitsShiftedOff, RHS: Zero);
5138 CGF.EmitBlock(BB: Cont);
5139 llvm::PHINode *BaseCheck = Builder.CreatePHI(Ty: ValidBase->getType(), NumReservedValues: 2);
5140 BaseCheck->addIncoming(V: Builder.getTrue(), BB: Orig);
5141 BaseCheck->addIncoming(V: ValidBase, BB: CheckShiftBase);
5142 Checks.push_back(Elt: std::make_pair(
5143 x&: BaseCheck, y: SanitizeSignedBase ? SanitizerKind::SO_ShiftBase
5144 : SanitizerKind::SO_UnsignedShiftBase));
5145 }
5146
5147 assert(!Checks.empty());
5148 EmitBinOpCheck(Checks, Info: Ops);
5149 }
5150
5151 return Builder.CreateShl(LHS: Ops.LHS, RHS, Name: "shl");
5152}
5153
5154Value *ScalarExprEmitter::EmitShr(const BinOpInfo &Ops) {
5155 // TODO: This misses out on the sanitizer check below.
5156 if (Ops.isFixedPointOp())
5157 return EmitFixedPointBinOp(op: Ops);
5158
5159 // LLVM requires the LHS and RHS to be the same type: promote or truncate the
5160 // RHS to the same size as the LHS.
5161 Value *RHS = Ops.RHS;
5162 if (Ops.LHS->getType() != RHS->getType())
5163 RHS = Builder.CreateIntCast(V: RHS, DestTy: Ops.LHS->getType(), isSigned: false, Name: "sh_prom");
5164
5165 // OpenCL 6.3j: shift values are effectively % word size of LHS.
5166 if (CGF.getLangOpts().OpenCL || CGF.getLangOpts().HLSL)
5167 RHS = ConstrainShiftValue(LHS: Ops.LHS, RHS, Name: "shr.mask");
5168 else if (CGF.SanOpts.has(K: SanitizerKind::ShiftExponent) &&
5169 isa<llvm::IntegerType>(Val: Ops.LHS->getType())) {
5170 SanitizerDebugLocation SanScope(&CGF, {SanitizerKind::SO_ShiftExponent},
5171 SanitizerHandler::ShiftOutOfBounds);
5172 bool RHSIsSigned = Ops.rhsHasSignedIntegerRepresentation();
5173 llvm::Value *Valid = Builder.CreateICmpULE(
5174 LHS: Ops.RHS, RHS: GetMaximumShiftAmount(LHS: Ops.LHS, RHS: Ops.RHS, RHSIsSigned));
5175 EmitBinOpCheck(Checks: std::make_pair(x&: Valid, y: SanitizerKind::SO_ShiftExponent), Info: Ops);
5176 }
5177
5178 if (Ops.Ty->hasUnsignedIntegerRepresentation())
5179 return Builder.CreateLShr(LHS: Ops.LHS, RHS, Name: "shr");
5180 return Builder.CreateAShr(LHS: Ops.LHS, RHS, Name: "shr");
5181}
5182
5183enum IntrinsicType { VCMPEQ, VCMPGT };
5184// return corresponding comparison intrinsic for given vector type
5185static llvm::Intrinsic::ID GetIntrinsic(IntrinsicType IT,
5186 BuiltinType::Kind ElemKind) {
5187 switch (ElemKind) {
5188 default: llvm_unreachable("unexpected element type");
5189 case BuiltinType::Char_U:
5190 case BuiltinType::UChar:
5191 return (IT == VCMPEQ) ? llvm::Intrinsic::ppc_altivec_vcmpequb_p :
5192 llvm::Intrinsic::ppc_altivec_vcmpgtub_p;
5193 case BuiltinType::Char_S:
5194 case BuiltinType::SChar:
5195 return (IT == VCMPEQ) ? llvm::Intrinsic::ppc_altivec_vcmpequb_p :
5196 llvm::Intrinsic::ppc_altivec_vcmpgtsb_p;
5197 case BuiltinType::UShort:
5198 return (IT == VCMPEQ) ? llvm::Intrinsic::ppc_altivec_vcmpequh_p :
5199 llvm::Intrinsic::ppc_altivec_vcmpgtuh_p;
5200 case BuiltinType::Short:
5201 return (IT == VCMPEQ) ? llvm::Intrinsic::ppc_altivec_vcmpequh_p :
5202 llvm::Intrinsic::ppc_altivec_vcmpgtsh_p;
5203 case BuiltinType::UInt:
5204 return (IT == VCMPEQ) ? llvm::Intrinsic::ppc_altivec_vcmpequw_p :
5205 llvm::Intrinsic::ppc_altivec_vcmpgtuw_p;
5206 case BuiltinType::Int:
5207 return (IT == VCMPEQ) ? llvm::Intrinsic::ppc_altivec_vcmpequw_p :
5208 llvm::Intrinsic::ppc_altivec_vcmpgtsw_p;
5209 case BuiltinType::ULong:
5210 case BuiltinType::ULongLong:
5211 return (IT == VCMPEQ) ? llvm::Intrinsic::ppc_altivec_vcmpequd_p :
5212 llvm::Intrinsic::ppc_altivec_vcmpgtud_p;
5213 case BuiltinType::Long:
5214 case BuiltinType::LongLong:
5215 return (IT == VCMPEQ) ? llvm::Intrinsic::ppc_altivec_vcmpequd_p :
5216 llvm::Intrinsic::ppc_altivec_vcmpgtsd_p;
5217 case BuiltinType::Float:
5218 return (IT == VCMPEQ) ? llvm::Intrinsic::ppc_altivec_vcmpeqfp_p :
5219 llvm::Intrinsic::ppc_altivec_vcmpgtfp_p;
5220 case BuiltinType::Double:
5221 return (IT == VCMPEQ) ? llvm::Intrinsic::ppc_vsx_xvcmpeqdp_p :
5222 llvm::Intrinsic::ppc_vsx_xvcmpgtdp_p;
5223 case BuiltinType::UInt128:
5224 return (IT == VCMPEQ) ? llvm::Intrinsic::ppc_altivec_vcmpequq_p
5225 : llvm::Intrinsic::ppc_altivec_vcmpgtuq_p;
5226 case BuiltinType::Int128:
5227 return (IT == VCMPEQ) ? llvm::Intrinsic::ppc_altivec_vcmpequq_p
5228 : llvm::Intrinsic::ppc_altivec_vcmpgtsq_p;
5229 }
5230}
5231
5232Value *ScalarExprEmitter::EmitCompare(const BinaryOperator *E,
5233 llvm::CmpInst::Predicate UICmpOpc,
5234 llvm::CmpInst::Predicate SICmpOpc,
5235 llvm::CmpInst::Predicate FCmpOpc,
5236 bool IsSignaling) {
5237 TestAndClearIgnoreResultAssign();
5238 Value *Result;
5239 QualType LHSTy = E->getLHS()->getType();
5240 QualType RHSTy = E->getRHS()->getType();
5241 if (const MemberPointerType *MPT = LHSTy->getAs<MemberPointerType>()) {
5242 assert(E->getOpcode() == BO_EQ ||
5243 E->getOpcode() == BO_NE);
5244 Value *LHS = CGF.EmitScalarExpr(E: E->getLHS());
5245 Value *RHS = CGF.EmitScalarExpr(E: E->getRHS());
5246 Result = CGF.CGM.getCXXABI().EmitMemberPointerComparison(
5247 CGF, L: LHS, R: RHS, MPT, Inequality: E->getOpcode() == BO_NE);
5248 } else if (!LHSTy->isAnyComplexType() && !RHSTy->isAnyComplexType()) {
5249 BinOpInfo BOInfo = EmitBinOps(E);
5250 Value *LHS = BOInfo.LHS;
5251 Value *RHS = BOInfo.RHS;
5252
5253 // If AltiVec, the comparison results in a numeric type, so we use
5254 // intrinsics comparing vectors and giving 0 or 1 as a result
5255 if (LHSTy->isVectorType() && !E->getType()->isVectorType()) {
5256 // constants for mapping CR6 register bits to predicate result
5257 enum { CR6_EQ=0, CR6_EQ_REV, CR6_LT, CR6_LT_REV } CR6;
5258
5259 llvm::Intrinsic::ID ID = llvm::Intrinsic::not_intrinsic;
5260
5261 // in several cases vector arguments order will be reversed
5262 Value *FirstVecArg = LHS,
5263 *SecondVecArg = RHS;
5264
5265 QualType ElTy = LHSTy->castAs<VectorType>()->getElementType();
5266 BuiltinType::Kind ElementKind = ElTy->castAs<BuiltinType>()->getKind();
5267
5268 switch(E->getOpcode()) {
5269 default: llvm_unreachable("is not a comparison operation");
5270 case BO_EQ:
5271 CR6 = CR6_LT;
5272 ID = GetIntrinsic(IT: VCMPEQ, ElemKind: ElementKind);
5273 break;
5274 case BO_NE:
5275 CR6 = CR6_EQ;
5276 ID = GetIntrinsic(IT: VCMPEQ, ElemKind: ElementKind);
5277 break;
5278 case BO_LT:
5279 CR6 = CR6_LT;
5280 ID = GetIntrinsic(IT: VCMPGT, ElemKind: ElementKind);
5281 std::swap(a&: FirstVecArg, b&: SecondVecArg);
5282 break;
5283 case BO_GT:
5284 CR6 = CR6_LT;
5285 ID = GetIntrinsic(IT: VCMPGT, ElemKind: ElementKind);
5286 break;
5287 case BO_LE:
5288 if (ElementKind == BuiltinType::Float) {
5289 CR6 = CR6_LT;
5290 ID = llvm::Intrinsic::ppc_altivec_vcmpgefp_p;
5291 std::swap(a&: FirstVecArg, b&: SecondVecArg);
5292 }
5293 else {
5294 CR6 = CR6_EQ;
5295 ID = GetIntrinsic(IT: VCMPGT, ElemKind: ElementKind);
5296 }
5297 break;
5298 case BO_GE:
5299 if (ElementKind == BuiltinType::Float) {
5300 CR6 = CR6_LT;
5301 ID = llvm::Intrinsic::ppc_altivec_vcmpgefp_p;
5302 }
5303 else {
5304 CR6 = CR6_EQ;
5305 ID = GetIntrinsic(IT: VCMPGT, ElemKind: ElementKind);
5306 std::swap(a&: FirstVecArg, b&: SecondVecArg);
5307 }
5308 break;
5309 }
5310
5311 Value *CR6Param = Builder.getInt32(C: CR6);
5312 llvm::Function *F = CGF.CGM.getIntrinsic(IID: ID);
5313 Result = Builder.CreateCall(Callee: F, Args: {CR6Param, FirstVecArg, SecondVecArg});
5314
5315 // The result type of intrinsic may not be same as E->getType().
5316 // If E->getType() is not BoolTy, EmitScalarConversion will do the
5317 // conversion work. If E->getType() is BoolTy, EmitScalarConversion will
5318 // do nothing, if ResultTy is not i1 at the same time, it will cause
5319 // crash later.
5320 llvm::IntegerType *ResultTy = cast<llvm::IntegerType>(Val: Result->getType());
5321 if (ResultTy->getBitWidth() > 1 &&
5322 E->getType() == CGF.getContext().BoolTy)
5323 Result = Builder.CreateTrunc(V: Result, DestTy: Builder.getInt1Ty());
5324 return EmitScalarConversion(Src: Result, SrcType: CGF.getContext().BoolTy, DstType: E->getType(),
5325 Loc: E->getExprLoc());
5326 }
5327
5328 if (BOInfo.isFixedPointOp()) {
5329 Result = EmitFixedPointBinOp(op: BOInfo);
5330 } else if (LHS->getType()->isFPOrFPVectorTy()) {
5331 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, BOInfo.FPFeatures);
5332 if (!IsSignaling)
5333 Result = Builder.CreateFCmp(P: FCmpOpc, LHS, RHS, Name: "cmp");
5334 else
5335 Result = Builder.CreateFCmpS(P: FCmpOpc, LHS, RHS, Name: "cmp");
5336 } else if (LHSTy->hasSignedIntegerRepresentation()) {
5337 Result = Builder.CreateICmp(P: SICmpOpc, LHS, RHS, Name: "cmp");
5338 } else {
5339 // Unsigned integers and pointers.
5340
5341 if (CGF.CGM.getCodeGenOpts().StrictVTablePointers &&
5342 !isa<llvm::ConstantPointerNull>(Val: LHS) &&
5343 !isa<llvm::ConstantPointerNull>(Val: RHS)) {
5344
5345 // Dynamic information is required to be stripped for comparisons,
5346 // because it could leak the dynamic information. Based on comparisons
5347 // of pointers to dynamic objects, the optimizer can replace one pointer
5348 // with another, which might be incorrect in presence of invariant
5349 // groups. Comparison with null is safe because null does not carry any
5350 // dynamic information.
5351 if (LHSTy.mayBeDynamicClass())
5352 LHS = Builder.CreateStripInvariantGroup(Ptr: LHS);
5353 if (RHSTy.mayBeDynamicClass())
5354 RHS = Builder.CreateStripInvariantGroup(Ptr: RHS);
5355 }
5356
5357 Result = Builder.CreateICmp(P: UICmpOpc, LHS, RHS, Name: "cmp");
5358 }
5359
5360 // If this is a vector comparison, sign extend the result to the appropriate
5361 // vector integer type and return it (don't convert to bool).
5362 if (LHSTy->isVectorType() || LHSTy->isSveVLSBuiltinType())
5363 return Builder.CreateSExt(V: Result, DestTy: ConvertType(T: E->getType()), Name: "sext");
5364
5365 } else {
5366 // Complex Comparison: can only be an equality comparison.
5367 CodeGenFunction::ComplexPairTy LHS, RHS;
5368 QualType CETy;
5369 if (auto *CTy = LHSTy->getAs<ComplexType>()) {
5370 LHS = CGF.EmitComplexExpr(E: E->getLHS());
5371 CETy = CTy->getElementType();
5372 } else {
5373 LHS.first = Visit(E: E->getLHS());
5374 LHS.second = llvm::Constant::getNullValue(Ty: LHS.first->getType());
5375 CETy = LHSTy;
5376 }
5377 if (auto *CTy = RHSTy->getAs<ComplexType>()) {
5378 RHS = CGF.EmitComplexExpr(E: E->getRHS());
5379 assert(CGF.getContext().hasSameUnqualifiedType(CETy,
5380 CTy->getElementType()) &&
5381 "The element types must always match.");
5382 (void)CTy;
5383 } else {
5384 RHS.first = Visit(E: E->getRHS());
5385 RHS.second = llvm::Constant::getNullValue(Ty: RHS.first->getType());
5386 assert(CGF.getContext().hasSameUnqualifiedType(CETy, RHSTy) &&
5387 "The element types must always match.");
5388 }
5389
5390 Value *ResultR, *ResultI;
5391 if (CETy->isRealFloatingType()) {
5392 // As complex comparisons can only be equality comparisons, they
5393 // are never signaling comparisons.
5394 ResultR = Builder.CreateFCmp(P: FCmpOpc, LHS: LHS.first, RHS: RHS.first, Name: "cmp.r");
5395 ResultI = Builder.CreateFCmp(P: FCmpOpc, LHS: LHS.second, RHS: RHS.second, Name: "cmp.i");
5396 } else {
5397 // Complex comparisons can only be equality comparisons. As such, signed
5398 // and unsigned opcodes are the same.
5399 ResultR = Builder.CreateICmp(P: UICmpOpc, LHS: LHS.first, RHS: RHS.first, Name: "cmp.r");
5400 ResultI = Builder.CreateICmp(P: UICmpOpc, LHS: LHS.second, RHS: RHS.second, Name: "cmp.i");
5401 }
5402
5403 if (E->getOpcode() == BO_EQ) {
5404 Result = Builder.CreateAnd(LHS: ResultR, RHS: ResultI, Name: "and.ri");
5405 } else {
5406 assert(E->getOpcode() == BO_NE &&
5407 "Complex comparison other than == or != ?");
5408 Result = Builder.CreateOr(LHS: ResultR, RHS: ResultI, Name: "or.ri");
5409 }
5410 }
5411
5412 return EmitScalarConversion(Src: Result, SrcType: CGF.getContext().BoolTy, DstType: E->getType(),
5413 Loc: E->getExprLoc());
5414}
5415
5416llvm::Value *CodeGenFunction::EmitWithOriginalRHSBitfieldAssignment(
5417 const BinaryOperator *E, Value **Previous, QualType *SrcType) {
5418 // In case we have the integer or bitfield sanitizer checks enabled
5419 // we want to get the expression before scalar conversion.
5420 if (auto *ICE = dyn_cast<ImplicitCastExpr>(Val: E->getRHS())) {
5421 CastKind Kind = ICE->getCastKind();
5422 if (Kind == CK_IntegralCast || Kind == CK_LValueToRValue) {
5423 *SrcType = ICE->getSubExpr()->getType();
5424 *Previous = EmitScalarExpr(E: ICE->getSubExpr());
5425 // Pass default ScalarConversionOpts to avoid emitting
5426 // integer sanitizer checks as E refers to bitfield.
5427 return EmitScalarConversion(Src: *Previous, SrcTy: *SrcType, DstTy: ICE->getType(),
5428 Loc: ICE->getExprLoc());
5429 }
5430 }
5431 return EmitScalarExpr(E: E->getRHS());
5432}
5433
5434Value *ScalarExprEmitter::VisitBinAssign(const BinaryOperator *E) {
5435 ApplyAtomGroup Grp(CGF.getDebugInfo());
5436 bool Ignore = TestAndClearIgnoreResultAssign();
5437
5438 Value *RHS;
5439 LValue LHS;
5440
5441 if (PointerAuthQualifier PtrAuth = E->getLHS()->getType().getPointerAuth()) {
5442 LValue LV = CGF.EmitCheckedLValue(E: E->getLHS(), TCK: CodeGenFunction::TCK_Store);
5443 LV.getQuals().removePointerAuth();
5444 llvm::Value *RV =
5445 CGF.EmitPointerAuthQualify(Qualifier: PtrAuth, PointerExpr: E->getRHS(), StorageAddress: LV.getAddress());
5446 CGF.EmitNullabilityCheck(LHS: LV, RHS: RV, Loc: E->getExprLoc());
5447 CGF.EmitStoreThroughLValue(Src: RValue::get(V: RV), Dst: LV);
5448
5449 if (Ignore)
5450 return nullptr;
5451 RV = CGF.EmitPointerAuthUnqualify(Qualifier: PtrAuth, Pointer: RV, PointerType: LV.getType(),
5452 StorageAddress: LV.getAddress(), /*nonnull*/ IsKnownNonNull: false);
5453 return RV;
5454 }
5455
5456 switch (E->getLHS()->getType().getObjCLifetime()) {
5457 case Qualifiers::OCL_Strong:
5458 std::tie(args&: LHS, args&: RHS) = CGF.EmitARCStoreStrong(e: E, ignored: Ignore);
5459 break;
5460
5461 case Qualifiers::OCL_Autoreleasing:
5462 std::tie(args&: LHS, args&: RHS) = CGF.EmitARCStoreAutoreleasing(e: E);
5463 break;
5464
5465 case Qualifiers::OCL_ExplicitNone:
5466 std::tie(args&: LHS, args&: RHS) = CGF.EmitARCStoreUnsafeUnretained(e: E, ignored: Ignore);
5467 break;
5468
5469 case Qualifiers::OCL_Weak:
5470 RHS = Visit(E: E->getRHS());
5471 LHS = EmitCheckedLValue(E: E->getLHS(), TCK: CodeGenFunction::TCK_Store);
5472 RHS = CGF.EmitARCStoreWeak(addr: LHS.getAddress(), value: RHS, ignored: Ignore);
5473 break;
5474
5475 case Qualifiers::OCL_None:
5476 // __block variables need to have the rhs evaluated first, plus
5477 // this should improve codegen just a little.
5478 Value *Previous = nullptr;
5479 QualType SrcType = E->getRHS()->getType();
5480 // Check if LHS is a bitfield, if RHS contains an implicit cast expression
5481 // we want to extract that value and potentially (if the bitfield sanitizer
5482 // is enabled) use it to check for an implicit conversion.
5483 if (E->getLHS()->refersToBitField())
5484 RHS = CGF.EmitWithOriginalRHSBitfieldAssignment(E, Previous: &Previous, SrcType: &SrcType);
5485 else
5486 RHS = Visit(E: E->getRHS());
5487
5488 LHS = EmitCheckedLValue(E: E->getLHS(), TCK: CodeGenFunction::TCK_Store);
5489
5490 // Store the value into the LHS. Bit-fields are handled specially
5491 // because the result is altered by the store, i.e., [C99 6.5.16p1]
5492 // 'An assignment expression has the value of the left operand after
5493 // the assignment...'.
5494 if (LHS.isBitField()) {
5495 CGF.EmitStoreThroughBitfieldLValue(Src: RValue::get(V: RHS), Dst: LHS, Result: &RHS);
5496 // If the expression contained an implicit conversion, make sure
5497 // to use the value before the scalar conversion.
5498 Value *Src = Previous ? Previous : RHS;
5499 QualType DstType = E->getLHS()->getType();
5500 CGF.EmitBitfieldConversionCheck(Src, SrcType, Dst: RHS, DstType,
5501 Info: LHS.getBitFieldInfo(), Loc: E->getExprLoc());
5502 } else {
5503 CGF.EmitNullabilityCheck(LHS, RHS, Loc: E->getExprLoc());
5504 CGF.EmitStoreThroughLValue(Src: RValue::get(V: RHS), Dst: LHS);
5505 }
5506 }
5507 // OpenMP: Handle lastprivate(condition:) in scalar assignment
5508 if (CGF.getLangOpts().OpenMP) {
5509 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF,
5510 LHS: E->getLHS());
5511 }
5512
5513 // If the result is clearly ignored, return now.
5514 if (Ignore)
5515 return nullptr;
5516
5517 // The result of an assignment in C is the assigned r-value.
5518 if (!CGF.getLangOpts().CPlusPlus)
5519 return RHS;
5520
5521 // If the lvalue is non-volatile, return the computed value of the assignment.
5522 if (!LHS.isVolatileQualified())
5523 return RHS;
5524
5525 // Otherwise, reload the value.
5526 return EmitLoadOfLValue(LV: LHS, Loc: E->getExprLoc());
5527}
5528
5529Value *ScalarExprEmitter::VisitBinLAnd(const BinaryOperator *E) {
5530 auto HasLHSSkip = CGF.hasSkipCounter(S: E);
5531 auto HasRHSSkip = CGF.hasSkipCounter(S: E->getRHS());
5532
5533 // Perform vector logical and on comparisons with zero vectors.
5534 if (E->getType()->isVectorType()) {
5535 CGF.incrementProfileCounter(S: E);
5536
5537 Value *LHS = Visit(E: E->getLHS());
5538 Value *RHS = Visit(E: E->getRHS());
5539 Value *Zero = llvm::ConstantAggregateZero::get(Ty: LHS->getType());
5540 if (LHS->getType()->isFPOrFPVectorTy()) {
5541 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(
5542 CGF, E->getFPFeaturesInEffect(LO: CGF.getLangOpts()));
5543 LHS = Builder.CreateFCmp(P: llvm::CmpInst::FCMP_UNE, LHS, RHS: Zero, Name: "cmp");
5544 RHS = Builder.CreateFCmp(P: llvm::CmpInst::FCMP_UNE, LHS: RHS, RHS: Zero, Name: "cmp");
5545 } else {
5546 LHS = Builder.CreateICmp(P: llvm::CmpInst::ICMP_NE, LHS, RHS: Zero, Name: "cmp");
5547 RHS = Builder.CreateICmp(P: llvm::CmpInst::ICMP_NE, LHS: RHS, RHS: Zero, Name: "cmp");
5548 }
5549 Value *And = Builder.CreateAnd(LHS, RHS);
5550 return Builder.CreateSExt(V: And, DestTy: ConvertType(T: E->getType()), Name: "sext");
5551 }
5552
5553 bool InstrumentRegions = CGF.CGM.getCodeGenOpts().hasProfileClangInstr();
5554 llvm::Type *ResTy = ConvertType(T: E->getType());
5555
5556 // If we have 0 && RHS, see if we can elide RHS, if so, just return 0.
5557 // If we have 1 && X, just emit X without inserting the control flow.
5558 bool LHSCondVal;
5559 if (CGF.ConstantFoldsToSimpleInteger(Cond: E->getLHS(), Result&: LHSCondVal)) {
5560 if (LHSCondVal) { // If we have 1 && X, just emit X.
5561 CGF.incrementProfileCounter(ExecSkip: CGF.UseExecPath, S: E, /*UseBoth=*/true);
5562
5563 // If the top of the logical operator nest, reset the MCDC temp to 0.
5564 if (CGF.isMCDCDecisionExpr(E))
5565 CGF.maybeResetMCDCCondBitmap(E);
5566
5567 Value *RHSCond = CGF.EvaluateExprAsBool(E: E->getRHS());
5568
5569 // If we're generating for profiling or coverage, generate a branch to a
5570 // block that increments the RHS counter needed to track branch condition
5571 // coverage. In this case, use "FBlock" as both the final "TrueBlock" and
5572 // "FalseBlock" after the increment is done.
5573 if (InstrumentRegions &&
5574 CodeGenFunction::isInstrumentedCondition(C: E->getRHS())) {
5575 CGF.maybeUpdateMCDCCondBitmap(E: E->getRHS(), Val: RHSCond);
5576 llvm::BasicBlock *FBlock = CGF.createBasicBlock(name: "land.end");
5577 llvm::BasicBlock *RHSSkip =
5578 (HasRHSSkip ? CGF.createBasicBlock(name: "land.rhsskip") : FBlock);
5579 llvm::BasicBlock *RHSBlockCnt = CGF.createBasicBlock(name: "land.rhscnt");
5580 Builder.CreateCondBr(Cond: RHSCond, True: RHSBlockCnt, False: RHSSkip);
5581 CGF.EmitBlock(BB: RHSBlockCnt);
5582 CGF.incrementProfileCounter(ExecSkip: CGF.UseExecPath, S: E->getRHS());
5583 CGF.EmitBranch(Block: FBlock);
5584 if (HasRHSSkip) {
5585 CGF.EmitBlock(BB: RHSSkip);
5586 CGF.incrementProfileCounter(ExecSkip: CGF.UseSkipPath, S: E->getRHS());
5587 }
5588 CGF.EmitBlock(BB: FBlock);
5589 } else
5590 CGF.markStmtMaybeUsed(S: E->getRHS());
5591
5592 // If the top of the logical operator nest, update the MCDC bitmap.
5593 if (CGF.isMCDCDecisionExpr(E))
5594 CGF.maybeUpdateMCDCTestVectorBitmap(E);
5595
5596 // ZExt result to int or bool.
5597 return Builder.CreateZExtOrBitCast(V: RHSCond, DestTy: ResTy, Name: "land.ext");
5598 }
5599
5600 // 0 && RHS: If it is safe, just elide the RHS, and return 0/false.
5601 if (!CGF.ContainsLabel(S: E->getRHS())) {
5602 CGF.markStmtAsUsed(Skipped: false, S: E);
5603 if (HasLHSSkip)
5604 CGF.incrementProfileCounter(ExecSkip: CGF.UseSkipPath, S: E);
5605
5606 CGF.markStmtMaybeUsed(S: E->getRHS());
5607
5608 return llvm::Constant::getNullValue(Ty: ResTy);
5609 }
5610 }
5611
5612 // If the top of the logical operator nest, reset the MCDC temp to 0.
5613 if (CGF.isMCDCDecisionExpr(E))
5614 CGF.maybeResetMCDCCondBitmap(E);
5615
5616 llvm::BasicBlock *ContBlock = CGF.createBasicBlock(name: "land.end");
5617 llvm::BasicBlock *RHSBlock = CGF.createBasicBlock(name: "land.rhs");
5618
5619 llvm::BasicBlock *LHSFalseBlock =
5620 (HasLHSSkip ? CGF.createBasicBlock(name: "land.lhsskip") : ContBlock);
5621
5622 CodeGenFunction::ConditionalEvaluation eval(CGF);
5623
5624 // Branch on the LHS first. If it is false, go to the failure (cont) block.
5625 CGF.EmitBranchOnBoolExpr(Cond: E->getLHS(), TrueBlock: RHSBlock, FalseBlock: LHSFalseBlock,
5626 TrueCount: CGF.getProfileCount(S: E->getRHS()));
5627
5628 if (HasLHSSkip) {
5629 CGF.EmitBlock(BB: LHSFalseBlock);
5630 CGF.incrementProfileCounter(ExecSkip: CGF.UseSkipPath, S: E);
5631 CGF.EmitBranch(Block: ContBlock);
5632 }
5633
5634 // Any edges into the ContBlock are now from an (indeterminate number of)
5635 // edges from this first condition. All of these values will be false. Start
5636 // setting up the PHI node in the Cont Block for this.
5637 llvm::PHINode *PN = llvm::PHINode::Create(Ty: llvm::Type::getInt1Ty(C&: VMContext), NumReservedValues: 2,
5638 NameStr: "", InsertBefore: ContBlock);
5639 for (llvm::pred_iterator PI = pred_begin(BB: ContBlock), PE = pred_end(BB: ContBlock);
5640 PI != PE; ++PI)
5641 PN->addIncoming(V: llvm::ConstantInt::getFalse(Context&: VMContext), BB: *PI);
5642
5643 eval.begin(CGF);
5644 CGF.EmitBlock(BB: RHSBlock);
5645 CGF.incrementProfileCounter(ExecSkip: CGF.UseExecPath, S: E);
5646 Value *RHSCond = CGF.EvaluateExprAsBool(E: E->getRHS());
5647 eval.end(CGF);
5648
5649 // Reaquire the RHS block, as there may be subblocks inserted.
5650 RHSBlock = Builder.GetInsertBlock();
5651
5652 // If we're generating for profiling or coverage, generate a branch on the
5653 // RHS to a block that increments the RHS true counter needed to track branch
5654 // condition coverage.
5655 llvm::BasicBlock *ContIncoming = RHSBlock;
5656 if (InstrumentRegions &&
5657 CodeGenFunction::isInstrumentedCondition(C: E->getRHS())) {
5658 CGF.maybeUpdateMCDCCondBitmap(E: E->getRHS(), Val: RHSCond);
5659 llvm::BasicBlock *RHSBlockCnt = CGF.createBasicBlock(name: "land.rhscnt");
5660 llvm::BasicBlock *RHSBlockSkip =
5661 (HasRHSSkip ? CGF.createBasicBlock(name: "land.rhsskip") : ContBlock);
5662 Builder.CreateCondBr(Cond: RHSCond, True: RHSBlockCnt, False: RHSBlockSkip);
5663 CGF.EmitBlock(BB: RHSBlockCnt);
5664 CGF.incrementProfileCounter(ExecSkip: CGF.UseExecPath, S: E->getRHS());
5665 CGF.EmitBranch(Block: ContBlock);
5666 PN->addIncoming(V: RHSCond, BB: RHSBlockCnt);
5667 if (HasRHSSkip) {
5668 CGF.EmitBlock(BB: RHSBlockSkip);
5669 CGF.incrementProfileCounter(ExecSkip: CGF.UseSkipPath, S: E->getRHS());
5670 CGF.EmitBranch(Block: ContBlock);
5671 ContIncoming = RHSBlockSkip;
5672 }
5673 }
5674
5675 // Emit an unconditional branch from this block to ContBlock.
5676 {
5677 // There is no need to emit line number for unconditional branch.
5678 auto NL = ApplyDebugLocation::CreateEmpty(CGF);
5679 CGF.EmitBlock(BB: ContBlock);
5680 }
5681 // Insert an entry into the phi node for the edge with the value of RHSCond.
5682 PN->addIncoming(V: RHSCond, BB: ContIncoming);
5683
5684 // If the top of the logical operator nest, update the MCDC bitmap.
5685 if (CGF.isMCDCDecisionExpr(E))
5686 CGF.maybeUpdateMCDCTestVectorBitmap(E);
5687
5688 // Artificial location to preserve the scope information
5689 {
5690 auto NL = ApplyDebugLocation::CreateArtificial(CGF);
5691 PN->setDebugLoc(Builder.getCurrentDebugLocation());
5692 }
5693
5694 // ZExt result to int.
5695 return Builder.CreateZExtOrBitCast(V: PN, DestTy: ResTy, Name: "land.ext");
5696}
5697
5698Value *ScalarExprEmitter::VisitBinLOr(const BinaryOperator *E) {
5699 auto HasLHSSkip = CGF.hasSkipCounter(S: E);
5700 auto HasRHSSkip = CGF.hasSkipCounter(S: E->getRHS());
5701
5702 // Perform vector logical or on comparisons with zero vectors.
5703 if (E->getType()->isVectorType()) {
5704 CGF.incrementProfileCounter(S: E);
5705
5706 Value *LHS = Visit(E: E->getLHS());
5707 Value *RHS = Visit(E: E->getRHS());
5708 Value *Zero = llvm::ConstantAggregateZero::get(Ty: LHS->getType());
5709 if (LHS->getType()->isFPOrFPVectorTy()) {
5710 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(
5711 CGF, E->getFPFeaturesInEffect(LO: CGF.getLangOpts()));
5712 LHS = Builder.CreateFCmp(P: llvm::CmpInst::FCMP_UNE, LHS, RHS: Zero, Name: "cmp");
5713 RHS = Builder.CreateFCmp(P: llvm::CmpInst::FCMP_UNE, LHS: RHS, RHS: Zero, Name: "cmp");
5714 } else {
5715 LHS = Builder.CreateICmp(P: llvm::CmpInst::ICMP_NE, LHS, RHS: Zero, Name: "cmp");
5716 RHS = Builder.CreateICmp(P: llvm::CmpInst::ICMP_NE, LHS: RHS, RHS: Zero, Name: "cmp");
5717 }
5718 Value *Or = Builder.CreateOr(LHS, RHS);
5719 return Builder.CreateSExt(V: Or, DestTy: ConvertType(T: E->getType()), Name: "sext");
5720 }
5721
5722 bool InstrumentRegions = CGF.CGM.getCodeGenOpts().hasProfileClangInstr();
5723 llvm::Type *ResTy = ConvertType(T: E->getType());
5724
5725 // If we have 1 || RHS, see if we can elide RHS, if so, just return 1.
5726 // If we have 0 || X, just emit X without inserting the control flow.
5727 bool LHSCondVal;
5728 if (CGF.ConstantFoldsToSimpleInteger(Cond: E->getLHS(), Result&: LHSCondVal)) {
5729 if (!LHSCondVal) { // If we have 0 || X, just emit X.
5730 CGF.incrementProfileCounter(ExecSkip: CGF.UseExecPath, S: E, /*UseBoth=*/true);
5731
5732 // If the top of the logical operator nest, reset the MCDC temp to 0.
5733 if (CGF.isMCDCDecisionExpr(E))
5734 CGF.maybeResetMCDCCondBitmap(E);
5735
5736 Value *RHSCond = CGF.EvaluateExprAsBool(E: E->getRHS());
5737
5738 // If we're generating for profiling or coverage, generate a branch to a
5739 // block that increments the RHS counter need to track branch condition
5740 // coverage. In this case, use "FBlock" as both the final "TrueBlock" and
5741 // "FalseBlock" after the increment is done.
5742 if (InstrumentRegions &&
5743 CodeGenFunction::isInstrumentedCondition(C: E->getRHS())) {
5744 CGF.maybeUpdateMCDCCondBitmap(E: E->getRHS(), Val: RHSCond);
5745 llvm::BasicBlock *FBlock = CGF.createBasicBlock(name: "lor.end");
5746 llvm::BasicBlock *RHSSkip =
5747 (HasRHSSkip ? CGF.createBasicBlock(name: "lor.rhsskip") : FBlock);
5748 llvm::BasicBlock *RHSBlockCnt = CGF.createBasicBlock(name: "lor.rhscnt");
5749 Builder.CreateCondBr(Cond: RHSCond, True: RHSSkip, False: RHSBlockCnt);
5750 CGF.EmitBlock(BB: RHSBlockCnt);
5751 CGF.incrementProfileCounter(ExecSkip: CGF.UseExecPath, S: E->getRHS());
5752 CGF.EmitBranch(Block: FBlock);
5753 if (HasRHSSkip) {
5754 CGF.EmitBlock(BB: RHSSkip);
5755 CGF.incrementProfileCounter(ExecSkip: CGF.UseSkipPath, S: E->getRHS());
5756 }
5757 CGF.EmitBlock(BB: FBlock);
5758 } else
5759 CGF.markStmtMaybeUsed(S: E->getRHS());
5760
5761 // If the top of the logical operator nest, update the MCDC bitmap.
5762 if (CGF.isMCDCDecisionExpr(E))
5763 CGF.maybeUpdateMCDCTestVectorBitmap(E);
5764
5765 // ZExt result to int or bool.
5766 return Builder.CreateZExtOrBitCast(V: RHSCond, DestTy: ResTy, Name: "lor.ext");
5767 }
5768
5769 // 1 || RHS: If it is safe, just elide the RHS, and return 1/true.
5770 if (!CGF.ContainsLabel(S: E->getRHS())) {
5771 CGF.markStmtAsUsed(Skipped: false, S: E);
5772 if (HasLHSSkip)
5773 CGF.incrementProfileCounter(ExecSkip: CGF.UseSkipPath, S: E);
5774
5775 CGF.markStmtMaybeUsed(S: E->getRHS());
5776
5777 return llvm::ConstantInt::get(Ty: ResTy, V: 1);
5778 }
5779 }
5780
5781 // If the top of the logical operator nest, reset the MCDC temp to 0.
5782 if (CGF.isMCDCDecisionExpr(E))
5783 CGF.maybeResetMCDCCondBitmap(E);
5784
5785 llvm::BasicBlock *ContBlock = CGF.createBasicBlock(name: "lor.end");
5786 llvm::BasicBlock *RHSBlock = CGF.createBasicBlock(name: "lor.rhs");
5787 llvm::BasicBlock *LHSTrueBlock =
5788 (HasLHSSkip ? CGF.createBasicBlock(name: "lor.lhsskip") : ContBlock);
5789
5790 CodeGenFunction::ConditionalEvaluation eval(CGF);
5791
5792 // Branch on the LHS first. If it is true, go to the success (cont) block.
5793 CGF.EmitBranchOnBoolExpr(Cond: E->getLHS(), TrueBlock: LHSTrueBlock, FalseBlock: RHSBlock,
5794 TrueCount: CGF.getCurrentProfileCount() -
5795 CGF.getProfileCount(S: E->getRHS()));
5796
5797 if (HasLHSSkip) {
5798 CGF.EmitBlock(BB: LHSTrueBlock);
5799 CGF.incrementProfileCounter(ExecSkip: CGF.UseSkipPath, S: E);
5800 CGF.EmitBranch(Block: ContBlock);
5801 }
5802
5803 // Any edges into the ContBlock are now from an (indeterminate number of)
5804 // edges from this first condition. All of these values will be true. Start
5805 // setting up the PHI node in the Cont Block for this.
5806 llvm::PHINode *PN = llvm::PHINode::Create(Ty: llvm::Type::getInt1Ty(C&: VMContext), NumReservedValues: 2,
5807 NameStr: "", InsertBefore: ContBlock);
5808 for (llvm::pred_iterator PI = pred_begin(BB: ContBlock), PE = pred_end(BB: ContBlock);
5809 PI != PE; ++PI)
5810 PN->addIncoming(V: llvm::ConstantInt::getTrue(Context&: VMContext), BB: *PI);
5811
5812 eval.begin(CGF);
5813
5814 // Emit the RHS condition as a bool value.
5815 CGF.EmitBlock(BB: RHSBlock);
5816 CGF.incrementProfileCounter(ExecSkip: CGF.UseExecPath, S: E);
5817 Value *RHSCond = CGF.EvaluateExprAsBool(E: E->getRHS());
5818
5819 eval.end(CGF);
5820
5821 // Reaquire the RHS block, as there may be subblocks inserted.
5822 RHSBlock = Builder.GetInsertBlock();
5823
5824 // If we're generating for profiling or coverage, generate a branch on the
5825 // RHS to a block that increments the RHS true counter needed to track branch
5826 // condition coverage.
5827 llvm::BasicBlock *ContIncoming = RHSBlock;
5828 if (InstrumentRegions &&
5829 CodeGenFunction::isInstrumentedCondition(C: E->getRHS())) {
5830 CGF.maybeUpdateMCDCCondBitmap(E: E->getRHS(), Val: RHSCond);
5831 llvm::BasicBlock *RHSBlockCnt = CGF.createBasicBlock(name: "lor.rhscnt");
5832 llvm::BasicBlock *RHSTrueBlock =
5833 (HasRHSSkip ? CGF.createBasicBlock(name: "lor.rhsskip") : ContBlock);
5834 Builder.CreateCondBr(Cond: RHSCond, True: RHSTrueBlock, False: RHSBlockCnt);
5835 CGF.EmitBlock(BB: RHSBlockCnt);
5836 CGF.incrementProfileCounter(ExecSkip: CGF.UseExecPath, S: E->getRHS());
5837 CGF.EmitBranch(Block: ContBlock);
5838 PN->addIncoming(V: RHSCond, BB: RHSBlockCnt);
5839 if (HasRHSSkip) {
5840 CGF.EmitBlock(BB: RHSTrueBlock);
5841 CGF.incrementProfileCounter(ExecSkip: CGF.UseSkipPath, S: E->getRHS());
5842 CGF.EmitBranch(Block: ContBlock);
5843 ContIncoming = RHSTrueBlock;
5844 }
5845 }
5846
5847 // Emit an unconditional branch from this block to ContBlock. Insert an entry
5848 // into the phi node for the edge with the value of RHSCond.
5849 CGF.EmitBlock(BB: ContBlock);
5850 PN->addIncoming(V: RHSCond, BB: ContIncoming);
5851
5852 // If the top of the logical operator nest, update the MCDC bitmap.
5853 if (CGF.isMCDCDecisionExpr(E))
5854 CGF.maybeUpdateMCDCTestVectorBitmap(E);
5855
5856 // ZExt result to int.
5857 return Builder.CreateZExtOrBitCast(V: PN, DestTy: ResTy, Name: "lor.ext");
5858}
5859
5860Value *ScalarExprEmitter::VisitBinComma(const BinaryOperator *E) {
5861 CGF.EmitIgnoredExpr(E: E->getLHS());
5862 CGF.EnsureInsertPoint();
5863 return Visit(E: E->getRHS());
5864}
5865
5866//===----------------------------------------------------------------------===//
5867// Other Operators
5868//===----------------------------------------------------------------------===//
5869
5870/// isCheapEnoughToEvaluateUnconditionally - Return true if the specified
5871/// expression is cheap enough and side-effect-free enough to evaluate
5872/// unconditionally instead of conditionally. This is used to convert control
5873/// flow into selects in some cases.
5874static bool isCheapEnoughToEvaluateUnconditionally(const Expr *E,
5875 CodeGenFunction &CGF) {
5876 // Anything that is an integer or floating point constant is fine.
5877 return E->IgnoreParens()->isEvaluatable(Ctx: CGF.getContext());
5878
5879 // Even non-volatile automatic variables can't be evaluated unconditionally.
5880 // Referencing a thread_local may cause non-trivial initialization work to
5881 // occur. If we're inside a lambda and one of the variables is from the scope
5882 // outside the lambda, that function may have returned already. Reading its
5883 // locals is a bad idea. Also, these reads may introduce races there didn't
5884 // exist in the source-level program.
5885}
5886
5887
5888Value *ScalarExprEmitter::
5889VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) {
5890 TestAndClearIgnoreResultAssign();
5891
5892 // Bind the common expression if necessary.
5893 CodeGenFunction::OpaqueValueMapping binding(CGF, E);
5894
5895 Expr *condExpr = E->getCond();
5896 Expr *lhsExpr = E->getTrueExpr();
5897 Expr *rhsExpr = E->getFalseExpr();
5898
5899 // If the condition constant folds and can be elided, try to avoid emitting
5900 // the condition and the dead arm.
5901 bool CondExprBool;
5902 if (CGF.ConstantFoldsToSimpleInteger(Cond: condExpr, Result&: CondExprBool)) {
5903 Expr *live = lhsExpr, *dead = rhsExpr;
5904 if (!CondExprBool) std::swap(a&: live, b&: dead);
5905
5906 // If the dead side doesn't have labels we need, just emit the Live part.
5907 if (!CGF.ContainsLabel(S: dead)) {
5908 CGF.incrementProfileCounter(ExecSkip: CondExprBool ? CGF.UseExecPath
5909 : CGF.UseSkipPath,
5910 S: E, /*UseBoth=*/true);
5911 Value *Result = Visit(E: live);
5912 CGF.markStmtMaybeUsed(S: dead);
5913
5914 // If the live part is a throw expression, it acts like it has a void
5915 // type, so evaluating it returns a null Value*. However, a conditional
5916 // with non-void type must return a non-null Value*.
5917 if (!Result && !E->getType()->isVoidType())
5918 Result = llvm::UndefValue::get(T: CGF.ConvertType(T: E->getType()));
5919
5920 return Result;
5921 }
5922 }
5923
5924 // OpenCL: If the condition is a vector, we can treat this condition like
5925 // the select function.
5926 if (CGF.getLangOpts().OpenCL && (condExpr->getType()->isVectorType() ||
5927 condExpr->getType()->isExtVectorType())) {
5928 CGF.incrementProfileCounter(S: E);
5929
5930 llvm::Value *CondV = CGF.EmitScalarExpr(E: condExpr);
5931 llvm::Value *LHS = Visit(E: lhsExpr);
5932 llvm::Value *RHS = Visit(E: rhsExpr);
5933
5934 llvm::Type *condType = ConvertType(T: condExpr->getType());
5935 auto *vecTy = cast<llvm::FixedVectorType>(Val: condType);
5936
5937 unsigned numElem = vecTy->getNumElements();
5938 llvm::Type *elemType = vecTy->getElementType();
5939
5940 llvm::Value *zeroVec = llvm::Constant::getNullValue(Ty: vecTy);
5941 llvm::Value *TestMSB = Builder.CreateICmpSLT(LHS: CondV, RHS: zeroVec);
5942 llvm::Value *tmp = Builder.CreateSExt(
5943 V: TestMSB, DestTy: llvm::FixedVectorType::get(ElementType: elemType, NumElts: numElem), Name: "sext");
5944 llvm::Value *tmp2 = Builder.CreateNot(V: tmp);
5945
5946 // Cast float to int to perform ANDs if necessary.
5947 llvm::Value *RHSTmp = RHS;
5948 llvm::Value *LHSTmp = LHS;
5949 bool wasCast = false;
5950 llvm::VectorType *rhsVTy = cast<llvm::VectorType>(Val: RHS->getType());
5951 if (rhsVTy->getElementType()->isFloatingPointTy()) {
5952 RHSTmp = Builder.CreateBitCast(V: RHS, DestTy: tmp2->getType());
5953 LHSTmp = Builder.CreateBitCast(V: LHS, DestTy: tmp->getType());
5954 wasCast = true;
5955 }
5956
5957 llvm::Value *tmp3 = Builder.CreateAnd(LHS: RHSTmp, RHS: tmp2);
5958 llvm::Value *tmp4 = Builder.CreateAnd(LHS: LHSTmp, RHS: tmp);
5959 llvm::Value *tmp5 = Builder.CreateOr(LHS: tmp3, RHS: tmp4, Name: "cond");
5960 if (wasCast)
5961 tmp5 = Builder.CreateBitCast(V: tmp5, DestTy: RHS->getType());
5962
5963 return tmp5;
5964 }
5965
5966 if (condExpr->getType()->isVectorType() ||
5967 condExpr->getType()->isSveVLSBuiltinType()) {
5968 CGF.incrementProfileCounter(S: E);
5969
5970 llvm::Value *CondV = CGF.EmitScalarExpr(E: condExpr);
5971 llvm::Value *LHS = Visit(E: lhsExpr);
5972 llvm::Value *RHS = Visit(E: rhsExpr);
5973
5974 llvm::Type *CondType = ConvertType(T: condExpr->getType());
5975 auto *VecTy = cast<llvm::VectorType>(Val: CondType);
5976
5977 if (VecTy->getElementType()->isIntegerTy(BitWidth: 1))
5978 return Builder.CreateSelect(C: CondV, True: LHS, False: RHS, Name: "vector_select");
5979
5980 // OpenCL uses the MSB of the mask vector.
5981 llvm::Value *ZeroVec = llvm::Constant::getNullValue(Ty: VecTy);
5982 if (condExpr->getType()->isExtVectorType())
5983 CondV = Builder.CreateICmpSLT(LHS: CondV, RHS: ZeroVec, Name: "vector_cond");
5984 else
5985 CondV = Builder.CreateICmpNE(LHS: CondV, RHS: ZeroVec, Name: "vector_cond");
5986 return Builder.CreateSelect(C: CondV, True: LHS, False: RHS, Name: "vector_select");
5987 }
5988
5989 // If this is a really simple expression (like x ? 4 : 5), emit this as a
5990 // select instead of as control flow. We can only do this if it is cheap and
5991 // safe to evaluate the LHS and RHS unconditionally.
5992 if (!llvm::EnableSingleByteCoverage &&
5993 isCheapEnoughToEvaluateUnconditionally(E: lhsExpr, CGF) &&
5994 isCheapEnoughToEvaluateUnconditionally(E: rhsExpr, CGF)) {
5995 llvm::Value *CondV = CGF.EvaluateExprAsBool(E: condExpr);
5996 llvm::Value *StepV = Builder.CreateZExtOrBitCast(V: CondV, DestTy: CGF.Int64Ty);
5997
5998 CGF.incrementProfileCounter(S: E, StepV);
5999
6000 llvm::Value *LHS = Visit(E: lhsExpr);
6001 llvm::Value *RHS = Visit(E: rhsExpr);
6002 if (!LHS) {
6003 // If the conditional has void type, make sure we return a null Value*.
6004 assert(!RHS && "LHS and RHS types must match");
6005 return nullptr;
6006 }
6007 return Builder.CreateSelect(C: CondV, True: LHS, False: RHS, Name: "cond");
6008 }
6009
6010 // If the top of the logical operator nest, reset the MCDC temp to 0.
6011 if (auto E = CGF.stripCond(C: condExpr); CGF.isMCDCDecisionExpr(E))
6012 CGF.maybeResetMCDCCondBitmap(E);
6013
6014 llvm::BasicBlock *LHSBlock = CGF.createBasicBlock(name: "cond.true");
6015 llvm::BasicBlock *RHSBlock = CGF.createBasicBlock(name: "cond.false");
6016 llvm::BasicBlock *ContBlock = CGF.createBasicBlock(name: "cond.end");
6017
6018 CodeGenFunction::ConditionalEvaluation eval(CGF);
6019 CGF.EmitBranchOnBoolExpr(Cond: condExpr, TrueBlock: LHSBlock, FalseBlock: RHSBlock,
6020 TrueCount: CGF.getProfileCount(S: lhsExpr));
6021
6022 CGF.EmitBlock(BB: LHSBlock);
6023
6024 // If the top of the logical operator nest, update the MCDC bitmap for the
6025 // ConditionalOperator prior to visiting its LHS and RHS blocks, since they
6026 // may also contain a boolean expression.
6027 if (auto E = CGF.stripCond(C: condExpr); CGF.isMCDCDecisionExpr(E))
6028 CGF.maybeUpdateMCDCTestVectorBitmap(E);
6029
6030 CGF.incrementProfileCounter(ExecSkip: CGF.UseExecPath, S: E);
6031 eval.begin(CGF);
6032 Value *LHS = Visit(E: lhsExpr);
6033 eval.end(CGF);
6034
6035 LHSBlock = Builder.GetInsertBlock();
6036 Builder.CreateBr(Dest: ContBlock);
6037
6038 CGF.EmitBlock(BB: RHSBlock);
6039
6040 // If the top of the logical operator nest, update the MCDC bitmap for the
6041 // ConditionalOperator prior to visiting its LHS and RHS blocks, since they
6042 // may also contain a boolean expression.
6043 if (auto E = CGF.stripCond(C: condExpr); CGF.isMCDCDecisionExpr(E))
6044 CGF.maybeUpdateMCDCTestVectorBitmap(E);
6045
6046 CGF.incrementProfileCounter(ExecSkip: CGF.UseSkipPath, S: E);
6047 eval.begin(CGF);
6048 Value *RHS = Visit(E: rhsExpr);
6049 eval.end(CGF);
6050
6051 RHSBlock = Builder.GetInsertBlock();
6052 CGF.EmitBlock(BB: ContBlock);
6053
6054 // If the LHS or RHS is a throw expression, it will be legitimately null.
6055 if (!LHS)
6056 return RHS;
6057 if (!RHS)
6058 return LHS;
6059
6060 // Create a PHI node for the real part.
6061 llvm::PHINode *PN = Builder.CreatePHI(Ty: LHS->getType(), NumReservedValues: 2, Name: "cond");
6062 PN->addIncoming(V: LHS, BB: LHSBlock);
6063 PN->addIncoming(V: RHS, BB: RHSBlock);
6064
6065 return PN;
6066}
6067
6068Value *ScalarExprEmitter::VisitChooseExpr(ChooseExpr *E) {
6069 return Visit(E: E->getChosenSubExpr());
6070}
6071
6072Value *ScalarExprEmitter::VisitVAArgExpr(VAArgExpr *VE) {
6073 Address ArgValue = Address::invalid();
6074 RValue ArgPtr = CGF.EmitVAArg(VE, VAListAddr&: ArgValue);
6075
6076 return ArgPtr.getScalarVal();
6077}
6078
6079Value *ScalarExprEmitter::VisitBlockExpr(const BlockExpr *block) {
6080 return CGF.EmitBlockLiteral(block);
6081}
6082
6083// Convert a vec3 to vec4, or vice versa.
6084static Value *ConvertVec3AndVec4(CGBuilderTy &Builder, CodeGenFunction &CGF,
6085 Value *Src, unsigned NumElementsDst) {
6086 static constexpr int Mask[] = {0, 1, 2, -1};
6087 return Builder.CreateShuffleVector(V: Src, Mask: llvm::ArrayRef(Mask, NumElementsDst));
6088}
6089
6090// Create cast instructions for converting LLVM value \p Src to LLVM type \p
6091// DstTy. \p Src has the same size as \p DstTy. Both are single value types
6092// but could be scalar or vectors of different lengths, and either can be
6093// pointer.
6094// There are 4 cases:
6095// 1. non-pointer -> non-pointer : needs 1 bitcast
6096// 2. pointer -> pointer : needs 1 bitcast or addrspacecast
6097// 3. pointer -> non-pointer
6098// a) pointer -> intptr_t : needs 1 ptrtoint
6099// b) pointer -> non-intptr_t : needs 1 ptrtoint then 1 bitcast
6100// 4. non-pointer -> pointer
6101// a) intptr_t -> pointer : needs 1 inttoptr
6102// b) non-intptr_t -> pointer : needs 1 bitcast then 1 inttoptr
6103// Note: for cases 3b and 4b two casts are required since LLVM casts do not
6104// allow casting directly between pointer types and non-integer non-pointer
6105// types.
6106static Value *createCastsForTypeOfSameSize(CGBuilderTy &Builder,
6107 const llvm::DataLayout &DL,
6108 Value *Src, llvm::Type *DstTy,
6109 StringRef Name = "") {
6110 auto SrcTy = Src->getType();
6111
6112 // Case 1.
6113 if (!SrcTy->isPointerTy() && !DstTy->isPointerTy())
6114 return Builder.CreateBitCast(V: Src, DestTy: DstTy, Name);
6115
6116 // Case 2.
6117 if (SrcTy->isPointerTy() && DstTy->isPointerTy())
6118 return Builder.CreatePointerBitCastOrAddrSpaceCast(V: Src, DestTy: DstTy, Name);
6119
6120 // Case 3.
6121 if (SrcTy->isPointerTy() && !DstTy->isPointerTy()) {
6122 // Case 3b.
6123 if (!DstTy->isIntegerTy())
6124 Src = Builder.CreatePtrToInt(V: Src, DestTy: DL.getIntPtrType(SrcTy));
6125 // Cases 3a and 3b.
6126 return Builder.CreateBitOrPointerCast(V: Src, DestTy: DstTy, Name);
6127 }
6128
6129 // Case 4b.
6130 if (!SrcTy->isIntegerTy())
6131 Src = Builder.CreateBitCast(V: Src, DestTy: DL.getIntPtrType(DstTy));
6132 // Cases 4a and 4b.
6133 return Builder.CreateIntToPtr(V: Src, DestTy: DstTy, Name);
6134}
6135
6136Value *ScalarExprEmitter::VisitAsTypeExpr(AsTypeExpr *E) {
6137 Value *Src = CGF.EmitScalarExpr(E: E->getSrcExpr());
6138 llvm::Type *DstTy = ConvertType(T: E->getType());
6139
6140 llvm::Type *SrcTy = Src->getType();
6141 unsigned NumElementsSrc =
6142 isa<llvm::VectorType>(Val: SrcTy)
6143 ? cast<llvm::FixedVectorType>(Val: SrcTy)->getNumElements()
6144 : 0;
6145 unsigned NumElementsDst =
6146 isa<llvm::VectorType>(Val: DstTy)
6147 ? cast<llvm::FixedVectorType>(Val: DstTy)->getNumElements()
6148 : 0;
6149
6150 // Use bit vector expansion for ext_vector_type boolean vectors.
6151 if (E->getType()->isExtVectorBoolType())
6152 return CGF.emitBoolVecConversion(SrcVec: Src, NumElementsDst, Name: "astype");
6153
6154 // Going from vec3 to non-vec3 is a special case and requires a shuffle
6155 // vector to get a vec4, then a bitcast if the target type is different.
6156 if (NumElementsSrc == 3 && NumElementsDst != 3) {
6157 Src = ConvertVec3AndVec4(Builder, CGF, Src, NumElementsDst: 4);
6158 Src = createCastsForTypeOfSameSize(Builder, DL: CGF.CGM.getDataLayout(), Src,
6159 DstTy);
6160
6161 Src->setName("astype");
6162 return Src;
6163 }
6164
6165 // Going from non-vec3 to vec3 is a special case and requires a bitcast
6166 // to vec4 if the original type is not vec4, then a shuffle vector to
6167 // get a vec3.
6168 if (NumElementsSrc != 3 && NumElementsDst == 3) {
6169 auto *Vec4Ty = llvm::FixedVectorType::get(
6170 ElementType: cast<llvm::VectorType>(Val: DstTy)->getElementType(), NumElts: 4);
6171 Src = createCastsForTypeOfSameSize(Builder, DL: CGF.CGM.getDataLayout(), Src,
6172 DstTy: Vec4Ty);
6173
6174 Src = ConvertVec3AndVec4(Builder, CGF, Src, NumElementsDst: 3);
6175 Src->setName("astype");
6176 return Src;
6177 }
6178
6179 return createCastsForTypeOfSameSize(Builder, DL: CGF.CGM.getDataLayout(),
6180 Src, DstTy, Name: "astype");
6181}
6182
6183Value *ScalarExprEmitter::VisitAtomicExpr(AtomicExpr *E) {
6184 return CGF.EmitAtomicExpr(E).getScalarVal();
6185}
6186
6187//===----------------------------------------------------------------------===//
6188// Entry Point into this File
6189//===----------------------------------------------------------------------===//
6190
6191/// Emit the computation of the specified expression of scalar type, ignoring
6192/// the result.
6193Value *CodeGenFunction::EmitScalarExpr(const Expr *E, bool IgnoreResultAssign) {
6194 assert(E && hasScalarEvaluationKind(E->getType()) &&
6195 "Invalid scalar expression to emit");
6196
6197 return ScalarExprEmitter(*this, IgnoreResultAssign)
6198 .Visit(E: const_cast<Expr *>(E));
6199}
6200
6201/// Emit a conversion from the specified type to the specified destination type,
6202/// both of which are LLVM scalar types.
6203Value *CodeGenFunction::EmitScalarConversion(Value *Src, QualType SrcTy,
6204 QualType DstTy,
6205 SourceLocation Loc) {
6206 assert(hasScalarEvaluationKind(SrcTy) && hasScalarEvaluationKind(DstTy) &&
6207 "Invalid scalar expression to emit");
6208 return ScalarExprEmitter(*this).EmitScalarConversion(Src, SrcType: SrcTy, DstType: DstTy, Loc);
6209}
6210
6211/// Emit a conversion from the specified complex type to the specified
6212/// destination type, where the destination type is an LLVM scalar type.
6213Value *CodeGenFunction::EmitComplexToScalarConversion(ComplexPairTy Src,
6214 QualType SrcTy,
6215 QualType DstTy,
6216 SourceLocation Loc) {
6217 assert(SrcTy->isAnyComplexType() && hasScalarEvaluationKind(DstTy) &&
6218 "Invalid complex -> scalar conversion");
6219 return ScalarExprEmitter(*this)
6220 .EmitComplexToScalarConversion(Src, SrcTy, DstTy, Loc);
6221}
6222
6223
6224Value *
6225CodeGenFunction::EmitPromotedScalarExpr(const Expr *E,
6226 QualType PromotionType) {
6227 if (!PromotionType.isNull())
6228 return ScalarExprEmitter(*this).EmitPromoted(E, PromotionType);
6229 else
6230 return ScalarExprEmitter(*this).Visit(E: const_cast<Expr *>(E));
6231}
6232
6233
6234llvm::Value *CodeGenFunction::
6235EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV,
6236 bool isInc, bool isPre) {
6237 return ScalarExprEmitter(*this).EmitScalarPrePostIncDec(E, LV, isInc, isPre);
6238}
6239
6240LValue CodeGenFunction::EmitObjCIsaExpr(const ObjCIsaExpr *E) {
6241 // object->isa or (*object).isa
6242 // Generate code as for: *(Class*)object
6243
6244 Expr *BaseExpr = E->getBase();
6245 Address Addr = Address::invalid();
6246 if (BaseExpr->isPRValue()) {
6247 llvm::Type *BaseTy =
6248 ConvertTypeForMem(T: BaseExpr->getType()->getPointeeType());
6249 Addr = Address(EmitScalarExpr(E: BaseExpr), BaseTy, getPointerAlign());
6250 } else {
6251 Addr = EmitLValue(E: BaseExpr).getAddress();
6252 }
6253
6254 // Cast the address to Class*.
6255 Addr = Addr.withElementType(ElemTy: ConvertType(T: E->getType()));
6256 return MakeAddrLValue(Addr, T: E->getType());
6257}
6258
6259
6260LValue CodeGenFunction::EmitCompoundAssignmentLValue(
6261 const CompoundAssignOperator *E) {
6262 ApplyAtomGroup Grp(getDebugInfo());
6263 ScalarExprEmitter Scalar(*this);
6264 Value *Result = nullptr;
6265 switch (E->getOpcode()) {
6266#define COMPOUND_OP(Op) \
6267 case BO_##Op##Assign: \
6268 return Scalar.EmitCompoundAssignLValue(E, &ScalarExprEmitter::Emit##Op, \
6269 Result)
6270 COMPOUND_OP(Mul);
6271 COMPOUND_OP(Div);
6272 COMPOUND_OP(Rem);
6273 COMPOUND_OP(Add);
6274 COMPOUND_OP(Sub);
6275 COMPOUND_OP(Shl);
6276 COMPOUND_OP(Shr);
6277 COMPOUND_OP(And);
6278 COMPOUND_OP(Xor);
6279 COMPOUND_OP(Or);
6280#undef COMPOUND_OP
6281
6282 case BO_PtrMemD:
6283 case BO_PtrMemI:
6284 case BO_Mul:
6285 case BO_Div:
6286 case BO_Rem:
6287 case BO_Add:
6288 case BO_Sub:
6289 case BO_Shl:
6290 case BO_Shr:
6291 case BO_LT:
6292 case BO_GT:
6293 case BO_LE:
6294 case BO_GE:
6295 case BO_EQ:
6296 case BO_NE:
6297 case BO_Cmp:
6298 case BO_And:
6299 case BO_Xor:
6300 case BO_Or:
6301 case BO_LAnd:
6302 case BO_LOr:
6303 case BO_Assign:
6304 case BO_Comma:
6305 llvm_unreachable("Not valid compound assignment operators");
6306 }
6307
6308 llvm_unreachable("Unhandled compound assignment operator");
6309}
6310
6311struct GEPOffsetAndOverflow {
6312 // The total (signed) byte offset for the GEP.
6313 llvm::Value *TotalOffset;
6314 // The offset overflow flag - true if the total offset overflows.
6315 llvm::Value *OffsetOverflows;
6316};
6317
6318/// Evaluate given GEPVal, which is either an inbounds GEP, or a constant,
6319/// and compute the total offset it applies from it's base pointer BasePtr.
6320/// Returns offset in bytes and a boolean flag whether an overflow happened
6321/// during evaluation.
6322static GEPOffsetAndOverflow EmitGEPOffsetInBytes(Value *BasePtr, Value *GEPVal,
6323 llvm::LLVMContext &VMContext,
6324 CodeGenModule &CGM,
6325 CGBuilderTy &Builder) {
6326 const auto &DL = CGM.getDataLayout();
6327
6328 // The total (signed) byte offset for the GEP.
6329 llvm::Value *TotalOffset = nullptr;
6330
6331 // Was the GEP already reduced to a constant?
6332 if (isa<llvm::Constant>(Val: GEPVal)) {
6333 // Compute the offset by casting both pointers to integers and subtracting:
6334 // GEPVal = BasePtr + ptr(Offset) <--> Offset = int(GEPVal) - int(BasePtr)
6335 Value *BasePtr_int =
6336 Builder.CreatePtrToInt(V: BasePtr, DestTy: DL.getIntPtrType(BasePtr->getType()));
6337 Value *GEPVal_int =
6338 Builder.CreatePtrToInt(V: GEPVal, DestTy: DL.getIntPtrType(GEPVal->getType()));
6339 TotalOffset = Builder.CreateSub(LHS: GEPVal_int, RHS: BasePtr_int);
6340 return {.TotalOffset: TotalOffset, /*OffsetOverflows=*/Builder.getFalse()};
6341 }
6342
6343 auto *GEP = cast<llvm::GEPOperator>(Val: GEPVal);
6344 assert(GEP->getPointerOperand() == BasePtr &&
6345 "BasePtr must be the base of the GEP.");
6346 assert(GEP->isInBounds() && "Expected inbounds GEP");
6347
6348 auto *IntPtrTy = DL.getIntPtrType(GEP->getPointerOperandType());
6349
6350 // Grab references to the signed add/mul overflow intrinsics for intptr_t.
6351 auto *Zero = llvm::ConstantInt::getNullValue(Ty: IntPtrTy);
6352 auto *SAddIntrinsic =
6353 CGM.getIntrinsic(IID: llvm::Intrinsic::sadd_with_overflow, Tys: IntPtrTy);
6354 auto *SMulIntrinsic =
6355 CGM.getIntrinsic(IID: llvm::Intrinsic::smul_with_overflow, Tys: IntPtrTy);
6356
6357 // The offset overflow flag - true if the total offset overflows.
6358 llvm::Value *OffsetOverflows = Builder.getFalse();
6359
6360 /// Return the result of the given binary operation.
6361 auto eval = [&](BinaryOperator::Opcode Opcode, llvm::Value *LHS,
6362 llvm::Value *RHS) -> llvm::Value * {
6363 assert((Opcode == BO_Add || Opcode == BO_Mul) && "Can't eval binop");
6364
6365 // If the operands are constants, return a constant result.
6366 if (auto *LHSCI = dyn_cast<llvm::ConstantInt>(Val: LHS)) {
6367 if (auto *RHSCI = dyn_cast<llvm::ConstantInt>(Val: RHS)) {
6368 llvm::APInt N;
6369 bool HasOverflow = mayHaveIntegerOverflow(LHS: LHSCI, RHS: RHSCI, Opcode,
6370 /*Signed=*/true, Result&: N);
6371 if (HasOverflow)
6372 OffsetOverflows = Builder.getTrue();
6373 return llvm::ConstantInt::get(Context&: VMContext, V: N);
6374 }
6375 }
6376
6377 // Otherwise, compute the result with checked arithmetic.
6378 auto *ResultAndOverflow = Builder.CreateCall(
6379 Callee: (Opcode == BO_Add) ? SAddIntrinsic : SMulIntrinsic, Args: {LHS, RHS});
6380 OffsetOverflows = Builder.CreateOr(
6381 LHS: Builder.CreateExtractValue(Agg: ResultAndOverflow, Idxs: 1), RHS: OffsetOverflows);
6382 return Builder.CreateExtractValue(Agg: ResultAndOverflow, Idxs: 0);
6383 };
6384
6385 // Determine the total byte offset by looking at each GEP operand.
6386 for (auto GTI = llvm::gep_type_begin(GEP), GTE = llvm::gep_type_end(GEP);
6387 GTI != GTE; ++GTI) {
6388 llvm::Value *LocalOffset;
6389 auto *Index = GTI.getOperand();
6390 // Compute the local offset contributed by this indexing step:
6391 if (auto *STy = GTI.getStructTypeOrNull()) {
6392 // For struct indexing, the local offset is the byte position of the
6393 // specified field.
6394 unsigned FieldNo = cast<llvm::ConstantInt>(Val: Index)->getZExtValue();
6395 LocalOffset = llvm::ConstantInt::get(
6396 Ty: IntPtrTy, V: DL.getStructLayout(Ty: STy)->getElementOffset(Idx: FieldNo));
6397 } else {
6398 // Otherwise this is array-like indexing. The local offset is the index
6399 // multiplied by the element size.
6400 auto *ElementSize =
6401 llvm::ConstantInt::get(Ty: IntPtrTy, V: GTI.getSequentialElementStride(DL));
6402 auto *IndexS = Builder.CreateIntCast(V: Index, DestTy: IntPtrTy, /*isSigned=*/true);
6403 LocalOffset = eval(BO_Mul, ElementSize, IndexS);
6404 }
6405
6406 // If this is the first offset, set it as the total offset. Otherwise, add
6407 // the local offset into the running total.
6408 if (!TotalOffset || TotalOffset == Zero)
6409 TotalOffset = LocalOffset;
6410 else
6411 TotalOffset = eval(BO_Add, TotalOffset, LocalOffset);
6412 }
6413
6414 return {.TotalOffset: TotalOffset, .OffsetOverflows: OffsetOverflows};
6415}
6416
6417Value *
6418CodeGenFunction::EmitCheckedInBoundsGEP(llvm::Type *ElemTy, Value *Ptr,
6419 ArrayRef<Value *> IdxList,
6420 bool SignedIndices, bool IsSubtraction,
6421 SourceLocation Loc, const Twine &Name) {
6422 llvm::Type *PtrTy = Ptr->getType();
6423
6424 llvm::GEPNoWrapFlags NWFlags = llvm::GEPNoWrapFlags::inBounds();
6425 if (!SignedIndices && !IsSubtraction)
6426 NWFlags |= llvm::GEPNoWrapFlags::noUnsignedWrap();
6427
6428 Value *GEPVal = Builder.CreateGEP(Ty: ElemTy, Ptr, IdxList, Name, NW: NWFlags);
6429
6430 // If the pointer overflow sanitizer isn't enabled, do nothing.
6431 if (!SanOpts.has(K: SanitizerKind::PointerOverflow))
6432 return GEPVal;
6433
6434 // Perform nullptr-and-offset check unless the nullptr is defined.
6435 bool PerformNullCheck = !NullPointerIsDefined(
6436 F: Builder.GetInsertBlock()->getParent(), AS: PtrTy->getPointerAddressSpace());
6437 // Check for overflows unless the GEP got constant-folded,
6438 // and only in the default address space
6439 bool PerformOverflowCheck =
6440 !isa<llvm::Constant>(Val: GEPVal) && PtrTy->getPointerAddressSpace() == 0;
6441
6442 if (!(PerformNullCheck || PerformOverflowCheck))
6443 return GEPVal;
6444
6445 const auto &DL = CGM.getDataLayout();
6446
6447 auto CheckOrdinal = SanitizerKind::SO_PointerOverflow;
6448 auto CheckHandler = SanitizerHandler::PointerOverflow;
6449 SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);
6450 llvm::Type *IntPtrTy = DL.getIntPtrType(PtrTy);
6451
6452 GEPOffsetAndOverflow EvaluatedGEP =
6453 EmitGEPOffsetInBytes(BasePtr: Ptr, GEPVal, VMContext&: getLLVMContext(), CGM, Builder);
6454
6455 assert((!isa<llvm::Constant>(EvaluatedGEP.TotalOffset) ||
6456 EvaluatedGEP.OffsetOverflows == Builder.getFalse()) &&
6457 "If the offset got constant-folded, we don't expect that there was an "
6458 "overflow.");
6459
6460 auto *Zero = llvm::ConstantInt::getNullValue(Ty: IntPtrTy);
6461
6462 // Common case: if the total offset is zero, don't emit a check.
6463 if (EvaluatedGEP.TotalOffset == Zero)
6464 return GEPVal;
6465
6466 // Now that we've computed the total offset, add it to the base pointer (with
6467 // wrapping semantics).
6468 auto *IntPtr = Builder.CreatePtrToInt(V: Ptr, DestTy: IntPtrTy);
6469 auto *ComputedGEP = Builder.CreateAdd(LHS: IntPtr, RHS: EvaluatedGEP.TotalOffset);
6470
6471 llvm::SmallVector<std::pair<llvm::Value *, SanitizerKind::SanitizerOrdinal>,
6472 2>
6473 Checks;
6474
6475 if (PerformNullCheck) {
6476 // If the base pointer evaluates to a null pointer value,
6477 // the only valid pointer this inbounds GEP can produce is also
6478 // a null pointer, so the offset must also evaluate to zero.
6479 // Likewise, if we have non-zero base pointer, we can not get null pointer
6480 // as a result, so the offset can not be -intptr_t(BasePtr).
6481 // In other words, both pointers are either null, or both are non-null,
6482 // or the behaviour is undefined.
6483 auto *BaseIsNotNullptr = Builder.CreateIsNotNull(Arg: Ptr);
6484 auto *ResultIsNotNullptr = Builder.CreateIsNotNull(Arg: ComputedGEP);
6485 auto *Valid = Builder.CreateICmpEQ(LHS: BaseIsNotNullptr, RHS: ResultIsNotNullptr);
6486 Checks.emplace_back(Args&: Valid, Args&: CheckOrdinal);
6487 }
6488
6489 if (PerformOverflowCheck) {
6490 // The GEP is valid if:
6491 // 1) The total offset doesn't overflow, and
6492 // 2) The sign of the difference between the computed address and the base
6493 // pointer matches the sign of the total offset.
6494 llvm::Value *ValidGEP;
6495 auto *NoOffsetOverflow = Builder.CreateNot(V: EvaluatedGEP.OffsetOverflows);
6496 if (SignedIndices) {
6497 // GEP is computed as `unsigned base + signed offset`, therefore:
6498 // * If offset was positive, then the computed pointer can not be
6499 // [unsigned] less than the base pointer, unless it overflowed.
6500 // * If offset was negative, then the computed pointer can not be
6501 // [unsigned] greater than the bas pointere, unless it overflowed.
6502 auto *PosOrZeroValid = Builder.CreateICmpUGE(LHS: ComputedGEP, RHS: IntPtr);
6503 auto *PosOrZeroOffset =
6504 Builder.CreateICmpSGE(LHS: EvaluatedGEP.TotalOffset, RHS: Zero);
6505 llvm::Value *NegValid = Builder.CreateICmpULT(LHS: ComputedGEP, RHS: IntPtr);
6506 ValidGEP =
6507 Builder.CreateSelect(C: PosOrZeroOffset, True: PosOrZeroValid, False: NegValid);
6508 } else if (!IsSubtraction) {
6509 // GEP is computed as `unsigned base + unsigned offset`, therefore the
6510 // computed pointer can not be [unsigned] less than base pointer,
6511 // unless there was an overflow.
6512 // Equivalent to `@llvm.uadd.with.overflow(%base, %offset)`.
6513 ValidGEP = Builder.CreateICmpUGE(LHS: ComputedGEP, RHS: IntPtr);
6514 } else {
6515 // GEP is computed as `unsigned base - unsigned offset`, therefore the
6516 // computed pointer can not be [unsigned] greater than base pointer,
6517 // unless there was an overflow.
6518 // Equivalent to `@llvm.usub.with.overflow(%base, sub(0, %offset))`.
6519 ValidGEP = Builder.CreateICmpULE(LHS: ComputedGEP, RHS: IntPtr);
6520 }
6521 ValidGEP = Builder.CreateAnd(LHS: ValidGEP, RHS: NoOffsetOverflow);
6522 Checks.emplace_back(Args&: ValidGEP, Args&: CheckOrdinal);
6523 }
6524
6525 assert(!Checks.empty() && "Should have produced some checks.");
6526
6527 llvm::Constant *StaticArgs[] = {EmitCheckSourceLocation(Loc)};
6528 // Pass the computed GEP to the runtime to avoid emitting poisoned arguments.
6529 llvm::Value *DynamicArgs[] = {IntPtr, ComputedGEP};
6530 EmitCheck(Checked: Checks, Check: CheckHandler, StaticArgs, DynamicArgs);
6531
6532 return GEPVal;
6533}
6534
6535Address CodeGenFunction::EmitCheckedInBoundsGEP(
6536 Address Addr, ArrayRef<Value *> IdxList, llvm::Type *elementType,
6537 bool SignedIndices, bool IsSubtraction, SourceLocation Loc, CharUnits Align,
6538 const Twine &Name) {
6539 if (!SanOpts.has(K: SanitizerKind::PointerOverflow)) {
6540 llvm::GEPNoWrapFlags NWFlags = llvm::GEPNoWrapFlags::inBounds();
6541 if (!SignedIndices && !IsSubtraction)
6542 NWFlags |= llvm::GEPNoWrapFlags::noUnsignedWrap();
6543
6544 return Builder.CreateGEP(Addr, IdxList, ElementType: elementType, Align, Name, NW: NWFlags);
6545 }
6546
6547 return RawAddress(
6548 EmitCheckedInBoundsGEP(ElemTy: Addr.getElementType(), Ptr: Addr.emitRawPointer(CGF&: *this),
6549 IdxList, SignedIndices, IsSubtraction, Loc, Name),
6550 elementType, Align);
6551}
6552