1//===---- CGBuiltin.cpp - Emit LLVM Code for builtins ---------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit Builtin calls as LLVM code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGBuiltin.h"
14#include "ABIInfo.h"
15#include "CGCUDARuntime.h"
16#include "CGCXXABI.h"
17#include "CGDebugInfo.h"
18#include "CGObjCRuntime.h"
19#include "CGOpenCLRuntime.h"
20#include "CGRecordLayout.h"
21#include "CGValue.h"
22#include "CodeGenFunction.h"
23#include "CodeGenModule.h"
24#include "ConstantEmitter.h"
25#include "PatternInit.h"
26#include "TargetInfo.h"
27#include "clang/AST/OSLog.h"
28#include "clang/AST/StmtVisitor.h"
29#include "clang/Basic/DiagnosticFrontend.h"
30#include "clang/Basic/TargetInfo.h"
31#include "clang/CodeGenUtils/TargetUtils.h"
32#include "llvm/ADT/APFloat.h"
33#include "llvm/IR/InlineAsm.h"
34#include "llvm/IR/Instruction.h"
35#include "llvm/IR/Intrinsics.h"
36#include "llvm/IR/IntrinsicsX86.h"
37#include "llvm/IR/MatrixBuilder.h"
38#include "llvm/Support/ConvertUTF.h"
39#include "llvm/Support/ScopedPrinter.h"
40#include <algorithm>
41#include <optional>
42#include <utility>
43
44using namespace clang;
45using namespace CodeGen;
46using namespace llvm;
47
48/// Some builtins do not have library implementation on some targets and
49/// are instead emitted as LLVM IRs by some target builtin emitters.
50/// FIXME: Remove this when library support is added
51static bool shouldEmitBuiltinAsIR(unsigned BuiltinID,
52 const Builtin::Context &BI,
53 const CodeGenFunction &CGF) {
54 if (!CGF.CGM.getLangOpts().MathErrno &&
55 CGF.CurFPFeatures.getExceptionMode() ==
56 LangOptions::FPExceptionModeKind::FPE_Ignore &&
57 !CGF.CGM.getTargetCodeGenInfo().supportsLibCall()) {
58 switch (BuiltinID) {
59 default:
60 return false;
61 case Builtin::BIlogbf:
62 case Builtin::BI__builtin_logbf:
63 case Builtin::BIlogb:
64 case Builtin::BI__builtin_logb:
65 case Builtin::BIscalbnf:
66 case Builtin::BI__builtin_scalbnf:
67 case Builtin::BIscalbn:
68 case Builtin::BI__builtin_scalbn:
69 return true;
70 }
71 }
72 return false;
73}
74
75static Value *EmitTargetArchBuiltinExpr(CodeGenFunction *CGF,
76 unsigned BuiltinID, const CallExpr *E,
77 ReturnValueSlot ReturnValue,
78 llvm::Triple::ArchType Arch) {
79 // When compiling in HipStdPar mode we have to be conservative in rejecting
80 // target specific features in the FE, and defer the possible error to the
81 // AcceleratorCodeSelection pass, wherein iff an unsupported target builtin is
82 // referenced by an accelerator executable function, we emit an error.
83 // Returning nullptr here leads to the builtin being handled in
84 // EmitStdParUnsupportedBuiltin.
85 if (CGF->getLangOpts().HIPStdPar && CGF->getLangOpts().CUDAIsDevice &&
86 Arch != CGF->getTarget().getTriple().getArch())
87 return nullptr;
88
89 switch (Arch) {
90 case llvm::Triple::arm:
91 case llvm::Triple::armeb:
92 case llvm::Triple::thumb:
93 case llvm::Triple::thumbeb:
94 return CGF->EmitARMBuiltinExpr(BuiltinID, E, ReturnValue, Arch);
95 case llvm::Triple::aarch64:
96 case llvm::Triple::aarch64_32:
97 case llvm::Triple::aarch64_be:
98 return CGF->EmitAArch64BuiltinExpr(BuiltinID, E, Arch);
99 case llvm::Triple::bpfeb:
100 case llvm::Triple::bpfel:
101 return CGF->EmitBPFBuiltinExpr(BuiltinID, E);
102 case llvm::Triple::dxil:
103 return CGF->EmitDirectXBuiltinExpr(BuiltinID, E);
104 case llvm::Triple::x86:
105 case llvm::Triple::x86_64:
106 return CGF->EmitX86BuiltinExpr(BuiltinID, E);
107 case llvm::Triple::ppc:
108 case llvm::Triple::ppcle:
109 case llvm::Triple::ppc64:
110 case llvm::Triple::ppc64le:
111 return CGF->EmitPPCBuiltinExpr(BuiltinID, E);
112 case llvm::Triple::amdgpu:
113 case llvm::Triple::r600:
114 return CGF->EmitAMDGPUBuiltinExpr(BuiltinID, E);
115 case llvm::Triple::systemz:
116 return CGF->EmitSystemZBuiltinExpr(BuiltinID, E);
117 case llvm::Triple::nvptx:
118 case llvm::Triple::nvptx64:
119 return CGF->EmitNVPTXBuiltinExpr(BuiltinID, E);
120 case llvm::Triple::wasm32:
121 case llvm::Triple::wasm64:
122 return CGF->EmitWebAssemblyBuiltinExpr(BuiltinID, E);
123 case llvm::Triple::hexagon:
124 return CGF->EmitHexagonBuiltinExpr(BuiltinID, E);
125 case llvm::Triple::riscv32:
126 case llvm::Triple::riscv64:
127 case llvm::Triple::riscv32be:
128 case llvm::Triple::riscv64be:
129 return CGF->EmitRISCVBuiltinExpr(BuiltinID, E, ReturnValue);
130 case llvm::Triple::spirv32:
131 case llvm::Triple::spirv64:
132 if (CGF->getTarget().getTriple().getOS() == llvm::Triple::OSType::AMDHSA)
133 return CGF->EmitAMDGPUBuiltinExpr(BuiltinID, E);
134 [[fallthrough]];
135 case llvm::Triple::spirv:
136 return CGF->EmitSPIRVBuiltinExpr(BuiltinID, E);
137 case llvm::Triple::avr:
138 return CGF->EmitAVRBuiltinExpr(BuiltinID, E);
139 default:
140 return nullptr;
141 }
142}
143
144Value *CodeGenFunction::EmitTargetBuiltinExpr(unsigned BuiltinID,
145 const CallExpr *E,
146 ReturnValueSlot ReturnValue) {
147 if (getContext().BuiltinInfo.isAuxBuiltinID(ID: BuiltinID)) {
148 assert(getContext().getAuxTargetInfo() && "Missing aux target info");
149 return EmitTargetArchBuiltinExpr(
150 CGF: this, BuiltinID: getContext().BuiltinInfo.getAuxBuiltinID(ID: BuiltinID), E,
151 ReturnValue, Arch: getContext().getAuxTargetInfo()->getTriple().getArch());
152 }
153
154 return EmitTargetArchBuiltinExpr(CGF: this, BuiltinID, E, ReturnValue,
155 Arch: getTarget().getTriple().getArch());
156}
157
158static void initializeAlloca(CodeGenFunction &CGF, AllocaInst *AI, Value *Size,
159 Align AlignmentInBytes) {
160 ConstantInt *Byte;
161 switch (CGF.getLangOpts().getTrivialAutoVarInit()) {
162 case LangOptions::TrivialAutoVarInitKind::Uninitialized:
163 // Nothing to initialize.
164 return;
165 case LangOptions::TrivialAutoVarInitKind::Zero:
166 Byte = CGF.Builder.getInt8(C: 0x00);
167 break;
168 case LangOptions::TrivialAutoVarInitKind::Pattern: {
169 llvm::Type *Int8 = llvm::IntegerType::getInt8Ty(C&: CGF.CGM.getLLVMContext());
170 Byte = llvm::dyn_cast<llvm::ConstantInt>(
171 Val: initializationPatternFor(CGF.CGM, Int8));
172 break;
173 }
174 }
175 if (CGF.CGM.stopAutoInit())
176 return;
177 auto *I = CGF.Builder.CreateMemSet(Ptr: AI, Val: Byte, Size, Align: AlignmentInBytes);
178 I->addAnnotationMetadata(Annotation: "auto-init");
179}
180
181/// getBuiltinLibFunction - Given a builtin id for a function like
182/// "__builtin_fabsf", return a Function* for "fabsf".
183llvm::Constant *CodeGenModule::getBuiltinLibFunction(const FunctionDecl *FD,
184 unsigned BuiltinID) {
185 assert(Context.BuiltinInfo.isLibFunction(BuiltinID));
186
187 // Get the name, skip over the __builtin_ prefix (if necessary). We may have
188 // to build this up so provide a small stack buffer to handle the vast
189 // majority of names.
190 llvm::SmallString<64> Name;
191 GlobalDecl D(FD);
192
193 // If the builtin has been declared explicitly with an assembler label,
194 // use the mangled name. This differs from the plain label on platforms
195 // that prefix labels.
196 if (FD->hasAttr<AsmLabelAttr>())
197 Name = getMangledName(GD: D);
198 else if (StringRef TargetName = CodeGenUtils::getTargetBuiltinLibFunctionName(
199 BuiltinID, Target: getTarget());
200 !TargetName.empty())
201 Name = TargetName;
202 else
203 Name = Context.BuiltinInfo.getName(ID: BuiltinID).substr(pos: 10);
204
205 llvm::FunctionType *Ty =
206 cast<llvm::FunctionType>(Val: getTypes().ConvertType(T: FD->getType()));
207
208 return GetOrCreateLLVMFunction(MangledName: Name, Ty, D, /*ForVTable=*/false);
209}
210
211void appendDefaultIntrinsicArgs(SmallVectorImpl<llvm::Value *> &Args,
212 llvm::Function *F) {
213 llvm::FunctionType *FTy = F->getFunctionType();
214 unsigned NumParams = FTy->getNumParams();
215 if (Args.size() >= NumParams)
216 return;
217
218 auto [FirstDefault, Defaults] =
219 Intrinsic::getAllDefaultArgValues(IID: F->getIntrinsicID());
220 assert(Args.size() >= FirstDefault &&
221 "builtin passes fewer arguments than the intrinsic requires");
222
223 for (unsigned I = Args.size(); I != NumParams; ++I) {
224 llvm::Type *ParamTy = FTy->getParamType(i: I);
225 unsigned DefaultIdx = I - FirstDefault;
226 assert(ParamTy->isIntegerTy() &&
227 "intrinsic default arguments must be integer-typed");
228 Args.push_back(Elt: llvm::ConstantInt::get(Ty: ParamTy, V: Defaults[DefaultIdx]));
229 }
230}
231
232/// Emit the conversions required to turn the given value into an
233/// integer of the given size.
234Value *EmitToInt(CodeGenFunction &CGF, llvm::Value *V,
235 QualType T, llvm::IntegerType *IntType) {
236 V = CGF.EmitToMemory(Value: V, Ty: T);
237
238 if (V->getType()->isPointerTy())
239 return CGF.Builder.CreatePtrToInt(V, DestTy: IntType);
240
241 assert(V->getType() == IntType);
242 return V;
243}
244
245Value *EmitFromInt(CodeGenFunction &CGF, llvm::Value *V,
246 QualType T, llvm::Type *ResultType) {
247 V = CGF.EmitFromMemory(Value: V, Ty: T);
248
249 if (ResultType->isPointerTy())
250 return CGF.Builder.CreateIntToPtr(V, DestTy: ResultType);
251
252 assert(V->getType() == ResultType);
253 return V;
254}
255
256Address CheckAtomicAlignment(CodeGenFunction &CGF, const CallExpr *E) {
257 ASTContext &Ctx = CGF.getContext();
258 Address Ptr = CGF.EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
259 const llvm::DataLayout &DL = CGF.CGM.getDataLayout();
260 unsigned Bytes = Ptr.getElementType()->isPointerTy()
261 ? Ctx.getTypeSizeInChars(T: Ctx.VoidPtrTy).getQuantity()
262 : DL.getTypeStoreSize(Ty: Ptr.getElementType());
263 unsigned Align = Ptr.getAlignment().getQuantity();
264 if (Align % Bytes != 0) {
265 DiagnosticsEngine &Diags = CGF.CGM.getDiags();
266 Diags.Report(Loc: E->getBeginLoc(), DiagID: diag::warn_sync_op_misaligned);
267 // Force address to be at least naturally-aligned.
268 return Ptr.withAlignment(NewAlignment: CharUnits::fromQuantity(Quantity: Bytes));
269 }
270 return Ptr;
271}
272
273/// Utility to insert an atomic instruction based on Intrinsic::ID
274/// and the expression node.
275Value *MakeBinaryAtomicValue(
276 CodeGenFunction &CGF, llvm::AtomicRMWInst::BinOp Kind, const CallExpr *E,
277 AtomicOrdering Ordering) {
278
279 QualType T = E->getType();
280 assert(E->getArg(0)->getType()->isPointerType());
281 assert(CGF.getContext().hasSameUnqualifiedType(T,
282 E->getArg(0)->getType()->getPointeeType()));
283 assert(CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType()));
284
285 Address DestAddr = CheckAtomicAlignment(CGF, E);
286
287 llvm::IntegerType *IntType = llvm::IntegerType::get(
288 C&: CGF.getLLVMContext(), NumBits: CGF.getContext().getTypeSize(T));
289
290 llvm::Value *Val = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
291 llvm::Type *ValueType = Val->getType();
292 Val = EmitToInt(CGF, V: Val, T, IntType);
293
294 llvm::Value *Result =
295 CGF.Builder.CreateAtomicRMW(Op: Kind, Addr: DestAddr, Val, Ordering);
296 // Consider atomics to be volatile in MS kernel mode.
297 if (CGF.CGM.getLangOpts().Kernel)
298 cast<llvm::AtomicRMWInst>(Val: Result)->setVolatile(true);
299 return EmitFromInt(CGF, V: Result, T, ResultType: ValueType);
300}
301
302static Value *EmitNontemporalStore(CodeGenFunction &CGF, const CallExpr *E) {
303 Value *Val = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
304 Address Addr = CGF.EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
305
306 Val = CGF.EmitToMemory(Value: Val, Ty: E->getArg(Arg: 0)->getType());
307 LValue LV = CGF.MakeAddrLValue(Addr, T: E->getArg(Arg: 0)->getType());
308 LV.setNontemporal(true);
309 CGF.EmitStoreOfScalar(value: Val, lvalue: LV, isInit: false);
310 return nullptr;
311}
312
313static Value *EmitNontemporalLoad(CodeGenFunction &CGF, const CallExpr *E) {
314 Address Addr = CGF.EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
315
316 LValue LV = CGF.MakeAddrLValue(Addr, T: E->getType());
317 LV.setNontemporal(true);
318 return CGF.EmitLoadOfScalar(lvalue: LV, Loc: E->getExprLoc());
319}
320
321static RValue EmitBinaryAtomic(CodeGenFunction &CGF,
322 llvm::AtomicRMWInst::BinOp Kind,
323 const CallExpr *E) {
324 return RValue::get(V: MakeBinaryAtomicValue(CGF, Kind, E));
325}
326
327/// Utility to insert an atomic instruction based Intrinsic::ID and
328/// the expression node, where the return value is the result of the
329/// operation.
330static RValue EmitBinaryAtomicPost(CodeGenFunction &CGF,
331 llvm::AtomicRMWInst::BinOp Kind,
332 const CallExpr *E,
333 Instruction::BinaryOps Op,
334 bool Invert = false) {
335 QualType T = E->getType();
336 assert(E->getArg(0)->getType()->isPointerType());
337 assert(CGF.getContext().hasSameUnqualifiedType(T,
338 E->getArg(0)->getType()->getPointeeType()));
339 assert(CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType()));
340
341 Address DestAddr = CheckAtomicAlignment(CGF, E);
342
343 llvm::IntegerType *IntType = llvm::IntegerType::get(
344 C&: CGF.getLLVMContext(), NumBits: CGF.getContext().getTypeSize(T));
345
346 llvm::Value *Val = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
347 llvm::Type *ValueType = Val->getType();
348 Val = EmitToInt(CGF, V: Val, T, IntType);
349
350 llvm::Value *Result = CGF.Builder.CreateAtomicRMW(
351 Op: Kind, Addr: DestAddr, Val, Ordering: llvm::AtomicOrdering::SequentiallyConsistent);
352 Result = CGF.Builder.CreateBinOp(Opc: Op, LHS: Result, RHS: Val);
353 if (Invert)
354 Result =
355 CGF.Builder.CreateBinOp(Opc: llvm::Instruction::Xor, LHS: Result,
356 RHS: llvm::ConstantInt::getAllOnesValue(Ty: IntType));
357 Result = EmitFromInt(CGF, V: Result, T, ResultType: ValueType);
358 return RValue::get(V: Result);
359}
360
361/// Utility to insert an atomic cmpxchg instruction.
362///
363/// @param CGF The current codegen function.
364/// @param E Builtin call expression to convert to cmpxchg.
365/// arg0 - address to operate on
366/// arg1 - value to compare with
367/// arg2 - new value
368/// @param ReturnBool Specifies whether to return success flag of
369/// cmpxchg result or the old value.
370///
371/// @returns result of cmpxchg, according to ReturnBool
372///
373/// Note: In order to lower Microsoft's _InterlockedCompareExchange* intrinsics
374/// invoke the function EmitAtomicCmpXchgForMSIntrin.
375Value *MakeAtomicCmpXchgValue(CodeGenFunction &CGF, const CallExpr *E,
376 bool ReturnBool,
377 llvm::AtomicOrdering SuccessOrdering,
378 llvm::AtomicOrdering FailureOrdering) {
379 QualType T = ReturnBool ? E->getArg(Arg: 1)->getType() : E->getType();
380 Address DestAddr = CheckAtomicAlignment(CGF, E);
381
382 llvm::IntegerType *IntType = llvm::IntegerType::get(
383 C&: CGF.getLLVMContext(), NumBits: CGF.getContext().getTypeSize(T));
384
385 Value *Cmp = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
386 llvm::Type *ValueType = Cmp->getType();
387 Cmp = EmitToInt(CGF, V: Cmp, T, IntType);
388 Value *New = EmitToInt(CGF, V: CGF.EmitScalarExpr(E: E->getArg(Arg: 2)), T, IntType);
389
390 Value *Pair = CGF.Builder.CreateAtomicCmpXchg(
391 Addr: DestAddr, Cmp, New, SuccessOrdering, FailureOrdering);
392 if (ReturnBool)
393 // Extract boolean success flag and zext it to int.
394 return CGF.Builder.CreateZExt(V: CGF.Builder.CreateExtractValue(Agg: Pair, Idxs: 1),
395 DestTy: CGF.ConvertType(T: E->getType()));
396 else
397 // Extract old value and emit it using the same type as compare value.
398 return EmitFromInt(CGF, V: CGF.Builder.CreateExtractValue(Agg: Pair, Idxs: 0), T,
399 ResultType: ValueType);
400}
401
402/// This function should be invoked to emit atomic cmpxchg for Microsoft's
403/// _InterlockedCompareExchange* intrinsics which have the following signature:
404/// T _InterlockedCompareExchange(T volatile *Destination,
405/// T Exchange,
406/// T Comparand);
407///
408/// Whereas the llvm 'cmpxchg' instruction has the following syntax:
409/// cmpxchg *Destination, Comparand, Exchange.
410/// So we need to swap Comparand and Exchange when invoking
411/// CreateAtomicCmpXchg. That is the reason we could not use the above utility
412/// function MakeAtomicCmpXchgValue since it expects the arguments to be
413/// already swapped.
414
415static
416Value *EmitAtomicCmpXchgForMSIntrin(CodeGenFunction &CGF, const CallExpr *E,
417 AtomicOrdering SuccessOrdering = AtomicOrdering::SequentiallyConsistent) {
418 assert(E->getArg(0)->getType()->isPointerType());
419 assert(CGF.getContext().hasSameUnqualifiedType(
420 E->getType(), E->getArg(0)->getType()->getPointeeType()));
421 assert(CGF.getContext().hasSameUnqualifiedType(E->getType(),
422 E->getArg(1)->getType()));
423 assert(CGF.getContext().hasSameUnqualifiedType(E->getType(),
424 E->getArg(2)->getType()));
425
426 Address DestAddr = CheckAtomicAlignment(CGF, E);
427
428 auto *Exchange = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
429 auto *RTy = Exchange->getType();
430
431 auto *Comparand = CGF.EmitScalarExpr(E: E->getArg(Arg: 2));
432
433 if (RTy->isPointerTy()) {
434 Exchange = CGF.Builder.CreatePtrToInt(V: Exchange, DestTy: CGF.IntPtrTy);
435 Comparand = CGF.Builder.CreatePtrToInt(V: Comparand, DestTy: CGF.IntPtrTy);
436 }
437
438 // For Release ordering, the failure ordering should be Monotonic.
439 auto FailureOrdering = SuccessOrdering == AtomicOrdering::Release ?
440 AtomicOrdering::Monotonic :
441 SuccessOrdering;
442
443 // The atomic instruction is marked volatile for consistency with MSVC. This
444 // blocks the few atomics optimizations that LLVM has. If we want to optimize
445 // _Interlocked* operations in the future, we will have to remove the volatile
446 // marker.
447 auto *CmpXchg = CGF.Builder.CreateAtomicCmpXchg(
448 Addr: DestAddr, Cmp: Comparand, New: Exchange, SuccessOrdering, FailureOrdering);
449 CmpXchg->setVolatile(true);
450
451 auto *Result = CGF.Builder.CreateExtractValue(Agg: CmpXchg, Idxs: 0);
452 if (RTy->isPointerTy()) {
453 Result = CGF.Builder.CreateIntToPtr(V: Result, DestTy: RTy);
454 }
455
456 return Result;
457}
458
459// 64-bit Microsoft platforms support 128 bit cmpxchg operations. They are
460// prototyped like this:
461//
462// unsigned char _InterlockedCompareExchange128...(
463// __int64 volatile * _Destination,
464// __int64 _ExchangeHigh,
465// __int64 _ExchangeLow,
466// __int64 * _ComparandResult);
467//
468// Note that Destination is assumed to be at least 16-byte aligned, despite
469// being typed int64.
470
471static Value *EmitAtomicCmpXchg128ForMSIntrin(CodeGenFunction &CGF,
472 const CallExpr *E,
473 AtomicOrdering SuccessOrdering) {
474 assert(E->getNumArgs() == 4);
475 llvm::Value *DestPtr = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
476 llvm::Value *ExchangeHigh = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
477 llvm::Value *ExchangeLow = CGF.EmitScalarExpr(E: E->getArg(Arg: 2));
478 Address ComparandAddr = CGF.EmitPointerWithAlignment(Addr: E->getArg(Arg: 3));
479
480 assert(DestPtr->getType()->isPointerTy());
481 assert(!ExchangeHigh->getType()->isPointerTy());
482 assert(!ExchangeLow->getType()->isPointerTy());
483
484 // For Release ordering, the failure ordering should be Monotonic.
485 auto FailureOrdering = SuccessOrdering == AtomicOrdering::Release
486 ? AtomicOrdering::Monotonic
487 : SuccessOrdering;
488
489 // Convert to i128 pointers and values. Alignment is also overridden for
490 // destination pointer.
491 llvm::Type *Int128Ty = llvm::IntegerType::get(C&: CGF.getLLVMContext(), NumBits: 128);
492 Address DestAddr(DestPtr, Int128Ty,
493 CGF.getContext().toCharUnitsFromBits(BitSize: 128));
494 ComparandAddr = ComparandAddr.withElementType(ElemTy: Int128Ty);
495
496 // (((i128)hi) << 64) | ((i128)lo)
497 ExchangeHigh = CGF.Builder.CreateZExt(V: ExchangeHigh, DestTy: Int128Ty);
498 ExchangeLow = CGF.Builder.CreateZExt(V: ExchangeLow, DestTy: Int128Ty);
499 ExchangeHigh =
500 CGF.Builder.CreateShl(LHS: ExchangeHigh, RHS: llvm::ConstantInt::get(Ty: Int128Ty, V: 64));
501 llvm::Value *Exchange = CGF.Builder.CreateOr(LHS: ExchangeHigh, RHS: ExchangeLow);
502
503 // Load the comparand for the instruction.
504 llvm::Value *Comparand = CGF.Builder.CreateLoad(Addr: ComparandAddr);
505
506 auto *CXI = CGF.Builder.CreateAtomicCmpXchg(Addr: DestAddr, Cmp: Comparand, New: Exchange,
507 SuccessOrdering, FailureOrdering);
508
509 // The atomic instruction is marked volatile for consistency with MSVC. This
510 // blocks the few atomics optimizations that LLVM has. If we want to optimize
511 // _Interlocked* operations in the future, we will have to remove the volatile
512 // marker.
513 CXI->setVolatile(true);
514
515 // Store the result as an outparameter.
516 CGF.Builder.CreateStore(Val: CGF.Builder.CreateExtractValue(Agg: CXI, Idxs: 0),
517 Addr: ComparandAddr);
518
519 // Get the success boolean and zero extend it to i8.
520 Value *Success = CGF.Builder.CreateExtractValue(Agg: CXI, Idxs: 1);
521 return CGF.Builder.CreateZExt(V: Success, DestTy: CGF.Int8Ty);
522}
523
524static Value *EmitAtomicIncrementValue(CodeGenFunction &CGF, const CallExpr *E,
525 AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
526 assert(E->getArg(0)->getType()->isPointerType());
527
528 auto *IntTy = CGF.ConvertType(T: E->getType());
529 Address DestAddr = CheckAtomicAlignment(CGF, E);
530 auto *Result = CGF.Builder.CreateAtomicRMW(
531 Op: AtomicRMWInst::Add, Addr: DestAddr, Val: ConstantInt::get(Ty: IntTy, V: 1), Ordering);
532 return CGF.Builder.CreateAdd(LHS: Result, RHS: ConstantInt::get(Ty: IntTy, V: 1));
533}
534
535static Value *EmitAtomicDecrementValue(
536 CodeGenFunction &CGF, const CallExpr *E,
537 AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
538 assert(E->getArg(0)->getType()->isPointerType());
539
540 auto *IntTy = CGF.ConvertType(T: E->getType());
541 Address DestAddr = CheckAtomicAlignment(CGF, E);
542 auto *Result = CGF.Builder.CreateAtomicRMW(
543 Op: AtomicRMWInst::Sub, Addr: DestAddr, Val: ConstantInt::get(Ty: IntTy, V: 1), Ordering);
544 return CGF.Builder.CreateSub(LHS: Result, RHS: ConstantInt::get(Ty: IntTy, V: 1));
545}
546
547// Build a plain volatile load.
548static Value *EmitISOVolatileLoad(CodeGenFunction &CGF, const CallExpr *E) {
549 Value *Ptr = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
550 QualType ElTy = E->getArg(Arg: 0)->getType()->getPointeeType();
551 CharUnits LoadSize = CGF.getContext().getTypeSizeInChars(T: ElTy);
552 llvm::Type *ITy =
553 llvm::IntegerType::get(C&: CGF.getLLVMContext(), NumBits: LoadSize.getQuantity() * 8);
554 llvm::LoadInst *Load = CGF.Builder.CreateAlignedLoad(Ty: ITy, Addr: Ptr, Align: LoadSize);
555 Load->setAtomic(Ordering: llvm::AtomicOrdering::Monotonic);
556 Load->setVolatile(true);
557 return Load;
558}
559
560// Build a plain volatile store.
561static Value *EmitISOVolatileStore(CodeGenFunction &CGF, const CallExpr *E) {
562 Value *Ptr = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
563 Value *Value = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
564 QualType ElTy = E->getArg(Arg: 0)->getType()->getPointeeType();
565 CharUnits StoreSize = CGF.getContext().getTypeSizeInChars(T: ElTy);
566 llvm::StoreInst *Store =
567 CGF.Builder.CreateAlignedStore(Val: Value, Addr: Ptr, Align: StoreSize);
568 Store->setAtomic(Ordering: llvm::AtomicOrdering::Monotonic);
569 Store->setVolatile(true);
570 return Store;
571}
572
573// Emit a simple mangled intrinsic that has 1 argument and a return type
574// matching the argument type. Depending on mode, this may be a constrained
575// floating-point intrinsic.
576Value *emitUnaryMaybeConstrainedFPBuiltin(CodeGenFunction &CGF,
577 const CallExpr *E, unsigned IntrinsicID,
578 unsigned ConstrainedIntrinsicID) {
579 llvm::Value *Src0 = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
580
581 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
582 if (CGF.Builder.getIsFPConstrained()) {
583 Function *F = CGF.CGM.getIntrinsic(IID: ConstrainedIntrinsicID, Tys: Src0->getType());
584 return CGF.Builder.CreateConstrainedFPCall(Callee: F, Args: { Src0 });
585 } else {
586 Function *F = CGF.CGM.getIntrinsic(IID: IntrinsicID, Tys: Src0->getType());
587 return CGF.Builder.CreateCall(Callee: F, Args: Src0);
588 }
589}
590
591// Emit an intrinsic that has 2 operands of the same type as its result.
592// Depending on mode, this may be a constrained floating-point intrinsic.
593static Value *emitBinaryMaybeConstrainedFPBuiltin(CodeGenFunction &CGF,
594 const CallExpr *E, unsigned IntrinsicID,
595 unsigned ConstrainedIntrinsicID) {
596 llvm::Value *Src0 = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
597 llvm::Value *Src1 = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
598
599 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
600 if (CGF.Builder.getIsFPConstrained()) {
601 Function *F = CGF.CGM.getIntrinsic(IID: ConstrainedIntrinsicID, Tys: Src0->getType());
602 return CGF.Builder.CreateConstrainedFPCall(Callee: F, Args: { Src0, Src1 });
603 } else {
604 Function *F = CGF.CGM.getIntrinsic(IID: IntrinsicID, Tys: Src0->getType());
605 return CGF.Builder.CreateCall(Callee: F, Args: { Src0, Src1 });
606 }
607}
608
609// Has second type mangled argument.
610static Value *
611emitBinaryExpMaybeConstrainedFPBuiltin(CodeGenFunction &CGF, const CallExpr *E,
612 Intrinsic::ID IntrinsicID,
613 Intrinsic::ID ConstrainedIntrinsicID) {
614 llvm::Value *Src0 = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
615 llvm::Value *Src1 = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
616
617 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
618 if (CGF.Builder.getIsFPConstrained()) {
619 Function *F = CGF.CGM.getIntrinsic(IID: ConstrainedIntrinsicID,
620 Tys: {Src0->getType(), Src1->getType()});
621 return CGF.Builder.CreateConstrainedFPCall(Callee: F, Args: {Src0, Src1});
622 }
623
624 Function *F =
625 CGF.CGM.getIntrinsic(IID: IntrinsicID, Tys: {Src0->getType(), Src1->getType()});
626 return CGF.Builder.CreateCall(Callee: F, Args: {Src0, Src1});
627}
628
629// Emit an intrinsic that has 3 operands of the same type as its result.
630// Depending on mode, this may be a constrained floating-point intrinsic.
631static Value *emitTernaryMaybeConstrainedFPBuiltin(CodeGenFunction &CGF,
632 const CallExpr *E, unsigned IntrinsicID,
633 unsigned ConstrainedIntrinsicID) {
634 llvm::Value *Src0 = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
635 llvm::Value *Src1 = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
636 llvm::Value *Src2 = CGF.EmitScalarExpr(E: E->getArg(Arg: 2));
637
638 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
639 if (CGF.Builder.getIsFPConstrained()) {
640 Function *F = CGF.CGM.getIntrinsic(IID: ConstrainedIntrinsicID, Tys: Src0->getType());
641 return CGF.Builder.CreateConstrainedFPCall(Callee: F, Args: { Src0, Src1, Src2 });
642 } else {
643 Function *F = CGF.CGM.getIntrinsic(IID: IntrinsicID, Tys: Src0->getType());
644 return CGF.Builder.CreateCall(Callee: F, Args: { Src0, Src1, Src2 });
645 }
646}
647
648// Emit an intrinsic that has overloaded integer result and fp operand.
649static Value *
650emitMaybeConstrainedFPToIntRoundBuiltin(CodeGenFunction &CGF, const CallExpr *E,
651 unsigned IntrinsicID,
652 unsigned ConstrainedIntrinsicID) {
653 llvm::Type *ResultType = CGF.ConvertType(T: E->getType());
654 llvm::Value *Src0 = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
655
656 if (CGF.Builder.getIsFPConstrained()) {
657 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
658 Function *F = CGF.CGM.getIntrinsic(IID: ConstrainedIntrinsicID,
659 Tys: {ResultType, Src0->getType()});
660 return CGF.Builder.CreateConstrainedFPCall(Callee: F, Args: {Src0});
661 } else {
662 Function *F =
663 CGF.CGM.getIntrinsic(IID: IntrinsicID, Tys: {ResultType, Src0->getType()});
664 return CGF.Builder.CreateCall(Callee: F, Args: Src0);
665 }
666}
667
668static Value *emitFrexpBuiltin(CodeGenFunction &CGF, const CallExpr *E,
669 Intrinsic::ID IntrinsicID) {
670 llvm::Value *Src0 = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
671 llvm::Value *Src1 = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
672
673 QualType IntPtrTy = E->getArg(Arg: 1)->getType()->getPointeeType();
674 llvm::Type *IntTy = CGF.ConvertType(T: IntPtrTy);
675 llvm::Function *F =
676 CGF.CGM.getIntrinsic(IID: IntrinsicID, Tys: {Src0->getType(), IntTy});
677 llvm::Value *Call = CGF.Builder.CreateCall(Callee: F, Args: Src0);
678
679 llvm::Value *Exp = CGF.Builder.CreateExtractValue(Agg: Call, Idxs: 1);
680 LValue LV = CGF.MakeNaturalAlignAddrLValue(V: Src1, T: IntPtrTy);
681 CGF.EmitStoreOfScalar(value: Exp, lvalue: LV);
682
683 return CGF.Builder.CreateExtractValue(Agg: Call, Idxs: 0);
684}
685
686static void emitSincosBuiltin(CodeGenFunction &CGF, const CallExpr *E,
687 Intrinsic::ID IntrinsicID) {
688 llvm::Value *Val = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
689 llvm::Value *Dest0 = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
690 llvm::Value *Dest1 = CGF.EmitScalarExpr(E: E->getArg(Arg: 2));
691
692 llvm::Function *F = CGF.CGM.getIntrinsic(IID: IntrinsicID, Tys: {Val->getType()});
693 llvm::Value *Call = CGF.Builder.CreateCall(Callee: F, Args: Val);
694
695 llvm::Value *SinResult = CGF.Builder.CreateExtractValue(Agg: Call, Idxs: 0);
696 llvm::Value *CosResult = CGF.Builder.CreateExtractValue(Agg: Call, Idxs: 1);
697
698 QualType DestPtrType = E->getArg(Arg: 1)->getType()->getPointeeType();
699 LValue SinLV = CGF.MakeNaturalAlignAddrLValue(V: Dest0, T: DestPtrType);
700 LValue CosLV = CGF.MakeNaturalAlignAddrLValue(V: Dest1, T: DestPtrType);
701
702 llvm::StoreInst *StoreSin =
703 CGF.Builder.CreateStore(Val: SinResult, Addr: SinLV.getAddress());
704 llvm::StoreInst *StoreCos =
705 CGF.Builder.CreateStore(Val: CosResult, Addr: CosLV.getAddress());
706
707 // Mark the two stores as non-aliasing with each other. The order of stores
708 // emitted by this builtin is arbitrary, enforcing a particular order will
709 // prevent optimizations later on.
710 llvm::MDBuilder MDHelper(CGF.getLLVMContext());
711 MDNode *Domain = MDHelper.createAnonymousAliasScopeDomain();
712 MDNode *AliasScope = MDHelper.createAnonymousAliasScope(Domain);
713 MDNode *AliasScopeList = MDNode::get(Context&: Call->getContext(), MDs: AliasScope);
714 StoreSin->setMetadata(KindID: LLVMContext::MD_alias_scope, Node: AliasScopeList);
715 StoreCos->setMetadata(KindID: LLVMContext::MD_noalias, Node: AliasScopeList);
716}
717
718static llvm::Value *emitModfBuiltin(CodeGenFunction &CGF, const CallExpr *E,
719 Intrinsic::ID IntrinsicID) {
720 llvm::Value *Val = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
721 llvm::Value *IntPartDest = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
722
723 llvm::Value *Call =
724 CGF.Builder.CreateIntrinsic(ID: IntrinsicID, OverloadTypes: {Val->getType()}, Args: Val);
725
726 llvm::Value *FractionalResult = CGF.Builder.CreateExtractValue(Agg: Call, Idxs: 0);
727 llvm::Value *IntegralResult = CGF.Builder.CreateExtractValue(Agg: Call, Idxs: 1);
728
729 QualType DestPtrType = E->getArg(Arg: 1)->getType()->getPointeeType();
730 LValue IntegralLV = CGF.MakeNaturalAlignAddrLValue(V: IntPartDest, T: DestPtrType);
731 CGF.EmitStoreOfScalar(value: IntegralResult, lvalue: IntegralLV);
732
733 return FractionalResult;
734}
735
736/// EmitFAbs - Emit a call to @llvm.fabs().
737static Value *EmitFAbs(CodeGenFunction &CGF, Value *V) {
738 llvm::Value *Call = CGF.Builder.CreateFAbs(V);
739 if (auto *CallI = dyn_cast<llvm::CallInst>(Val: Call))
740 CallI->setDoesNotAccessMemory();
741 return Call;
742}
743
744/// Emit the computation of the sign bit for a floating point value. Returns
745/// the i1 sign bit value.
746static Value *EmitSignBit(CodeGenFunction &CGF, Value *V) {
747 LLVMContext &C = CGF.CGM.getLLVMContext();
748
749 llvm::Type *Ty = V->getType();
750 int Width = Ty->getPrimitiveSizeInBits();
751 llvm::Type *IntTy = llvm::IntegerType::get(C, NumBits: Width);
752 V = CGF.Builder.CreateBitCast(V, DestTy: IntTy);
753 if (Ty->isPPC_FP128Ty()) {
754 // We want the sign bit of the higher-order double. The bitcast we just
755 // did works as if the double-double was stored to memory and then
756 // read as an i128. The "store" will put the higher-order double in the
757 // lower address in both little- and big-Endian modes, but the "load"
758 // will treat those bits as a different part of the i128: the low bits in
759 // little-Endian, the high bits in big-Endian. Therefore, on big-Endian
760 // we need to shift the high bits down to the low before truncating.
761 Width >>= 1;
762 if (CGF.getTarget().isBigEndian()) {
763 Value *ShiftCst = llvm::ConstantInt::get(Ty: IntTy, V: Width);
764 V = CGF.Builder.CreateLShr(LHS: V, RHS: ShiftCst);
765 }
766 // We are truncating value in order to extract the higher-order
767 // double, which we will be using to extract the sign from.
768 IntTy = llvm::IntegerType::get(C, NumBits: Width);
769 V = CGF.Builder.CreateTrunc(V, DestTy: IntTy);
770 }
771 Value *Zero = llvm::Constant::getNullValue(Ty: IntTy);
772 return CGF.Builder.CreateICmpSLT(LHS: V, RHS: Zero);
773}
774
775static RValue emitLibraryCall(CodeGenFunction &CGF, const FunctionDecl *FD,
776 const CallExpr *E, llvm::Constant *calleeValue) {
777 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
778 CGCallee callee = CGCallee::forDirect(functionPtr: calleeValue, abstractInfo: GlobalDecl(FD));
779 llvm::CallBase *callOrInvoke = nullptr;
780 CGFunctionInfo const *FnInfo = nullptr;
781 return CGF.EmitCall(FnType: E->getCallee()->getType(), Callee: callee, E, ReturnValue: ReturnValueSlot(),
782 /*Chain=*/nullptr, CallOrInvoke: &callOrInvoke, ResolvedFnInfo: &FnInfo);
783}
784
785/// Emit a call to llvm.{sadd,uadd,ssub,usub,smul,umul}.with.overflow.*
786/// depending on IntrinsicID.
787///
788/// \arg CGF The current codegen function.
789/// \arg IntrinsicID The ID for the Intrinsic we wish to generate.
790/// \arg X The first argument to the llvm.*.with.overflow.*.
791/// \arg Y The second argument to the llvm.*.with.overflow.*.
792/// \arg Carry The carry returned by the llvm.*.with.overflow.*.
793/// \returns The result (i.e. sum/product) returned by the intrinsic.
794llvm::Value *EmitOverflowIntrinsic(CodeGenFunction &CGF,
795 const Intrinsic::ID IntrinsicID,
796 llvm::Value *X, llvm::Value *Y,
797 llvm::Value *&Carry) {
798 // Make sure we have integers of the same width.
799 assert(X->getType() == Y->getType() &&
800 "Arguments must be the same type. (Did you forget to make sure both "
801 "arguments have the same integer width?)");
802
803 Function *Callee = CGF.CGM.getIntrinsic(IID: IntrinsicID, Tys: X->getType());
804 llvm::Value *Tmp = CGF.Builder.CreateCall(Callee, Args: {X, Y});
805 Carry = CGF.Builder.CreateExtractValue(Agg: Tmp, Idxs: 1);
806 return CGF.Builder.CreateExtractValue(Agg: Tmp, Idxs: 0);
807}
808
809namespace {
810 struct WidthAndSignedness {
811 unsigned Width;
812 bool Signed;
813 };
814}
815
816static WidthAndSignedness
817getIntegerWidthAndSignedness(const clang::ASTContext &context,
818 const clang::QualType Type) {
819 assert(Type->isIntegerType() && "Given type is not an integer.");
820 unsigned Width = context.getIntWidth(T: Type);
821 bool Signed = Type->isSignedIntegerType();
822 return {.Width: Width, .Signed: Signed};
823}
824
825// Given one or more integer types, this function produces an integer type that
826// encompasses them: any value in one of the given types could be expressed in
827// the encompassing type.
828static struct WidthAndSignedness
829EncompassingIntegerType(ArrayRef<struct WidthAndSignedness> Types) {
830 assert(Types.size() > 0 && "Empty list of types.");
831
832 // If any of the given types is signed, we must return a signed type.
833 bool Signed = false;
834 for (const auto &Type : Types) {
835 Signed |= Type.Signed;
836 }
837
838 // The encompassing type must have a width greater than or equal to the width
839 // of the specified types. Additionally, if the encompassing type is signed,
840 // its width must be strictly greater than the width of any unsigned types
841 // given.
842 unsigned Width = 0;
843 for (const auto &Type : Types) {
844 unsigned MinWidth = Type.Width + (Signed && !Type.Signed);
845 if (Width < MinWidth) {
846 Width = MinWidth;
847 }
848 }
849
850 return {.Width: Width, .Signed: Signed};
851}
852
853Value *CodeGenFunction::EmitVAStartEnd(Value *ArgValue, bool IsStart) {
854 Intrinsic::ID inst = IsStart ? Intrinsic::vastart : Intrinsic::vaend;
855 return Builder.CreateCall(Callee: CGM.getIntrinsic(IID: inst, Tys: {ArgValue->getType()}),
856 Args: ArgValue);
857}
858
859/// Checks if using the result of __builtin_object_size(p, @p From) in place of
860/// __builtin_object_size(p, @p To) is correct
861static bool areBOSTypesCompatible(int From, int To) {
862 // Note: Our __builtin_object_size implementation currently treats Type=0 and
863 // Type=2 identically. Encoding this implementation detail here may make
864 // improving __builtin_object_size difficult in the future, so it's omitted.
865 return From == To || (From == 0 && To == 1) || (From == 3 && To == 2);
866}
867
868static llvm::Value *
869getDefaultBuiltinObjectSizeResult(unsigned Type, llvm::IntegerType *ResType) {
870 return ConstantInt::get(Ty: ResType, V: (Type & 2) ? 0 : -1, /*isSigned=*/IsSigned: true);
871}
872
873llvm::Value *
874CodeGenFunction::evaluateOrEmitBuiltinObjectSize(const Expr *E, unsigned Type,
875 llvm::IntegerType *ResType,
876 llvm::Value *EmittedE,
877 bool IsDynamic) {
878 if (std::optional<uint64_t> ObjectSize =
879 E->tryEvaluateObjectSize(Ctx: getContext(), Type))
880 return ConstantInt::get(Ty: ResType, V: *ObjectSize, /*isSigned=*/IsSigned: true);
881 return emitBuiltinObjectSize(E, Type, ResType, EmittedE, IsDynamic);
882}
883
884/// Find a struct's flexible array member. It may be embedded inside multiple
885/// sub-structs, but must still be the last field.
886static const FieldDecl *FindFlexibleArrayMemberField(CodeGenFunction &CGF,
887 ASTContext &Ctx,
888 const RecordDecl *RD) {
889 const LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel =
890 CGF.getLangOpts().getStrictFlexArraysLevel();
891
892 if (RD->isImplicit())
893 return nullptr;
894
895 for (const FieldDecl *FD : RD->fields()) {
896 if (Decl::isFlexibleArrayMemberLike(
897 Context: Ctx, D: FD, Ty: FD->getType(), StrictFlexArraysLevel,
898 /*IgnoreTemplateOrMacroSubstitution=*/true))
899 return FD;
900
901 if (const auto *RD = FD->getType()->getAsRecordDecl())
902 if (const FieldDecl *FD = FindFlexibleArrayMemberField(CGF, Ctx, RD))
903 return FD;
904 }
905
906 return nullptr;
907}
908
909/// Calculate the offset of a struct field. It may be embedded inside multiple
910/// sub-structs.
911static bool GetFieldOffset(ASTContext &Ctx, const RecordDecl *RD,
912 const FieldDecl *FD, int64_t &Offset) {
913 if (RD->isImplicit())
914 return false;
915
916 // Keep track of the field number ourselves, because the other methods
917 // (CGRecordLayout::getLLVMFieldNo) aren't always equivalent to how the AST
918 // is laid out.
919 uint32_t FieldNo = 0;
920 const ASTRecordLayout &Layout = Ctx.getASTRecordLayout(D: RD);
921
922 for (const FieldDecl *Field : RD->fields()) {
923 if (Field == FD) {
924 Offset += Layout.getFieldOffset(FieldNo);
925 return true;
926 }
927
928 if (const auto *RD = Field->getType()->getAsRecordDecl()) {
929 if (GetFieldOffset(Ctx, RD, FD, Offset)) {
930 Offset += Layout.getFieldOffset(FieldNo);
931 return true;
932 }
933 }
934
935 if (!RD->isUnion())
936 ++FieldNo;
937 }
938
939 return false;
940}
941
942static std::optional<int64_t>
943GetFieldOffset(ASTContext &Ctx, const RecordDecl *RD, const FieldDecl *FD) {
944 int64_t Offset = 0;
945
946 if (GetFieldOffset(Ctx, RD, FD, Offset))
947 return std::optional<int64_t>(Offset);
948
949 return std::nullopt;
950}
951
952llvm::Value *CodeGenFunction::emitCountedBySize(const Expr *E,
953 llvm::Value *EmittedE,
954 unsigned Type,
955 llvm::IntegerType *ResType) {
956 // Note: If the whole struct is specificed in the __bdos (i.e. Visitor
957 // returns a DeclRefExpr). The calculation of the whole size of the structure
958 // with a flexible array member can be done in two ways:
959 //
960 // 1) sizeof(struct S) + count * sizeof(typeof(fam))
961 // 2) offsetof(struct S, fam) + count * sizeof(typeof(fam))
962 //
963 // The first will add additional padding after the end of the array
964 // allocation while the second method is more precise, but not quite expected
965 // from programmers. See
966 // https://lore.kernel.org/lkml/ZvV6X5FPBBW7CO1f@archlinux/ for a discussion
967 // of the topic.
968 //
969 // GCC isn't (currently) able to calculate __bdos on a pointer to the whole
970 // structure. Therefore, because of the above issue, we choose to match what
971 // GCC does for consistency's sake.
972
973 const Expr *Idx = nullptr;
974 // FIXME: `ArrayElementTy` is misleadingly named. `findStructFieldAccess()`
975 // sets it to the type of the array-subscript base, i.e. the (possibly cast)
976 // *pointer* being indexed (not an element type) or a null QualType when there
977 // is no subscript.
978 QualType ArrayElementTy;
979 E = findStructFieldAccess(E, OutArrayIndex: &Idx, OutArrayElementTy: &ArrayElementTy);
980 if (!E)
981 return nullptr;
982
983 if (Idx) {
984 if (Idx->HasSideEffects(Ctx: getContext()))
985 // We can't have side-effects.
986 return getDefaultBuiltinObjectSizeResult(Type, ResType);
987
988 if (const auto *IL = dyn_cast<IntegerLiteral>(Val: Idx)) {
989 int64_t Val = IL->getValue().getSExtValue();
990 if (Val < 0)
991 return getDefaultBuiltinObjectSizeResult(Type, ResType);
992
993 // The index is 0, so we don't need to take it into account.
994 if (Val == 0)
995 Idx = nullptr;
996 }
997 }
998
999 // __counted_by on either a flexible array member or a pointer into a struct
1000 // with a flexible array member.
1001 if (const auto *ME = dyn_cast<MemberExpr>(Val: E))
1002 return emitCountedByMemberSize(E: ME, Idx, EmittedE, CastedArrayElementTy: ArrayElementTy, Type,
1003 ResType);
1004
1005 // __counted_by on a pointer in a struct.
1006 if (const auto *ICE = dyn_cast<ImplicitCastExpr>(Val: E);
1007 ICE && ICE->getCastKind() == CK_LValueToRValue)
1008 return emitCountedByPointerSize(E: ICE, Idx, EmittedE, CastedArrayElementTy: ArrayElementTy, Type,
1009 ResType);
1010
1011 return nullptr;
1012}
1013
1014static llvm::Value *EmitPositiveResultOrZero(CodeGenFunction &CGF,
1015 llvm::Value *Res,
1016 llvm::Value *Index,
1017 llvm::IntegerType *ResType,
1018 bool IsSigned) {
1019 // cmp = (array_size >= 0)
1020 Value *Cmp = CGF.Builder.CreateIsNotNeg(Arg: Res);
1021 if (Index)
1022 // cmp = (cmp && index >= 0)
1023 Cmp = CGF.Builder.CreateAnd(LHS: CGF.Builder.CreateIsNotNeg(Arg: Index), RHS: Cmp);
1024
1025 // return cmp ? result : 0
1026 return CGF.Builder.CreateSelect(C: Cmp, True: Res,
1027 False: ConstantInt::get(Ty: ResType, V: 0, IsSigned));
1028}
1029
1030static std::pair<llvm::Value *, llvm::Value *>
1031GetCountFieldAndIndex(CodeGenFunction &CGF, const MemberExpr *ME,
1032 const FieldDecl *ArrayFD, const FieldDecl *CountFD,
1033 const Expr *Idx, llvm::IntegerType *ResType,
1034 bool IsSigned) {
1035 // count = ptr->count;
1036 Value *Count = CGF.EmitLoadOfCountedByField(Base: ME, FD: ArrayFD, CountDecl: CountFD);
1037 if (!Count)
1038 return std::make_pair<Value *>(x: nullptr, y: nullptr);
1039 Count = CGF.Builder.CreateIntCast(V: Count, DestTy: ResType, isSigned: IsSigned, Name: "count");
1040
1041 // index = idx;
1042 Value *Index = nullptr;
1043 if (Idx) {
1044 bool IdxSigned = Idx->getType()->isSignedIntegerType();
1045 Index = CGF.EmitScalarExpr(E: Idx);
1046 Index = CGF.Builder.CreateIntCast(V: Index, DestTy: ResType, isSigned: IdxSigned, Name: "index");
1047 }
1048
1049 return std::make_pair(x&: Count, y&: Index);
1050}
1051
1052llvm::Value *CodeGenFunction::emitCountedByPointerSize(
1053 const ImplicitCastExpr *E, const Expr *Idx, llvm::Value *EmittedE,
1054 QualType CastedArrayElementTy, unsigned Type, llvm::IntegerType *ResType) {
1055 assert(E->getCastKind() == CK_LValueToRValue &&
1056 "must be an LValue to RValue cast");
1057 assert(EmittedE && "emitted must not be null");
1058
1059 const MemberExpr *ME =
1060 dyn_cast<MemberExpr>(Val: E->getSubExpr()->IgnoreParenNoopCasts(Ctx: getContext()));
1061 if (!ME)
1062 return nullptr;
1063
1064 const auto *ArrayBaseFD = dyn_cast<FieldDecl>(Val: ME->getMemberDecl());
1065 if (!ArrayBaseFD || !ArrayBaseFD->getType()->isPointerType() ||
1066 !ArrayBaseFD->getType()->isCountAttributedType())
1067 return nullptr;
1068
1069 // Get the 'count' FieldDecl.
1070 const FieldDecl *CountFD = ArrayBaseFD->findCountedByField();
1071 if (!CountFD)
1072 // Can't find the field referenced by the "counted_by" attribute.
1073 return nullptr;
1074
1075 // Calculate the array's object size using these formulae. (Note: if the
1076 // calculation is negative, we return 0.):
1077 //
1078 // struct p;
1079 // struct s {
1080 // /* ... */
1081 // struct p **array __attribute__((ATTR(count)));
1082 // int count;
1083 // };
1084 //
1085 // 1) 'ptr->array':
1086 //
1087 // #if ATTR is counted_by_or_null || ATTR is sized_by_or_null
1088 // count = ptr->array ? ptr->count : 0;
1089 // #else
1090 // count = ptr->count;
1091 // #endif
1092 //
1093 // #if ATTR is counted_by || ATTR is counted_by_or_null
1094 // array_element_size = sizeof (*ptr->array);
1095 // array_size = count * array_element_size;
1096 // #else
1097 // array_size = count;
1098 // #endif
1099 //
1100 // result = array_size;
1101 //
1102 // cmp = (result >= 0)
1103 // return cmp ? result : 0;
1104 //
1105 // 2) '&((cast) ptr->array)[idx]':
1106 //
1107 // #if ATTR is counted_by_or_null || ATTR is sized_by_or_null
1108 // count = ptr->array ? ptr->count : 0;
1109 // #else
1110 // count = ptr->count;
1111 // #endif
1112 // index = idx;
1113 //
1114 // #if ATTR is counted_by || ATTR is counted_by_or_null
1115 // array_element_size = sizeof (*ptr->array);
1116 // array_size = count * array_element_size;
1117 // #else
1118 // array_size = count;
1119 // #endif
1120 //
1121 // casted_array_element_size = sizeof (*((cast) ptr->array));
1122 //
1123 // index_size = index * casted_array_element_size;
1124 // result = array_size - index_size;
1125 //
1126 // cmp = (result >= 0)
1127 // if (index)
1128 // cmp = (cmp && index > 0)
1129 // return cmp ? result : 0;
1130
1131 auto GetPointeeSize = [&](QualType PtrTy) -> CharUnits {
1132 assert(!PtrTy.isNull());
1133 QualType PointeeTy = PtrTy->getPointeeType();
1134 assert(!PointeeTy.isNull() &&
1135 (PointeeTy->isVoidType() || !PointeeTy->isIncompleteType()) &&
1136 "pointee type must have a computable size");
1137
1138 CharUnits PointeeSize = getContext().getTypeSizeInChars(T: PointeeTy);
1139 if (PointeeSize.isZero()) {
1140 // Support GNU extension of treating `void` having size 1.
1141 PointeeSize = CharUnits::One();
1142 }
1143
1144 return PointeeSize;
1145 };
1146
1147 bool IsSigned = CountFD->getType()->isSignedIntegerType();
1148 const auto *CountAttributedTy =
1149 ArrayBaseFD->getType()->getAs<CountAttributedType>();
1150 assert(CountAttributedTy && "the field's type is not a CountAttributedType");
1151
1152 // count = ptr->count;
1153 // index = idx;
1154 Value *Count, *Index;
1155 std::tie(args&: Count, args&: Index) = GetCountFieldAndIndex(
1156 CGF&: *this, ME, ArrayFD: ArrayBaseFD, CountFD, Idx, ResType, IsSigned);
1157 if (!Count)
1158 return nullptr;
1159
1160 // For the _or_null variants, a null pointer describes no accessible memory:
1161 // count = ptr->array ? count : 0;
1162 if (CountAttributedTy->isOrNull()) {
1163 Value *Ptr = nullptr;
1164 if (!Idx) {
1165 // 1) 'ptr->array'
1166 // Reuse the already-emitted pointer value rather than re-loading `ME`.
1167 // Re-loading would produce a second, observable access for a volatile
1168 // pointer field
1169 Ptr = EmittedE;
1170 } else {
1171 // 2) '&((cast) ptr->array)[idx]'
1172 // FIXME: `EmittedE` is the element address, not `ptr->array`, so we fall
1173 // back to re-emitting `ME` and the pointer field is loaded twice. This is
1174 // normally harmless except when the pointer is `volatile`. Avoiding that
1175 // would require restructuring how the base pointer is emitted (it is
1176 // handled elsewhere in the callstack), so it is left as-is for now.
1177 Ptr = EmitScalarExpr(E: ME);
1178 }
1179 Value *IsNull = Builder.CreateIsNull(Arg: Ptr);
1180 Count = Builder.CreateSelect(C: IsNull, True: ConstantInt::get(Ty: ResType, V: 0, IsSigned),
1181 False: Count, Name: "count.or.null");
1182 }
1183
1184 // #if ATTR is counted_by || ATTR is counted_by_or_null
1185 // array_element_size = sizeof (*ptr->array);
1186 // array_size = count * array_element_size;
1187 // #else
1188 // array_size = count;
1189 // #endif
1190 Value *ArraySize;
1191 if (!CountAttributedTy->isCountInBytes()) {
1192 // `__counted_by`/`__counted_by_or_null` require a complete pointee at use
1193 // sites (enforced by Sema) so the element size is computable.
1194 CharUnits ArrayElementBaseSize = GetPointeeSize(ArrayBaseFD->getType());
1195
1196 // array_element_size = sizeof (*ptr->array)
1197 auto *ArrayElementSize = llvm::ConstantInt::get(
1198 Ty: ResType, V: ArrayElementBaseSize.getQuantity(), IsSigned);
1199
1200 // array_size = count * array_element_size;
1201 ArraySize = Builder.CreateMul(LHS: Count, RHS: ArrayElementSize, Name: "array_size",
1202 HasNUW: !IsSigned, HasNSW: IsSigned);
1203 } else {
1204 // array_size = count;
1205 ArraySize = Count;
1206 }
1207
1208 // Option (1) 'ptr->array'
1209 // result = array_size
1210 Value *Result = ArraySize;
1211
1212 if (Idx) { // Option (2) '&((cast) ptr->array)[idx]'
1213 // FIXME: CastedArrayElementTy is confusingly named. It's actually the base
1214 // expression of the ArraySubscriptExpr, not the element (pointee) type.
1215 CharUnits CastedArrayElementSizeInChars =
1216 GetPointeeSize(CastedArrayElementTy);
1217
1218 // casted_array_element_size = sizeof (*((cast) ptr->array));
1219 auto *CastedArrayElementSize = llvm::ConstantInt::get(
1220 Ty: ResType, V: CastedArrayElementSizeInChars.getQuantity(), IsSigned);
1221
1222 // index_size = index * casted_array_element_size;
1223 Value *IndexSize = Builder.CreateMul(LHS: Index, RHS: CastedArrayElementSize,
1224 Name: "index_size", HasNUW: !IsSigned, HasNSW: IsSigned);
1225
1226 // result = result - index_size;
1227 Result =
1228 Builder.CreateSub(LHS: Result, RHS: IndexSize, Name: "result", HasNUW: !IsSigned, HasNSW: IsSigned);
1229 }
1230
1231 return EmitPositiveResultOrZero(CGF&: *this, Res: Result, Index, ResType, IsSigned);
1232}
1233
1234llvm::Value *CodeGenFunction::emitCountedByMemberSize(
1235 const MemberExpr *ME, const Expr *Idx, llvm::Value *EmittedE,
1236 QualType CastedArrayElementTy, unsigned Type, llvm::IntegerType *ResType) {
1237 const auto *FD = dyn_cast<FieldDecl>(Val: ME->getMemberDecl());
1238 if (!FD)
1239 return nullptr;
1240
1241 // Find the flexible array member and check that it has the __counted_by
1242 // attribute.
1243 ASTContext &Ctx = getContext();
1244 const RecordDecl *RD = FD->getDeclContext()->getOuterLexicalRecordContext();
1245 const FieldDecl *FlexibleArrayMemberFD = nullptr;
1246
1247 if (Decl::isFlexibleArrayMemberLike(
1248 Context: Ctx, D: FD, Ty: FD->getType(), StrictFlexArraysLevel: getLangOpts().getStrictFlexArraysLevel(),
1249 /*IgnoreTemplateOrMacroSubstitution=*/true))
1250 FlexibleArrayMemberFD = FD;
1251 else
1252 FlexibleArrayMemberFD = FindFlexibleArrayMemberField(CGF&: *this, Ctx, RD);
1253
1254 if (!FlexibleArrayMemberFD ||
1255 !FlexibleArrayMemberFD->getType()->isCountAttributedType())
1256 return nullptr;
1257
1258 // Get the 'count' FieldDecl.
1259 const FieldDecl *CountFD = FlexibleArrayMemberFD->findCountedByField();
1260 if (!CountFD)
1261 // Can't find the field referenced by the "counted_by" attribute.
1262 return nullptr;
1263
1264 // Calculate the flexible array member's object size using these formulae.
1265 // (Note: if the calculation is negative, we return 0.):
1266 //
1267 // struct p;
1268 // struct s {
1269 // /* ... */
1270 // int count;
1271 // struct p *array[] __attribute__((counted_by(count)));
1272 // };
1273 //
1274 // 1) 'ptr->array':
1275 //
1276 // count = ptr->count;
1277 //
1278 // flexible_array_member_element_size = sizeof (*ptr->array);
1279 // flexible_array_member_size =
1280 // count * flexible_array_member_element_size;
1281 //
1282 // result = flexible_array_member_size;
1283 //
1284 // cmp = (result >= 0)
1285 // return cmp ? result : 0;
1286 //
1287 // 2) '&((cast) ptr->array)[idx]':
1288 //
1289 // count = ptr->count;
1290 // index = idx;
1291 //
1292 // flexible_array_member_element_size = sizeof (*ptr->array);
1293 // flexible_array_member_size =
1294 // count * flexible_array_member_element_size;
1295 //
1296 // casted_flexible_array_member_element_size =
1297 // sizeof (*((cast) ptr->array));
1298 // index_size = index * casted_flexible_array_member_element_size;
1299 //
1300 // result = flexible_array_member_size - index_size;
1301 //
1302 // cmp = (result >= 0)
1303 // if (index != 0)
1304 // cmp = (cmp && index >= 0)
1305 // return cmp ? result : 0;
1306 //
1307 // 3) '&ptr->field':
1308 //
1309 // count = ptr->count;
1310 // sizeof_struct = sizeof (struct s);
1311 //
1312 // flexible_array_member_element_size = sizeof (*ptr->array);
1313 // flexible_array_member_size =
1314 // count * flexible_array_member_element_size;
1315 //
1316 // field_offset = offsetof (struct s, field);
1317 // offset_diff = sizeof_struct - field_offset;
1318 //
1319 // result = offset_diff + flexible_array_member_size;
1320 //
1321 // cmp = (result >= 0)
1322 // return cmp ? result : 0;
1323 //
1324 // 4) '&((cast) ptr->field_array)[idx]':
1325 //
1326 // count = ptr->count;
1327 // index = idx;
1328 // sizeof_struct = sizeof (struct s);
1329 //
1330 // flexible_array_member_element_size = sizeof (*ptr->array);
1331 // flexible_array_member_size =
1332 // count * flexible_array_member_element_size;
1333 //
1334 // casted_field_element_size = sizeof (*((cast) ptr->field_array));
1335 // field_offset = offsetof (struct s, field)
1336 // field_offset += index * casted_field_element_size;
1337 //
1338 // offset_diff = sizeof_struct - field_offset;
1339 //
1340 // result = offset_diff + flexible_array_member_size;
1341 //
1342 // cmp = (result >= 0)
1343 // if (index != 0)
1344 // cmp = (cmp && index >= 0)
1345 // return cmp ? result : 0;
1346
1347 bool IsSigned = CountFD->getType()->isSignedIntegerType();
1348
1349 QualType FlexibleArrayMemberTy = FlexibleArrayMemberFD->getType();
1350
1351 // Explicit cast because otherwise the CharWidth will promote an i32's into
1352 // u64's leading to overflows.
1353 int64_t CharWidth = static_cast<int64_t>(CGM.getContext().getCharWidth());
1354
1355 // field_offset = offsetof (struct s, field);
1356 Value *FieldOffset = nullptr;
1357 if (FlexibleArrayMemberFD != FD) {
1358 std::optional<int64_t> Offset = GetFieldOffset(Ctx, RD, FD);
1359 if (!Offset)
1360 return nullptr;
1361 FieldOffset =
1362 llvm::ConstantInt::get(Ty: ResType, V: *Offset / CharWidth, IsSigned);
1363 }
1364
1365 // count = ptr->count;
1366 // index = ptr->index;
1367 Value *Count, *Index;
1368 std::tie(args&: Count, args&: Index) = GetCountFieldAndIndex(
1369 CGF&: *this, ME, ArrayFD: FlexibleArrayMemberFD, CountFD, Idx, ResType, IsSigned);
1370 if (!Count)
1371 return nullptr;
1372
1373 // flexible_array_member_element_size = sizeof (*ptr->array);
1374 const ArrayType *ArrayTy = Ctx.getAsArrayType(T: FlexibleArrayMemberTy);
1375 CharUnits BaseSize = Ctx.getTypeSizeInChars(T: ArrayTy->getElementType());
1376 auto *FlexibleArrayMemberElementSize =
1377 llvm::ConstantInt::get(Ty: ResType, V: BaseSize.getQuantity(), IsSigned);
1378
1379 // flexible_array_member_size = count * flexible_array_member_element_size;
1380 Value *FlexibleArrayMemberSize =
1381 Builder.CreateMul(LHS: Count, RHS: FlexibleArrayMemberElementSize,
1382 Name: "flexible_array_member_size", HasNUW: !IsSigned, HasNSW: IsSigned);
1383
1384 Value *Result = nullptr;
1385 if (FlexibleArrayMemberFD == FD) {
1386 if (Idx) { // Option (2) '&((cast) ptr->array)[idx]'
1387 // casted_flexible_array_member_element_size =
1388 // sizeof (*((cast) ptr->array));
1389 llvm::ConstantInt *CastedFlexibleArrayMemberElementSize =
1390 FlexibleArrayMemberElementSize;
1391 if (!CastedArrayElementTy.isNull() &&
1392 CastedArrayElementTy->isPointerType()) {
1393 CharUnits BaseSize =
1394 Ctx.getTypeSizeInChars(T: CastedArrayElementTy->getPointeeType());
1395 CastedFlexibleArrayMemberElementSize =
1396 llvm::ConstantInt::get(Ty: ResType, V: BaseSize.getQuantity(), IsSigned);
1397 }
1398
1399 // index_size = index * casted_flexible_array_member_element_size;
1400 Value *IndexSize =
1401 Builder.CreateMul(LHS: Index, RHS: CastedFlexibleArrayMemberElementSize,
1402 Name: "index_size", HasNUW: !IsSigned, HasNSW: IsSigned);
1403
1404 // result = flexible_array_member_size - index_size;
1405 Result = Builder.CreateSub(LHS: FlexibleArrayMemberSize, RHS: IndexSize, Name: "result",
1406 HasNUW: !IsSigned, HasNSW: IsSigned);
1407 } else { // Option (1) 'ptr->array'
1408 // result = flexible_array_member_size;
1409 Result = FlexibleArrayMemberSize;
1410 }
1411 } else {
1412 // sizeof_struct = sizeof (struct s);
1413 llvm::StructType *StructTy = getTypes().getCGRecordLayout(RD).getLLVMType();
1414 const llvm::DataLayout &Layout = CGM.getDataLayout();
1415 TypeSize Size = Layout.getTypeSizeInBits(Ty: StructTy);
1416 Value *SizeofStruct =
1417 llvm::ConstantInt::get(Ty: ResType, V: Size.getKnownMinValue() / CharWidth);
1418
1419 if (Idx) { // Option (4) '&((cast) ptr->field_array)[idx]'
1420 // casted_field_element_size = sizeof (*((cast) ptr->field_array));
1421 CharUnits BaseSize;
1422 if (!CastedArrayElementTy.isNull() &&
1423 CastedArrayElementTy->isPointerType()) {
1424 BaseSize =
1425 Ctx.getTypeSizeInChars(T: CastedArrayElementTy->getPointeeType());
1426 } else {
1427 const ArrayType *ArrayTy = Ctx.getAsArrayType(T: FD->getType());
1428 BaseSize = Ctx.getTypeSizeInChars(T: ArrayTy->getElementType());
1429 }
1430
1431 llvm::ConstantInt *CastedFieldElementSize =
1432 llvm::ConstantInt::get(Ty: ResType, V: BaseSize.getQuantity(), IsSigned);
1433
1434 // field_offset += index * casted_field_element_size;
1435 Value *Mul = Builder.CreateMul(LHS: Index, RHS: CastedFieldElementSize,
1436 Name: "field_offset", HasNUW: !IsSigned, HasNSW: IsSigned);
1437 FieldOffset = Builder.CreateAdd(LHS: FieldOffset, RHS: Mul);
1438 }
1439 // Option (3) '&ptr->field', and Option (4) continuation.
1440 // offset_diff = flexible_array_member_offset - field_offset;
1441 Value *OffsetDiff = Builder.CreateSub(LHS: SizeofStruct, RHS: FieldOffset,
1442 Name: "offset_diff", HasNUW: !IsSigned, HasNSW: IsSigned);
1443
1444 // result = offset_diff + flexible_array_member_size;
1445 Result = Builder.CreateAdd(LHS: FlexibleArrayMemberSize, RHS: OffsetDiff, Name: "result");
1446 }
1447
1448 return EmitPositiveResultOrZero(CGF&: *this, Res: Result, Index, ResType, IsSigned);
1449}
1450
1451/// Returns a Value corresponding to the size of the given expression.
1452/// This Value may be either of the following:
1453/// - A llvm::Argument (if E is a param with the pass_object_size attribute on
1454/// it)
1455/// - A call to the @llvm.objectsize intrinsic
1456///
1457/// EmittedE is the result of emitting `E` as a scalar expr. If it's non-null
1458/// and we wouldn't otherwise try to reference a pass_object_size parameter,
1459/// we'll call @llvm.objectsize on EmittedE, rather than emitting E.
1460llvm::Value *
1461CodeGenFunction::emitBuiltinObjectSize(const Expr *E, unsigned Type,
1462 llvm::IntegerType *ResType,
1463 llvm::Value *EmittedE, bool IsDynamic) {
1464 // We need to reference an argument if the pointer is a parameter with the
1465 // pass_object_size attribute.
1466 if (auto *D = dyn_cast<DeclRefExpr>(Val: E->IgnoreParenImpCasts())) {
1467 auto *Param = dyn_cast<ParmVarDecl>(Val: D->getDecl());
1468 auto *PS = D->getDecl()->getAttr<PassObjectSizeAttr>();
1469 if (Param != nullptr && PS != nullptr &&
1470 areBOSTypesCompatible(From: PS->getType(), To: Type)) {
1471 auto Iter = SizeArguments.find(Val: Param);
1472 assert(Iter != SizeArguments.end());
1473
1474 const ImplicitParamDecl *D = Iter->second;
1475 auto DIter = LocalDeclMap.find(Val: D);
1476 assert(DIter != LocalDeclMap.end());
1477
1478 return EmitLoadOfScalar(Addr: DIter->second, /*Volatile=*/false,
1479 Ty: getContext().getSizeType(), Loc: E->getBeginLoc());
1480 }
1481 }
1482
1483 // LLVM can't handle Type=3 appropriately, and __builtin_object_size shouldn't
1484 // evaluate E for side-effects. In either case, we shouldn't lower to
1485 // @llvm.objectsize.
1486 if (Type == 3 || (!EmittedE && E->HasSideEffects(Ctx: getContext())))
1487 return getDefaultBuiltinObjectSizeResult(Type, ResType);
1488
1489 Value *Ptr = EmittedE ? EmittedE : EmitScalarExpr(E);
1490 assert(Ptr->getType()->isPointerTy() &&
1491 "Non-pointer passed to __builtin_object_size?");
1492
1493 if (IsDynamic)
1494 // Emit special code for a flexible array member with the "counted_by"
1495 // attribute.
1496 if (Value *V = emitCountedBySize(E, EmittedE: Ptr, Type, ResType))
1497 return V;
1498
1499 Function *F =
1500 CGM.getIntrinsic(IID: Intrinsic::objectsize, Tys: {ResType, Ptr->getType()});
1501
1502 // LLVM only supports 0 and 2, make sure that we pass along that as a boolean.
1503 Value *Min = Builder.getInt1(V: (Type & 2) != 0);
1504 // For GCC compatibility, __builtin_object_size treat NULL as unknown size.
1505 Value *NullIsUnknown = Builder.getTrue();
1506 Value *Dynamic = Builder.getInt1(V: IsDynamic);
1507 return Builder.CreateCall(Callee: F, Args: {Ptr, Min, NullIsUnknown, Dynamic});
1508}
1509
1510namespace {
1511/// A struct to generically describe a bit test intrinsic.
1512struct BitTest {
1513 enum ActionKind : uint8_t { TestOnly, Complement, Reset, Set };
1514 enum InterlockingKind : uint8_t {
1515 Unlocked,
1516 Sequential,
1517 Acquire,
1518 Release,
1519 NoFence
1520 };
1521
1522 ActionKind Action;
1523 InterlockingKind Interlocking;
1524 bool Is64Bit;
1525
1526 static BitTest decodeBitTestBuiltin(unsigned BuiltinID);
1527};
1528
1529} // namespace
1530
1531BitTest BitTest::decodeBitTestBuiltin(unsigned BuiltinID) {
1532 switch (BuiltinID) {
1533 // Main portable variants.
1534 case Builtin::BI_bittest:
1535 return {.Action: TestOnly, .Interlocking: Unlocked, .Is64Bit: false};
1536 case Builtin::BI_bittestandcomplement:
1537 return {.Action: Complement, .Interlocking: Unlocked, .Is64Bit: false};
1538 case Builtin::BI_bittestandreset:
1539 return {.Action: Reset, .Interlocking: Unlocked, .Is64Bit: false};
1540 case Builtin::BI_bittestandset:
1541 return {.Action: Set, .Interlocking: Unlocked, .Is64Bit: false};
1542 case Builtin::BI_interlockedbittestandreset:
1543 return {.Action: Reset, .Interlocking: Sequential, .Is64Bit: false};
1544 case Builtin::BI_interlockedbittestandset:
1545 return {.Action: Set, .Interlocking: Sequential, .Is64Bit: false};
1546
1547 // 64-bit variants.
1548 case Builtin::BI_bittest64:
1549 return {.Action: TestOnly, .Interlocking: Unlocked, .Is64Bit: true};
1550 case Builtin::BI_bittestandcomplement64:
1551 return {.Action: Complement, .Interlocking: Unlocked, .Is64Bit: true};
1552 case Builtin::BI_bittestandreset64:
1553 return {.Action: Reset, .Interlocking: Unlocked, .Is64Bit: true};
1554 case Builtin::BI_bittestandset64:
1555 return {.Action: Set, .Interlocking: Unlocked, .Is64Bit: true};
1556 case Builtin::BI_interlockedbittestandreset64:
1557 return {.Action: Reset, .Interlocking: Sequential, .Is64Bit: true};
1558 case Builtin::BI_interlockedbittestandset64:
1559 return {.Action: Set, .Interlocking: Sequential, .Is64Bit: true};
1560
1561 // ARM/AArch64-specific ordering variants.
1562 case Builtin::BI_interlockedbittestandset_acq:
1563 return {.Action: Set, .Interlocking: Acquire, .Is64Bit: false};
1564 case Builtin::BI_interlockedbittestandset_rel:
1565 return {.Action: Set, .Interlocking: Release, .Is64Bit: false};
1566 case Builtin::BI_interlockedbittestandset_nf:
1567 return {.Action: Set, .Interlocking: NoFence, .Is64Bit: false};
1568 case Builtin::BI_interlockedbittestandreset_acq:
1569 return {.Action: Reset, .Interlocking: Acquire, .Is64Bit: false};
1570 case Builtin::BI_interlockedbittestandreset_rel:
1571 return {.Action: Reset, .Interlocking: Release, .Is64Bit: false};
1572 case Builtin::BI_interlockedbittestandreset_nf:
1573 return {.Action: Reset, .Interlocking: NoFence, .Is64Bit: false};
1574 case Builtin::BI_interlockedbittestandreset64_acq:
1575 return {.Action: Reset, .Interlocking: Acquire, .Is64Bit: false};
1576 case Builtin::BI_interlockedbittestandreset64_rel:
1577 return {.Action: Reset, .Interlocking: Release, .Is64Bit: false};
1578 case Builtin::BI_interlockedbittestandreset64_nf:
1579 return {.Action: Reset, .Interlocking: NoFence, .Is64Bit: false};
1580 case Builtin::BI_interlockedbittestandset64_acq:
1581 return {.Action: Set, .Interlocking: Acquire, .Is64Bit: false};
1582 case Builtin::BI_interlockedbittestandset64_rel:
1583 return {.Action: Set, .Interlocking: Release, .Is64Bit: false};
1584 case Builtin::BI_interlockedbittestandset64_nf:
1585 return {.Action: Set, .Interlocking: NoFence, .Is64Bit: false};
1586 }
1587 llvm_unreachable("expected only bittest intrinsics");
1588}
1589
1590static char bitActionToX86BTCode(BitTest::ActionKind A) {
1591 switch (A) {
1592 case BitTest::TestOnly: return '\0';
1593 case BitTest::Complement: return 'c';
1594 case BitTest::Reset: return 'r';
1595 case BitTest::Set: return 's';
1596 }
1597 llvm_unreachable("invalid action");
1598}
1599
1600static llvm::Value *EmitX86BitTestIntrinsic(CodeGenFunction &CGF,
1601 BitTest BT,
1602 const CallExpr *E, Value *BitBase,
1603 Value *BitPos) {
1604 char Action = bitActionToX86BTCode(A: BT.Action);
1605 char SizeSuffix = BT.Is64Bit ? 'q' : 'l';
1606
1607 // Build the assembly.
1608 SmallString<64> Asm;
1609 raw_svector_ostream AsmOS(Asm);
1610 if (BT.Interlocking != BitTest::Unlocked)
1611 AsmOS << "lock ";
1612 AsmOS << "bt";
1613 if (Action)
1614 AsmOS << Action;
1615 AsmOS << SizeSuffix << " $2, ($1)";
1616
1617 // Build the constraints. FIXME: We should support immediates when possible.
1618 std::string Constraints = "={@ccc},r,r,~{cc},~{memory}";
1619 std::string_view MachineClobbers = CGF.getTarget().getClobbers();
1620 if (!MachineClobbers.empty()) {
1621 Constraints += ',';
1622 Constraints += MachineClobbers;
1623 }
1624 llvm::IntegerType *IntType = llvm::IntegerType::get(
1625 C&: CGF.getLLVMContext(),
1626 NumBits: CGF.getContext().getTypeSize(T: E->getArg(Arg: 1)->getType()));
1627 llvm::FunctionType *FTy =
1628 llvm::FunctionType::get(Result: CGF.Int8Ty, Params: {CGF.DefaultPtrTy, IntType}, isVarArg: false);
1629
1630 llvm::InlineAsm *IA =
1631 llvm::InlineAsm::get(Ty: FTy, AsmString: Asm, Constraints, /*hasSideEffects=*/true);
1632 return CGF.Builder.CreateCall(Callee: IA, Args: {BitBase, BitPos});
1633}
1634
1635static llvm::AtomicOrdering
1636getBitTestAtomicOrdering(BitTest::InterlockingKind I) {
1637 switch (I) {
1638 case BitTest::Unlocked: return llvm::AtomicOrdering::NotAtomic;
1639 case BitTest::Sequential: return llvm::AtomicOrdering::SequentiallyConsistent;
1640 case BitTest::Acquire: return llvm::AtomicOrdering::Acquire;
1641 case BitTest::Release: return llvm::AtomicOrdering::Release;
1642 case BitTest::NoFence: return llvm::AtomicOrdering::Monotonic;
1643 }
1644 llvm_unreachable("invalid interlocking");
1645}
1646
1647static llvm::Value *EmitBitCountExpr(CodeGenFunction &CGF, const Expr *E) {
1648 llvm::Value *ArgValue = CGF.EmitScalarExpr(E);
1649 llvm::Type *ArgType = ArgValue->getType();
1650
1651 // Boolean vectors can be casted directly to its bitfield representation. We
1652 // intentionally do not round up to the next power of two size and let LLVM
1653 // handle the trailing bits.
1654 //
1655 // In big endian mode, the bitfield representation has a reversed bit order,
1656 // hence the need to add an operation to reverse it back to the expected
1657 // order.
1658 if (auto *VT = dyn_cast<llvm::FixedVectorType>(Val: ArgType);
1659 VT && VT->getElementType()->isIntegerTy(BitWidth: 1)) {
1660 llvm::Type *StorageType =
1661 llvm::Type::getIntNTy(C&: CGF.getLLVMContext(), N: VT->getNumElements());
1662 ArgValue = CGF.Builder.CreateBitCast(V: ArgValue, DestTy: StorageType);
1663
1664 if (CGF.getTarget().isBigEndian())
1665 ArgValue = CGF.Builder.CreateIntrinsic(ID: Intrinsic::bitreverse,
1666 OverloadTypes: {StorageType}, Args: ArgValue);
1667 }
1668
1669 return ArgValue;
1670}
1671
1672/// Emit a _bittest* intrinsic. These intrinsics take a pointer to an array of
1673/// bits and a bit position and read and optionally modify the bit at that
1674/// position. The position index can be arbitrarily large, i.e. it can be larger
1675/// than 31 or 63, so we need an indexed load in the general case.
1676static llvm::Value *EmitBitTestIntrinsic(CodeGenFunction &CGF,
1677 unsigned BuiltinID,
1678 const CallExpr *E) {
1679 Value *BitBase = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
1680 Value *BitPos = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
1681
1682 BitTest BT = BitTest::decodeBitTestBuiltin(BuiltinID);
1683
1684 // X86 has special BT, BTC, BTR, and BTS instructions that handle the array
1685 // indexing operation internally. Use them if possible.
1686 if (CGF.getTarget().getTriple().isX86())
1687 return EmitX86BitTestIntrinsic(CGF, BT, E, BitBase, BitPos);
1688
1689 // Otherwise, use generic code to load one byte and test the bit. Use all but
1690 // the bottom three bits as the array index, and the bottom three bits to form
1691 // a mask.
1692 // Bit = BitBaseI8[BitPos >> 3] & (1 << (BitPos & 0x7)) != 0;
1693 Value *ByteIndex = CGF.Builder.CreateAShr(
1694 LHS: BitPos, RHS: llvm::ConstantInt::get(Ty: BitPos->getType(), V: 3), Name: "bittest.byteidx");
1695 Address ByteAddr(CGF.Builder.CreateInBoundsGEP(Ty: CGF.Int8Ty, Ptr: BitBase, IdxList: ByteIndex,
1696 Name: "bittest.byteaddr"),
1697 CGF.Int8Ty, CharUnits::One());
1698 Value *PosLow =
1699 CGF.Builder.CreateAnd(LHS: CGF.Builder.CreateTrunc(V: BitPos, DestTy: CGF.Int8Ty),
1700 RHS: llvm::ConstantInt::get(Ty: CGF.Int8Ty, V: 0x7));
1701
1702 // The updating instructions will need a mask.
1703 Value *Mask = nullptr;
1704 if (BT.Action != BitTest::TestOnly) {
1705 Mask = CGF.Builder.CreateShl(LHS: llvm::ConstantInt::get(Ty: CGF.Int8Ty, V: 1), RHS: PosLow,
1706 Name: "bittest.mask");
1707 }
1708
1709 // Check the action and ordering of the interlocked intrinsics.
1710 llvm::AtomicOrdering Ordering = getBitTestAtomicOrdering(I: BT.Interlocking);
1711
1712 Value *OldByte = nullptr;
1713 if (Ordering != llvm::AtomicOrdering::NotAtomic) {
1714 // Emit a combined atomicrmw load/store operation for the interlocked
1715 // intrinsics.
1716 llvm::AtomicRMWInst::BinOp RMWOp = llvm::AtomicRMWInst::Or;
1717 if (BT.Action == BitTest::Reset) {
1718 Mask = CGF.Builder.CreateNot(V: Mask);
1719 RMWOp = llvm::AtomicRMWInst::And;
1720 }
1721 OldByte = CGF.Builder.CreateAtomicRMW(Op: RMWOp, Addr: ByteAddr, Val: Mask, Ordering);
1722 } else {
1723 // Emit a plain load for the non-interlocked intrinsics.
1724 OldByte = CGF.Builder.CreateLoad(Addr: ByteAddr, Name: "bittest.byte");
1725 Value *NewByte = nullptr;
1726 switch (BT.Action) {
1727 case BitTest::TestOnly:
1728 // Don't store anything.
1729 break;
1730 case BitTest::Complement:
1731 NewByte = CGF.Builder.CreateXor(LHS: OldByte, RHS: Mask);
1732 break;
1733 case BitTest::Reset:
1734 NewByte = CGF.Builder.CreateAnd(LHS: OldByte, RHS: CGF.Builder.CreateNot(V: Mask));
1735 break;
1736 case BitTest::Set:
1737 NewByte = CGF.Builder.CreateOr(LHS: OldByte, RHS: Mask);
1738 break;
1739 }
1740 if (NewByte)
1741 CGF.Builder.CreateStore(Val: NewByte, Addr: ByteAddr);
1742 }
1743
1744 // However we loaded the old byte, either by plain load or atomicrmw, shift
1745 // the bit into the low position and mask it to 0 or 1.
1746 Value *ShiftedByte = CGF.Builder.CreateLShr(LHS: OldByte, RHS: PosLow, Name: "bittest.shr");
1747 return CGF.Builder.CreateAnd(
1748 LHS: ShiftedByte, RHS: llvm::ConstantInt::get(Ty: CGF.Int8Ty, V: 1), Name: "bittest.res");
1749}
1750
1751namespace {
1752enum class MSVCSetJmpKind {
1753 _setjmpex,
1754 _setjmp3,
1755 _setjmp
1756};
1757}
1758
1759/// MSVC handles setjmp a bit differently on different platforms. On 32-bit x86
1760/// extra parameters can be passed as variadic arguments, but we always pass
1761/// none. Everywhere else a frame value is passed: the stack pointer as it was
1762/// on entry to the function for AArch64 and 32-bit Arm, and the frame address
1763/// for the rest.
1764static RValue EmitMSVCRTSetJmp(CodeGenFunction &CGF, MSVCSetJmpKind SJKind,
1765 const CallExpr *E) {
1766 llvm::Value *Arg1 = nullptr;
1767 llvm::Type *Arg1Ty = nullptr;
1768 StringRef Name;
1769 bool IsVarArg = false;
1770 if (SJKind == MSVCSetJmpKind::_setjmp3) {
1771 Name = "_setjmp3";
1772 Arg1Ty = CGF.Int32Ty;
1773 Arg1 = llvm::ConstantInt::get(Ty: CGF.IntTy, V: 0);
1774 IsVarArg = true;
1775 } else {
1776 Name = SJKind == MSVCSetJmpKind::_setjmp ? "_setjmp" : "_setjmpex";
1777 Arg1Ty = CGF.Int8PtrTy;
1778 const llvm::Triple &T = CGF.getTarget().getTriple();
1779 if (T.getArch() == llvm::Triple::aarch64 || T.isARM() || T.isThumb()) {
1780 Arg1 = CGF.Builder.CreateCall(
1781 Callee: CGF.CGM.getIntrinsic(IID: Intrinsic::sponentry, Tys: CGF.AllocaInt8PtrTy));
1782 } else
1783 Arg1 = CGF.Builder.CreateCall(
1784 Callee: CGF.CGM.getIntrinsic(IID: Intrinsic::frameaddress, Tys: CGF.AllocaInt8PtrTy),
1785 Args: llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: 0));
1786 }
1787
1788 // Mark the call site and declaration with ReturnsTwice.
1789 llvm::Type *ArgTypes[2] = {CGF.Int8PtrTy, Arg1Ty};
1790 llvm::AttributeList ReturnsTwiceAttr = llvm::AttributeList::get(
1791 C&: CGF.getLLVMContext(), Index: llvm::AttributeList::FunctionIndex,
1792 Kinds: llvm::Attribute::ReturnsTwice);
1793 llvm::FunctionCallee SetJmpFn = CGF.CGM.CreateRuntimeFunction(
1794 Ty: llvm::FunctionType::get(Result: CGF.IntTy, Params: ArgTypes, isVarArg: IsVarArg), Name,
1795 ExtraAttrs: ReturnsTwiceAttr, /*Local=*/true);
1796
1797 llvm::Value *Buf = CGF.Builder.CreateBitOrPointerCast(
1798 V: CGF.EmitScalarExpr(E: E->getArg(Arg: 0)), DestTy: CGF.Int8PtrTy);
1799 llvm::Value *Args[] = {Buf, Arg1};
1800 llvm::CallBase *CB = CGF.EmitRuntimeCallOrInvoke(callee: SetJmpFn, args: Args);
1801 CB->setAttributes(ReturnsTwiceAttr);
1802 return RValue::get(V: CB);
1803}
1804
1805// Emit an MSVC intrinsic. Assumes that arguments have *not* been evaluated.
1806Value *CodeGenFunction::EmitMSVCBuiltinExpr(MSVCIntrin BuiltinID,
1807 const CallExpr *E) {
1808 switch (BuiltinID) {
1809 case MSVCIntrin::_BitScanForward:
1810 case MSVCIntrin::_BitScanReverse: {
1811 Address IndexAddress(EmitPointerWithAlignment(Addr: E->getArg(Arg: 0)));
1812 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 1));
1813
1814 llvm::Type *ArgType = ArgValue->getType();
1815 llvm::Type *IndexType = IndexAddress.getElementType();
1816 llvm::Type *ResultType = ConvertType(T: E->getType());
1817
1818 Value *ArgZero = llvm::Constant::getNullValue(Ty: ArgType);
1819 Value *ResZero = llvm::Constant::getNullValue(Ty: ResultType);
1820 Value *ResOne = llvm::ConstantInt::get(Ty: ResultType, V: 1);
1821
1822 BasicBlock *Begin = Builder.GetInsertBlock();
1823 BasicBlock *End = createBasicBlock(name: "bitscan_end", parent: this->CurFn);
1824 Builder.SetInsertPoint(End);
1825 PHINode *Result = Builder.CreatePHI(Ty: ResultType, NumReservedValues: 2, Name: "bitscan_result");
1826
1827 Builder.SetInsertPoint(Begin);
1828 Value *IsZero = Builder.CreateICmpEQ(LHS: ArgValue, RHS: ArgZero);
1829 BasicBlock *NotZero = createBasicBlock(name: "bitscan_not_zero", parent: this->CurFn);
1830 Builder.CreateCondBr(Cond: IsZero, True: End, False: NotZero);
1831 Result->addIncoming(V: ResZero, BB: Begin);
1832
1833 Builder.SetInsertPoint(NotZero);
1834
1835 if (BuiltinID == MSVCIntrin::_BitScanForward) {
1836 Function *F = CGM.getIntrinsic(IID: Intrinsic::cttz, Tys: ArgType);
1837 Value *ZeroCount = Builder.CreateCall(Callee: F, Args: {ArgValue, Builder.getTrue()});
1838 ZeroCount = Builder.CreateIntCast(V: ZeroCount, DestTy: IndexType, isSigned: false);
1839 Builder.CreateStore(Val: ZeroCount, Addr: IndexAddress, IsVolatile: false);
1840 } else {
1841 unsigned ArgWidth = cast<llvm::IntegerType>(Val: ArgType)->getBitWidth();
1842 Value *ArgTypeLastIndex = llvm::ConstantInt::get(Ty: IndexType, V: ArgWidth - 1);
1843
1844 Function *F = CGM.getIntrinsic(IID: Intrinsic::ctlz, Tys: ArgType);
1845 Value *ZeroCount = Builder.CreateCall(Callee: F, Args: {ArgValue, Builder.getTrue()});
1846 ZeroCount = Builder.CreateIntCast(V: ZeroCount, DestTy: IndexType, isSigned: false);
1847 Value *Index = Builder.CreateNSWSub(LHS: ArgTypeLastIndex, RHS: ZeroCount);
1848 Builder.CreateStore(Val: Index, Addr: IndexAddress, IsVolatile: false);
1849 }
1850 Builder.CreateBr(Dest: End);
1851 Result->addIncoming(V: ResOne, BB: NotZero);
1852
1853 Builder.SetInsertPoint(End);
1854 return Result;
1855 }
1856 case MSVCIntrin::_InterlockedAnd:
1857 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::And, E);
1858 case MSVCIntrin::_InterlockedExchange:
1859 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Xchg, E);
1860 case MSVCIntrin::_InterlockedExchangeAdd:
1861 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Add, E);
1862 case MSVCIntrin::_InterlockedExchangeSub:
1863 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Sub, E);
1864 case MSVCIntrin::_InterlockedOr:
1865 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Or, E);
1866 case MSVCIntrin::_InterlockedXor:
1867 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Xor, E);
1868 case MSVCIntrin::_InterlockedExchangeAdd_acq:
1869 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Add, E,
1870 Ordering: AtomicOrdering::Acquire);
1871 case MSVCIntrin::_InterlockedExchangeAdd_rel:
1872 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Add, E,
1873 Ordering: AtomicOrdering::Release);
1874 case MSVCIntrin::_InterlockedExchangeAdd_nf:
1875 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Add, E,
1876 Ordering: AtomicOrdering::Monotonic);
1877 case MSVCIntrin::_InterlockedExchange_acq:
1878 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Xchg, E,
1879 Ordering: AtomicOrdering::Acquire);
1880 case MSVCIntrin::_InterlockedExchange_rel:
1881 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Xchg, E,
1882 Ordering: AtomicOrdering::Release);
1883 case MSVCIntrin::_InterlockedExchange_nf:
1884 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Xchg, E,
1885 Ordering: AtomicOrdering::Monotonic);
1886 case MSVCIntrin::_InterlockedCompareExchange:
1887 return EmitAtomicCmpXchgForMSIntrin(CGF&: *this, E);
1888 case MSVCIntrin::_InterlockedCompareExchange_acq:
1889 return EmitAtomicCmpXchgForMSIntrin(CGF&: *this, E, SuccessOrdering: AtomicOrdering::Acquire);
1890 case MSVCIntrin::_InterlockedCompareExchange_rel:
1891 return EmitAtomicCmpXchgForMSIntrin(CGF&: *this, E, SuccessOrdering: AtomicOrdering::Release);
1892 case MSVCIntrin::_InterlockedCompareExchange_nf:
1893 return EmitAtomicCmpXchgForMSIntrin(CGF&: *this, E, SuccessOrdering: AtomicOrdering::Monotonic);
1894 case MSVCIntrin::_InterlockedCompareExchange128:
1895 return EmitAtomicCmpXchg128ForMSIntrin(
1896 CGF&: *this, E, SuccessOrdering: AtomicOrdering::SequentiallyConsistent);
1897 case MSVCIntrin::_InterlockedCompareExchange128_acq:
1898 return EmitAtomicCmpXchg128ForMSIntrin(CGF&: *this, E, SuccessOrdering: AtomicOrdering::Acquire);
1899 case MSVCIntrin::_InterlockedCompareExchange128_rel:
1900 return EmitAtomicCmpXchg128ForMSIntrin(CGF&: *this, E, SuccessOrdering: AtomicOrdering::Release);
1901 case MSVCIntrin::_InterlockedCompareExchange128_nf:
1902 return EmitAtomicCmpXchg128ForMSIntrin(CGF&: *this, E, SuccessOrdering: AtomicOrdering::Monotonic);
1903 case MSVCIntrin::_InterlockedOr_acq:
1904 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Or, E,
1905 Ordering: AtomicOrdering::Acquire);
1906 case MSVCIntrin::_InterlockedOr_rel:
1907 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Or, E,
1908 Ordering: AtomicOrdering::Release);
1909 case MSVCIntrin::_InterlockedOr_nf:
1910 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Or, E,
1911 Ordering: AtomicOrdering::Monotonic);
1912 case MSVCIntrin::_InterlockedXor_acq:
1913 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Xor, E,
1914 Ordering: AtomicOrdering::Acquire);
1915 case MSVCIntrin::_InterlockedXor_rel:
1916 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Xor, E,
1917 Ordering: AtomicOrdering::Release);
1918 case MSVCIntrin::_InterlockedXor_nf:
1919 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Xor, E,
1920 Ordering: AtomicOrdering::Monotonic);
1921 case MSVCIntrin::_InterlockedAnd_acq:
1922 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::And, E,
1923 Ordering: AtomicOrdering::Acquire);
1924 case MSVCIntrin::_InterlockedAnd_rel:
1925 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::And, E,
1926 Ordering: AtomicOrdering::Release);
1927 case MSVCIntrin::_InterlockedAnd_nf:
1928 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::And, E,
1929 Ordering: AtomicOrdering::Monotonic);
1930 case MSVCIntrin::_InterlockedIncrement_acq:
1931 return EmitAtomicIncrementValue(CGF&: *this, E, Ordering: AtomicOrdering::Acquire);
1932 case MSVCIntrin::_InterlockedIncrement_rel:
1933 return EmitAtomicIncrementValue(CGF&: *this, E, Ordering: AtomicOrdering::Release);
1934 case MSVCIntrin::_InterlockedIncrement_nf:
1935 return EmitAtomicIncrementValue(CGF&: *this, E, Ordering: AtomicOrdering::Monotonic);
1936 case MSVCIntrin::_InterlockedDecrement_acq:
1937 return EmitAtomicDecrementValue(CGF&: *this, E, Ordering: AtomicOrdering::Acquire);
1938 case MSVCIntrin::_InterlockedDecrement_rel:
1939 return EmitAtomicDecrementValue(CGF&: *this, E, Ordering: AtomicOrdering::Release);
1940 case MSVCIntrin::_InterlockedDecrement_nf:
1941 return EmitAtomicDecrementValue(CGF&: *this, E, Ordering: AtomicOrdering::Monotonic);
1942
1943 case MSVCIntrin::_InterlockedDecrement:
1944 return EmitAtomicDecrementValue(CGF&: *this, E);
1945 case MSVCIntrin::_InterlockedIncrement:
1946 return EmitAtomicIncrementValue(CGF&: *this, E);
1947
1948 case MSVCIntrin::__fastfail: {
1949 // Request immediate process termination from the kernel. The instruction
1950 // sequences to do this are documented on MSDN:
1951 // https://msdn.microsoft.com/en-us/library/dn774154.aspx
1952 llvm::Triple::ArchType ISA = getTarget().getTriple().getArch();
1953 StringRef Asm, Constraints;
1954 switch (ISA) {
1955 default:
1956 ErrorUnsupported(S: E, Type: "__fastfail call for this architecture");
1957 break;
1958 case llvm::Triple::x86:
1959 case llvm::Triple::x86_64:
1960 Asm = "int $$0x29";
1961 Constraints = "{cx}";
1962 break;
1963 case llvm::Triple::thumb:
1964 Asm = "udf #251";
1965 Constraints = "{r0}";
1966 break;
1967 case llvm::Triple::aarch64:
1968 Asm = "brk #0xF003";
1969 Constraints = "{w0}";
1970 }
1971 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: VoidTy, Params: {Int32Ty}, isVarArg: false);
1972 llvm::InlineAsm *IA =
1973 llvm::InlineAsm::get(Ty: FTy, AsmString: Asm, Constraints, /*hasSideEffects=*/true);
1974 llvm::AttributeList NoReturnAttr = llvm::AttributeList::get(
1975 C&: getLLVMContext(), Index: llvm::AttributeList::FunctionIndex,
1976 Kinds: llvm::Attribute::NoReturn);
1977 llvm::CallInst *CI = Builder.CreateCall(Callee: IA, Args: EmitScalarExpr(E: E->getArg(Arg: 0)));
1978 CI->setAttributes(NoReturnAttr);
1979 return CI;
1980 }
1981 }
1982 llvm_unreachable("Incorrect MSVC intrinsic!");
1983}
1984
1985namespace {
1986// ARC cleanup for __builtin_os_log_format
1987struct CallObjCArcUse final : EHScopeStack::Cleanup {
1988 CallObjCArcUse(llvm::Value *object) : object(object) {}
1989 llvm::Value *object;
1990
1991 void Emit(CodeGenFunction &CGF, Flags flags) override {
1992 CGF.EmitARCIntrinsicUse(values: object);
1993 }
1994};
1995}
1996
1997Value *CodeGenFunction::EmitCheckedArgForBuiltin(const Expr *E,
1998 BuiltinCheckKind Kind) {
1999 assert((Kind == BCK_CLZPassedZero || Kind == BCK_CTZPassedZero) &&
2000 "Unsupported builtin check kind");
2001
2002 Value *ArgValue = EmitBitCountExpr(CGF&: *this, E);
2003 if (!SanOpts.has(K: SanitizerKind::Builtin))
2004 return ArgValue;
2005
2006 auto CheckOrdinal = SanitizerKind::SO_Builtin;
2007 auto CheckHandler = SanitizerHandler::InvalidBuiltin;
2008 SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);
2009 Value *Cond = Builder.CreateICmpNE(
2010 LHS: ArgValue, RHS: llvm::Constant::getNullValue(Ty: ArgValue->getType()));
2011 EmitCheck(Checked: std::make_pair(x&: Cond, y&: CheckOrdinal), Check: CheckHandler,
2012 StaticArgs: {EmitCheckSourceLocation(Loc: E->getExprLoc()),
2013 llvm::ConstantInt::get(Ty: Builder.getInt8Ty(), V: Kind)},
2014 DynamicArgs: {});
2015 return ArgValue;
2016}
2017
2018Value *CodeGenFunction::EmitCheckedArgForAssume(const Expr *E) {
2019 Value *ArgValue = EvaluateExprAsBool(E);
2020 if (!SanOpts.has(K: SanitizerKind::Builtin))
2021 return ArgValue;
2022
2023 auto CheckOrdinal = SanitizerKind::SO_Builtin;
2024 auto CheckHandler = SanitizerHandler::InvalidBuiltin;
2025 SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);
2026 EmitCheck(
2027 Checked: std::make_pair(x&: ArgValue, y&: CheckOrdinal), Check: CheckHandler,
2028 StaticArgs: {EmitCheckSourceLocation(Loc: E->getExprLoc()),
2029 llvm::ConstantInt::get(Ty: Builder.getInt8Ty(), V: BCK_AssumePassedFalse)},
2030 DynamicArgs: {});
2031 return ArgValue;
2032}
2033
2034static Value *EmitAbs(CodeGenFunction &CGF, Value *ArgValue, bool HasNSW) {
2035 return CGF.Builder.CreateBinaryIntrinsic(
2036 ID: Intrinsic::abs, LHS: ArgValue,
2037 RHS: ConstantInt::get(Ty: CGF.Builder.getInt1Ty(), V: HasNSW));
2038}
2039
2040static Value *EmitOverflowCheckedAbs(CodeGenFunction &CGF, const CallExpr *E,
2041 bool SanitizeOverflow) {
2042 Value *ArgValue = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
2043
2044 // Try to eliminate overflow check.
2045 if (const auto *VCI = dyn_cast<llvm::ConstantInt>(Val: ArgValue)) {
2046 if (!VCI->isMinSignedValue())
2047 return EmitAbs(CGF, ArgValue, HasNSW: true);
2048 }
2049
2050 SmallVector<SanitizerKind::SanitizerOrdinal, 1> Ordinals;
2051 SanitizerHandler CheckHandler;
2052 if (SanitizeOverflow) {
2053 Ordinals.push_back(Elt: SanitizerKind::SO_SignedIntegerOverflow);
2054 CheckHandler = SanitizerHandler::NegateOverflow;
2055 } else
2056 CheckHandler = SanitizerHandler::SubOverflow;
2057
2058 SanitizerDebugLocation SanScope(&CGF, Ordinals, CheckHandler);
2059
2060 Constant *Zero = Constant::getNullValue(Ty: ArgValue->getType());
2061 Value *ResultAndOverflow = CGF.Builder.CreateBinaryIntrinsic(
2062 ID: Intrinsic::ssub_with_overflow, LHS: Zero, RHS: ArgValue);
2063 Value *Result = CGF.Builder.CreateExtractValue(Agg: ResultAndOverflow, Idxs: 0);
2064 Value *NotOverflow = CGF.Builder.CreateNot(
2065 V: CGF.Builder.CreateExtractValue(Agg: ResultAndOverflow, Idxs: 1));
2066
2067 // TODO: support -ftrapv-handler.
2068 if (SanitizeOverflow) {
2069 CGF.EmitCheck(Checked: {{NotOverflow, SanitizerKind::SO_SignedIntegerOverflow}},
2070 Check: CheckHandler,
2071 StaticArgs: {CGF.EmitCheckSourceLocation(Loc: E->getArg(Arg: 0)->getExprLoc()),
2072 CGF.EmitCheckTypeDescriptor(T: E->getType())},
2073 DynamicArgs: {ArgValue});
2074 } else
2075 CGF.EmitTrapCheck(Checked: NotOverflow, CheckHandlerID: CheckHandler);
2076
2077 Value *CmpResult = CGF.Builder.CreateICmpSLT(LHS: ArgValue, RHS: Zero, Name: "abscond");
2078 return CGF.Builder.CreateSelect(C: CmpResult, True: Result, False: ArgValue, Name: "abs");
2079}
2080
2081/// Get the argument type for arguments to os_log_helper.
2082static CanQualType getOSLogArgType(ASTContext &C, int Size) {
2083 QualType UnsignedTy = C.getIntTypeForBitwidth(DestWidth: Size * 8, /*Signed=*/false);
2084 return C.getCanonicalType(T: UnsignedTy);
2085}
2086
2087llvm::Function *CodeGenFunction::generateBuiltinOSLogHelperFunction(
2088 const analyze_os_log::OSLogBufferLayout &Layout,
2089 CharUnits BufferAlignment) {
2090 ASTContext &Ctx = getContext();
2091
2092 llvm::SmallString<64> Name;
2093 {
2094 raw_svector_ostream OS(Name);
2095 OS << "__os_log_helper";
2096 OS << "_" << BufferAlignment.getQuantity();
2097 OS << "_" << int(Layout.getSummaryByte());
2098 OS << "_" << int(Layout.getNumArgsByte());
2099 for (const auto &Item : Layout.Items)
2100 OS << "_" << int(Item.getSizeByte()) << "_"
2101 << int(Item.getDescriptorByte());
2102 }
2103
2104 if (llvm::Function *F = CGM.getModule().getFunction(Name))
2105 return F;
2106
2107 llvm::SmallVector<QualType, 4> ArgTys;
2108 FunctionArgList Args;
2109 Args.push_back(Elt: ImplicitParamDecl::Create(
2110 C&: Ctx, DC: nullptr, IdLoc: SourceLocation(), Id: &Ctx.Idents.get(Name: "buffer"), T: Ctx.VoidPtrTy,
2111 ParamKind: ImplicitParamKind::Other));
2112 ArgTys.emplace_back(Args&: Ctx.VoidPtrTy);
2113
2114 for (unsigned int I = 0, E = Layout.Items.size(); I < E; ++I) {
2115 char Size = Layout.Items[I].getSizeByte();
2116 if (!Size)
2117 continue;
2118
2119 QualType ArgTy = getOSLogArgType(C&: Ctx, Size);
2120 Args.push_back(Elt: ImplicitParamDecl::Create(
2121 C&: Ctx, DC: nullptr, IdLoc: SourceLocation(),
2122 Id: &Ctx.Idents.get(Name: std::string("arg") + llvm::to_string(Value: I)), T: ArgTy,
2123 ParamKind: ImplicitParamKind::Other));
2124 ArgTys.emplace_back(Args&: ArgTy);
2125 }
2126
2127 QualType ReturnTy = Ctx.VoidTy;
2128
2129 // The helper function has linkonce_odr linkage to enable the linker to merge
2130 // identical functions. To ensure the merging always happens, 'noinline' is
2131 // attached to the function when compiling with -Oz.
2132 const CGFunctionInfo &FI =
2133 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: ReturnTy, args: Args);
2134 llvm::FunctionType *FuncTy = CGM.getTypes().GetFunctionType(Info: FI);
2135 llvm::Function *Fn = llvm::Function::Create(
2136 Ty: FuncTy, Linkage: llvm::GlobalValue::LinkOnceODRLinkage, N: Name, M: &CGM.getModule());
2137 Fn->setVisibility(llvm::GlobalValue::HiddenVisibility);
2138 CGM.SetLLVMFunctionAttributes(GD: GlobalDecl(), Info: FI, F: Fn, /*IsThunk=*/false);
2139 CGM.SetLLVMFunctionAttributesForDefinition(D: nullptr, F: Fn);
2140 Fn->setDoesNotThrow();
2141
2142 // Attach 'noinline' at -Oz.
2143 if (CGM.getCodeGenOpts().OptimizeSize == 2)
2144 Fn->addFnAttr(Kind: llvm::Attribute::NoInline);
2145
2146 auto NL = ApplyDebugLocation::CreateEmpty(CGF&: *this);
2147 StartFunction(GD: GlobalDecl(), RetTy: ReturnTy, Fn, FnInfo: FI, Args);
2148
2149 // Create a scope with an artificial location for the body of this function.
2150 auto AL = ApplyDebugLocation::CreateArtificial(CGF&: *this);
2151
2152 CharUnits Offset;
2153 Address BufAddr = makeNaturalAddressForPointer(
2154 Ptr: Builder.CreateLoad(Addr: GetAddrOfLocalVar(VD: Args[0]), Name: "buf"), T: Ctx.VoidTy,
2155 Alignment: BufferAlignment);
2156 Builder.CreateStore(Val: Builder.getInt8(C: Layout.getSummaryByte()),
2157 Addr: Builder.CreateConstByteGEP(Addr: BufAddr, Offset: Offset++, Name: "summary"));
2158 Builder.CreateStore(Val: Builder.getInt8(C: Layout.getNumArgsByte()),
2159 Addr: Builder.CreateConstByteGEP(Addr: BufAddr, Offset: Offset++, Name: "numArgs"));
2160
2161 unsigned I = 1;
2162 for (const auto &Item : Layout.Items) {
2163 Builder.CreateStore(
2164 Val: Builder.getInt8(C: Item.getDescriptorByte()),
2165 Addr: Builder.CreateConstByteGEP(Addr: BufAddr, Offset: Offset++, Name: "argDescriptor"));
2166 Builder.CreateStore(
2167 Val: Builder.getInt8(C: Item.getSizeByte()),
2168 Addr: Builder.CreateConstByteGEP(Addr: BufAddr, Offset: Offset++, Name: "argSize"));
2169
2170 CharUnits Size = Item.size();
2171 if (!Size.getQuantity())
2172 continue;
2173
2174 Address Arg = GetAddrOfLocalVar(VD: Args[I]);
2175 Address Addr = Builder.CreateConstByteGEP(Addr: BufAddr, Offset, Name: "argData");
2176 Addr = Addr.withElementType(ElemTy: Arg.getElementType());
2177 Builder.CreateStore(Val: Builder.CreateLoad(Addr: Arg), Addr);
2178 Offset += Size;
2179 ++I;
2180 }
2181
2182 FinishFunction();
2183
2184 return Fn;
2185}
2186
2187RValue CodeGenFunction::emitBuiltinOSLogFormat(const CallExpr &E) {
2188 assert(E.getNumArgs() >= 2 &&
2189 "__builtin_os_log_format takes at least 2 arguments");
2190 ASTContext &Ctx = getContext();
2191 analyze_os_log::OSLogBufferLayout Layout;
2192 analyze_os_log::computeOSLogBufferLayout(Ctx, E: &E, layout&: Layout);
2193 Address BufAddr = EmitPointerWithAlignment(Addr: E.getArg(Arg: 0));
2194
2195 // Ignore argument 1, the format string. It is not currently used.
2196 CallArgList Args;
2197 Args.add(rvalue: RValue::get(V: BufAddr.emitRawPointer(CGF&: *this)), type: Ctx.VoidPtrTy);
2198
2199 for (const auto &Item : Layout.Items) {
2200 int Size = Item.getSizeByte();
2201 if (!Size)
2202 continue;
2203
2204 llvm::Value *ArgVal;
2205
2206 if (Item.getKind() == analyze_os_log::OSLogBufferItem::MaskKind) {
2207 uint64_t Val = 0;
2208 for (unsigned I = 0, E = Item.getMaskType().size(); I < E; ++I)
2209 Val |= ((uint64_t)Item.getMaskType()[I]) << I * 8;
2210 ArgVal = llvm::Constant::getIntegerValue(Ty: Int64Ty, V: llvm::APInt(64, Val));
2211 } else if (const Expr *TheExpr = Item.getExpr()) {
2212 ArgVal = EmitScalarExpr(E: TheExpr, /*Ignore*/ IgnoreResultAssign: false);
2213
2214 // If a temporary object that requires destruction after the full
2215 // expression is passed, push a lifetime-extended cleanup to extend its
2216 // lifetime to the end of the enclosing block scope.
2217 auto LifetimeExtendObject = [&](const Expr *E) {
2218 E = E->IgnoreParenCasts();
2219 // Extend lifetimes of objects returned by function calls and message
2220 // sends.
2221
2222 // FIXME: We should do this in other cases in which temporaries are
2223 // created including arguments of non-ARC types (e.g., C++
2224 // temporaries).
2225 if (isa<CallExpr>(Val: E) || isa<ObjCMessageExpr>(Val: E))
2226 return true;
2227 return false;
2228 };
2229
2230 if (TheExpr->getType()->isObjCRetainableType() &&
2231 getLangOpts().ObjCAutoRefCount && LifetimeExtendObject(TheExpr)) {
2232 assert(getEvaluationKind(TheExpr->getType()) == TEK_Scalar &&
2233 "Only scalar can be a ObjC retainable type");
2234 if (!isa<Constant>(Val: ArgVal)) {
2235 CleanupKind Cleanup = getARCCleanupKind();
2236 QualType Ty = TheExpr->getType();
2237 RawAddress Alloca = CreateMemTempWithoutCast(T: Ty, Name: "os.log.arg");
2238 ArgVal = EmitARCRetain(type: Ty, value: ArgVal);
2239 Builder.CreateStore(Val: ArgVal, Addr: Alloca);
2240 pushLifetimeExtendedDestroy(kind: Cleanup, addr: Alloca, type: Ty,
2241 destroyer: CodeGenFunction::destroyARCStrongPrecise,
2242 useEHCleanupForArray: Cleanup & EHCleanup);
2243
2244 // Push a clang.arc.use call to ensure ARC optimizer knows that the
2245 // argument has to be alive.
2246 if (CGM.getCodeGenOpts().OptimizationLevel != 0)
2247 pushCleanupAfterFullExpr<CallObjCArcUse>(Kind: Cleanup, A: ArgVal);
2248 }
2249 }
2250 } else {
2251 ArgVal = Builder.getInt32(C: Item.getConstValue().getQuantity());
2252 }
2253
2254 unsigned ArgValSize =
2255 CGM.getDataLayout().getTypeSizeInBits(Ty: ArgVal->getType());
2256 llvm::IntegerType *IntTy = llvm::Type::getIntNTy(C&: getLLVMContext(),
2257 N: ArgValSize);
2258 ArgVal = Builder.CreateBitOrPointerCast(V: ArgVal, DestTy: IntTy);
2259 CanQualType ArgTy = getOSLogArgType(C&: Ctx, Size);
2260 // If ArgVal has type x86_fp80, zero-extend ArgVal.
2261 ArgVal = Builder.CreateZExtOrBitCast(V: ArgVal, DestTy: ConvertType(T: ArgTy));
2262 Args.add(rvalue: RValue::get(V: ArgVal), type: ArgTy);
2263 }
2264
2265 const CGFunctionInfo &FI =
2266 CGM.getTypes().arrangeBuiltinFunctionCall(resultType: Ctx.VoidTy, args: Args);
2267 llvm::Function *F = CodeGenFunction(CGM).generateBuiltinOSLogHelperFunction(
2268 Layout, BufferAlignment: BufAddr.getAlignment());
2269 EmitCall(CallInfo: FI, Callee: CGCallee::forDirect(functionPtr: F), ReturnValue: ReturnValueSlot(), Args);
2270 return RValue::get(Addr: BufAddr, CGF&: *this);
2271}
2272
2273static bool isSpecialUnsignedMultiplySignedResult(
2274 unsigned BuiltinID, WidthAndSignedness Op1Info, WidthAndSignedness Op2Info,
2275 WidthAndSignedness ResultInfo) {
2276 return BuiltinID == Builtin::BI__builtin_mul_overflow &&
2277 Op1Info.Width == Op2Info.Width && Op2Info.Width == ResultInfo.Width &&
2278 !Op1Info.Signed && !Op2Info.Signed && ResultInfo.Signed;
2279}
2280
2281static RValue EmitCheckedUnsignedMultiplySignedResult(
2282 CodeGenFunction &CGF, const clang::Expr *Op1, WidthAndSignedness Op1Info,
2283 const clang::Expr *Op2, WidthAndSignedness Op2Info,
2284 const clang::Expr *ResultArg, QualType ResultQTy,
2285 WidthAndSignedness ResultInfo) {
2286 assert(isSpecialUnsignedMultiplySignedResult(
2287 Builtin::BI__builtin_mul_overflow, Op1Info, Op2Info, ResultInfo) &&
2288 "Cannot specialize this multiply");
2289
2290 llvm::Value *V1 = CGF.EmitScalarExpr(E: Op1);
2291 llvm::Value *V2 = CGF.EmitScalarExpr(E: Op2);
2292
2293 llvm::Value *HasOverflow;
2294 llvm::Value *Result = EmitOverflowIntrinsic(
2295 CGF, IntrinsicID: Intrinsic::umul_with_overflow, X: V1, Y: V2, Carry&: HasOverflow);
2296
2297 // The intrinsic call will detect overflow when the value is > UINT_MAX,
2298 // however, since the original builtin had a signed result, we need to report
2299 // an overflow when the result is greater than INT_MAX.
2300 auto IntMax = llvm::APInt::getSignedMaxValue(numBits: ResultInfo.Width);
2301 llvm::Value *IntMaxValue = llvm::ConstantInt::get(Ty: Result->getType(), V: IntMax);
2302
2303 llvm::Value *IntMaxOverflow = CGF.Builder.CreateICmpUGT(LHS: Result, RHS: IntMaxValue);
2304 HasOverflow = CGF.Builder.CreateOr(LHS: HasOverflow, RHS: IntMaxOverflow);
2305
2306 bool isVolatile =
2307 ResultArg->getType()->getPointeeType().isVolatileQualified();
2308 Address ResultPtr = CGF.EmitPointerWithAlignment(Addr: ResultArg);
2309 CGF.Builder.CreateStore(Val: CGF.EmitToMemory(Value: Result, Ty: ResultQTy), Addr: ResultPtr,
2310 IsVolatile: isVolatile);
2311 return RValue::get(V: HasOverflow);
2312}
2313
2314/// Determine if a binop is a checked mixed-sign multiply we can specialize.
2315static bool isSpecialMixedSignMultiply(unsigned BuiltinID,
2316 WidthAndSignedness Op1Info,
2317 WidthAndSignedness Op2Info,
2318 WidthAndSignedness ResultInfo) {
2319 return BuiltinID == Builtin::BI__builtin_mul_overflow &&
2320 std::max(a: Op1Info.Width, b: Op2Info.Width) >= ResultInfo.Width &&
2321 Op1Info.Signed != Op2Info.Signed;
2322}
2323
2324/// Emit a checked mixed-sign multiply. This is a cheaper specialization of
2325/// the generic checked-binop irgen.
2326static RValue
2327EmitCheckedMixedSignMultiply(CodeGenFunction &CGF, const clang::Expr *Op1,
2328 WidthAndSignedness Op1Info, const clang::Expr *Op2,
2329 WidthAndSignedness Op2Info,
2330 const clang::Expr *ResultArg, QualType ResultQTy,
2331 WidthAndSignedness ResultInfo) {
2332 assert(isSpecialMixedSignMultiply(Builtin::BI__builtin_mul_overflow, Op1Info,
2333 Op2Info, ResultInfo) &&
2334 "Not a mixed-sign multipliction we can specialize");
2335
2336 // Emit the signed and unsigned operands.
2337 const clang::Expr *SignedOp = Op1Info.Signed ? Op1 : Op2;
2338 const clang::Expr *UnsignedOp = Op1Info.Signed ? Op2 : Op1;
2339 llvm::Value *Signed = CGF.EmitScalarExpr(E: SignedOp);
2340 llvm::Value *Unsigned = CGF.EmitScalarExpr(E: UnsignedOp);
2341 unsigned SignedOpWidth = Op1Info.Signed ? Op1Info.Width : Op2Info.Width;
2342 unsigned UnsignedOpWidth = Op1Info.Signed ? Op2Info.Width : Op1Info.Width;
2343
2344 // One of the operands may be smaller than the other. If so, [s|z]ext it.
2345 if (SignedOpWidth < UnsignedOpWidth)
2346 Signed = CGF.Builder.CreateSExt(V: Signed, DestTy: Unsigned->getType(), Name: "op.sext");
2347 if (UnsignedOpWidth < SignedOpWidth)
2348 Unsigned = CGF.Builder.CreateZExt(V: Unsigned, DestTy: Signed->getType(), Name: "op.zext");
2349
2350 llvm::Type *OpTy = Signed->getType();
2351 llvm::Value *Zero = llvm::Constant::getNullValue(Ty: OpTy);
2352 Address ResultPtr = CGF.EmitPointerWithAlignment(Addr: ResultArg);
2353 llvm::Type *ResTy = CGF.getTypes().ConvertType(T: ResultQTy);
2354 unsigned OpWidth = std::max(a: Op1Info.Width, b: Op2Info.Width);
2355
2356 // Take the absolute value of the signed operand.
2357 llvm::Value *IsNegative = CGF.Builder.CreateICmpSLT(LHS: Signed, RHS: Zero);
2358 llvm::Value *AbsOfNegative = CGF.Builder.CreateSub(LHS: Zero, RHS: Signed);
2359 llvm::Value *AbsSigned =
2360 CGF.Builder.CreateSelect(C: IsNegative, True: AbsOfNegative, False: Signed);
2361
2362 // Perform a checked unsigned multiplication.
2363 llvm::Value *UnsignedOverflow;
2364 llvm::Value *UnsignedResult =
2365 EmitOverflowIntrinsic(CGF, IntrinsicID: Intrinsic::umul_with_overflow, X: AbsSigned,
2366 Y: Unsigned, Carry&: UnsignedOverflow);
2367
2368 llvm::Value *Overflow, *Result;
2369 if (ResultInfo.Signed) {
2370 // Signed overflow occurs if the result is greater than INT_MAX or lesser
2371 // than INT_MIN, i.e when |Result| > (INT_MAX + IsNegative).
2372 auto IntMax =
2373 llvm::APInt::getSignedMaxValue(numBits: ResultInfo.Width).zext(width: OpWidth);
2374 llvm::Value *MaxResult =
2375 CGF.Builder.CreateAdd(LHS: llvm::ConstantInt::get(Ty: OpTy, V: IntMax),
2376 RHS: CGF.Builder.CreateZExt(V: IsNegative, DestTy: OpTy));
2377 llvm::Value *SignedOverflow =
2378 CGF.Builder.CreateICmpUGT(LHS: UnsignedResult, RHS: MaxResult);
2379 Overflow = CGF.Builder.CreateOr(LHS: UnsignedOverflow, RHS: SignedOverflow);
2380
2381 // Prepare the signed result (possibly by negating it).
2382 llvm::Value *NegativeResult = CGF.Builder.CreateNeg(V: UnsignedResult);
2383 llvm::Value *SignedResult =
2384 CGF.Builder.CreateSelect(C: IsNegative, True: NegativeResult, False: UnsignedResult);
2385 Result = CGF.Builder.CreateTrunc(V: SignedResult, DestTy: ResTy);
2386 } else {
2387 // Unsigned overflow occurs if the result is < 0 or greater than UINT_MAX.
2388 llvm::Value *Underflow = CGF.Builder.CreateAnd(
2389 LHS: IsNegative, RHS: CGF.Builder.CreateIsNotNull(Arg: UnsignedResult));
2390 Overflow = CGF.Builder.CreateOr(LHS: UnsignedOverflow, RHS: Underflow);
2391 if (ResultInfo.Width < OpWidth) {
2392 auto IntMax =
2393 llvm::APInt::getMaxValue(numBits: ResultInfo.Width).zext(width: OpWidth);
2394 llvm::Value *TruncOverflow = CGF.Builder.CreateICmpUGT(
2395 LHS: UnsignedResult, RHS: llvm::ConstantInt::get(Ty: OpTy, V: IntMax));
2396 Overflow = CGF.Builder.CreateOr(LHS: Overflow, RHS: TruncOverflow);
2397 }
2398
2399 // Negate the product if it would be negative in infinite precision.
2400 Result = CGF.Builder.CreateSelect(
2401 C: IsNegative, True: CGF.Builder.CreateNeg(V: UnsignedResult), False: UnsignedResult);
2402
2403 Result = CGF.Builder.CreateTrunc(V: Result, DestTy: ResTy);
2404 }
2405 assert(Overflow && Result && "Missing overflow or result");
2406
2407 bool isVolatile =
2408 ResultArg->getType()->getPointeeType().isVolatileQualified();
2409 CGF.Builder.CreateStore(Val: CGF.EmitToMemory(Value: Result, Ty: ResultQTy), Addr: ResultPtr,
2410 IsVolatile: isVolatile);
2411 return RValue::get(V: Overflow);
2412}
2413
2414/// Determine if the specified type requires laundering by checking if it is a
2415/// dynamic class type or contains a subobject which is a dynamic class type.
2416static bool TypeRequiresBuiltinLaunder(CodeGenModule &CGM, QualType Ty) {
2417 if (!CGM.getCodeGenOpts().StrictVTablePointers)
2418 return false;
2419 return Ty.requiresBuiltinLaunder(Context: CGM.getContext());
2420}
2421
2422RValue CodeGenFunction::emitRotate(const CallExpr *E, bool IsRotateRight) {
2423 llvm::Value *Src = EmitScalarExpr(E: E->getArg(Arg: 0));
2424 llvm::Value *ShiftAmt = EmitScalarExpr(E: E->getArg(Arg: 1));
2425
2426 // The builtin's shift arg may have a different type than the source arg and
2427 // result, but the LLVM intrinsic uses the same type for all values.
2428 llvm::Type *Ty = Src->getType();
2429 llvm::Type *ShiftTy = ShiftAmt->getType();
2430
2431 unsigned BitWidth = Ty->getIntegerBitWidth();
2432
2433 // Normalize shift amount to [0, BitWidth) range to match runtime behavior.
2434 // This matches the algorithm in ExprConstant.cpp for constant evaluation.
2435 if (BitWidth == 1) {
2436 // Rotating a 1-bit value is always a no-op
2437 ShiftAmt = ConstantInt::get(Ty: ShiftTy, V: 0);
2438 } else if (BitWidth == 2) {
2439 // For 2-bit values: rotation amount is 0 or 1 based on
2440 // whether the amount is even or odd. We can't use srem here because
2441 // the divisor (2) would be misinterpreted as -2 in 2-bit signed arithmetic.
2442 llvm::Value *One = ConstantInt::get(Ty: ShiftTy, V: 1);
2443 ShiftAmt = Builder.CreateAnd(LHS: ShiftAmt, RHS: One);
2444 } else {
2445 unsigned ShiftAmtBitWidth = ShiftTy->getIntegerBitWidth();
2446 bool ShiftAmtIsSigned = E->getArg(Arg: 1)->getType()->isSignedIntegerType();
2447
2448 // Choose the wider type for the divisor to avoid truncation
2449 llvm::Type *DivisorTy = ShiftAmtBitWidth > BitWidth ? ShiftTy : Ty;
2450 llvm::Value *Divisor = ConstantInt::get(Ty: DivisorTy, V: BitWidth);
2451
2452 // Extend ShiftAmt to match Divisor width if needed
2453 if (ShiftAmtBitWidth < DivisorTy->getIntegerBitWidth()) {
2454 ShiftAmt = Builder.CreateIntCast(V: ShiftAmt, DestTy: DivisorTy, isSigned: ShiftAmtIsSigned);
2455 }
2456
2457 // Normalize to [0, BitWidth)
2458 llvm::Value *RemResult;
2459 if (ShiftAmtIsSigned) {
2460 RemResult = Builder.CreateSRem(LHS: ShiftAmt, RHS: Divisor);
2461 // Signed remainder can be negative, convert to positive equivalent
2462 llvm::Value *Zero = ConstantInt::get(Ty: DivisorTy, V: 0);
2463 llvm::Value *IsNegative = Builder.CreateICmpSLT(LHS: RemResult, RHS: Zero);
2464 llvm::Value *PositiveShift = Builder.CreateAdd(LHS: RemResult, RHS: Divisor);
2465 ShiftAmt = Builder.CreateSelect(C: IsNegative, True: PositiveShift, False: RemResult);
2466 } else {
2467 ShiftAmt = Builder.CreateURem(LHS: ShiftAmt, RHS: Divisor);
2468 }
2469 }
2470
2471 // Convert to the source type if needed
2472 if (ShiftAmt->getType() != Ty) {
2473 ShiftAmt = Builder.CreateIntCast(V: ShiftAmt, DestTy: Ty, isSigned: false);
2474 }
2475
2476 // Rotate is a special case of LLVM funnel shift - 1st 2 args are the same.
2477 unsigned IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;
2478 Function *F = CGM.getIntrinsic(IID, Tys: Ty);
2479 return RValue::get(V: Builder.CreateCall(Callee: F, Args: {Src, Src, ShiftAmt}));
2480}
2481
2482// Map math builtins for long-double to f128 version.
2483static unsigned mutateLongDoubleBuiltin(unsigned BuiltinID) {
2484 switch (BuiltinID) {
2485#define MUTATE_LDBL(func) \
2486 case Builtin::BI__builtin_##func##l: \
2487 return Builtin::BI__builtin_##func##f128;
2488 MUTATE_LDBL(sqrt)
2489 MUTATE_LDBL(cbrt)
2490 MUTATE_LDBL(fabs)
2491 MUTATE_LDBL(log)
2492 MUTATE_LDBL(log2)
2493 MUTATE_LDBL(log10)
2494 MUTATE_LDBL(log1p)
2495 MUTATE_LDBL(logb)
2496 MUTATE_LDBL(exp)
2497 MUTATE_LDBL(exp2)
2498 MUTATE_LDBL(expm1)
2499 MUTATE_LDBL(fdim)
2500 MUTATE_LDBL(hypot)
2501 MUTATE_LDBL(ilogb)
2502 MUTATE_LDBL(pow)
2503 MUTATE_LDBL(fmin)
2504 MUTATE_LDBL(fmax)
2505 MUTATE_LDBL(ceil)
2506 MUTATE_LDBL(trunc)
2507 MUTATE_LDBL(rint)
2508 MUTATE_LDBL(nearbyint)
2509 MUTATE_LDBL(round)
2510 MUTATE_LDBL(floor)
2511 MUTATE_LDBL(lround)
2512 MUTATE_LDBL(llround)
2513 MUTATE_LDBL(lrint)
2514 MUTATE_LDBL(llrint)
2515 MUTATE_LDBL(fmod)
2516 MUTATE_LDBL(modf)
2517 MUTATE_LDBL(nan)
2518 MUTATE_LDBL(nans)
2519 MUTATE_LDBL(inf)
2520 MUTATE_LDBL(fma)
2521 MUTATE_LDBL(sin)
2522 MUTATE_LDBL(cos)
2523 MUTATE_LDBL(tan)
2524 MUTATE_LDBL(sinh)
2525 MUTATE_LDBL(cosh)
2526 MUTATE_LDBL(tanh)
2527 MUTATE_LDBL(asin)
2528 MUTATE_LDBL(acos)
2529 MUTATE_LDBL(atan)
2530 MUTATE_LDBL(asinh)
2531 MUTATE_LDBL(acosh)
2532 MUTATE_LDBL(atanh)
2533 MUTATE_LDBL(atan2)
2534 MUTATE_LDBL(erf)
2535 MUTATE_LDBL(erfc)
2536 MUTATE_LDBL(ldexp)
2537 MUTATE_LDBL(frexp)
2538 MUTATE_LDBL(huge_val)
2539 MUTATE_LDBL(copysign)
2540 MUTATE_LDBL(nextafter)
2541 MUTATE_LDBL(nexttoward)
2542 MUTATE_LDBL(remainder)
2543 MUTATE_LDBL(remquo)
2544 MUTATE_LDBL(scalbln)
2545 MUTATE_LDBL(scalbn)
2546 MUTATE_LDBL(tgamma)
2547 MUTATE_LDBL(lgamma)
2548#undef MUTATE_LDBL
2549 default:
2550 return BuiltinID;
2551 }
2552}
2553
2554static Value *tryUseTestFPKind(CodeGenFunction &CGF, unsigned BuiltinID,
2555 Value *V) {
2556 if (CGF.Builder.getIsFPConstrained() &&
2557 CGF.Builder.getDefaultConstrainedExcept() != fp::ebIgnore) {
2558 if (Value *Result =
2559 CGF.getTargetHooks().testFPKind(V, BuiltinID, Builder&: CGF.Builder, CGM&: CGF.CGM))
2560 return Result;
2561 }
2562 return nullptr;
2563}
2564
2565static RValue EmitHipStdParUnsupportedBuiltin(CodeGenFunction *CGF,
2566 const FunctionDecl *FD) {
2567 auto Name = FD->getNameAsString() + "__hipstdpar_unsupported";
2568 auto FnTy = CGF->CGM.getTypes().GetFunctionType(GD: FD);
2569 auto UBF = CGF->CGM.getModule().getOrInsertFunction(Name, T: FnTy);
2570
2571 SmallVector<Value *, 16> Args;
2572 for (auto &&FormalTy : FnTy->params())
2573 Args.push_back(Elt: llvm::PoisonValue::get(T: FormalTy));
2574
2575 return RValue::get(V: CGF->Builder.CreateCall(Callee: UBF, Args));
2576}
2577
2578// stdc_{leading,trailing}_{zeros,ones} and stdc_count_ones: counts bits using
2579// ctlz, cttz, or ctpop (IsPop). InvertArg flips the input to count the
2580// opposite bit value.
2581RValue CodeGenFunction::emitStdcCountIntrinsic(const CallExpr *E,
2582 Intrinsic::ID IntID,
2583 bool InvertArg, bool IsPop) {
2584 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
2585 llvm::Type *ArgType = ArgValue->getType();
2586 llvm::Type *ResultType = ConvertType(T: E->getType());
2587 Value *ActualArg = InvertArg ? Builder.CreateNot(V: ArgValue) : ArgValue;
2588 Function *F = CGM.getIntrinsic(IID: IntID, Tys: ArgType);
2589 Value *Result = IsPop
2590 ? Builder.CreateCall(Callee: F, Args: ActualArg)
2591 : Builder.CreateCall(Callee: F, Args: {ActualArg, Builder.getFalse()});
2592 if (Result->getType() != ResultType)
2593 Result = Builder.CreateIntCast(V: Result, DestTy: ResultType, isSigned: false);
2594 return RValue::get(V: Result);
2595}
2596
2597// stdc_count_zeros (BitWidth - ctpop) and stdc_bit_width (BitWidth - ctlz).
2598// IsPop selects ctpop; otherwise ctlz is used.
2599RValue CodeGenFunction::emitStdcBitWidthMinus(const CallExpr *E,
2600 Intrinsic::ID IntID, bool IsPop) {
2601 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
2602 llvm::Type *ArgType = ArgValue->getType();
2603 llvm::Type *ResultType = ConvertType(T: E->getType());
2604 unsigned BitWidth = ArgType->getIntegerBitWidth();
2605 Function *F = CGM.getIntrinsic(IID: IntID, Tys: ArgType);
2606 Value *Cnt = IsPop ? Builder.CreateCall(Callee: F, Args: ArgValue)
2607 : Builder.CreateCall(Callee: F, Args: {ArgValue, Builder.getFalse()});
2608 Value *Result = Builder.CreateSub(LHS: ConstantInt::get(Ty: ArgType, V: BitWidth), RHS: Cnt);
2609 if (Result->getType() != ResultType)
2610 Result = Builder.CreateIntCast(V: Result, DestTy: ResultType, isSigned: false);
2611 return RValue::get(V: Result);
2612}
2613
2614// stdc_first_{leading,trailing}_{zero,one}: returns the 1-based position of
2615// the first matching bit, or 0 if no such bit exists. InvertArg flips the
2616// input to search for zeros instead of ones.
2617RValue CodeGenFunction::emitStdcFirstBit(const CallExpr *E, Intrinsic::ID IntID,
2618 bool InvertArg) {
2619 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
2620 llvm::Type *ArgType = ArgValue->getType();
2621 llvm::Type *ResultType = ConvertType(T: E->getType());
2622 Value *Zero = ConstantInt::get(Ty: ArgType, V: 0);
2623 Value *One = ConstantInt::get(Ty: ArgType, V: 1);
2624 Value *ActualArg = InvertArg ? Builder.CreateNot(V: ArgValue) : ArgValue;
2625 Function *F = CGM.getIntrinsic(IID: IntID, Tys: ArgType);
2626 Value *Cnt = Builder.CreateCall(Callee: F, Args: {ActualArg, Builder.getFalse()});
2627 Value *Tmp = Builder.CreateAdd(LHS: Cnt, RHS: One);
2628 Value *IsZero = Builder.CreateICmpEQ(LHS: ActualArg, RHS: Zero);
2629 Value *Result = Builder.CreateSelect(C: IsZero, True: Zero, False: Tmp);
2630 if (Result->getType() != ResultType)
2631 Result = Builder.CreateIntCast(V: Result, DestTy: ResultType, isSigned: false);
2632 return RValue::get(V: Result);
2633}
2634
2635static void ClearPadding(CodeGenFunction &CGF, Address Src,
2636 const ASTContext::BitInterval &PaddingInterval) {
2637 uint64_t CharWidth = CGF.getContext().getCharWidth();
2638
2639 auto *I8Ptr = CGF.Builder.CreateBitCast(V: Src.getBasePointer(), DestTy: CGF.Int8PtrTy);
2640 auto *Zero = ConstantInt::get(Ty: CGF.Int8Ty, V: 0);
2641
2642 // Calculate byte indices and bit positions
2643 auto StartByte = PaddingInterval.First / CharWidth;
2644 auto StartBit = PaddingInterval.First % CharWidth;
2645 auto EndByte = PaddingInterval.Last / CharWidth;
2646 auto EndBit = PaddingInterval.Last % CharWidth;
2647
2648 if (StartByte == EndByte) {
2649 // Interval is within a single byte
2650 auto *Index = ConstantInt::get(Ty: CGF.IntTy, V: StartByte);
2651 auto *Element = CGF.Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: I8Ptr, IdxList: Index);
2652 Address ElementAddr(Element, CGF.Int8Ty,
2653 Src.getAlignment().alignmentAtOffset(
2654 offset: CharUnits::fromQuantity(Quantity: StartByte)));
2655
2656 auto *Value = CGF.Builder.CreateLoad(Addr: ElementAddr);
2657
2658 // Create mask to clear bits within the byte
2659 // We want to clear bits from StartBit to EndBit-1
2660 uint8_t bitsToClear = ((1 << EndBit) - 1) & ~((1 << StartBit) - 1);
2661 uint8_t bitsToKeep = ~bitsToClear;
2662 auto *MaskValue = ConstantInt::get(Ty: CGF.Int8Ty, V: bitsToKeep);
2663 auto *NewValue = CGF.Builder.CreateAnd(LHS: Value, RHS: MaskValue);
2664
2665 CGF.Builder.CreateStore(Val: NewValue, Addr: ElementAddr);
2666 } else {
2667 // Handle the start byte
2668 if (StartBit != 0) {
2669 auto *Index = ConstantInt::get(Ty: CGF.IntTy, V: StartByte);
2670 auto *Element = CGF.Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: I8Ptr, IdxList: Index);
2671 Address ElementAddr(Element, CGF.Int8Ty,
2672 Src.getAlignment().alignmentAtOffset(
2673 offset: CharUnits::fromQuantity(Quantity: StartByte)));
2674
2675 auto *Value = CGF.Builder.CreateLoad(Addr: ElementAddr);
2676
2677 uint8_t bitsToClear = ((1 << (CharWidth - StartBit)) - 1) << StartBit;
2678 uint8_t bitsToKeep = ~bitsToClear;
2679 auto *MaskValue = ConstantInt::get(Ty: CGF.Int8Ty, V: bitsToKeep);
2680 auto *NewValue = CGF.Builder.CreateAnd(LHS: Value, RHS: MaskValue);
2681
2682 CGF.Builder.CreateStore(Val: NewValue, Addr: ElementAddr);
2683 ++StartByte;
2684 }
2685
2686 // Handle full bytes in the middle
2687 for (auto Offset = StartByte; Offset < EndByte; ++Offset) {
2688 auto *Index = ConstantInt::get(Ty: CGF.IntTy, V: Offset);
2689 auto *Element = CGF.Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: I8Ptr, IdxList: Index);
2690 Address ElementAddr(Element, CGF.Int8Ty,
2691 Src.getAlignment().alignmentAtOffset(
2692 offset: CharUnits::fromQuantity(Quantity: Offset)));
2693
2694 CGF.Builder.CreateStore(Val: Zero, Addr: ElementAddr);
2695 }
2696
2697 // Handle the end byte
2698 if (EndBit != 0) {
2699 auto *Index = ConstantInt::get(Ty: CGF.IntTy, V: EndByte);
2700 auto *Element = CGF.Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: I8Ptr, IdxList: Index);
2701 Address ElementAddr(Element, CGF.Int8Ty,
2702 Src.getAlignment().alignmentAtOffset(
2703 offset: CharUnits::fromQuantity(Quantity: EndByte)));
2704
2705 auto *Value = CGF.Builder.CreateLoad(Addr: ElementAddr);
2706
2707 uint8_t bitsToClear = (1 << EndBit) - 1;
2708 uint8_t bitsToKeep = ~bitsToClear;
2709 auto *MaskValue = ConstantInt::get(Ty: CGF.Int8Ty, V: bitsToKeep);
2710 auto *NewValue = CGF.Builder.CreateAnd(LHS: Value, RHS: MaskValue);
2711
2712 CGF.Builder.CreateStore(Val: NewValue, Addr: ElementAddr);
2713 }
2714 }
2715}
2716
2717RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
2718 const CallExpr *E,
2719 ReturnValueSlot ReturnValue) {
2720 assert(!getContext().BuiltinInfo.isImmediate(BuiltinID) &&
2721 "Should not codegen for consteval builtins");
2722
2723 const FunctionDecl *FD = GD.getDecl()->getAsFunction();
2724 // See if we can constant fold this builtin. If so, don't emit it at all.
2725 // TODO: Extend this handling to all builtin calls that we can constant-fold.
2726 Expr::EvalResult Result;
2727 if (E->isPRValue() && E->EvaluateAsRValue(Result, Ctx: CGM.getContext()) &&
2728 !Result.hasSideEffects()) {
2729 if (Result.Val.isInt())
2730 return RValue::get(V: llvm::ConstantInt::get(Context&: getLLVMContext(),
2731 V: Result.Val.getInt()));
2732 if (Result.Val.isFloat())
2733 return RValue::get(V: llvm::ConstantFP::get(Context&: getLLVMContext(),
2734 V: Result.Val.getFloat()));
2735 }
2736
2737 // If current long-double semantics is IEEE 128-bit, replace math builtins
2738 // of long-double with f128 equivalent.
2739 // TODO: This mutation should also be applied to other targets other than PPC,
2740 // after backend supports IEEE 128-bit style libcalls.
2741 if (getTarget().getTriple().isPPC64() &&
2742 &getTarget().getLongDoubleFormat() == &llvm::APFloat::IEEEquad())
2743 BuiltinID = mutateLongDoubleBuiltin(BuiltinID);
2744
2745 // If the builtin has been declared explicitly with an assembler label,
2746 // disable the specialized emitting below. Ideally we should communicate the
2747 // rename in IR, or at least avoid generating the intrinsic calls that are
2748 // likely to get lowered to the renamed library functions.
2749 const unsigned BuiltinIDIfNoAsmLabel =
2750 FD->hasAttr<AsmLabelAttr>() ? 0 : BuiltinID;
2751
2752 std::optional<bool> ErrnoOverriden;
2753 // ErrnoOverriden is true if math-errno is overriden via the
2754 // '#pragma float_control(precise, on)'. This pragma disables fast-math,
2755 // which implies math-errno.
2756 if (E->hasStoredFPFeatures()) {
2757 FPOptionsOverride OP = E->getFPFeatures();
2758 if (OP.hasMathErrnoOverride())
2759 ErrnoOverriden = OP.getMathErrnoOverride();
2760 }
2761 // True if 'attribute__((optnone))' is used. This attribute overrides
2762 // fast-math which implies math-errno.
2763 bool OptNone = CurFuncDecl && CurFuncDecl->hasAttr<OptimizeNoneAttr>();
2764
2765 bool IsOptimizationEnabled = CGM.getCodeGenOpts().OptimizationLevel != 0;
2766
2767 bool GenerateFPMathIntrinsics =
2768 getContext().BuiltinInfo.shouldGenerateFPMathIntrinsic(
2769 BuiltinID, Trip: CGM.getTriple(), ErrnoOverwritten: ErrnoOverriden, MathErrnoEnabled: getLangOpts().MathErrno,
2770 HasOptNoneAttr: OptNone, IsOptimizationEnabled);
2771
2772 if (GenerateFPMathIntrinsics) {
2773 switch (BuiltinIDIfNoAsmLabel) {
2774 case Builtin::BIacos:
2775 case Builtin::BIacosf:
2776 case Builtin::BIacosl:
2777 case Builtin::BI__builtin_acos:
2778 case Builtin::BI__builtin_acosf:
2779 case Builtin::BI__builtin_acosf16:
2780 case Builtin::BI__builtin_acosl:
2781 case Builtin::BI__builtin_acosf128:
2782 case Builtin::BI__builtin_elementwise_acos:
2783 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(
2784 CGF&: *this, E, IntrinsicID: Intrinsic::acos, ConstrainedIntrinsicID: Intrinsic::experimental_constrained_acos));
2785
2786 case Builtin::BIasin:
2787 case Builtin::BIasinf:
2788 case Builtin::BIasinl:
2789 case Builtin::BI__builtin_asin:
2790 case Builtin::BI__builtin_asinf:
2791 case Builtin::BI__builtin_asinf16:
2792 case Builtin::BI__builtin_asinl:
2793 case Builtin::BI__builtin_asinf128:
2794 case Builtin::BI__builtin_elementwise_asin:
2795 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(
2796 CGF&: *this, E, IntrinsicID: Intrinsic::asin, ConstrainedIntrinsicID: Intrinsic::experimental_constrained_asin));
2797
2798 case Builtin::BIatan:
2799 case Builtin::BIatanf:
2800 case Builtin::BIatanl:
2801 case Builtin::BI__builtin_atan:
2802 case Builtin::BI__builtin_atanf:
2803 case Builtin::BI__builtin_atanf16:
2804 case Builtin::BI__builtin_atanl:
2805 case Builtin::BI__builtin_atanf128:
2806 case Builtin::BI__builtin_elementwise_atan:
2807 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(
2808 CGF&: *this, E, IntrinsicID: Intrinsic::atan, ConstrainedIntrinsicID: Intrinsic::experimental_constrained_atan));
2809
2810 case Builtin::BIatan2:
2811 case Builtin::BIatan2f:
2812 case Builtin::BIatan2l:
2813 case Builtin::BI__builtin_atan2:
2814 case Builtin::BI__builtin_atan2f:
2815 case Builtin::BI__builtin_atan2f16:
2816 case Builtin::BI__builtin_atan2l:
2817 case Builtin::BI__builtin_atan2f128:
2818 case Builtin::BI__builtin_elementwise_atan2:
2819 return RValue::get(V: emitBinaryMaybeConstrainedFPBuiltin(
2820 CGF&: *this, E, IntrinsicID: Intrinsic::atan2,
2821 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_atan2));
2822
2823 case Builtin::BIceil:
2824 case Builtin::BIceilf:
2825 case Builtin::BIceill:
2826 case Builtin::BI__builtin_ceil:
2827 case Builtin::BI__builtin_ceilf:
2828 case Builtin::BI__builtin_ceilf16:
2829 case Builtin::BI__builtin_ceill:
2830 case Builtin::BI__builtin_ceilf128:
2831 case Builtin::BI__builtin_elementwise_ceil:
2832 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
2833 IntrinsicID: Intrinsic::ceil,
2834 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_ceil));
2835
2836 case Builtin::BIcopysign:
2837 case Builtin::BIcopysignf:
2838 case Builtin::BIcopysignl:
2839 case Builtin::BI__builtin_copysign:
2840 case Builtin::BI__builtin_copysignf:
2841 case Builtin::BI__builtin_copysignf16:
2842 case Builtin::BI__builtin_copysignl:
2843 case Builtin::BI__builtin_copysignf128:
2844 return RValue::get(
2845 V: emitBuiltinWithOneOverloadedType<2>(CGF&: *this, E, IntrinsicID: Intrinsic::copysign));
2846
2847 case Builtin::BIcos:
2848 case Builtin::BIcosf:
2849 case Builtin::BIcosl:
2850 case Builtin::BI__builtin_cos:
2851 case Builtin::BI__builtin_cosf:
2852 case Builtin::BI__builtin_cosf16:
2853 case Builtin::BI__builtin_cosl:
2854 case Builtin::BI__builtin_cosf128:
2855 case Builtin::BI__builtin_elementwise_cos:
2856 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
2857 IntrinsicID: Intrinsic::cos,
2858 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_cos));
2859
2860 case Builtin::BIcosh:
2861 case Builtin::BIcoshf:
2862 case Builtin::BIcoshl:
2863 case Builtin::BI__builtin_cosh:
2864 case Builtin::BI__builtin_coshf:
2865 case Builtin::BI__builtin_coshf16:
2866 case Builtin::BI__builtin_coshl:
2867 case Builtin::BI__builtin_coshf128:
2868 case Builtin::BI__builtin_elementwise_cosh:
2869 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(
2870 CGF&: *this, E, IntrinsicID: Intrinsic::cosh, ConstrainedIntrinsicID: Intrinsic::experimental_constrained_cosh));
2871
2872 case Builtin::BIexp:
2873 case Builtin::BIexpf:
2874 case Builtin::BIexpl:
2875 case Builtin::BI__builtin_exp:
2876 case Builtin::BI__builtin_expf:
2877 case Builtin::BI__builtin_expf16:
2878 case Builtin::BI__builtin_expl:
2879 case Builtin::BI__builtin_expf128:
2880 case Builtin::BI__builtin_elementwise_exp:
2881 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
2882 IntrinsicID: Intrinsic::exp,
2883 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_exp));
2884
2885 case Builtin::BIexp2:
2886 case Builtin::BIexp2f:
2887 case Builtin::BIexp2l:
2888 case Builtin::BI__builtin_exp2:
2889 case Builtin::BI__builtin_exp2f:
2890 case Builtin::BI__builtin_exp2f16:
2891 case Builtin::BI__builtin_exp2l:
2892 case Builtin::BI__builtin_exp2f128:
2893 case Builtin::BI__builtin_elementwise_exp2:
2894 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
2895 IntrinsicID: Intrinsic::exp2,
2896 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_exp2));
2897 case Builtin::BI__builtin_exp10:
2898 case Builtin::BI__builtin_exp10f:
2899 case Builtin::BI__builtin_exp10f16:
2900 case Builtin::BI__builtin_exp10l:
2901 case Builtin::BI__builtin_exp10f128:
2902 case Builtin::BI__builtin_elementwise_exp10: {
2903 // TODO: strictfp support
2904 if (Builder.getIsFPConstrained())
2905 break;
2906 return RValue::get(
2907 V: emitBuiltinWithOneOverloadedType<1>(CGF&: *this, E, IntrinsicID: Intrinsic::exp10));
2908 }
2909 case Builtin::BIfabs:
2910 case Builtin::BIfabsf:
2911 case Builtin::BIfabsl:
2912 case Builtin::BI__builtin_fabs:
2913 case Builtin::BI__builtin_fabsf:
2914 case Builtin::BI__builtin_fabsf16:
2915 case Builtin::BI__builtin_fabsl:
2916 case Builtin::BI__builtin_fabsf128:
2917 return RValue::get(
2918 V: emitBuiltinWithOneOverloadedType<1>(CGF&: *this, E, IntrinsicID: Intrinsic::fabs));
2919
2920 case Builtin::BIfloor:
2921 case Builtin::BIfloorf:
2922 case Builtin::BIfloorl:
2923 case Builtin::BI__builtin_floor:
2924 case Builtin::BI__builtin_floorf:
2925 case Builtin::BI__builtin_floorf16:
2926 case Builtin::BI__builtin_floorl:
2927 case Builtin::BI__builtin_floorf128:
2928 case Builtin::BI__builtin_elementwise_floor:
2929 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
2930 IntrinsicID: Intrinsic::floor,
2931 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_floor));
2932
2933 case Builtin::BIfma:
2934 case Builtin::BIfmaf:
2935 case Builtin::BIfmal:
2936 case Builtin::BI__builtin_fma:
2937 case Builtin::BI__builtin_fmaf:
2938 case Builtin::BI__builtin_fmaf16:
2939 case Builtin::BI__builtin_fmal:
2940 case Builtin::BI__builtin_fmaf128:
2941 case Builtin::BI__builtin_elementwise_fma:
2942 return RValue::get(V: emitTernaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
2943 IntrinsicID: Intrinsic::fma,
2944 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_fma));
2945
2946 case Builtin::BIfmax:
2947 case Builtin::BIfmaxf:
2948 case Builtin::BIfmaxl:
2949 case Builtin::BI__builtin_fmax:
2950 case Builtin::BI__builtin_fmaxf:
2951 case Builtin::BI__builtin_fmaxf16:
2952 case Builtin::BI__builtin_fmaxl:
2953 case Builtin::BI__builtin_fmaxf128: {
2954 IRBuilder<>::FastMathFlagGuard FMFGuard(Builder);
2955 Builder.getFastMathFlags().setNoSignedZeros();
2956 return RValue::get(V: emitBinaryMaybeConstrainedFPBuiltin(
2957 CGF&: *this, E, IntrinsicID: Intrinsic::maxnum,
2958 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_maxnum));
2959 }
2960
2961 case Builtin::BIfmin:
2962 case Builtin::BIfminf:
2963 case Builtin::BIfminl:
2964 case Builtin::BI__builtin_fmin:
2965 case Builtin::BI__builtin_fminf:
2966 case Builtin::BI__builtin_fminf16:
2967 case Builtin::BI__builtin_fminl:
2968 case Builtin::BI__builtin_fminf128: {
2969 IRBuilder<>::FastMathFlagGuard FMFGuard(Builder);
2970 Builder.getFastMathFlags().setNoSignedZeros();
2971 return RValue::get(V: emitBinaryMaybeConstrainedFPBuiltin(
2972 CGF&: *this, E, IntrinsicID: Intrinsic::minnum,
2973 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_minnum));
2974 }
2975
2976 case Builtin::BIfmaximum_num:
2977 case Builtin::BIfmaximum_numf:
2978 case Builtin::BIfmaximum_numl:
2979 case Builtin::BI__builtin_fmaximum_num:
2980 case Builtin::BI__builtin_fmaximum_numf:
2981 case Builtin::BI__builtin_fmaximum_numf16:
2982 case Builtin::BI__builtin_fmaximum_numl:
2983 case Builtin::BI__builtin_fmaximum_numf128:
2984 return RValue::get(
2985 V: emitBuiltinWithOneOverloadedType<2>(CGF&: *this, E, IntrinsicID: Intrinsic::maximumnum));
2986
2987 case Builtin::BIfminimum_num:
2988 case Builtin::BIfminimum_numf:
2989 case Builtin::BIfminimum_numl:
2990 case Builtin::BI__builtin_fminimum_num:
2991 case Builtin::BI__builtin_fminimum_numf:
2992 case Builtin::BI__builtin_fminimum_numf16:
2993 case Builtin::BI__builtin_fminimum_numl:
2994 case Builtin::BI__builtin_fminimum_numf128:
2995 return RValue::get(
2996 V: emitBuiltinWithOneOverloadedType<2>(CGF&: *this, E, IntrinsicID: Intrinsic::minimumnum));
2997
2998 // fmod() is a special-case. It maps to the frem instruction rather than an
2999 // LLVM intrinsic.
3000 case Builtin::BIfmod:
3001 case Builtin::BIfmodf:
3002 case Builtin::BIfmodl:
3003 case Builtin::BI__builtin_fmod:
3004 case Builtin::BI__builtin_fmodf:
3005 case Builtin::BI__builtin_fmodf16:
3006 case Builtin::BI__builtin_fmodl:
3007 case Builtin::BI__builtin_fmodf128:
3008 case Builtin::BI__builtin_elementwise_fmod: {
3009 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
3010 Value *Arg1 = EmitScalarExpr(E: E->getArg(Arg: 0));
3011 Value *Arg2 = EmitScalarExpr(E: E->getArg(Arg: 1));
3012 if (Builder.getIsFPConstrained()) {
3013 Function *F = CGM.getIntrinsic(IID: Intrinsic::experimental_constrained_frem,
3014 Tys: Arg1->getType());
3015 return RValue::get(V: Builder.CreateConstrainedFPCall(Callee: F, Args: {Arg1, Arg2}));
3016 } else {
3017 return RValue::get(V: Builder.CreateFRem(L: Arg1, R: Arg2, Name: "fmod"));
3018 }
3019 }
3020
3021 case Builtin::BIlog:
3022 case Builtin::BIlogf:
3023 case Builtin::BIlogl:
3024 case Builtin::BI__builtin_log:
3025 case Builtin::BI__builtin_logf:
3026 case Builtin::BI__builtin_logf16:
3027 case Builtin::BI__builtin_logl:
3028 case Builtin::BI__builtin_logf128:
3029 case Builtin::BI__builtin_elementwise_log:
3030 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
3031 IntrinsicID: Intrinsic::log,
3032 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_log));
3033
3034 case Builtin::BIlog10:
3035 case Builtin::BIlog10f:
3036 case Builtin::BIlog10l:
3037 case Builtin::BI__builtin_log10:
3038 case Builtin::BI__builtin_log10f:
3039 case Builtin::BI__builtin_log10f16:
3040 case Builtin::BI__builtin_log10l:
3041 case Builtin::BI__builtin_log10f128:
3042 case Builtin::BI__builtin_elementwise_log10:
3043 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
3044 IntrinsicID: Intrinsic::log10,
3045 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_log10));
3046
3047 case Builtin::BIlog2:
3048 case Builtin::BIlog2f:
3049 case Builtin::BIlog2l:
3050 case Builtin::BI__builtin_log2:
3051 case Builtin::BI__builtin_log2f:
3052 case Builtin::BI__builtin_log2f16:
3053 case Builtin::BI__builtin_log2l:
3054 case Builtin::BI__builtin_log2f128:
3055 case Builtin::BI__builtin_elementwise_log2:
3056 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
3057 IntrinsicID: Intrinsic::log2,
3058 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_log2));
3059
3060 case Builtin::BInearbyint:
3061 case Builtin::BInearbyintf:
3062 case Builtin::BInearbyintl:
3063 case Builtin::BI__builtin_nearbyint:
3064 case Builtin::BI__builtin_nearbyintf:
3065 case Builtin::BI__builtin_nearbyintl:
3066 case Builtin::BI__builtin_nearbyintf128:
3067 case Builtin::BI__builtin_elementwise_nearbyint:
3068 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
3069 IntrinsicID: Intrinsic::nearbyint,
3070 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_nearbyint));
3071
3072 case Builtin::BIpow:
3073 case Builtin::BIpowf:
3074 case Builtin::BIpowl:
3075 case Builtin::BI__builtin_pow:
3076 case Builtin::BI__builtin_powf:
3077 case Builtin::BI__builtin_powf16:
3078 case Builtin::BI__builtin_powl:
3079 case Builtin::BI__builtin_powf128:
3080 case Builtin::BI__builtin_elementwise_pow:
3081 return RValue::get(V: emitBinaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
3082 IntrinsicID: Intrinsic::pow,
3083 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_pow));
3084
3085 case Builtin::BIrint:
3086 case Builtin::BIrintf:
3087 case Builtin::BIrintl:
3088 case Builtin::BI__builtin_rint:
3089 case Builtin::BI__builtin_rintf:
3090 case Builtin::BI__builtin_rintf16:
3091 case Builtin::BI__builtin_rintl:
3092 case Builtin::BI__builtin_rintf128:
3093 case Builtin::BI__builtin_elementwise_rint:
3094 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
3095 IntrinsicID: Intrinsic::rint,
3096 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_rint));
3097
3098 case Builtin::BIround:
3099 case Builtin::BIroundf:
3100 case Builtin::BIroundl:
3101 case Builtin::BI__builtin_round:
3102 case Builtin::BI__builtin_roundf:
3103 case Builtin::BI__builtin_roundf16:
3104 case Builtin::BI__builtin_roundl:
3105 case Builtin::BI__builtin_roundf128:
3106 case Builtin::BI__builtin_elementwise_round:
3107 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
3108 IntrinsicID: Intrinsic::round,
3109 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_round));
3110
3111 case Builtin::BIroundeven:
3112 case Builtin::BIroundevenf:
3113 case Builtin::BIroundevenl:
3114 case Builtin::BI__builtin_roundeven:
3115 case Builtin::BI__builtin_roundevenf:
3116 case Builtin::BI__builtin_roundevenf16:
3117 case Builtin::BI__builtin_roundevenl:
3118 case Builtin::BI__builtin_roundevenf128:
3119 case Builtin::BI__builtin_elementwise_roundeven:
3120 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
3121 IntrinsicID: Intrinsic::roundeven,
3122 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_roundeven));
3123
3124 case Builtin::BIsin:
3125 case Builtin::BIsinf:
3126 case Builtin::BIsinl:
3127 case Builtin::BI__builtin_sin:
3128 case Builtin::BI__builtin_sinf:
3129 case Builtin::BI__builtin_sinf16:
3130 case Builtin::BI__builtin_sinl:
3131 case Builtin::BI__builtin_sinf128:
3132 case Builtin::BI__builtin_elementwise_sin:
3133 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
3134 IntrinsicID: Intrinsic::sin,
3135 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_sin));
3136
3137 case Builtin::BIsinh:
3138 case Builtin::BIsinhf:
3139 case Builtin::BIsinhl:
3140 case Builtin::BI__builtin_sinh:
3141 case Builtin::BI__builtin_sinhf:
3142 case Builtin::BI__builtin_sinhf16:
3143 case Builtin::BI__builtin_sinhl:
3144 case Builtin::BI__builtin_sinhf128:
3145 case Builtin::BI__builtin_elementwise_sinh:
3146 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(
3147 CGF&: *this, E, IntrinsicID: Intrinsic::sinh, ConstrainedIntrinsicID: Intrinsic::experimental_constrained_sinh));
3148
3149 case Builtin::BI__builtin_sincospi:
3150 case Builtin::BI__builtin_sincospif:
3151 case Builtin::BI__builtin_sincospil:
3152 if (Builder.getIsFPConstrained())
3153 break; // TODO: Emit constrained sincospi intrinsic once one exists.
3154 emitSincosBuiltin(CGF&: *this, E, IntrinsicID: Intrinsic::sincospi);
3155 return RValue::get(V: nullptr);
3156
3157 case Builtin::BIsincos:
3158 case Builtin::BIsincosf:
3159 case Builtin::BIsincosl:
3160 case Builtin::BI__builtin_sincos:
3161 case Builtin::BI__builtin_sincosf:
3162 case Builtin::BI__builtin_sincosf16:
3163 case Builtin::BI__builtin_sincosl:
3164 case Builtin::BI__builtin_sincosf128:
3165 if (Builder.getIsFPConstrained())
3166 break; // TODO: Emit constrained sincos intrinsic once one exists.
3167 emitSincosBuiltin(CGF&: *this, E, IntrinsicID: Intrinsic::sincos);
3168 return RValue::get(V: nullptr);
3169
3170 case Builtin::BIsqrt:
3171 case Builtin::BIsqrtf:
3172 case Builtin::BIsqrtl:
3173 case Builtin::BI__builtin_sqrt:
3174 case Builtin::BI__builtin_sqrtf:
3175 case Builtin::BI__builtin_sqrtf16:
3176 case Builtin::BI__builtin_sqrtl:
3177 case Builtin::BI__builtin_sqrtf128:
3178 case Builtin::BI__builtin_elementwise_sqrt: {
3179 llvm::Value *Call = emitUnaryMaybeConstrainedFPBuiltin(
3180 CGF&: *this, E, IntrinsicID: Intrinsic::sqrt, ConstrainedIntrinsicID: Intrinsic::experimental_constrained_sqrt);
3181 SetSqrtFPAccuracy(Call);
3182 return RValue::get(V: Call);
3183 }
3184
3185 case Builtin::BItan:
3186 case Builtin::BItanf:
3187 case Builtin::BItanl:
3188 case Builtin::BI__builtin_tan:
3189 case Builtin::BI__builtin_tanf:
3190 case Builtin::BI__builtin_tanf16:
3191 case Builtin::BI__builtin_tanl:
3192 case Builtin::BI__builtin_tanf128:
3193 case Builtin::BI__builtin_elementwise_tan:
3194 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(
3195 CGF&: *this, E, IntrinsicID: Intrinsic::tan, ConstrainedIntrinsicID: Intrinsic::experimental_constrained_tan));
3196
3197 case Builtin::BItanh:
3198 case Builtin::BItanhf:
3199 case Builtin::BItanhl:
3200 case Builtin::BI__builtin_tanh:
3201 case Builtin::BI__builtin_tanhf:
3202 case Builtin::BI__builtin_tanhf16:
3203 case Builtin::BI__builtin_tanhl:
3204 case Builtin::BI__builtin_tanhf128:
3205 case Builtin::BI__builtin_elementwise_tanh:
3206 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(
3207 CGF&: *this, E, IntrinsicID: Intrinsic::tanh, ConstrainedIntrinsicID: Intrinsic::experimental_constrained_tanh));
3208
3209 case Builtin::BItrunc:
3210 case Builtin::BItruncf:
3211 case Builtin::BItruncl:
3212 case Builtin::BI__builtin_trunc:
3213 case Builtin::BI__builtin_truncf:
3214 case Builtin::BI__builtin_truncf16:
3215 case Builtin::BI__builtin_truncl:
3216 case Builtin::BI__builtin_truncf128:
3217 case Builtin::BI__builtin_elementwise_trunc:
3218 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
3219 IntrinsicID: Intrinsic::trunc,
3220 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_trunc));
3221
3222 case Builtin::BIlround:
3223 case Builtin::BIlroundf:
3224 case Builtin::BIlroundl:
3225 case Builtin::BI__builtin_lround:
3226 case Builtin::BI__builtin_lroundf:
3227 case Builtin::BI__builtin_lroundl:
3228 case Builtin::BI__builtin_lroundf128:
3229 return RValue::get(V: emitMaybeConstrainedFPToIntRoundBuiltin(
3230 CGF&: *this, E, IntrinsicID: Intrinsic::lround,
3231 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_lround));
3232
3233 case Builtin::BIllround:
3234 case Builtin::BIllroundf:
3235 case Builtin::BIllroundl:
3236 case Builtin::BI__builtin_llround:
3237 case Builtin::BI__builtin_llroundf:
3238 case Builtin::BI__builtin_llroundl:
3239 case Builtin::BI__builtin_llroundf128:
3240 return RValue::get(V: emitMaybeConstrainedFPToIntRoundBuiltin(
3241 CGF&: *this, E, IntrinsicID: Intrinsic::llround,
3242 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_llround));
3243
3244 case Builtin::BIlrint:
3245 case Builtin::BIlrintf:
3246 case Builtin::BIlrintl:
3247 case Builtin::BI__builtin_lrint:
3248 case Builtin::BI__builtin_lrintf:
3249 case Builtin::BI__builtin_lrintl:
3250 case Builtin::BI__builtin_lrintf128:
3251 return RValue::get(V: emitMaybeConstrainedFPToIntRoundBuiltin(
3252 CGF&: *this, E, IntrinsicID: Intrinsic::lrint,
3253 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_lrint));
3254
3255 case Builtin::BIllrint:
3256 case Builtin::BIllrintf:
3257 case Builtin::BIllrintl:
3258 case Builtin::BI__builtin_llrint:
3259 case Builtin::BI__builtin_llrintf:
3260 case Builtin::BI__builtin_llrintl:
3261 case Builtin::BI__builtin_llrintf128:
3262 return RValue::get(V: emitMaybeConstrainedFPToIntRoundBuiltin(
3263 CGF&: *this, E, IntrinsicID: Intrinsic::llrint,
3264 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_llrint));
3265 case Builtin::BI__builtin_ldexp:
3266 case Builtin::BI__builtin_ldexpf:
3267 case Builtin::BI__builtin_ldexpl:
3268 case Builtin::BI__builtin_ldexpf16:
3269 case Builtin::BI__builtin_ldexpf128:
3270 case Builtin::BI__builtin_elementwise_ldexp:
3271 return RValue::get(V: emitBinaryExpMaybeConstrainedFPBuiltin(
3272 CGF&: *this, E, IntrinsicID: Intrinsic::ldexp,
3273 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_ldexp));
3274 default:
3275 break;
3276 }
3277 }
3278
3279 // Check NonnullAttribute/NullabilityArg and Alignment.
3280 auto EmitArgCheck = [&](TypeCheckKind Kind, Address A, const Expr *Arg,
3281 unsigned ParmNum) {
3282 Value *Val = A.emitRawPointer(CGF&: *this);
3283 EmitNonNullArgCheck(RV: RValue::get(V: Val), ArgType: Arg->getType(), ArgLoc: Arg->getExprLoc(), AC: FD,
3284 ParmNum);
3285
3286 if (SanOpts.has(K: SanitizerKind::Alignment)) {
3287 SanitizerSet SkippedChecks;
3288 SkippedChecks.set(SanitizerKind::All);
3289 SkippedChecks.clear(K: SanitizerKind::Alignment);
3290 SourceLocation Loc = Arg->getExprLoc();
3291 // Strip an implicit cast.
3292 if (auto *CE = dyn_cast<ImplicitCastExpr>(Val: Arg))
3293 if (CE->getCastKind() == CK_BitCast)
3294 Arg = CE->getSubExpr();
3295 EmitTypeCheck(TCK: Kind, Loc, V: Val, Type: Arg->getType(), Alignment: A.getAlignment(),
3296 SkippedChecks);
3297 }
3298 };
3299
3300 switch (BuiltinIDIfNoAsmLabel) {
3301 default: break;
3302 case Builtin::BI__builtin___CFStringMakeConstantString:
3303 case Builtin::BI__builtin___NSStringMakeConstantString:
3304 return RValue::get(V: ConstantEmitter(*this).emitAbstract(E, T: E->getType()));
3305 case Builtin::BI__builtin_stdarg_start:
3306 case Builtin::BI__builtin_va_start:
3307 case Builtin::BI__va_start:
3308 case Builtin::BI__builtin_c23_va_start:
3309 case Builtin::BI__builtin_va_end:
3310 EmitVAStartEnd(ArgValue: BuiltinID == Builtin::BI__va_start
3311 ? EmitScalarExpr(E: E->getArg(Arg: 0))
3312 : EmitVAListRef(E: E->getArg(Arg: 0)).emitRawPointer(CGF&: *this),
3313 IsStart: BuiltinID != Builtin::BI__builtin_va_end);
3314 return RValue::get(V: nullptr);
3315 case Builtin::BI__builtin_va_copy: {
3316 Value *DstPtr = EmitVAListRef(E: E->getArg(Arg: 0)).emitRawPointer(CGF&: *this);
3317 Value *SrcPtr = EmitVAListRef(E: E->getArg(Arg: 1)).emitRawPointer(CGF&: *this);
3318 Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Intrinsic::vacopy, Tys: {DstPtr->getType()}),
3319 Args: {DstPtr, SrcPtr});
3320 return RValue::get(V: nullptr);
3321 }
3322 case Builtin::BIabs:
3323 case Builtin::BIlabs:
3324 case Builtin::BIllabs:
3325 case Builtin::BI__builtin_abs:
3326 case Builtin::BI__builtin_labs:
3327 case Builtin::BI__builtin_llabs: {
3328 bool SanitizeOverflow = SanOpts.has(K: SanitizerKind::SignedIntegerOverflow);
3329
3330 Value *Result;
3331 switch (getLangOpts().getSignedOverflowBehavior()) {
3332 case LangOptions::SOB_Defined:
3333 Result = EmitAbs(CGF&: *this, ArgValue: EmitScalarExpr(E: E->getArg(Arg: 0)), HasNSW: false);
3334 break;
3335 case LangOptions::SOB_Undefined:
3336 if (!SanitizeOverflow) {
3337 Result = EmitAbs(CGF&: *this, ArgValue: EmitScalarExpr(E: E->getArg(Arg: 0)), HasNSW: true);
3338 break;
3339 }
3340 [[fallthrough]];
3341 case LangOptions::SOB_Trapping:
3342 // TODO: Somehow handle the corner case when the address of abs is taken.
3343 Result = EmitOverflowCheckedAbs(CGF&: *this, E, SanitizeOverflow);
3344 break;
3345 }
3346 return RValue::get(V: Result);
3347 }
3348 case Builtin::BI__builtin_complex: {
3349 Value *Real = EmitScalarExpr(E: E->getArg(Arg: 0));
3350 Value *Imag = EmitScalarExpr(E: E->getArg(Arg: 1));
3351 return RValue::getComplex(C: {Real, Imag});
3352 }
3353 case Builtin::BI__builtin_conj:
3354 case Builtin::BI__builtin_conjf:
3355 case Builtin::BI__builtin_conjl:
3356 case Builtin::BIconj:
3357 case Builtin::BIconjf:
3358 case Builtin::BIconjl: {
3359 ComplexPairTy ComplexVal = EmitComplexExpr(E: E->getArg(Arg: 0));
3360 Value *Real = ComplexVal.first;
3361 Value *Imag = ComplexVal.second;
3362 Imag = Builder.CreateFNeg(V: Imag, Name: "neg");
3363 return RValue::getComplex(C: std::make_pair(x&: Real, y&: Imag));
3364 }
3365 case Builtin::BI__builtin_creal:
3366 case Builtin::BI__builtin_crealf:
3367 case Builtin::BI__builtin_creall:
3368 case Builtin::BIcreal:
3369 case Builtin::BIcrealf:
3370 case Builtin::BIcreall: {
3371 ComplexPairTy ComplexVal = EmitComplexExpr(E: E->getArg(Arg: 0));
3372 return RValue::get(V: ComplexVal.first);
3373 }
3374
3375 case Builtin::BI__builtin_preserve_access_index: {
3376 // Only enabled preserved access index region when debuginfo
3377 // is available as debuginfo is needed to preserve user-level
3378 // access pattern.
3379 if (!getDebugInfo()) {
3380 CGM.Error(loc: E->getExprLoc(), error: "using builtin_preserve_access_index() without -g");
3381 return RValue::get(V: EmitScalarExpr(E: E->getArg(Arg: 0)));
3382 }
3383
3384 // Nested builtin_preserve_access_index() not supported
3385 if (IsInPreservedAIRegion) {
3386 CGM.Error(loc: E->getExprLoc(), error: "nested builtin_preserve_access_index() not supported");
3387 return RValue::get(V: EmitScalarExpr(E: E->getArg(Arg: 0)));
3388 }
3389
3390 IsInPreservedAIRegion = true;
3391 Value *Res = EmitScalarExpr(E: E->getArg(Arg: 0));
3392 IsInPreservedAIRegion = false;
3393 return RValue::get(V: Res);
3394 }
3395
3396 case Builtin::BI__builtin_cimag:
3397 case Builtin::BI__builtin_cimagf:
3398 case Builtin::BI__builtin_cimagl:
3399 case Builtin::BIcimag:
3400 case Builtin::BIcimagf:
3401 case Builtin::BIcimagl: {
3402 ComplexPairTy ComplexVal = EmitComplexExpr(E: E->getArg(Arg: 0));
3403 return RValue::get(V: ComplexVal.second);
3404 }
3405
3406 case Builtin::BI__builtin_clrsb:
3407 case Builtin::BI__builtin_clrsbl:
3408 case Builtin::BI__builtin_clrsbll: {
3409 // clrsb(x) -> clz(x < 0 ? ~x : x) - 1 or
3410 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
3411
3412 llvm::Type *ArgType = ArgValue->getType();
3413 Function *F = CGM.getIntrinsic(IID: Intrinsic::ctlz, Tys: ArgType);
3414
3415 llvm::Type *ResultType = ConvertType(T: E->getType());
3416 Value *Zero = llvm::Constant::getNullValue(Ty: ArgType);
3417 Value *IsNeg = Builder.CreateICmpSLT(LHS: ArgValue, RHS: Zero, Name: "isneg");
3418 Value *Inverse = Builder.CreateNot(V: ArgValue, Name: "not");
3419 Value *Tmp = Builder.CreateSelect(C: IsNeg, True: Inverse, False: ArgValue);
3420 Value *Ctlz = Builder.CreateCall(Callee: F, Args: {Tmp, Builder.getFalse()});
3421 Value *Result =
3422 Builder.CreateNUWSub(LHS: Ctlz, RHS: llvm::ConstantInt::get(Ty: ArgType, V: 1));
3423 Result = Builder.CreateIntCast(V: Result, DestTy: ResultType, /*isSigned*/true,
3424 Name: "cast");
3425 return RValue::get(V: Result);
3426 }
3427 case Builtin::BI__builtin_ctzs:
3428 case Builtin::BI__builtin_ctz:
3429 case Builtin::BI__builtin_ctzl:
3430 case Builtin::BI__builtin_ctzll:
3431 case Builtin::BI__builtin_ctzg:
3432 case Builtin::BI__builtin_elementwise_ctzg: {
3433 bool HasFallback =
3434 (BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_ctzg ||
3435 BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_elementwise_ctzg) &&
3436 E->getNumArgs() > 1;
3437
3438 Value *ArgValue =
3439 HasFallback ? EmitBitCountExpr(CGF&: *this, E: E->getArg(Arg: 0))
3440 : EmitCheckedArgForBuiltin(E: E->getArg(Arg: 0), Kind: BCK_CTZPassedZero);
3441
3442 llvm::Type *ArgType = ArgValue->getType();
3443 Function *F = CGM.getIntrinsic(IID: Intrinsic::cttz, Tys: ArgType);
3444
3445 llvm::Type *ResultType = ConvertType(T: E->getType());
3446 // The elementwise builtins always exhibit zero-is-undef behaviour
3447 Value *ZeroUndef = Builder.getInt1(
3448 V: HasFallback || getTarget().isCLZForZeroUndef() ||
3449 BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_elementwise_ctzg);
3450 Value *Result = Builder.CreateCall(Callee: F, Args: {ArgValue, ZeroUndef});
3451 if (Result->getType() != ResultType)
3452 Result =
3453 Builder.CreateIntCast(V: Result, DestTy: ResultType, /*isSigned*/ false, Name: "cast");
3454 if (!HasFallback)
3455 return RValue::get(V: Result);
3456
3457 Value *Zero = Constant::getNullValue(Ty: ArgType);
3458 Value *IsZero = Builder.CreateICmpEQ(LHS: ArgValue, RHS: Zero, Name: "iszero");
3459 Value *FallbackValue = EmitScalarExpr(E: E->getArg(Arg: 1));
3460 Value *ResultOrFallback =
3461 Builder.CreateSelect(C: IsZero, True: FallbackValue, False: Result, Name: "ctzg");
3462 return RValue::get(V: ResultOrFallback);
3463 }
3464 case Builtin::BI__builtin_clzs:
3465 case Builtin::BI__builtin_clz:
3466 case Builtin::BI__builtin_clzl:
3467 case Builtin::BI__builtin_clzll:
3468 case Builtin::BI__builtin_clzg:
3469 case Builtin::BI__builtin_elementwise_clzg: {
3470 bool HasFallback =
3471 (BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_clzg ||
3472 BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_elementwise_clzg) &&
3473 E->getNumArgs() > 1;
3474
3475 Value *ArgValue =
3476 HasFallback ? EmitBitCountExpr(CGF&: *this, E: E->getArg(Arg: 0))
3477 : EmitCheckedArgForBuiltin(E: E->getArg(Arg: 0), Kind: BCK_CLZPassedZero);
3478
3479 llvm::Type *ArgType = ArgValue->getType();
3480 Function *F = CGM.getIntrinsic(IID: Intrinsic::ctlz, Tys: ArgType);
3481
3482 llvm::Type *ResultType = ConvertType(T: E->getType());
3483 // The elementwise builtins always exhibit zero-is-undef behaviour
3484 Value *ZeroUndef = Builder.getInt1(
3485 V: HasFallback || getTarget().isCLZForZeroUndef() ||
3486 BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_elementwise_clzg);
3487 Value *Result = Builder.CreateCall(Callee: F, Args: {ArgValue, ZeroUndef});
3488 if (Result->getType() != ResultType)
3489 Result =
3490 Builder.CreateIntCast(V: Result, DestTy: ResultType, /*isSigned*/ false, Name: "cast");
3491 if (!HasFallback)
3492 return RValue::get(V: Result);
3493
3494 Value *Zero = Constant::getNullValue(Ty: ArgType);
3495 Value *IsZero = Builder.CreateICmpEQ(LHS: ArgValue, RHS: Zero, Name: "iszero");
3496 Value *FallbackValue = EmitScalarExpr(E: E->getArg(Arg: 1));
3497 Value *ResultOrFallback =
3498 Builder.CreateSelect(C: IsZero, True: FallbackValue, False: Result, Name: "clzg");
3499 return RValue::get(V: ResultOrFallback);
3500 }
3501 case Builtin::BI__builtin_ffs:
3502 case Builtin::BI__builtin_ffsl:
3503 case Builtin::BI__builtin_ffsll: {
3504 // ffs(x) -> x ? cttz(x) + 1 : 0
3505 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
3506
3507 llvm::Type *ArgType = ArgValue->getType();
3508 Function *F = CGM.getIntrinsic(IID: Intrinsic::cttz, Tys: ArgType);
3509
3510 llvm::Type *ResultType = ConvertType(T: E->getType());
3511 Value *Tmp =
3512 Builder.CreateAdd(LHS: Builder.CreateCall(Callee: F, Args: {ArgValue, Builder.getTrue()}),
3513 RHS: llvm::ConstantInt::get(Ty: ArgType, V: 1));
3514 Value *Zero = llvm::Constant::getNullValue(Ty: ArgType);
3515 Value *IsZero = Builder.CreateICmpEQ(LHS: ArgValue, RHS: Zero, Name: "iszero");
3516 Value *Result = Builder.CreateSelect(C: IsZero, True: Zero, False: Tmp, Name: "ffs");
3517 if (Result->getType() != ResultType)
3518 Result = Builder.CreateIntCast(V: Result, DestTy: ResultType, /*isSigned*/true,
3519 Name: "cast");
3520 return RValue::get(V: Result);
3521 }
3522 case Builtin::BI__builtin_parity:
3523 case Builtin::BI__builtin_parityl:
3524 case Builtin::BI__builtin_parityll: {
3525 // parity(x) -> ctpop(x) & 1
3526 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
3527
3528 llvm::Type *ArgType = ArgValue->getType();
3529 Function *F = CGM.getIntrinsic(IID: Intrinsic::ctpop, Tys: ArgType);
3530
3531 llvm::Type *ResultType = ConvertType(T: E->getType());
3532 Value *Tmp = Builder.CreateCall(Callee: F, Args: ArgValue);
3533 Value *Result = Builder.CreateAnd(LHS: Tmp, RHS: llvm::ConstantInt::get(Ty: ArgType, V: 1));
3534 if (Result->getType() != ResultType)
3535 Result = Builder.CreateIntCast(V: Result, DestTy: ResultType, /*isSigned*/true,
3536 Name: "cast");
3537 return RValue::get(V: Result);
3538 }
3539 case Builtin::BI__lzcnt16:
3540 case Builtin::BI__lzcnt:
3541 case Builtin::BI__lzcnt64: {
3542 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
3543
3544 llvm::Type *ArgType = ArgValue->getType();
3545 Function *F = CGM.getIntrinsic(IID: Intrinsic::ctlz, Tys: ArgType);
3546
3547 llvm::Type *ResultType = ConvertType(T: E->getType());
3548 Value *Result = Builder.CreateCall(Callee: F, Args: {ArgValue, Builder.getFalse()});
3549 if (Result->getType() != ResultType)
3550 Result = Builder.CreateIntCast(V: Result, DestTy: ResultType, /*isSigned*/true,
3551 Name: "cast");
3552 return RValue::get(V: Result);
3553 }
3554 case Builtin::BI__popcnt16:
3555 case Builtin::BI__popcnt:
3556 case Builtin::BI__popcnt64:
3557 case Builtin::BI__builtin_popcount:
3558 case Builtin::BI__builtin_popcountl:
3559 case Builtin::BI__builtin_popcountll:
3560 case Builtin::BI__builtin_popcountg: {
3561 Value *ArgValue = EmitBitCountExpr(CGF&: *this, E: E->getArg(Arg: 0));
3562
3563 llvm::Type *ArgType = ArgValue->getType();
3564 Function *F = CGM.getIntrinsic(IID: Intrinsic::ctpop, Tys: ArgType);
3565
3566 llvm::Type *ResultType = ConvertType(T: E->getType());
3567 Value *Result = Builder.CreateCall(Callee: F, Args: ArgValue);
3568 if (Result->getType() != ResultType)
3569 Result =
3570 Builder.CreateIntCast(V: Result, DestTy: ResultType, /*isSigned*/ false, Name: "cast");
3571 return RValue::get(V: Result);
3572 }
3573 case Builtin::BI__builtin_unpredictable: {
3574 // Always return the argument of __builtin_unpredictable. LLVM does not
3575 // handle this builtin. Metadata for this builtin should be added directly
3576 // to instructions such as branches or switches that use it.
3577 return RValue::get(V: EmitScalarExpr(E: E->getArg(Arg: 0)));
3578 }
3579 case Builtin::BI__builtin_expect: {
3580 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
3581 llvm::Type *ArgType = ArgValue->getType();
3582
3583 Value *ExpectedValue = EmitScalarExpr(E: E->getArg(Arg: 1));
3584 // Don't generate llvm.expect on -O0 as the backend won't use it for
3585 // anything.
3586 // Note, we still IRGen ExpectedValue because it could have side-effects.
3587 if (CGM.getCodeGenOpts().OptimizationLevel == 0)
3588 return RValue::get(V: ArgValue);
3589
3590 Function *FnExpect = CGM.getIntrinsic(IID: Intrinsic::expect, Tys: ArgType);
3591 Value *Result =
3592 Builder.CreateCall(Callee: FnExpect, Args: {ArgValue, ExpectedValue}, Name: "expval");
3593 return RValue::get(V: Result);
3594 }
3595 case Builtin::BI__builtin_expect_with_probability: {
3596 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
3597 llvm::Type *ArgType = ArgValue->getType();
3598
3599 Value *ExpectedValue = EmitScalarExpr(E: E->getArg(Arg: 1));
3600 llvm::APFloat Probability(0.0);
3601 const Expr *ProbArg = E->getArg(Arg: 2);
3602 bool EvalSucceed = ProbArg->EvaluateAsFloat(Result&: Probability, Ctx: CGM.getContext());
3603 assert(EvalSucceed && "probability should be able to evaluate as float");
3604 (void)EvalSucceed;
3605 bool LoseInfo = false;
3606 Probability.convert(ToSemantics: llvm::APFloat::IEEEdouble(),
3607 RM: llvm::RoundingMode::Dynamic, losesInfo: &LoseInfo);
3608 llvm::Type *Ty = ConvertType(T: ProbArg->getType());
3609 Constant *Confidence = ConstantFP::get(Ty, V: Probability);
3610 // Don't generate llvm.expect.with.probability on -O0 as the backend
3611 // won't use it for anything.
3612 // Note, we still IRGen ExpectedValue because it could have side-effects.
3613 if (CGM.getCodeGenOpts().OptimizationLevel == 0)
3614 return RValue::get(V: ArgValue);
3615
3616 Function *FnExpect =
3617 CGM.getIntrinsic(IID: Intrinsic::expect_with_probability, Tys: ArgType);
3618 Value *Result = Builder.CreateCall(
3619 Callee: FnExpect, Args: {ArgValue, ExpectedValue, Confidence}, Name: "expval");
3620 return RValue::get(V: Result);
3621 }
3622 case Builtin::BI__builtin_assume_aligned: {
3623 const Expr *Ptr = E->getArg(Arg: 0);
3624 Value *PtrValue = EmitScalarExpr(E: Ptr);
3625 Value *OffsetValue =
3626 (E->getNumArgs() > 2) ? EmitScalarExpr(E: E->getArg(Arg: 2)) : nullptr;
3627
3628 Value *AlignmentValue = EmitScalarExpr(E: E->getArg(Arg: 1));
3629 ConstantInt *AlignmentCI = cast<ConstantInt>(Val: AlignmentValue);
3630 if (AlignmentCI->getValue().ugt(RHS: llvm::Value::MaximumAlignment))
3631 AlignmentCI = ConstantInt::get(Ty: AlignmentCI->getIntegerType(),
3632 V: llvm::Value::MaximumAlignment);
3633
3634 emitAlignmentAssumption(PtrValue, E: Ptr,
3635 /*The expr loc is sufficient.*/ AssumptionLoc: SourceLocation(),
3636 Alignment: AlignmentCI, OffsetValue);
3637 return RValue::get(V: PtrValue);
3638 }
3639 case Builtin::BI__builtin_assume_dereferenceable: {
3640 const Expr *Ptr = E->getArg(Arg: 0);
3641 const Expr *Size = E->getArg(Arg: 1);
3642 Value *PtrValue = EmitScalarExpr(E: Ptr);
3643 Value *SizeValue = EmitScalarExpr(E: Size);
3644 if (SizeValue->getType() != IntPtrTy)
3645 SizeValue =
3646 Builder.CreateIntCast(V: SizeValue, DestTy: IntPtrTy, isSigned: false, Name: "casted.size");
3647 Builder.CreateDereferenceableAssumption(PtrValue, SizeValue);
3648 return RValue::get(V: nullptr);
3649 }
3650 case Builtin::BI__assume:
3651 case Builtin::BI__builtin_assume: {
3652 if (E->getArg(Arg: 0)->HasSideEffects(Ctx: getContext()))
3653 return RValue::get(V: nullptr);
3654
3655 Value *ArgValue = EmitCheckedArgForAssume(E: E->getArg(Arg: 0));
3656 Function *FnAssume = CGM.getIntrinsic(IID: Intrinsic::assume);
3657 Builder.CreateCall(Callee: FnAssume, Args: ArgValue);
3658 return RValue::get(V: nullptr);
3659 }
3660 case Builtin::BI__builtin_assume_separate_storage: {
3661 const Expr *Arg0 = E->getArg(Arg: 0);
3662 const Expr *Arg1 = E->getArg(Arg: 1);
3663
3664 Value *Value0 = EmitScalarExpr(E: Arg0);
3665 Value *Value1 = EmitScalarExpr(E: Arg1);
3666
3667 Value *Values[] = {Value0, Value1};
3668 OperandBundleDefT<Value *> OBD("separate_storage", Values);
3669 Builder.CreateAssumption(OpBundles: {OBD});
3670 return RValue::get(V: nullptr);
3671 }
3672 case Builtin::BI__builtin_allow_runtime_check: {
3673 StringRef Kind =
3674 cast<StringLiteral>(Val: E->getArg(Arg: 0)->IgnoreParenCasts())->getString();
3675 LLVMContext &Ctx = CGM.getLLVMContext();
3676 llvm::Value *Allow = Builder.CreateCall(
3677 Callee: CGM.getIntrinsic(IID: Intrinsic::allow_runtime_check),
3678 Args: llvm::MetadataAsValue::get(Context&: Ctx, MD: llvm::MDString::get(Context&: Ctx, Str: Kind)));
3679 return RValue::get(V: Allow);
3680 }
3681 case Builtin::BI__builtin_allow_sanitize_check: {
3682 Intrinsic::ID IntrID = Intrinsic::not_intrinsic;
3683 StringRef Name =
3684 cast<StringLiteral>(Val: E->getArg(Arg: 0)->IgnoreParenCasts())->getString();
3685
3686 // We deliberately allow the use of kernel- and non-kernel names
3687 // interchangably, even when one or the other is enabled. This is consistent
3688 // with the no_sanitize-attribute, which allows either kernel- or non-kernel
3689 // name to disable instrumentation (see CodeGenFunction::StartFunction).
3690 if (getLangOpts().Sanitize.hasOneOf(K: SanitizerKind::Address |
3691 SanitizerKind::KernelAddress) &&
3692 (Name == "address" || Name == "kernel-address")) {
3693 IntrID = Intrinsic::allow_sanitize_address;
3694 } else if (getLangOpts().Sanitize.has(K: SanitizerKind::Thread) &&
3695 Name == "thread") {
3696 IntrID = Intrinsic::allow_sanitize_thread;
3697 } else if (getLangOpts().Sanitize.hasOneOf(K: SanitizerKind::Memory |
3698 SanitizerKind::KernelMemory) &&
3699 (Name == "memory" || Name == "kernel-memory")) {
3700 IntrID = Intrinsic::allow_sanitize_memory;
3701 } else if (getLangOpts().Sanitize.hasOneOf(
3702 K: SanitizerKind::HWAddress | SanitizerKind::KernelHWAddress) &&
3703 (Name == "hwaddress" || Name == "kernel-hwaddress")) {
3704 IntrID = Intrinsic::allow_sanitize_hwaddress;
3705 }
3706
3707 if (IntrID != Intrinsic::not_intrinsic) {
3708 llvm::Value *Allow = Builder.CreateCall(Callee: CGM.getIntrinsic(IID: IntrID));
3709 return RValue::get(V: Allow);
3710 }
3711 // If the checked sanitizer is not enabled, we can safely lower to false
3712 // right away. This is also more efficient, since the LowerAllowCheckPass
3713 // must not always be enabled if none of the above sanitizers are enabled.
3714 return RValue::get(V: Builder.getFalse());
3715 }
3716 case Builtin::BI__arithmetic_fence: {
3717 // Create the builtin call if FastMath is selected, and the target
3718 // supports the builtin, otherwise just return the argument.
3719 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
3720 llvm::FastMathFlags FMF = Builder.getFastMathFlags();
3721 bool isArithmeticFenceEnabled =
3722 FMF.allowReassoc() &&
3723 getContext().getTargetInfo().checkArithmeticFenceSupported();
3724 QualType ArgType = E->getArg(Arg: 0)->getType();
3725 if (ArgType->isComplexType()) {
3726 if (isArithmeticFenceEnabled) {
3727 QualType ElementType = ArgType->castAs<ComplexType>()->getElementType();
3728 ComplexPairTy ComplexVal = EmitComplexExpr(E: E->getArg(Arg: 0));
3729 Value *Real = Builder.CreateArithmeticFence(Val: ComplexVal.first,
3730 DstType: ConvertType(T: ElementType));
3731 Value *Imag = Builder.CreateArithmeticFence(Val: ComplexVal.second,
3732 DstType: ConvertType(T: ElementType));
3733 return RValue::getComplex(C: std::make_pair(x&: Real, y&: Imag));
3734 }
3735 ComplexPairTy ComplexVal = EmitComplexExpr(E: E->getArg(Arg: 0));
3736 Value *Real = ComplexVal.first;
3737 Value *Imag = ComplexVal.second;
3738 return RValue::getComplex(C: std::make_pair(x&: Real, y&: Imag));
3739 }
3740 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
3741 if (isArithmeticFenceEnabled)
3742 return RValue::get(
3743 V: Builder.CreateArithmeticFence(Val: ArgValue, DstType: ConvertType(T: ArgType)));
3744 return RValue::get(V: ArgValue);
3745 }
3746 case Builtin::BI__builtin_bswapg: {
3747 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
3748 llvm::IntegerType *IntTy = cast<llvm::IntegerType>(Val: ArgValue->getType());
3749 assert(IntTy && "LLVM's __builtin_bswapg only supports integer variants");
3750 if (IntTy->getBitWidth() == 1 || IntTy->getBitWidth() == 8)
3751 return RValue::get(V: ArgValue);
3752 assert(((IntTy->getBitWidth() % 16 == 0 && IntTy->getBitWidth() != 0)) &&
3753 "LLVM's __builtin_bswapg only supports integer variants that has a "
3754 "multiple of 16 bits as well as a single byte");
3755 return RValue::get(
3756 V: emitBuiltinWithOneOverloadedType<1>(CGF&: *this, E, IntrinsicID: Intrinsic::bswap));
3757 }
3758 case Builtin::BI__builtin_bswap16:
3759 case Builtin::BI__builtin_bswap32:
3760 case Builtin::BI__builtin_bswap64:
3761 case Builtin::BI_byteswap_ushort:
3762 case Builtin::BI_byteswap_ulong:
3763 case Builtin::BI_byteswap_uint64: {
3764 return RValue::get(
3765 V: emitBuiltinWithOneOverloadedType<1>(CGF&: *this, E, IntrinsicID: Intrinsic::bswap));
3766 }
3767 case Builtin::BI__builtin_bitreverseg: {
3768 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
3769 llvm::IntegerType *IntTy = cast<llvm::IntegerType>(Val: ArgValue->getType());
3770 assert(IntTy &&
3771 "LLVM's __builtin_bitreverseg only support integer variants");
3772 if (IntTy->getBitWidth() == 1)
3773 return RValue::get(V: ArgValue);
3774 return RValue::get(
3775 V: emitBuiltinWithOneOverloadedType<1>(CGF&: *this, E, IntrinsicID: Intrinsic::bitreverse));
3776 }
3777 case Builtin::BI__builtin_bitreverse8:
3778 case Builtin::BI__builtin_bitreverse16:
3779 case Builtin::BI__builtin_bitreverse32:
3780 case Builtin::BI__builtin_bitreverse64: {
3781 return RValue::get(
3782 V: emitBuiltinWithOneOverloadedType<1>(CGF&: *this, E, IntrinsicID: Intrinsic::bitreverse));
3783 }
3784 case Builtin::BI__builtin_rotateleft8:
3785 case Builtin::BI__builtin_rotateleft16:
3786 case Builtin::BI__builtin_rotateleft32:
3787 case Builtin::BI__builtin_rotateleft64:
3788 case Builtin::BI__builtin_stdc_rotate_left:
3789 case Builtin::BIstdc_rotate_left_uc:
3790 case Builtin::BIstdc_rotate_left_us:
3791 case Builtin::BIstdc_rotate_left_ui:
3792 case Builtin::BIstdc_rotate_left_ul:
3793 case Builtin::BIstdc_rotate_left_ull:
3794 case Builtin::BI_rotl8: // Microsoft variants of rotate left
3795 case Builtin::BI_rotl16:
3796 case Builtin::BI_rotl:
3797 case Builtin::BI_lrotl:
3798 case Builtin::BI_rotl64:
3799 return emitRotate(E, IsRotateRight: false);
3800
3801 case Builtin::BI__builtin_rotateright8:
3802 case Builtin::BI__builtin_rotateright16:
3803 case Builtin::BI__builtin_rotateright32:
3804 case Builtin::BI__builtin_rotateright64:
3805 case Builtin::BI__builtin_stdc_rotate_right:
3806 case Builtin::BIstdc_rotate_right_uc:
3807 case Builtin::BIstdc_rotate_right_us:
3808 case Builtin::BIstdc_rotate_right_ui:
3809 case Builtin::BIstdc_rotate_right_ul:
3810 case Builtin::BIstdc_rotate_right_ull:
3811 case Builtin::BI_rotr8: // Microsoft variants of rotate right
3812 case Builtin::BI_rotr16:
3813 case Builtin::BI_rotr:
3814 case Builtin::BI_lrotr:
3815 case Builtin::BI_rotr64:
3816 return emitRotate(E, IsRotateRight: true);
3817
3818 case Builtin::BIstdc_leading_zeros_uc:
3819 case Builtin::BIstdc_leading_zeros_us:
3820 case Builtin::BIstdc_leading_zeros_ui:
3821 case Builtin::BIstdc_leading_zeros_ul:
3822 case Builtin::BIstdc_leading_zeros_ull:
3823 case Builtin::BI__builtin_stdc_leading_zeros:
3824 return emitStdcCountIntrinsic(E, IntID: Intrinsic::ctlz, /*InvertArg=*/false);
3825 case Builtin::BIstdc_leading_ones_uc:
3826 case Builtin::BIstdc_leading_ones_us:
3827 case Builtin::BIstdc_leading_ones_ui:
3828 case Builtin::BIstdc_leading_ones_ul:
3829 case Builtin::BIstdc_leading_ones_ull:
3830 case Builtin::BI__builtin_stdc_leading_ones:
3831 return emitStdcCountIntrinsic(E, IntID: Intrinsic::ctlz, /*InvertArg=*/true);
3832 case Builtin::BIstdc_trailing_zeros_uc:
3833 case Builtin::BIstdc_trailing_zeros_us:
3834 case Builtin::BIstdc_trailing_zeros_ui:
3835 case Builtin::BIstdc_trailing_zeros_ul:
3836 case Builtin::BIstdc_trailing_zeros_ull:
3837 case Builtin::BI__builtin_stdc_trailing_zeros:
3838 return emitStdcCountIntrinsic(E, IntID: Intrinsic::cttz, /*InvertArg=*/false);
3839 case Builtin::BIstdc_trailing_ones_uc:
3840 case Builtin::BIstdc_trailing_ones_us:
3841 case Builtin::BIstdc_trailing_ones_ui:
3842 case Builtin::BIstdc_trailing_ones_ul:
3843 case Builtin::BIstdc_trailing_ones_ull:
3844 case Builtin::BI__builtin_stdc_trailing_ones:
3845 return emitStdcCountIntrinsic(E, IntID: Intrinsic::cttz, /*InvertArg=*/true);
3846 case Builtin::BIstdc_first_leading_zero_uc:
3847 case Builtin::BIstdc_first_leading_zero_us:
3848 case Builtin::BIstdc_first_leading_zero_ui:
3849 case Builtin::BIstdc_first_leading_zero_ul:
3850 case Builtin::BIstdc_first_leading_zero_ull:
3851 case Builtin::BI__builtin_stdc_first_leading_zero:
3852 return emitStdcFirstBit(E, IntID: Intrinsic::ctlz, /*InvertArg=*/true);
3853 case Builtin::BIstdc_first_leading_one_uc:
3854 case Builtin::BIstdc_first_leading_one_us:
3855 case Builtin::BIstdc_first_leading_one_ui:
3856 case Builtin::BIstdc_first_leading_one_ul:
3857 case Builtin::BIstdc_first_leading_one_ull:
3858 case Builtin::BI__builtin_stdc_first_leading_one:
3859 return emitStdcFirstBit(E, IntID: Intrinsic::ctlz, /*InvertArg=*/false);
3860 case Builtin::BIstdc_first_trailing_zero_uc:
3861 case Builtin::BIstdc_first_trailing_zero_us:
3862 case Builtin::BIstdc_first_trailing_zero_ui:
3863 case Builtin::BIstdc_first_trailing_zero_ul:
3864 case Builtin::BIstdc_first_trailing_zero_ull:
3865 case Builtin::BI__builtin_stdc_first_trailing_zero:
3866 return emitStdcFirstBit(E, IntID: Intrinsic::cttz, /*InvertArg=*/true);
3867 case Builtin::BIstdc_first_trailing_one_uc:
3868 case Builtin::BIstdc_first_trailing_one_us:
3869 case Builtin::BIstdc_first_trailing_one_ui:
3870 case Builtin::BIstdc_first_trailing_one_ul:
3871 case Builtin::BIstdc_first_trailing_one_ull:
3872 case Builtin::BI__builtin_stdc_first_trailing_one:
3873 return emitStdcFirstBit(E, IntID: Intrinsic::cttz, /*InvertArg=*/false);
3874 case Builtin::BIstdc_count_zeros_uc:
3875 case Builtin::BIstdc_count_zeros_us:
3876 case Builtin::BIstdc_count_zeros_ui:
3877 case Builtin::BIstdc_count_zeros_ul:
3878 case Builtin::BIstdc_count_zeros_ull:
3879 case Builtin::BI__builtin_stdc_count_zeros:
3880 return emitStdcBitWidthMinus(E, IntID: Intrinsic::ctpop, /*IsPop=*/true);
3881 case Builtin::BIstdc_count_ones_uc:
3882 case Builtin::BIstdc_count_ones_us:
3883 case Builtin::BIstdc_count_ones_ui:
3884 case Builtin::BIstdc_count_ones_ul:
3885 case Builtin::BIstdc_count_ones_ull:
3886 case Builtin::BI__builtin_stdc_count_ones:
3887 return emitStdcCountIntrinsic(E, IntID: Intrinsic::ctpop, /*InvertArg=*/false,
3888 /*IsPop=*/true);
3889 case Builtin::BIstdc_has_single_bit_uc:
3890 case Builtin::BIstdc_has_single_bit_us:
3891 case Builtin::BIstdc_has_single_bit_ui:
3892 case Builtin::BIstdc_has_single_bit_ul:
3893 case Builtin::BIstdc_has_single_bit_ull:
3894 case Builtin::BI__builtin_stdc_has_single_bit: {
3895 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
3896 llvm::Type *ArgType = ArgValue->getType();
3897 Value *One = ConstantInt::get(Ty: ArgType, V: 1);
3898 Function *F = CGM.getIntrinsic(IID: Intrinsic::ctpop, Tys: ArgType);
3899 Value *PopCnt = Builder.CreateCall(Callee: F, Args: ArgValue);
3900 return RValue::get(V: Builder.CreateICmpEQ(LHS: PopCnt, RHS: One));
3901 }
3902 case Builtin::BIstdc_bit_width_uc:
3903 case Builtin::BIstdc_bit_width_us:
3904 case Builtin::BIstdc_bit_width_ui:
3905 case Builtin::BIstdc_bit_width_ul:
3906 case Builtin::BIstdc_bit_width_ull:
3907 case Builtin::BI__builtin_stdc_bit_width:
3908 return emitStdcBitWidthMinus(E, IntID: Intrinsic::ctlz, /*IsPop=*/false);
3909 case Builtin::BIstdc_bit_floor_uc:
3910 case Builtin::BIstdc_bit_floor_us:
3911 case Builtin::BIstdc_bit_floor_ui:
3912 case Builtin::BIstdc_bit_floor_ul:
3913 case Builtin::BIstdc_bit_floor_ull:
3914 case Builtin::BI__builtin_stdc_bit_floor: {
3915 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
3916 llvm::Type *ArgType = ArgValue->getType();
3917 unsigned BitWidth = ArgType->getIntegerBitWidth();
3918 Value *Zero = ConstantInt::get(Ty: ArgType, V: 0);
3919 Value *One = ConstantInt::get(Ty: ArgType, V: 1);
3920 Function *F = CGM.getIntrinsic(IID: Intrinsic::ctlz, Tys: ArgType);
3921 Value *LZ = Builder.CreateCall(Callee: F, Args: {ArgValue, Builder.getTrue()});
3922 Value *ShiftAmt =
3923 Builder.CreateSub(LHS: ConstantInt::get(Ty: ArgType, V: BitWidth - 1), RHS: LZ);
3924 Value *Shifted = Builder.CreateShl(LHS: One, RHS: ShiftAmt);
3925 Value *IsZero = Builder.CreateICmpEQ(LHS: ArgValue, RHS: Zero);
3926 Value *Result = Builder.CreateSelect(C: IsZero, True: Zero, False: Shifted);
3927 return RValue::get(V: Result);
3928 }
3929 case Builtin::BIstdc_bit_ceil_uc:
3930 case Builtin::BIstdc_bit_ceil_us:
3931 case Builtin::BIstdc_bit_ceil_ui:
3932 case Builtin::BIstdc_bit_ceil_ul:
3933 case Builtin::BIstdc_bit_ceil_ull:
3934 case Builtin::BI__builtin_stdc_bit_ceil: {
3935 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
3936 llvm::Type *ArgType = ArgValue->getType();
3937 Value *One = ConstantInt::get(Ty: ArgType, V: 1);
3938 Value *IsLEOne = Builder.CreateICmpULE(LHS: ArgValue, RHS: One, Name: "isleone");
3939
3940 BasicBlock *EntryBB = Builder.GetInsertBlock();
3941 BasicBlock *CalcBB = createBasicBlock(name: "bitceil.calc", parent: CurFn);
3942 BasicBlock *MergeBB = createBasicBlock(name: "bitceil.merge", parent: CurFn);
3943
3944 Builder.CreateCondBr(Cond: IsLEOne, True: MergeBB, False: CalcBB);
3945
3946 Builder.SetInsertPoint(CalcBB);
3947 Function *F = CGM.getIntrinsic(IID: Intrinsic::ctlz, Tys: ArgType);
3948 Value *ArgMinusOne = Builder.CreateSub(LHS: ArgValue, RHS: One);
3949 Value *LZ = Builder.CreateCall(Callee: F, Args: {ArgMinusOne, Builder.getFalse()});
3950 // 2<<(BitWidth-1-LZ) to get the next power of two. The shift
3951 // amount is always in [0, BitWidth-1], so when LZ==0 (argument has its MSB
3952 // set), the result wraps to 0
3953 unsigned BitWidth = ArgType->getIntegerBitWidth();
3954 Value *ShiftAmt =
3955 Builder.CreateSub(LHS: ConstantInt::get(Ty: ArgType, V: BitWidth - 1), RHS: LZ);
3956 Value *Two = Builder.CreateShl(LHS: One, RHS: One);
3957 Value *Tmp = Builder.CreateShl(LHS: Two, RHS: ShiftAmt);
3958 Builder.CreateBr(Dest: MergeBB);
3959
3960 Builder.SetInsertPoint(MergeBB);
3961 PHINode *Phi = Builder.CreatePHI(Ty: ArgType, NumReservedValues: 2);
3962 Phi->addIncoming(V: One, BB: EntryBB);
3963 Phi->addIncoming(V: Tmp, BB: CalcBB);
3964 return RValue::get(V: Phi);
3965 }
3966
3967 // stdc_memreverse8u8 is a no-op (single byte, nothing to swap).
3968 case Builtin::BIstdc_memreverse8u8:
3969 return RValue::get(V: EmitScalarExpr(E: E->getArg(Arg: 0)));
3970
3971 case Builtin::BIstdc_memreverse8u16:
3972 case Builtin::BIstdc_memreverse8u32:
3973 case Builtin::BIstdc_memreverse8u64:
3974 return RValue::get(
3975 V: emitBuiltinWithOneOverloadedType<1>(CGF&: *this, E, IntrinsicID: Intrinsic::bswap));
3976
3977 case Builtin::BIstdc_memreverse8:
3978 case Builtin::BI__builtin_stdc_memreverse8: {
3979 Expr::EvalResult R;
3980 if (E->getArg(Arg: 0)->EvaluateAsInt(Result&: R, Ctx: getContext())) {
3981 uint64_t Size = R.Val.getInt().getZExtValue();
3982 if (Size <= 1) {
3983 EmitIgnoredExpr(E: E->getArg(Arg: 1));
3984 return RValue::get(V: nullptr);
3985 }
3986 if (Size == 2 || Size == 4 || Size == 8) {
3987 llvm::Type *IntTy = Builder.getIntNTy(N: Size * 8);
3988 Address PtrAddr = EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
3989 Address Addr = PtrAddr.withElementType(ElemTy: IntTy);
3990 Value *Val = Builder.CreateLoad(Addr);
3991 Function *F = CGM.getIntrinsic(IID: Intrinsic::bswap, Tys: IntTy);
3992 Value *Swapped = Builder.CreateCall(Callee: F, Args: Val);
3993 Builder.CreateStore(Val: Swapped, Addr);
3994 return RValue::get(V: nullptr);
3995 }
3996 }
3997
3998 // General case: fall back to the library function stdc_memreverse8.
3999 break;
4000 }
4001
4002 // stdc_load8_*: load N bytes from a const unsigned char* and assemble
4003 // into an integer using little-endian or big-endian byte order.
4004 case Builtin::BIstdc_load8_leu8:
4005 case Builtin::BIstdc_load8_les8:
4006 case Builtin::BIstdc_load8_beu8:
4007 case Builtin::BIstdc_load8_bes8:
4008 case Builtin::BIstdc_load8_aligned_leu8:
4009 case Builtin::BIstdc_load8_aligned_les8:
4010 case Builtin::BIstdc_load8_aligned_beu8:
4011 case Builtin::BIstdc_load8_aligned_bes8:
4012 case Builtin::BIstdc_load8_leu16:
4013 case Builtin::BIstdc_load8_les16:
4014 case Builtin::BIstdc_load8_beu16:
4015 case Builtin::BIstdc_load8_bes16:
4016 case Builtin::BIstdc_load8_aligned_leu16:
4017 case Builtin::BIstdc_load8_aligned_les16:
4018 case Builtin::BIstdc_load8_aligned_beu16:
4019 case Builtin::BIstdc_load8_aligned_bes16:
4020 case Builtin::BIstdc_load8_leu32:
4021 case Builtin::BIstdc_load8_les32:
4022 case Builtin::BIstdc_load8_beu32:
4023 case Builtin::BIstdc_load8_bes32:
4024 case Builtin::BIstdc_load8_aligned_leu32:
4025 case Builtin::BIstdc_load8_aligned_les32:
4026 case Builtin::BIstdc_load8_aligned_beu32:
4027 case Builtin::BIstdc_load8_aligned_bes32:
4028 case Builtin::BIstdc_load8_leu64:
4029 case Builtin::BIstdc_load8_les64:
4030 case Builtin::BIstdc_load8_beu64:
4031 case Builtin::BIstdc_load8_bes64:
4032 case Builtin::BIstdc_load8_aligned_leu64:
4033 case Builtin::BIstdc_load8_aligned_les64:
4034 case Builtin::BIstdc_load8_aligned_beu64:
4035 case Builtin::BIstdc_load8_aligned_bes64: {
4036 bool IsBE = BuiltinID == Builtin::BIstdc_load8_beu8 ||
4037 BuiltinID == Builtin::BIstdc_load8_bes8 ||
4038 BuiltinID == Builtin::BIstdc_load8_beu16 ||
4039 BuiltinID == Builtin::BIstdc_load8_bes16 ||
4040 BuiltinID == Builtin::BIstdc_load8_beu32 ||
4041 BuiltinID == Builtin::BIstdc_load8_bes32 ||
4042 BuiltinID == Builtin::BIstdc_load8_beu64 ||
4043 BuiltinID == Builtin::BIstdc_load8_bes64 ||
4044 BuiltinID == Builtin::BIstdc_load8_aligned_beu8 ||
4045 BuiltinID == Builtin::BIstdc_load8_aligned_bes8 ||
4046 BuiltinID == Builtin::BIstdc_load8_aligned_beu16 ||
4047 BuiltinID == Builtin::BIstdc_load8_aligned_bes16 ||
4048 BuiltinID == Builtin::BIstdc_load8_aligned_beu32 ||
4049 BuiltinID == Builtin::BIstdc_load8_aligned_bes32 ||
4050 BuiltinID == Builtin::BIstdc_load8_aligned_beu64 ||
4051 BuiltinID == Builtin::BIstdc_load8_aligned_bes64;
4052 bool IsAligned = BuiltinID == Builtin::BIstdc_load8_aligned_leu8 ||
4053 BuiltinID == Builtin::BIstdc_load8_aligned_les8 ||
4054 BuiltinID == Builtin::BIstdc_load8_aligned_beu8 ||
4055 BuiltinID == Builtin::BIstdc_load8_aligned_bes8 ||
4056 BuiltinID == Builtin::BIstdc_load8_aligned_leu16 ||
4057 BuiltinID == Builtin::BIstdc_load8_aligned_les16 ||
4058 BuiltinID == Builtin::BIstdc_load8_aligned_beu16 ||
4059 BuiltinID == Builtin::BIstdc_load8_aligned_bes16 ||
4060 BuiltinID == Builtin::BIstdc_load8_aligned_leu32 ||
4061 BuiltinID == Builtin::BIstdc_load8_aligned_les32 ||
4062 BuiltinID == Builtin::BIstdc_load8_aligned_beu32 ||
4063 BuiltinID == Builtin::BIstdc_load8_aligned_bes32 ||
4064 BuiltinID == Builtin::BIstdc_load8_aligned_leu64 ||
4065 BuiltinID == Builtin::BIstdc_load8_aligned_les64 ||
4066 BuiltinID == Builtin::BIstdc_load8_aligned_beu64 ||
4067 BuiltinID == Builtin::BIstdc_load8_aligned_bes64;
4068
4069 llvm::Type *IntTy = ConvertType(T: E->getType());
4070 unsigned BitWidth = IntTy->getIntegerBitWidth();
4071 assert(
4072 (BitWidth == 8 || BitWidth == 16 || BitWidth == 32 || BitWidth == 64) &&
4073 "unexpected bit width for stdc_load8_*");
4074 Address Addr = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4075 if (IsAligned) {
4076 CharUnits TypeAlign = getContext().getTypeAlignInChars(T: E->getType());
4077 Addr = Addr.withAlignment(NewAlignment: std::max(a: Addr.getAlignment(), b: TypeAlign));
4078 }
4079 Addr = Addr.withElementType(ElemTy: IntTy);
4080 Value *Val = Builder.CreateLoad(Addr);
4081 if ((BitWidth == 16 || BitWidth == 32 || BitWidth == 64) &&
4082 IsBE != getTarget().isBigEndian())
4083 Val = Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Intrinsic::bswap, Tys: IntTy), Args: Val);
4084
4085 return RValue::get(V: Val);
4086 }
4087
4088 case Builtin::BI__builtin_constant_p: {
4089 llvm::Type *ResultType = ConvertType(T: E->getType());
4090
4091 const Expr *Arg = E->getArg(Arg: 0);
4092 QualType ArgType = Arg->getType();
4093 // FIXME: The allowance for Obj-C pointers and block pointers is historical
4094 // and likely a mistake.
4095 if (!ArgType->isIntegralOrEnumerationType() && !ArgType->isFloatingType() &&
4096 !ArgType->isObjCObjectPointerType() && !ArgType->isBlockPointerType())
4097 // Per the GCC documentation, only numeric constants are recognized after
4098 // inlining.
4099 return RValue::get(V: ConstantInt::get(Ty: ResultType, V: 0));
4100
4101 if (Arg->HasSideEffects(Ctx: getContext()))
4102 // The argument is unevaluated, so be conservative if it might have
4103 // side-effects.
4104 return RValue::get(V: ConstantInt::get(Ty: ResultType, V: 0));
4105
4106 Value *ArgValue = EmitScalarExpr(E: Arg);
4107 if (ArgType->isObjCObjectPointerType()) {
4108 // Convert Objective-C objects to id because we cannot distinguish between
4109 // LLVM types for Obj-C classes as they are opaque.
4110 ArgType = CGM.getContext().getObjCIdType();
4111 ArgValue = Builder.CreateBitCast(V: ArgValue, DestTy: ConvertType(T: ArgType));
4112 }
4113 Function *F =
4114 CGM.getIntrinsic(IID: Intrinsic::is_constant, Tys: ConvertType(T: ArgType));
4115 Value *Result = Builder.CreateCall(Callee: F, Args: ArgValue);
4116 if (Result->getType() != ResultType)
4117 Result = Builder.CreateIntCast(V: Result, DestTy: ResultType, /*isSigned*/false);
4118 return RValue::get(V: Result);
4119 }
4120 case Builtin::BI__builtin_dynamic_object_size:
4121 case Builtin::BI__builtin_object_size: {
4122 unsigned Type =
4123 E->getArg(Arg: 1)->EvaluateKnownConstInt(Ctx: getContext()).getZExtValue();
4124 auto *ResType = cast<llvm::IntegerType>(Val: ConvertType(T: E->getType()));
4125
4126 // We pass this builtin onto the optimizer so that it can figure out the
4127 // object size in more complex cases.
4128 bool IsDynamic = BuiltinID == Builtin::BI__builtin_dynamic_object_size;
4129 return RValue::get(V: emitBuiltinObjectSize(E: E->getArg(Arg: 0), Type, ResType,
4130 /*EmittedE=*/nullptr, IsDynamic));
4131 }
4132 case Builtin::BI__builtin_counted_by_ref: {
4133 // Default to returning '(void *) 0'.
4134 llvm::Value *Result = llvm::ConstantPointerNull::get(
4135 T: llvm::PointerType::getUnqual(C&: getLLVMContext()));
4136
4137 const Expr *Arg = E->getArg(Arg: 0)->IgnoreParenImpCasts();
4138
4139 if (auto *UO = dyn_cast<UnaryOperator>(Val: Arg);
4140 UO && UO->getOpcode() == UO_AddrOf) {
4141 Arg = UO->getSubExpr()->IgnoreParenImpCasts();
4142
4143 if (auto *ASE = dyn_cast<ArraySubscriptExpr>(Val: Arg))
4144 Arg = ASE->getBase()->IgnoreParenImpCasts();
4145 }
4146
4147 if (const MemberExpr *ME = dyn_cast_if_present<MemberExpr>(Val: Arg)) {
4148 if (auto *CATy =
4149 ME->getMemberDecl()->getType()->getAs<CountAttributedType>();
4150 CATy && CATy->getKind() == CountAttributedType::CountedBy) {
4151 const auto *MemberDecl = cast<FieldDecl>(Val: ME->getMemberDecl());
4152 if (const FieldDecl *CountFD = MemberDecl->findCountedByField())
4153 Result = GetCountedByFieldExprGEP(Base: Arg, FD: MemberDecl, CountDecl: CountFD);
4154 else
4155 llvm::report_fatal_error(reason: "Cannot find the counted_by 'count' field");
4156 }
4157 }
4158
4159 return RValue::get(V: Result);
4160 }
4161 case Builtin::BI__builtin_prefetch: {
4162 Value *Locality, *RW, *Address = EmitScalarExpr(E: E->getArg(Arg: 0));
4163 unsigned ICEArguments = (1 << 1) | (1 << 2);
4164 RW = (E->getNumArgs() > 1) ? EmitScalarOrConstFoldImmArg(ICEArguments, Idx: 1, E)
4165 : llvm::ConstantInt::get(Ty: Int32Ty, V: 0);
4166 RW = Builder.CreateZExtOrTrunc(V: RW, DestTy: Int32Ty);
4167 Locality = (E->getNumArgs() > 2)
4168 ? EmitScalarOrConstFoldImmArg(ICEArguments, Idx: 2, E)
4169 : llvm::ConstantInt::get(Ty: Int32Ty, V: 3);
4170 Locality = Builder.CreateZExtOrTrunc(V: Locality, DestTy: Int32Ty);
4171 Value *Data = llvm::ConstantInt::get(Ty: Int32Ty, V: 1);
4172 Function *F = CGM.getIntrinsic(IID: Intrinsic::prefetch, Tys: Address->getType());
4173 Builder.CreateCall(Callee: F, Args: {Address, RW, Locality, Data});
4174 return RValue::get(V: nullptr);
4175 }
4176 case Builtin::BI__builtin_readcyclecounter: {
4177 Function *F = CGM.getIntrinsic(IID: Intrinsic::readcyclecounter);
4178 return RValue::get(V: Builder.CreateCall(Callee: F));
4179 }
4180 case Builtin::BI__builtin_readsteadycounter: {
4181 Function *F = CGM.getIntrinsic(IID: Intrinsic::readsteadycounter);
4182 return RValue::get(V: Builder.CreateCall(Callee: F));
4183 }
4184 case Builtin::BI__builtin___clear_cache: {
4185 Value *Begin = EmitScalarExpr(E: E->getArg(Arg: 0));
4186 Value *End = EmitScalarExpr(E: E->getArg(Arg: 1));
4187 Function *F = CGM.getIntrinsic(IID: Intrinsic::clear_cache, Tys: {CGM.DefaultPtrTy});
4188 return RValue::get(V: Builder.CreateCall(Callee: F, Args: {Begin, End}));
4189 }
4190 case Builtin::BI__builtin_trap:
4191 EmitTrapCall(IntrID: Intrinsic::trap);
4192 return RValue::get(V: nullptr);
4193 case Builtin::BI__builtin_verbose_trap: {
4194 llvm::DebugLoc CallLocation = Builder.getCurrentDebugLocation();
4195 llvm::DILocation *TrapLocation = CallLocation;
4196 if (getDebugInfo()) {
4197 TrapLocation = getDebugInfo()->CreateTrapFailureMessageFor(
4198 TrapLocation, Category: *E->getArg(Arg: 0)->tryEvaluateString(Ctx&: getContext()),
4199 FailureMsg: *E->getArg(Arg: 1)->tryEvaluateString(Ctx&: getContext()));
4200 // Keep the trap on the builtin's source line. A line-zero location would
4201 // leave the trap attributed to the preceding line in the line table.
4202 TrapLocation = llvm::DILocation::get(
4203 Context&: getLLVMContext(), Line: CallLocation.getLine(), Column: CallLocation.getCol(),
4204 Scope: TrapLocation->getScope(), InlinedAt: TrapLocation->getInlinedAt());
4205 }
4206 ApplyDebugLocation ApplyTrapDI(*this, TrapLocation);
4207 // Currently no attempt is made to prevent traps from being merged.
4208 EmitTrapCall(IntrID: Intrinsic::trap);
4209 return RValue::get(V: nullptr);
4210 }
4211 case Builtin::BI__debugbreak:
4212 EmitTrapCall(IntrID: Intrinsic::debugtrap);
4213 return RValue::get(V: nullptr);
4214 case Builtin::BI__builtin_unreachable: {
4215 EmitUnreachable(Loc: E->getExprLoc());
4216
4217 // We do need to preserve an insertion point.
4218 EmitBlock(BB: createBasicBlock(name: "unreachable.cont"));
4219
4220 return RValue::get(V: nullptr);
4221 }
4222
4223 case Builtin::BI__builtin_powi:
4224 case Builtin::BI__builtin_powif:
4225 case Builtin::BI__builtin_powil: {
4226 llvm::Value *Src0 = EmitScalarExpr(E: E->getArg(Arg: 0));
4227 llvm::Value *Src1 = EmitScalarExpr(E: E->getArg(Arg: 1));
4228
4229 if (Builder.getIsFPConstrained()) {
4230 // FIXME: llvm.powi has 2 mangling types,
4231 // llvm.experimental.constrained.powi has one.
4232 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
4233 Function *F = CGM.getIntrinsic(IID: Intrinsic::experimental_constrained_powi,
4234 Tys: Src0->getType());
4235 return RValue::get(V: Builder.CreateConstrainedFPCall(Callee: F, Args: { Src0, Src1 }));
4236 }
4237
4238 Function *F = CGM.getIntrinsic(IID: Intrinsic::powi,
4239 Tys: { Src0->getType(), Src1->getType() });
4240 return RValue::get(V: Builder.CreateCall(Callee: F, Args: { Src0, Src1 }));
4241 }
4242 case Builtin::BI__builtin_frexpl: {
4243 // Linux PPC will not be adding additional PPCDoubleDouble support.
4244 // WIP to switch default to IEEE long double. Will emit libcall for
4245 // frexpl instead of legalizing this type in the BE.
4246 if (&getTarget().getLongDoubleFormat() == &llvm::APFloat::PPCDoubleDouble())
4247 break;
4248 [[fallthrough]];
4249 }
4250 case Builtin::BI__builtin_frexp:
4251 case Builtin::BI__builtin_frexpf:
4252 case Builtin::BI__builtin_frexpf128:
4253 case Builtin::BI__builtin_frexpf16:
4254 return RValue::get(V: emitFrexpBuiltin(CGF&: *this, E, IntrinsicID: Intrinsic::frexp));
4255 case Builtin::BImodf:
4256 case Builtin::BImodff:
4257 case Builtin::BImodfl:
4258 case Builtin::BI__builtin_modf:
4259 case Builtin::BI__builtin_modff:
4260 case Builtin::BI__builtin_modfl:
4261 if (Builder.getIsFPConstrained())
4262 break; // TODO: Emit constrained modf intrinsic once one exists.
4263 return RValue::get(V: emitModfBuiltin(CGF&: *this, E, IntrinsicID: Intrinsic::modf));
4264 case Builtin::BI__builtin_isgreater:
4265 case Builtin::BI__builtin_isgreaterequal:
4266 case Builtin::BI__builtin_isless:
4267 case Builtin::BI__builtin_islessequal:
4268 case Builtin::BI__builtin_islessgreater:
4269 case Builtin::BI__builtin_isunordered: {
4270 // Ordered comparisons: we know the arguments to these are matching scalar
4271 // floating point values.
4272 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
4273 Value *LHS = EmitScalarExpr(E: E->getArg(Arg: 0));
4274 Value *RHS = EmitScalarExpr(E: E->getArg(Arg: 1));
4275
4276 switch (BuiltinID) {
4277 default: llvm_unreachable("Unknown ordered comparison");
4278 case Builtin::BI__builtin_isgreater:
4279 LHS = Builder.CreateFCmpOGT(LHS, RHS, Name: "cmp");
4280 break;
4281 case Builtin::BI__builtin_isgreaterequal:
4282 LHS = Builder.CreateFCmpOGE(LHS, RHS, Name: "cmp");
4283 break;
4284 case Builtin::BI__builtin_isless:
4285 LHS = Builder.CreateFCmpOLT(LHS, RHS, Name: "cmp");
4286 break;
4287 case Builtin::BI__builtin_islessequal:
4288 LHS = Builder.CreateFCmpOLE(LHS, RHS, Name: "cmp");
4289 break;
4290 case Builtin::BI__builtin_islessgreater:
4291 LHS = Builder.CreateFCmpONE(LHS, RHS, Name: "cmp");
4292 break;
4293 case Builtin::BI__builtin_isunordered:
4294 LHS = Builder.CreateFCmpUNO(LHS, RHS, Name: "cmp");
4295 break;
4296 }
4297 // ZExt bool to int type.
4298 return RValue::get(V: Builder.CreateZExt(V: LHS, DestTy: ConvertType(T: E->getType())));
4299 }
4300
4301 case Builtin::BI__builtin_isnan: {
4302 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
4303 Value *V = EmitScalarExpr(E: E->getArg(Arg: 0));
4304 if (Value *Result = tryUseTestFPKind(CGF&: *this, BuiltinID, V))
4305 return RValue::get(V: Result);
4306 return RValue::get(
4307 V: Builder.CreateZExt(V: Builder.createIsFPClass(FPNum: V, Test: FPClassTest::fcNan),
4308 DestTy: ConvertType(T: E->getType())));
4309 }
4310
4311 case Builtin::BI__builtin_issignaling: {
4312 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
4313 Value *V = EmitScalarExpr(E: E->getArg(Arg: 0));
4314 return RValue::get(
4315 V: Builder.CreateZExt(V: Builder.createIsFPClass(FPNum: V, Test: FPClassTest::fcSNan),
4316 DestTy: ConvertType(T: E->getType())));
4317 }
4318
4319 case Builtin::BI__builtin_isinf: {
4320 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
4321 Value *V = EmitScalarExpr(E: E->getArg(Arg: 0));
4322 if (Value *Result = tryUseTestFPKind(CGF&: *this, BuiltinID, V))
4323 return RValue::get(V: Result);
4324 return RValue::get(
4325 V: Builder.CreateZExt(V: Builder.createIsFPClass(FPNum: V, Test: FPClassTest::fcInf),
4326 DestTy: ConvertType(T: E->getType())));
4327 }
4328
4329 case Builtin::BIfinite:
4330 case Builtin::BI__finite:
4331 case Builtin::BIfinitef:
4332 case Builtin::BI__finitef:
4333 case Builtin::BIfinitel:
4334 case Builtin::BI__finitel:
4335 case Builtin::BI__builtin_isfinite: {
4336 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
4337 Value *V = EmitScalarExpr(E: E->getArg(Arg: 0));
4338 if (Value *Result = tryUseTestFPKind(CGF&: *this, BuiltinID, V))
4339 return RValue::get(V: Result);
4340 return RValue::get(
4341 V: Builder.CreateZExt(V: Builder.createIsFPClass(FPNum: V, Test: FPClassTest::fcFinite),
4342 DestTy: ConvertType(T: E->getType())));
4343 }
4344
4345 case Builtin::BI__builtin_isnormal: {
4346 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
4347 Value *V = EmitScalarExpr(E: E->getArg(Arg: 0));
4348 return RValue::get(
4349 V: Builder.CreateZExt(V: Builder.createIsFPClass(FPNum: V, Test: FPClassTest::fcNormal),
4350 DestTy: ConvertType(T: E->getType())));
4351 }
4352
4353 case Builtin::BI__builtin_issubnormal: {
4354 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
4355 Value *V = EmitScalarExpr(E: E->getArg(Arg: 0));
4356 return RValue::get(
4357 V: Builder.CreateZExt(V: Builder.createIsFPClass(FPNum: V, Test: FPClassTest::fcSubnormal),
4358 DestTy: ConvertType(T: E->getType())));
4359 }
4360
4361 case Builtin::BI__builtin_iszero: {
4362 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
4363 Value *V = EmitScalarExpr(E: E->getArg(Arg: 0));
4364 return RValue::get(
4365 V: Builder.CreateZExt(V: Builder.createIsFPClass(FPNum: V, Test: FPClassTest::fcZero),
4366 DestTy: ConvertType(T: E->getType())));
4367 }
4368
4369 case Builtin::BI__builtin_isfpclass: {
4370 Expr::EvalResult Result;
4371 if (!E->getArg(Arg: 1)->EvaluateAsInt(Result, Ctx: CGM.getContext()))
4372 break;
4373 uint64_t Test = Result.Val.getInt().getLimitedValue();
4374 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
4375 Value *V = EmitScalarExpr(E: E->getArg(Arg: 0));
4376 return RValue::get(V: Builder.CreateZExt(V: Builder.createIsFPClass(FPNum: V, Test),
4377 DestTy: ConvertType(T: E->getType())));
4378 }
4379
4380 case Builtin::BI__builtin_nondeterministic_value: {
4381 llvm::Type *Ty = ConvertType(T: E->getArg(Arg: 0)->getType());
4382
4383 Value *Result = PoisonValue::get(T: Ty);
4384 Result = Builder.CreateFreeze(V: Result);
4385
4386 return RValue::get(V: Result);
4387 }
4388
4389 case Builtin::BI__builtin_elementwise_abs: {
4390 Value *Result;
4391 QualType QT = E->getArg(Arg: 0)->getType();
4392
4393 if (auto *VecTy = QT->getAs<VectorType>())
4394 QT = VecTy->getElementType();
4395 else if (auto *MatTy = QT->getAs<ConstantMatrixType>())
4396 QT = MatTy->getElementType();
4397 if (QT->isIntegerType())
4398 Result = Builder.CreateBinaryIntrinsic(
4399 ID: Intrinsic::abs, LHS: EmitScalarExpr(E: E->getArg(Arg: 0)), RHS: Builder.getFalse(),
4400 FMFSource: nullptr, Name: "elt.abs");
4401 else
4402 Result = emitBuiltinWithOneOverloadedType<1>(CGF&: *this, E, IntrinsicID: Intrinsic::fabs,
4403 Name: "elt.abs");
4404
4405 return RValue::get(V: Result);
4406 }
4407 case Builtin::BI__builtin_elementwise_bitreverse:
4408 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4409 CGF&: *this, E, IntrinsicID: Intrinsic::bitreverse, Name: "elt.bitreverse"));
4410 case Builtin::BI__builtin_elementwise_popcount:
4411 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4412 CGF&: *this, E, IntrinsicID: Intrinsic::ctpop, Name: "elt.ctpop"));
4413 case Builtin::BI__builtin_elementwise_canonicalize:
4414 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4415 CGF&: *this, E, IntrinsicID: Intrinsic::canonicalize, Name: "elt.canonicalize"));
4416 case Builtin::BI__builtin_elementwise_copysign:
4417 return RValue::get(
4418 V: emitBuiltinWithOneOverloadedType<2>(CGF&: *this, E, IntrinsicID: Intrinsic::copysign));
4419 case Builtin::BI__builtin_elementwise_fshl:
4420 return RValue::get(
4421 V: emitBuiltinWithOneOverloadedType<3>(CGF&: *this, E, IntrinsicID: Intrinsic::fshl));
4422 case Builtin::BI__builtin_elementwise_fshr:
4423 return RValue::get(
4424 V: emitBuiltinWithOneOverloadedType<3>(CGF&: *this, E, IntrinsicID: Intrinsic::fshr));
4425 case Builtin::BI__builtin_elementwise_clmul:
4426 return RValue::get(
4427 V: emitBuiltinWithOneOverloadedType<2>(CGF&: *this, E, IntrinsicID: Intrinsic::clmul));
4428 case Builtin::BI__builtin_elementwise_pext:
4429 return RValue::get(
4430 V: emitBuiltinWithOneOverloadedType<2>(CGF&: *this, E, IntrinsicID: Intrinsic::pext));
4431 case Builtin::BI__builtin_elementwise_pdep:
4432 return RValue::get(
4433 V: emitBuiltinWithOneOverloadedType<2>(CGF&: *this, E, IntrinsicID: Intrinsic::pdep));
4434
4435 case Builtin::BI__builtin_elementwise_add_sat:
4436 case Builtin::BI__builtin_elementwise_sub_sat: {
4437 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
4438 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
4439 Value *Result;
4440 assert(Op0->getType()->isIntOrIntVectorTy() && "integer type expected");
4441 QualType Ty = E->getArg(Arg: 0)->getType();
4442 if (auto *VecTy = Ty->getAs<VectorType>())
4443 Ty = VecTy->getElementType();
4444 bool IsSigned = Ty->isSignedIntegerType();
4445 unsigned Opc;
4446 if (BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_elementwise_add_sat)
4447 Opc = IsSigned ? Intrinsic::sadd_sat : Intrinsic::uadd_sat;
4448 else
4449 Opc = IsSigned ? Intrinsic::ssub_sat : Intrinsic::usub_sat;
4450 Result = Builder.CreateBinaryIntrinsic(ID: Opc, LHS: Op0, RHS: Op1, FMFSource: nullptr, Name: "elt.sat");
4451 return RValue::get(V: Result);
4452 }
4453
4454 case Builtin::BI__builtin_elementwise_max: {
4455 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
4456 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
4457 Value *Result;
4458 if (Op0->getType()->isIntOrIntVectorTy()) {
4459 QualType Ty = E->getArg(Arg: 0)->getType();
4460 Result = Builder.CreateBinaryIntrinsic(
4461 ID: Ty->hasSignedIntegerRepresentation() ? Intrinsic::smax
4462 : Intrinsic::umax,
4463 LHS: Op0, RHS: Op1, FMFSource: nullptr, Name: "elt.max");
4464 } else
4465 Result = Builder.CreateMaxNum(LHS: Op0, RHS: Op1, /*FMFSource=*/nullptr, Name: "elt.max");
4466 return RValue::get(V: Result);
4467 }
4468 case Builtin::BI__builtin_elementwise_min: {
4469 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
4470 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
4471 Value *Result;
4472 if (Op0->getType()->isIntOrIntVectorTy()) {
4473 QualType Ty = E->getArg(Arg: 0)->getType();
4474 Result = Builder.CreateBinaryIntrinsic(
4475 ID: Ty->hasSignedIntegerRepresentation() ? Intrinsic::smin
4476 : Intrinsic::umin,
4477 LHS: Op0, RHS: Op1, FMFSource: nullptr, Name: "elt.min");
4478 } else
4479 Result = Builder.CreateMinNum(LHS: Op0, RHS: Op1, /*FMFSource=*/nullptr, Name: "elt.min");
4480 return RValue::get(V: Result);
4481 }
4482
4483 case Builtin::BI__builtin_elementwise_maxnum: {
4484 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
4485 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
4486 Value *Result = Builder.CreateBinaryIntrinsic(ID: llvm::Intrinsic::maxnum, LHS: Op0,
4487 RHS: Op1, FMFSource: nullptr, Name: "elt.maxnum");
4488 return RValue::get(V: Result);
4489 }
4490
4491 case Builtin::BI__builtin_elementwise_minnum: {
4492 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
4493 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
4494 Value *Result = Builder.CreateBinaryIntrinsic(ID: llvm::Intrinsic::minnum, LHS: Op0,
4495 RHS: Op1, FMFSource: nullptr, Name: "elt.minnum");
4496 return RValue::get(V: Result);
4497 }
4498
4499 case Builtin::BI__builtin_elementwise_maximum: {
4500 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
4501 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
4502 Value *Result = Builder.CreateBinaryIntrinsic(ID: Intrinsic::maximum, LHS: Op0, RHS: Op1,
4503 FMFSource: nullptr, Name: "elt.maximum");
4504 return RValue::get(V: Result);
4505 }
4506
4507 case Builtin::BI__builtin_elementwise_minimum: {
4508 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
4509 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
4510 Value *Result = Builder.CreateBinaryIntrinsic(ID: Intrinsic::minimum, LHS: Op0, RHS: Op1,
4511 FMFSource: nullptr, Name: "elt.minimum");
4512 return RValue::get(V: Result);
4513 }
4514
4515 case Builtin::BI__builtin_elementwise_maximumnum: {
4516 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
4517 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
4518 Value *Result = Builder.CreateBinaryIntrinsic(
4519 ID: Intrinsic::maximumnum, LHS: Op0, RHS: Op1, FMFSource: nullptr, Name: "elt.maximumnum");
4520 return RValue::get(V: Result);
4521 }
4522
4523 case Builtin::BI__builtin_elementwise_minimumnum: {
4524 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
4525 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
4526 Value *Result = Builder.CreateBinaryIntrinsic(
4527 ID: Intrinsic::minimumnum, LHS: Op0, RHS: Op1, FMFSource: nullptr, Name: "elt.minimumnum");
4528 return RValue::get(V: Result);
4529 }
4530
4531 case Builtin::BI__builtin_reduce_max: {
4532 auto GetIntrinsicID = [this](QualType QT) {
4533 if (auto *VecTy = QT->getAs<VectorType>())
4534 QT = VecTy->getElementType();
4535 else if (QT->isSizelessVectorType())
4536 QT = QT->getSizelessVectorEltType(Ctx: CGM.getContext());
4537
4538 if (QT->isSignedIntegerType())
4539 return Intrinsic::vector_reduce_smax;
4540 if (QT->isUnsignedIntegerType())
4541 return Intrinsic::vector_reduce_umax;
4542 assert(QT->isFloatingType() && "must have a float here");
4543 return Intrinsic::vector_reduce_fmax;
4544 };
4545 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4546 CGF&: *this, E, IntrinsicID: GetIntrinsicID(E->getArg(Arg: 0)->getType()), Name: "rdx.min"));
4547 }
4548
4549 case Builtin::BI__builtin_reduce_min: {
4550 auto GetIntrinsicID = [this](QualType QT) {
4551 if (auto *VecTy = QT->getAs<VectorType>())
4552 QT = VecTy->getElementType();
4553 else if (QT->isSizelessVectorType())
4554 QT = QT->getSizelessVectorEltType(Ctx: CGM.getContext());
4555
4556 if (QT->isSignedIntegerType())
4557 return Intrinsic::vector_reduce_smin;
4558 if (QT->isUnsignedIntegerType())
4559 return Intrinsic::vector_reduce_umin;
4560 assert(QT->isFloatingType() && "must have a float here");
4561 return Intrinsic::vector_reduce_fmin;
4562 };
4563
4564 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4565 CGF&: *this, E, IntrinsicID: GetIntrinsicID(E->getArg(Arg: 0)->getType()), Name: "rdx.min"));
4566 }
4567
4568 case Builtin::BI__builtin_reduce_add:
4569 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4570 CGF&: *this, E, IntrinsicID: Intrinsic::vector_reduce_add, Name: "rdx.add"));
4571 case Builtin::BI__builtin_reduce_mul:
4572 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4573 CGF&: *this, E, IntrinsicID: Intrinsic::vector_reduce_mul, Name: "rdx.mul"));
4574 case Builtin::BI__builtin_reduce_xor:
4575 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4576 CGF&: *this, E, IntrinsicID: Intrinsic::vector_reduce_xor, Name: "rdx.xor"));
4577 case Builtin::BI__builtin_reduce_or:
4578 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4579 CGF&: *this, E, IntrinsicID: Intrinsic::vector_reduce_or, Name: "rdx.or"));
4580 case Builtin::BI__builtin_reduce_and:
4581 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4582 CGF&: *this, E, IntrinsicID: Intrinsic::vector_reduce_and, Name: "rdx.and"));
4583 case Builtin::BI__builtin_reduce_maximum:
4584 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4585 CGF&: *this, E, IntrinsicID: Intrinsic::vector_reduce_fmaximum, Name: "rdx.maximum"));
4586 case Builtin::BI__builtin_reduce_minimum:
4587 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4588 CGF&: *this, E, IntrinsicID: Intrinsic::vector_reduce_fminimum, Name: "rdx.minimum"));
4589 case Builtin::BI__builtin_reduce_assoc_fadd:
4590 case Builtin::BI__builtin_reduce_in_order_fadd: {
4591 llvm::Value *Vector = EmitScalarExpr(E: E->getArg(Arg: 0));
4592 llvm::Type *ScalarTy = Vector->getType()->getScalarType();
4593 llvm::Value *StartValue = nullptr;
4594 if (E->getNumArgs() == 2)
4595 StartValue = Builder.CreateFPCast(V: EmitScalarExpr(E: E->getArg(Arg: 1)), DestTy: ScalarTy);
4596 llvm::Value *Args[] = {/*start_value=*/StartValue
4597 ? StartValue
4598 : llvm::ConstantFP::get(Ty: ScalarTy, V: -0.0F),
4599 /*vector=*/Vector};
4600 llvm::Function *F =
4601 CGM.getIntrinsic(IID: Intrinsic::vector_reduce_fadd, Tys: Vector->getType());
4602 llvm::CallBase *Reduce = Builder.CreateCall(Callee: F, Args, Name: "rdx.addf");
4603 if (BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_reduce_assoc_fadd) {
4604 // `__builtin_reduce_assoc_fadd` is an associative reduction which
4605 // requires the reassoc FMF flag.
4606 llvm::FastMathFlags FMF;
4607 FMF.setAllowReassoc();
4608 cast<llvm::CallBase>(Val: Reduce)->setFastMathFlags(FMF);
4609 }
4610 return RValue::get(V: Reduce);
4611 }
4612
4613 case Builtin::BI__builtin_matrix_transpose: {
4614 auto *MatrixTy = E->getArg(Arg: 0)->getType()->castAs<ConstantMatrixType>();
4615 Value *MatValue = EmitScalarExpr(E: E->getArg(Arg: 0));
4616 MatrixBuilder MB(Builder);
4617 Value *Result = MB.CreateMatrixTranspose(Matrix: MatValue, Rows: MatrixTy->getNumRows(),
4618 Columns: MatrixTy->getNumColumns());
4619 return RValue::get(V: Result);
4620 }
4621
4622 case Builtin::BI__builtin_matrix_column_major_load: {
4623 MatrixBuilder MB(Builder);
4624 // Emit everything that isn't dependent on the first parameter type
4625 Value *Stride = EmitScalarExpr(E: E->getArg(Arg: 3));
4626 const auto *ResultTy = E->getType()->getAs<ConstantMatrixType>();
4627 auto *PtrTy = E->getArg(Arg: 0)->getType()->getAs<PointerType>();
4628 assert(PtrTy && "arg0 must be of pointer type");
4629 bool IsVolatile = PtrTy->getPointeeType().isVolatileQualified();
4630
4631 Address Src = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4632 EmitNonNullArgCheck(RV: RValue::get(V: Src.emitRawPointer(CGF&: *this)),
4633 ArgType: E->getArg(Arg: 0)->getType(), ArgLoc: E->getArg(Arg: 0)->getExprLoc(), AC: FD,
4634 ParmNum: 0);
4635 Value *Result = MB.CreateColumnMajorLoad(
4636 EltTy: Src.getElementType(), DataPtr: Src.emitRawPointer(CGF&: *this),
4637 Alignment: Align(Src.getAlignment().getQuantity()), Stride, IsVolatile,
4638 Rows: ResultTy->getNumRows(), Columns: ResultTy->getNumColumns(), Name: "matrix");
4639 return RValue::get(V: Result);
4640 }
4641
4642 case Builtin::BI__builtin_matrix_column_major_store: {
4643 MatrixBuilder MB(Builder);
4644 Value *Matrix = EmitScalarExpr(E: E->getArg(Arg: 0));
4645 Address Dst = EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
4646 Value *Stride = EmitScalarExpr(E: E->getArg(Arg: 2));
4647
4648 const auto *MatrixTy = E->getArg(Arg: 0)->getType()->getAs<ConstantMatrixType>();
4649 auto *PtrTy = E->getArg(Arg: 1)->getType()->getAs<PointerType>();
4650 assert(PtrTy && "arg1 must be of pointer type");
4651 bool IsVolatile = PtrTy->getPointeeType().isVolatileQualified();
4652
4653 EmitNonNullArgCheck(RV: RValue::get(V: Dst.emitRawPointer(CGF&: *this)),
4654 ArgType: E->getArg(Arg: 1)->getType(), ArgLoc: E->getArg(Arg: 1)->getExprLoc(), AC: FD,
4655 ParmNum: 0);
4656 Value *Result = MB.CreateColumnMajorStore(
4657 Matrix, Ptr: Dst.emitRawPointer(CGF&: *this),
4658 Alignment: Align(Dst.getAlignment().getQuantity()), Stride, IsVolatile,
4659 Rows: MatrixTy->getNumRows(), Columns: MatrixTy->getNumColumns());
4660 addInstToNewSourceAtom(KeyInstruction: cast<Instruction>(Val: Result), Backup: Matrix);
4661 return RValue::get(V: Result);
4662 }
4663
4664 case Builtin::BI__builtin_masked_load:
4665 case Builtin::BI__builtin_masked_expand_load: {
4666 llvm::Value *Mask = EmitScalarExpr(E: E->getArg(Arg: 0));
4667 llvm::Value *Ptr = EmitScalarExpr(E: E->getArg(Arg: 1));
4668
4669 llvm::Type *RetTy = CGM.getTypes().ConvertType(T: E->getType());
4670 llvm::Value *PassThru = llvm::PoisonValue::get(T: RetTy);
4671 if (E->getNumArgs() > 2)
4672 PassThru = EmitScalarExpr(E: E->getArg(Arg: 2));
4673
4674 CharUnits Align = CGM.getNaturalTypeAlignment(
4675 T: E->getType()->getAs<VectorType>()->getElementType(), BaseInfo: nullptr);
4676
4677 llvm::Value *Result;
4678 if (BuiltinID == Builtin::BI__builtin_masked_load)
4679 Result = Builder.CreateMaskedLoad(Ty: RetTy, Ptr, Alignment: Align.getAsAlign(), Mask,
4680 PassThru, Name: "masked_load");
4681 else
4682 Result = Builder.CreateMaskedExpandLoad(Ty: RetTy, Ptr, Align: MaybeAlign(), Mask,
4683 PassThru, Name: "masked_expand_load");
4684
4685 return RValue::get(V: Result);
4686 };
4687 case Builtin::BI__builtin_masked_gather: {
4688 llvm::Value *Mask = EmitScalarExpr(E: E->getArg(Arg: 0));
4689 llvm::Value *Idx = EmitScalarExpr(E: E->getArg(Arg: 1));
4690 llvm::Value *Ptr = EmitScalarExpr(E: E->getArg(Arg: 2));
4691
4692 llvm::Type *RetTy = CGM.getTypes().ConvertType(T: E->getType());
4693 CharUnits Align = CGM.getNaturalTypeAlignment(
4694 T: E->getType()->getAs<VectorType>()->getElementType(), BaseInfo: nullptr);
4695
4696 llvm::Value *PassThru = llvm::PoisonValue::get(T: RetTy);
4697 if (E->getNumArgs() > 3)
4698 PassThru = EmitScalarExpr(E: E->getArg(Arg: 3));
4699
4700 llvm::Type *ElemTy = CGM.getTypes().ConvertType(
4701 T: E->getType()->getAs<VectorType>()->getElementType());
4702 llvm::Value *PtrVec = Builder.CreateGEP(Ty: ElemTy, Ptr, IdxList: Idx);
4703
4704 llvm::Value *Result = Builder.CreateMaskedGather(
4705 Ty: RetTy, Ptrs: PtrVec, Alignment: Align.getAsAlign(), Mask, PassThru, Name: "masked_gather");
4706 return RValue::get(V: Result);
4707 }
4708 case Builtin::BI__builtin_masked_store:
4709 case Builtin::BI__builtin_masked_compress_store: {
4710 llvm::Value *Mask = EmitScalarExpr(E: E->getArg(Arg: 0));
4711 llvm::Value *Val = EmitScalarExpr(E: E->getArg(Arg: 1));
4712 llvm::Value *Ptr = EmitScalarExpr(E: E->getArg(Arg: 2));
4713
4714 CharUnits Align = CGM.getNaturalTypeAlignment(
4715 T: E->getArg(Arg: 1)->getType()->getAs<VectorType>()->getElementType(),
4716 BaseInfo: nullptr);
4717
4718 if (BuiltinID == Builtin::BI__builtin_masked_store)
4719 Builder.CreateMaskedStore(Val, Ptr, Alignment: Align.getAsAlign(), Mask);
4720 else
4721 Builder.CreateMaskedCompressStore(Val, Ptr, Align: MaybeAlign(), Mask);
4722
4723 return RValue::get(V: nullptr);
4724 }
4725 case Builtin::BI__builtin_masked_scatter: {
4726 llvm::Value *Mask = EmitScalarExpr(E: E->getArg(Arg: 0));
4727 llvm::Value *Idx = EmitScalarExpr(E: E->getArg(Arg: 1));
4728 llvm::Value *Val = EmitScalarExpr(E: E->getArg(Arg: 2));
4729 llvm::Value *Ptr = EmitScalarExpr(E: E->getArg(Arg: 3));
4730
4731 CharUnits Align = CGM.getNaturalTypeAlignment(
4732 T: E->getArg(Arg: 2)->getType()->getAs<VectorType>()->getElementType(),
4733 BaseInfo: nullptr);
4734
4735 llvm::Type *ElemTy = CGM.getTypes().ConvertType(
4736 T: E->getArg(Arg: 1)->getType()->getAs<VectorType>()->getElementType());
4737 llvm::Value *PtrVec = Builder.CreateGEP(Ty: ElemTy, Ptr, IdxList: Idx);
4738
4739 Builder.CreateMaskedScatter(Val, Ptrs: PtrVec, Alignment: Align.getAsAlign(), Mask);
4740 return RValue();
4741 }
4742 case Builtin::BI__builtin_isinf_sign: {
4743 // isinf_sign(x) -> fabs(x) == infinity ? (signbit(x) ? -1 : 1) : 0
4744 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
4745 // FIXME: for strictfp/IEEE-754 we need to not trap on SNaN here.
4746 Value *Arg = EmitScalarExpr(E: E->getArg(Arg: 0));
4747 Value *AbsArg = EmitFAbs(CGF&: *this, V: Arg);
4748 Value *IsInf = Builder.CreateFCmpOEQ(
4749 LHS: AbsArg, RHS: ConstantFP::getInfinity(Ty: Arg->getType()), Name: "isinf");
4750 Value *IsNeg = EmitSignBit(CGF&: *this, V: Arg);
4751
4752 llvm::Type *IntTy = ConvertType(T: E->getType());
4753 Value *Zero = Constant::getNullValue(Ty: IntTy);
4754 Value *One = ConstantInt::get(Ty: IntTy, V: 1);
4755 Value *NegativeOne = ConstantInt::getAllOnesValue(Ty: IntTy);
4756 Value *SignResult = Builder.CreateSelect(C: IsNeg, True: NegativeOne, False: One);
4757 Value *Result = Builder.CreateSelect(C: IsInf, True: SignResult, False: Zero);
4758 return RValue::get(V: Result);
4759 }
4760
4761 case Builtin::BI__builtin_flt_rounds: {
4762 Function *F = CGM.getIntrinsic(IID: Intrinsic::get_rounding);
4763
4764 llvm::Type *ResultType = ConvertType(T: E->getType());
4765 Value *Result = Builder.CreateCall(Callee: F);
4766 if (Result->getType() != ResultType)
4767 Result = Builder.CreateIntCast(V: Result, DestTy: ResultType, /*isSigned*/true,
4768 Name: "cast");
4769 return RValue::get(V: Result);
4770 }
4771
4772 case Builtin::BI__builtin_set_flt_rounds: {
4773 Function *F = CGM.getIntrinsic(IID: Intrinsic::set_rounding);
4774
4775 Value *V = EmitScalarExpr(E: E->getArg(Arg: 0));
4776 Builder.CreateCall(Callee: F, Args: V);
4777 return RValue::get(V: nullptr);
4778 }
4779
4780 case Builtin::BI__builtin_fpclassify: {
4781 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
4782 // FIXME: for strictfp/IEEE-754 we need to not trap on SNaN here.
4783 Value *V = EmitScalarExpr(E: E->getArg(Arg: 5));
4784 llvm::Type *Ty = ConvertType(T: E->getArg(Arg: 5)->getType());
4785
4786 // Create Result
4787 BasicBlock *Begin = Builder.GetInsertBlock();
4788 BasicBlock *End = createBasicBlock(name: "fpclassify_end", parent: this->CurFn);
4789 Builder.SetInsertPoint(End);
4790 PHINode *Result =
4791 Builder.CreatePHI(Ty: ConvertType(T: E->getArg(Arg: 0)->getType()), NumReservedValues: 4,
4792 Name: "fpclassify_result");
4793
4794 // if (V==0) return FP_ZERO
4795 Builder.SetInsertPoint(Begin);
4796 Value *IsZero = Builder.CreateFCmpOEQ(LHS: V, RHS: Constant::getNullValue(Ty),
4797 Name: "iszero");
4798 Value *ZeroLiteral = EmitScalarExpr(E: E->getArg(Arg: 4));
4799 BasicBlock *NotZero = createBasicBlock(name: "fpclassify_not_zero", parent: this->CurFn);
4800 Builder.CreateCondBr(Cond: IsZero, True: End, False: NotZero);
4801 Result->addIncoming(V: ZeroLiteral, BB: Begin);
4802
4803 // if (V != V) return FP_NAN
4804 Builder.SetInsertPoint(NotZero);
4805 Value *IsNan = Builder.CreateFCmpUNO(LHS: V, RHS: V, Name: "cmp");
4806 Value *NanLiteral = EmitScalarExpr(E: E->getArg(Arg: 0));
4807 BasicBlock *NotNan = createBasicBlock(name: "fpclassify_not_nan", parent: this->CurFn);
4808 Builder.CreateCondBr(Cond: IsNan, True: End, False: NotNan);
4809 Result->addIncoming(V: NanLiteral, BB: NotZero);
4810
4811 // if (fabs(V) == infinity) return FP_INFINITY
4812 Builder.SetInsertPoint(NotNan);
4813 Value *VAbs = EmitFAbs(CGF&: *this, V);
4814 Value *IsInf =
4815 Builder.CreateFCmpOEQ(LHS: VAbs, RHS: ConstantFP::getInfinity(Ty: V->getType()),
4816 Name: "isinf");
4817 Value *InfLiteral = EmitScalarExpr(E: E->getArg(Arg: 1));
4818 BasicBlock *NotInf = createBasicBlock(name: "fpclassify_not_inf", parent: this->CurFn);
4819 Builder.CreateCondBr(Cond: IsInf, True: End, False: NotInf);
4820 Result->addIncoming(V: InfLiteral, BB: NotNan);
4821
4822 // if (fabs(V) >= MIN_NORMAL) return FP_NORMAL else FP_SUBNORMAL
4823 Builder.SetInsertPoint(NotInf);
4824 APFloat Smallest = APFloat::getSmallestNormalized(
4825 Sem: getContext().getFloatTypeSemantics(T: E->getArg(Arg: 5)->getType()));
4826 Value *IsNormal =
4827 Builder.CreateFCmpUGE(LHS: VAbs, RHS: ConstantFP::get(Context&: V->getContext(), V: Smallest),
4828 Name: "isnormal");
4829 Value *NormalResult =
4830 Builder.CreateSelect(C: IsNormal, True: EmitScalarExpr(E: E->getArg(Arg: 2)),
4831 False: EmitScalarExpr(E: E->getArg(Arg: 3)));
4832 Builder.CreateBr(Dest: End);
4833 Result->addIncoming(V: NormalResult, BB: NotInf);
4834
4835 // return Result
4836 Builder.SetInsertPoint(End);
4837 return RValue::get(V: Result);
4838 }
4839
4840 // An alloca will always return a pointer to the alloca (stack) address
4841 // space. This address space need not be the same as the AST / Language
4842 // default (e.g. in C / C++ auto vars are in the generic address space). At
4843 // the AST level this is handled within CreateTempAlloca et al., but for the
4844 // builtin / dynamic alloca we have to handle it here. We use an explicit cast
4845 // instead of passing an AS to CreateAlloca so as to not inhibit optimisation.
4846 case Builtin::BIalloca:
4847 case Builtin::BI_alloca:
4848 case Builtin::BI__builtin_alloca_uninitialized:
4849 case Builtin::BI__builtin_alloca: {
4850 Value *Size = EmitScalarExpr(E: E->getArg(Arg: 0));
4851 const TargetInfo &TI = getContext().getTargetInfo();
4852 // The alignment of the alloca should correspond to __BIGGEST_ALIGNMENT__.
4853 const Align SuitableAlignmentInBytes =
4854 CGM.getContext()
4855 .toCharUnitsFromBits(BitSize: TI.getSuitableAlign())
4856 .getAsAlign();
4857 AllocaInst *AI = Builder.CreateAlloca(Ty: Builder.getInt8Ty(), ArraySize: Size);
4858 AI->setAlignment(SuitableAlignmentInBytes);
4859 if (BuiltinID != Builtin::BI__builtin_alloca_uninitialized)
4860 initializeAlloca(CGF&: *this, AI, Size, AlignmentInBytes: SuitableAlignmentInBytes);
4861 if (AI->getAddressSpace() !=
4862 CGM.getContext().getTargetAddressSpace(
4863 AS: E->getType()->getPointeeType().getAddressSpace())) {
4864 llvm::Type *Ty = CGM.getTypes().ConvertType(T: E->getType());
4865 return RValue::get(V: performAddrSpaceCast(Src: AI, DestTy: Ty));
4866 }
4867 return RValue::get(V: AI);
4868 }
4869
4870 case Builtin::BI__builtin_alloca_with_align_uninitialized:
4871 case Builtin::BI__builtin_alloca_with_align: {
4872 Value *Size = EmitScalarExpr(E: E->getArg(Arg: 0));
4873 Value *AlignmentInBitsValue = EmitScalarExpr(E: E->getArg(Arg: 1));
4874 auto *AlignmentInBitsCI = cast<ConstantInt>(Val: AlignmentInBitsValue);
4875 unsigned AlignmentInBits = AlignmentInBitsCI->getZExtValue();
4876 const Align AlignmentInBytes =
4877 CGM.getContext().toCharUnitsFromBits(BitSize: AlignmentInBits).getAsAlign();
4878 AllocaInst *AI = Builder.CreateAlloca(Ty: Builder.getInt8Ty(), ArraySize: Size);
4879 AI->setAlignment(AlignmentInBytes);
4880 if (BuiltinID != Builtin::BI__builtin_alloca_with_align_uninitialized)
4881 initializeAlloca(CGF&: *this, AI, Size, AlignmentInBytes);
4882 if (AI->getAddressSpace() !=
4883 CGM.getContext().getTargetAddressSpace(
4884 AS: E->getType()->getPointeeType().getAddressSpace())) {
4885 llvm::Type *Ty = CGM.getTypes().ConvertType(T: E->getType());
4886 return RValue::get(V: performAddrSpaceCast(Src: AI, DestTy: Ty));
4887 }
4888 return RValue::get(V: AI);
4889 }
4890
4891 case Builtin::BI__builtin_infer_alloc_token: {
4892 llvm::MDNode *MDN = buildAllocToken(E);
4893 llvm::Value *MDV = MetadataAsValue::get(Context&: getLLVMContext(), MD: MDN);
4894 llvm::Function *F =
4895 CGM.getIntrinsic(IID: llvm::Intrinsic::alloc_token_id, Tys: {IntPtrTy});
4896 llvm::CallBase *TokenID = Builder.CreateCall(Callee: F, Args: MDV);
4897 return RValue::get(V: TokenID);
4898 }
4899
4900 case Builtin::BIbzero:
4901 case Builtin::BI__builtin_bzero: {
4902 Address Dest = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4903 Value *SizeVal = EmitScalarExpr(E: E->getArg(Arg: 1));
4904 EmitNonNullArgCheck(Addr: Dest, ArgType: E->getArg(Arg: 0)->getType(),
4905 ArgLoc: E->getArg(Arg: 0)->getExprLoc(), AC: FD, ParmNum: 0);
4906 auto *I = Builder.CreateMemSet(Dest, Value: Builder.getInt8(C: 0), Size: SizeVal, IsVolatile: false);
4907 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
4908 return RValue::get(V: nullptr);
4909 }
4910
4911 case Builtin::BIbcopy:
4912 case Builtin::BI__builtin_bcopy: {
4913 Address Src = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4914 Address Dest = EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
4915 Value *SizeVal = EmitScalarExpr(E: E->getArg(Arg: 2));
4916 EmitNonNullArgCheck(RV: RValue::get(V: Src.emitRawPointer(CGF&: *this)),
4917 ArgType: E->getArg(Arg: 0)->getType(), ArgLoc: E->getArg(Arg: 0)->getExprLoc(), AC: FD,
4918 ParmNum: 0);
4919 EmitNonNullArgCheck(RV: RValue::get(V: Dest.emitRawPointer(CGF&: *this)),
4920 ArgType: E->getArg(Arg: 1)->getType(), ArgLoc: E->getArg(Arg: 1)->getExprLoc(), AC: FD,
4921 ParmNum: 0);
4922 auto *I = Builder.CreateMemMove(Dest, Src, Size: SizeVal, IsVolatile: false);
4923 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
4924 return RValue::get(V: nullptr);
4925 }
4926
4927 case Builtin::BImemcpy:
4928 case Builtin::BI__builtin_memcpy:
4929 case Builtin::BImempcpy:
4930 case Builtin::BI__builtin_mempcpy: {
4931 Address Dest = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4932 Address Src = EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
4933 Value *SizeVal = EmitScalarExpr(E: E->getArg(Arg: 2));
4934 EmitArgCheck(TCK_Store, Dest, E->getArg(Arg: 0), 0);
4935 EmitArgCheck(TCK_Load, Src, E->getArg(Arg: 1), 1);
4936 auto *I = Builder.CreateMemCpy(Dest, Src, Size: SizeVal, IsVolatile: false);
4937 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
4938 if (BuiltinID == Builtin::BImempcpy ||
4939 BuiltinID == Builtin::BI__builtin_mempcpy)
4940 return RValue::get(V: Builder.CreateInBoundsGEP(
4941 Ty: Dest.getElementType(), Ptr: Dest.emitRawPointer(CGF&: *this), IdxList: SizeVal));
4942 else
4943 return RValue::get(Addr: Dest, CGF&: *this);
4944 }
4945
4946 case Builtin::BI__builtin_memcpy_inline: {
4947 Address Dest = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4948 Address Src = EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
4949 uint64_t Size =
4950 E->getArg(Arg: 2)->EvaluateKnownConstInt(Ctx: getContext()).getZExtValue();
4951 EmitArgCheck(TCK_Store, Dest, E->getArg(Arg: 0), 0);
4952 EmitArgCheck(TCK_Load, Src, E->getArg(Arg: 1), 1);
4953 auto *I = Builder.CreateMemCpyInline(Dest, Src, Size);
4954 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
4955 return RValue::get(V: nullptr);
4956 }
4957
4958 case Builtin::BI__builtin_char_memchr:
4959 BuiltinID = Builtin::BI__builtin_memchr;
4960 break;
4961
4962 case Builtin::BI__builtin___memcpy_chk: {
4963 // fold __builtin_memcpy_chk(x, y, cst1, cst2) to memcpy iff cst1<=cst2.
4964 Expr::EvalResult SizeResult, DstSizeResult;
4965 if (!E->getArg(Arg: 2)->EvaluateAsInt(Result&: SizeResult, Ctx: CGM.getContext()) ||
4966 !E->getArg(Arg: 3)->EvaluateAsInt(Result&: DstSizeResult, Ctx: CGM.getContext()))
4967 break;
4968 llvm::APSInt Size = SizeResult.Val.getInt();
4969 llvm::APSInt DstSize = DstSizeResult.Val.getInt();
4970 if (Size.ugt(RHS: DstSize))
4971 break;
4972 Address Dest = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4973 Address Src = EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
4974 Value *SizeVal = llvm::ConstantInt::get(Context&: Builder.getContext(), V: Size);
4975 auto *I = Builder.CreateMemCpy(Dest, Src, Size: SizeVal, IsVolatile: false);
4976 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
4977 return RValue::get(Addr: Dest, CGF&: *this);
4978 }
4979
4980 case Builtin::BI__builtin_objc_memmove_collectable: {
4981 Address DestAddr = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4982 Address SrcAddr = EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
4983 Value *SizeVal = EmitScalarExpr(E: E->getArg(Arg: 2));
4984 CGM.getObjCRuntime().EmitGCMemmoveCollectable(CGF&: *this,
4985 DestPtr: DestAddr, SrcPtr: SrcAddr, Size: SizeVal);
4986 return RValue::get(Addr: DestAddr, CGF&: *this);
4987 }
4988
4989 case Builtin::BI__builtin___memmove_chk: {
4990 // fold __builtin_memmove_chk(x, y, cst1, cst2) to memmove iff cst1<=cst2.
4991 Expr::EvalResult SizeResult, DstSizeResult;
4992 if (!E->getArg(Arg: 2)->EvaluateAsInt(Result&: SizeResult, Ctx: CGM.getContext()) ||
4993 !E->getArg(Arg: 3)->EvaluateAsInt(Result&: DstSizeResult, Ctx: CGM.getContext()))
4994 break;
4995 llvm::APSInt Size = SizeResult.Val.getInt();
4996 llvm::APSInt DstSize = DstSizeResult.Val.getInt();
4997 if (Size.ugt(RHS: DstSize))
4998 break;
4999 Address Dest = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
5000 Address Src = EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
5001 Value *SizeVal = llvm::ConstantInt::get(Context&: Builder.getContext(), V: Size);
5002 auto *I = Builder.CreateMemMove(Dest, Src, Size: SizeVal, IsVolatile: false);
5003 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
5004 return RValue::get(Addr: Dest, CGF&: *this);
5005 }
5006
5007 case Builtin::BI__builtin_trivially_relocate:
5008 case Builtin::BImemmove:
5009 case Builtin::BI__builtin_memmove: {
5010 Address Dest = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
5011 Address Src = EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
5012 Value *SizeVal = EmitScalarExpr(E: E->getArg(Arg: 2));
5013 if (BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_trivially_relocate)
5014 SizeVal = Builder.CreateMul(
5015 LHS: SizeVal,
5016 RHS: ConstantInt::get(
5017 Ty: SizeVal->getType(),
5018 V: getContext()
5019 .getTypeSizeInChars(T: E->getArg(Arg: 0)->getType()->getPointeeType())
5020 .getQuantity()));
5021 EmitArgCheck(TCK_Store, Dest, E->getArg(Arg: 0), 0);
5022 EmitArgCheck(TCK_Load, Src, E->getArg(Arg: 1), 1);
5023 auto *I = Builder.CreateMemMove(Dest, Src, Size: SizeVal, IsVolatile: false);
5024 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
5025 return RValue::get(Addr: Dest, CGF&: *this);
5026 }
5027 case Builtin::BImemset:
5028 case Builtin::BI__builtin_memset: {
5029 Address Dest = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
5030 Value *ByteVal = Builder.CreateTrunc(V: EmitScalarExpr(E: E->getArg(Arg: 1)),
5031 DestTy: Builder.getInt8Ty());
5032 Value *SizeVal = EmitScalarExpr(E: E->getArg(Arg: 2));
5033 EmitNonNullArgCheck(Addr: Dest, ArgType: E->getArg(Arg: 0)->getType(),
5034 ArgLoc: E->getArg(Arg: 0)->getExprLoc(), AC: FD, ParmNum: 0);
5035 auto *I = Builder.CreateMemSet(Dest, Value: ByteVal, Size: SizeVal, IsVolatile: false);
5036 addInstToNewSourceAtom(KeyInstruction: I, Backup: ByteVal);
5037 return RValue::get(Addr: Dest, CGF&: *this);
5038 }
5039 case Builtin::BI__builtin_memset_inline: {
5040 Address Dest = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
5041 Value *ByteVal =
5042 Builder.CreateTrunc(V: EmitScalarExpr(E: E->getArg(Arg: 1)), DestTy: Builder.getInt8Ty());
5043 uint64_t Size =
5044 E->getArg(Arg: 2)->EvaluateKnownConstInt(Ctx: getContext()).getZExtValue();
5045 EmitNonNullArgCheck(RV: RValue::get(V: Dest.emitRawPointer(CGF&: *this)),
5046 ArgType: E->getArg(Arg: 0)->getType(), ArgLoc: E->getArg(Arg: 0)->getExprLoc(), AC: FD,
5047 ParmNum: 0);
5048 auto *I = Builder.CreateMemSetInline(Dest, Value: ByteVal, Size);
5049 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
5050 return RValue::get(V: nullptr);
5051 }
5052 case Builtin::BI__builtin___memset_chk: {
5053 // fold __builtin_memset_chk(x, y, cst1, cst2) to memset iff cst1<=cst2.
5054 Expr::EvalResult SizeResult, DstSizeResult;
5055 if (!E->getArg(Arg: 2)->EvaluateAsInt(Result&: SizeResult, Ctx: CGM.getContext()) ||
5056 !E->getArg(Arg: 3)->EvaluateAsInt(Result&: DstSizeResult, Ctx: CGM.getContext()))
5057 break;
5058 llvm::APSInt Size = SizeResult.Val.getInt();
5059 llvm::APSInt DstSize = DstSizeResult.Val.getInt();
5060 if (Size.ugt(RHS: DstSize))
5061 break;
5062 Address Dest = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
5063 Value *ByteVal = Builder.CreateTrunc(V: EmitScalarExpr(E: E->getArg(Arg: 1)),
5064 DestTy: Builder.getInt8Ty());
5065 Value *SizeVal = llvm::ConstantInt::get(Context&: Builder.getContext(), V: Size);
5066 auto *I = Builder.CreateMemSet(Dest, Value: ByteVal, Size: SizeVal, IsVolatile: false);
5067 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
5068 return RValue::get(Addr: Dest, CGF&: *this);
5069 }
5070 case Builtin::BI__builtin_wmemchr: {
5071 // The MSVC runtime library does not provide a definition of wmemchr, so we
5072 // need an inline implementation.
5073 if (!getTarget().getTriple().isOSMSVCRT())
5074 break;
5075
5076 llvm::Type *WCharTy = ConvertType(T: getContext().WCharTy);
5077 Value *Str = EmitScalarExpr(E: E->getArg(Arg: 0));
5078 Value *Chr = EmitScalarExpr(E: E->getArg(Arg: 1));
5079 Value *Size = EmitScalarExpr(E: E->getArg(Arg: 2));
5080
5081 BasicBlock *Entry = Builder.GetInsertBlock();
5082 BasicBlock *CmpEq = createBasicBlock(name: "wmemchr.eq");
5083 BasicBlock *Next = createBasicBlock(name: "wmemchr.next");
5084 BasicBlock *Exit = createBasicBlock(name: "wmemchr.exit");
5085 Value *SizeEq0 = Builder.CreateICmpEQ(LHS: Size, RHS: ConstantInt::get(Ty: SizeTy, V: 0));
5086 Builder.CreateCondBr(Cond: SizeEq0, True: Exit, False: CmpEq);
5087
5088 EmitBlock(BB: CmpEq);
5089 PHINode *StrPhi = Builder.CreatePHI(Ty: Str->getType(), NumReservedValues: 2);
5090 StrPhi->addIncoming(V: Str, BB: Entry);
5091 PHINode *SizePhi = Builder.CreatePHI(Ty: SizeTy, NumReservedValues: 2);
5092 SizePhi->addIncoming(V: Size, BB: Entry);
5093 CharUnits WCharAlign =
5094 getContext().getTypeAlignInChars(T: getContext().WCharTy);
5095 Value *StrCh = Builder.CreateAlignedLoad(Ty: WCharTy, Addr: StrPhi, Align: WCharAlign);
5096 Value *FoundChr = Builder.CreateConstInBoundsGEP1_32(Ty: WCharTy, Ptr: StrPhi, Idx0: 0);
5097 Value *StrEqChr = Builder.CreateICmpEQ(LHS: StrCh, RHS: Chr);
5098 Builder.CreateCondBr(Cond: StrEqChr, True: Exit, False: Next);
5099
5100 EmitBlock(BB: Next);
5101 Value *NextStr = Builder.CreateConstInBoundsGEP1_32(Ty: WCharTy, Ptr: StrPhi, Idx0: 1);
5102 Value *NextSize = Builder.CreateSub(LHS: SizePhi, RHS: ConstantInt::get(Ty: SizeTy, V: 1));
5103 Value *NextSizeEq0 =
5104 Builder.CreateICmpEQ(LHS: NextSize, RHS: ConstantInt::get(Ty: SizeTy, V: 0));
5105 Builder.CreateCondBr(Cond: NextSizeEq0, True: Exit, False: CmpEq);
5106 StrPhi->addIncoming(V: NextStr, BB: Next);
5107 SizePhi->addIncoming(V: NextSize, BB: Next);
5108
5109 EmitBlock(BB: Exit);
5110 PHINode *Ret = Builder.CreatePHI(Ty: Str->getType(), NumReservedValues: 3);
5111 Ret->addIncoming(V: llvm::Constant::getNullValue(Ty: Str->getType()), BB: Entry);
5112 Ret->addIncoming(V: llvm::Constant::getNullValue(Ty: Str->getType()), BB: Next);
5113 Ret->addIncoming(V: FoundChr, BB: CmpEq);
5114 return RValue::get(V: Ret);
5115 }
5116 case Builtin::BI__builtin_wmemcmp: {
5117 // The MSVC runtime library does not provide a definition of wmemcmp, so we
5118 // need an inline implementation.
5119 if (!getTarget().getTriple().isOSMSVCRT())
5120 break;
5121
5122 llvm::Type *WCharTy = ConvertType(T: getContext().WCharTy);
5123
5124 Value *Dst = EmitScalarExpr(E: E->getArg(Arg: 0));
5125 Value *Src = EmitScalarExpr(E: E->getArg(Arg: 1));
5126 Value *Size = EmitScalarExpr(E: E->getArg(Arg: 2));
5127
5128 BasicBlock *Entry = Builder.GetInsertBlock();
5129 BasicBlock *CmpGT = createBasicBlock(name: "wmemcmp.gt");
5130 BasicBlock *CmpLT = createBasicBlock(name: "wmemcmp.lt");
5131 BasicBlock *Next = createBasicBlock(name: "wmemcmp.next");
5132 BasicBlock *Exit = createBasicBlock(name: "wmemcmp.exit");
5133 Value *SizeEq0 = Builder.CreateICmpEQ(LHS: Size, RHS: ConstantInt::get(Ty: SizeTy, V: 0));
5134 Builder.CreateCondBr(Cond: SizeEq0, True: Exit, False: CmpGT);
5135
5136 EmitBlock(BB: CmpGT);
5137 PHINode *DstPhi = Builder.CreatePHI(Ty: Dst->getType(), NumReservedValues: 2);
5138 DstPhi->addIncoming(V: Dst, BB: Entry);
5139 PHINode *SrcPhi = Builder.CreatePHI(Ty: Src->getType(), NumReservedValues: 2);
5140 SrcPhi->addIncoming(V: Src, BB: Entry);
5141 PHINode *SizePhi = Builder.CreatePHI(Ty: SizeTy, NumReservedValues: 2);
5142 SizePhi->addIncoming(V: Size, BB: Entry);
5143 CharUnits WCharAlign =
5144 getContext().getTypeAlignInChars(T: getContext().WCharTy);
5145 Value *DstCh = Builder.CreateAlignedLoad(Ty: WCharTy, Addr: DstPhi, Align: WCharAlign);
5146 Value *SrcCh = Builder.CreateAlignedLoad(Ty: WCharTy, Addr: SrcPhi, Align: WCharAlign);
5147 Value *DstGtSrc = Builder.CreateICmpUGT(LHS: DstCh, RHS: SrcCh);
5148 Builder.CreateCondBr(Cond: DstGtSrc, True: Exit, False: CmpLT);
5149
5150 EmitBlock(BB: CmpLT);
5151 Value *DstLtSrc = Builder.CreateICmpULT(LHS: DstCh, RHS: SrcCh);
5152 Builder.CreateCondBr(Cond: DstLtSrc, True: Exit, False: Next);
5153
5154 EmitBlock(BB: Next);
5155 Value *NextDst = Builder.CreateConstInBoundsGEP1_32(Ty: WCharTy, Ptr: DstPhi, Idx0: 1);
5156 Value *NextSrc = Builder.CreateConstInBoundsGEP1_32(Ty: WCharTy, Ptr: SrcPhi, Idx0: 1);
5157 Value *NextSize = Builder.CreateSub(LHS: SizePhi, RHS: ConstantInt::get(Ty: SizeTy, V: 1));
5158 Value *NextSizeEq0 =
5159 Builder.CreateICmpEQ(LHS: NextSize, RHS: ConstantInt::get(Ty: SizeTy, V: 0));
5160 Builder.CreateCondBr(Cond: NextSizeEq0, True: Exit, False: CmpGT);
5161 DstPhi->addIncoming(V: NextDst, BB: Next);
5162 SrcPhi->addIncoming(V: NextSrc, BB: Next);
5163 SizePhi->addIncoming(V: NextSize, BB: Next);
5164
5165 EmitBlock(BB: Exit);
5166 PHINode *Ret = Builder.CreatePHI(Ty: IntTy, NumReservedValues: 4);
5167 Ret->addIncoming(V: ConstantInt::get(Ty: IntTy, V: 0), BB: Entry);
5168 Ret->addIncoming(V: ConstantInt::get(Ty: IntTy, V: 1), BB: CmpGT);
5169 Ret->addIncoming(V: ConstantInt::getAllOnesValue(Ty: IntTy), BB: CmpLT);
5170 Ret->addIncoming(V: ConstantInt::get(Ty: IntTy, V: 0), BB: Next);
5171 return RValue::get(V: Ret);
5172 }
5173 case Builtin::BI__builtin_dwarf_cfa: {
5174 // The offset in bytes from the first argument to the CFA.
5175 //
5176 // Why on earth is this in the frontend? Is there any reason at
5177 // all that the backend can't reasonably determine this while
5178 // lowering llvm.eh.dwarf.cfa()?
5179 //
5180 // TODO: If there's a satisfactory reason, add a target hook for
5181 // this instead of hard-coding 0, which is correct for most targets.
5182 int32_t Offset = 0;
5183
5184 Function *F = CGM.getIntrinsic(IID: Intrinsic::eh_dwarf_cfa);
5185 return RValue::get(V: Builder.CreateCall(Callee: F,
5186 Args: llvm::ConstantInt::get(Ty: Int32Ty, V: Offset)));
5187 }
5188 case Builtin::BI__builtin_return_address: {
5189 Value *Depth = ConstantEmitter(*this).emitAbstract(E: E->getArg(Arg: 0),
5190 T: getContext().UnsignedIntTy);
5191 Depth = Builder.CreateZExtOrTrunc(V: Depth, DestTy: Int32Ty);
5192 Function *F =
5193 CGM.getIntrinsic(IID: Intrinsic::returnaddress, Tys: {CGM.ProgramPtrTy});
5194 return RValue::get(V: Builder.CreateCall(Callee: F, Args: Depth));
5195 }
5196 case Builtin::BI_ReturnAddress: {
5197 Function *F =
5198 CGM.getIntrinsic(IID: Intrinsic::returnaddress, Tys: {CGM.ProgramPtrTy});
5199 return RValue::get(V: Builder.CreateCall(Callee: F, Args: Builder.getInt32(C: 0)));
5200 }
5201 case Builtin::BI__builtin_frame_address: {
5202 Value *Depth = ConstantEmitter(*this).emitAbstract(E: E->getArg(Arg: 0),
5203 T: getContext().UnsignedIntTy);
5204 Depth = Builder.CreateZExtOrTrunc(V: Depth, DestTy: Int32Ty);
5205 Function *F = CGM.getIntrinsic(IID: Intrinsic::frameaddress, Tys: AllocaInt8PtrTy);
5206 return RValue::get(V: Builder.CreateCall(Callee: F, Args: Depth));
5207 }
5208 case Builtin::BI__builtin_stack_address: {
5209 return RValue::get(V: Builder.CreateCall(
5210 Callee: CGM.getIntrinsic(IID: Intrinsic::stackaddress, Tys: AllocaInt8PtrTy)));
5211 }
5212 case Builtin::BI__builtin_extract_return_addr: {
5213 Value *Address = EmitScalarExpr(E: E->getArg(Arg: 0));
5214 Value *Result = getTargetHooks().decodeReturnAddress(CGF&: *this, Address);
5215 return RValue::get(V: Result);
5216 }
5217 case Builtin::BI__builtin_frob_return_addr: {
5218 Value *Address = EmitScalarExpr(E: E->getArg(Arg: 0));
5219 Value *Result = getTargetHooks().encodeReturnAddress(CGF&: *this, Address);
5220 return RValue::get(V: Result);
5221 }
5222 case Builtin::BI__builtin_dwarf_sp_column: {
5223 llvm::IntegerType *Ty
5224 = cast<llvm::IntegerType>(Val: ConvertType(T: E->getType()));
5225 int Column = getTargetHooks().getDwarfEHStackPointer(M&: CGM);
5226 if (Column == -1) {
5227 CGM.ErrorUnsupported(S: E, Type: "__builtin_dwarf_sp_column");
5228 return RValue::get(V: llvm::UndefValue::get(T: Ty));
5229 }
5230 return RValue::get(V: llvm::ConstantInt::get(Ty, V: Column, IsSigned: true));
5231 }
5232 case Builtin::BI__builtin_init_dwarf_reg_size_table: {
5233 Value *Address = EmitScalarExpr(E: E->getArg(Arg: 0));
5234 if (getTargetHooks().initDwarfEHRegSizeTable(CGF&: *this, Address))
5235 CGM.ErrorUnsupported(S: E, Type: "__builtin_init_dwarf_reg_size_table");
5236 return RValue::get(V: llvm::UndefValue::get(T: ConvertType(T: E->getType())));
5237 }
5238 case Builtin::BI__builtin_eh_return: {
5239 Value *Int = EmitScalarExpr(E: E->getArg(Arg: 0));
5240 Value *Ptr = EmitScalarExpr(E: E->getArg(Arg: 1));
5241
5242 llvm::IntegerType *IntTy = cast<llvm::IntegerType>(Val: Int->getType());
5243 assert((IntTy->getBitWidth() == 32 || IntTy->getBitWidth() == 64) &&
5244 "LLVM's __builtin_eh_return only supports 32- and 64-bit variants");
5245 Function *F =
5246 CGM.getIntrinsic(IID: IntTy->getBitWidth() == 32 ? Intrinsic::eh_return_i32
5247 : Intrinsic::eh_return_i64);
5248 Builder.CreateCall(Callee: F, Args: {Int, Ptr});
5249 Builder.CreateUnreachable();
5250
5251 // We do need to preserve an insertion point.
5252 EmitBlock(BB: createBasicBlock(name: "builtin_eh_return.cont"));
5253
5254 return RValue::get(V: nullptr);
5255 }
5256 case Builtin::BI__builtin_unwind_init: {
5257 Function *F = CGM.getIntrinsic(IID: Intrinsic::eh_unwind_init);
5258 Builder.CreateCall(Callee: F);
5259 return RValue::get(V: nullptr);
5260 }
5261 case Builtin::BI__builtin_extend_pointer: {
5262 // Extends a pointer to the size of an _Unwind_Word, which is
5263 // uint64_t on all platforms. Generally this gets poked into a
5264 // register and eventually used as an address, so if the
5265 // addressing registers are wider than pointers and the platform
5266 // doesn't implicitly ignore high-order bits when doing
5267 // addressing, we need to make sure we zext / sext based on
5268 // the platform's expectations.
5269 //
5270 // See: http://gcc.gnu.org/ml/gcc-bugs/2002-02/msg00237.html
5271
5272 // Cast the pointer to intptr_t.
5273 Value *Ptr = EmitScalarExpr(E: E->getArg(Arg: 0));
5274 Value *Result = Builder.CreatePtrToInt(V: Ptr, DestTy: IntPtrTy, Name: "extend.cast");
5275
5276 // If that's 64 bits, we're done.
5277 if (IntPtrTy->getBitWidth() == 64)
5278 return RValue::get(V: Result);
5279
5280 // Otherwise, ask the codegen data what to do.
5281 if (getTargetHooks().extendPointerWithSExt())
5282 return RValue::get(V: Builder.CreateSExt(V: Result, DestTy: Int64Ty, Name: "extend.sext"));
5283 else
5284 return RValue::get(V: Builder.CreateZExt(V: Result, DestTy: Int64Ty, Name: "extend.zext"));
5285 }
5286 case Builtin::BI__builtin_setjmp: {
5287 // Buffer is a void**.
5288 Address Buf = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
5289
5290 if (getTarget().getTriple().getArch() == llvm::Triple::systemz) {
5291 // On this target, the back end fills in the context buffer completely.
5292 // It doesn't really matter if the frontend stores to the buffer before
5293 // calling setjmp, the back-end is going to overwrite them anyway.
5294 Function *F = CGM.getIntrinsic(IID: Intrinsic::eh_sjlj_setjmp);
5295 return RValue::get(V: Builder.CreateCall(Callee: F, Args: Buf.emitRawPointer(CGF&: *this)));
5296 }
5297
5298 // Store the frame pointer to the setjmp buffer.
5299 Value *FrameAddr = Builder.CreateCall(
5300 Callee: CGM.getIntrinsic(IID: Intrinsic::frameaddress, Tys: AllocaInt8PtrTy),
5301 Args: ConstantInt::get(Ty: Int32Ty, V: 0));
5302 Builder.CreateStore(Val: FrameAddr, Addr: Buf);
5303
5304 // Store the stack pointer to the setjmp buffer.
5305 Value *StackAddr = Builder.CreateStackSave();
5306 assert(Buf.emitRawPointer(*this)->getType() == StackAddr->getType());
5307
5308 Address StackSaveSlot = Builder.CreateConstInBoundsGEP(Addr: Buf, Index: 2);
5309 Builder.CreateStore(Val: StackAddr, Addr: StackSaveSlot);
5310
5311 // Call LLVM's EH setjmp, which is lightweight.
5312 Function *F = CGM.getIntrinsic(IID: Intrinsic::eh_sjlj_setjmp);
5313 return RValue::get(V: Builder.CreateCall(Callee: F, Args: Buf.emitRawPointer(CGF&: *this)));
5314 }
5315 case Builtin::BI__builtin_longjmp: {
5316 Value *Buf = EmitScalarExpr(E: E->getArg(Arg: 0));
5317
5318 // Call LLVM's EH longjmp, which is lightweight.
5319 Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Intrinsic::eh_sjlj_longjmp), Args: Buf);
5320
5321 // longjmp doesn't return; mark this as unreachable.
5322 Builder.CreateUnreachable();
5323
5324 // We do need to preserve an insertion point.
5325 EmitBlock(BB: createBasicBlock(name: "longjmp.cont"));
5326
5327 return RValue::get(V: nullptr);
5328 }
5329 case Builtin::BI__builtin_launder: {
5330 const Expr *Arg = E->getArg(Arg: 0);
5331 QualType ArgTy = Arg->getType()->getPointeeType();
5332 Value *Ptr = EmitScalarExpr(E: Arg);
5333 if (TypeRequiresBuiltinLaunder(CGM, Ty: ArgTy))
5334 Ptr = Builder.CreateLaunderInvariantGroup(Ptr);
5335
5336 return RValue::get(V: Ptr);
5337 }
5338 case Builtin::BI__builtin_clear_padding: {
5339 Address Src = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
5340 auto PointeeTy = E->getArg(Arg: 0)->getType()->getPointeeType();
5341
5342 llvm::ArrayRef<ASTContext::BitInterval> Padding =
5343 getContext().getPaddingIntervals(Ty: PointeeTy);
5344 for (const auto &Interval : Padding)
5345 ClearPadding(CGF&: *this, Src, PaddingInterval: Interval);
5346
5347 return RValue::get(V: nullptr);
5348 }
5349 case Builtin::BI__sync_fetch_and_add:
5350 case Builtin::BI__sync_fetch_and_sub:
5351 case Builtin::BI__sync_fetch_and_or:
5352 case Builtin::BI__sync_fetch_and_and:
5353 case Builtin::BI__sync_fetch_and_xor:
5354 case Builtin::BI__sync_fetch_and_nand:
5355 case Builtin::BI__sync_add_and_fetch:
5356 case Builtin::BI__sync_sub_and_fetch:
5357 case Builtin::BI__sync_and_and_fetch:
5358 case Builtin::BI__sync_or_and_fetch:
5359 case Builtin::BI__sync_xor_and_fetch:
5360 case Builtin::BI__sync_nand_and_fetch:
5361 case Builtin::BI__sync_val_compare_and_swap:
5362 case Builtin::BI__sync_bool_compare_and_swap:
5363 case Builtin::BI__sync_lock_test_and_set:
5364 case Builtin::BI__sync_lock_release:
5365 case Builtin::BI__sync_swap:
5366 llvm_unreachable("Shouldn't make it through sema");
5367 case Builtin::BI__sync_fetch_and_add_1:
5368 case Builtin::BI__sync_fetch_and_add_2:
5369 case Builtin::BI__sync_fetch_and_add_4:
5370 case Builtin::BI__sync_fetch_and_add_8:
5371 case Builtin::BI__sync_fetch_and_add_16:
5372 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::Add, E);
5373 case Builtin::BI__sync_fetch_and_sub_1:
5374 case Builtin::BI__sync_fetch_and_sub_2:
5375 case Builtin::BI__sync_fetch_and_sub_4:
5376 case Builtin::BI__sync_fetch_and_sub_8:
5377 case Builtin::BI__sync_fetch_and_sub_16:
5378 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::Sub, E);
5379 case Builtin::BI__sync_fetch_and_or_1:
5380 case Builtin::BI__sync_fetch_and_or_2:
5381 case Builtin::BI__sync_fetch_and_or_4:
5382 case Builtin::BI__sync_fetch_and_or_8:
5383 case Builtin::BI__sync_fetch_and_or_16:
5384 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::Or, E);
5385 case Builtin::BI__sync_fetch_and_and_1:
5386 case Builtin::BI__sync_fetch_and_and_2:
5387 case Builtin::BI__sync_fetch_and_and_4:
5388 case Builtin::BI__sync_fetch_and_and_8:
5389 case Builtin::BI__sync_fetch_and_and_16:
5390 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::And, E);
5391 case Builtin::BI__sync_fetch_and_xor_1:
5392 case Builtin::BI__sync_fetch_and_xor_2:
5393 case Builtin::BI__sync_fetch_and_xor_4:
5394 case Builtin::BI__sync_fetch_and_xor_8:
5395 case Builtin::BI__sync_fetch_and_xor_16:
5396 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::Xor, E);
5397 case Builtin::BI__sync_fetch_and_nand_1:
5398 case Builtin::BI__sync_fetch_and_nand_2:
5399 case Builtin::BI__sync_fetch_and_nand_4:
5400 case Builtin::BI__sync_fetch_and_nand_8:
5401 case Builtin::BI__sync_fetch_and_nand_16:
5402 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::Nand, E);
5403
5404 // Clang extensions: not overloaded yet.
5405 case Builtin::BI__sync_fetch_and_min:
5406 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::Min, E);
5407 case Builtin::BI__sync_fetch_and_max:
5408 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::Max, E);
5409 case Builtin::BI__sync_fetch_and_umin:
5410 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::UMin, E);
5411 case Builtin::BI__sync_fetch_and_umax:
5412 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::UMax, E);
5413
5414 case Builtin::BI__sync_add_and_fetch_1:
5415 case Builtin::BI__sync_add_and_fetch_2:
5416 case Builtin::BI__sync_add_and_fetch_4:
5417 case Builtin::BI__sync_add_and_fetch_8:
5418 case Builtin::BI__sync_add_and_fetch_16:
5419 return EmitBinaryAtomicPost(CGF&: *this, Kind: llvm::AtomicRMWInst::Add, E,
5420 Op: llvm::Instruction::Add);
5421 case Builtin::BI__sync_sub_and_fetch_1:
5422 case Builtin::BI__sync_sub_and_fetch_2:
5423 case Builtin::BI__sync_sub_and_fetch_4:
5424 case Builtin::BI__sync_sub_and_fetch_8:
5425 case Builtin::BI__sync_sub_and_fetch_16:
5426 return EmitBinaryAtomicPost(CGF&: *this, Kind: llvm::AtomicRMWInst::Sub, E,
5427 Op: llvm::Instruction::Sub);
5428 case Builtin::BI__sync_and_and_fetch_1:
5429 case Builtin::BI__sync_and_and_fetch_2:
5430 case Builtin::BI__sync_and_and_fetch_4:
5431 case Builtin::BI__sync_and_and_fetch_8:
5432 case Builtin::BI__sync_and_and_fetch_16:
5433 return EmitBinaryAtomicPost(CGF&: *this, Kind: llvm::AtomicRMWInst::And, E,
5434 Op: llvm::Instruction::And);
5435 case Builtin::BI__sync_or_and_fetch_1:
5436 case Builtin::BI__sync_or_and_fetch_2:
5437 case Builtin::BI__sync_or_and_fetch_4:
5438 case Builtin::BI__sync_or_and_fetch_8:
5439 case Builtin::BI__sync_or_and_fetch_16:
5440 return EmitBinaryAtomicPost(CGF&: *this, Kind: llvm::AtomicRMWInst::Or, E,
5441 Op: llvm::Instruction::Or);
5442 case Builtin::BI__sync_xor_and_fetch_1:
5443 case Builtin::BI__sync_xor_and_fetch_2:
5444 case Builtin::BI__sync_xor_and_fetch_4:
5445 case Builtin::BI__sync_xor_and_fetch_8:
5446 case Builtin::BI__sync_xor_and_fetch_16:
5447 return EmitBinaryAtomicPost(CGF&: *this, Kind: llvm::AtomicRMWInst::Xor, E,
5448 Op: llvm::Instruction::Xor);
5449 case Builtin::BI__sync_nand_and_fetch_1:
5450 case Builtin::BI__sync_nand_and_fetch_2:
5451 case Builtin::BI__sync_nand_and_fetch_4:
5452 case Builtin::BI__sync_nand_and_fetch_8:
5453 case Builtin::BI__sync_nand_and_fetch_16:
5454 return EmitBinaryAtomicPost(CGF&: *this, Kind: llvm::AtomicRMWInst::Nand, E,
5455 Op: llvm::Instruction::And, Invert: true);
5456
5457 case Builtin::BI__sync_val_compare_and_swap_1:
5458 case Builtin::BI__sync_val_compare_and_swap_2:
5459 case Builtin::BI__sync_val_compare_and_swap_4:
5460 case Builtin::BI__sync_val_compare_and_swap_8:
5461 case Builtin::BI__sync_val_compare_and_swap_16:
5462 return RValue::get(V: MakeAtomicCmpXchgValue(
5463 CGF&: *this, E, ReturnBool: false, SuccessOrdering: AtomicOrdering::SequentiallyConsistent,
5464 FailureOrdering: AtomicOrdering::SequentiallyConsistent));
5465
5466 case Builtin::BI__sync_bool_compare_and_swap_1:
5467 case Builtin::BI__sync_bool_compare_and_swap_2:
5468 case Builtin::BI__sync_bool_compare_and_swap_4:
5469 case Builtin::BI__sync_bool_compare_and_swap_8:
5470 case Builtin::BI__sync_bool_compare_and_swap_16:
5471 return RValue::get(V: MakeAtomicCmpXchgValue(
5472 CGF&: *this, E, ReturnBool: true, SuccessOrdering: AtomicOrdering::SequentiallyConsistent,
5473 FailureOrdering: AtomicOrdering::SequentiallyConsistent));
5474
5475 case Builtin::BI__sync_swap_1:
5476 case Builtin::BI__sync_swap_2:
5477 case Builtin::BI__sync_swap_4:
5478 case Builtin::BI__sync_swap_8:
5479 case Builtin::BI__sync_swap_16:
5480 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::Xchg, E);
5481
5482 case Builtin::BI__sync_lock_test_and_set_1:
5483 case Builtin::BI__sync_lock_test_and_set_2:
5484 case Builtin::BI__sync_lock_test_and_set_4:
5485 case Builtin::BI__sync_lock_test_and_set_8:
5486 case Builtin::BI__sync_lock_test_and_set_16:
5487 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::Xchg, E);
5488
5489 case Builtin::BI__sync_lock_release_1:
5490 case Builtin::BI__sync_lock_release_2:
5491 case Builtin::BI__sync_lock_release_4:
5492 case Builtin::BI__sync_lock_release_8:
5493 case Builtin::BI__sync_lock_release_16: {
5494 Address Ptr = CheckAtomicAlignment(CGF&: *this, E);
5495 QualType ElTy = E->getArg(Arg: 0)->getType()->getPointeeType();
5496
5497 llvm::Type *ITy = llvm::IntegerType::get(C&: getLLVMContext(),
5498 NumBits: getContext().getTypeSize(T: ElTy));
5499 llvm::StoreInst *Store =
5500 Builder.CreateStore(Val: llvm::Constant::getNullValue(Ty: ITy), Addr: Ptr);
5501 Store->setAtomic(Ordering: llvm::AtomicOrdering::Release);
5502 return RValue::get(V: nullptr);
5503 }
5504
5505 case Builtin::BI__sync_synchronize: {
5506 // We assume this is supposed to correspond to a C++0x-style
5507 // sequentially-consistent fence (i.e. this is only usable for
5508 // synchronization, not device I/O or anything like that). This intrinsic
5509 // is really badly designed in the sense that in theory, there isn't
5510 // any way to safely use it... but in practice, it mostly works
5511 // to use it with non-atomic loads and stores to get acquire/release
5512 // semantics.
5513 Builder.CreateFence(Ordering: llvm::AtomicOrdering::SequentiallyConsistent);
5514 return RValue::get(V: nullptr);
5515 }
5516
5517 case Builtin::BI__builtin_nontemporal_load:
5518 return RValue::get(V: EmitNontemporalLoad(CGF&: *this, E));
5519 case Builtin::BI__builtin_nontemporal_store:
5520 return RValue::get(V: EmitNontemporalStore(CGF&: *this, E));
5521 case Builtin::BI__c11_atomic_is_lock_free:
5522 case Builtin::BI__atomic_is_lock_free: {
5523 // Call "bool __atomic_is_lock_free(size_t size, void *ptr)". For the
5524 // __c11 builtin, ptr is 0 (indicating a properly-aligned object), since
5525 // _Atomic(T) is always properly-aligned.
5526 const char *LibCallName = "__atomic_is_lock_free";
5527 CallArgList Args;
5528 Args.add(rvalue: RValue::get(V: EmitScalarExpr(E: E->getArg(Arg: 0))),
5529 type: getContext().getSizeType());
5530 if (BuiltinID == Builtin::BI__atomic_is_lock_free)
5531 Args.add(rvalue: RValue::get(V: EmitScalarExpr(E: E->getArg(Arg: 1))),
5532 type: getContext().VoidPtrTy);
5533 else
5534 Args.add(rvalue: RValue::get(V: llvm::Constant::getNullValue(Ty: VoidPtrTy)),
5535 type: getContext().VoidPtrTy);
5536 const CGFunctionInfo &FuncInfo =
5537 CGM.getTypes().arrangeBuiltinFunctionCall(resultType: E->getType(), args: Args);
5538 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(Info: FuncInfo);
5539 llvm::FunctionCallee Func = CGM.CreateRuntimeFunction(Ty: FTy, Name: LibCallName);
5540 return EmitCall(CallInfo: FuncInfo, Callee: CGCallee::forDirect(functionPtr: Func),
5541 ReturnValue: ReturnValueSlot(), Args);
5542 }
5543
5544 case Builtin::BI__atomic_thread_fence:
5545 case Builtin::BI__atomic_signal_fence:
5546 case Builtin::BI__c11_atomic_thread_fence:
5547 case Builtin::BI__c11_atomic_signal_fence: {
5548 llvm::SyncScope::ID SSID;
5549 if (BuiltinID == Builtin::BI__atomic_signal_fence ||
5550 BuiltinID == Builtin::BI__c11_atomic_signal_fence)
5551 SSID = llvm::SyncScope::SingleThread;
5552 else
5553 SSID = llvm::SyncScope::System;
5554 Value *Order = EmitScalarExpr(E: E->getArg(Arg: 0));
5555 if (isa<llvm::ConstantInt>(Val: Order)) {
5556 int ord = cast<llvm::ConstantInt>(Val: Order)->getZExtValue();
5557 switch (ord) {
5558 case 0: // memory_order_relaxed
5559 default: // invalid order
5560 break;
5561 case 1: // memory_order_consume
5562 case 2: // memory_order_acquire
5563 emitAtomicFence(Order: llvm::AtomicOrdering::Acquire, SSID);
5564 break;
5565 case 3: // memory_order_release
5566 emitAtomicFence(Order: llvm::AtomicOrdering::Release, SSID);
5567 break;
5568 case 4: // memory_order_acq_rel
5569 emitAtomicFence(Order: llvm::AtomicOrdering::AcquireRelease, SSID);
5570 break;
5571 case 5: // memory_order_seq_cst
5572 emitAtomicFence(Order: llvm::AtomicOrdering::SequentiallyConsistent, SSID);
5573 break;
5574 }
5575 return RValue::get(V: nullptr);
5576 }
5577
5578 llvm::BasicBlock *AcquireBB, *ReleaseBB, *AcqRelBB, *SeqCstBB;
5579 AcquireBB = createBasicBlock(name: "acquire", parent: CurFn);
5580 ReleaseBB = createBasicBlock(name: "release", parent: CurFn);
5581 AcqRelBB = createBasicBlock(name: "acqrel", parent: CurFn);
5582 SeqCstBB = createBasicBlock(name: "seqcst", parent: CurFn);
5583 llvm::BasicBlock *ContBB = createBasicBlock(name: "atomic.continue", parent: CurFn);
5584
5585 Order = Builder.CreateIntCast(V: Order, DestTy: Builder.getInt32Ty(), isSigned: false);
5586 llvm::SwitchInst *SI = Builder.CreateSwitch(V: Order, Dest: ContBB);
5587
5588 Builder.SetInsertPoint(AcquireBB);
5589 emitAtomicFence(Order: llvm::AtomicOrdering::Acquire, SSID);
5590 Builder.CreateBr(Dest: ContBB);
5591 SI->addCase(OnVal: Builder.getInt32(C: 1), Dest: AcquireBB);
5592 SI->addCase(OnVal: Builder.getInt32(C: 2), Dest: AcquireBB);
5593
5594 Builder.SetInsertPoint(ReleaseBB);
5595 emitAtomicFence(Order: llvm::AtomicOrdering::Release, SSID);
5596 Builder.CreateBr(Dest: ContBB);
5597 SI->addCase(OnVal: Builder.getInt32(C: 3), Dest: ReleaseBB);
5598
5599 Builder.SetInsertPoint(AcqRelBB);
5600 emitAtomicFence(Order: llvm::AtomicOrdering::AcquireRelease, SSID);
5601 Builder.CreateBr(Dest: ContBB);
5602 SI->addCase(OnVal: Builder.getInt32(C: 4), Dest: AcqRelBB);
5603
5604 Builder.SetInsertPoint(SeqCstBB);
5605 emitAtomicFence(Order: llvm::AtomicOrdering::SequentiallyConsistent, SSID);
5606 Builder.CreateBr(Dest: ContBB);
5607 SI->addCase(OnVal: Builder.getInt32(C: 5), Dest: SeqCstBB);
5608
5609 Builder.SetInsertPoint(ContBB);
5610 return RValue::get(V: nullptr);
5611 }
5612 case Builtin::BI__scoped_atomic_thread_fence: {
5613 auto ScopeModel = AtomicScopeModel::create(K: AtomicScopeModelKind::Generic);
5614
5615 Value *Order = EmitScalarExpr(E: E->getArg(Arg: 0));
5616 Value *Scope = EmitScalarExpr(E: E->getArg(Arg: 1));
5617 auto Ord = dyn_cast<llvm::ConstantInt>(Val: Order);
5618 auto Scp = dyn_cast<llvm::ConstantInt>(Val: Scope);
5619 if (Ord && Scp) {
5620 SyncScope SS = ScopeModel->isValid(S: Scp->getZExtValue())
5621 ? ScopeModel->map(S: Scp->getZExtValue())
5622 : ScopeModel->map(S: ScopeModel->getFallBackValue());
5623 switch (Ord->getZExtValue()) {
5624 case 0: // memory_order_relaxed
5625 default: // invalid order
5626 break;
5627 case 1: // memory_order_consume
5628 case 2: // memory_order_acquire
5629 emitAtomicFence(Order: llvm::AtomicOrdering::Acquire,
5630 SSID: getTargetHooks().getLLVMSyncScopeID(
5631 LangOpts: getLangOpts(), Scope: SS, Ordering: llvm::AtomicOrdering::Acquire,
5632 Ctx&: getLLVMContext()));
5633 break;
5634 case 3: // memory_order_release
5635 emitAtomicFence(Order: llvm::AtomicOrdering::Release,
5636 SSID: getTargetHooks().getLLVMSyncScopeID(
5637 LangOpts: getLangOpts(), Scope: SS, Ordering: llvm::AtomicOrdering::Release,
5638 Ctx&: getLLVMContext()));
5639 break;
5640 case 4: // memory_order_acq_rel
5641 emitAtomicFence(Order: llvm::AtomicOrdering::AcquireRelease,
5642 SSID: getTargetHooks().getLLVMSyncScopeID(
5643 LangOpts: getLangOpts(), Scope: SS,
5644 Ordering: llvm::AtomicOrdering::AcquireRelease,
5645 Ctx&: getLLVMContext()));
5646 break;
5647 case 5: // memory_order_seq_cst
5648 emitAtomicFence(Order: llvm::AtomicOrdering::SequentiallyConsistent,
5649 SSID: getTargetHooks().getLLVMSyncScopeID(
5650 LangOpts: getLangOpts(), Scope: SS,
5651 Ordering: llvm::AtomicOrdering::SequentiallyConsistent,
5652 Ctx&: getLLVMContext()));
5653 break;
5654 }
5655 return RValue::get(V: nullptr);
5656 }
5657
5658 llvm::BasicBlock *ContBB = createBasicBlock(name: "atomic.scope.continue", parent: CurFn);
5659
5660 llvm::SmallVector<std::pair<llvm::BasicBlock *, llvm::AtomicOrdering>>
5661 OrderBBs;
5662 if (Ord) {
5663 switch (Ord->getZExtValue()) {
5664 case 0: // memory_order_relaxed
5665 default: // invalid order
5666 ContBB->eraseFromParent();
5667 return RValue::get(V: nullptr);
5668 case 1: // memory_order_consume
5669 case 2: // memory_order_acquire
5670 OrderBBs.emplace_back(Args: Builder.GetInsertBlock(),
5671 Args: llvm::AtomicOrdering::Acquire);
5672 break;
5673 case 3: // memory_order_release
5674 OrderBBs.emplace_back(Args: Builder.GetInsertBlock(),
5675 Args: llvm::AtomicOrdering::Release);
5676 break;
5677 case 4: // memory_order_acq_rel
5678 OrderBBs.emplace_back(Args: Builder.GetInsertBlock(),
5679 Args: llvm::AtomicOrdering::AcquireRelease);
5680 break;
5681 case 5: // memory_order_seq_cst
5682 OrderBBs.emplace_back(Args: Builder.GetInsertBlock(),
5683 Args: llvm::AtomicOrdering::SequentiallyConsistent);
5684 break;
5685 }
5686 } else {
5687 llvm::BasicBlock *AcquireBB = createBasicBlock(name: "acquire", parent: CurFn);
5688 llvm::BasicBlock *ReleaseBB = createBasicBlock(name: "release", parent: CurFn);
5689 llvm::BasicBlock *AcqRelBB = createBasicBlock(name: "acqrel", parent: CurFn);
5690 llvm::BasicBlock *SeqCstBB = createBasicBlock(name: "seqcst", parent: CurFn);
5691
5692 Order = Builder.CreateIntCast(V: Order, DestTy: Builder.getInt32Ty(), isSigned: false);
5693 llvm::SwitchInst *SI = Builder.CreateSwitch(V: Order, Dest: ContBB);
5694 SI->addCase(OnVal: Builder.getInt32(C: 1), Dest: AcquireBB);
5695 SI->addCase(OnVal: Builder.getInt32(C: 2), Dest: AcquireBB);
5696 SI->addCase(OnVal: Builder.getInt32(C: 3), Dest: ReleaseBB);
5697 SI->addCase(OnVal: Builder.getInt32(C: 4), Dest: AcqRelBB);
5698 SI->addCase(OnVal: Builder.getInt32(C: 5), Dest: SeqCstBB);
5699
5700 OrderBBs.emplace_back(Args&: AcquireBB, Args: llvm::AtomicOrdering::Acquire);
5701 OrderBBs.emplace_back(Args&: ReleaseBB, Args: llvm::AtomicOrdering::Release);
5702 OrderBBs.emplace_back(Args&: AcqRelBB, Args: llvm::AtomicOrdering::AcquireRelease);
5703 OrderBBs.emplace_back(Args&: SeqCstBB,
5704 Args: llvm::AtomicOrdering::SequentiallyConsistent);
5705 }
5706
5707 for (auto &[OrderBB, Ordering] : OrderBBs) {
5708 Builder.SetInsertPoint(OrderBB);
5709 if (Scp) {
5710 SyncScope SS = ScopeModel->isValid(S: Scp->getZExtValue())
5711 ? ScopeModel->map(S: Scp->getZExtValue())
5712 : ScopeModel->map(S: ScopeModel->getFallBackValue());
5713 emitAtomicFence(Order: Ordering,
5714 SSID: getTargetHooks().getLLVMSyncScopeID(
5715 LangOpts: getLangOpts(), Scope: SS, Ordering, Ctx&: getLLVMContext()));
5716 Builder.CreateBr(Dest: ContBB);
5717 } else {
5718 llvm::DenseMap<unsigned, llvm::BasicBlock *> BBs;
5719 for (unsigned Scp : ScopeModel->getRuntimeValues())
5720 BBs[Scp] = createBasicBlock(name: getAsString(S: ScopeModel->map(S: Scp)), parent: CurFn);
5721
5722 auto *SC = Builder.CreateIntCast(V: Scope, DestTy: Builder.getInt32Ty(), isSigned: false);
5723 llvm::SwitchInst *SI = Builder.CreateSwitch(V: SC, Dest: ContBB);
5724 for (unsigned Scp : ScopeModel->getRuntimeValues()) {
5725 auto *B = BBs[Scp];
5726 SI->addCase(OnVal: Builder.getInt32(C: Scp), Dest: B);
5727
5728 Builder.SetInsertPoint(B);
5729 emitAtomicFence(Order: Ordering, SSID: getTargetHooks().getLLVMSyncScopeID(
5730 LangOpts: getLangOpts(), Scope: ScopeModel->map(S: Scp),
5731 Ordering, Ctx&: getLLVMContext()));
5732 Builder.CreateBr(Dest: ContBB);
5733 }
5734 }
5735 }
5736
5737 Builder.SetInsertPoint(ContBB);
5738 return RValue::get(V: nullptr);
5739 }
5740
5741 case Builtin::BI__builtin_signbit:
5742 case Builtin::BI__builtin_signbitf:
5743 case Builtin::BI__builtin_signbitl: {
5744 return RValue::get(
5745 V: Builder.CreateZExt(V: EmitSignBit(CGF&: *this, V: EmitScalarExpr(E: E->getArg(Arg: 0))),
5746 DestTy: ConvertType(T: E->getType())));
5747 }
5748 case Builtin::BI__warn_memset_zero_len:
5749 return RValue::getIgnored();
5750 case Builtin::BI__annotation: {
5751 // Re-encode each wide string to UTF8 and make an MDString.
5752 SmallVector<Metadata *, 1> Strings;
5753 for (const Expr *Arg : E->arguments()) {
5754 const auto *Str = cast<StringLiteral>(Val: Arg->IgnoreParenCasts());
5755 assert(Str->getCharByteWidth() == 2 || Str->getCharByteWidth() == 4);
5756 StringRef WideBytes = Str->getBytes();
5757 std::string StrUtf8;
5758 bool Converted =
5759 (Str->getCharByteWidth() == 2)
5760 ? convertUTF16ToUTF8String(
5761 SrcBytes: ArrayRef(WideBytes.data(), WideBytes.size()), Out&: StrUtf8)
5762 : convertUTF32ToUTF8String(
5763 SrcBytes: ArrayRef(WideBytes.data(), WideBytes.size()), Out&: StrUtf8);
5764 if (!Converted) {
5765 CGM.ErrorUnsupported(S: E, Type: "non-Unicode __annotation argument");
5766 continue;
5767 }
5768 Strings.push_back(Elt: llvm::MDString::get(Context&: getLLVMContext(), Str: StrUtf8));
5769 }
5770
5771 // Build and MDTuple of MDStrings and emit the intrinsic call.
5772 llvm::Function *F = CGM.getIntrinsic(IID: Intrinsic::codeview_annotation, Tys: {});
5773 MDTuple *StrTuple = MDTuple::get(Context&: getLLVMContext(), MDs: Strings);
5774 Builder.CreateCall(Callee: F, Args: MetadataAsValue::get(Context&: getLLVMContext(), MD: StrTuple));
5775 return RValue::getIgnored();
5776 }
5777 case Builtin::BI__builtin_annotation: {
5778 llvm::Value *AnnVal = EmitScalarExpr(E: E->getArg(Arg: 0));
5779 llvm::Function *F = CGM.getIntrinsic(
5780 IID: Intrinsic::annotation, Tys: {AnnVal->getType(), CGM.ConstGlobalsPtrTy});
5781
5782 // Get the annotation string, go through casts. Sema requires this to be a
5783 // non-wide string literal, potentially casted, so the cast<> is safe.
5784 const Expr *AnnotationStrExpr = E->getArg(Arg: 1)->IgnoreParenCasts();
5785 StringRef Str = cast<StringLiteral>(Val: AnnotationStrExpr)->getString();
5786 return RValue::get(
5787 V: EmitAnnotationCall(AnnotationFn: F, AnnotatedVal: AnnVal, AnnotationStr: Str, Location: E->getExprLoc(), Attr: nullptr));
5788 }
5789 case Builtin::BI__builtin_addcb:
5790 case Builtin::BI__builtin_addcs:
5791 case Builtin::BI__builtin_addc:
5792 case Builtin::BI__builtin_addcl:
5793 case Builtin::BI__builtin_addcll:
5794 case Builtin::BI__builtin_subcb:
5795 case Builtin::BI__builtin_subcs:
5796 case Builtin::BI__builtin_subc:
5797 case Builtin::BI__builtin_subcl:
5798 case Builtin::BI__builtin_subcll: {
5799
5800 // We translate all of these builtins from expressions of the form:
5801 // int x = ..., y = ..., carryin = ..., carryout, result;
5802 // result = __builtin_addc(x, y, carryin, &carryout);
5803 //
5804 // to LLVM IR of the form:
5805 //
5806 // %tmp1 = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %x, i32 %y)
5807 // %tmpsum1 = extractvalue {i32, i1} %tmp1, 0
5808 // %carry1 = extractvalue {i32, i1} %tmp1, 1
5809 // %tmp2 = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %tmpsum1,
5810 // i32 %carryin)
5811 // %result = extractvalue {i32, i1} %tmp2, 0
5812 // %carry2 = extractvalue {i32, i1} %tmp2, 1
5813 // %tmp3 = or i1 %carry1, %carry2
5814 // %tmp4 = zext i1 %tmp3 to i32
5815 // store i32 %tmp4, i32* %carryout
5816
5817 // Scalarize our inputs.
5818 llvm::Value *X = EmitScalarExpr(E: E->getArg(Arg: 0));
5819 llvm::Value *Y = EmitScalarExpr(E: E->getArg(Arg: 1));
5820 llvm::Value *Carryin = EmitScalarExpr(E: E->getArg(Arg: 2));
5821 Address CarryOutPtr = EmitPointerWithAlignment(Addr: E->getArg(Arg: 3));
5822
5823 // Decide if we are lowering to a uadd.with.overflow or usub.with.overflow.
5824 Intrinsic::ID IntrinsicId;
5825 switch (BuiltinID) {
5826 default: llvm_unreachable("Unknown multiprecision builtin id.");
5827 case Builtin::BI__builtin_addcb:
5828 case Builtin::BI__builtin_addcs:
5829 case Builtin::BI__builtin_addc:
5830 case Builtin::BI__builtin_addcl:
5831 case Builtin::BI__builtin_addcll:
5832 IntrinsicId = Intrinsic::uadd_with_overflow;
5833 break;
5834 case Builtin::BI__builtin_subcb:
5835 case Builtin::BI__builtin_subcs:
5836 case Builtin::BI__builtin_subc:
5837 case Builtin::BI__builtin_subcl:
5838 case Builtin::BI__builtin_subcll:
5839 IntrinsicId = Intrinsic::usub_with_overflow;
5840 break;
5841 }
5842
5843 // Construct our resulting LLVM IR expression.
5844 llvm::Value *Carry1;
5845 llvm::Value *Sum1 = EmitOverflowIntrinsic(CGF&: *this, IntrinsicID: IntrinsicId,
5846 X, Y, Carry&: Carry1);
5847 llvm::Value *Carry2;
5848 llvm::Value *Sum2 = EmitOverflowIntrinsic(CGF&: *this, IntrinsicID: IntrinsicId,
5849 X: Sum1, Y: Carryin, Carry&: Carry2);
5850 llvm::Value *CarryOut = Builder.CreateZExt(V: Builder.CreateOr(LHS: Carry1, RHS: Carry2),
5851 DestTy: X->getType());
5852 Builder.CreateStore(Val: CarryOut, Addr: CarryOutPtr);
5853 return RValue::get(V: Sum2);
5854 }
5855
5856 case Builtin::BI__builtin_add_overflow:
5857 case Builtin::BI__builtin_sub_overflow:
5858 case Builtin::BI__builtin_mul_overflow: {
5859 const clang::Expr *LeftArg = E->getArg(Arg: 0);
5860 const clang::Expr *RightArg = E->getArg(Arg: 1);
5861 const clang::Expr *ResultArg = E->getArg(Arg: 2);
5862
5863 clang::QualType ResultQTy =
5864 ResultArg->getType()->castAs<PointerType>()->getPointeeType();
5865
5866 WidthAndSignedness LeftInfo =
5867 getIntegerWidthAndSignedness(context: CGM.getContext(), Type: LeftArg->getType());
5868 WidthAndSignedness RightInfo =
5869 getIntegerWidthAndSignedness(context: CGM.getContext(), Type: RightArg->getType());
5870 WidthAndSignedness ResultInfo =
5871 getIntegerWidthAndSignedness(context: CGM.getContext(), Type: ResultQTy);
5872
5873 // Handle mixed-sign multiplication as a special case, because adding
5874 // runtime or backend support for our generic irgen would be too expensive.
5875 if (isSpecialMixedSignMultiply(BuiltinID, Op1Info: LeftInfo, Op2Info: RightInfo, ResultInfo))
5876 return EmitCheckedMixedSignMultiply(CGF&: *this, Op1: LeftArg, Op1Info: LeftInfo, Op2: RightArg,
5877 Op2Info: RightInfo, ResultArg, ResultQTy,
5878 ResultInfo);
5879
5880 if (isSpecialUnsignedMultiplySignedResult(BuiltinID, Op1Info: LeftInfo, Op2Info: RightInfo,
5881 ResultInfo))
5882 return EmitCheckedUnsignedMultiplySignedResult(
5883 CGF&: *this, Op1: LeftArg, Op1Info: LeftInfo, Op2: RightArg, Op2Info: RightInfo, ResultArg, ResultQTy,
5884 ResultInfo);
5885
5886 WidthAndSignedness EncompassingInfo =
5887 EncompassingIntegerType(Types: {LeftInfo, RightInfo, ResultInfo});
5888
5889 llvm::Type *EncompassingLLVMTy =
5890 llvm::IntegerType::get(C&: CGM.getLLVMContext(), NumBits: EncompassingInfo.Width);
5891
5892 llvm::Type *ResultLLVMTy = CGM.getTypes().ConvertType(T: ResultQTy);
5893
5894 Intrinsic::ID IntrinsicId;
5895 switch (BuiltinID) {
5896 default:
5897 llvm_unreachable("Unknown overflow builtin id.");
5898 case Builtin::BI__builtin_add_overflow:
5899 IntrinsicId = EncompassingInfo.Signed ? Intrinsic::sadd_with_overflow
5900 : Intrinsic::uadd_with_overflow;
5901 break;
5902 case Builtin::BI__builtin_sub_overflow:
5903 IntrinsicId = EncompassingInfo.Signed ? Intrinsic::ssub_with_overflow
5904 : Intrinsic::usub_with_overflow;
5905 break;
5906 case Builtin::BI__builtin_mul_overflow:
5907 IntrinsicId = EncompassingInfo.Signed ? Intrinsic::smul_with_overflow
5908 : Intrinsic::umul_with_overflow;
5909 break;
5910 }
5911
5912 llvm::Value *Left = EmitScalarExpr(E: LeftArg);
5913 llvm::Value *Right = EmitScalarExpr(E: RightArg);
5914 Address ResultPtr = EmitPointerWithAlignment(Addr: ResultArg);
5915
5916 // Extend each operand to the encompassing type.
5917 Left = Builder.CreateIntCast(V: Left, DestTy: EncompassingLLVMTy, isSigned: LeftInfo.Signed);
5918 Right = Builder.CreateIntCast(V: Right, DestTy: EncompassingLLVMTy, isSigned: RightInfo.Signed);
5919
5920 // Perform the operation on the extended values.
5921 llvm::Value *Overflow, *Result;
5922 Result = EmitOverflowIntrinsic(CGF&: *this, IntrinsicID: IntrinsicId, X: Left, Y: Right, Carry&: Overflow);
5923
5924 if (EncompassingInfo.Width > ResultInfo.Width) {
5925 // The encompassing type is wider than the result type, so we need to
5926 // truncate it.
5927 llvm::Value *ResultTrunc = Builder.CreateTrunc(V: Result, DestTy: ResultLLVMTy);
5928
5929 // To see if the truncation caused an overflow, we will extend
5930 // the result and then compare it to the original result.
5931 llvm::Value *ResultTruncExt = Builder.CreateIntCast(
5932 V: ResultTrunc, DestTy: EncompassingLLVMTy, isSigned: ResultInfo.Signed);
5933 llvm::Value *TruncationOverflow =
5934 Builder.CreateICmpNE(LHS: Result, RHS: ResultTruncExt);
5935
5936 Overflow = Builder.CreateOr(LHS: Overflow, RHS: TruncationOverflow);
5937 Result = ResultTrunc;
5938 }
5939
5940 // Finally, store the result using the pointer.
5941 bool isVolatile =
5942 ResultArg->getType()->getPointeeType().isVolatileQualified();
5943 Builder.CreateStore(Val: EmitToMemory(Value: Result, Ty: ResultQTy), Addr: ResultPtr, IsVolatile: isVolatile);
5944
5945 return RValue::get(V: Overflow);
5946 }
5947
5948 case Builtin::BI__builtin_uadd_overflow:
5949 case Builtin::BI__builtin_uaddl_overflow:
5950 case Builtin::BI__builtin_uaddll_overflow:
5951 case Builtin::BI__builtin_usub_overflow:
5952 case Builtin::BI__builtin_usubl_overflow:
5953 case Builtin::BI__builtin_usubll_overflow:
5954 case Builtin::BI__builtin_umul_overflow:
5955 case Builtin::BI__builtin_umull_overflow:
5956 case Builtin::BI__builtin_umulll_overflow:
5957 case Builtin::BI__builtin_sadd_overflow:
5958 case Builtin::BI__builtin_saddl_overflow:
5959 case Builtin::BI__builtin_saddll_overflow:
5960 case Builtin::BI__builtin_ssub_overflow:
5961 case Builtin::BI__builtin_ssubl_overflow:
5962 case Builtin::BI__builtin_ssubll_overflow:
5963 case Builtin::BI__builtin_smul_overflow:
5964 case Builtin::BI__builtin_smull_overflow:
5965 case Builtin::BI__builtin_smulll_overflow: {
5966
5967 // We translate all of these builtins directly to the relevant llvm IR node.
5968
5969 // Scalarize our inputs.
5970 llvm::Value *X = EmitScalarExpr(E: E->getArg(Arg: 0));
5971 llvm::Value *Y = EmitScalarExpr(E: E->getArg(Arg: 1));
5972 Address SumOutPtr = EmitPointerWithAlignment(Addr: E->getArg(Arg: 2));
5973
5974 // Decide which of the overflow intrinsics we are lowering to:
5975 Intrinsic::ID IntrinsicId;
5976 switch (BuiltinID) {
5977 default: llvm_unreachable("Unknown overflow builtin id.");
5978 case Builtin::BI__builtin_uadd_overflow:
5979 case Builtin::BI__builtin_uaddl_overflow:
5980 case Builtin::BI__builtin_uaddll_overflow:
5981 IntrinsicId = Intrinsic::uadd_with_overflow;
5982 break;
5983 case Builtin::BI__builtin_usub_overflow:
5984 case Builtin::BI__builtin_usubl_overflow:
5985 case Builtin::BI__builtin_usubll_overflow:
5986 IntrinsicId = Intrinsic::usub_with_overflow;
5987 break;
5988 case Builtin::BI__builtin_umul_overflow:
5989 case Builtin::BI__builtin_umull_overflow:
5990 case Builtin::BI__builtin_umulll_overflow:
5991 IntrinsicId = Intrinsic::umul_with_overflow;
5992 break;
5993 case Builtin::BI__builtin_sadd_overflow:
5994 case Builtin::BI__builtin_saddl_overflow:
5995 case Builtin::BI__builtin_saddll_overflow:
5996 IntrinsicId = Intrinsic::sadd_with_overflow;
5997 break;
5998 case Builtin::BI__builtin_ssub_overflow:
5999 case Builtin::BI__builtin_ssubl_overflow:
6000 case Builtin::BI__builtin_ssubll_overflow:
6001 IntrinsicId = Intrinsic::ssub_with_overflow;
6002 break;
6003 case Builtin::BI__builtin_smul_overflow:
6004 case Builtin::BI__builtin_smull_overflow:
6005 case Builtin::BI__builtin_smulll_overflow:
6006 IntrinsicId = Intrinsic::smul_with_overflow;
6007 break;
6008 }
6009
6010
6011 llvm::Value *Carry;
6012 llvm::Value *Sum = EmitOverflowIntrinsic(CGF&: *this, IntrinsicID: IntrinsicId, X, Y, Carry);
6013 Builder.CreateStore(Val: Sum, Addr: SumOutPtr);
6014
6015 return RValue::get(V: Carry);
6016 }
6017 case Builtin::BIaddressof:
6018 case Builtin::BI__addressof:
6019 case Builtin::BI__builtin_addressof:
6020 return RValue::get(V: EmitLValue(E: E->getArg(Arg: 0)).getPointer(CGF&: *this));
6021 case Builtin::BI__builtin_function_start:
6022 return RValue::get(V: CGM.GetFunctionStart(
6023 Decl: E->getArg(Arg: 0)->getAsBuiltinConstantDeclRef(Context: CGM.getContext())));
6024 case Builtin::BI__builtin_operator_new:
6025 return EmitBuiltinNewDeleteCall(
6026 Type: E->getCallee()->getType()->castAs<FunctionProtoType>(), TheCallExpr: E, IsDelete: false);
6027 case Builtin::BI__builtin_operator_delete:
6028 EmitBuiltinNewDeleteCall(
6029 Type: E->getCallee()->getType()->castAs<FunctionProtoType>(), TheCallExpr: E, IsDelete: true);
6030 return RValue::get(V: nullptr);
6031
6032 case Builtin::BI__builtin_is_aligned:
6033 return EmitBuiltinIsAligned(E);
6034 case Builtin::BI__builtin_align_up:
6035 return EmitBuiltinAlignTo(E, AlignUp: true);
6036 case Builtin::BI__builtin_align_down:
6037 return EmitBuiltinAlignTo(E, AlignUp: false);
6038
6039 case Builtin::BI__noop:
6040 // __noop always evaluates to an integer literal zero.
6041 return RValue::get(V: ConstantInt::get(Ty: IntTy, V: 0));
6042 case Builtin::BI__builtin_call_with_static_chain: {
6043 const CallExpr *Call = cast<CallExpr>(Val: E->getArg(Arg: 0));
6044 const Expr *Chain = E->getArg(Arg: 1);
6045 return EmitCall(FnType: Call->getCallee()->getType(),
6046 Callee: EmitCallee(E: Call->getCallee()), E: Call, ReturnValue,
6047 Chain: EmitScalarExpr(E: Chain));
6048 }
6049 case Builtin::BI_InterlockedExchange8:
6050 case Builtin::BI_InterlockedExchange16:
6051 case Builtin::BI_InterlockedExchange:
6052 case Builtin::BI_InterlockedExchangePointer:
6053 return RValue::get(
6054 V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::_InterlockedExchange, E));
6055 case Builtin::BI_InterlockedCompareExchangePointer:
6056 return RValue::get(
6057 V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::_InterlockedCompareExchange, E));
6058 case Builtin::BI_InterlockedCompareExchangePointer_nf:
6059 return RValue::get(
6060 V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::_InterlockedCompareExchange_nf, E));
6061 case Builtin::BI_InterlockedCompareExchange8:
6062 case Builtin::BI_InterlockedCompareExchange16:
6063 case Builtin::BI_InterlockedCompareExchange:
6064 case Builtin::BI_InterlockedCompareExchange64:
6065 return RValue::get(V: EmitAtomicCmpXchgForMSIntrin(CGF&: *this, E));
6066 case Builtin::BI_InterlockedIncrement16:
6067 case Builtin::BI_InterlockedIncrement:
6068 return RValue::get(
6069 V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::_InterlockedIncrement, E));
6070 case Builtin::BI_InterlockedDecrement16:
6071 case Builtin::BI_InterlockedDecrement:
6072 return RValue::get(
6073 V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::_InterlockedDecrement, E));
6074 case Builtin::BI_InterlockedAnd8:
6075 case Builtin::BI_InterlockedAnd16:
6076 case Builtin::BI_InterlockedAnd:
6077 return RValue::get(V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::_InterlockedAnd, E));
6078 case Builtin::BI_InterlockedExchangeAdd8:
6079 case Builtin::BI_InterlockedExchangeAdd16:
6080 case Builtin::BI_InterlockedExchangeAdd:
6081 return RValue::get(
6082 V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::_InterlockedExchangeAdd, E));
6083 case Builtin::BI_InterlockedExchangeSub8:
6084 case Builtin::BI_InterlockedExchangeSub16:
6085 case Builtin::BI_InterlockedExchangeSub:
6086 return RValue::get(
6087 V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::_InterlockedExchangeSub, E));
6088 case Builtin::BI_InterlockedOr8:
6089 case Builtin::BI_InterlockedOr16:
6090 case Builtin::BI_InterlockedOr:
6091 return RValue::get(V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::_InterlockedOr, E));
6092 case Builtin::BI_InterlockedXor8:
6093 case Builtin::BI_InterlockedXor16:
6094 case Builtin::BI_InterlockedXor:
6095 return RValue::get(V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::_InterlockedXor, E));
6096
6097 case Builtin::BI_bittest64:
6098 case Builtin::BI_bittest:
6099 case Builtin::BI_bittestandcomplement64:
6100 case Builtin::BI_bittestandcomplement:
6101 case Builtin::BI_bittestandreset64:
6102 case Builtin::BI_bittestandreset:
6103 case Builtin::BI_bittestandset64:
6104 case Builtin::BI_bittestandset:
6105 case Builtin::BI_interlockedbittestandreset:
6106 case Builtin::BI_interlockedbittestandreset64:
6107 case Builtin::BI_interlockedbittestandreset64_acq:
6108 case Builtin::BI_interlockedbittestandreset64_rel:
6109 case Builtin::BI_interlockedbittestandreset64_nf:
6110 case Builtin::BI_interlockedbittestandset64:
6111 case Builtin::BI_interlockedbittestandset64_acq:
6112 case Builtin::BI_interlockedbittestandset64_rel:
6113 case Builtin::BI_interlockedbittestandset64_nf:
6114 case Builtin::BI_interlockedbittestandset:
6115 case Builtin::BI_interlockedbittestandset_acq:
6116 case Builtin::BI_interlockedbittestandset_rel:
6117 case Builtin::BI_interlockedbittestandset_nf:
6118 case Builtin::BI_interlockedbittestandreset_acq:
6119 case Builtin::BI_interlockedbittestandreset_rel:
6120 case Builtin::BI_interlockedbittestandreset_nf:
6121 return RValue::get(V: EmitBitTestIntrinsic(CGF&: *this, BuiltinID, E));
6122
6123 // These builtins exist to emit regular volatile loads and stores not
6124 // affected by the -fms-volatile setting.
6125 case Builtin::BI__iso_volatile_load8:
6126 case Builtin::BI__iso_volatile_load16:
6127 case Builtin::BI__iso_volatile_load32:
6128 case Builtin::BI__iso_volatile_load64:
6129 return RValue::get(V: EmitISOVolatileLoad(CGF&: *this, E));
6130 case Builtin::BI__iso_volatile_store8:
6131 case Builtin::BI__iso_volatile_store16:
6132 case Builtin::BI__iso_volatile_store32:
6133 case Builtin::BI__iso_volatile_store64:
6134 return RValue::get(V: EmitISOVolatileStore(CGF&: *this, E));
6135
6136 case Builtin::BI__builtin_ptrauth_sign_constant:
6137 return RValue::get(V: ConstantEmitter(*this).emitAbstract(E, T: E->getType()));
6138
6139 case Builtin::BI__builtin_ptrauth_auth:
6140 case Builtin::BI__builtin_ptrauth_auth_and_resign:
6141 case Builtin::BI__builtin_ptrauth_auth_with_pc_and_resign:
6142 case Builtin::BI__builtin_ptrauth_auth_load_relative_and_sign:
6143 case Builtin::BI__builtin_ptrauth_blend_discriminator:
6144 case Builtin::BI__builtin_ptrauth_sign_generic_data:
6145 case Builtin::BI__builtin_ptrauth_sign_unauthenticated:
6146 case Builtin::BI__builtin_ptrauth_strip: {
6147 // Emit the arguments.
6148 SmallVector<llvm::Value *, 6> Args;
6149 for (auto argExpr : E->arguments())
6150 Args.push_back(Elt: EmitScalarExpr(E: argExpr));
6151
6152 // Cast the value to intptr_t, saving its original type.
6153 llvm::Type *OrigValueType = Args[0]->getType();
6154 if (OrigValueType->isPointerTy())
6155 Args[0] = Builder.CreatePtrToInt(V: Args[0], DestTy: IntPtrTy);
6156
6157 switch (BuiltinID) {
6158 case Builtin::BI__builtin_ptrauth_auth_with_pc_and_resign:
6159 // Convert oldDiscriminator (arg 2), oldPC (arg 3) and newDiscriminator
6160 // (arg 5) to intptr_t
6161 if (Args[2]->getType()->isPointerTy())
6162 Args[2] = Builder.CreatePtrToInt(V: Args[2], DestTy: IntPtrTy);
6163 if (Args[3]->getType()->isPointerTy())
6164 Args[3] = Builder.CreatePtrToInt(V: Args[3], DestTy: IntPtrTy);
6165 if (Args[5]->getType()->isPointerTy())
6166 Args[5] = Builder.CreatePtrToInt(V: Args[5], DestTy: IntPtrTy);
6167 break;
6168
6169 case Builtin::BI__builtin_ptrauth_auth_and_resign:
6170 case Builtin::BI__builtin_ptrauth_auth_load_relative_and_sign:
6171 if (Args[4]->getType()->isPointerTy())
6172 Args[4] = Builder.CreatePtrToInt(V: Args[4], DestTy: IntPtrTy);
6173 [[fallthrough]];
6174
6175 case Builtin::BI__builtin_ptrauth_auth:
6176 case Builtin::BI__builtin_ptrauth_sign_unauthenticated:
6177 if (Args[2]->getType()->isPointerTy())
6178 Args[2] = Builder.CreatePtrToInt(V: Args[2], DestTy: IntPtrTy);
6179 break;
6180
6181 case Builtin::BI__builtin_ptrauth_sign_generic_data:
6182 if (Args[1]->getType()->isPointerTy())
6183 Args[1] = Builder.CreatePtrToInt(V: Args[1], DestTy: IntPtrTy);
6184 break;
6185
6186 case Builtin::BI__builtin_ptrauth_blend_discriminator:
6187 case Builtin::BI__builtin_ptrauth_strip:
6188 break;
6189 }
6190
6191 // Call the intrinsic.
6192 auto IntrinsicID = [&]() -> unsigned {
6193 switch (BuiltinID) {
6194 case Builtin::BI__builtin_ptrauth_auth:
6195 return Intrinsic::ptrauth_auth;
6196 case Builtin::BI__builtin_ptrauth_auth_and_resign:
6197 return Intrinsic::ptrauth_resign;
6198 case Builtin::BI__builtin_ptrauth_auth_with_pc_and_resign:
6199 return Intrinsic::ptrauth_auth_with_pc_and_resign;
6200 case Builtin::BI__builtin_ptrauth_auth_load_relative_and_sign:
6201 return Intrinsic::ptrauth_resign_load_relative;
6202 case Builtin::BI__builtin_ptrauth_blend_discriminator:
6203 return Intrinsic::ptrauth_blend;
6204 case Builtin::BI__builtin_ptrauth_sign_generic_data:
6205 return Intrinsic::ptrauth_sign_generic;
6206 case Builtin::BI__builtin_ptrauth_sign_unauthenticated:
6207 return Intrinsic::ptrauth_sign;
6208 case Builtin::BI__builtin_ptrauth_strip:
6209 return Intrinsic::ptrauth_strip;
6210 }
6211 llvm_unreachable("bad ptrauth intrinsic");
6212 }();
6213 auto Intrinsic = CGM.getIntrinsic(IID: IntrinsicID);
6214 llvm::Value *Result = EmitRuntimeCall(callee: Intrinsic, args: Args);
6215
6216 if (BuiltinID != Builtin::BI__builtin_ptrauth_sign_generic_data &&
6217 BuiltinID != Builtin::BI__builtin_ptrauth_blend_discriminator &&
6218 OrigValueType->isPointerTy()) {
6219 Result = Builder.CreateIntToPtr(V: Result, DestTy: OrigValueType);
6220 }
6221 return RValue::get(V: Result);
6222 }
6223
6224 case Builtin::BI__builtin_get_vtable_pointer: {
6225 const Expr *Target = E->getArg(Arg: 0);
6226 QualType TargetType = Target->getType();
6227 const CXXRecordDecl *Decl = TargetType->getPointeeCXXRecordDecl();
6228 assert(Decl);
6229 auto ThisAddress = EmitPointerWithAlignment(Addr: Target);
6230 assert(ThisAddress.isValid());
6231 llvm::Value *VTablePointer =
6232 GetVTablePtr(This: ThisAddress, VTableTy: Int8PtrTy, VTableClass: Decl, AuthMode: VTableAuthMode::MustTrap);
6233 return RValue::get(V: VTablePointer);
6234 }
6235
6236 case Builtin::BI__exception_code:
6237 case Builtin::BI_exception_code:
6238 return RValue::get(V: EmitSEHExceptionCode());
6239 case Builtin::BI__exception_info:
6240 case Builtin::BI_exception_info:
6241 return RValue::get(V: EmitSEHExceptionInfo());
6242 case Builtin::BI__abnormal_termination:
6243 case Builtin::BI_abnormal_termination:
6244 return RValue::get(V: EmitSEHAbnormalTermination());
6245 case Builtin::BI_setjmpex:
6246 if (getTarget().getTriple().isOSMSVCRT() && E->getNumArgs() == 1 &&
6247 E->getArg(Arg: 0)->getType()->isPointerType())
6248 return EmitMSVCRTSetJmp(CGF&: *this, SJKind: MSVCSetJmpKind::_setjmpex, E);
6249 break;
6250 case Builtin::BI_setjmp:
6251 if (getTarget().getTriple().isOSMSVCRT() && E->getNumArgs() == 1 &&
6252 E->getArg(Arg: 0)->getType()->isPointerType()) {
6253 if (getTarget().getTriple().getArch() == llvm::Triple::x86)
6254 return EmitMSVCRTSetJmp(CGF&: *this, SJKind: MSVCSetJmpKind::_setjmp3, E);
6255 else if (getTarget().getTriple().getArch() == llvm::Triple::aarch64)
6256 return EmitMSVCRTSetJmp(CGF&: *this, SJKind: MSVCSetJmpKind::_setjmpex, E);
6257 return EmitMSVCRTSetJmp(CGF&: *this, SJKind: MSVCSetJmpKind::_setjmp, E);
6258 }
6259 break;
6260
6261 // C++ std:: builtins.
6262 case Builtin::BImove:
6263 case Builtin::BImove_if_noexcept:
6264 case Builtin::BIforward:
6265 case Builtin::BIforward_like:
6266 case Builtin::BIas_const:
6267 return RValue::get(V: EmitLValue(E: E->getArg(Arg: 0)).getPointer(CGF&: *this));
6268 case Builtin::BI__GetExceptionInfo: {
6269 if (llvm::GlobalVariable *GV =
6270 CGM.getCXXABI().getThrowInfo(T: FD->getParamDecl(i: 0)->getType()))
6271 return RValue::get(V: GV);
6272 break;
6273 }
6274
6275 case Builtin::BI__fastfail:
6276 return RValue::get(V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::__fastfail, E));
6277
6278 case Builtin::BI__builtin_coro_id:
6279 return EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_id);
6280 case Builtin::BI__builtin_coro_promise:
6281 return EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_promise);
6282 case Builtin::BI__builtin_coro_resume:
6283 EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_resume);
6284 return RValue::get(V: nullptr);
6285 case Builtin::BI__builtin_coro_frame:
6286 return EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_frame);
6287 case Builtin::BI__builtin_coro_noop:
6288 return EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_noop);
6289 case Builtin::BI__builtin_coro_free:
6290 return EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_free);
6291 case Builtin::BI__builtin_coro_destroy:
6292 EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_destroy);
6293 return RValue::get(V: nullptr);
6294 case Builtin::BI__builtin_coro_done:
6295 return EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_done);
6296 case Builtin::BI__builtin_coro_alloc:
6297 return EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_alloc);
6298 case Builtin::BI__builtin_coro_begin:
6299 return EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_begin);
6300 case Builtin::BI__builtin_coro_end:
6301 return EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_end);
6302 case Builtin::BI__builtin_coro_suspend:
6303 return EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_suspend);
6304 case Builtin::BI__builtin_coro_size:
6305 return EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_size);
6306 case Builtin::BI__builtin_coro_align:
6307 return EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_align);
6308
6309 // OpenCL v2.0 s6.13.16.2, Built-in pipe read and write functions
6310 case Builtin::BIread_pipe:
6311 case Builtin::BIwrite_pipe: {
6312 Value *Arg0 = EmitScalarExpr(E: E->getArg(Arg: 0)),
6313 *Arg1 = EmitScalarExpr(E: E->getArg(Arg: 1));
6314 CGOpenCLRuntime OpenCLRT(CGM);
6315 Value *PacketSize = OpenCLRT.getPipeElemSize(PipeArg: E->getArg(Arg: 0));
6316 Value *PacketAlign = OpenCLRT.getPipeElemAlign(PipeArg: E->getArg(Arg: 0));
6317
6318 // Type of the generic packet parameter.
6319 unsigned GenericAS =
6320 getContext().getTargetAddressSpace(AS: LangAS::opencl_generic);
6321 llvm::Type *I8PTy = llvm::PointerType::get(C&: getLLVMContext(), AddressSpace: GenericAS);
6322
6323 // Testing which overloaded version we should generate the call for.
6324 if (2U == E->getNumArgs()) {
6325 const char *Name = (BuiltinID == Builtin::BIread_pipe) ? "__read_pipe_2"
6326 : "__write_pipe_2";
6327 // Creating a generic function type to be able to call with any builtin or
6328 // user defined type.
6329 llvm::Type *ArgTys[] = {Arg0->getType(), I8PTy, Int32Ty, Int32Ty};
6330 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: Int32Ty, Params: ArgTys, isVarArg: false);
6331 Value *ACast = Builder.CreateAddrSpaceCast(V: Arg1, DestTy: I8PTy);
6332 return RValue::get(
6333 V: EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(Ty: FTy, Name),
6334 args: {Arg0, ACast, PacketSize, PacketAlign}));
6335 } else {
6336 assert(4 == E->getNumArgs() &&
6337 "Illegal number of parameters to pipe function");
6338 const char *Name = (BuiltinID == Builtin::BIread_pipe) ? "__read_pipe_4"
6339 : "__write_pipe_4";
6340
6341 llvm::Type *ArgTys[] = {Arg0->getType(), Arg1->getType(), Int32Ty, I8PTy,
6342 Int32Ty, Int32Ty};
6343 Value *Arg2 = EmitScalarExpr(E: E->getArg(Arg: 2)),
6344 *Arg3 = EmitScalarExpr(E: E->getArg(Arg: 3));
6345 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: Int32Ty, Params: ArgTys, isVarArg: false);
6346 Value *ACast = Builder.CreateAddrSpaceCast(V: Arg3, DestTy: I8PTy);
6347 // We know the third argument is an integer type, but we may need to cast
6348 // it to i32.
6349 if (Arg2->getType() != Int32Ty)
6350 Arg2 = Builder.CreateZExtOrTrunc(V: Arg2, DestTy: Int32Ty);
6351 return RValue::get(
6352 V: EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(Ty: FTy, Name),
6353 args: {Arg0, Arg1, Arg2, ACast, PacketSize, PacketAlign}));
6354 }
6355 }
6356 // OpenCL v2.0 s6.13.16 ,s9.17.3.5 - Built-in pipe reserve read and write
6357 // functions
6358 case Builtin::BIreserve_read_pipe:
6359 case Builtin::BIreserve_write_pipe:
6360 case Builtin::BIwork_group_reserve_read_pipe:
6361 case Builtin::BIwork_group_reserve_write_pipe:
6362 case Builtin::BIsub_group_reserve_read_pipe:
6363 case Builtin::BIsub_group_reserve_write_pipe: {
6364 // Composing the mangled name for the function.
6365 const char *Name;
6366 if (BuiltinID == Builtin::BIreserve_read_pipe)
6367 Name = "__reserve_read_pipe";
6368 else if (BuiltinID == Builtin::BIreserve_write_pipe)
6369 Name = "__reserve_write_pipe";
6370 else if (BuiltinID == Builtin::BIwork_group_reserve_read_pipe)
6371 Name = "__work_group_reserve_read_pipe";
6372 else if (BuiltinID == Builtin::BIwork_group_reserve_write_pipe)
6373 Name = "__work_group_reserve_write_pipe";
6374 else if (BuiltinID == Builtin::BIsub_group_reserve_read_pipe)
6375 Name = "__sub_group_reserve_read_pipe";
6376 else
6377 Name = "__sub_group_reserve_write_pipe";
6378
6379 Value *Arg0 = EmitScalarExpr(E: E->getArg(Arg: 0)),
6380 *Arg1 = EmitScalarExpr(E: E->getArg(Arg: 1));
6381 llvm::Type *ReservedIDTy = ConvertType(T: getContext().OCLReserveIDTy);
6382 CGOpenCLRuntime OpenCLRT(CGM);
6383 Value *PacketSize = OpenCLRT.getPipeElemSize(PipeArg: E->getArg(Arg: 0));
6384 Value *PacketAlign = OpenCLRT.getPipeElemAlign(PipeArg: E->getArg(Arg: 0));
6385
6386 // Building the generic function prototype.
6387 llvm::Type *ArgTys[] = {Arg0->getType(), Int32Ty, Int32Ty, Int32Ty};
6388 llvm::FunctionType *FTy =
6389 llvm::FunctionType::get(Result: ReservedIDTy, Params: ArgTys, isVarArg: false);
6390 // We know the second argument is an integer type, but we may need to cast
6391 // it to i32.
6392 if (Arg1->getType() != Int32Ty)
6393 Arg1 = Builder.CreateZExtOrTrunc(V: Arg1, DestTy: Int32Ty);
6394 return RValue::get(V: EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(Ty: FTy, Name),
6395 args: {Arg0, Arg1, PacketSize, PacketAlign}));
6396 }
6397 // OpenCL v2.0 s6.13.16, s9.17.3.5 - Built-in pipe commit read and write
6398 // functions
6399 case Builtin::BIcommit_read_pipe:
6400 case Builtin::BIcommit_write_pipe:
6401 case Builtin::BIwork_group_commit_read_pipe:
6402 case Builtin::BIwork_group_commit_write_pipe:
6403 case Builtin::BIsub_group_commit_read_pipe:
6404 case Builtin::BIsub_group_commit_write_pipe: {
6405 const char *Name;
6406 if (BuiltinID == Builtin::BIcommit_read_pipe)
6407 Name = "__commit_read_pipe";
6408 else if (BuiltinID == Builtin::BIcommit_write_pipe)
6409 Name = "__commit_write_pipe";
6410 else if (BuiltinID == Builtin::BIwork_group_commit_read_pipe)
6411 Name = "__work_group_commit_read_pipe";
6412 else if (BuiltinID == Builtin::BIwork_group_commit_write_pipe)
6413 Name = "__work_group_commit_write_pipe";
6414 else if (BuiltinID == Builtin::BIsub_group_commit_read_pipe)
6415 Name = "__sub_group_commit_read_pipe";
6416 else
6417 Name = "__sub_group_commit_write_pipe";
6418
6419 Value *Arg0 = EmitScalarExpr(E: E->getArg(Arg: 0)),
6420 *Arg1 = EmitScalarExpr(E: E->getArg(Arg: 1));
6421 CGOpenCLRuntime OpenCLRT(CGM);
6422 Value *PacketSize = OpenCLRT.getPipeElemSize(PipeArg: E->getArg(Arg: 0));
6423 Value *PacketAlign = OpenCLRT.getPipeElemAlign(PipeArg: E->getArg(Arg: 0));
6424
6425 // Building the generic function prototype.
6426 llvm::Type *ArgTys[] = {Arg0->getType(), Arg1->getType(), Int32Ty, Int32Ty};
6427 llvm::FunctionType *FTy = llvm::FunctionType::get(
6428 Result: llvm::Type::getVoidTy(C&: getLLVMContext()), Params: ArgTys, isVarArg: false);
6429
6430 return RValue::get(V: EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(Ty: FTy, Name),
6431 args: {Arg0, Arg1, PacketSize, PacketAlign}));
6432 }
6433 // OpenCL v2.0 s6.13.16.4 Built-in pipe query functions
6434 case Builtin::BIget_pipe_num_packets:
6435 case Builtin::BIget_pipe_max_packets: {
6436 const char *BaseName;
6437 const auto *PipeTy = E->getArg(Arg: 0)->getType()->castAs<PipeType>();
6438 if (BuiltinID == Builtin::BIget_pipe_num_packets)
6439 BaseName = "__get_pipe_num_packets";
6440 else
6441 BaseName = "__get_pipe_max_packets";
6442 std::string Name = std::string(BaseName) +
6443 std::string(PipeTy->isReadOnly() ? "_ro" : "_wo");
6444
6445 // Building the generic function prototype.
6446 Value *Arg0 = EmitScalarExpr(E: E->getArg(Arg: 0));
6447 CGOpenCLRuntime OpenCLRT(CGM);
6448 Value *PacketSize = OpenCLRT.getPipeElemSize(PipeArg: E->getArg(Arg: 0));
6449 Value *PacketAlign = OpenCLRT.getPipeElemAlign(PipeArg: E->getArg(Arg: 0));
6450 llvm::Type *ArgTys[] = {Arg0->getType(), Int32Ty, Int32Ty};
6451 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: Int32Ty, Params: ArgTys, isVarArg: false);
6452
6453 return RValue::get(V: EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(Ty: FTy, Name),
6454 args: {Arg0, PacketSize, PacketAlign}));
6455 }
6456
6457 // OpenCL v2.0 s6.13.9 - Address space qualifier functions.
6458 case Builtin::BIto_global:
6459 case Builtin::BIto_local:
6460 case Builtin::BIto_private: {
6461 auto Arg0 = EmitScalarExpr(E: E->getArg(Arg: 0));
6462 auto NewArgT = llvm::PointerType::get(
6463 C&: getLLVMContext(),
6464 AddressSpace: CGM.getContext().getTargetAddressSpace(AS: LangAS::opencl_generic));
6465 auto NewRetT = llvm::PointerType::get(
6466 C&: getLLVMContext(),
6467 AddressSpace: CGM.getContext().getTargetAddressSpace(
6468 AS: E->getType()->getPointeeType().getAddressSpace()));
6469 auto FTy = llvm::FunctionType::get(Result: NewRetT, Params: {NewArgT}, isVarArg: false);
6470 llvm::Value *NewArg;
6471 if (Arg0->getType()->getPointerAddressSpace() !=
6472 NewArgT->getPointerAddressSpace())
6473 NewArg = Builder.CreateAddrSpaceCast(V: Arg0, DestTy: NewArgT);
6474 else
6475 NewArg = Builder.CreateBitOrPointerCast(V: Arg0, DestTy: NewArgT);
6476 auto NewName = std::string("__") + E->getDirectCallee()->getName().str();
6477 auto NewCall =
6478 EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(Ty: FTy, Name: NewName), args: {NewArg});
6479 return RValue::get(V: Builder.CreateBitOrPointerCast(V: NewCall,
6480 DestTy: ConvertType(T: E->getType())));
6481 }
6482
6483 // OpenCL v2.0, s6.13.17 - Enqueue kernel function.
6484 // Table 6.13.17.1 specifies four overload forms of enqueue_kernel.
6485 // The code below expands the builtin call to a call to one of the following
6486 // functions that an OpenCL runtime library will have to provide:
6487 // __enqueue_kernel_basic
6488 // __enqueue_kernel_varargs
6489 // __enqueue_kernel_basic_events
6490 // __enqueue_kernel_events_varargs
6491 case Builtin::BIenqueue_kernel: {
6492 StringRef Name; // Generated function call name
6493 unsigned NumArgs = E->getNumArgs();
6494
6495 llvm::Type *QueueTy = ConvertType(T: getContext().OCLQueueTy);
6496 llvm::Type *GenericVoidPtrTy = Builder.getPtrTy(
6497 AddrSpace: getContext().getTargetAddressSpace(AS: LangAS::opencl_generic));
6498
6499 llvm::Value *Queue = EmitScalarExpr(E: E->getArg(Arg: 0));
6500 llvm::Value *Flags = EmitScalarExpr(E: E->getArg(Arg: 1));
6501 LValue NDRangeL = EmitAggExprToLValue(E: E->getArg(Arg: 2));
6502 llvm::Value *Range = NDRangeL.getAddress().emitRawPointer(CGF&: *this);
6503
6504 // FIXME: Look through the addrspacecast which may exist to the stack
6505 // temporary as a hack.
6506 //
6507 // This is hardcoding the assumed ABI of the target function. This assumes
6508 // direct passing for every argument except NDRange, which is assumed to be
6509 // byval or byref indirect passed.
6510 //
6511 // This should be fixed to query a signature from CGOpenCLRuntime, and go
6512 // through EmitCallArgs to get the correct target ABI.
6513 Range = Range->stripPointerCasts();
6514
6515 llvm::Type *RangePtrTy = Range->getType();
6516
6517 if (NumArgs == 4) {
6518 // The most basic form of the call with parameters:
6519 // queue_t, kernel_enqueue_flags_t, ndrange_t, block(void)
6520 Name = "__enqueue_kernel_basic";
6521 llvm::Type *ArgTys[] = {QueueTy, Int32Ty, RangePtrTy, GenericVoidPtrTy,
6522 GenericVoidPtrTy};
6523 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: Int32Ty, Params: ArgTys, isVarArg: false);
6524
6525 auto Info =
6526 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(CGF&: *this, E: E->getArg(Arg: 3));
6527 llvm::Value *Kernel =
6528 Builder.CreatePointerCast(V: Info.KernelHandle, DestTy: GenericVoidPtrTy);
6529 llvm::Value *Block =
6530 Builder.CreatePointerCast(V: Info.BlockArg, DestTy: GenericVoidPtrTy);
6531
6532 auto RTCall = EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(Ty: FTy, Name),
6533 args: {Queue, Flags, Range, Kernel, Block});
6534 return RValue::get(V: RTCall);
6535 }
6536 assert(NumArgs >= 5 && "Invalid enqueue_kernel signature");
6537
6538 // Create a temporary array to hold the sizes of local pointer arguments
6539 // for the block. \p First is the position of the first size argument.
6540 auto CreateArrayForSizeVar =
6541 [=](unsigned First) -> std::pair<llvm::Value *, llvm::Value *> {
6542 llvm::APInt ArraySize(32, NumArgs - First);
6543 QualType SizeArrayTy = getContext().getConstantArrayType(
6544 EltTy: getContext().getSizeType(), ArySize: ArraySize, SizeExpr: nullptr,
6545 ASM: ArraySizeModifier::Normal,
6546 /*IndexTypeQuals=*/0);
6547 auto Tmp = CreateMemTempWithoutCast(T: SizeArrayTy, Name: "block_sizes");
6548 llvm::Value *Alloca = Tmp.getPointer();
6549 llvm::Value *ElemPtr;
6550 EmitLifetimeStart(Addr: Alloca);
6551 // Each of the following arguments specifies the size of the corresponding
6552 // argument passed to the enqueued block.
6553 auto *Zero = llvm::ConstantInt::get(Ty: IntTy, V: 0);
6554 for (unsigned I = First; I < NumArgs; ++I) {
6555 auto *Index = llvm::ConstantInt::get(Ty: IntTy, V: I - First);
6556 auto *GEP =
6557 Builder.CreateGEP(Ty: Tmp.getElementType(), Ptr: Alloca, IdxList: {Zero, Index});
6558 if (I == First)
6559 ElemPtr = GEP;
6560 auto *V =
6561 Builder.CreateZExtOrTrunc(V: EmitScalarExpr(E: E->getArg(Arg: I)), DestTy: SizeTy);
6562 Builder.CreateAlignedStore(
6563 Val: V, Ptr: GEP, Align: CGM.getDataLayout().getPrefTypeAlign(Ty: SizeTy));
6564 }
6565 return {ElemPtr, Alloca};
6566 };
6567
6568 // Could have events and/or varargs.
6569 if (E->getArg(Arg: 3)->getType()->isBlockPointerType()) {
6570 // No events passed, but has variadic arguments.
6571 Name = "__enqueue_kernel_varargs";
6572 auto Info =
6573 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(CGF&: *this, E: E->getArg(Arg: 3));
6574 llvm::Value *Kernel =
6575 Builder.CreatePointerCast(V: Info.KernelHandle, DestTy: GenericVoidPtrTy);
6576 auto *Block = Builder.CreatePointerCast(V: Info.BlockArg, DestTy: GenericVoidPtrTy);
6577 auto [ElemPtr, TmpPtr] = CreateArrayForSizeVar(4);
6578
6579 // Create a vector of the arguments, as well as a constant value to
6580 // express to the runtime the number of variadic arguments.
6581 llvm::Value *const Args[] = {Queue, Flags,
6582 Range, Kernel,
6583 Block, ConstantInt::get(Ty: IntTy, V: NumArgs - 4),
6584 ElemPtr};
6585 llvm::Type *const ArgTys[] = {
6586 QueueTy, IntTy, RangePtrTy, GenericVoidPtrTy,
6587 GenericVoidPtrTy, IntTy, ElemPtr->getType()};
6588
6589 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: Int32Ty, Params: ArgTys, isVarArg: false);
6590 auto Call = RValue::get(
6591 V: EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(Ty: FTy, Name), args: Args));
6592 EmitLifetimeEnd(Addr: TmpPtr);
6593 return Call;
6594 }
6595 // Any calls now have event arguments passed.
6596 if (NumArgs >= 7) {
6597 llvm::PointerType *PtrTy = llvm::PointerType::get(
6598 C&: CGM.getLLVMContext(),
6599 AddressSpace: CGM.getContext().getTargetAddressSpace(AS: LangAS::opencl_generic));
6600
6601 llvm::Value *NumEvents =
6602 Builder.CreateZExtOrTrunc(V: EmitScalarExpr(E: E->getArg(Arg: 3)), DestTy: Int32Ty);
6603
6604 // Since SemaOpenCLBuiltinEnqueueKernel allows fifth and sixth arguments
6605 // to be a null pointer constant (including `0` literal), we can take it
6606 // into account and emit null pointer directly.
6607 llvm::Value *EventWaitList = nullptr;
6608 if (E->getArg(Arg: 4)->isNullPointerConstant(
6609 Ctx&: getContext(), NPC: Expr::NPC_ValueDependentIsNotNull)) {
6610 EventWaitList = llvm::ConstantPointerNull::get(T: PtrTy);
6611 } else {
6612 EventWaitList =
6613 E->getArg(Arg: 4)->getType()->isArrayType()
6614 ? EmitArrayToPointerDecay(Array: E->getArg(Arg: 4)).emitRawPointer(CGF&: *this)
6615 : EmitScalarExpr(E: E->getArg(Arg: 4));
6616 // Convert to generic address space.
6617 EventWaitList = Builder.CreatePointerCast(V: EventWaitList, DestTy: PtrTy);
6618 }
6619 llvm::Value *EventRet = nullptr;
6620 if (E->getArg(Arg: 5)->isNullPointerConstant(
6621 Ctx&: getContext(), NPC: Expr::NPC_ValueDependentIsNotNull)) {
6622 EventRet = llvm::ConstantPointerNull::get(T: PtrTy);
6623 } else {
6624 EventRet =
6625 Builder.CreatePointerCast(V: EmitScalarExpr(E: E->getArg(Arg: 5)), DestTy: PtrTy);
6626 }
6627
6628 auto Info =
6629 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(CGF&: *this, E: E->getArg(Arg: 6));
6630 llvm::Value *Kernel =
6631 Builder.CreatePointerCast(V: Info.KernelHandle, DestTy: GenericVoidPtrTy);
6632 llvm::Value *Block =
6633 Builder.CreatePointerCast(V: Info.BlockArg, DestTy: GenericVoidPtrTy);
6634
6635 std::vector<llvm::Type *> ArgTys = {
6636 QueueTy, Int32Ty, RangePtrTy, Int32Ty,
6637 PtrTy, PtrTy, GenericVoidPtrTy, GenericVoidPtrTy};
6638
6639 std::vector<llvm::Value *> Args = {Queue, Flags, Range,
6640 NumEvents, EventWaitList, EventRet,
6641 Kernel, Block};
6642
6643 if (NumArgs == 7) {
6644 // Has events but no variadics.
6645 Name = "__enqueue_kernel_basic_events";
6646 llvm::FunctionType *FTy =
6647 llvm::FunctionType::get(Result: Int32Ty, Params: ArgTys, isVarArg: false);
6648 return RValue::get(
6649 V: EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(Ty: FTy, Name), args: Args));
6650 }
6651 // Has event info and variadics
6652 // Pass the number of variadics to the runtime function too.
6653 Args.push_back(x: ConstantInt::get(Ty: Int32Ty, V: NumArgs - 7));
6654 ArgTys.push_back(x: Int32Ty);
6655 Name = "__enqueue_kernel_events_varargs";
6656
6657 auto [ElemPtr, TmpPtr] = CreateArrayForSizeVar(7);
6658 Args.push_back(x: ElemPtr);
6659 ArgTys.push_back(x: ElemPtr->getType());
6660
6661 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: Int32Ty, Params: ArgTys, isVarArg: false);
6662 auto Call = RValue::get(
6663 V: EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(Ty: FTy, Name), args: Args));
6664 EmitLifetimeEnd(Addr: TmpPtr);
6665 return Call;
6666 }
6667 llvm_unreachable("Unexpected enqueue_kernel signature");
6668 }
6669 // OpenCL v2.0 s6.13.17.6 - Kernel query functions need bitcast of block
6670 // parameter.
6671 case Builtin::BIget_kernel_work_group_size: {
6672 llvm::Type *GenericVoidPtrTy = Builder.getPtrTy(
6673 AddrSpace: getContext().getTargetAddressSpace(AS: LangAS::opencl_generic));
6674 auto Info =
6675 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(CGF&: *this, E: E->getArg(Arg: 0));
6676 Value *Kernel =
6677 Builder.CreatePointerCast(V: Info.KernelHandle, DestTy: GenericVoidPtrTy);
6678 Value *Arg = Builder.CreatePointerCast(V: Info.BlockArg, DestTy: GenericVoidPtrTy);
6679 return RValue::get(V: EmitRuntimeCall(
6680 callee: CGM.CreateRuntimeFunction(
6681 Ty: llvm::FunctionType::get(Result: IntTy, Params: {GenericVoidPtrTy, GenericVoidPtrTy},
6682 isVarArg: false),
6683 Name: "__get_kernel_work_group_size_impl"),
6684 args: {Kernel, Arg}));
6685 }
6686 case Builtin::BIget_kernel_preferred_work_group_size_multiple: {
6687 llvm::Type *GenericVoidPtrTy = Builder.getPtrTy(
6688 AddrSpace: getContext().getTargetAddressSpace(AS: LangAS::opencl_generic));
6689 auto Info =
6690 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(CGF&: *this, E: E->getArg(Arg: 0));
6691 Value *Kernel =
6692 Builder.CreatePointerCast(V: Info.KernelHandle, DestTy: GenericVoidPtrTy);
6693 Value *Arg = Builder.CreatePointerCast(V: Info.BlockArg, DestTy: GenericVoidPtrTy);
6694 return RValue::get(V: EmitRuntimeCall(
6695 callee: CGM.CreateRuntimeFunction(
6696 Ty: llvm::FunctionType::get(Result: IntTy, Params: {GenericVoidPtrTy, GenericVoidPtrTy},
6697 isVarArg: false),
6698 Name: "__get_kernel_preferred_work_group_size_multiple_impl"),
6699 args: {Kernel, Arg}));
6700 }
6701 case Builtin::BIget_kernel_max_sub_group_size_for_ndrange:
6702 case Builtin::BIget_kernel_sub_group_count_for_ndrange: {
6703 llvm::Type *GenericVoidPtrTy = Builder.getPtrTy(
6704 AddrSpace: getContext().getTargetAddressSpace(AS: LangAS::opencl_generic));
6705 LValue NDRangeL = EmitAggExprToLValue(E: E->getArg(Arg: 0));
6706 llvm::Value *NDRange = NDRangeL.getAddress().emitRawPointer(CGF&: *this);
6707 auto Info =
6708 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(CGF&: *this, E: E->getArg(Arg: 1));
6709 Value *Kernel =
6710 Builder.CreatePointerCast(V: Info.KernelHandle, DestTy: GenericVoidPtrTy);
6711 Value *Block = Builder.CreatePointerCast(V: Info.BlockArg, DestTy: GenericVoidPtrTy);
6712 const char *Name =
6713 BuiltinID == Builtin::BIget_kernel_max_sub_group_size_for_ndrange
6714 ? "__get_kernel_max_sub_group_size_for_ndrange_impl"
6715 : "__get_kernel_sub_group_count_for_ndrange_impl";
6716 return RValue::get(V: EmitRuntimeCall(
6717 callee: CGM.CreateRuntimeFunction(
6718 Ty: llvm::FunctionType::get(
6719 Result: IntTy, Params: {NDRange->getType(), GenericVoidPtrTy, GenericVoidPtrTy},
6720 isVarArg: false),
6721 Name),
6722 args: {NDRange, Kernel, Block}));
6723 }
6724 case Builtin::BI__builtin_store_half:
6725 case Builtin::BI__builtin_store_halff: {
6726 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
6727 Value *Val = EmitScalarExpr(E: E->getArg(Arg: 0));
6728 Address Address = EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
6729 Value *HalfVal = Builder.CreateFPTrunc(V: Val, DestTy: Builder.getHalfTy());
6730 Builder.CreateStore(Val: HalfVal, Addr: Address);
6731 return RValue::get(V: nullptr);
6732 }
6733 case Builtin::BI__builtin_load_half: {
6734 Address Address = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
6735 Value *HalfVal = Builder.CreateLoad(Addr: Address);
6736 return RValue::get(V: Builder.CreateFPExt(V: HalfVal, DestTy: Builder.getDoubleTy()));
6737 }
6738 case Builtin::BI__builtin_load_halff: {
6739 Address Address = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
6740 Value *HalfVal = Builder.CreateLoad(Addr: Address);
6741 return RValue::get(V: Builder.CreateFPExt(V: HalfVal, DestTy: Builder.getFloatTy()));
6742 }
6743 case Builtin::BI__builtin_printf:
6744 case Builtin::BIprintf:
6745 if (getTarget().getTriple().isNVPTX() ||
6746 getTarget().getTriple().isAMDGCN() ||
6747 (getTarget().getTriple().isSPIRV() &&
6748 getTarget().getTriple().getVendor() == Triple::VendorType::AMD)) {
6749 if (getTarget().getTriple().isNVPTX())
6750 return EmitNVPTXDevicePrintfCallExpr(E);
6751 if ((getTarget().getTriple().isAMDGCN() ||
6752 getTarget().getTriple().isSPIRV()) &&
6753 getLangOpts().HIP)
6754 return EmitAMDGPUDevicePrintfCallExpr(E);
6755 }
6756
6757 break;
6758 case Builtin::BI__builtin_canonicalize:
6759 case Builtin::BI__builtin_canonicalizef:
6760 case Builtin::BI__builtin_canonicalizef16:
6761 case Builtin::BI__builtin_canonicalizel:
6762 return RValue::get(
6763 V: emitBuiltinWithOneOverloadedType<1>(CGF&: *this, E, IntrinsicID: Intrinsic::canonicalize));
6764
6765 case Builtin::BI__builtin_thread_pointer: {
6766 if (!getContext().getTargetInfo().isTLSSupported())
6767 CGM.ErrorUnsupported(S: E, Type: "__builtin_thread_pointer");
6768
6769 return RValue::get(V: Builder.CreateIntrinsic(ID: llvm::Intrinsic::thread_pointer,
6770 OverloadTypes: {GlobalsInt8PtrTy}, Args: {}));
6771 }
6772 case Builtin::BI__builtin_os_log_format:
6773 return emitBuiltinOSLogFormat(E: *E);
6774
6775 case Builtin::BI__xray_customevent: {
6776 if (!ShouldXRayInstrumentFunction())
6777 return RValue::getIgnored();
6778
6779 if (!CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
6780 K: XRayInstrKind::Custom))
6781 return RValue::getIgnored();
6782
6783 if (const auto *XRayAttr = CurFuncDecl->getAttr<XRayInstrumentAttr>())
6784 if (XRayAttr->neverXRayInstrument() && !AlwaysEmitXRayCustomEvents())
6785 return RValue::getIgnored();
6786
6787 Function *F = CGM.getIntrinsic(IID: Intrinsic::xray_customevent);
6788 auto FTy = F->getFunctionType();
6789 auto Arg0 = E->getArg(Arg: 0);
6790 auto Arg0Val = EmitScalarExpr(E: Arg0);
6791 auto Arg0Ty = Arg0->getType();
6792 auto PTy0 = FTy->getParamType(i: 0);
6793 if (PTy0 != Arg0Val->getType()) {
6794 if (Arg0Ty->isArrayType())
6795 Arg0Val = EmitArrayToPointerDecay(Array: Arg0).emitRawPointer(CGF&: *this);
6796 else
6797 Arg0Val = Builder.CreatePointerCast(V: Arg0Val, DestTy: PTy0);
6798 }
6799 auto Arg1 = EmitScalarExpr(E: E->getArg(Arg: 1));
6800 auto PTy1 = FTy->getParamType(i: 1);
6801 if (PTy1 != Arg1->getType())
6802 Arg1 = Builder.CreateTruncOrBitCast(V: Arg1, DestTy: PTy1);
6803 return RValue::get(V: Builder.CreateCall(Callee: F, Args: {Arg0Val, Arg1}));
6804 }
6805
6806 case Builtin::BI__xray_typedevent: {
6807 // TODO: There should be a way to always emit events even if the current
6808 // function is not instrumented. Losing events in a stream can cripple
6809 // a trace.
6810 if (!ShouldXRayInstrumentFunction())
6811 return RValue::getIgnored();
6812
6813 if (!CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
6814 K: XRayInstrKind::Typed))
6815 return RValue::getIgnored();
6816
6817 if (const auto *XRayAttr = CurFuncDecl->getAttr<XRayInstrumentAttr>())
6818 if (XRayAttr->neverXRayInstrument() && !AlwaysEmitXRayTypedEvents())
6819 return RValue::getIgnored();
6820
6821 Function *F = CGM.getIntrinsic(IID: Intrinsic::xray_typedevent);
6822 auto FTy = F->getFunctionType();
6823 auto Arg0 = EmitScalarExpr(E: E->getArg(Arg: 0));
6824 auto PTy0 = FTy->getParamType(i: 0);
6825 if (PTy0 != Arg0->getType())
6826 Arg0 = Builder.CreateTruncOrBitCast(V: Arg0, DestTy: PTy0);
6827 auto Arg1 = E->getArg(Arg: 1);
6828 auto Arg1Val = EmitScalarExpr(E: Arg1);
6829 auto Arg1Ty = Arg1->getType();
6830 auto PTy1 = FTy->getParamType(i: 1);
6831 if (PTy1 != Arg1Val->getType()) {
6832 if (Arg1Ty->isArrayType())
6833 Arg1Val = EmitArrayToPointerDecay(Array: Arg1).emitRawPointer(CGF&: *this);
6834 else
6835 Arg1Val = Builder.CreatePointerCast(V: Arg1Val, DestTy: PTy1);
6836 }
6837 auto Arg2 = EmitScalarExpr(E: E->getArg(Arg: 2));
6838 auto PTy2 = FTy->getParamType(i: 2);
6839 if (PTy2 != Arg2->getType())
6840 Arg2 = Builder.CreateTruncOrBitCast(V: Arg2, DestTy: PTy2);
6841 return RValue::get(V: Builder.CreateCall(Callee: F, Args: {Arg0, Arg1Val, Arg2}));
6842 }
6843
6844 case Builtin::BI__builtin_ms_va_start:
6845 case Builtin::BI__builtin_ms_va_end:
6846 return RValue::get(
6847 V: EmitVAStartEnd(ArgValue: EmitMSVAListRef(E: E->getArg(Arg: 0)).emitRawPointer(CGF&: *this),
6848 IsStart: BuiltinID == Builtin::BI__builtin_ms_va_start));
6849
6850 case Builtin::BI__builtin_ms_va_copy: {
6851 // Lower this manually. We can't reliably determine whether or not any
6852 // given va_copy() is for a Win64 va_list from the calling convention
6853 // alone, because it's legal to do this from a System V ABI function.
6854 // With opaque pointer types, we won't have enough information in LLVM
6855 // IR to determine this from the argument types, either. Best to do it
6856 // now, while we have enough information.
6857 Address DestAddr = EmitMSVAListRef(E: E->getArg(Arg: 0));
6858 Address SrcAddr = EmitMSVAListRef(E: E->getArg(Arg: 1));
6859
6860 DestAddr = DestAddr.withElementType(ElemTy: Int8PtrTy);
6861 SrcAddr = SrcAddr.withElementType(ElemTy: Int8PtrTy);
6862
6863 Value *ArgPtr = Builder.CreateLoad(Addr: SrcAddr, Name: "ap.val");
6864 return RValue::get(V: Builder.CreateStore(Val: ArgPtr, Addr: DestAddr));
6865 }
6866
6867 case Builtin::BI__builtin_zos_va_start:
6868 case Builtin::BI__builtin_zos_va_end: {
6869 // The va_list is an array with 2 elements, called curr and next.
6870 // Element curr is set to 0. For builtin_zos_va_start, next is initialized
6871 // with a call to @llvm.va_start. Otherwise, next is passed to @llvm.va_end.
6872 Address VAList = EmitZOSVAListRef(E: E->getArg(Arg: 0));
6873 llvm::Type *VAListTy = ConvertType(T: getContext().getBuiltinZOSVaListType());
6874 VAList = VAList.withElementType(ElemTy: VAListTy);
6875 Address Curr = Builder.CreateConstArrayGEP(Addr: VAList, Index: 0, Name: "curr");
6876 Value *Zero = llvm::Constant::getNullValue(Ty: VoidPtrTy);
6877 Builder.CreateStore(Val: Zero, Addr: Curr);
6878 Address Next = Builder.CreateConstArrayGEP(Addr: VAList, Index: 1, Name: "next");
6879 return RValue::get(
6880 V: EmitVAStartEnd(ArgValue: Next.emitRawPointer(CGF&: *this),
6881 IsStart: BuiltinID == Builtin::BI__builtin_zos_va_start));
6882 }
6883 case Builtin::BI__builtin_zos_va_copy: {
6884 // Lower this manually because later can't reliably determine the type.
6885 Address Dest = EmitZOSVAListRef(E: E->getArg(Arg: 0));
6886 Address Src = EmitZOSVAListRef(E: E->getArg(Arg: 1));
6887 llvm::Type *VAListTy = ConvertType(T: getContext().getBuiltinZOSVaListType());
6888 uint64_t SizeBytes =
6889 CGM.getDataLayout().getTypeAllocSize(Ty: VAListTy).getFixedValue();
6890 Value *SizeVal = llvm::ConstantInt::get(Ty: Int64Ty, V: SizeBytes);
6891 Builder.CreateMemCpy(Dest, Src, Size: SizeVal, IsVolatile: false);
6892 return RValue::get(V: Dest.emitRawPointer(CGF&: *this));
6893 }
6894
6895 case Builtin::BI__builtin_get_device_side_mangled_name: {
6896 auto Name = CGM.getCUDARuntime().getDeviceSideName(
6897 ND: cast<DeclRefExpr>(Val: E->getArg(Arg: 0)->IgnoreImpCasts())->getDecl());
6898 auto Str = CGM.GetAddrOfConstantCString(Str: Name, GlobalName: "");
6899 return RValue::get(V: Str.getPointer());
6900 }
6901 }
6902
6903 // If this is an alias for a lib function (e.g. __builtin_sin), emit
6904 // the call using the normal call path, but using the unmangled
6905 // version of the function name.
6906 const auto &BI = getContext().BuiltinInfo;
6907 if (!shouldEmitBuiltinAsIR(BuiltinID, BI, CGF: *this) &&
6908 BI.isLibFunction(ID: BuiltinID))
6909 return emitLibraryCall(CGF&: *this, FD, E,
6910 calleeValue: CGM.getBuiltinLibFunction(FD, BuiltinID));
6911
6912 // If this is a predefined lib function (e.g. malloc), emit the call
6913 // using exactly the normal call path.
6914 if (BI.isPredefinedLibFunction(ID: BuiltinID))
6915 return emitLibraryCall(CGF&: *this, FD, E, calleeValue: CGM.getRawFunctionPointer(GD: FD));
6916
6917 // Check that a call to a target specific builtin has the correct target
6918 // features.
6919 // This is down here to avoid non-target specific builtins, however, if
6920 // generic builtins start to require generic target features then we
6921 // can move this up to the beginning of the function.
6922 checkTargetFeatures(E, TargetDecl: FD);
6923
6924 if (unsigned VectorWidth = getContext().BuiltinInfo.getRequiredVectorWidth(ID: BuiltinID))
6925 LargestVectorWidth = std::max(a: LargestVectorWidth, b: VectorWidth);
6926
6927 // See if we have a target specific intrinsic.
6928 std::string Name = getContext().BuiltinInfo.getName(ID: BuiltinID);
6929 Intrinsic::ID IntrinsicID = Intrinsic::not_intrinsic;
6930 StringRef Prefix =
6931 llvm::Triple::getArchTypePrefix(Kind: getTarget().getTriple().getArch());
6932 if (!Prefix.empty()) {
6933 IntrinsicID = Intrinsic::getIntrinsicForClangBuiltin(TargetPrefix: Prefix.data(), BuiltinName: Name);
6934 if (IntrinsicID == Intrinsic::not_intrinsic && Prefix == "spv" &&
6935 getTarget().getTriple().getOS() == llvm::Triple::OSType::AMDHSA)
6936 IntrinsicID = Intrinsic::getIntrinsicForClangBuiltin(TargetPrefix: "amdgcn", BuiltinName: Name);
6937 // NOTE we don't need to perform a compatibility flag check here since the
6938 // intrinsics are declared in Builtins*.def via LANGBUILTIN which filter the
6939 // MS builtins via ALL_MS_LANGUAGES and are filtered earlier.
6940 if (IntrinsicID == Intrinsic::not_intrinsic)
6941 IntrinsicID = Intrinsic::getIntrinsicForMSBuiltin(TargetPrefix: Prefix.data(), BuiltinName: Name);
6942 }
6943
6944 if (IntrinsicID != Intrinsic::not_intrinsic) {
6945 SmallVector<Value*, 16> Args;
6946
6947 // Find out if any arguments are required to be integer constant
6948 // expressions.
6949 unsigned ICEArguments = 0;
6950 ASTContext::GetBuiltinTypeError Error;
6951 getContext().GetBuiltinType(ID: BuiltinID, Error, IntegerConstantArgs: &ICEArguments);
6952 assert(Error == ASTContext::GE_None && "Should not codegen an error");
6953
6954 Function *F = CGM.getIntrinsic(IID: IntrinsicID);
6955 llvm::FunctionType *FTy = F->getFunctionType();
6956
6957 for (unsigned i = 0, e = E->getNumArgs(); i != e; ++i) {
6958 Value *ArgValue = EmitScalarOrConstFoldImmArg(ICEArguments, Idx: i, E);
6959 // If the intrinsic arg type is different from the builtin arg type
6960 // we need to do a bit cast.
6961 llvm::Type *PTy = FTy->getParamType(i);
6962 if (PTy != ArgValue->getType()) {
6963 // XXX - vector of pointers?
6964 if (auto *PtrTy = dyn_cast<llvm::PointerType>(Val: PTy)) {
6965 if (PtrTy->getAddressSpace() !=
6966 ArgValue->getType()->getPointerAddressSpace()) {
6967 ArgValue = Builder.CreateAddrSpaceCast(
6968 V: ArgValue, DestTy: llvm::PointerType::get(C&: getLLVMContext(),
6969 AddressSpace: PtrTy->getAddressSpace()));
6970 }
6971 }
6972
6973 // Cast vector type (e.g., v256i32) to x86_amx, this only happen
6974 // in amx intrinsics.
6975 if (PTy->isX86_AMXTy())
6976 ArgValue = Builder.CreateIntrinsic(ID: Intrinsic::x86_cast_vector_to_tile,
6977 OverloadTypes: {ArgValue->getType()}, Args: {ArgValue});
6978 else
6979 ArgValue = Builder.CreateBitCast(V: ArgValue, DestTy: PTy);
6980 }
6981
6982 Args.push_back(Elt: ArgValue);
6983 }
6984
6985 appendDefaultIntrinsicArgs(Args, F);
6986
6987 Value *V = Builder.CreateCall(Callee: F, Args);
6988 QualType BuiltinRetType = E->getType();
6989
6990 llvm::Type *RetTy = VoidTy;
6991 if (!BuiltinRetType->isVoidType())
6992 RetTy = ConvertType(T: BuiltinRetType);
6993
6994 if (RetTy != V->getType()) {
6995 // XXX - vector of pointers?
6996 if (auto *PtrTy = dyn_cast<llvm::PointerType>(Val: RetTy)) {
6997 if (PtrTy->getAddressSpace() != V->getType()->getPointerAddressSpace()) {
6998 V = Builder.CreateAddrSpaceCast(
6999 V, DestTy: llvm::PointerType::get(C&: getLLVMContext(),
7000 AddressSpace: PtrTy->getAddressSpace()));
7001 }
7002 }
7003
7004 // Cast x86_amx to vector type (e.g., v256i32), this only happen
7005 // in amx intrinsics.
7006 if (V->getType()->isX86_AMXTy())
7007 V = Builder.CreateIntrinsic(ID: Intrinsic::x86_cast_tile_to_vector, OverloadTypes: {RetTy},
7008 Args: {V});
7009 else
7010 V = Builder.CreateBitCast(V, DestTy: RetTy);
7011 }
7012
7013 if (RetTy->isVoidTy())
7014 return RValue::get(V: nullptr);
7015
7016 return RValue::get(V);
7017 }
7018
7019 // Some target-specific builtins can have aggregate return values, e.g.
7020 // __builtin_arm_mve_vld2q_u32. So if the result is an aggregate, force
7021 // ReturnValue to be non-null, so that the target-specific emission code can
7022 // always just emit into it.
7023 TypeEvaluationKind EvalKind = getEvaluationKind(T: E->getType());
7024 if (EvalKind == TEK_Aggregate && ReturnValue.isNull()) {
7025 Address DestPtr = CreateMemTemp(T: E->getType(), Name: "agg.tmp");
7026 ReturnValue = ReturnValueSlot(DestPtr, false);
7027 }
7028
7029 // Now see if we can emit a target-specific builtin.
7030 if (Value *V = EmitTargetBuiltinExpr(BuiltinID, E, ReturnValue)) {
7031 switch (EvalKind) {
7032 case TEK_Scalar:
7033 if (V->getType()->isVoidTy())
7034 return RValue::get(V: nullptr);
7035 return RValue::get(V);
7036 case TEK_Aggregate:
7037 return RValue::getAggregate(addr: ReturnValue.getAddress(),
7038 isVolatile: ReturnValue.isVolatile());
7039 case TEK_Complex:
7040 llvm_unreachable("No current target builtin returns complex");
7041 }
7042 llvm_unreachable("Bad evaluation kind in EmitBuiltinExpr");
7043 }
7044
7045 // EmitHLSLBuiltinExpr will check getLangOpts().HLSL
7046 if (Value *V = EmitHLSLBuiltinExpr(BuiltinID, E, ReturnValue)) {
7047 switch (EvalKind) {
7048 case TEK_Scalar:
7049 if (V->getType()->isVoidTy())
7050 return RValue::get(V: nullptr);
7051 return RValue::get(V);
7052 case TEK_Aggregate:
7053 return RValue::getAggregate(addr: ReturnValue.getAddress(),
7054 isVolatile: ReturnValue.isVolatile());
7055 case TEK_Complex:
7056 llvm_unreachable("No current hlsl builtin returns complex");
7057 }
7058 llvm_unreachable("Bad evaluation kind in EmitBuiltinExpr");
7059 }
7060
7061 if (getLangOpts().HIPStdPar && getLangOpts().CUDAIsDevice)
7062 return EmitHipStdParUnsupportedBuiltin(CGF: this, FD);
7063
7064 ErrorUnsupported(S: E, Type: "builtin function");
7065
7066 // Unknown builtin, for now just dump it out and return undef.
7067 return GetUndefRValue(Ty: E->getType());
7068}
7069
7070namespace {
7071struct BuiltinAlignArgs {
7072 llvm::Value *Src = nullptr;
7073 llvm::Type *SrcType = nullptr;
7074 llvm::Value *Alignment = nullptr;
7075 llvm::Value *Mask = nullptr;
7076 llvm::IntegerType *IntType = nullptr;
7077
7078 BuiltinAlignArgs(const CallExpr *E, CodeGenFunction &CGF) {
7079 QualType AstType = E->getArg(Arg: 0)->getType();
7080 if (AstType->isArrayType())
7081 Src = CGF.EmitArrayToPointerDecay(Array: E->getArg(Arg: 0)).emitRawPointer(CGF);
7082 else
7083 Src = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
7084 SrcType = Src->getType();
7085 if (SrcType->isPointerTy()) {
7086 IntType = IntegerType::get(
7087 C&: CGF.getLLVMContext(),
7088 NumBits: CGF.CGM.getDataLayout().getIndexTypeSizeInBits(Ty: SrcType));
7089 } else {
7090 assert(SrcType->isIntegerTy());
7091 IntType = cast<llvm::IntegerType>(Val: SrcType);
7092 }
7093 Alignment = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
7094 Alignment = CGF.Builder.CreateZExtOrTrunc(V: Alignment, DestTy: IntType, Name: "alignment");
7095 auto *One = llvm::ConstantInt::get(Ty: IntType, V: 1);
7096 Mask = CGF.Builder.CreateSub(LHS: Alignment, RHS: One, Name: "mask");
7097 }
7098};
7099} // namespace
7100
7101/// Generate (x & (y-1)) == 0.
7102RValue CodeGenFunction::EmitBuiltinIsAligned(const CallExpr *E) {
7103 BuiltinAlignArgs Args(E, *this);
7104 llvm::Value *SrcAddress = Args.Src;
7105 if (Args.SrcType->isPointerTy())
7106 SrcAddress =
7107 Builder.CreateBitOrPointerCast(V: Args.Src, DestTy: Args.IntType, Name: "src_addr");
7108 return RValue::get(V: Builder.CreateICmpEQ(
7109 LHS: Builder.CreateAnd(LHS: SrcAddress, RHS: Args.Mask, Name: "set_bits"),
7110 RHS: llvm::Constant::getNullValue(Ty: Args.IntType), Name: "is_aligned"));
7111}
7112
7113/// Generate (x & ~(y-1)) to align down or ((x+(y-1)) & ~(y-1)) to align up.
7114/// Note: For pointer types we can avoid ptrtoint/inttoptr pairs by using the
7115/// llvm.ptrmask intrinsic (with a GEP before in the align_up case).
7116RValue CodeGenFunction::EmitBuiltinAlignTo(const CallExpr *E, bool AlignUp) {
7117 BuiltinAlignArgs Args(E, *this);
7118 llvm::Value *SrcForMask = Args.Src;
7119 if (AlignUp) {
7120 // When aligning up we have to first add the mask to ensure we go over the
7121 // next alignment value and then align down to the next valid multiple.
7122 // By adding the mask, we ensure that align_up on an already aligned
7123 // value will not change the value.
7124 if (Args.Src->getType()->isPointerTy()) {
7125 if (getLangOpts().PointerOverflowDefined)
7126 SrcForMask =
7127 Builder.CreateGEP(Ty: Int8Ty, Ptr: SrcForMask, IdxList: Args.Mask, Name: "over_boundary");
7128 else
7129 SrcForMask = EmitCheckedInBoundsGEP(ElemTy: Int8Ty, Ptr: SrcForMask, IdxList: Args.Mask,
7130 /*SignedIndices=*/true,
7131 /*isSubtraction=*/IsSubtraction: false,
7132 Loc: E->getExprLoc(), Name: "over_boundary");
7133 } else {
7134 SrcForMask = Builder.CreateAdd(LHS: SrcForMask, RHS: Args.Mask, Name: "over_boundary");
7135 }
7136 }
7137 // Invert the mask to only clear the lower bits.
7138 llvm::Value *InvertedMask = Builder.CreateNot(V: Args.Mask, Name: "inverted_mask");
7139 llvm::Value *Result = nullptr;
7140 if (Args.Src->getType()->isPointerTy()) {
7141 Result = Builder.CreateIntrinsic(
7142 ID: Intrinsic::ptrmask, OverloadTypes: {Args.SrcType, Args.IntType},
7143 Args: {SrcForMask, InvertedMask}, FMFSource: nullptr, Name: "aligned_result");
7144 } else {
7145 Result = Builder.CreateAnd(LHS: SrcForMask, RHS: InvertedMask, Name: "aligned_result");
7146 }
7147 assert(Result->getType() == Args.SrcType);
7148 return RValue::get(V: Result);
7149}
7150