1//===---- CGBuiltin.cpp - Emit LLVM Code for builtins ---------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit Builtin calls as LLVM code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGBuiltin.h"
14#include "ABIInfo.h"
15#include "CGCUDARuntime.h"
16#include "CGCXXABI.h"
17#include "CGDebugInfo.h"
18#include "CGObjCRuntime.h"
19#include "CGOpenCLRuntime.h"
20#include "CGRecordLayout.h"
21#include "CGValue.h"
22#include "CodeGenFunction.h"
23#include "CodeGenModule.h"
24#include "ConstantEmitter.h"
25#include "PatternInit.h"
26#include "TargetInfo.h"
27#include "clang/AST/OSLog.h"
28#include "clang/AST/StmtVisitor.h"
29#include "clang/Basic/DiagnosticFrontend.h"
30#include "clang/Basic/TargetInfo.h"
31#include "llvm/ADT/APFloat.h"
32#include "llvm/IR/InlineAsm.h"
33#include "llvm/IR/Instruction.h"
34#include "llvm/IR/Intrinsics.h"
35#include "llvm/IR/IntrinsicsX86.h"
36#include "llvm/IR/MatrixBuilder.h"
37#include "llvm/Support/ConvertUTF.h"
38#include "llvm/Support/ScopedPrinter.h"
39#include <algorithm>
40#include <optional>
41#include <utility>
42
43using namespace clang;
44using namespace CodeGen;
45using namespace llvm;
46
47/// Some builtins do not have library implementation on some targets and
48/// are instead emitted as LLVM IRs by some target builtin emitters.
49/// FIXME: Remove this when library support is added
50static bool shouldEmitBuiltinAsIR(unsigned BuiltinID,
51 const Builtin::Context &BI,
52 const CodeGenFunction &CGF) {
53 if (!CGF.CGM.getLangOpts().MathErrno &&
54 CGF.CurFPFeatures.getExceptionMode() ==
55 LangOptions::FPExceptionModeKind::FPE_Ignore &&
56 !CGF.CGM.getTargetCodeGenInfo().supportsLibCall()) {
57 switch (BuiltinID) {
58 default:
59 return false;
60 case Builtin::BIlogbf:
61 case Builtin::BI__builtin_logbf:
62 case Builtin::BIlogb:
63 case Builtin::BI__builtin_logb:
64 case Builtin::BIscalbnf:
65 case Builtin::BI__builtin_scalbnf:
66 case Builtin::BIscalbn:
67 case Builtin::BI__builtin_scalbn:
68 return true;
69 }
70 }
71 return false;
72}
73
74static Value *EmitTargetArchBuiltinExpr(CodeGenFunction *CGF,
75 unsigned BuiltinID, const CallExpr *E,
76 ReturnValueSlot ReturnValue,
77 llvm::Triple::ArchType Arch) {
78 // When compiling in HipStdPar mode we have to be conservative in rejecting
79 // target specific features in the FE, and defer the possible error to the
80 // AcceleratorCodeSelection pass, wherein iff an unsupported target builtin is
81 // referenced by an accelerator executable function, we emit an error.
82 // Returning nullptr here leads to the builtin being handled in
83 // EmitStdParUnsupportedBuiltin.
84 if (CGF->getLangOpts().HIPStdPar && CGF->getLangOpts().CUDAIsDevice &&
85 Arch != CGF->getTarget().getTriple().getArch())
86 return nullptr;
87
88 switch (Arch) {
89 case llvm::Triple::arm:
90 case llvm::Triple::armeb:
91 case llvm::Triple::thumb:
92 case llvm::Triple::thumbeb:
93 return CGF->EmitARMBuiltinExpr(BuiltinID, E, ReturnValue, Arch);
94 case llvm::Triple::aarch64:
95 case llvm::Triple::aarch64_32:
96 case llvm::Triple::aarch64_be:
97 return CGF->EmitAArch64BuiltinExpr(BuiltinID, E, Arch);
98 case llvm::Triple::bpfeb:
99 case llvm::Triple::bpfel:
100 return CGF->EmitBPFBuiltinExpr(BuiltinID, E);
101 case llvm::Triple::dxil:
102 return CGF->EmitDirectXBuiltinExpr(BuiltinID, E);
103 case llvm::Triple::x86:
104 case llvm::Triple::x86_64:
105 return CGF->EmitX86BuiltinExpr(BuiltinID, E);
106 case llvm::Triple::ppc:
107 case llvm::Triple::ppcle:
108 case llvm::Triple::ppc64:
109 case llvm::Triple::ppc64le:
110 return CGF->EmitPPCBuiltinExpr(BuiltinID, E);
111 case llvm::Triple::amdgpu:
112 case llvm::Triple::r600:
113 return CGF->EmitAMDGPUBuiltinExpr(BuiltinID, E);
114 case llvm::Triple::systemz:
115 return CGF->EmitSystemZBuiltinExpr(BuiltinID, E);
116 case llvm::Triple::nvptx:
117 case llvm::Triple::nvptx64:
118 return CGF->EmitNVPTXBuiltinExpr(BuiltinID, E);
119 case llvm::Triple::wasm32:
120 case llvm::Triple::wasm64:
121 return CGF->EmitWebAssemblyBuiltinExpr(BuiltinID, E);
122 case llvm::Triple::hexagon:
123 return CGF->EmitHexagonBuiltinExpr(BuiltinID, E);
124 case llvm::Triple::riscv32:
125 case llvm::Triple::riscv64:
126 case llvm::Triple::riscv32be:
127 case llvm::Triple::riscv64be:
128 return CGF->EmitRISCVBuiltinExpr(BuiltinID, E, ReturnValue);
129 case llvm::Triple::spirv32:
130 case llvm::Triple::spirv64:
131 if (CGF->getTarget().getTriple().getOS() == llvm::Triple::OSType::AMDHSA)
132 return CGF->EmitAMDGPUBuiltinExpr(BuiltinID, E);
133 [[fallthrough]];
134 case llvm::Triple::spirv:
135 return CGF->EmitSPIRVBuiltinExpr(BuiltinID, E);
136 case llvm::Triple::avr:
137 return CGF->EmitAVRBuiltinExpr(BuiltinID, E);
138 default:
139 return nullptr;
140 }
141}
142
143Value *CodeGenFunction::EmitTargetBuiltinExpr(unsigned BuiltinID,
144 const CallExpr *E,
145 ReturnValueSlot ReturnValue) {
146 if (getContext().BuiltinInfo.isAuxBuiltinID(ID: BuiltinID)) {
147 assert(getContext().getAuxTargetInfo() && "Missing aux target info");
148 return EmitTargetArchBuiltinExpr(
149 CGF: this, BuiltinID: getContext().BuiltinInfo.getAuxBuiltinID(ID: BuiltinID), E,
150 ReturnValue, Arch: getContext().getAuxTargetInfo()->getTriple().getArch());
151 }
152
153 return EmitTargetArchBuiltinExpr(CGF: this, BuiltinID, E, ReturnValue,
154 Arch: getTarget().getTriple().getArch());
155}
156
157static void initializeAlloca(CodeGenFunction &CGF, AllocaInst *AI, Value *Size,
158 Align AlignmentInBytes) {
159 ConstantInt *Byte;
160 switch (CGF.getLangOpts().getTrivialAutoVarInit()) {
161 case LangOptions::TrivialAutoVarInitKind::Uninitialized:
162 // Nothing to initialize.
163 return;
164 case LangOptions::TrivialAutoVarInitKind::Zero:
165 Byte = CGF.Builder.getInt8(C: 0x00);
166 break;
167 case LangOptions::TrivialAutoVarInitKind::Pattern: {
168 llvm::Type *Int8 = llvm::IntegerType::getInt8Ty(C&: CGF.CGM.getLLVMContext());
169 Byte = llvm::dyn_cast<llvm::ConstantInt>(
170 Val: initializationPatternFor(CGF.CGM, Int8));
171 break;
172 }
173 }
174 if (CGF.CGM.stopAutoInit())
175 return;
176 auto *I = CGF.Builder.CreateMemSet(Ptr: AI, Val: Byte, Size, Align: AlignmentInBytes);
177 I->addAnnotationMetadata(Annotation: "auto-init");
178}
179
180/// getBuiltinLibFunction - Given a builtin id for a function like
181/// "__builtin_fabsf", return a Function* for "fabsf".
182llvm::Constant *CodeGenModule::getBuiltinLibFunction(const FunctionDecl *FD,
183 unsigned BuiltinID) {
184 assert(Context.BuiltinInfo.isLibFunction(BuiltinID));
185
186 // Get the name, skip over the __builtin_ prefix (if necessary). We may have
187 // to build this up so provide a small stack buffer to handle the vast
188 // majority of names.
189 llvm::SmallString<64> Name;
190 GlobalDecl D(FD);
191
192 // TODO: This list should be expanded or refactored after all GCC-compatible
193 // std libcall builtins are implemented.
194 static const SmallDenseMap<unsigned, StringRef, 64> F128Builtins{
195 {Builtin::BI__builtin___fprintf_chk, "__fprintf_chkieee128"},
196 {Builtin::BI__builtin___printf_chk, "__printf_chkieee128"},
197 {Builtin::BI__builtin___snprintf_chk, "__snprintf_chkieee128"},
198 {Builtin::BI__builtin___sprintf_chk, "__sprintf_chkieee128"},
199 {Builtin::BI__builtin___vfprintf_chk, "__vfprintf_chkieee128"},
200 {Builtin::BI__builtin___vprintf_chk, "__vprintf_chkieee128"},
201 {Builtin::BI__builtin___vsnprintf_chk, "__vsnprintf_chkieee128"},
202 {Builtin::BI__builtin___vsprintf_chk, "__vsprintf_chkieee128"},
203 {Builtin::BI__builtin_fprintf, "__fprintfieee128"},
204 {Builtin::BI__builtin_printf, "__printfieee128"},
205 {Builtin::BI__builtin_snprintf, "__snprintfieee128"},
206 {Builtin::BI__builtin_sprintf, "__sprintfieee128"},
207 {Builtin::BI__builtin_vfprintf, "__vfprintfieee128"},
208 {Builtin::BI__builtin_vprintf, "__vprintfieee128"},
209 {Builtin::BI__builtin_vsnprintf, "__vsnprintfieee128"},
210 {Builtin::BI__builtin_vsprintf, "__vsprintfieee128"},
211 {Builtin::BI__builtin_fscanf, "__fscanfieee128"},
212 {Builtin::BI__builtin_scanf, "__scanfieee128"},
213 {Builtin::BI__builtin_sscanf, "__sscanfieee128"},
214 {Builtin::BI__builtin_vfscanf, "__vfscanfieee128"},
215 {Builtin::BI__builtin_vscanf, "__vscanfieee128"},
216 {Builtin::BI__builtin_vsscanf, "__vsscanfieee128"},
217 {Builtin::BI__builtin_nexttowardf128, "__nexttowardieee128"},
218 };
219
220 // The AIX library functions frexpl, ldexpl, and modfl are for 128-bit
221 // IBM 'long double' (i.e. __ibm128). Map to the 'double' versions
222 // if it is 64-bit 'long double' mode.
223 static const SmallDenseMap<unsigned, StringRef, 4> AIXLongDouble64Builtins{
224 {Builtin::BI__builtin_frexpl, "frexp"},
225 {Builtin::BI__builtin_ldexpl, "ldexp"},
226 {Builtin::BI__builtin_modfl, "modf"},
227 };
228
229 // If the builtin has been declared explicitly with an assembler label,
230 // use the mangled name. This differs from the plain label on platforms
231 // that prefix labels.
232 if (FD->hasAttr<AsmLabelAttr>())
233 Name = getMangledName(GD: D);
234 else {
235 // TODO: This mutation should also be applied to other targets other than
236 // PPC, after backend supports IEEE 128-bit style libcalls.
237 if (getTriple().isPPC64() &&
238 &getTarget().getLongDoubleFormat() == &llvm::APFloat::IEEEquad() &&
239 F128Builtins.contains(Val: BuiltinID))
240 Name = F128Builtins.lookup(Val: BuiltinID);
241 else if (getTriple().isOSAIX() &&
242 &getTarget().getLongDoubleFormat() ==
243 &llvm::APFloat::IEEEdouble() &&
244 AIXLongDouble64Builtins.contains(Val: BuiltinID))
245 Name = AIXLongDouble64Builtins.lookup(Val: BuiltinID);
246 else
247 Name = Context.BuiltinInfo.getName(ID: BuiltinID).substr(pos: 10);
248 }
249
250 llvm::FunctionType *Ty =
251 cast<llvm::FunctionType>(Val: getTypes().ConvertType(T: FD->getType()));
252
253 return GetOrCreateLLVMFunction(MangledName: Name, Ty, D, /*ForVTable=*/false);
254}
255
256void appendDefaultIntrinsicArgs(SmallVectorImpl<llvm::Value *> &Args,
257 llvm::Function *F) {
258 llvm::FunctionType *FTy = F->getFunctionType();
259 unsigned NumParams = FTy->getNumParams();
260 if (Args.size() >= NumParams)
261 return;
262
263 auto [FirstDefault, Defaults] =
264 Intrinsic::getAllDefaultArgValues(IID: F->getIntrinsicID());
265 assert(Args.size() >= FirstDefault &&
266 "builtin passes fewer arguments than the intrinsic requires");
267
268 for (unsigned I = Args.size(); I != NumParams; ++I) {
269 llvm::Type *ParamTy = FTy->getParamType(i: I);
270 unsigned DefaultIdx = I - FirstDefault;
271 assert(ParamTy->isIntegerTy() &&
272 "intrinsic default arguments must be integer-typed");
273 Args.push_back(Elt: llvm::ConstantInt::get(Ty: ParamTy, V: Defaults[DefaultIdx]));
274 }
275}
276
277/// Emit the conversions required to turn the given value into an
278/// integer of the given size.
279Value *EmitToInt(CodeGenFunction &CGF, llvm::Value *V,
280 QualType T, llvm::IntegerType *IntType) {
281 V = CGF.EmitToMemory(Value: V, Ty: T);
282
283 if (V->getType()->isPointerTy())
284 return CGF.Builder.CreatePtrToInt(V, DestTy: IntType);
285
286 assert(V->getType() == IntType);
287 return V;
288}
289
290Value *EmitFromInt(CodeGenFunction &CGF, llvm::Value *V,
291 QualType T, llvm::Type *ResultType) {
292 V = CGF.EmitFromMemory(Value: V, Ty: T);
293
294 if (ResultType->isPointerTy())
295 return CGF.Builder.CreateIntToPtr(V, DestTy: ResultType);
296
297 assert(V->getType() == ResultType);
298 return V;
299}
300
301Address CheckAtomicAlignment(CodeGenFunction &CGF, const CallExpr *E) {
302 ASTContext &Ctx = CGF.getContext();
303 Address Ptr = CGF.EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
304 const llvm::DataLayout &DL = CGF.CGM.getDataLayout();
305 unsigned Bytes = Ptr.getElementType()->isPointerTy()
306 ? Ctx.getTypeSizeInChars(T: Ctx.VoidPtrTy).getQuantity()
307 : DL.getTypeStoreSize(Ty: Ptr.getElementType());
308 unsigned Align = Ptr.getAlignment().getQuantity();
309 if (Align % Bytes != 0) {
310 DiagnosticsEngine &Diags = CGF.CGM.getDiags();
311 Diags.Report(Loc: E->getBeginLoc(), DiagID: diag::warn_sync_op_misaligned);
312 // Force address to be at least naturally-aligned.
313 return Ptr.withAlignment(NewAlignment: CharUnits::fromQuantity(Quantity: Bytes));
314 }
315 return Ptr;
316}
317
318/// Utility to insert an atomic instruction based on Intrinsic::ID
319/// and the expression node.
320Value *MakeBinaryAtomicValue(
321 CodeGenFunction &CGF, llvm::AtomicRMWInst::BinOp Kind, const CallExpr *E,
322 AtomicOrdering Ordering) {
323
324 QualType T = E->getType();
325 assert(E->getArg(0)->getType()->isPointerType());
326 assert(CGF.getContext().hasSameUnqualifiedType(T,
327 E->getArg(0)->getType()->getPointeeType()));
328 assert(CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType()));
329
330 Address DestAddr = CheckAtomicAlignment(CGF, E);
331
332 llvm::IntegerType *IntType = llvm::IntegerType::get(
333 C&: CGF.getLLVMContext(), NumBits: CGF.getContext().getTypeSize(T));
334
335 llvm::Value *Val = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
336 llvm::Type *ValueType = Val->getType();
337 Val = EmitToInt(CGF, V: Val, T, IntType);
338
339 llvm::Value *Result =
340 CGF.Builder.CreateAtomicRMW(Op: Kind, Addr: DestAddr, Val, Ordering);
341 // Consider atomics to be volatile in MS kernel mode.
342 if (CGF.CGM.getLangOpts().Kernel)
343 cast<llvm::AtomicRMWInst>(Val: Result)->setVolatile(true);
344 return EmitFromInt(CGF, V: Result, T, ResultType: ValueType);
345}
346
347static Value *EmitNontemporalStore(CodeGenFunction &CGF, const CallExpr *E) {
348 Value *Val = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
349 Address Addr = CGF.EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
350
351 Val = CGF.EmitToMemory(Value: Val, Ty: E->getArg(Arg: 0)->getType());
352 LValue LV = CGF.MakeAddrLValue(Addr, T: E->getArg(Arg: 0)->getType());
353 LV.setNontemporal(true);
354 CGF.EmitStoreOfScalar(value: Val, lvalue: LV, isInit: false);
355 return nullptr;
356}
357
358static Value *EmitNontemporalLoad(CodeGenFunction &CGF, const CallExpr *E) {
359 Address Addr = CGF.EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
360
361 LValue LV = CGF.MakeAddrLValue(Addr, T: E->getType());
362 LV.setNontemporal(true);
363 return CGF.EmitLoadOfScalar(lvalue: LV, Loc: E->getExprLoc());
364}
365
366static RValue EmitBinaryAtomic(CodeGenFunction &CGF,
367 llvm::AtomicRMWInst::BinOp Kind,
368 const CallExpr *E) {
369 return RValue::get(V: MakeBinaryAtomicValue(CGF, Kind, E));
370}
371
372/// Utility to insert an atomic instruction based Intrinsic::ID and
373/// the expression node, where the return value is the result of the
374/// operation.
375static RValue EmitBinaryAtomicPost(CodeGenFunction &CGF,
376 llvm::AtomicRMWInst::BinOp Kind,
377 const CallExpr *E,
378 Instruction::BinaryOps Op,
379 bool Invert = false) {
380 QualType T = E->getType();
381 assert(E->getArg(0)->getType()->isPointerType());
382 assert(CGF.getContext().hasSameUnqualifiedType(T,
383 E->getArg(0)->getType()->getPointeeType()));
384 assert(CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType()));
385
386 Address DestAddr = CheckAtomicAlignment(CGF, E);
387
388 llvm::IntegerType *IntType = llvm::IntegerType::get(
389 C&: CGF.getLLVMContext(), NumBits: CGF.getContext().getTypeSize(T));
390
391 llvm::Value *Val = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
392 llvm::Type *ValueType = Val->getType();
393 Val = EmitToInt(CGF, V: Val, T, IntType);
394
395 llvm::Value *Result = CGF.Builder.CreateAtomicRMW(
396 Op: Kind, Addr: DestAddr, Val, Ordering: llvm::AtomicOrdering::SequentiallyConsistent);
397 Result = CGF.Builder.CreateBinOp(Opc: Op, LHS: Result, RHS: Val);
398 if (Invert)
399 Result =
400 CGF.Builder.CreateBinOp(Opc: llvm::Instruction::Xor, LHS: Result,
401 RHS: llvm::ConstantInt::getAllOnesValue(Ty: IntType));
402 Result = EmitFromInt(CGF, V: Result, T, ResultType: ValueType);
403 return RValue::get(V: Result);
404}
405
406/// Utility to insert an atomic cmpxchg instruction.
407///
408/// @param CGF The current codegen function.
409/// @param E Builtin call expression to convert to cmpxchg.
410/// arg0 - address to operate on
411/// arg1 - value to compare with
412/// arg2 - new value
413/// @param ReturnBool Specifies whether to return success flag of
414/// cmpxchg result or the old value.
415///
416/// @returns result of cmpxchg, according to ReturnBool
417///
418/// Note: In order to lower Microsoft's _InterlockedCompareExchange* intrinsics
419/// invoke the function EmitAtomicCmpXchgForMSIntrin.
420Value *MakeAtomicCmpXchgValue(CodeGenFunction &CGF, const CallExpr *E,
421 bool ReturnBool,
422 llvm::AtomicOrdering SuccessOrdering,
423 llvm::AtomicOrdering FailureOrdering) {
424 QualType T = ReturnBool ? E->getArg(Arg: 1)->getType() : E->getType();
425 Address DestAddr = CheckAtomicAlignment(CGF, E);
426
427 llvm::IntegerType *IntType = llvm::IntegerType::get(
428 C&: CGF.getLLVMContext(), NumBits: CGF.getContext().getTypeSize(T));
429
430 Value *Cmp = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
431 llvm::Type *ValueType = Cmp->getType();
432 Cmp = EmitToInt(CGF, V: Cmp, T, IntType);
433 Value *New = EmitToInt(CGF, V: CGF.EmitScalarExpr(E: E->getArg(Arg: 2)), T, IntType);
434
435 Value *Pair = CGF.Builder.CreateAtomicCmpXchg(
436 Addr: DestAddr, Cmp, New, SuccessOrdering, FailureOrdering);
437 if (ReturnBool)
438 // Extract boolean success flag and zext it to int.
439 return CGF.Builder.CreateZExt(V: CGF.Builder.CreateExtractValue(Agg: Pair, Idxs: 1),
440 DestTy: CGF.ConvertType(T: E->getType()));
441 else
442 // Extract old value and emit it using the same type as compare value.
443 return EmitFromInt(CGF, V: CGF.Builder.CreateExtractValue(Agg: Pair, Idxs: 0), T,
444 ResultType: ValueType);
445}
446
447/// This function should be invoked to emit atomic cmpxchg for Microsoft's
448/// _InterlockedCompareExchange* intrinsics which have the following signature:
449/// T _InterlockedCompareExchange(T volatile *Destination,
450/// T Exchange,
451/// T Comparand);
452///
453/// Whereas the llvm 'cmpxchg' instruction has the following syntax:
454/// cmpxchg *Destination, Comparand, Exchange.
455/// So we need to swap Comparand and Exchange when invoking
456/// CreateAtomicCmpXchg. That is the reason we could not use the above utility
457/// function MakeAtomicCmpXchgValue since it expects the arguments to be
458/// already swapped.
459
460static
461Value *EmitAtomicCmpXchgForMSIntrin(CodeGenFunction &CGF, const CallExpr *E,
462 AtomicOrdering SuccessOrdering = AtomicOrdering::SequentiallyConsistent) {
463 assert(E->getArg(0)->getType()->isPointerType());
464 assert(CGF.getContext().hasSameUnqualifiedType(
465 E->getType(), E->getArg(0)->getType()->getPointeeType()));
466 assert(CGF.getContext().hasSameUnqualifiedType(E->getType(),
467 E->getArg(1)->getType()));
468 assert(CGF.getContext().hasSameUnqualifiedType(E->getType(),
469 E->getArg(2)->getType()));
470
471 Address DestAddr = CheckAtomicAlignment(CGF, E);
472
473 auto *Exchange = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
474 auto *RTy = Exchange->getType();
475
476 auto *Comparand = CGF.EmitScalarExpr(E: E->getArg(Arg: 2));
477
478 if (RTy->isPointerTy()) {
479 Exchange = CGF.Builder.CreatePtrToInt(V: Exchange, DestTy: CGF.IntPtrTy);
480 Comparand = CGF.Builder.CreatePtrToInt(V: Comparand, DestTy: CGF.IntPtrTy);
481 }
482
483 // For Release ordering, the failure ordering should be Monotonic.
484 auto FailureOrdering = SuccessOrdering == AtomicOrdering::Release ?
485 AtomicOrdering::Monotonic :
486 SuccessOrdering;
487
488 // The atomic instruction is marked volatile for consistency with MSVC. This
489 // blocks the few atomics optimizations that LLVM has. If we want to optimize
490 // _Interlocked* operations in the future, we will have to remove the volatile
491 // marker.
492 auto *CmpXchg = CGF.Builder.CreateAtomicCmpXchg(
493 Addr: DestAddr, Cmp: Comparand, New: Exchange, SuccessOrdering, FailureOrdering);
494 CmpXchg->setVolatile(true);
495
496 auto *Result = CGF.Builder.CreateExtractValue(Agg: CmpXchg, Idxs: 0);
497 if (RTy->isPointerTy()) {
498 Result = CGF.Builder.CreateIntToPtr(V: Result, DestTy: RTy);
499 }
500
501 return Result;
502}
503
504// 64-bit Microsoft platforms support 128 bit cmpxchg operations. They are
505// prototyped like this:
506//
507// unsigned char _InterlockedCompareExchange128...(
508// __int64 volatile * _Destination,
509// __int64 _ExchangeHigh,
510// __int64 _ExchangeLow,
511// __int64 * _ComparandResult);
512//
513// Note that Destination is assumed to be at least 16-byte aligned, despite
514// being typed int64.
515
516static Value *EmitAtomicCmpXchg128ForMSIntrin(CodeGenFunction &CGF,
517 const CallExpr *E,
518 AtomicOrdering SuccessOrdering) {
519 assert(E->getNumArgs() == 4);
520 llvm::Value *DestPtr = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
521 llvm::Value *ExchangeHigh = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
522 llvm::Value *ExchangeLow = CGF.EmitScalarExpr(E: E->getArg(Arg: 2));
523 Address ComparandAddr = CGF.EmitPointerWithAlignment(Addr: E->getArg(Arg: 3));
524
525 assert(DestPtr->getType()->isPointerTy());
526 assert(!ExchangeHigh->getType()->isPointerTy());
527 assert(!ExchangeLow->getType()->isPointerTy());
528
529 // For Release ordering, the failure ordering should be Monotonic.
530 auto FailureOrdering = SuccessOrdering == AtomicOrdering::Release
531 ? AtomicOrdering::Monotonic
532 : SuccessOrdering;
533
534 // Convert to i128 pointers and values. Alignment is also overridden for
535 // destination pointer.
536 llvm::Type *Int128Ty = llvm::IntegerType::get(C&: CGF.getLLVMContext(), NumBits: 128);
537 Address DestAddr(DestPtr, Int128Ty,
538 CGF.getContext().toCharUnitsFromBits(BitSize: 128));
539 ComparandAddr = ComparandAddr.withElementType(ElemTy: Int128Ty);
540
541 // (((i128)hi) << 64) | ((i128)lo)
542 ExchangeHigh = CGF.Builder.CreateZExt(V: ExchangeHigh, DestTy: Int128Ty);
543 ExchangeLow = CGF.Builder.CreateZExt(V: ExchangeLow, DestTy: Int128Ty);
544 ExchangeHigh =
545 CGF.Builder.CreateShl(LHS: ExchangeHigh, RHS: llvm::ConstantInt::get(Ty: Int128Ty, V: 64));
546 llvm::Value *Exchange = CGF.Builder.CreateOr(LHS: ExchangeHigh, RHS: ExchangeLow);
547
548 // Load the comparand for the instruction.
549 llvm::Value *Comparand = CGF.Builder.CreateLoad(Addr: ComparandAddr);
550
551 auto *CXI = CGF.Builder.CreateAtomicCmpXchg(Addr: DestAddr, Cmp: Comparand, New: Exchange,
552 SuccessOrdering, FailureOrdering);
553
554 // The atomic instruction is marked volatile for consistency with MSVC. This
555 // blocks the few atomics optimizations that LLVM has. If we want to optimize
556 // _Interlocked* operations in the future, we will have to remove the volatile
557 // marker.
558 CXI->setVolatile(true);
559
560 // Store the result as an outparameter.
561 CGF.Builder.CreateStore(Val: CGF.Builder.CreateExtractValue(Agg: CXI, Idxs: 0),
562 Addr: ComparandAddr);
563
564 // Get the success boolean and zero extend it to i8.
565 Value *Success = CGF.Builder.CreateExtractValue(Agg: CXI, Idxs: 1);
566 return CGF.Builder.CreateZExt(V: Success, DestTy: CGF.Int8Ty);
567}
568
569static Value *EmitAtomicIncrementValue(CodeGenFunction &CGF, const CallExpr *E,
570 AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
571 assert(E->getArg(0)->getType()->isPointerType());
572
573 auto *IntTy = CGF.ConvertType(T: E->getType());
574 Address DestAddr = CheckAtomicAlignment(CGF, E);
575 auto *Result = CGF.Builder.CreateAtomicRMW(
576 Op: AtomicRMWInst::Add, Addr: DestAddr, Val: ConstantInt::get(Ty: IntTy, V: 1), Ordering);
577 return CGF.Builder.CreateAdd(LHS: Result, RHS: ConstantInt::get(Ty: IntTy, V: 1));
578}
579
580static Value *EmitAtomicDecrementValue(
581 CodeGenFunction &CGF, const CallExpr *E,
582 AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
583 assert(E->getArg(0)->getType()->isPointerType());
584
585 auto *IntTy = CGF.ConvertType(T: E->getType());
586 Address DestAddr = CheckAtomicAlignment(CGF, E);
587 auto *Result = CGF.Builder.CreateAtomicRMW(
588 Op: AtomicRMWInst::Sub, Addr: DestAddr, Val: ConstantInt::get(Ty: IntTy, V: 1), Ordering);
589 return CGF.Builder.CreateSub(LHS: Result, RHS: ConstantInt::get(Ty: IntTy, V: 1));
590}
591
592// Build a plain volatile load.
593static Value *EmitISOVolatileLoad(CodeGenFunction &CGF, const CallExpr *E) {
594 Value *Ptr = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
595 QualType ElTy = E->getArg(Arg: 0)->getType()->getPointeeType();
596 CharUnits LoadSize = CGF.getContext().getTypeSizeInChars(T: ElTy);
597 llvm::Type *ITy =
598 llvm::IntegerType::get(C&: CGF.getLLVMContext(), NumBits: LoadSize.getQuantity() * 8);
599 llvm::LoadInst *Load = CGF.Builder.CreateAlignedLoad(Ty: ITy, Addr: Ptr, Align: LoadSize);
600 Load->setAtomic(Ordering: llvm::AtomicOrdering::Monotonic);
601 Load->setVolatile(true);
602 return Load;
603}
604
605// Build a plain volatile store.
606static Value *EmitISOVolatileStore(CodeGenFunction &CGF, const CallExpr *E) {
607 Value *Ptr = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
608 Value *Value = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
609 QualType ElTy = E->getArg(Arg: 0)->getType()->getPointeeType();
610 CharUnits StoreSize = CGF.getContext().getTypeSizeInChars(T: ElTy);
611 llvm::StoreInst *Store =
612 CGF.Builder.CreateAlignedStore(Val: Value, Addr: Ptr, Align: StoreSize);
613 Store->setAtomic(Ordering: llvm::AtomicOrdering::Monotonic);
614 Store->setVolatile(true);
615 return Store;
616}
617
618// Emit a simple mangled intrinsic that has 1 argument and a return type
619// matching the argument type. Depending on mode, this may be a constrained
620// floating-point intrinsic.
621Value *emitUnaryMaybeConstrainedFPBuiltin(CodeGenFunction &CGF,
622 const CallExpr *E, unsigned IntrinsicID,
623 unsigned ConstrainedIntrinsicID) {
624 llvm::Value *Src0 = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
625
626 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
627 if (CGF.Builder.getIsFPConstrained()) {
628 Function *F = CGF.CGM.getIntrinsic(IID: ConstrainedIntrinsicID, Tys: Src0->getType());
629 return CGF.Builder.CreateConstrainedFPCall(Callee: F, Args: { Src0 });
630 } else {
631 Function *F = CGF.CGM.getIntrinsic(IID: IntrinsicID, Tys: Src0->getType());
632 return CGF.Builder.CreateCall(Callee: F, Args: Src0);
633 }
634}
635
636// Emit an intrinsic that has 2 operands of the same type as its result.
637// Depending on mode, this may be a constrained floating-point intrinsic.
638static Value *emitBinaryMaybeConstrainedFPBuiltin(CodeGenFunction &CGF,
639 const CallExpr *E, unsigned IntrinsicID,
640 unsigned ConstrainedIntrinsicID) {
641 llvm::Value *Src0 = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
642 llvm::Value *Src1 = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
643
644 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
645 if (CGF.Builder.getIsFPConstrained()) {
646 Function *F = CGF.CGM.getIntrinsic(IID: ConstrainedIntrinsicID, Tys: Src0->getType());
647 return CGF.Builder.CreateConstrainedFPCall(Callee: F, Args: { Src0, Src1 });
648 } else {
649 Function *F = CGF.CGM.getIntrinsic(IID: IntrinsicID, Tys: Src0->getType());
650 return CGF.Builder.CreateCall(Callee: F, Args: { Src0, Src1 });
651 }
652}
653
654// Has second type mangled argument.
655static Value *
656emitBinaryExpMaybeConstrainedFPBuiltin(CodeGenFunction &CGF, const CallExpr *E,
657 Intrinsic::ID IntrinsicID,
658 Intrinsic::ID ConstrainedIntrinsicID) {
659 llvm::Value *Src0 = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
660 llvm::Value *Src1 = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
661
662 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
663 if (CGF.Builder.getIsFPConstrained()) {
664 Function *F = CGF.CGM.getIntrinsic(IID: ConstrainedIntrinsicID,
665 Tys: {Src0->getType(), Src1->getType()});
666 return CGF.Builder.CreateConstrainedFPCall(Callee: F, Args: {Src0, Src1});
667 }
668
669 Function *F =
670 CGF.CGM.getIntrinsic(IID: IntrinsicID, Tys: {Src0->getType(), Src1->getType()});
671 return CGF.Builder.CreateCall(Callee: F, Args: {Src0, Src1});
672}
673
674// Emit an intrinsic that has 3 operands of the same type as its result.
675// Depending on mode, this may be a constrained floating-point intrinsic.
676static Value *emitTernaryMaybeConstrainedFPBuiltin(CodeGenFunction &CGF,
677 const CallExpr *E, unsigned IntrinsicID,
678 unsigned ConstrainedIntrinsicID) {
679 llvm::Value *Src0 = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
680 llvm::Value *Src1 = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
681 llvm::Value *Src2 = CGF.EmitScalarExpr(E: E->getArg(Arg: 2));
682
683 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
684 if (CGF.Builder.getIsFPConstrained()) {
685 Function *F = CGF.CGM.getIntrinsic(IID: ConstrainedIntrinsicID, Tys: Src0->getType());
686 return CGF.Builder.CreateConstrainedFPCall(Callee: F, Args: { Src0, Src1, Src2 });
687 } else {
688 Function *F = CGF.CGM.getIntrinsic(IID: IntrinsicID, Tys: Src0->getType());
689 return CGF.Builder.CreateCall(Callee: F, Args: { Src0, Src1, Src2 });
690 }
691}
692
693// Emit an intrinsic that has overloaded integer result and fp operand.
694static Value *
695emitMaybeConstrainedFPToIntRoundBuiltin(CodeGenFunction &CGF, const CallExpr *E,
696 unsigned IntrinsicID,
697 unsigned ConstrainedIntrinsicID) {
698 llvm::Type *ResultType = CGF.ConvertType(T: E->getType());
699 llvm::Value *Src0 = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
700
701 if (CGF.Builder.getIsFPConstrained()) {
702 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
703 Function *F = CGF.CGM.getIntrinsic(IID: ConstrainedIntrinsicID,
704 Tys: {ResultType, Src0->getType()});
705 return CGF.Builder.CreateConstrainedFPCall(Callee: F, Args: {Src0});
706 } else {
707 Function *F =
708 CGF.CGM.getIntrinsic(IID: IntrinsicID, Tys: {ResultType, Src0->getType()});
709 return CGF.Builder.CreateCall(Callee: F, Args: Src0);
710 }
711}
712
713static Value *emitFrexpBuiltin(CodeGenFunction &CGF, const CallExpr *E,
714 Intrinsic::ID IntrinsicID) {
715 llvm::Value *Src0 = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
716 llvm::Value *Src1 = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
717
718 QualType IntPtrTy = E->getArg(Arg: 1)->getType()->getPointeeType();
719 llvm::Type *IntTy = CGF.ConvertType(T: IntPtrTy);
720 llvm::Function *F =
721 CGF.CGM.getIntrinsic(IID: IntrinsicID, Tys: {Src0->getType(), IntTy});
722 llvm::Value *Call = CGF.Builder.CreateCall(Callee: F, Args: Src0);
723
724 llvm::Value *Exp = CGF.Builder.CreateExtractValue(Agg: Call, Idxs: 1);
725 LValue LV = CGF.MakeNaturalAlignAddrLValue(V: Src1, T: IntPtrTy);
726 CGF.EmitStoreOfScalar(value: Exp, lvalue: LV);
727
728 return CGF.Builder.CreateExtractValue(Agg: Call, Idxs: 0);
729}
730
731static void emitSincosBuiltin(CodeGenFunction &CGF, const CallExpr *E,
732 Intrinsic::ID IntrinsicID) {
733 llvm::Value *Val = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
734 llvm::Value *Dest0 = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
735 llvm::Value *Dest1 = CGF.EmitScalarExpr(E: E->getArg(Arg: 2));
736
737 llvm::Function *F = CGF.CGM.getIntrinsic(IID: IntrinsicID, Tys: {Val->getType()});
738 llvm::Value *Call = CGF.Builder.CreateCall(Callee: F, Args: Val);
739
740 llvm::Value *SinResult = CGF.Builder.CreateExtractValue(Agg: Call, Idxs: 0);
741 llvm::Value *CosResult = CGF.Builder.CreateExtractValue(Agg: Call, Idxs: 1);
742
743 QualType DestPtrType = E->getArg(Arg: 1)->getType()->getPointeeType();
744 LValue SinLV = CGF.MakeNaturalAlignAddrLValue(V: Dest0, T: DestPtrType);
745 LValue CosLV = CGF.MakeNaturalAlignAddrLValue(V: Dest1, T: DestPtrType);
746
747 llvm::StoreInst *StoreSin =
748 CGF.Builder.CreateStore(Val: SinResult, Addr: SinLV.getAddress());
749 llvm::StoreInst *StoreCos =
750 CGF.Builder.CreateStore(Val: CosResult, Addr: CosLV.getAddress());
751
752 // Mark the two stores as non-aliasing with each other. The order of stores
753 // emitted by this builtin is arbitrary, enforcing a particular order will
754 // prevent optimizations later on.
755 llvm::MDBuilder MDHelper(CGF.getLLVMContext());
756 MDNode *Domain = MDHelper.createAnonymousAliasScopeDomain();
757 MDNode *AliasScope = MDHelper.createAnonymousAliasScope(Domain);
758 MDNode *AliasScopeList = MDNode::get(Context&: Call->getContext(), MDs: AliasScope);
759 StoreSin->setMetadata(KindID: LLVMContext::MD_alias_scope, Node: AliasScopeList);
760 StoreCos->setMetadata(KindID: LLVMContext::MD_noalias, Node: AliasScopeList);
761}
762
763static llvm::Value *emitModfBuiltin(CodeGenFunction &CGF, const CallExpr *E,
764 Intrinsic::ID IntrinsicID) {
765 llvm::Value *Val = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
766 llvm::Value *IntPartDest = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
767
768 llvm::Value *Call =
769 CGF.Builder.CreateIntrinsic(ID: IntrinsicID, OverloadTypes: {Val->getType()}, Args: Val);
770
771 llvm::Value *FractionalResult = CGF.Builder.CreateExtractValue(Agg: Call, Idxs: 0);
772 llvm::Value *IntegralResult = CGF.Builder.CreateExtractValue(Agg: Call, Idxs: 1);
773
774 QualType DestPtrType = E->getArg(Arg: 1)->getType()->getPointeeType();
775 LValue IntegralLV = CGF.MakeNaturalAlignAddrLValue(V: IntPartDest, T: DestPtrType);
776 CGF.EmitStoreOfScalar(value: IntegralResult, lvalue: IntegralLV);
777
778 return FractionalResult;
779}
780
781/// EmitFAbs - Emit a call to @llvm.fabs().
782static Value *EmitFAbs(CodeGenFunction &CGF, Value *V) {
783 llvm::Value *Call = CGF.Builder.CreateFAbs(V);
784 if (auto *CallI = dyn_cast<llvm::CallInst>(Val: Call))
785 CallI->setDoesNotAccessMemory();
786 return Call;
787}
788
789/// Emit the computation of the sign bit for a floating point value. Returns
790/// the i1 sign bit value.
791static Value *EmitSignBit(CodeGenFunction &CGF, Value *V) {
792 LLVMContext &C = CGF.CGM.getLLVMContext();
793
794 llvm::Type *Ty = V->getType();
795 int Width = Ty->getPrimitiveSizeInBits();
796 llvm::Type *IntTy = llvm::IntegerType::get(C, NumBits: Width);
797 V = CGF.Builder.CreateBitCast(V, DestTy: IntTy);
798 if (Ty->isPPC_FP128Ty()) {
799 // We want the sign bit of the higher-order double. The bitcast we just
800 // did works as if the double-double was stored to memory and then
801 // read as an i128. The "store" will put the higher-order double in the
802 // lower address in both little- and big-Endian modes, but the "load"
803 // will treat those bits as a different part of the i128: the low bits in
804 // little-Endian, the high bits in big-Endian. Therefore, on big-Endian
805 // we need to shift the high bits down to the low before truncating.
806 Width >>= 1;
807 if (CGF.getTarget().isBigEndian()) {
808 Value *ShiftCst = llvm::ConstantInt::get(Ty: IntTy, V: Width);
809 V = CGF.Builder.CreateLShr(LHS: V, RHS: ShiftCst);
810 }
811 // We are truncating value in order to extract the higher-order
812 // double, which we will be using to extract the sign from.
813 IntTy = llvm::IntegerType::get(C, NumBits: Width);
814 V = CGF.Builder.CreateTrunc(V, DestTy: IntTy);
815 }
816 Value *Zero = llvm::Constant::getNullValue(Ty: IntTy);
817 return CGF.Builder.CreateICmpSLT(LHS: V, RHS: Zero);
818}
819
820static RValue emitLibraryCall(CodeGenFunction &CGF, const FunctionDecl *FD,
821 const CallExpr *E, llvm::Constant *calleeValue) {
822 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
823 CGCallee callee = CGCallee::forDirect(functionPtr: calleeValue, abstractInfo: GlobalDecl(FD));
824 llvm::CallBase *callOrInvoke = nullptr;
825 CGFunctionInfo const *FnInfo = nullptr;
826 return CGF.EmitCall(FnType: E->getCallee()->getType(), Callee: callee, E, ReturnValue: ReturnValueSlot(),
827 /*Chain=*/nullptr, CallOrInvoke: &callOrInvoke, ResolvedFnInfo: &FnInfo);
828}
829
830/// Emit a call to llvm.{sadd,uadd,ssub,usub,smul,umul}.with.overflow.*
831/// depending on IntrinsicID.
832///
833/// \arg CGF The current codegen function.
834/// \arg IntrinsicID The ID for the Intrinsic we wish to generate.
835/// \arg X The first argument to the llvm.*.with.overflow.*.
836/// \arg Y The second argument to the llvm.*.with.overflow.*.
837/// \arg Carry The carry returned by the llvm.*.with.overflow.*.
838/// \returns The result (i.e. sum/product) returned by the intrinsic.
839llvm::Value *EmitOverflowIntrinsic(CodeGenFunction &CGF,
840 const Intrinsic::ID IntrinsicID,
841 llvm::Value *X, llvm::Value *Y,
842 llvm::Value *&Carry) {
843 // Make sure we have integers of the same width.
844 assert(X->getType() == Y->getType() &&
845 "Arguments must be the same type. (Did you forget to make sure both "
846 "arguments have the same integer width?)");
847
848 Function *Callee = CGF.CGM.getIntrinsic(IID: IntrinsicID, Tys: X->getType());
849 llvm::Value *Tmp = CGF.Builder.CreateCall(Callee, Args: {X, Y});
850 Carry = CGF.Builder.CreateExtractValue(Agg: Tmp, Idxs: 1);
851 return CGF.Builder.CreateExtractValue(Agg: Tmp, Idxs: 0);
852}
853
854namespace {
855 struct WidthAndSignedness {
856 unsigned Width;
857 bool Signed;
858 };
859}
860
861static WidthAndSignedness
862getIntegerWidthAndSignedness(const clang::ASTContext &context,
863 const clang::QualType Type) {
864 assert(Type->isIntegerType() && "Given type is not an integer.");
865 unsigned Width = context.getIntWidth(T: Type);
866 bool Signed = Type->isSignedIntegerType();
867 return {.Width: Width, .Signed: Signed};
868}
869
870// Given one or more integer types, this function produces an integer type that
871// encompasses them: any value in one of the given types could be expressed in
872// the encompassing type.
873static struct WidthAndSignedness
874EncompassingIntegerType(ArrayRef<struct WidthAndSignedness> Types) {
875 assert(Types.size() > 0 && "Empty list of types.");
876
877 // If any of the given types is signed, we must return a signed type.
878 bool Signed = false;
879 for (const auto &Type : Types) {
880 Signed |= Type.Signed;
881 }
882
883 // The encompassing type must have a width greater than or equal to the width
884 // of the specified types. Additionally, if the encompassing type is signed,
885 // its width must be strictly greater than the width of any unsigned types
886 // given.
887 unsigned Width = 0;
888 for (const auto &Type : Types) {
889 unsigned MinWidth = Type.Width + (Signed && !Type.Signed);
890 if (Width < MinWidth) {
891 Width = MinWidth;
892 }
893 }
894
895 return {.Width: Width, .Signed: Signed};
896}
897
898Value *CodeGenFunction::EmitVAStartEnd(Value *ArgValue, bool IsStart) {
899 Intrinsic::ID inst = IsStart ? Intrinsic::vastart : Intrinsic::vaend;
900 return Builder.CreateCall(Callee: CGM.getIntrinsic(IID: inst, Tys: {ArgValue->getType()}),
901 Args: ArgValue);
902}
903
904/// Checks if using the result of __builtin_object_size(p, @p From) in place of
905/// __builtin_object_size(p, @p To) is correct
906static bool areBOSTypesCompatible(int From, int To) {
907 // Note: Our __builtin_object_size implementation currently treats Type=0 and
908 // Type=2 identically. Encoding this implementation detail here may make
909 // improving __builtin_object_size difficult in the future, so it's omitted.
910 return From == To || (From == 0 && To == 1) || (From == 3 && To == 2);
911}
912
913static llvm::Value *
914getDefaultBuiltinObjectSizeResult(unsigned Type, llvm::IntegerType *ResType) {
915 return ConstantInt::get(Ty: ResType, V: (Type & 2) ? 0 : -1, /*isSigned=*/IsSigned: true);
916}
917
918llvm::Value *
919CodeGenFunction::evaluateOrEmitBuiltinObjectSize(const Expr *E, unsigned Type,
920 llvm::IntegerType *ResType,
921 llvm::Value *EmittedE,
922 bool IsDynamic) {
923 if (std::optional<uint64_t> ObjectSize =
924 E->tryEvaluateObjectSize(Ctx: getContext(), Type))
925 return ConstantInt::get(Ty: ResType, V: *ObjectSize, /*isSigned=*/IsSigned: true);
926 return emitBuiltinObjectSize(E, Type, ResType, EmittedE, IsDynamic);
927}
928
929/// Find a struct's flexible array member. It may be embedded inside multiple
930/// sub-structs, but must still be the last field.
931static const FieldDecl *FindFlexibleArrayMemberField(CodeGenFunction &CGF,
932 ASTContext &Ctx,
933 const RecordDecl *RD) {
934 const LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel =
935 CGF.getLangOpts().getStrictFlexArraysLevel();
936
937 if (RD->isImplicit())
938 return nullptr;
939
940 for (const FieldDecl *FD : RD->fields()) {
941 if (Decl::isFlexibleArrayMemberLike(
942 Context: Ctx, D: FD, Ty: FD->getType(), StrictFlexArraysLevel,
943 /*IgnoreTemplateOrMacroSubstitution=*/true))
944 return FD;
945
946 if (const auto *RD = FD->getType()->getAsRecordDecl())
947 if (const FieldDecl *FD = FindFlexibleArrayMemberField(CGF, Ctx, RD))
948 return FD;
949 }
950
951 return nullptr;
952}
953
954/// Calculate the offset of a struct field. It may be embedded inside multiple
955/// sub-structs.
956static bool GetFieldOffset(ASTContext &Ctx, const RecordDecl *RD,
957 const FieldDecl *FD, int64_t &Offset) {
958 if (RD->isImplicit())
959 return false;
960
961 // Keep track of the field number ourselves, because the other methods
962 // (CGRecordLayout::getLLVMFieldNo) aren't always equivalent to how the AST
963 // is laid out.
964 uint32_t FieldNo = 0;
965 const ASTRecordLayout &Layout = Ctx.getASTRecordLayout(D: RD);
966
967 for (const FieldDecl *Field : RD->fields()) {
968 if (Field == FD) {
969 Offset += Layout.getFieldOffset(FieldNo);
970 return true;
971 }
972
973 if (const auto *RD = Field->getType()->getAsRecordDecl()) {
974 if (GetFieldOffset(Ctx, RD, FD, Offset)) {
975 Offset += Layout.getFieldOffset(FieldNo);
976 return true;
977 }
978 }
979
980 if (!RD->isUnion())
981 ++FieldNo;
982 }
983
984 return false;
985}
986
987static std::optional<int64_t>
988GetFieldOffset(ASTContext &Ctx, const RecordDecl *RD, const FieldDecl *FD) {
989 int64_t Offset = 0;
990
991 if (GetFieldOffset(Ctx, RD, FD, Offset))
992 return std::optional<int64_t>(Offset);
993
994 return std::nullopt;
995}
996
997llvm::Value *CodeGenFunction::emitCountedBySize(const Expr *E,
998 llvm::Value *EmittedE,
999 unsigned Type,
1000 llvm::IntegerType *ResType) {
1001 // Note: If the whole struct is specificed in the __bdos (i.e. Visitor
1002 // returns a DeclRefExpr). The calculation of the whole size of the structure
1003 // with a flexible array member can be done in two ways:
1004 //
1005 // 1) sizeof(struct S) + count * sizeof(typeof(fam))
1006 // 2) offsetof(struct S, fam) + count * sizeof(typeof(fam))
1007 //
1008 // The first will add additional padding after the end of the array
1009 // allocation while the second method is more precise, but not quite expected
1010 // from programmers. See
1011 // https://lore.kernel.org/lkml/ZvV6X5FPBBW7CO1f@archlinux/ for a discussion
1012 // of the topic.
1013 //
1014 // GCC isn't (currently) able to calculate __bdos on a pointer to the whole
1015 // structure. Therefore, because of the above issue, we choose to match what
1016 // GCC does for consistency's sake.
1017
1018 const Expr *Idx = nullptr;
1019 // FIXME: `ArrayElementTy` is misleadingly named. `findStructFieldAccess()`
1020 // sets it to the type of the array-subscript base, i.e. the (possibly cast)
1021 // *pointer* being indexed (not an element type) or a null QualType when there
1022 // is no subscript.
1023 QualType ArrayElementTy;
1024 E = findStructFieldAccess(E, OutArrayIndex: &Idx, OutArrayElementTy: &ArrayElementTy);
1025 if (!E)
1026 return nullptr;
1027
1028 if (Idx) {
1029 if (Idx->HasSideEffects(Ctx: getContext()))
1030 // We can't have side-effects.
1031 return getDefaultBuiltinObjectSizeResult(Type, ResType);
1032
1033 if (const auto *IL = dyn_cast<IntegerLiteral>(Val: Idx)) {
1034 int64_t Val = IL->getValue().getSExtValue();
1035 if (Val < 0)
1036 return getDefaultBuiltinObjectSizeResult(Type, ResType);
1037
1038 // The index is 0, so we don't need to take it into account.
1039 if (Val == 0)
1040 Idx = nullptr;
1041 }
1042 }
1043
1044 // __counted_by on either a flexible array member or a pointer into a struct
1045 // with a flexible array member.
1046 if (const auto *ME = dyn_cast<MemberExpr>(Val: E))
1047 return emitCountedByMemberSize(E: ME, Idx, EmittedE, CastedArrayElementTy: ArrayElementTy, Type,
1048 ResType);
1049
1050 // __counted_by on a pointer in a struct.
1051 if (const auto *ICE = dyn_cast<ImplicitCastExpr>(Val: E);
1052 ICE && ICE->getCastKind() == CK_LValueToRValue)
1053 return emitCountedByPointerSize(E: ICE, Idx, EmittedE, CastedArrayElementTy: ArrayElementTy, Type,
1054 ResType);
1055
1056 return nullptr;
1057}
1058
1059static llvm::Value *EmitPositiveResultOrZero(CodeGenFunction &CGF,
1060 llvm::Value *Res,
1061 llvm::Value *Index,
1062 llvm::IntegerType *ResType,
1063 bool IsSigned) {
1064 // cmp = (array_size >= 0)
1065 Value *Cmp = CGF.Builder.CreateIsNotNeg(Arg: Res);
1066 if (Index)
1067 // cmp = (cmp && index >= 0)
1068 Cmp = CGF.Builder.CreateAnd(LHS: CGF.Builder.CreateIsNotNeg(Arg: Index), RHS: Cmp);
1069
1070 // return cmp ? result : 0
1071 return CGF.Builder.CreateSelect(C: Cmp, True: Res,
1072 False: ConstantInt::get(Ty: ResType, V: 0, IsSigned));
1073}
1074
1075static std::pair<llvm::Value *, llvm::Value *>
1076GetCountFieldAndIndex(CodeGenFunction &CGF, const MemberExpr *ME,
1077 const FieldDecl *ArrayFD, const FieldDecl *CountFD,
1078 const Expr *Idx, llvm::IntegerType *ResType,
1079 bool IsSigned) {
1080 // count = ptr->count;
1081 Value *Count = CGF.EmitLoadOfCountedByField(Base: ME, FD: ArrayFD, CountDecl: CountFD);
1082 if (!Count)
1083 return std::make_pair<Value *>(x: nullptr, y: nullptr);
1084 Count = CGF.Builder.CreateIntCast(V: Count, DestTy: ResType, isSigned: IsSigned, Name: "count");
1085
1086 // index = idx;
1087 Value *Index = nullptr;
1088 if (Idx) {
1089 bool IdxSigned = Idx->getType()->isSignedIntegerType();
1090 Index = CGF.EmitScalarExpr(E: Idx);
1091 Index = CGF.Builder.CreateIntCast(V: Index, DestTy: ResType, isSigned: IdxSigned, Name: "index");
1092 }
1093
1094 return std::make_pair(x&: Count, y&: Index);
1095}
1096
1097llvm::Value *CodeGenFunction::emitCountedByPointerSize(
1098 const ImplicitCastExpr *E, const Expr *Idx, llvm::Value *EmittedE,
1099 QualType CastedArrayElementTy, unsigned Type, llvm::IntegerType *ResType) {
1100 assert(E->getCastKind() == CK_LValueToRValue &&
1101 "must be an LValue to RValue cast");
1102 assert(EmittedE && "emitted must not be null");
1103
1104 const MemberExpr *ME =
1105 dyn_cast<MemberExpr>(Val: E->getSubExpr()->IgnoreParenNoopCasts(Ctx: getContext()));
1106 if (!ME)
1107 return nullptr;
1108
1109 const auto *ArrayBaseFD = dyn_cast<FieldDecl>(Val: ME->getMemberDecl());
1110 if (!ArrayBaseFD || !ArrayBaseFD->getType()->isPointerType() ||
1111 !ArrayBaseFD->getType()->isCountAttributedType())
1112 return nullptr;
1113
1114 // Get the 'count' FieldDecl.
1115 const FieldDecl *CountFD = ArrayBaseFD->findCountedByField();
1116 if (!CountFD)
1117 // Can't find the field referenced by the "counted_by" attribute.
1118 return nullptr;
1119
1120 // Calculate the array's object size using these formulae. (Note: if the
1121 // calculation is negative, we return 0.):
1122 //
1123 // struct p;
1124 // struct s {
1125 // /* ... */
1126 // struct p **array __attribute__((ATTR(count)));
1127 // int count;
1128 // };
1129 //
1130 // 1) 'ptr->array':
1131 //
1132 // #if ATTR is counted_by_or_null || ATTR is sized_by_or_null
1133 // count = ptr->array ? ptr->count : 0;
1134 // #else
1135 // count = ptr->count;
1136 // #endif
1137 //
1138 // #if ATTR is counted_by || ATTR is counted_by_or_null
1139 // array_element_size = sizeof (*ptr->array);
1140 // array_size = count * array_element_size;
1141 // #else
1142 // array_size = count;
1143 // #endif
1144 //
1145 // result = array_size;
1146 //
1147 // cmp = (result >= 0)
1148 // return cmp ? result : 0;
1149 //
1150 // 2) '&((cast) ptr->array)[idx]':
1151 //
1152 // #if ATTR is counted_by_or_null || ATTR is sized_by_or_null
1153 // count = ptr->array ? ptr->count : 0;
1154 // #else
1155 // count = ptr->count;
1156 // #endif
1157 // index = idx;
1158 //
1159 // #if ATTR is counted_by || ATTR is counted_by_or_null
1160 // array_element_size = sizeof (*ptr->array);
1161 // array_size = count * array_element_size;
1162 // #else
1163 // array_size = count;
1164 // #endif
1165 //
1166 // casted_array_element_size = sizeof (*((cast) ptr->array));
1167 //
1168 // index_size = index * casted_array_element_size;
1169 // result = array_size - index_size;
1170 //
1171 // cmp = (result >= 0)
1172 // if (index)
1173 // cmp = (cmp && index > 0)
1174 // return cmp ? result : 0;
1175
1176 auto GetPointeeSize = [&](QualType PtrTy) -> CharUnits {
1177 assert(!PtrTy.isNull());
1178 QualType PointeeTy = PtrTy->getPointeeType();
1179 assert(!PointeeTy.isNull() &&
1180 (PointeeTy->isVoidType() || !PointeeTy->isIncompleteType()) &&
1181 "pointee type must have a computable size");
1182
1183 CharUnits PointeeSize = getContext().getTypeSizeInChars(T: PointeeTy);
1184 if (PointeeSize.isZero()) {
1185 // Support GNU extension of treating `void` having size 1.
1186 PointeeSize = CharUnits::One();
1187 }
1188
1189 return PointeeSize;
1190 };
1191
1192 bool IsSigned = CountFD->getType()->isSignedIntegerType();
1193 const auto *CountAttributedTy =
1194 ArrayBaseFD->getType()->getAs<CountAttributedType>();
1195 assert(CountAttributedTy && "the field's type is not a CountAttributedType");
1196
1197 // count = ptr->count;
1198 // index = idx;
1199 Value *Count, *Index;
1200 std::tie(args&: Count, args&: Index) = GetCountFieldAndIndex(
1201 CGF&: *this, ME, ArrayFD: ArrayBaseFD, CountFD, Idx, ResType, IsSigned);
1202 if (!Count)
1203 return nullptr;
1204
1205 // For the _or_null variants, a null pointer describes no accessible memory:
1206 // count = ptr->array ? count : 0;
1207 if (CountAttributedTy->isOrNull()) {
1208 Value *Ptr = nullptr;
1209 if (!Idx) {
1210 // 1) 'ptr->array'
1211 // Reuse the already-emitted pointer value rather than re-loading `ME`.
1212 // Re-loading would produce a second, observable access for a volatile
1213 // pointer field
1214 Ptr = EmittedE;
1215 } else {
1216 // 2) '&((cast) ptr->array)[idx]'
1217 // FIXME: `EmittedE` is the element address, not `ptr->array`, so we fall
1218 // back to re-emitting `ME` and the pointer field is loaded twice. This is
1219 // normally harmless except when the pointer is `volatile`. Avoiding that
1220 // would require restructuring how the base pointer is emitted (it is
1221 // handled elsewhere in the callstack), so it is left as-is for now.
1222 Ptr = EmitScalarExpr(E: ME);
1223 }
1224 Value *IsNull = Builder.CreateIsNull(Arg: Ptr);
1225 Count = Builder.CreateSelect(C: IsNull, True: ConstantInt::get(Ty: ResType, V: 0, IsSigned),
1226 False: Count, Name: "count.or.null");
1227 }
1228
1229 // #if ATTR is counted_by || ATTR is counted_by_or_null
1230 // array_element_size = sizeof (*ptr->array);
1231 // array_size = count * array_element_size;
1232 // #else
1233 // array_size = count;
1234 // #endif
1235 Value *ArraySize;
1236 if (!CountAttributedTy->isCountInBytes()) {
1237 // `__counted_by`/`__counted_by_or_null` require a complete pointee at use
1238 // sites (enforced by Sema) so the element size is computable.
1239 CharUnits ArrayElementBaseSize = GetPointeeSize(ArrayBaseFD->getType());
1240
1241 // array_element_size = sizeof (*ptr->array)
1242 auto *ArrayElementSize = llvm::ConstantInt::get(
1243 Ty: ResType, V: ArrayElementBaseSize.getQuantity(), IsSigned);
1244
1245 // array_size = count * array_element_size;
1246 ArraySize = Builder.CreateMul(LHS: Count, RHS: ArrayElementSize, Name: "array_size",
1247 HasNUW: !IsSigned, HasNSW: IsSigned);
1248 } else {
1249 // array_size = count;
1250 ArraySize = Count;
1251 }
1252
1253 // Option (1) 'ptr->array'
1254 // result = array_size
1255 Value *Result = ArraySize;
1256
1257 if (Idx) { // Option (2) '&((cast) ptr->array)[idx]'
1258 // FIXME: CastedArrayElementTy is confusingly named. It's actually the base
1259 // expression of the ArraySubscriptExpr, not the element (pointee) type.
1260 CharUnits CastedArrayElementSizeInChars =
1261 GetPointeeSize(CastedArrayElementTy);
1262
1263 // casted_array_element_size = sizeof (*((cast) ptr->array));
1264 auto *CastedArrayElementSize = llvm::ConstantInt::get(
1265 Ty: ResType, V: CastedArrayElementSizeInChars.getQuantity(), IsSigned);
1266
1267 // index_size = index * casted_array_element_size;
1268 Value *IndexSize = Builder.CreateMul(LHS: Index, RHS: CastedArrayElementSize,
1269 Name: "index_size", HasNUW: !IsSigned, HasNSW: IsSigned);
1270
1271 // result = result - index_size;
1272 Result =
1273 Builder.CreateSub(LHS: Result, RHS: IndexSize, Name: "result", HasNUW: !IsSigned, HasNSW: IsSigned);
1274 }
1275
1276 return EmitPositiveResultOrZero(CGF&: *this, Res: Result, Index, ResType, IsSigned);
1277}
1278
1279llvm::Value *CodeGenFunction::emitCountedByMemberSize(
1280 const MemberExpr *ME, const Expr *Idx, llvm::Value *EmittedE,
1281 QualType CastedArrayElementTy, unsigned Type, llvm::IntegerType *ResType) {
1282 const auto *FD = dyn_cast<FieldDecl>(Val: ME->getMemberDecl());
1283 if (!FD)
1284 return nullptr;
1285
1286 // Find the flexible array member and check that it has the __counted_by
1287 // attribute.
1288 ASTContext &Ctx = getContext();
1289 const RecordDecl *RD = FD->getDeclContext()->getOuterLexicalRecordContext();
1290 const FieldDecl *FlexibleArrayMemberFD = nullptr;
1291
1292 if (Decl::isFlexibleArrayMemberLike(
1293 Context: Ctx, D: FD, Ty: FD->getType(), StrictFlexArraysLevel: getLangOpts().getStrictFlexArraysLevel(),
1294 /*IgnoreTemplateOrMacroSubstitution=*/true))
1295 FlexibleArrayMemberFD = FD;
1296 else
1297 FlexibleArrayMemberFD = FindFlexibleArrayMemberField(CGF&: *this, Ctx, RD);
1298
1299 if (!FlexibleArrayMemberFD ||
1300 !FlexibleArrayMemberFD->getType()->isCountAttributedType())
1301 return nullptr;
1302
1303 // Get the 'count' FieldDecl.
1304 const FieldDecl *CountFD = FlexibleArrayMemberFD->findCountedByField();
1305 if (!CountFD)
1306 // Can't find the field referenced by the "counted_by" attribute.
1307 return nullptr;
1308
1309 // Calculate the flexible array member's object size using these formulae.
1310 // (Note: if the calculation is negative, we return 0.):
1311 //
1312 // struct p;
1313 // struct s {
1314 // /* ... */
1315 // int count;
1316 // struct p *array[] __attribute__((counted_by(count)));
1317 // };
1318 //
1319 // 1) 'ptr->array':
1320 //
1321 // count = ptr->count;
1322 //
1323 // flexible_array_member_element_size = sizeof (*ptr->array);
1324 // flexible_array_member_size =
1325 // count * flexible_array_member_element_size;
1326 //
1327 // result = flexible_array_member_size;
1328 //
1329 // cmp = (result >= 0)
1330 // return cmp ? result : 0;
1331 //
1332 // 2) '&((cast) ptr->array)[idx]':
1333 //
1334 // count = ptr->count;
1335 // index = idx;
1336 //
1337 // flexible_array_member_element_size = sizeof (*ptr->array);
1338 // flexible_array_member_size =
1339 // count * flexible_array_member_element_size;
1340 //
1341 // casted_flexible_array_member_element_size =
1342 // sizeof (*((cast) ptr->array));
1343 // index_size = index * casted_flexible_array_member_element_size;
1344 //
1345 // result = flexible_array_member_size - index_size;
1346 //
1347 // cmp = (result >= 0)
1348 // if (index != 0)
1349 // cmp = (cmp && index >= 0)
1350 // return cmp ? result : 0;
1351 //
1352 // 3) '&ptr->field':
1353 //
1354 // count = ptr->count;
1355 // sizeof_struct = sizeof (struct s);
1356 //
1357 // flexible_array_member_element_size = sizeof (*ptr->array);
1358 // flexible_array_member_size =
1359 // count * flexible_array_member_element_size;
1360 //
1361 // field_offset = offsetof (struct s, field);
1362 // offset_diff = sizeof_struct - field_offset;
1363 //
1364 // result = offset_diff + flexible_array_member_size;
1365 //
1366 // cmp = (result >= 0)
1367 // return cmp ? result : 0;
1368 //
1369 // 4) '&((cast) ptr->field_array)[idx]':
1370 //
1371 // count = ptr->count;
1372 // index = idx;
1373 // sizeof_struct = sizeof (struct s);
1374 //
1375 // flexible_array_member_element_size = sizeof (*ptr->array);
1376 // flexible_array_member_size =
1377 // count * flexible_array_member_element_size;
1378 //
1379 // casted_field_element_size = sizeof (*((cast) ptr->field_array));
1380 // field_offset = offsetof (struct s, field)
1381 // field_offset += index * casted_field_element_size;
1382 //
1383 // offset_diff = sizeof_struct - field_offset;
1384 //
1385 // result = offset_diff + flexible_array_member_size;
1386 //
1387 // cmp = (result >= 0)
1388 // if (index != 0)
1389 // cmp = (cmp && index >= 0)
1390 // return cmp ? result : 0;
1391
1392 bool IsSigned = CountFD->getType()->isSignedIntegerType();
1393
1394 QualType FlexibleArrayMemberTy = FlexibleArrayMemberFD->getType();
1395
1396 // Explicit cast because otherwise the CharWidth will promote an i32's into
1397 // u64's leading to overflows.
1398 int64_t CharWidth = static_cast<int64_t>(CGM.getContext().getCharWidth());
1399
1400 // field_offset = offsetof (struct s, field);
1401 Value *FieldOffset = nullptr;
1402 if (FlexibleArrayMemberFD != FD) {
1403 std::optional<int64_t> Offset = GetFieldOffset(Ctx, RD, FD);
1404 if (!Offset)
1405 return nullptr;
1406 FieldOffset =
1407 llvm::ConstantInt::get(Ty: ResType, V: *Offset / CharWidth, IsSigned);
1408 }
1409
1410 // count = ptr->count;
1411 // index = ptr->index;
1412 Value *Count, *Index;
1413 std::tie(args&: Count, args&: Index) = GetCountFieldAndIndex(
1414 CGF&: *this, ME, ArrayFD: FlexibleArrayMemberFD, CountFD, Idx, ResType, IsSigned);
1415 if (!Count)
1416 return nullptr;
1417
1418 // flexible_array_member_element_size = sizeof (*ptr->array);
1419 const ArrayType *ArrayTy = Ctx.getAsArrayType(T: FlexibleArrayMemberTy);
1420 CharUnits BaseSize = Ctx.getTypeSizeInChars(T: ArrayTy->getElementType());
1421 auto *FlexibleArrayMemberElementSize =
1422 llvm::ConstantInt::get(Ty: ResType, V: BaseSize.getQuantity(), IsSigned);
1423
1424 // flexible_array_member_size = count * flexible_array_member_element_size;
1425 Value *FlexibleArrayMemberSize =
1426 Builder.CreateMul(LHS: Count, RHS: FlexibleArrayMemberElementSize,
1427 Name: "flexible_array_member_size", HasNUW: !IsSigned, HasNSW: IsSigned);
1428
1429 Value *Result = nullptr;
1430 if (FlexibleArrayMemberFD == FD) {
1431 if (Idx) { // Option (2) '&((cast) ptr->array)[idx]'
1432 // casted_flexible_array_member_element_size =
1433 // sizeof (*((cast) ptr->array));
1434 llvm::ConstantInt *CastedFlexibleArrayMemberElementSize =
1435 FlexibleArrayMemberElementSize;
1436 if (!CastedArrayElementTy.isNull() &&
1437 CastedArrayElementTy->isPointerType()) {
1438 CharUnits BaseSize =
1439 Ctx.getTypeSizeInChars(T: CastedArrayElementTy->getPointeeType());
1440 CastedFlexibleArrayMemberElementSize =
1441 llvm::ConstantInt::get(Ty: ResType, V: BaseSize.getQuantity(), IsSigned);
1442 }
1443
1444 // index_size = index * casted_flexible_array_member_element_size;
1445 Value *IndexSize =
1446 Builder.CreateMul(LHS: Index, RHS: CastedFlexibleArrayMemberElementSize,
1447 Name: "index_size", HasNUW: !IsSigned, HasNSW: IsSigned);
1448
1449 // result = flexible_array_member_size - index_size;
1450 Result = Builder.CreateSub(LHS: FlexibleArrayMemberSize, RHS: IndexSize, Name: "result",
1451 HasNUW: !IsSigned, HasNSW: IsSigned);
1452 } else { // Option (1) 'ptr->array'
1453 // result = flexible_array_member_size;
1454 Result = FlexibleArrayMemberSize;
1455 }
1456 } else {
1457 // sizeof_struct = sizeof (struct s);
1458 llvm::StructType *StructTy = getTypes().getCGRecordLayout(RD).getLLVMType();
1459 const llvm::DataLayout &Layout = CGM.getDataLayout();
1460 TypeSize Size = Layout.getTypeSizeInBits(Ty: StructTy);
1461 Value *SizeofStruct =
1462 llvm::ConstantInt::get(Ty: ResType, V: Size.getKnownMinValue() / CharWidth);
1463
1464 if (Idx) { // Option (4) '&((cast) ptr->field_array)[idx]'
1465 // casted_field_element_size = sizeof (*((cast) ptr->field_array));
1466 CharUnits BaseSize;
1467 if (!CastedArrayElementTy.isNull() &&
1468 CastedArrayElementTy->isPointerType()) {
1469 BaseSize =
1470 Ctx.getTypeSizeInChars(T: CastedArrayElementTy->getPointeeType());
1471 } else {
1472 const ArrayType *ArrayTy = Ctx.getAsArrayType(T: FD->getType());
1473 BaseSize = Ctx.getTypeSizeInChars(T: ArrayTy->getElementType());
1474 }
1475
1476 llvm::ConstantInt *CastedFieldElementSize =
1477 llvm::ConstantInt::get(Ty: ResType, V: BaseSize.getQuantity(), IsSigned);
1478
1479 // field_offset += index * casted_field_element_size;
1480 Value *Mul = Builder.CreateMul(LHS: Index, RHS: CastedFieldElementSize,
1481 Name: "field_offset", HasNUW: !IsSigned, HasNSW: IsSigned);
1482 FieldOffset = Builder.CreateAdd(LHS: FieldOffset, RHS: Mul);
1483 }
1484 // Option (3) '&ptr->field', and Option (4) continuation.
1485 // offset_diff = flexible_array_member_offset - field_offset;
1486 Value *OffsetDiff = Builder.CreateSub(LHS: SizeofStruct, RHS: FieldOffset,
1487 Name: "offset_diff", HasNUW: !IsSigned, HasNSW: IsSigned);
1488
1489 // result = offset_diff + flexible_array_member_size;
1490 Result = Builder.CreateAdd(LHS: FlexibleArrayMemberSize, RHS: OffsetDiff, Name: "result");
1491 }
1492
1493 return EmitPositiveResultOrZero(CGF&: *this, Res: Result, Index, ResType, IsSigned);
1494}
1495
1496/// Returns a Value corresponding to the size of the given expression.
1497/// This Value may be either of the following:
1498/// - A llvm::Argument (if E is a param with the pass_object_size attribute on
1499/// it)
1500/// - A call to the @llvm.objectsize intrinsic
1501///
1502/// EmittedE is the result of emitting `E` as a scalar expr. If it's non-null
1503/// and we wouldn't otherwise try to reference a pass_object_size parameter,
1504/// we'll call @llvm.objectsize on EmittedE, rather than emitting E.
1505llvm::Value *
1506CodeGenFunction::emitBuiltinObjectSize(const Expr *E, unsigned Type,
1507 llvm::IntegerType *ResType,
1508 llvm::Value *EmittedE, bool IsDynamic) {
1509 // We need to reference an argument if the pointer is a parameter with the
1510 // pass_object_size attribute.
1511 if (auto *D = dyn_cast<DeclRefExpr>(Val: E->IgnoreParenImpCasts())) {
1512 auto *Param = dyn_cast<ParmVarDecl>(Val: D->getDecl());
1513 auto *PS = D->getDecl()->getAttr<PassObjectSizeAttr>();
1514 if (Param != nullptr && PS != nullptr &&
1515 areBOSTypesCompatible(From: PS->getType(), To: Type)) {
1516 auto Iter = SizeArguments.find(Val: Param);
1517 assert(Iter != SizeArguments.end());
1518
1519 const ImplicitParamDecl *D = Iter->second;
1520 auto DIter = LocalDeclMap.find(Val: D);
1521 assert(DIter != LocalDeclMap.end());
1522
1523 return EmitLoadOfScalar(Addr: DIter->second, /*Volatile=*/false,
1524 Ty: getContext().getSizeType(), Loc: E->getBeginLoc());
1525 }
1526 }
1527
1528 // LLVM can't handle Type=3 appropriately, and __builtin_object_size shouldn't
1529 // evaluate E for side-effects. In either case, we shouldn't lower to
1530 // @llvm.objectsize.
1531 if (Type == 3 || (!EmittedE && E->HasSideEffects(Ctx: getContext())))
1532 return getDefaultBuiltinObjectSizeResult(Type, ResType);
1533
1534 Value *Ptr = EmittedE ? EmittedE : EmitScalarExpr(E);
1535 assert(Ptr->getType()->isPointerTy() &&
1536 "Non-pointer passed to __builtin_object_size?");
1537
1538 if (IsDynamic)
1539 // Emit special code for a flexible array member with the "counted_by"
1540 // attribute.
1541 if (Value *V = emitCountedBySize(E, EmittedE: Ptr, Type, ResType))
1542 return V;
1543
1544 Function *F =
1545 CGM.getIntrinsic(IID: Intrinsic::objectsize, Tys: {ResType, Ptr->getType()});
1546
1547 // LLVM only supports 0 and 2, make sure that we pass along that as a boolean.
1548 Value *Min = Builder.getInt1(V: (Type & 2) != 0);
1549 // For GCC compatibility, __builtin_object_size treat NULL as unknown size.
1550 Value *NullIsUnknown = Builder.getTrue();
1551 Value *Dynamic = Builder.getInt1(V: IsDynamic);
1552 return Builder.CreateCall(Callee: F, Args: {Ptr, Min, NullIsUnknown, Dynamic});
1553}
1554
1555namespace {
1556/// A struct to generically describe a bit test intrinsic.
1557struct BitTest {
1558 enum ActionKind : uint8_t { TestOnly, Complement, Reset, Set };
1559 enum InterlockingKind : uint8_t {
1560 Unlocked,
1561 Sequential,
1562 Acquire,
1563 Release,
1564 NoFence
1565 };
1566
1567 ActionKind Action;
1568 InterlockingKind Interlocking;
1569 bool Is64Bit;
1570
1571 static BitTest decodeBitTestBuiltin(unsigned BuiltinID);
1572};
1573
1574} // namespace
1575
1576BitTest BitTest::decodeBitTestBuiltin(unsigned BuiltinID) {
1577 switch (BuiltinID) {
1578 // Main portable variants.
1579 case Builtin::BI_bittest:
1580 return {.Action: TestOnly, .Interlocking: Unlocked, .Is64Bit: false};
1581 case Builtin::BI_bittestandcomplement:
1582 return {.Action: Complement, .Interlocking: Unlocked, .Is64Bit: false};
1583 case Builtin::BI_bittestandreset:
1584 return {.Action: Reset, .Interlocking: Unlocked, .Is64Bit: false};
1585 case Builtin::BI_bittestandset:
1586 return {.Action: Set, .Interlocking: Unlocked, .Is64Bit: false};
1587 case Builtin::BI_interlockedbittestandreset:
1588 return {.Action: Reset, .Interlocking: Sequential, .Is64Bit: false};
1589 case Builtin::BI_interlockedbittestandset:
1590 return {.Action: Set, .Interlocking: Sequential, .Is64Bit: false};
1591
1592 // 64-bit variants.
1593 case Builtin::BI_bittest64:
1594 return {.Action: TestOnly, .Interlocking: Unlocked, .Is64Bit: true};
1595 case Builtin::BI_bittestandcomplement64:
1596 return {.Action: Complement, .Interlocking: Unlocked, .Is64Bit: true};
1597 case Builtin::BI_bittestandreset64:
1598 return {.Action: Reset, .Interlocking: Unlocked, .Is64Bit: true};
1599 case Builtin::BI_bittestandset64:
1600 return {.Action: Set, .Interlocking: Unlocked, .Is64Bit: true};
1601 case Builtin::BI_interlockedbittestandreset64:
1602 return {.Action: Reset, .Interlocking: Sequential, .Is64Bit: true};
1603 case Builtin::BI_interlockedbittestandset64:
1604 return {.Action: Set, .Interlocking: Sequential, .Is64Bit: true};
1605
1606 // ARM/AArch64-specific ordering variants.
1607 case Builtin::BI_interlockedbittestandset_acq:
1608 return {.Action: Set, .Interlocking: Acquire, .Is64Bit: false};
1609 case Builtin::BI_interlockedbittestandset_rel:
1610 return {.Action: Set, .Interlocking: Release, .Is64Bit: false};
1611 case Builtin::BI_interlockedbittestandset_nf:
1612 return {.Action: Set, .Interlocking: NoFence, .Is64Bit: false};
1613 case Builtin::BI_interlockedbittestandreset_acq:
1614 return {.Action: Reset, .Interlocking: Acquire, .Is64Bit: false};
1615 case Builtin::BI_interlockedbittestandreset_rel:
1616 return {.Action: Reset, .Interlocking: Release, .Is64Bit: false};
1617 case Builtin::BI_interlockedbittestandreset_nf:
1618 return {.Action: Reset, .Interlocking: NoFence, .Is64Bit: false};
1619 case Builtin::BI_interlockedbittestandreset64_acq:
1620 return {.Action: Reset, .Interlocking: Acquire, .Is64Bit: false};
1621 case Builtin::BI_interlockedbittestandreset64_rel:
1622 return {.Action: Reset, .Interlocking: Release, .Is64Bit: false};
1623 case Builtin::BI_interlockedbittestandreset64_nf:
1624 return {.Action: Reset, .Interlocking: NoFence, .Is64Bit: false};
1625 case Builtin::BI_interlockedbittestandset64_acq:
1626 return {.Action: Set, .Interlocking: Acquire, .Is64Bit: false};
1627 case Builtin::BI_interlockedbittestandset64_rel:
1628 return {.Action: Set, .Interlocking: Release, .Is64Bit: false};
1629 case Builtin::BI_interlockedbittestandset64_nf:
1630 return {.Action: Set, .Interlocking: NoFence, .Is64Bit: false};
1631 }
1632 llvm_unreachable("expected only bittest intrinsics");
1633}
1634
1635static char bitActionToX86BTCode(BitTest::ActionKind A) {
1636 switch (A) {
1637 case BitTest::TestOnly: return '\0';
1638 case BitTest::Complement: return 'c';
1639 case BitTest::Reset: return 'r';
1640 case BitTest::Set: return 's';
1641 }
1642 llvm_unreachable("invalid action");
1643}
1644
1645static llvm::Value *EmitX86BitTestIntrinsic(CodeGenFunction &CGF,
1646 BitTest BT,
1647 const CallExpr *E, Value *BitBase,
1648 Value *BitPos) {
1649 char Action = bitActionToX86BTCode(A: BT.Action);
1650 char SizeSuffix = BT.Is64Bit ? 'q' : 'l';
1651
1652 // Build the assembly.
1653 SmallString<64> Asm;
1654 raw_svector_ostream AsmOS(Asm);
1655 if (BT.Interlocking != BitTest::Unlocked)
1656 AsmOS << "lock ";
1657 AsmOS << "bt";
1658 if (Action)
1659 AsmOS << Action;
1660 AsmOS << SizeSuffix << " $2, ($1)";
1661
1662 // Build the constraints. FIXME: We should support immediates when possible.
1663 std::string Constraints = "={@ccc},r,r,~{cc},~{memory}";
1664 std::string_view MachineClobbers = CGF.getTarget().getClobbers();
1665 if (!MachineClobbers.empty()) {
1666 Constraints += ',';
1667 Constraints += MachineClobbers;
1668 }
1669 llvm::IntegerType *IntType = llvm::IntegerType::get(
1670 C&: CGF.getLLVMContext(),
1671 NumBits: CGF.getContext().getTypeSize(T: E->getArg(Arg: 1)->getType()));
1672 llvm::FunctionType *FTy =
1673 llvm::FunctionType::get(Result: CGF.Int8Ty, Params: {CGF.DefaultPtrTy, IntType}, isVarArg: false);
1674
1675 llvm::InlineAsm *IA =
1676 llvm::InlineAsm::get(Ty: FTy, AsmString: Asm, Constraints, /*hasSideEffects=*/true);
1677 return CGF.Builder.CreateCall(Callee: IA, Args: {BitBase, BitPos});
1678}
1679
1680static llvm::AtomicOrdering
1681getBitTestAtomicOrdering(BitTest::InterlockingKind I) {
1682 switch (I) {
1683 case BitTest::Unlocked: return llvm::AtomicOrdering::NotAtomic;
1684 case BitTest::Sequential: return llvm::AtomicOrdering::SequentiallyConsistent;
1685 case BitTest::Acquire: return llvm::AtomicOrdering::Acquire;
1686 case BitTest::Release: return llvm::AtomicOrdering::Release;
1687 case BitTest::NoFence: return llvm::AtomicOrdering::Monotonic;
1688 }
1689 llvm_unreachable("invalid interlocking");
1690}
1691
1692static llvm::Value *EmitBitCountExpr(CodeGenFunction &CGF, const Expr *E) {
1693 llvm::Value *ArgValue = CGF.EmitScalarExpr(E);
1694 llvm::Type *ArgType = ArgValue->getType();
1695
1696 // Boolean vectors can be casted directly to its bitfield representation. We
1697 // intentionally do not round up to the next power of two size and let LLVM
1698 // handle the trailing bits.
1699 //
1700 // In big endian mode, the bitfield representation has a reversed bit order,
1701 // hence the need to add an operation to reverse it back to the expected
1702 // order.
1703 if (auto *VT = dyn_cast<llvm::FixedVectorType>(Val: ArgType);
1704 VT && VT->getElementType()->isIntegerTy(BitWidth: 1)) {
1705 llvm::Type *StorageType =
1706 llvm::Type::getIntNTy(C&: CGF.getLLVMContext(), N: VT->getNumElements());
1707 ArgValue = CGF.Builder.CreateBitCast(V: ArgValue, DestTy: StorageType);
1708
1709 if (CGF.getTarget().isBigEndian())
1710 ArgValue = CGF.Builder.CreateIntrinsic(ID: Intrinsic::bitreverse,
1711 OverloadTypes: {StorageType}, Args: ArgValue);
1712 }
1713
1714 return ArgValue;
1715}
1716
1717/// Emit a _bittest* intrinsic. These intrinsics take a pointer to an array of
1718/// bits and a bit position and read and optionally modify the bit at that
1719/// position. The position index can be arbitrarily large, i.e. it can be larger
1720/// than 31 or 63, so we need an indexed load in the general case.
1721static llvm::Value *EmitBitTestIntrinsic(CodeGenFunction &CGF,
1722 unsigned BuiltinID,
1723 const CallExpr *E) {
1724 Value *BitBase = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
1725 Value *BitPos = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
1726
1727 BitTest BT = BitTest::decodeBitTestBuiltin(BuiltinID);
1728
1729 // X86 has special BT, BTC, BTR, and BTS instructions that handle the array
1730 // indexing operation internally. Use them if possible.
1731 if (CGF.getTarget().getTriple().isX86())
1732 return EmitX86BitTestIntrinsic(CGF, BT, E, BitBase, BitPos);
1733
1734 // Otherwise, use generic code to load one byte and test the bit. Use all but
1735 // the bottom three bits as the array index, and the bottom three bits to form
1736 // a mask.
1737 // Bit = BitBaseI8[BitPos >> 3] & (1 << (BitPos & 0x7)) != 0;
1738 Value *ByteIndex = CGF.Builder.CreateAShr(
1739 LHS: BitPos, RHS: llvm::ConstantInt::get(Ty: BitPos->getType(), V: 3), Name: "bittest.byteidx");
1740 Address ByteAddr(CGF.Builder.CreateInBoundsGEP(Ty: CGF.Int8Ty, Ptr: BitBase, IdxList: ByteIndex,
1741 Name: "bittest.byteaddr"),
1742 CGF.Int8Ty, CharUnits::One());
1743 Value *PosLow =
1744 CGF.Builder.CreateAnd(LHS: CGF.Builder.CreateTrunc(V: BitPos, DestTy: CGF.Int8Ty),
1745 RHS: llvm::ConstantInt::get(Ty: CGF.Int8Ty, V: 0x7));
1746
1747 // The updating instructions will need a mask.
1748 Value *Mask = nullptr;
1749 if (BT.Action != BitTest::TestOnly) {
1750 Mask = CGF.Builder.CreateShl(LHS: llvm::ConstantInt::get(Ty: CGF.Int8Ty, V: 1), RHS: PosLow,
1751 Name: "bittest.mask");
1752 }
1753
1754 // Check the action and ordering of the interlocked intrinsics.
1755 llvm::AtomicOrdering Ordering = getBitTestAtomicOrdering(I: BT.Interlocking);
1756
1757 Value *OldByte = nullptr;
1758 if (Ordering != llvm::AtomicOrdering::NotAtomic) {
1759 // Emit a combined atomicrmw load/store operation for the interlocked
1760 // intrinsics.
1761 llvm::AtomicRMWInst::BinOp RMWOp = llvm::AtomicRMWInst::Or;
1762 if (BT.Action == BitTest::Reset) {
1763 Mask = CGF.Builder.CreateNot(V: Mask);
1764 RMWOp = llvm::AtomicRMWInst::And;
1765 }
1766 OldByte = CGF.Builder.CreateAtomicRMW(Op: RMWOp, Addr: ByteAddr, Val: Mask, Ordering);
1767 } else {
1768 // Emit a plain load for the non-interlocked intrinsics.
1769 OldByte = CGF.Builder.CreateLoad(Addr: ByteAddr, Name: "bittest.byte");
1770 Value *NewByte = nullptr;
1771 switch (BT.Action) {
1772 case BitTest::TestOnly:
1773 // Don't store anything.
1774 break;
1775 case BitTest::Complement:
1776 NewByte = CGF.Builder.CreateXor(LHS: OldByte, RHS: Mask);
1777 break;
1778 case BitTest::Reset:
1779 NewByte = CGF.Builder.CreateAnd(LHS: OldByte, RHS: CGF.Builder.CreateNot(V: Mask));
1780 break;
1781 case BitTest::Set:
1782 NewByte = CGF.Builder.CreateOr(LHS: OldByte, RHS: Mask);
1783 break;
1784 }
1785 if (NewByte)
1786 CGF.Builder.CreateStore(Val: NewByte, Addr: ByteAddr);
1787 }
1788
1789 // However we loaded the old byte, either by plain load or atomicrmw, shift
1790 // the bit into the low position and mask it to 0 or 1.
1791 Value *ShiftedByte = CGF.Builder.CreateLShr(LHS: OldByte, RHS: PosLow, Name: "bittest.shr");
1792 return CGF.Builder.CreateAnd(
1793 LHS: ShiftedByte, RHS: llvm::ConstantInt::get(Ty: CGF.Int8Ty, V: 1), Name: "bittest.res");
1794}
1795
1796namespace {
1797enum class MSVCSetJmpKind {
1798 _setjmpex,
1799 _setjmp3,
1800 _setjmp
1801};
1802}
1803
1804/// MSVC handles setjmp a bit differently on different platforms. On 32-bit x86
1805/// extra parameters can be passed as variadic arguments, but we always pass
1806/// none. Everywhere else a frame value is passed: the stack pointer as it was
1807/// on entry to the function for AArch64 and 32-bit Arm, and the frame address
1808/// for the rest.
1809static RValue EmitMSVCRTSetJmp(CodeGenFunction &CGF, MSVCSetJmpKind SJKind,
1810 const CallExpr *E) {
1811 llvm::Value *Arg1 = nullptr;
1812 llvm::Type *Arg1Ty = nullptr;
1813 StringRef Name;
1814 bool IsVarArg = false;
1815 if (SJKind == MSVCSetJmpKind::_setjmp3) {
1816 Name = "_setjmp3";
1817 Arg1Ty = CGF.Int32Ty;
1818 Arg1 = llvm::ConstantInt::get(Ty: CGF.IntTy, V: 0);
1819 IsVarArg = true;
1820 } else {
1821 Name = SJKind == MSVCSetJmpKind::_setjmp ? "_setjmp" : "_setjmpex";
1822 Arg1Ty = CGF.Int8PtrTy;
1823 const llvm::Triple &T = CGF.getTarget().getTriple();
1824 if (T.getArch() == llvm::Triple::aarch64 || T.isARM() || T.isThumb()) {
1825 Arg1 = CGF.Builder.CreateCall(
1826 Callee: CGF.CGM.getIntrinsic(IID: Intrinsic::sponentry, Tys: CGF.AllocaInt8PtrTy));
1827 } else
1828 Arg1 = CGF.Builder.CreateCall(
1829 Callee: CGF.CGM.getIntrinsic(IID: Intrinsic::frameaddress, Tys: CGF.AllocaInt8PtrTy),
1830 Args: llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: 0));
1831 }
1832
1833 // Mark the call site and declaration with ReturnsTwice.
1834 llvm::Type *ArgTypes[2] = {CGF.Int8PtrTy, Arg1Ty};
1835 llvm::AttributeList ReturnsTwiceAttr = llvm::AttributeList::get(
1836 C&: CGF.getLLVMContext(), Index: llvm::AttributeList::FunctionIndex,
1837 Kinds: llvm::Attribute::ReturnsTwice);
1838 llvm::FunctionCallee SetJmpFn = CGF.CGM.CreateRuntimeFunction(
1839 Ty: llvm::FunctionType::get(Result: CGF.IntTy, Params: ArgTypes, isVarArg: IsVarArg), Name,
1840 ExtraAttrs: ReturnsTwiceAttr, /*Local=*/true);
1841
1842 llvm::Value *Buf = CGF.Builder.CreateBitOrPointerCast(
1843 V: CGF.EmitScalarExpr(E: E->getArg(Arg: 0)), DestTy: CGF.Int8PtrTy);
1844 llvm::Value *Args[] = {Buf, Arg1};
1845 llvm::CallBase *CB = CGF.EmitRuntimeCallOrInvoke(callee: SetJmpFn, args: Args);
1846 CB->setAttributes(ReturnsTwiceAttr);
1847 return RValue::get(V: CB);
1848}
1849
1850// Emit an MSVC intrinsic. Assumes that arguments have *not* been evaluated.
1851Value *CodeGenFunction::EmitMSVCBuiltinExpr(MSVCIntrin BuiltinID,
1852 const CallExpr *E) {
1853 switch (BuiltinID) {
1854 case MSVCIntrin::_BitScanForward:
1855 case MSVCIntrin::_BitScanReverse: {
1856 Address IndexAddress(EmitPointerWithAlignment(Addr: E->getArg(Arg: 0)));
1857 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 1));
1858
1859 llvm::Type *ArgType = ArgValue->getType();
1860 llvm::Type *IndexType = IndexAddress.getElementType();
1861 llvm::Type *ResultType = ConvertType(T: E->getType());
1862
1863 Value *ArgZero = llvm::Constant::getNullValue(Ty: ArgType);
1864 Value *ResZero = llvm::Constant::getNullValue(Ty: ResultType);
1865 Value *ResOne = llvm::ConstantInt::get(Ty: ResultType, V: 1);
1866
1867 BasicBlock *Begin = Builder.GetInsertBlock();
1868 BasicBlock *End = createBasicBlock(name: "bitscan_end", parent: this->CurFn);
1869 Builder.SetInsertPoint(End);
1870 PHINode *Result = Builder.CreatePHI(Ty: ResultType, NumReservedValues: 2, Name: "bitscan_result");
1871
1872 Builder.SetInsertPoint(Begin);
1873 Value *IsZero = Builder.CreateICmpEQ(LHS: ArgValue, RHS: ArgZero);
1874 BasicBlock *NotZero = createBasicBlock(name: "bitscan_not_zero", parent: this->CurFn);
1875 Builder.CreateCondBr(Cond: IsZero, True: End, False: NotZero);
1876 Result->addIncoming(V: ResZero, BB: Begin);
1877
1878 Builder.SetInsertPoint(NotZero);
1879
1880 if (BuiltinID == MSVCIntrin::_BitScanForward) {
1881 Function *F = CGM.getIntrinsic(IID: Intrinsic::cttz, Tys: ArgType);
1882 Value *ZeroCount = Builder.CreateCall(Callee: F, Args: {ArgValue, Builder.getTrue()});
1883 ZeroCount = Builder.CreateIntCast(V: ZeroCount, DestTy: IndexType, isSigned: false);
1884 Builder.CreateStore(Val: ZeroCount, Addr: IndexAddress, IsVolatile: false);
1885 } else {
1886 unsigned ArgWidth = cast<llvm::IntegerType>(Val: ArgType)->getBitWidth();
1887 Value *ArgTypeLastIndex = llvm::ConstantInt::get(Ty: IndexType, V: ArgWidth - 1);
1888
1889 Function *F = CGM.getIntrinsic(IID: Intrinsic::ctlz, Tys: ArgType);
1890 Value *ZeroCount = Builder.CreateCall(Callee: F, Args: {ArgValue, Builder.getTrue()});
1891 ZeroCount = Builder.CreateIntCast(V: ZeroCount, DestTy: IndexType, isSigned: false);
1892 Value *Index = Builder.CreateNSWSub(LHS: ArgTypeLastIndex, RHS: ZeroCount);
1893 Builder.CreateStore(Val: Index, Addr: IndexAddress, IsVolatile: false);
1894 }
1895 Builder.CreateBr(Dest: End);
1896 Result->addIncoming(V: ResOne, BB: NotZero);
1897
1898 Builder.SetInsertPoint(End);
1899 return Result;
1900 }
1901 case MSVCIntrin::_InterlockedAnd:
1902 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::And, E);
1903 case MSVCIntrin::_InterlockedExchange:
1904 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Xchg, E);
1905 case MSVCIntrin::_InterlockedExchangeAdd:
1906 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Add, E);
1907 case MSVCIntrin::_InterlockedExchangeSub:
1908 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Sub, E);
1909 case MSVCIntrin::_InterlockedOr:
1910 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Or, E);
1911 case MSVCIntrin::_InterlockedXor:
1912 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Xor, E);
1913 case MSVCIntrin::_InterlockedExchangeAdd_acq:
1914 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Add, E,
1915 Ordering: AtomicOrdering::Acquire);
1916 case MSVCIntrin::_InterlockedExchangeAdd_rel:
1917 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Add, E,
1918 Ordering: AtomicOrdering::Release);
1919 case MSVCIntrin::_InterlockedExchangeAdd_nf:
1920 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Add, E,
1921 Ordering: AtomicOrdering::Monotonic);
1922 case MSVCIntrin::_InterlockedExchange_acq:
1923 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Xchg, E,
1924 Ordering: AtomicOrdering::Acquire);
1925 case MSVCIntrin::_InterlockedExchange_rel:
1926 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Xchg, E,
1927 Ordering: AtomicOrdering::Release);
1928 case MSVCIntrin::_InterlockedExchange_nf:
1929 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Xchg, E,
1930 Ordering: AtomicOrdering::Monotonic);
1931 case MSVCIntrin::_InterlockedCompareExchange:
1932 return EmitAtomicCmpXchgForMSIntrin(CGF&: *this, E);
1933 case MSVCIntrin::_InterlockedCompareExchange_acq:
1934 return EmitAtomicCmpXchgForMSIntrin(CGF&: *this, E, SuccessOrdering: AtomicOrdering::Acquire);
1935 case MSVCIntrin::_InterlockedCompareExchange_rel:
1936 return EmitAtomicCmpXchgForMSIntrin(CGF&: *this, E, SuccessOrdering: AtomicOrdering::Release);
1937 case MSVCIntrin::_InterlockedCompareExchange_nf:
1938 return EmitAtomicCmpXchgForMSIntrin(CGF&: *this, E, SuccessOrdering: AtomicOrdering::Monotonic);
1939 case MSVCIntrin::_InterlockedCompareExchange128:
1940 return EmitAtomicCmpXchg128ForMSIntrin(
1941 CGF&: *this, E, SuccessOrdering: AtomicOrdering::SequentiallyConsistent);
1942 case MSVCIntrin::_InterlockedCompareExchange128_acq:
1943 return EmitAtomicCmpXchg128ForMSIntrin(CGF&: *this, E, SuccessOrdering: AtomicOrdering::Acquire);
1944 case MSVCIntrin::_InterlockedCompareExchange128_rel:
1945 return EmitAtomicCmpXchg128ForMSIntrin(CGF&: *this, E, SuccessOrdering: AtomicOrdering::Release);
1946 case MSVCIntrin::_InterlockedCompareExchange128_nf:
1947 return EmitAtomicCmpXchg128ForMSIntrin(CGF&: *this, E, SuccessOrdering: AtomicOrdering::Monotonic);
1948 case MSVCIntrin::_InterlockedOr_acq:
1949 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Or, E,
1950 Ordering: AtomicOrdering::Acquire);
1951 case MSVCIntrin::_InterlockedOr_rel:
1952 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Or, E,
1953 Ordering: AtomicOrdering::Release);
1954 case MSVCIntrin::_InterlockedOr_nf:
1955 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Or, E,
1956 Ordering: AtomicOrdering::Monotonic);
1957 case MSVCIntrin::_InterlockedXor_acq:
1958 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Xor, E,
1959 Ordering: AtomicOrdering::Acquire);
1960 case MSVCIntrin::_InterlockedXor_rel:
1961 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Xor, E,
1962 Ordering: AtomicOrdering::Release);
1963 case MSVCIntrin::_InterlockedXor_nf:
1964 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Xor, E,
1965 Ordering: AtomicOrdering::Monotonic);
1966 case MSVCIntrin::_InterlockedAnd_acq:
1967 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::And, E,
1968 Ordering: AtomicOrdering::Acquire);
1969 case MSVCIntrin::_InterlockedAnd_rel:
1970 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::And, E,
1971 Ordering: AtomicOrdering::Release);
1972 case MSVCIntrin::_InterlockedAnd_nf:
1973 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::And, E,
1974 Ordering: AtomicOrdering::Monotonic);
1975 case MSVCIntrin::_InterlockedIncrement_acq:
1976 return EmitAtomicIncrementValue(CGF&: *this, E, Ordering: AtomicOrdering::Acquire);
1977 case MSVCIntrin::_InterlockedIncrement_rel:
1978 return EmitAtomicIncrementValue(CGF&: *this, E, Ordering: AtomicOrdering::Release);
1979 case MSVCIntrin::_InterlockedIncrement_nf:
1980 return EmitAtomicIncrementValue(CGF&: *this, E, Ordering: AtomicOrdering::Monotonic);
1981 case MSVCIntrin::_InterlockedDecrement_acq:
1982 return EmitAtomicDecrementValue(CGF&: *this, E, Ordering: AtomicOrdering::Acquire);
1983 case MSVCIntrin::_InterlockedDecrement_rel:
1984 return EmitAtomicDecrementValue(CGF&: *this, E, Ordering: AtomicOrdering::Release);
1985 case MSVCIntrin::_InterlockedDecrement_nf:
1986 return EmitAtomicDecrementValue(CGF&: *this, E, Ordering: AtomicOrdering::Monotonic);
1987
1988 case MSVCIntrin::_InterlockedDecrement:
1989 return EmitAtomicDecrementValue(CGF&: *this, E);
1990 case MSVCIntrin::_InterlockedIncrement:
1991 return EmitAtomicIncrementValue(CGF&: *this, E);
1992
1993 case MSVCIntrin::__fastfail: {
1994 // Request immediate process termination from the kernel. The instruction
1995 // sequences to do this are documented on MSDN:
1996 // https://msdn.microsoft.com/en-us/library/dn774154.aspx
1997 llvm::Triple::ArchType ISA = getTarget().getTriple().getArch();
1998 StringRef Asm, Constraints;
1999 switch (ISA) {
2000 default:
2001 ErrorUnsupported(S: E, Type: "__fastfail call for this architecture");
2002 break;
2003 case llvm::Triple::x86:
2004 case llvm::Triple::x86_64:
2005 Asm = "int $$0x29";
2006 Constraints = "{cx}";
2007 break;
2008 case llvm::Triple::thumb:
2009 Asm = "udf #251";
2010 Constraints = "{r0}";
2011 break;
2012 case llvm::Triple::aarch64:
2013 Asm = "brk #0xF003";
2014 Constraints = "{w0}";
2015 }
2016 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: VoidTy, Params: {Int32Ty}, isVarArg: false);
2017 llvm::InlineAsm *IA =
2018 llvm::InlineAsm::get(Ty: FTy, AsmString: Asm, Constraints, /*hasSideEffects=*/true);
2019 llvm::AttributeList NoReturnAttr = llvm::AttributeList::get(
2020 C&: getLLVMContext(), Index: llvm::AttributeList::FunctionIndex,
2021 Kinds: llvm::Attribute::NoReturn);
2022 llvm::CallInst *CI = Builder.CreateCall(Callee: IA, Args: EmitScalarExpr(E: E->getArg(Arg: 0)));
2023 CI->setAttributes(NoReturnAttr);
2024 return CI;
2025 }
2026 }
2027 llvm_unreachable("Incorrect MSVC intrinsic!");
2028}
2029
2030namespace {
2031// ARC cleanup for __builtin_os_log_format
2032struct CallObjCArcUse final : EHScopeStack::Cleanup {
2033 CallObjCArcUse(llvm::Value *object) : object(object) {}
2034 llvm::Value *object;
2035
2036 void Emit(CodeGenFunction &CGF, Flags flags) override {
2037 CGF.EmitARCIntrinsicUse(values: object);
2038 }
2039};
2040}
2041
2042Value *CodeGenFunction::EmitCheckedArgForBuiltin(const Expr *E,
2043 BuiltinCheckKind Kind) {
2044 assert((Kind == BCK_CLZPassedZero || Kind == BCK_CTZPassedZero) &&
2045 "Unsupported builtin check kind");
2046
2047 Value *ArgValue = EmitBitCountExpr(CGF&: *this, E);
2048 if (!SanOpts.has(K: SanitizerKind::Builtin))
2049 return ArgValue;
2050
2051 auto CheckOrdinal = SanitizerKind::SO_Builtin;
2052 auto CheckHandler = SanitizerHandler::InvalidBuiltin;
2053 SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);
2054 Value *Cond = Builder.CreateICmpNE(
2055 LHS: ArgValue, RHS: llvm::Constant::getNullValue(Ty: ArgValue->getType()));
2056 EmitCheck(Checked: std::make_pair(x&: Cond, y&: CheckOrdinal), Check: CheckHandler,
2057 StaticArgs: {EmitCheckSourceLocation(Loc: E->getExprLoc()),
2058 llvm::ConstantInt::get(Ty: Builder.getInt8Ty(), V: Kind)},
2059 DynamicArgs: {});
2060 return ArgValue;
2061}
2062
2063Value *CodeGenFunction::EmitCheckedArgForAssume(const Expr *E) {
2064 Value *ArgValue = EvaluateExprAsBool(E);
2065 if (!SanOpts.has(K: SanitizerKind::Builtin))
2066 return ArgValue;
2067
2068 auto CheckOrdinal = SanitizerKind::SO_Builtin;
2069 auto CheckHandler = SanitizerHandler::InvalidBuiltin;
2070 SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);
2071 EmitCheck(
2072 Checked: std::make_pair(x&: ArgValue, y&: CheckOrdinal), Check: CheckHandler,
2073 StaticArgs: {EmitCheckSourceLocation(Loc: E->getExprLoc()),
2074 llvm::ConstantInt::get(Ty: Builder.getInt8Ty(), V: BCK_AssumePassedFalse)},
2075 DynamicArgs: {});
2076 return ArgValue;
2077}
2078
2079static Value *EmitAbs(CodeGenFunction &CGF, Value *ArgValue, bool HasNSW) {
2080 return CGF.Builder.CreateBinaryIntrinsic(
2081 ID: Intrinsic::abs, LHS: ArgValue,
2082 RHS: ConstantInt::get(Ty: CGF.Builder.getInt1Ty(), V: HasNSW));
2083}
2084
2085static Value *EmitOverflowCheckedAbs(CodeGenFunction &CGF, const CallExpr *E,
2086 bool SanitizeOverflow) {
2087 Value *ArgValue = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
2088
2089 // Try to eliminate overflow check.
2090 if (const auto *VCI = dyn_cast<llvm::ConstantInt>(Val: ArgValue)) {
2091 if (!VCI->isMinSignedValue())
2092 return EmitAbs(CGF, ArgValue, HasNSW: true);
2093 }
2094
2095 SmallVector<SanitizerKind::SanitizerOrdinal, 1> Ordinals;
2096 SanitizerHandler CheckHandler;
2097 if (SanitizeOverflow) {
2098 Ordinals.push_back(Elt: SanitizerKind::SO_SignedIntegerOverflow);
2099 CheckHandler = SanitizerHandler::NegateOverflow;
2100 } else
2101 CheckHandler = SanitizerHandler::SubOverflow;
2102
2103 SanitizerDebugLocation SanScope(&CGF, Ordinals, CheckHandler);
2104
2105 Constant *Zero = Constant::getNullValue(Ty: ArgValue->getType());
2106 Value *ResultAndOverflow = CGF.Builder.CreateBinaryIntrinsic(
2107 ID: Intrinsic::ssub_with_overflow, LHS: Zero, RHS: ArgValue);
2108 Value *Result = CGF.Builder.CreateExtractValue(Agg: ResultAndOverflow, Idxs: 0);
2109 Value *NotOverflow = CGF.Builder.CreateNot(
2110 V: CGF.Builder.CreateExtractValue(Agg: ResultAndOverflow, Idxs: 1));
2111
2112 // TODO: support -ftrapv-handler.
2113 if (SanitizeOverflow) {
2114 CGF.EmitCheck(Checked: {{NotOverflow, SanitizerKind::SO_SignedIntegerOverflow}},
2115 Check: CheckHandler,
2116 StaticArgs: {CGF.EmitCheckSourceLocation(Loc: E->getArg(Arg: 0)->getExprLoc()),
2117 CGF.EmitCheckTypeDescriptor(T: E->getType())},
2118 DynamicArgs: {ArgValue});
2119 } else
2120 CGF.EmitTrapCheck(Checked: NotOverflow, CheckHandlerID: CheckHandler);
2121
2122 Value *CmpResult = CGF.Builder.CreateICmpSLT(LHS: ArgValue, RHS: Zero, Name: "abscond");
2123 return CGF.Builder.CreateSelect(C: CmpResult, True: Result, False: ArgValue, Name: "abs");
2124}
2125
2126/// Get the argument type for arguments to os_log_helper.
2127static CanQualType getOSLogArgType(ASTContext &C, int Size) {
2128 QualType UnsignedTy = C.getIntTypeForBitwidth(DestWidth: Size * 8, /*Signed=*/false);
2129 return C.getCanonicalType(T: UnsignedTy);
2130}
2131
2132llvm::Function *CodeGenFunction::generateBuiltinOSLogHelperFunction(
2133 const analyze_os_log::OSLogBufferLayout &Layout,
2134 CharUnits BufferAlignment) {
2135 ASTContext &Ctx = getContext();
2136
2137 llvm::SmallString<64> Name;
2138 {
2139 raw_svector_ostream OS(Name);
2140 OS << "__os_log_helper";
2141 OS << "_" << BufferAlignment.getQuantity();
2142 OS << "_" << int(Layout.getSummaryByte());
2143 OS << "_" << int(Layout.getNumArgsByte());
2144 for (const auto &Item : Layout.Items)
2145 OS << "_" << int(Item.getSizeByte()) << "_"
2146 << int(Item.getDescriptorByte());
2147 }
2148
2149 if (llvm::Function *F = CGM.getModule().getFunction(Name))
2150 return F;
2151
2152 llvm::SmallVector<QualType, 4> ArgTys;
2153 FunctionArgList Args;
2154 Args.push_back(Elt: ImplicitParamDecl::Create(
2155 C&: Ctx, DC: nullptr, IdLoc: SourceLocation(), Id: &Ctx.Idents.get(Name: "buffer"), T: Ctx.VoidPtrTy,
2156 ParamKind: ImplicitParamKind::Other));
2157 ArgTys.emplace_back(Args&: Ctx.VoidPtrTy);
2158
2159 for (unsigned int I = 0, E = Layout.Items.size(); I < E; ++I) {
2160 char Size = Layout.Items[I].getSizeByte();
2161 if (!Size)
2162 continue;
2163
2164 QualType ArgTy = getOSLogArgType(C&: Ctx, Size);
2165 Args.push_back(Elt: ImplicitParamDecl::Create(
2166 C&: Ctx, DC: nullptr, IdLoc: SourceLocation(),
2167 Id: &Ctx.Idents.get(Name: std::string("arg") + llvm::to_string(Value: I)), T: ArgTy,
2168 ParamKind: ImplicitParamKind::Other));
2169 ArgTys.emplace_back(Args&: ArgTy);
2170 }
2171
2172 QualType ReturnTy = Ctx.VoidTy;
2173
2174 // The helper function has linkonce_odr linkage to enable the linker to merge
2175 // identical functions. To ensure the merging always happens, 'noinline' is
2176 // attached to the function when compiling with -Oz.
2177 const CGFunctionInfo &FI =
2178 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: ReturnTy, args: Args);
2179 llvm::FunctionType *FuncTy = CGM.getTypes().GetFunctionType(Info: FI);
2180 llvm::Function *Fn = llvm::Function::Create(
2181 Ty: FuncTy, Linkage: llvm::GlobalValue::LinkOnceODRLinkage, N: Name, M: &CGM.getModule());
2182 Fn->setVisibility(llvm::GlobalValue::HiddenVisibility);
2183 CGM.SetLLVMFunctionAttributes(GD: GlobalDecl(), Info: FI, F: Fn, /*IsThunk=*/false);
2184 CGM.SetLLVMFunctionAttributesForDefinition(D: nullptr, F: Fn);
2185 Fn->setDoesNotThrow();
2186
2187 // Attach 'noinline' at -Oz.
2188 if (CGM.getCodeGenOpts().OptimizeSize == 2)
2189 Fn->addFnAttr(Kind: llvm::Attribute::NoInline);
2190
2191 auto NL = ApplyDebugLocation::CreateEmpty(CGF&: *this);
2192 StartFunction(GD: GlobalDecl(), RetTy: ReturnTy, Fn, FnInfo: FI, Args);
2193
2194 // Create a scope with an artificial location for the body of this function.
2195 auto AL = ApplyDebugLocation::CreateArtificial(CGF&: *this);
2196
2197 CharUnits Offset;
2198 Address BufAddr = makeNaturalAddressForPointer(
2199 Ptr: Builder.CreateLoad(Addr: GetAddrOfLocalVar(VD: Args[0]), Name: "buf"), T: Ctx.VoidTy,
2200 Alignment: BufferAlignment);
2201 Builder.CreateStore(Val: Builder.getInt8(C: Layout.getSummaryByte()),
2202 Addr: Builder.CreateConstByteGEP(Addr: BufAddr, Offset: Offset++, Name: "summary"));
2203 Builder.CreateStore(Val: Builder.getInt8(C: Layout.getNumArgsByte()),
2204 Addr: Builder.CreateConstByteGEP(Addr: BufAddr, Offset: Offset++, Name: "numArgs"));
2205
2206 unsigned I = 1;
2207 for (const auto &Item : Layout.Items) {
2208 Builder.CreateStore(
2209 Val: Builder.getInt8(C: Item.getDescriptorByte()),
2210 Addr: Builder.CreateConstByteGEP(Addr: BufAddr, Offset: Offset++, Name: "argDescriptor"));
2211 Builder.CreateStore(
2212 Val: Builder.getInt8(C: Item.getSizeByte()),
2213 Addr: Builder.CreateConstByteGEP(Addr: BufAddr, Offset: Offset++, Name: "argSize"));
2214
2215 CharUnits Size = Item.size();
2216 if (!Size.getQuantity())
2217 continue;
2218
2219 Address Arg = GetAddrOfLocalVar(VD: Args[I]);
2220 Address Addr = Builder.CreateConstByteGEP(Addr: BufAddr, Offset, Name: "argData");
2221 Addr = Addr.withElementType(ElemTy: Arg.getElementType());
2222 Builder.CreateStore(Val: Builder.CreateLoad(Addr: Arg), Addr);
2223 Offset += Size;
2224 ++I;
2225 }
2226
2227 FinishFunction();
2228
2229 return Fn;
2230}
2231
2232RValue CodeGenFunction::emitBuiltinOSLogFormat(const CallExpr &E) {
2233 assert(E.getNumArgs() >= 2 &&
2234 "__builtin_os_log_format takes at least 2 arguments");
2235 ASTContext &Ctx = getContext();
2236 analyze_os_log::OSLogBufferLayout Layout;
2237 analyze_os_log::computeOSLogBufferLayout(Ctx, E: &E, layout&: Layout);
2238 Address BufAddr = EmitPointerWithAlignment(Addr: E.getArg(Arg: 0));
2239
2240 // Ignore argument 1, the format string. It is not currently used.
2241 CallArgList Args;
2242 Args.add(rvalue: RValue::get(V: BufAddr.emitRawPointer(CGF&: *this)), type: Ctx.VoidPtrTy);
2243
2244 for (const auto &Item : Layout.Items) {
2245 int Size = Item.getSizeByte();
2246 if (!Size)
2247 continue;
2248
2249 llvm::Value *ArgVal;
2250
2251 if (Item.getKind() == analyze_os_log::OSLogBufferItem::MaskKind) {
2252 uint64_t Val = 0;
2253 for (unsigned I = 0, E = Item.getMaskType().size(); I < E; ++I)
2254 Val |= ((uint64_t)Item.getMaskType()[I]) << I * 8;
2255 ArgVal = llvm::Constant::getIntegerValue(Ty: Int64Ty, V: llvm::APInt(64, Val));
2256 } else if (const Expr *TheExpr = Item.getExpr()) {
2257 ArgVal = EmitScalarExpr(E: TheExpr, /*Ignore*/ IgnoreResultAssign: false);
2258
2259 // If a temporary object that requires destruction after the full
2260 // expression is passed, push a lifetime-extended cleanup to extend its
2261 // lifetime to the end of the enclosing block scope.
2262 auto LifetimeExtendObject = [&](const Expr *E) {
2263 E = E->IgnoreParenCasts();
2264 // Extend lifetimes of objects returned by function calls and message
2265 // sends.
2266
2267 // FIXME: We should do this in other cases in which temporaries are
2268 // created including arguments of non-ARC types (e.g., C++
2269 // temporaries).
2270 if (isa<CallExpr>(Val: E) || isa<ObjCMessageExpr>(Val: E))
2271 return true;
2272 return false;
2273 };
2274
2275 if (TheExpr->getType()->isObjCRetainableType() &&
2276 getLangOpts().ObjCAutoRefCount && LifetimeExtendObject(TheExpr)) {
2277 assert(getEvaluationKind(TheExpr->getType()) == TEK_Scalar &&
2278 "Only scalar can be a ObjC retainable type");
2279 if (!isa<Constant>(Val: ArgVal)) {
2280 CleanupKind Cleanup = getARCCleanupKind();
2281 QualType Ty = TheExpr->getType();
2282 RawAddress Alloca = CreateMemTempWithoutCast(T: Ty, Name: "os.log.arg");
2283 ArgVal = EmitARCRetain(type: Ty, value: ArgVal);
2284 Builder.CreateStore(Val: ArgVal, Addr: Alloca);
2285 pushLifetimeExtendedDestroy(kind: Cleanup, addr: Alloca, type: Ty,
2286 destroyer: CodeGenFunction::destroyARCStrongPrecise,
2287 useEHCleanupForArray: Cleanup & EHCleanup);
2288
2289 // Push a clang.arc.use call to ensure ARC optimizer knows that the
2290 // argument has to be alive.
2291 if (CGM.getCodeGenOpts().OptimizationLevel != 0)
2292 pushCleanupAfterFullExpr<CallObjCArcUse>(Kind: Cleanup, A: ArgVal);
2293 }
2294 }
2295 } else {
2296 ArgVal = Builder.getInt32(C: Item.getConstValue().getQuantity());
2297 }
2298
2299 unsigned ArgValSize =
2300 CGM.getDataLayout().getTypeSizeInBits(Ty: ArgVal->getType());
2301 llvm::IntegerType *IntTy = llvm::Type::getIntNTy(C&: getLLVMContext(),
2302 N: ArgValSize);
2303 ArgVal = Builder.CreateBitOrPointerCast(V: ArgVal, DestTy: IntTy);
2304 CanQualType ArgTy = getOSLogArgType(C&: Ctx, Size);
2305 // If ArgVal has type x86_fp80, zero-extend ArgVal.
2306 ArgVal = Builder.CreateZExtOrBitCast(V: ArgVal, DestTy: ConvertType(T: ArgTy));
2307 Args.add(rvalue: RValue::get(V: ArgVal), type: ArgTy);
2308 }
2309
2310 const CGFunctionInfo &FI =
2311 CGM.getTypes().arrangeBuiltinFunctionCall(resultType: Ctx.VoidTy, args: Args);
2312 llvm::Function *F = CodeGenFunction(CGM).generateBuiltinOSLogHelperFunction(
2313 Layout, BufferAlignment: BufAddr.getAlignment());
2314 EmitCall(CallInfo: FI, Callee: CGCallee::forDirect(functionPtr: F), ReturnValue: ReturnValueSlot(), Args);
2315 return RValue::get(Addr: BufAddr, CGF&: *this);
2316}
2317
2318static bool isSpecialUnsignedMultiplySignedResult(
2319 unsigned BuiltinID, WidthAndSignedness Op1Info, WidthAndSignedness Op2Info,
2320 WidthAndSignedness ResultInfo) {
2321 return BuiltinID == Builtin::BI__builtin_mul_overflow &&
2322 Op1Info.Width == Op2Info.Width && Op2Info.Width == ResultInfo.Width &&
2323 !Op1Info.Signed && !Op2Info.Signed && ResultInfo.Signed;
2324}
2325
2326static RValue EmitCheckedUnsignedMultiplySignedResult(
2327 CodeGenFunction &CGF, const clang::Expr *Op1, WidthAndSignedness Op1Info,
2328 const clang::Expr *Op2, WidthAndSignedness Op2Info,
2329 const clang::Expr *ResultArg, QualType ResultQTy,
2330 WidthAndSignedness ResultInfo) {
2331 assert(isSpecialUnsignedMultiplySignedResult(
2332 Builtin::BI__builtin_mul_overflow, Op1Info, Op2Info, ResultInfo) &&
2333 "Cannot specialize this multiply");
2334
2335 llvm::Value *V1 = CGF.EmitScalarExpr(E: Op1);
2336 llvm::Value *V2 = CGF.EmitScalarExpr(E: Op2);
2337
2338 llvm::Value *HasOverflow;
2339 llvm::Value *Result = EmitOverflowIntrinsic(
2340 CGF, IntrinsicID: Intrinsic::umul_with_overflow, X: V1, Y: V2, Carry&: HasOverflow);
2341
2342 // The intrinsic call will detect overflow when the value is > UINT_MAX,
2343 // however, since the original builtin had a signed result, we need to report
2344 // an overflow when the result is greater than INT_MAX.
2345 auto IntMax = llvm::APInt::getSignedMaxValue(numBits: ResultInfo.Width);
2346 llvm::Value *IntMaxValue = llvm::ConstantInt::get(Ty: Result->getType(), V: IntMax);
2347
2348 llvm::Value *IntMaxOverflow = CGF.Builder.CreateICmpUGT(LHS: Result, RHS: IntMaxValue);
2349 HasOverflow = CGF.Builder.CreateOr(LHS: HasOverflow, RHS: IntMaxOverflow);
2350
2351 bool isVolatile =
2352 ResultArg->getType()->getPointeeType().isVolatileQualified();
2353 Address ResultPtr = CGF.EmitPointerWithAlignment(Addr: ResultArg);
2354 CGF.Builder.CreateStore(Val: CGF.EmitToMemory(Value: Result, Ty: ResultQTy), Addr: ResultPtr,
2355 IsVolatile: isVolatile);
2356 return RValue::get(V: HasOverflow);
2357}
2358
2359/// Determine if a binop is a checked mixed-sign multiply we can specialize.
2360static bool isSpecialMixedSignMultiply(unsigned BuiltinID,
2361 WidthAndSignedness Op1Info,
2362 WidthAndSignedness Op2Info,
2363 WidthAndSignedness ResultInfo) {
2364 return BuiltinID == Builtin::BI__builtin_mul_overflow &&
2365 std::max(a: Op1Info.Width, b: Op2Info.Width) >= ResultInfo.Width &&
2366 Op1Info.Signed != Op2Info.Signed;
2367}
2368
2369/// Emit a checked mixed-sign multiply. This is a cheaper specialization of
2370/// the generic checked-binop irgen.
2371static RValue
2372EmitCheckedMixedSignMultiply(CodeGenFunction &CGF, const clang::Expr *Op1,
2373 WidthAndSignedness Op1Info, const clang::Expr *Op2,
2374 WidthAndSignedness Op2Info,
2375 const clang::Expr *ResultArg, QualType ResultQTy,
2376 WidthAndSignedness ResultInfo) {
2377 assert(isSpecialMixedSignMultiply(Builtin::BI__builtin_mul_overflow, Op1Info,
2378 Op2Info, ResultInfo) &&
2379 "Not a mixed-sign multipliction we can specialize");
2380
2381 // Emit the signed and unsigned operands.
2382 const clang::Expr *SignedOp = Op1Info.Signed ? Op1 : Op2;
2383 const clang::Expr *UnsignedOp = Op1Info.Signed ? Op2 : Op1;
2384 llvm::Value *Signed = CGF.EmitScalarExpr(E: SignedOp);
2385 llvm::Value *Unsigned = CGF.EmitScalarExpr(E: UnsignedOp);
2386 unsigned SignedOpWidth = Op1Info.Signed ? Op1Info.Width : Op2Info.Width;
2387 unsigned UnsignedOpWidth = Op1Info.Signed ? Op2Info.Width : Op1Info.Width;
2388
2389 // One of the operands may be smaller than the other. If so, [s|z]ext it.
2390 if (SignedOpWidth < UnsignedOpWidth)
2391 Signed = CGF.Builder.CreateSExt(V: Signed, DestTy: Unsigned->getType(), Name: "op.sext");
2392 if (UnsignedOpWidth < SignedOpWidth)
2393 Unsigned = CGF.Builder.CreateZExt(V: Unsigned, DestTy: Signed->getType(), Name: "op.zext");
2394
2395 llvm::Type *OpTy = Signed->getType();
2396 llvm::Value *Zero = llvm::Constant::getNullValue(Ty: OpTy);
2397 Address ResultPtr = CGF.EmitPointerWithAlignment(Addr: ResultArg);
2398 llvm::Type *ResTy = CGF.getTypes().ConvertType(T: ResultQTy);
2399 unsigned OpWidth = std::max(a: Op1Info.Width, b: Op2Info.Width);
2400
2401 // Take the absolute value of the signed operand.
2402 llvm::Value *IsNegative = CGF.Builder.CreateICmpSLT(LHS: Signed, RHS: Zero);
2403 llvm::Value *AbsOfNegative = CGF.Builder.CreateSub(LHS: Zero, RHS: Signed);
2404 llvm::Value *AbsSigned =
2405 CGF.Builder.CreateSelect(C: IsNegative, True: AbsOfNegative, False: Signed);
2406
2407 // Perform a checked unsigned multiplication.
2408 llvm::Value *UnsignedOverflow;
2409 llvm::Value *UnsignedResult =
2410 EmitOverflowIntrinsic(CGF, IntrinsicID: Intrinsic::umul_with_overflow, X: AbsSigned,
2411 Y: Unsigned, Carry&: UnsignedOverflow);
2412
2413 llvm::Value *Overflow, *Result;
2414 if (ResultInfo.Signed) {
2415 // Signed overflow occurs if the result is greater than INT_MAX or lesser
2416 // than INT_MIN, i.e when |Result| > (INT_MAX + IsNegative).
2417 auto IntMax =
2418 llvm::APInt::getSignedMaxValue(numBits: ResultInfo.Width).zext(width: OpWidth);
2419 llvm::Value *MaxResult =
2420 CGF.Builder.CreateAdd(LHS: llvm::ConstantInt::get(Ty: OpTy, V: IntMax),
2421 RHS: CGF.Builder.CreateZExt(V: IsNegative, DestTy: OpTy));
2422 llvm::Value *SignedOverflow =
2423 CGF.Builder.CreateICmpUGT(LHS: UnsignedResult, RHS: MaxResult);
2424 Overflow = CGF.Builder.CreateOr(LHS: UnsignedOverflow, RHS: SignedOverflow);
2425
2426 // Prepare the signed result (possibly by negating it).
2427 llvm::Value *NegativeResult = CGF.Builder.CreateNeg(V: UnsignedResult);
2428 llvm::Value *SignedResult =
2429 CGF.Builder.CreateSelect(C: IsNegative, True: NegativeResult, False: UnsignedResult);
2430 Result = CGF.Builder.CreateTrunc(V: SignedResult, DestTy: ResTy);
2431 } else {
2432 // Unsigned overflow occurs if the result is < 0 or greater than UINT_MAX.
2433 llvm::Value *Underflow = CGF.Builder.CreateAnd(
2434 LHS: IsNegative, RHS: CGF.Builder.CreateIsNotNull(Arg: UnsignedResult));
2435 Overflow = CGF.Builder.CreateOr(LHS: UnsignedOverflow, RHS: Underflow);
2436 if (ResultInfo.Width < OpWidth) {
2437 auto IntMax =
2438 llvm::APInt::getMaxValue(numBits: ResultInfo.Width).zext(width: OpWidth);
2439 llvm::Value *TruncOverflow = CGF.Builder.CreateICmpUGT(
2440 LHS: UnsignedResult, RHS: llvm::ConstantInt::get(Ty: OpTy, V: IntMax));
2441 Overflow = CGF.Builder.CreateOr(LHS: Overflow, RHS: TruncOverflow);
2442 }
2443
2444 // Negate the product if it would be negative in infinite precision.
2445 Result = CGF.Builder.CreateSelect(
2446 C: IsNegative, True: CGF.Builder.CreateNeg(V: UnsignedResult), False: UnsignedResult);
2447
2448 Result = CGF.Builder.CreateTrunc(V: Result, DestTy: ResTy);
2449 }
2450 assert(Overflow && Result && "Missing overflow or result");
2451
2452 bool isVolatile =
2453 ResultArg->getType()->getPointeeType().isVolatileQualified();
2454 CGF.Builder.CreateStore(Val: CGF.EmitToMemory(Value: Result, Ty: ResultQTy), Addr: ResultPtr,
2455 IsVolatile: isVolatile);
2456 return RValue::get(V: Overflow);
2457}
2458
2459/// Determine if the specified type requires laundering by checking if it is a
2460/// dynamic class type or contains a subobject which is a dynamic class type.
2461static bool TypeRequiresBuiltinLaunder(CodeGenModule &CGM, QualType Ty) {
2462 if (!CGM.getCodeGenOpts().StrictVTablePointers)
2463 return false;
2464 return Ty.requiresBuiltinLaunder(Context: CGM.getContext());
2465}
2466
2467RValue CodeGenFunction::emitRotate(const CallExpr *E, bool IsRotateRight) {
2468 llvm::Value *Src = EmitScalarExpr(E: E->getArg(Arg: 0));
2469 llvm::Value *ShiftAmt = EmitScalarExpr(E: E->getArg(Arg: 1));
2470
2471 // The builtin's shift arg may have a different type than the source arg and
2472 // result, but the LLVM intrinsic uses the same type for all values.
2473 llvm::Type *Ty = Src->getType();
2474 llvm::Type *ShiftTy = ShiftAmt->getType();
2475
2476 unsigned BitWidth = Ty->getIntegerBitWidth();
2477
2478 // Normalize shift amount to [0, BitWidth) range to match runtime behavior.
2479 // This matches the algorithm in ExprConstant.cpp for constant evaluation.
2480 if (BitWidth == 1) {
2481 // Rotating a 1-bit value is always a no-op
2482 ShiftAmt = ConstantInt::get(Ty: ShiftTy, V: 0);
2483 } else if (BitWidth == 2) {
2484 // For 2-bit values: rotation amount is 0 or 1 based on
2485 // whether the amount is even or odd. We can't use srem here because
2486 // the divisor (2) would be misinterpreted as -2 in 2-bit signed arithmetic.
2487 llvm::Value *One = ConstantInt::get(Ty: ShiftTy, V: 1);
2488 ShiftAmt = Builder.CreateAnd(LHS: ShiftAmt, RHS: One);
2489 } else {
2490 unsigned ShiftAmtBitWidth = ShiftTy->getIntegerBitWidth();
2491 bool ShiftAmtIsSigned = E->getArg(Arg: 1)->getType()->isSignedIntegerType();
2492
2493 // Choose the wider type for the divisor to avoid truncation
2494 llvm::Type *DivisorTy = ShiftAmtBitWidth > BitWidth ? ShiftTy : Ty;
2495 llvm::Value *Divisor = ConstantInt::get(Ty: DivisorTy, V: BitWidth);
2496
2497 // Extend ShiftAmt to match Divisor width if needed
2498 if (ShiftAmtBitWidth < DivisorTy->getIntegerBitWidth()) {
2499 ShiftAmt = Builder.CreateIntCast(V: ShiftAmt, DestTy: DivisorTy, isSigned: ShiftAmtIsSigned);
2500 }
2501
2502 // Normalize to [0, BitWidth)
2503 llvm::Value *RemResult;
2504 if (ShiftAmtIsSigned) {
2505 RemResult = Builder.CreateSRem(LHS: ShiftAmt, RHS: Divisor);
2506 // Signed remainder can be negative, convert to positive equivalent
2507 llvm::Value *Zero = ConstantInt::get(Ty: DivisorTy, V: 0);
2508 llvm::Value *IsNegative = Builder.CreateICmpSLT(LHS: RemResult, RHS: Zero);
2509 llvm::Value *PositiveShift = Builder.CreateAdd(LHS: RemResult, RHS: Divisor);
2510 ShiftAmt = Builder.CreateSelect(C: IsNegative, True: PositiveShift, False: RemResult);
2511 } else {
2512 ShiftAmt = Builder.CreateURem(LHS: ShiftAmt, RHS: Divisor);
2513 }
2514 }
2515
2516 // Convert to the source type if needed
2517 if (ShiftAmt->getType() != Ty) {
2518 ShiftAmt = Builder.CreateIntCast(V: ShiftAmt, DestTy: Ty, isSigned: false);
2519 }
2520
2521 // Rotate is a special case of LLVM funnel shift - 1st 2 args are the same.
2522 unsigned IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;
2523 Function *F = CGM.getIntrinsic(IID, Tys: Ty);
2524 return RValue::get(V: Builder.CreateCall(Callee: F, Args: {Src, Src, ShiftAmt}));
2525}
2526
2527// Map math builtins for long-double to f128 version.
2528static unsigned mutateLongDoubleBuiltin(unsigned BuiltinID) {
2529 switch (BuiltinID) {
2530#define MUTATE_LDBL(func) \
2531 case Builtin::BI__builtin_##func##l: \
2532 return Builtin::BI__builtin_##func##f128;
2533 MUTATE_LDBL(sqrt)
2534 MUTATE_LDBL(cbrt)
2535 MUTATE_LDBL(fabs)
2536 MUTATE_LDBL(log)
2537 MUTATE_LDBL(log2)
2538 MUTATE_LDBL(log10)
2539 MUTATE_LDBL(log1p)
2540 MUTATE_LDBL(logb)
2541 MUTATE_LDBL(exp)
2542 MUTATE_LDBL(exp2)
2543 MUTATE_LDBL(expm1)
2544 MUTATE_LDBL(fdim)
2545 MUTATE_LDBL(hypot)
2546 MUTATE_LDBL(ilogb)
2547 MUTATE_LDBL(pow)
2548 MUTATE_LDBL(fmin)
2549 MUTATE_LDBL(fmax)
2550 MUTATE_LDBL(ceil)
2551 MUTATE_LDBL(trunc)
2552 MUTATE_LDBL(rint)
2553 MUTATE_LDBL(nearbyint)
2554 MUTATE_LDBL(round)
2555 MUTATE_LDBL(floor)
2556 MUTATE_LDBL(lround)
2557 MUTATE_LDBL(llround)
2558 MUTATE_LDBL(lrint)
2559 MUTATE_LDBL(llrint)
2560 MUTATE_LDBL(fmod)
2561 MUTATE_LDBL(modf)
2562 MUTATE_LDBL(nan)
2563 MUTATE_LDBL(nans)
2564 MUTATE_LDBL(inf)
2565 MUTATE_LDBL(fma)
2566 MUTATE_LDBL(sin)
2567 MUTATE_LDBL(cos)
2568 MUTATE_LDBL(tan)
2569 MUTATE_LDBL(sinh)
2570 MUTATE_LDBL(cosh)
2571 MUTATE_LDBL(tanh)
2572 MUTATE_LDBL(asin)
2573 MUTATE_LDBL(acos)
2574 MUTATE_LDBL(atan)
2575 MUTATE_LDBL(asinh)
2576 MUTATE_LDBL(acosh)
2577 MUTATE_LDBL(atanh)
2578 MUTATE_LDBL(atan2)
2579 MUTATE_LDBL(erf)
2580 MUTATE_LDBL(erfc)
2581 MUTATE_LDBL(ldexp)
2582 MUTATE_LDBL(frexp)
2583 MUTATE_LDBL(huge_val)
2584 MUTATE_LDBL(copysign)
2585 MUTATE_LDBL(nextafter)
2586 MUTATE_LDBL(nexttoward)
2587 MUTATE_LDBL(remainder)
2588 MUTATE_LDBL(remquo)
2589 MUTATE_LDBL(scalbln)
2590 MUTATE_LDBL(scalbn)
2591 MUTATE_LDBL(tgamma)
2592 MUTATE_LDBL(lgamma)
2593#undef MUTATE_LDBL
2594 default:
2595 return BuiltinID;
2596 }
2597}
2598
2599static Value *tryUseTestFPKind(CodeGenFunction &CGF, unsigned BuiltinID,
2600 Value *V) {
2601 if (CGF.Builder.getIsFPConstrained() &&
2602 CGF.Builder.getDefaultConstrainedExcept() != fp::ebIgnore) {
2603 if (Value *Result =
2604 CGF.getTargetHooks().testFPKind(V, BuiltinID, Builder&: CGF.Builder, CGM&: CGF.CGM))
2605 return Result;
2606 }
2607 return nullptr;
2608}
2609
2610static RValue EmitHipStdParUnsupportedBuiltin(CodeGenFunction *CGF,
2611 const FunctionDecl *FD) {
2612 auto Name = FD->getNameAsString() + "__hipstdpar_unsupported";
2613 auto FnTy = CGF->CGM.getTypes().GetFunctionType(GD: FD);
2614 auto UBF = CGF->CGM.getModule().getOrInsertFunction(Name, T: FnTy);
2615
2616 SmallVector<Value *, 16> Args;
2617 for (auto &&FormalTy : FnTy->params())
2618 Args.push_back(Elt: llvm::PoisonValue::get(T: FormalTy));
2619
2620 return RValue::get(V: CGF->Builder.CreateCall(Callee: UBF, Args));
2621}
2622
2623// stdc_{leading,trailing}_{zeros,ones} and stdc_count_ones: counts bits using
2624// ctlz, cttz, or ctpop (IsPop). InvertArg flips the input to count the
2625// opposite bit value.
2626RValue CodeGenFunction::emitStdcCountIntrinsic(const CallExpr *E,
2627 Intrinsic::ID IntID,
2628 bool InvertArg, bool IsPop) {
2629 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
2630 llvm::Type *ArgType = ArgValue->getType();
2631 llvm::Type *ResultType = ConvertType(T: E->getType());
2632 Value *ActualArg = InvertArg ? Builder.CreateNot(V: ArgValue) : ArgValue;
2633 Function *F = CGM.getIntrinsic(IID: IntID, Tys: ArgType);
2634 Value *Result = IsPop
2635 ? Builder.CreateCall(Callee: F, Args: ActualArg)
2636 : Builder.CreateCall(Callee: F, Args: {ActualArg, Builder.getFalse()});
2637 if (Result->getType() != ResultType)
2638 Result = Builder.CreateIntCast(V: Result, DestTy: ResultType, isSigned: false);
2639 return RValue::get(V: Result);
2640}
2641
2642// stdc_count_zeros (BitWidth - ctpop) and stdc_bit_width (BitWidth - ctlz).
2643// IsPop selects ctpop; otherwise ctlz is used.
2644RValue CodeGenFunction::emitStdcBitWidthMinus(const CallExpr *E,
2645 Intrinsic::ID IntID, bool IsPop) {
2646 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
2647 llvm::Type *ArgType = ArgValue->getType();
2648 llvm::Type *ResultType = ConvertType(T: E->getType());
2649 unsigned BitWidth = ArgType->getIntegerBitWidth();
2650 Function *F = CGM.getIntrinsic(IID: IntID, Tys: ArgType);
2651 Value *Cnt = IsPop ? Builder.CreateCall(Callee: F, Args: ArgValue)
2652 : Builder.CreateCall(Callee: F, Args: {ArgValue, Builder.getFalse()});
2653 Value *Result = Builder.CreateSub(LHS: ConstantInt::get(Ty: ArgType, V: BitWidth), RHS: Cnt);
2654 if (Result->getType() != ResultType)
2655 Result = Builder.CreateIntCast(V: Result, DestTy: ResultType, isSigned: false);
2656 return RValue::get(V: Result);
2657}
2658
2659// stdc_first_{leading,trailing}_{zero,one}: returns the 1-based position of
2660// the first matching bit, or 0 if no such bit exists. InvertArg flips the
2661// input to search for zeros instead of ones.
2662RValue CodeGenFunction::emitStdcFirstBit(const CallExpr *E, Intrinsic::ID IntID,
2663 bool InvertArg) {
2664 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
2665 llvm::Type *ArgType = ArgValue->getType();
2666 llvm::Type *ResultType = ConvertType(T: E->getType());
2667 Value *Zero = ConstantInt::get(Ty: ArgType, V: 0);
2668 Value *One = ConstantInt::get(Ty: ArgType, V: 1);
2669 Value *ActualArg = InvertArg ? Builder.CreateNot(V: ArgValue) : ArgValue;
2670 Function *F = CGM.getIntrinsic(IID: IntID, Tys: ArgType);
2671 Value *Cnt = Builder.CreateCall(Callee: F, Args: {ActualArg, Builder.getFalse()});
2672 Value *Tmp = Builder.CreateAdd(LHS: Cnt, RHS: One);
2673 Value *IsZero = Builder.CreateICmpEQ(LHS: ActualArg, RHS: Zero);
2674 Value *Result = Builder.CreateSelect(C: IsZero, True: Zero, False: Tmp);
2675 if (Result->getType() != ResultType)
2676 Result = Builder.CreateIntCast(V: Result, DestTy: ResultType, isSigned: false);
2677 return RValue::get(V: Result);
2678}
2679
2680static void ClearPadding(CodeGenFunction &CGF, Address Src,
2681 const ASTContext::BitInterval &PaddingInterval) {
2682 uint64_t CharWidth = CGF.getContext().getCharWidth();
2683
2684 auto *I8Ptr = CGF.Builder.CreateBitCast(V: Src.getBasePointer(), DestTy: CGF.Int8PtrTy);
2685 auto *Zero = ConstantInt::get(Ty: CGF.Int8Ty, V: 0);
2686
2687 // Calculate byte indices and bit positions
2688 auto StartByte = PaddingInterval.First / CharWidth;
2689 auto StartBit = PaddingInterval.First % CharWidth;
2690 auto EndByte = PaddingInterval.Last / CharWidth;
2691 auto EndBit = PaddingInterval.Last % CharWidth;
2692
2693 if (StartByte == EndByte) {
2694 // Interval is within a single byte
2695 auto *Index = ConstantInt::get(Ty: CGF.IntTy, V: StartByte);
2696 auto *Element = CGF.Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: I8Ptr, IdxList: Index);
2697 Address ElementAddr(Element, CGF.Int8Ty,
2698 Src.getAlignment().alignmentAtOffset(
2699 offset: CharUnits::fromQuantity(Quantity: StartByte)));
2700
2701 auto *Value = CGF.Builder.CreateLoad(Addr: ElementAddr);
2702
2703 // Create mask to clear bits within the byte
2704 // We want to clear bits from StartBit to EndBit-1
2705 uint8_t bitsToClear = ((1 << EndBit) - 1) & ~((1 << StartBit) - 1);
2706 uint8_t bitsToKeep = ~bitsToClear;
2707 auto *MaskValue = ConstantInt::get(Ty: CGF.Int8Ty, V: bitsToKeep);
2708 auto *NewValue = CGF.Builder.CreateAnd(LHS: Value, RHS: MaskValue);
2709
2710 CGF.Builder.CreateStore(Val: NewValue, Addr: ElementAddr);
2711 } else {
2712 // Handle the start byte
2713 if (StartBit != 0) {
2714 auto *Index = ConstantInt::get(Ty: CGF.IntTy, V: StartByte);
2715 auto *Element = CGF.Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: I8Ptr, IdxList: Index);
2716 Address ElementAddr(Element, CGF.Int8Ty,
2717 Src.getAlignment().alignmentAtOffset(
2718 offset: CharUnits::fromQuantity(Quantity: StartByte)));
2719
2720 auto *Value = CGF.Builder.CreateLoad(Addr: ElementAddr);
2721
2722 uint8_t bitsToClear = ((1 << (CharWidth - StartBit)) - 1) << StartBit;
2723 uint8_t bitsToKeep = ~bitsToClear;
2724 auto *MaskValue = ConstantInt::get(Ty: CGF.Int8Ty, V: bitsToKeep);
2725 auto *NewValue = CGF.Builder.CreateAnd(LHS: Value, RHS: MaskValue);
2726
2727 CGF.Builder.CreateStore(Val: NewValue, Addr: ElementAddr);
2728 ++StartByte;
2729 }
2730
2731 // Handle full bytes in the middle
2732 for (auto Offset = StartByte; Offset < EndByte; ++Offset) {
2733 auto *Index = ConstantInt::get(Ty: CGF.IntTy, V: Offset);
2734 auto *Element = CGF.Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: I8Ptr, IdxList: Index);
2735 Address ElementAddr(Element, CGF.Int8Ty,
2736 Src.getAlignment().alignmentAtOffset(
2737 offset: CharUnits::fromQuantity(Quantity: Offset)));
2738
2739 CGF.Builder.CreateStore(Val: Zero, Addr: ElementAddr);
2740 }
2741
2742 // Handle the end byte
2743 if (EndBit != 0) {
2744 auto *Index = ConstantInt::get(Ty: CGF.IntTy, V: EndByte);
2745 auto *Element = CGF.Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: I8Ptr, IdxList: Index);
2746 Address ElementAddr(Element, CGF.Int8Ty,
2747 Src.getAlignment().alignmentAtOffset(
2748 offset: CharUnits::fromQuantity(Quantity: EndByte)));
2749
2750 auto *Value = CGF.Builder.CreateLoad(Addr: ElementAddr);
2751
2752 uint8_t bitsToClear = (1 << EndBit) - 1;
2753 uint8_t bitsToKeep = ~bitsToClear;
2754 auto *MaskValue = ConstantInt::get(Ty: CGF.Int8Ty, V: bitsToKeep);
2755 auto *NewValue = CGF.Builder.CreateAnd(LHS: Value, RHS: MaskValue);
2756
2757 CGF.Builder.CreateStore(Val: NewValue, Addr: ElementAddr);
2758 }
2759 }
2760}
2761
2762RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
2763 const CallExpr *E,
2764 ReturnValueSlot ReturnValue) {
2765 assert(!getContext().BuiltinInfo.isImmediate(BuiltinID) &&
2766 "Should not codegen for consteval builtins");
2767
2768 const FunctionDecl *FD = GD.getDecl()->getAsFunction();
2769 // See if we can constant fold this builtin. If so, don't emit it at all.
2770 // TODO: Extend this handling to all builtin calls that we can constant-fold.
2771 Expr::EvalResult Result;
2772 if (E->isPRValue() && E->EvaluateAsRValue(Result, Ctx: CGM.getContext()) &&
2773 !Result.hasSideEffects()) {
2774 if (Result.Val.isInt())
2775 return RValue::get(V: llvm::ConstantInt::get(Context&: getLLVMContext(),
2776 V: Result.Val.getInt()));
2777 if (Result.Val.isFloat())
2778 return RValue::get(V: llvm::ConstantFP::get(Context&: getLLVMContext(),
2779 V: Result.Val.getFloat()));
2780 }
2781
2782 // If current long-double semantics is IEEE 128-bit, replace math builtins
2783 // of long-double with f128 equivalent.
2784 // TODO: This mutation should also be applied to other targets other than PPC,
2785 // after backend supports IEEE 128-bit style libcalls.
2786 if (getTarget().getTriple().isPPC64() &&
2787 &getTarget().getLongDoubleFormat() == &llvm::APFloat::IEEEquad())
2788 BuiltinID = mutateLongDoubleBuiltin(BuiltinID);
2789
2790 // If the builtin has been declared explicitly with an assembler label,
2791 // disable the specialized emitting below. Ideally we should communicate the
2792 // rename in IR, or at least avoid generating the intrinsic calls that are
2793 // likely to get lowered to the renamed library functions.
2794 const unsigned BuiltinIDIfNoAsmLabel =
2795 FD->hasAttr<AsmLabelAttr>() ? 0 : BuiltinID;
2796
2797 std::optional<bool> ErrnoOverriden;
2798 // ErrnoOverriden is true if math-errno is overriden via the
2799 // '#pragma float_control(precise, on)'. This pragma disables fast-math,
2800 // which implies math-errno.
2801 if (E->hasStoredFPFeatures()) {
2802 FPOptionsOverride OP = E->getFPFeatures();
2803 if (OP.hasMathErrnoOverride())
2804 ErrnoOverriden = OP.getMathErrnoOverride();
2805 }
2806 // True if 'attribute__((optnone))' is used. This attribute overrides
2807 // fast-math which implies math-errno.
2808 bool OptNone = CurFuncDecl && CurFuncDecl->hasAttr<OptimizeNoneAttr>();
2809
2810 bool IsOptimizationEnabled = CGM.getCodeGenOpts().OptimizationLevel != 0;
2811
2812 bool GenerateFPMathIntrinsics =
2813 getContext().BuiltinInfo.shouldGenerateFPMathIntrinsic(
2814 BuiltinID, Trip: CGM.getTriple(), ErrnoOverwritten: ErrnoOverriden, MathErrnoEnabled: getLangOpts().MathErrno,
2815 HasOptNoneAttr: OptNone, IsOptimizationEnabled);
2816
2817 if (GenerateFPMathIntrinsics) {
2818 switch (BuiltinIDIfNoAsmLabel) {
2819 case Builtin::BIacos:
2820 case Builtin::BIacosf:
2821 case Builtin::BIacosl:
2822 case Builtin::BI__builtin_acos:
2823 case Builtin::BI__builtin_acosf:
2824 case Builtin::BI__builtin_acosf16:
2825 case Builtin::BI__builtin_acosl:
2826 case Builtin::BI__builtin_acosf128:
2827 case Builtin::BI__builtin_elementwise_acos:
2828 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(
2829 CGF&: *this, E, IntrinsicID: Intrinsic::acos, ConstrainedIntrinsicID: Intrinsic::experimental_constrained_acos));
2830
2831 case Builtin::BIasin:
2832 case Builtin::BIasinf:
2833 case Builtin::BIasinl:
2834 case Builtin::BI__builtin_asin:
2835 case Builtin::BI__builtin_asinf:
2836 case Builtin::BI__builtin_asinf16:
2837 case Builtin::BI__builtin_asinl:
2838 case Builtin::BI__builtin_asinf128:
2839 case Builtin::BI__builtin_elementwise_asin:
2840 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(
2841 CGF&: *this, E, IntrinsicID: Intrinsic::asin, ConstrainedIntrinsicID: Intrinsic::experimental_constrained_asin));
2842
2843 case Builtin::BIatan:
2844 case Builtin::BIatanf:
2845 case Builtin::BIatanl:
2846 case Builtin::BI__builtin_atan:
2847 case Builtin::BI__builtin_atanf:
2848 case Builtin::BI__builtin_atanf16:
2849 case Builtin::BI__builtin_atanl:
2850 case Builtin::BI__builtin_atanf128:
2851 case Builtin::BI__builtin_elementwise_atan:
2852 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(
2853 CGF&: *this, E, IntrinsicID: Intrinsic::atan, ConstrainedIntrinsicID: Intrinsic::experimental_constrained_atan));
2854
2855 case Builtin::BIatan2:
2856 case Builtin::BIatan2f:
2857 case Builtin::BIatan2l:
2858 case Builtin::BI__builtin_atan2:
2859 case Builtin::BI__builtin_atan2f:
2860 case Builtin::BI__builtin_atan2f16:
2861 case Builtin::BI__builtin_atan2l:
2862 case Builtin::BI__builtin_atan2f128:
2863 case Builtin::BI__builtin_elementwise_atan2:
2864 return RValue::get(V: emitBinaryMaybeConstrainedFPBuiltin(
2865 CGF&: *this, E, IntrinsicID: Intrinsic::atan2,
2866 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_atan2));
2867
2868 case Builtin::BIceil:
2869 case Builtin::BIceilf:
2870 case Builtin::BIceill:
2871 case Builtin::BI__builtin_ceil:
2872 case Builtin::BI__builtin_ceilf:
2873 case Builtin::BI__builtin_ceilf16:
2874 case Builtin::BI__builtin_ceill:
2875 case Builtin::BI__builtin_ceilf128:
2876 case Builtin::BI__builtin_elementwise_ceil:
2877 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
2878 IntrinsicID: Intrinsic::ceil,
2879 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_ceil));
2880
2881 case Builtin::BIcopysign:
2882 case Builtin::BIcopysignf:
2883 case Builtin::BIcopysignl:
2884 case Builtin::BI__builtin_copysign:
2885 case Builtin::BI__builtin_copysignf:
2886 case Builtin::BI__builtin_copysignf16:
2887 case Builtin::BI__builtin_copysignl:
2888 case Builtin::BI__builtin_copysignf128:
2889 return RValue::get(
2890 V: emitBuiltinWithOneOverloadedType<2>(CGF&: *this, E, IntrinsicID: Intrinsic::copysign));
2891
2892 case Builtin::BIcos:
2893 case Builtin::BIcosf:
2894 case Builtin::BIcosl:
2895 case Builtin::BI__builtin_cos:
2896 case Builtin::BI__builtin_cosf:
2897 case Builtin::BI__builtin_cosf16:
2898 case Builtin::BI__builtin_cosl:
2899 case Builtin::BI__builtin_cosf128:
2900 case Builtin::BI__builtin_elementwise_cos:
2901 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
2902 IntrinsicID: Intrinsic::cos,
2903 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_cos));
2904
2905 case Builtin::BIcosh:
2906 case Builtin::BIcoshf:
2907 case Builtin::BIcoshl:
2908 case Builtin::BI__builtin_cosh:
2909 case Builtin::BI__builtin_coshf:
2910 case Builtin::BI__builtin_coshf16:
2911 case Builtin::BI__builtin_coshl:
2912 case Builtin::BI__builtin_coshf128:
2913 case Builtin::BI__builtin_elementwise_cosh:
2914 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(
2915 CGF&: *this, E, IntrinsicID: Intrinsic::cosh, ConstrainedIntrinsicID: Intrinsic::experimental_constrained_cosh));
2916
2917 case Builtin::BIexp:
2918 case Builtin::BIexpf:
2919 case Builtin::BIexpl:
2920 case Builtin::BI__builtin_exp:
2921 case Builtin::BI__builtin_expf:
2922 case Builtin::BI__builtin_expf16:
2923 case Builtin::BI__builtin_expl:
2924 case Builtin::BI__builtin_expf128:
2925 case Builtin::BI__builtin_elementwise_exp:
2926 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
2927 IntrinsicID: Intrinsic::exp,
2928 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_exp));
2929
2930 case Builtin::BIexp2:
2931 case Builtin::BIexp2f:
2932 case Builtin::BIexp2l:
2933 case Builtin::BI__builtin_exp2:
2934 case Builtin::BI__builtin_exp2f:
2935 case Builtin::BI__builtin_exp2f16:
2936 case Builtin::BI__builtin_exp2l:
2937 case Builtin::BI__builtin_exp2f128:
2938 case Builtin::BI__builtin_elementwise_exp2:
2939 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
2940 IntrinsicID: Intrinsic::exp2,
2941 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_exp2));
2942 case Builtin::BI__builtin_exp10:
2943 case Builtin::BI__builtin_exp10f:
2944 case Builtin::BI__builtin_exp10f16:
2945 case Builtin::BI__builtin_exp10l:
2946 case Builtin::BI__builtin_exp10f128:
2947 case Builtin::BI__builtin_elementwise_exp10: {
2948 // TODO: strictfp support
2949 if (Builder.getIsFPConstrained())
2950 break;
2951 return RValue::get(
2952 V: emitBuiltinWithOneOverloadedType<1>(CGF&: *this, E, IntrinsicID: Intrinsic::exp10));
2953 }
2954 case Builtin::BIfabs:
2955 case Builtin::BIfabsf:
2956 case Builtin::BIfabsl:
2957 case Builtin::BI__builtin_fabs:
2958 case Builtin::BI__builtin_fabsf:
2959 case Builtin::BI__builtin_fabsf16:
2960 case Builtin::BI__builtin_fabsl:
2961 case Builtin::BI__builtin_fabsf128:
2962 return RValue::get(
2963 V: emitBuiltinWithOneOverloadedType<1>(CGF&: *this, E, IntrinsicID: Intrinsic::fabs));
2964
2965 case Builtin::BIfloor:
2966 case Builtin::BIfloorf:
2967 case Builtin::BIfloorl:
2968 case Builtin::BI__builtin_floor:
2969 case Builtin::BI__builtin_floorf:
2970 case Builtin::BI__builtin_floorf16:
2971 case Builtin::BI__builtin_floorl:
2972 case Builtin::BI__builtin_floorf128:
2973 case Builtin::BI__builtin_elementwise_floor:
2974 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
2975 IntrinsicID: Intrinsic::floor,
2976 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_floor));
2977
2978 case Builtin::BIfma:
2979 case Builtin::BIfmaf:
2980 case Builtin::BIfmal:
2981 case Builtin::BI__builtin_fma:
2982 case Builtin::BI__builtin_fmaf:
2983 case Builtin::BI__builtin_fmaf16:
2984 case Builtin::BI__builtin_fmal:
2985 case Builtin::BI__builtin_fmaf128:
2986 case Builtin::BI__builtin_elementwise_fma:
2987 return RValue::get(V: emitTernaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
2988 IntrinsicID: Intrinsic::fma,
2989 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_fma));
2990
2991 case Builtin::BIfmax:
2992 case Builtin::BIfmaxf:
2993 case Builtin::BIfmaxl:
2994 case Builtin::BI__builtin_fmax:
2995 case Builtin::BI__builtin_fmaxf:
2996 case Builtin::BI__builtin_fmaxf16:
2997 case Builtin::BI__builtin_fmaxl:
2998 case Builtin::BI__builtin_fmaxf128: {
2999 IRBuilder<>::FastMathFlagGuard FMFGuard(Builder);
3000 Builder.getFastMathFlags().setNoSignedZeros();
3001 return RValue::get(V: emitBinaryMaybeConstrainedFPBuiltin(
3002 CGF&: *this, E, IntrinsicID: Intrinsic::maxnum,
3003 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_maxnum));
3004 }
3005
3006 case Builtin::BIfmin:
3007 case Builtin::BIfminf:
3008 case Builtin::BIfminl:
3009 case Builtin::BI__builtin_fmin:
3010 case Builtin::BI__builtin_fminf:
3011 case Builtin::BI__builtin_fminf16:
3012 case Builtin::BI__builtin_fminl:
3013 case Builtin::BI__builtin_fminf128: {
3014 IRBuilder<>::FastMathFlagGuard FMFGuard(Builder);
3015 Builder.getFastMathFlags().setNoSignedZeros();
3016 return RValue::get(V: emitBinaryMaybeConstrainedFPBuiltin(
3017 CGF&: *this, E, IntrinsicID: Intrinsic::minnum,
3018 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_minnum));
3019 }
3020
3021 case Builtin::BIfmaximum_num:
3022 case Builtin::BIfmaximum_numf:
3023 case Builtin::BIfmaximum_numl:
3024 case Builtin::BI__builtin_fmaximum_num:
3025 case Builtin::BI__builtin_fmaximum_numf:
3026 case Builtin::BI__builtin_fmaximum_numf16:
3027 case Builtin::BI__builtin_fmaximum_numl:
3028 case Builtin::BI__builtin_fmaximum_numf128:
3029 return RValue::get(
3030 V: emitBuiltinWithOneOverloadedType<2>(CGF&: *this, E, IntrinsicID: Intrinsic::maximumnum));
3031
3032 case Builtin::BIfminimum_num:
3033 case Builtin::BIfminimum_numf:
3034 case Builtin::BIfminimum_numl:
3035 case Builtin::BI__builtin_fminimum_num:
3036 case Builtin::BI__builtin_fminimum_numf:
3037 case Builtin::BI__builtin_fminimum_numf16:
3038 case Builtin::BI__builtin_fminimum_numl:
3039 case Builtin::BI__builtin_fminimum_numf128:
3040 return RValue::get(
3041 V: emitBuiltinWithOneOverloadedType<2>(CGF&: *this, E, IntrinsicID: Intrinsic::minimumnum));
3042
3043 // fmod() is a special-case. It maps to the frem instruction rather than an
3044 // LLVM intrinsic.
3045 case Builtin::BIfmod:
3046 case Builtin::BIfmodf:
3047 case Builtin::BIfmodl:
3048 case Builtin::BI__builtin_fmod:
3049 case Builtin::BI__builtin_fmodf:
3050 case Builtin::BI__builtin_fmodf16:
3051 case Builtin::BI__builtin_fmodl:
3052 case Builtin::BI__builtin_fmodf128:
3053 case Builtin::BI__builtin_elementwise_fmod: {
3054 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
3055 Value *Arg1 = EmitScalarExpr(E: E->getArg(Arg: 0));
3056 Value *Arg2 = EmitScalarExpr(E: E->getArg(Arg: 1));
3057 if (Builder.getIsFPConstrained()) {
3058 Function *F = CGM.getIntrinsic(IID: Intrinsic::experimental_constrained_frem,
3059 Tys: Arg1->getType());
3060 return RValue::get(V: Builder.CreateConstrainedFPCall(Callee: F, Args: {Arg1, Arg2}));
3061 } else {
3062 return RValue::get(V: Builder.CreateFRem(L: Arg1, R: Arg2, Name: "fmod"));
3063 }
3064 }
3065
3066 case Builtin::BIlog:
3067 case Builtin::BIlogf:
3068 case Builtin::BIlogl:
3069 case Builtin::BI__builtin_log:
3070 case Builtin::BI__builtin_logf:
3071 case Builtin::BI__builtin_logf16:
3072 case Builtin::BI__builtin_logl:
3073 case Builtin::BI__builtin_logf128:
3074 case Builtin::BI__builtin_elementwise_log:
3075 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
3076 IntrinsicID: Intrinsic::log,
3077 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_log));
3078
3079 case Builtin::BIlog10:
3080 case Builtin::BIlog10f:
3081 case Builtin::BIlog10l:
3082 case Builtin::BI__builtin_log10:
3083 case Builtin::BI__builtin_log10f:
3084 case Builtin::BI__builtin_log10f16:
3085 case Builtin::BI__builtin_log10l:
3086 case Builtin::BI__builtin_log10f128:
3087 case Builtin::BI__builtin_elementwise_log10:
3088 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
3089 IntrinsicID: Intrinsic::log10,
3090 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_log10));
3091
3092 case Builtin::BIlog2:
3093 case Builtin::BIlog2f:
3094 case Builtin::BIlog2l:
3095 case Builtin::BI__builtin_log2:
3096 case Builtin::BI__builtin_log2f:
3097 case Builtin::BI__builtin_log2f16:
3098 case Builtin::BI__builtin_log2l:
3099 case Builtin::BI__builtin_log2f128:
3100 case Builtin::BI__builtin_elementwise_log2:
3101 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
3102 IntrinsicID: Intrinsic::log2,
3103 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_log2));
3104
3105 case Builtin::BInearbyint:
3106 case Builtin::BInearbyintf:
3107 case Builtin::BInearbyintl:
3108 case Builtin::BI__builtin_nearbyint:
3109 case Builtin::BI__builtin_nearbyintf:
3110 case Builtin::BI__builtin_nearbyintl:
3111 case Builtin::BI__builtin_nearbyintf128:
3112 case Builtin::BI__builtin_elementwise_nearbyint:
3113 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
3114 IntrinsicID: Intrinsic::nearbyint,
3115 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_nearbyint));
3116
3117 case Builtin::BIpow:
3118 case Builtin::BIpowf:
3119 case Builtin::BIpowl:
3120 case Builtin::BI__builtin_pow:
3121 case Builtin::BI__builtin_powf:
3122 case Builtin::BI__builtin_powf16:
3123 case Builtin::BI__builtin_powl:
3124 case Builtin::BI__builtin_powf128:
3125 case Builtin::BI__builtin_elementwise_pow:
3126 return RValue::get(V: emitBinaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
3127 IntrinsicID: Intrinsic::pow,
3128 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_pow));
3129
3130 case Builtin::BIrint:
3131 case Builtin::BIrintf:
3132 case Builtin::BIrintl:
3133 case Builtin::BI__builtin_rint:
3134 case Builtin::BI__builtin_rintf:
3135 case Builtin::BI__builtin_rintf16:
3136 case Builtin::BI__builtin_rintl:
3137 case Builtin::BI__builtin_rintf128:
3138 case Builtin::BI__builtin_elementwise_rint:
3139 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
3140 IntrinsicID: Intrinsic::rint,
3141 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_rint));
3142
3143 case Builtin::BIround:
3144 case Builtin::BIroundf:
3145 case Builtin::BIroundl:
3146 case Builtin::BI__builtin_round:
3147 case Builtin::BI__builtin_roundf:
3148 case Builtin::BI__builtin_roundf16:
3149 case Builtin::BI__builtin_roundl:
3150 case Builtin::BI__builtin_roundf128:
3151 case Builtin::BI__builtin_elementwise_round:
3152 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
3153 IntrinsicID: Intrinsic::round,
3154 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_round));
3155
3156 case Builtin::BIroundeven:
3157 case Builtin::BIroundevenf:
3158 case Builtin::BIroundevenl:
3159 case Builtin::BI__builtin_roundeven:
3160 case Builtin::BI__builtin_roundevenf:
3161 case Builtin::BI__builtin_roundevenf16:
3162 case Builtin::BI__builtin_roundevenl:
3163 case Builtin::BI__builtin_roundevenf128:
3164 case Builtin::BI__builtin_elementwise_roundeven:
3165 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
3166 IntrinsicID: Intrinsic::roundeven,
3167 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_roundeven));
3168
3169 case Builtin::BIsin:
3170 case Builtin::BIsinf:
3171 case Builtin::BIsinl:
3172 case Builtin::BI__builtin_sin:
3173 case Builtin::BI__builtin_sinf:
3174 case Builtin::BI__builtin_sinf16:
3175 case Builtin::BI__builtin_sinl:
3176 case Builtin::BI__builtin_sinf128:
3177 case Builtin::BI__builtin_elementwise_sin:
3178 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
3179 IntrinsicID: Intrinsic::sin,
3180 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_sin));
3181
3182 case Builtin::BIsinh:
3183 case Builtin::BIsinhf:
3184 case Builtin::BIsinhl:
3185 case Builtin::BI__builtin_sinh:
3186 case Builtin::BI__builtin_sinhf:
3187 case Builtin::BI__builtin_sinhf16:
3188 case Builtin::BI__builtin_sinhl:
3189 case Builtin::BI__builtin_sinhf128:
3190 case Builtin::BI__builtin_elementwise_sinh:
3191 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(
3192 CGF&: *this, E, IntrinsicID: Intrinsic::sinh, ConstrainedIntrinsicID: Intrinsic::experimental_constrained_sinh));
3193
3194 case Builtin::BI__builtin_sincospi:
3195 case Builtin::BI__builtin_sincospif:
3196 case Builtin::BI__builtin_sincospil:
3197 if (Builder.getIsFPConstrained())
3198 break; // TODO: Emit constrained sincospi intrinsic once one exists.
3199 emitSincosBuiltin(CGF&: *this, E, IntrinsicID: Intrinsic::sincospi);
3200 return RValue::get(V: nullptr);
3201
3202 case Builtin::BIsincos:
3203 case Builtin::BIsincosf:
3204 case Builtin::BIsincosl:
3205 case Builtin::BI__builtin_sincos:
3206 case Builtin::BI__builtin_sincosf:
3207 case Builtin::BI__builtin_sincosf16:
3208 case Builtin::BI__builtin_sincosl:
3209 case Builtin::BI__builtin_sincosf128:
3210 if (Builder.getIsFPConstrained())
3211 break; // TODO: Emit constrained sincos intrinsic once one exists.
3212 emitSincosBuiltin(CGF&: *this, E, IntrinsicID: Intrinsic::sincos);
3213 return RValue::get(V: nullptr);
3214
3215 case Builtin::BIsqrt:
3216 case Builtin::BIsqrtf:
3217 case Builtin::BIsqrtl:
3218 case Builtin::BI__builtin_sqrt:
3219 case Builtin::BI__builtin_sqrtf:
3220 case Builtin::BI__builtin_sqrtf16:
3221 case Builtin::BI__builtin_sqrtl:
3222 case Builtin::BI__builtin_sqrtf128:
3223 case Builtin::BI__builtin_elementwise_sqrt: {
3224 llvm::Value *Call = emitUnaryMaybeConstrainedFPBuiltin(
3225 CGF&: *this, E, IntrinsicID: Intrinsic::sqrt, ConstrainedIntrinsicID: Intrinsic::experimental_constrained_sqrt);
3226 SetSqrtFPAccuracy(Call);
3227 return RValue::get(V: Call);
3228 }
3229
3230 case Builtin::BItan:
3231 case Builtin::BItanf:
3232 case Builtin::BItanl:
3233 case Builtin::BI__builtin_tan:
3234 case Builtin::BI__builtin_tanf:
3235 case Builtin::BI__builtin_tanf16:
3236 case Builtin::BI__builtin_tanl:
3237 case Builtin::BI__builtin_tanf128:
3238 case Builtin::BI__builtin_elementwise_tan:
3239 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(
3240 CGF&: *this, E, IntrinsicID: Intrinsic::tan, ConstrainedIntrinsicID: Intrinsic::experimental_constrained_tan));
3241
3242 case Builtin::BItanh:
3243 case Builtin::BItanhf:
3244 case Builtin::BItanhl:
3245 case Builtin::BI__builtin_tanh:
3246 case Builtin::BI__builtin_tanhf:
3247 case Builtin::BI__builtin_tanhf16:
3248 case Builtin::BI__builtin_tanhl:
3249 case Builtin::BI__builtin_tanhf128:
3250 case Builtin::BI__builtin_elementwise_tanh:
3251 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(
3252 CGF&: *this, E, IntrinsicID: Intrinsic::tanh, ConstrainedIntrinsicID: Intrinsic::experimental_constrained_tanh));
3253
3254 case Builtin::BItrunc:
3255 case Builtin::BItruncf:
3256 case Builtin::BItruncl:
3257 case Builtin::BI__builtin_trunc:
3258 case Builtin::BI__builtin_truncf:
3259 case Builtin::BI__builtin_truncf16:
3260 case Builtin::BI__builtin_truncl:
3261 case Builtin::BI__builtin_truncf128:
3262 case Builtin::BI__builtin_elementwise_trunc:
3263 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
3264 IntrinsicID: Intrinsic::trunc,
3265 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_trunc));
3266
3267 case Builtin::BIlround:
3268 case Builtin::BIlroundf:
3269 case Builtin::BIlroundl:
3270 case Builtin::BI__builtin_lround:
3271 case Builtin::BI__builtin_lroundf:
3272 case Builtin::BI__builtin_lroundl:
3273 case Builtin::BI__builtin_lroundf128:
3274 return RValue::get(V: emitMaybeConstrainedFPToIntRoundBuiltin(
3275 CGF&: *this, E, IntrinsicID: Intrinsic::lround,
3276 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_lround));
3277
3278 case Builtin::BIllround:
3279 case Builtin::BIllroundf:
3280 case Builtin::BIllroundl:
3281 case Builtin::BI__builtin_llround:
3282 case Builtin::BI__builtin_llroundf:
3283 case Builtin::BI__builtin_llroundl:
3284 case Builtin::BI__builtin_llroundf128:
3285 return RValue::get(V: emitMaybeConstrainedFPToIntRoundBuiltin(
3286 CGF&: *this, E, IntrinsicID: Intrinsic::llround,
3287 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_llround));
3288
3289 case Builtin::BIlrint:
3290 case Builtin::BIlrintf:
3291 case Builtin::BIlrintl:
3292 case Builtin::BI__builtin_lrint:
3293 case Builtin::BI__builtin_lrintf:
3294 case Builtin::BI__builtin_lrintl:
3295 case Builtin::BI__builtin_lrintf128:
3296 return RValue::get(V: emitMaybeConstrainedFPToIntRoundBuiltin(
3297 CGF&: *this, E, IntrinsicID: Intrinsic::lrint,
3298 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_lrint));
3299
3300 case Builtin::BIllrint:
3301 case Builtin::BIllrintf:
3302 case Builtin::BIllrintl:
3303 case Builtin::BI__builtin_llrint:
3304 case Builtin::BI__builtin_llrintf:
3305 case Builtin::BI__builtin_llrintl:
3306 case Builtin::BI__builtin_llrintf128:
3307 return RValue::get(V: emitMaybeConstrainedFPToIntRoundBuiltin(
3308 CGF&: *this, E, IntrinsicID: Intrinsic::llrint,
3309 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_llrint));
3310 case Builtin::BI__builtin_ldexp:
3311 case Builtin::BI__builtin_ldexpf:
3312 case Builtin::BI__builtin_ldexpl:
3313 case Builtin::BI__builtin_ldexpf16:
3314 case Builtin::BI__builtin_ldexpf128:
3315 case Builtin::BI__builtin_elementwise_ldexp:
3316 return RValue::get(V: emitBinaryExpMaybeConstrainedFPBuiltin(
3317 CGF&: *this, E, IntrinsicID: Intrinsic::ldexp,
3318 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_ldexp));
3319 default:
3320 break;
3321 }
3322 }
3323
3324 // Check NonnullAttribute/NullabilityArg and Alignment.
3325 auto EmitArgCheck = [&](TypeCheckKind Kind, Address A, const Expr *Arg,
3326 unsigned ParmNum) {
3327 Value *Val = A.emitRawPointer(CGF&: *this);
3328 EmitNonNullArgCheck(RV: RValue::get(V: Val), ArgType: Arg->getType(), ArgLoc: Arg->getExprLoc(), AC: FD,
3329 ParmNum);
3330
3331 if (SanOpts.has(K: SanitizerKind::Alignment)) {
3332 SanitizerSet SkippedChecks;
3333 SkippedChecks.set(SanitizerKind::All);
3334 SkippedChecks.clear(K: SanitizerKind::Alignment);
3335 SourceLocation Loc = Arg->getExprLoc();
3336 // Strip an implicit cast.
3337 if (auto *CE = dyn_cast<ImplicitCastExpr>(Val: Arg))
3338 if (CE->getCastKind() == CK_BitCast)
3339 Arg = CE->getSubExpr();
3340 EmitTypeCheck(TCK: Kind, Loc, V: Val, Type: Arg->getType(), Alignment: A.getAlignment(),
3341 SkippedChecks);
3342 }
3343 };
3344
3345 switch (BuiltinIDIfNoAsmLabel) {
3346 default: break;
3347 case Builtin::BI__builtin___CFStringMakeConstantString:
3348 case Builtin::BI__builtin___NSStringMakeConstantString:
3349 return RValue::get(V: ConstantEmitter(*this).emitAbstract(E, T: E->getType()));
3350 case Builtin::BI__builtin_stdarg_start:
3351 case Builtin::BI__builtin_va_start:
3352 case Builtin::BI__va_start:
3353 case Builtin::BI__builtin_c23_va_start:
3354 case Builtin::BI__builtin_va_end:
3355 EmitVAStartEnd(ArgValue: BuiltinID == Builtin::BI__va_start
3356 ? EmitScalarExpr(E: E->getArg(Arg: 0))
3357 : EmitVAListRef(E: E->getArg(Arg: 0)).emitRawPointer(CGF&: *this),
3358 IsStart: BuiltinID != Builtin::BI__builtin_va_end);
3359 return RValue::get(V: nullptr);
3360 case Builtin::BI__builtin_va_copy: {
3361 Value *DstPtr = EmitVAListRef(E: E->getArg(Arg: 0)).emitRawPointer(CGF&: *this);
3362 Value *SrcPtr = EmitVAListRef(E: E->getArg(Arg: 1)).emitRawPointer(CGF&: *this);
3363 Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Intrinsic::vacopy, Tys: {DstPtr->getType()}),
3364 Args: {DstPtr, SrcPtr});
3365 return RValue::get(V: nullptr);
3366 }
3367 case Builtin::BIabs:
3368 case Builtin::BIlabs:
3369 case Builtin::BIllabs:
3370 case Builtin::BI__builtin_abs:
3371 case Builtin::BI__builtin_labs:
3372 case Builtin::BI__builtin_llabs: {
3373 bool SanitizeOverflow = SanOpts.has(K: SanitizerKind::SignedIntegerOverflow);
3374
3375 Value *Result;
3376 switch (getLangOpts().getSignedOverflowBehavior()) {
3377 case LangOptions::SOB_Defined:
3378 Result = EmitAbs(CGF&: *this, ArgValue: EmitScalarExpr(E: E->getArg(Arg: 0)), HasNSW: false);
3379 break;
3380 case LangOptions::SOB_Undefined:
3381 if (!SanitizeOverflow) {
3382 Result = EmitAbs(CGF&: *this, ArgValue: EmitScalarExpr(E: E->getArg(Arg: 0)), HasNSW: true);
3383 break;
3384 }
3385 [[fallthrough]];
3386 case LangOptions::SOB_Trapping:
3387 // TODO: Somehow handle the corner case when the address of abs is taken.
3388 Result = EmitOverflowCheckedAbs(CGF&: *this, E, SanitizeOverflow);
3389 break;
3390 }
3391 return RValue::get(V: Result);
3392 }
3393 case Builtin::BI__builtin_complex: {
3394 Value *Real = EmitScalarExpr(E: E->getArg(Arg: 0));
3395 Value *Imag = EmitScalarExpr(E: E->getArg(Arg: 1));
3396 return RValue::getComplex(C: {Real, Imag});
3397 }
3398 case Builtin::BI__builtin_conj:
3399 case Builtin::BI__builtin_conjf:
3400 case Builtin::BI__builtin_conjl:
3401 case Builtin::BIconj:
3402 case Builtin::BIconjf:
3403 case Builtin::BIconjl: {
3404 ComplexPairTy ComplexVal = EmitComplexExpr(E: E->getArg(Arg: 0));
3405 Value *Real = ComplexVal.first;
3406 Value *Imag = ComplexVal.second;
3407 Imag = Builder.CreateFNeg(V: Imag, Name: "neg");
3408 return RValue::getComplex(C: std::make_pair(x&: Real, y&: Imag));
3409 }
3410 case Builtin::BI__builtin_creal:
3411 case Builtin::BI__builtin_crealf:
3412 case Builtin::BI__builtin_creall:
3413 case Builtin::BIcreal:
3414 case Builtin::BIcrealf:
3415 case Builtin::BIcreall: {
3416 ComplexPairTy ComplexVal = EmitComplexExpr(E: E->getArg(Arg: 0));
3417 return RValue::get(V: ComplexVal.first);
3418 }
3419
3420 case Builtin::BI__builtin_preserve_access_index: {
3421 // Only enabled preserved access index region when debuginfo
3422 // is available as debuginfo is needed to preserve user-level
3423 // access pattern.
3424 if (!getDebugInfo()) {
3425 CGM.Error(loc: E->getExprLoc(), error: "using builtin_preserve_access_index() without -g");
3426 return RValue::get(V: EmitScalarExpr(E: E->getArg(Arg: 0)));
3427 }
3428
3429 // Nested builtin_preserve_access_index() not supported
3430 if (IsInPreservedAIRegion) {
3431 CGM.Error(loc: E->getExprLoc(), error: "nested builtin_preserve_access_index() not supported");
3432 return RValue::get(V: EmitScalarExpr(E: E->getArg(Arg: 0)));
3433 }
3434
3435 IsInPreservedAIRegion = true;
3436 Value *Res = EmitScalarExpr(E: E->getArg(Arg: 0));
3437 IsInPreservedAIRegion = false;
3438 return RValue::get(V: Res);
3439 }
3440
3441 case Builtin::BI__builtin_cimag:
3442 case Builtin::BI__builtin_cimagf:
3443 case Builtin::BI__builtin_cimagl:
3444 case Builtin::BIcimag:
3445 case Builtin::BIcimagf:
3446 case Builtin::BIcimagl: {
3447 ComplexPairTy ComplexVal = EmitComplexExpr(E: E->getArg(Arg: 0));
3448 return RValue::get(V: ComplexVal.second);
3449 }
3450
3451 case Builtin::BI__builtin_clrsb:
3452 case Builtin::BI__builtin_clrsbl:
3453 case Builtin::BI__builtin_clrsbll: {
3454 // clrsb(x) -> clz(x < 0 ? ~x : x) - 1 or
3455 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
3456
3457 llvm::Type *ArgType = ArgValue->getType();
3458 Function *F = CGM.getIntrinsic(IID: Intrinsic::ctlz, Tys: ArgType);
3459
3460 llvm::Type *ResultType = ConvertType(T: E->getType());
3461 Value *Zero = llvm::Constant::getNullValue(Ty: ArgType);
3462 Value *IsNeg = Builder.CreateICmpSLT(LHS: ArgValue, RHS: Zero, Name: "isneg");
3463 Value *Inverse = Builder.CreateNot(V: ArgValue, Name: "not");
3464 Value *Tmp = Builder.CreateSelect(C: IsNeg, True: Inverse, False: ArgValue);
3465 Value *Ctlz = Builder.CreateCall(Callee: F, Args: {Tmp, Builder.getFalse()});
3466 Value *Result =
3467 Builder.CreateNUWSub(LHS: Ctlz, RHS: llvm::ConstantInt::get(Ty: ArgType, V: 1));
3468 Result = Builder.CreateIntCast(V: Result, DestTy: ResultType, /*isSigned*/true,
3469 Name: "cast");
3470 return RValue::get(V: Result);
3471 }
3472 case Builtin::BI__builtin_ctzs:
3473 case Builtin::BI__builtin_ctz:
3474 case Builtin::BI__builtin_ctzl:
3475 case Builtin::BI__builtin_ctzll:
3476 case Builtin::BI__builtin_ctzg:
3477 case Builtin::BI__builtin_elementwise_ctzg: {
3478 bool HasFallback =
3479 (BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_ctzg ||
3480 BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_elementwise_ctzg) &&
3481 E->getNumArgs() > 1;
3482
3483 Value *ArgValue =
3484 HasFallback ? EmitBitCountExpr(CGF&: *this, E: E->getArg(Arg: 0))
3485 : EmitCheckedArgForBuiltin(E: E->getArg(Arg: 0), Kind: BCK_CTZPassedZero);
3486
3487 llvm::Type *ArgType = ArgValue->getType();
3488 Function *F = CGM.getIntrinsic(IID: Intrinsic::cttz, Tys: ArgType);
3489
3490 llvm::Type *ResultType = ConvertType(T: E->getType());
3491 // The elementwise builtins always exhibit zero-is-undef behaviour
3492 Value *ZeroUndef = Builder.getInt1(
3493 V: HasFallback || getTarget().isCLZForZeroUndef() ||
3494 BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_elementwise_ctzg);
3495 Value *Result = Builder.CreateCall(Callee: F, Args: {ArgValue, ZeroUndef});
3496 if (Result->getType() != ResultType)
3497 Result =
3498 Builder.CreateIntCast(V: Result, DestTy: ResultType, /*isSigned*/ false, Name: "cast");
3499 if (!HasFallback)
3500 return RValue::get(V: Result);
3501
3502 Value *Zero = Constant::getNullValue(Ty: ArgType);
3503 Value *IsZero = Builder.CreateICmpEQ(LHS: ArgValue, RHS: Zero, Name: "iszero");
3504 Value *FallbackValue = EmitScalarExpr(E: E->getArg(Arg: 1));
3505 Value *ResultOrFallback =
3506 Builder.CreateSelect(C: IsZero, True: FallbackValue, False: Result, Name: "ctzg");
3507 return RValue::get(V: ResultOrFallback);
3508 }
3509 case Builtin::BI__builtin_clzs:
3510 case Builtin::BI__builtin_clz:
3511 case Builtin::BI__builtin_clzl:
3512 case Builtin::BI__builtin_clzll:
3513 case Builtin::BI__builtin_clzg:
3514 case Builtin::BI__builtin_elementwise_clzg: {
3515 bool HasFallback =
3516 (BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_clzg ||
3517 BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_elementwise_clzg) &&
3518 E->getNumArgs() > 1;
3519
3520 Value *ArgValue =
3521 HasFallback ? EmitBitCountExpr(CGF&: *this, E: E->getArg(Arg: 0))
3522 : EmitCheckedArgForBuiltin(E: E->getArg(Arg: 0), Kind: BCK_CLZPassedZero);
3523
3524 llvm::Type *ArgType = ArgValue->getType();
3525 Function *F = CGM.getIntrinsic(IID: Intrinsic::ctlz, Tys: ArgType);
3526
3527 llvm::Type *ResultType = ConvertType(T: E->getType());
3528 // The elementwise builtins always exhibit zero-is-undef behaviour
3529 Value *ZeroUndef = Builder.getInt1(
3530 V: HasFallback || getTarget().isCLZForZeroUndef() ||
3531 BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_elementwise_clzg);
3532 Value *Result = Builder.CreateCall(Callee: F, Args: {ArgValue, ZeroUndef});
3533 if (Result->getType() != ResultType)
3534 Result =
3535 Builder.CreateIntCast(V: Result, DestTy: ResultType, /*isSigned*/ false, Name: "cast");
3536 if (!HasFallback)
3537 return RValue::get(V: Result);
3538
3539 Value *Zero = Constant::getNullValue(Ty: ArgType);
3540 Value *IsZero = Builder.CreateICmpEQ(LHS: ArgValue, RHS: Zero, Name: "iszero");
3541 Value *FallbackValue = EmitScalarExpr(E: E->getArg(Arg: 1));
3542 Value *ResultOrFallback =
3543 Builder.CreateSelect(C: IsZero, True: FallbackValue, False: Result, Name: "clzg");
3544 return RValue::get(V: ResultOrFallback);
3545 }
3546 case Builtin::BI__builtin_ffs:
3547 case Builtin::BI__builtin_ffsl:
3548 case Builtin::BI__builtin_ffsll: {
3549 // ffs(x) -> x ? cttz(x) + 1 : 0
3550 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
3551
3552 llvm::Type *ArgType = ArgValue->getType();
3553 Function *F = CGM.getIntrinsic(IID: Intrinsic::cttz, Tys: ArgType);
3554
3555 llvm::Type *ResultType = ConvertType(T: E->getType());
3556 Value *Tmp =
3557 Builder.CreateAdd(LHS: Builder.CreateCall(Callee: F, Args: {ArgValue, Builder.getTrue()}),
3558 RHS: llvm::ConstantInt::get(Ty: ArgType, V: 1));
3559 Value *Zero = llvm::Constant::getNullValue(Ty: ArgType);
3560 Value *IsZero = Builder.CreateICmpEQ(LHS: ArgValue, RHS: Zero, Name: "iszero");
3561 Value *Result = Builder.CreateSelect(C: IsZero, True: Zero, False: Tmp, Name: "ffs");
3562 if (Result->getType() != ResultType)
3563 Result = Builder.CreateIntCast(V: Result, DestTy: ResultType, /*isSigned*/true,
3564 Name: "cast");
3565 return RValue::get(V: Result);
3566 }
3567 case Builtin::BI__builtin_parity:
3568 case Builtin::BI__builtin_parityl:
3569 case Builtin::BI__builtin_parityll: {
3570 // parity(x) -> ctpop(x) & 1
3571 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
3572
3573 llvm::Type *ArgType = ArgValue->getType();
3574 Function *F = CGM.getIntrinsic(IID: Intrinsic::ctpop, Tys: ArgType);
3575
3576 llvm::Type *ResultType = ConvertType(T: E->getType());
3577 Value *Tmp = Builder.CreateCall(Callee: F, Args: ArgValue);
3578 Value *Result = Builder.CreateAnd(LHS: Tmp, RHS: llvm::ConstantInt::get(Ty: ArgType, V: 1));
3579 if (Result->getType() != ResultType)
3580 Result = Builder.CreateIntCast(V: Result, DestTy: ResultType, /*isSigned*/true,
3581 Name: "cast");
3582 return RValue::get(V: Result);
3583 }
3584 case Builtin::BI__lzcnt16:
3585 case Builtin::BI__lzcnt:
3586 case Builtin::BI__lzcnt64: {
3587 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
3588
3589 llvm::Type *ArgType = ArgValue->getType();
3590 Function *F = CGM.getIntrinsic(IID: Intrinsic::ctlz, Tys: ArgType);
3591
3592 llvm::Type *ResultType = ConvertType(T: E->getType());
3593 Value *Result = Builder.CreateCall(Callee: F, Args: {ArgValue, Builder.getFalse()});
3594 if (Result->getType() != ResultType)
3595 Result = Builder.CreateIntCast(V: Result, DestTy: ResultType, /*isSigned*/true,
3596 Name: "cast");
3597 return RValue::get(V: Result);
3598 }
3599 case Builtin::BI__popcnt16:
3600 case Builtin::BI__popcnt:
3601 case Builtin::BI__popcnt64:
3602 case Builtin::BI__builtin_popcount:
3603 case Builtin::BI__builtin_popcountl:
3604 case Builtin::BI__builtin_popcountll:
3605 case Builtin::BI__builtin_popcountg: {
3606 Value *ArgValue = EmitBitCountExpr(CGF&: *this, E: E->getArg(Arg: 0));
3607
3608 llvm::Type *ArgType = ArgValue->getType();
3609 Function *F = CGM.getIntrinsic(IID: Intrinsic::ctpop, Tys: ArgType);
3610
3611 llvm::Type *ResultType = ConvertType(T: E->getType());
3612 Value *Result = Builder.CreateCall(Callee: F, Args: ArgValue);
3613 if (Result->getType() != ResultType)
3614 Result =
3615 Builder.CreateIntCast(V: Result, DestTy: ResultType, /*isSigned*/ false, Name: "cast");
3616 return RValue::get(V: Result);
3617 }
3618 case Builtin::BI__builtin_unpredictable: {
3619 // Always return the argument of __builtin_unpredictable. LLVM does not
3620 // handle this builtin. Metadata for this builtin should be added directly
3621 // to instructions such as branches or switches that use it.
3622 return RValue::get(V: EmitScalarExpr(E: E->getArg(Arg: 0)));
3623 }
3624 case Builtin::BI__builtin_expect: {
3625 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
3626 llvm::Type *ArgType = ArgValue->getType();
3627
3628 Value *ExpectedValue = EmitScalarExpr(E: E->getArg(Arg: 1));
3629 // Don't generate llvm.expect on -O0 as the backend won't use it for
3630 // anything.
3631 // Note, we still IRGen ExpectedValue because it could have side-effects.
3632 if (CGM.getCodeGenOpts().OptimizationLevel == 0)
3633 return RValue::get(V: ArgValue);
3634
3635 Function *FnExpect = CGM.getIntrinsic(IID: Intrinsic::expect, Tys: ArgType);
3636 Value *Result =
3637 Builder.CreateCall(Callee: FnExpect, Args: {ArgValue, ExpectedValue}, Name: "expval");
3638 return RValue::get(V: Result);
3639 }
3640 case Builtin::BI__builtin_expect_with_probability: {
3641 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
3642 llvm::Type *ArgType = ArgValue->getType();
3643
3644 Value *ExpectedValue = EmitScalarExpr(E: E->getArg(Arg: 1));
3645 llvm::APFloat Probability(0.0);
3646 const Expr *ProbArg = E->getArg(Arg: 2);
3647 bool EvalSucceed = ProbArg->EvaluateAsFloat(Result&: Probability, Ctx: CGM.getContext());
3648 assert(EvalSucceed && "probability should be able to evaluate as float");
3649 (void)EvalSucceed;
3650 bool LoseInfo = false;
3651 Probability.convert(ToSemantics: llvm::APFloat::IEEEdouble(),
3652 RM: llvm::RoundingMode::Dynamic, losesInfo: &LoseInfo);
3653 llvm::Type *Ty = ConvertType(T: ProbArg->getType());
3654 Constant *Confidence = ConstantFP::get(Ty, V: Probability);
3655 // Don't generate llvm.expect.with.probability on -O0 as the backend
3656 // won't use it for anything.
3657 // Note, we still IRGen ExpectedValue because it could have side-effects.
3658 if (CGM.getCodeGenOpts().OptimizationLevel == 0)
3659 return RValue::get(V: ArgValue);
3660
3661 Function *FnExpect =
3662 CGM.getIntrinsic(IID: Intrinsic::expect_with_probability, Tys: ArgType);
3663 Value *Result = Builder.CreateCall(
3664 Callee: FnExpect, Args: {ArgValue, ExpectedValue, Confidence}, Name: "expval");
3665 return RValue::get(V: Result);
3666 }
3667 case Builtin::BI__builtin_assume_aligned: {
3668 const Expr *Ptr = E->getArg(Arg: 0);
3669 Value *PtrValue = EmitScalarExpr(E: Ptr);
3670 Value *OffsetValue =
3671 (E->getNumArgs() > 2) ? EmitScalarExpr(E: E->getArg(Arg: 2)) : nullptr;
3672
3673 Value *AlignmentValue = EmitScalarExpr(E: E->getArg(Arg: 1));
3674 ConstantInt *AlignmentCI = cast<ConstantInt>(Val: AlignmentValue);
3675 if (AlignmentCI->getValue().ugt(RHS: llvm::Value::MaximumAlignment))
3676 AlignmentCI = ConstantInt::get(Ty: AlignmentCI->getIntegerType(),
3677 V: llvm::Value::MaximumAlignment);
3678
3679 emitAlignmentAssumption(PtrValue, E: Ptr,
3680 /*The expr loc is sufficient.*/ AssumptionLoc: SourceLocation(),
3681 Alignment: AlignmentCI, OffsetValue);
3682 return RValue::get(V: PtrValue);
3683 }
3684 case Builtin::BI__builtin_assume_dereferenceable: {
3685 const Expr *Ptr = E->getArg(Arg: 0);
3686 const Expr *Size = E->getArg(Arg: 1);
3687 Value *PtrValue = EmitScalarExpr(E: Ptr);
3688 Value *SizeValue = EmitScalarExpr(E: Size);
3689 if (SizeValue->getType() != IntPtrTy)
3690 SizeValue =
3691 Builder.CreateIntCast(V: SizeValue, DestTy: IntPtrTy, isSigned: false, Name: "casted.size");
3692 Builder.CreateDereferenceableAssumption(PtrValue, SizeValue);
3693 return RValue::get(V: nullptr);
3694 }
3695 case Builtin::BI__assume:
3696 case Builtin::BI__builtin_assume: {
3697 if (E->getArg(Arg: 0)->HasSideEffects(Ctx: getContext()))
3698 return RValue::get(V: nullptr);
3699
3700 Value *ArgValue = EmitCheckedArgForAssume(E: E->getArg(Arg: 0));
3701 Function *FnAssume = CGM.getIntrinsic(IID: Intrinsic::assume);
3702 Builder.CreateCall(Callee: FnAssume, Args: ArgValue);
3703 return RValue::get(V: nullptr);
3704 }
3705 case Builtin::BI__builtin_assume_separate_storage: {
3706 const Expr *Arg0 = E->getArg(Arg: 0);
3707 const Expr *Arg1 = E->getArg(Arg: 1);
3708
3709 Value *Value0 = EmitScalarExpr(E: Arg0);
3710 Value *Value1 = EmitScalarExpr(E: Arg1);
3711
3712 Value *Values[] = {Value0, Value1};
3713 OperandBundleDefT<Value *> OBD("separate_storage", Values);
3714 Builder.CreateAssumption(OpBundles: {OBD});
3715 return RValue::get(V: nullptr);
3716 }
3717 case Builtin::BI__builtin_allow_runtime_check: {
3718 StringRef Kind =
3719 cast<StringLiteral>(Val: E->getArg(Arg: 0)->IgnoreParenCasts())->getString();
3720 LLVMContext &Ctx = CGM.getLLVMContext();
3721 llvm::Value *Allow = Builder.CreateCall(
3722 Callee: CGM.getIntrinsic(IID: Intrinsic::allow_runtime_check),
3723 Args: llvm::MetadataAsValue::get(Context&: Ctx, MD: llvm::MDString::get(Context&: Ctx, Str: Kind)));
3724 return RValue::get(V: Allow);
3725 }
3726 case Builtin::BI__builtin_allow_sanitize_check: {
3727 Intrinsic::ID IntrID = Intrinsic::not_intrinsic;
3728 StringRef Name =
3729 cast<StringLiteral>(Val: E->getArg(Arg: 0)->IgnoreParenCasts())->getString();
3730
3731 // We deliberately allow the use of kernel- and non-kernel names
3732 // interchangably, even when one or the other is enabled. This is consistent
3733 // with the no_sanitize-attribute, which allows either kernel- or non-kernel
3734 // name to disable instrumentation (see CodeGenFunction::StartFunction).
3735 if (getLangOpts().Sanitize.hasOneOf(K: SanitizerKind::Address |
3736 SanitizerKind::KernelAddress) &&
3737 (Name == "address" || Name == "kernel-address")) {
3738 IntrID = Intrinsic::allow_sanitize_address;
3739 } else if (getLangOpts().Sanitize.has(K: SanitizerKind::Thread) &&
3740 Name == "thread") {
3741 IntrID = Intrinsic::allow_sanitize_thread;
3742 } else if (getLangOpts().Sanitize.hasOneOf(K: SanitizerKind::Memory |
3743 SanitizerKind::KernelMemory) &&
3744 (Name == "memory" || Name == "kernel-memory")) {
3745 IntrID = Intrinsic::allow_sanitize_memory;
3746 } else if (getLangOpts().Sanitize.hasOneOf(
3747 K: SanitizerKind::HWAddress | SanitizerKind::KernelHWAddress) &&
3748 (Name == "hwaddress" || Name == "kernel-hwaddress")) {
3749 IntrID = Intrinsic::allow_sanitize_hwaddress;
3750 }
3751
3752 if (IntrID != Intrinsic::not_intrinsic) {
3753 llvm::Value *Allow = Builder.CreateCall(Callee: CGM.getIntrinsic(IID: IntrID));
3754 return RValue::get(V: Allow);
3755 }
3756 // If the checked sanitizer is not enabled, we can safely lower to false
3757 // right away. This is also more efficient, since the LowerAllowCheckPass
3758 // must not always be enabled if none of the above sanitizers are enabled.
3759 return RValue::get(V: Builder.getFalse());
3760 }
3761 case Builtin::BI__arithmetic_fence: {
3762 // Create the builtin call if FastMath is selected, and the target
3763 // supports the builtin, otherwise just return the argument.
3764 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
3765 llvm::FastMathFlags FMF = Builder.getFastMathFlags();
3766 bool isArithmeticFenceEnabled =
3767 FMF.allowReassoc() &&
3768 getContext().getTargetInfo().checkArithmeticFenceSupported();
3769 QualType ArgType = E->getArg(Arg: 0)->getType();
3770 if (ArgType->isComplexType()) {
3771 if (isArithmeticFenceEnabled) {
3772 QualType ElementType = ArgType->castAs<ComplexType>()->getElementType();
3773 ComplexPairTy ComplexVal = EmitComplexExpr(E: E->getArg(Arg: 0));
3774 Value *Real = Builder.CreateArithmeticFence(Val: ComplexVal.first,
3775 DstType: ConvertType(T: ElementType));
3776 Value *Imag = Builder.CreateArithmeticFence(Val: ComplexVal.second,
3777 DstType: ConvertType(T: ElementType));
3778 return RValue::getComplex(C: std::make_pair(x&: Real, y&: Imag));
3779 }
3780 ComplexPairTy ComplexVal = EmitComplexExpr(E: E->getArg(Arg: 0));
3781 Value *Real = ComplexVal.first;
3782 Value *Imag = ComplexVal.second;
3783 return RValue::getComplex(C: std::make_pair(x&: Real, y&: Imag));
3784 }
3785 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
3786 if (isArithmeticFenceEnabled)
3787 return RValue::get(
3788 V: Builder.CreateArithmeticFence(Val: ArgValue, DstType: ConvertType(T: ArgType)));
3789 return RValue::get(V: ArgValue);
3790 }
3791 case Builtin::BI__builtin_bswapg: {
3792 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
3793 llvm::IntegerType *IntTy = cast<llvm::IntegerType>(Val: ArgValue->getType());
3794 assert(IntTy && "LLVM's __builtin_bswapg only supports integer variants");
3795 if (IntTy->getBitWidth() == 1 || IntTy->getBitWidth() == 8)
3796 return RValue::get(V: ArgValue);
3797 assert(((IntTy->getBitWidth() % 16 == 0 && IntTy->getBitWidth() != 0)) &&
3798 "LLVM's __builtin_bswapg only supports integer variants that has a "
3799 "multiple of 16 bits as well as a single byte");
3800 return RValue::get(
3801 V: emitBuiltinWithOneOverloadedType<1>(CGF&: *this, E, IntrinsicID: Intrinsic::bswap));
3802 }
3803 case Builtin::BI__builtin_bswap16:
3804 case Builtin::BI__builtin_bswap32:
3805 case Builtin::BI__builtin_bswap64:
3806 case Builtin::BI_byteswap_ushort:
3807 case Builtin::BI_byteswap_ulong:
3808 case Builtin::BI_byteswap_uint64: {
3809 return RValue::get(
3810 V: emitBuiltinWithOneOverloadedType<1>(CGF&: *this, E, IntrinsicID: Intrinsic::bswap));
3811 }
3812 case Builtin::BI__builtin_bitreverseg: {
3813 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
3814 llvm::IntegerType *IntTy = cast<llvm::IntegerType>(Val: ArgValue->getType());
3815 assert(IntTy &&
3816 "LLVM's __builtin_bitreverseg only support integer variants");
3817 if (IntTy->getBitWidth() == 1)
3818 return RValue::get(V: ArgValue);
3819 return RValue::get(
3820 V: emitBuiltinWithOneOverloadedType<1>(CGF&: *this, E, IntrinsicID: Intrinsic::bitreverse));
3821 }
3822 case Builtin::BI__builtin_bitreverse8:
3823 case Builtin::BI__builtin_bitreverse16:
3824 case Builtin::BI__builtin_bitreverse32:
3825 case Builtin::BI__builtin_bitreverse64: {
3826 return RValue::get(
3827 V: emitBuiltinWithOneOverloadedType<1>(CGF&: *this, E, IntrinsicID: Intrinsic::bitreverse));
3828 }
3829 case Builtin::BI__builtin_rotateleft8:
3830 case Builtin::BI__builtin_rotateleft16:
3831 case Builtin::BI__builtin_rotateleft32:
3832 case Builtin::BI__builtin_rotateleft64:
3833 case Builtin::BI__builtin_stdc_rotate_left:
3834 case Builtin::BIstdc_rotate_left_uc:
3835 case Builtin::BIstdc_rotate_left_us:
3836 case Builtin::BIstdc_rotate_left_ui:
3837 case Builtin::BIstdc_rotate_left_ul:
3838 case Builtin::BIstdc_rotate_left_ull:
3839 case Builtin::BI_rotl8: // Microsoft variants of rotate left
3840 case Builtin::BI_rotl16:
3841 case Builtin::BI_rotl:
3842 case Builtin::BI_lrotl:
3843 case Builtin::BI_rotl64:
3844 return emitRotate(E, IsRotateRight: false);
3845
3846 case Builtin::BI__builtin_rotateright8:
3847 case Builtin::BI__builtin_rotateright16:
3848 case Builtin::BI__builtin_rotateright32:
3849 case Builtin::BI__builtin_rotateright64:
3850 case Builtin::BI__builtin_stdc_rotate_right:
3851 case Builtin::BIstdc_rotate_right_uc:
3852 case Builtin::BIstdc_rotate_right_us:
3853 case Builtin::BIstdc_rotate_right_ui:
3854 case Builtin::BIstdc_rotate_right_ul:
3855 case Builtin::BIstdc_rotate_right_ull:
3856 case Builtin::BI_rotr8: // Microsoft variants of rotate right
3857 case Builtin::BI_rotr16:
3858 case Builtin::BI_rotr:
3859 case Builtin::BI_lrotr:
3860 case Builtin::BI_rotr64:
3861 return emitRotate(E, IsRotateRight: true);
3862
3863 case Builtin::BIstdc_leading_zeros_uc:
3864 case Builtin::BIstdc_leading_zeros_us:
3865 case Builtin::BIstdc_leading_zeros_ui:
3866 case Builtin::BIstdc_leading_zeros_ul:
3867 case Builtin::BIstdc_leading_zeros_ull:
3868 case Builtin::BI__builtin_stdc_leading_zeros:
3869 return emitStdcCountIntrinsic(E, IntID: Intrinsic::ctlz, /*InvertArg=*/false);
3870 case Builtin::BIstdc_leading_ones_uc:
3871 case Builtin::BIstdc_leading_ones_us:
3872 case Builtin::BIstdc_leading_ones_ui:
3873 case Builtin::BIstdc_leading_ones_ul:
3874 case Builtin::BIstdc_leading_ones_ull:
3875 case Builtin::BI__builtin_stdc_leading_ones:
3876 return emitStdcCountIntrinsic(E, IntID: Intrinsic::ctlz, /*InvertArg=*/true);
3877 case Builtin::BIstdc_trailing_zeros_uc:
3878 case Builtin::BIstdc_trailing_zeros_us:
3879 case Builtin::BIstdc_trailing_zeros_ui:
3880 case Builtin::BIstdc_trailing_zeros_ul:
3881 case Builtin::BIstdc_trailing_zeros_ull:
3882 case Builtin::BI__builtin_stdc_trailing_zeros:
3883 return emitStdcCountIntrinsic(E, IntID: Intrinsic::cttz, /*InvertArg=*/false);
3884 case Builtin::BIstdc_trailing_ones_uc:
3885 case Builtin::BIstdc_trailing_ones_us:
3886 case Builtin::BIstdc_trailing_ones_ui:
3887 case Builtin::BIstdc_trailing_ones_ul:
3888 case Builtin::BIstdc_trailing_ones_ull:
3889 case Builtin::BI__builtin_stdc_trailing_ones:
3890 return emitStdcCountIntrinsic(E, IntID: Intrinsic::cttz, /*InvertArg=*/true);
3891 case Builtin::BIstdc_first_leading_zero_uc:
3892 case Builtin::BIstdc_first_leading_zero_us:
3893 case Builtin::BIstdc_first_leading_zero_ui:
3894 case Builtin::BIstdc_first_leading_zero_ul:
3895 case Builtin::BIstdc_first_leading_zero_ull:
3896 case Builtin::BI__builtin_stdc_first_leading_zero:
3897 return emitStdcFirstBit(E, IntID: Intrinsic::ctlz, /*InvertArg=*/true);
3898 case Builtin::BIstdc_first_leading_one_uc:
3899 case Builtin::BIstdc_first_leading_one_us:
3900 case Builtin::BIstdc_first_leading_one_ui:
3901 case Builtin::BIstdc_first_leading_one_ul:
3902 case Builtin::BIstdc_first_leading_one_ull:
3903 case Builtin::BI__builtin_stdc_first_leading_one:
3904 return emitStdcFirstBit(E, IntID: Intrinsic::ctlz, /*InvertArg=*/false);
3905 case Builtin::BIstdc_first_trailing_zero_uc:
3906 case Builtin::BIstdc_first_trailing_zero_us:
3907 case Builtin::BIstdc_first_trailing_zero_ui:
3908 case Builtin::BIstdc_first_trailing_zero_ul:
3909 case Builtin::BIstdc_first_trailing_zero_ull:
3910 case Builtin::BI__builtin_stdc_first_trailing_zero:
3911 return emitStdcFirstBit(E, IntID: Intrinsic::cttz, /*InvertArg=*/true);
3912 case Builtin::BIstdc_first_trailing_one_uc:
3913 case Builtin::BIstdc_first_trailing_one_us:
3914 case Builtin::BIstdc_first_trailing_one_ui:
3915 case Builtin::BIstdc_first_trailing_one_ul:
3916 case Builtin::BIstdc_first_trailing_one_ull:
3917 case Builtin::BI__builtin_stdc_first_trailing_one:
3918 return emitStdcFirstBit(E, IntID: Intrinsic::cttz, /*InvertArg=*/false);
3919 case Builtin::BIstdc_count_zeros_uc:
3920 case Builtin::BIstdc_count_zeros_us:
3921 case Builtin::BIstdc_count_zeros_ui:
3922 case Builtin::BIstdc_count_zeros_ul:
3923 case Builtin::BIstdc_count_zeros_ull:
3924 case Builtin::BI__builtin_stdc_count_zeros:
3925 return emitStdcBitWidthMinus(E, IntID: Intrinsic::ctpop, /*IsPop=*/true);
3926 case Builtin::BIstdc_count_ones_uc:
3927 case Builtin::BIstdc_count_ones_us:
3928 case Builtin::BIstdc_count_ones_ui:
3929 case Builtin::BIstdc_count_ones_ul:
3930 case Builtin::BIstdc_count_ones_ull:
3931 case Builtin::BI__builtin_stdc_count_ones:
3932 return emitStdcCountIntrinsic(E, IntID: Intrinsic::ctpop, /*InvertArg=*/false,
3933 /*IsPop=*/true);
3934 case Builtin::BIstdc_has_single_bit_uc:
3935 case Builtin::BIstdc_has_single_bit_us:
3936 case Builtin::BIstdc_has_single_bit_ui:
3937 case Builtin::BIstdc_has_single_bit_ul:
3938 case Builtin::BIstdc_has_single_bit_ull:
3939 case Builtin::BI__builtin_stdc_has_single_bit: {
3940 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
3941 llvm::Type *ArgType = ArgValue->getType();
3942 Value *One = ConstantInt::get(Ty: ArgType, V: 1);
3943 Function *F = CGM.getIntrinsic(IID: Intrinsic::ctpop, Tys: ArgType);
3944 Value *PopCnt = Builder.CreateCall(Callee: F, Args: ArgValue);
3945 return RValue::get(V: Builder.CreateICmpEQ(LHS: PopCnt, RHS: One));
3946 }
3947 case Builtin::BIstdc_bit_width_uc:
3948 case Builtin::BIstdc_bit_width_us:
3949 case Builtin::BIstdc_bit_width_ui:
3950 case Builtin::BIstdc_bit_width_ul:
3951 case Builtin::BIstdc_bit_width_ull:
3952 case Builtin::BI__builtin_stdc_bit_width:
3953 return emitStdcBitWidthMinus(E, IntID: Intrinsic::ctlz, /*IsPop=*/false);
3954 case Builtin::BIstdc_bit_floor_uc:
3955 case Builtin::BIstdc_bit_floor_us:
3956 case Builtin::BIstdc_bit_floor_ui:
3957 case Builtin::BIstdc_bit_floor_ul:
3958 case Builtin::BIstdc_bit_floor_ull:
3959 case Builtin::BI__builtin_stdc_bit_floor: {
3960 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
3961 llvm::Type *ArgType = ArgValue->getType();
3962 unsigned BitWidth = ArgType->getIntegerBitWidth();
3963 Value *Zero = ConstantInt::get(Ty: ArgType, V: 0);
3964 Value *One = ConstantInt::get(Ty: ArgType, V: 1);
3965 Function *F = CGM.getIntrinsic(IID: Intrinsic::ctlz, Tys: ArgType);
3966 Value *LZ = Builder.CreateCall(Callee: F, Args: {ArgValue, Builder.getTrue()});
3967 Value *ShiftAmt =
3968 Builder.CreateSub(LHS: ConstantInt::get(Ty: ArgType, V: BitWidth - 1), RHS: LZ);
3969 Value *Shifted = Builder.CreateShl(LHS: One, RHS: ShiftAmt);
3970 Value *IsZero = Builder.CreateICmpEQ(LHS: ArgValue, RHS: Zero);
3971 Value *Result = Builder.CreateSelect(C: IsZero, True: Zero, False: Shifted);
3972 return RValue::get(V: Result);
3973 }
3974 case Builtin::BIstdc_bit_ceil_uc:
3975 case Builtin::BIstdc_bit_ceil_us:
3976 case Builtin::BIstdc_bit_ceil_ui:
3977 case Builtin::BIstdc_bit_ceil_ul:
3978 case Builtin::BIstdc_bit_ceil_ull:
3979 case Builtin::BI__builtin_stdc_bit_ceil: {
3980 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
3981 llvm::Type *ArgType = ArgValue->getType();
3982 Value *One = ConstantInt::get(Ty: ArgType, V: 1);
3983 Value *IsLEOne = Builder.CreateICmpULE(LHS: ArgValue, RHS: One, Name: "isleone");
3984
3985 BasicBlock *EntryBB = Builder.GetInsertBlock();
3986 BasicBlock *CalcBB = createBasicBlock(name: "bitceil.calc", parent: CurFn);
3987 BasicBlock *MergeBB = createBasicBlock(name: "bitceil.merge", parent: CurFn);
3988
3989 Builder.CreateCondBr(Cond: IsLEOne, True: MergeBB, False: CalcBB);
3990
3991 Builder.SetInsertPoint(CalcBB);
3992 Function *F = CGM.getIntrinsic(IID: Intrinsic::ctlz, Tys: ArgType);
3993 Value *ArgMinusOne = Builder.CreateSub(LHS: ArgValue, RHS: One);
3994 Value *LZ = Builder.CreateCall(Callee: F, Args: {ArgMinusOne, Builder.getFalse()});
3995 // 2<<(BitWidth-1-LZ) to get the next power of two. The shift
3996 // amount is always in [0, BitWidth-1], so when LZ==0 (argument has its MSB
3997 // set), the result wraps to 0
3998 unsigned BitWidth = ArgType->getIntegerBitWidth();
3999 Value *ShiftAmt =
4000 Builder.CreateSub(LHS: ConstantInt::get(Ty: ArgType, V: BitWidth - 1), RHS: LZ);
4001 Value *Two = Builder.CreateShl(LHS: One, RHS: One);
4002 Value *Tmp = Builder.CreateShl(LHS: Two, RHS: ShiftAmt);
4003 Builder.CreateBr(Dest: MergeBB);
4004
4005 Builder.SetInsertPoint(MergeBB);
4006 PHINode *Phi = Builder.CreatePHI(Ty: ArgType, NumReservedValues: 2);
4007 Phi->addIncoming(V: One, BB: EntryBB);
4008 Phi->addIncoming(V: Tmp, BB: CalcBB);
4009 return RValue::get(V: Phi);
4010 }
4011
4012 // stdc_memreverse8u8 is a no-op (single byte, nothing to swap).
4013 case Builtin::BIstdc_memreverse8u8:
4014 return RValue::get(V: EmitScalarExpr(E: E->getArg(Arg: 0)));
4015
4016 case Builtin::BIstdc_memreverse8u16:
4017 case Builtin::BIstdc_memreverse8u32:
4018 case Builtin::BIstdc_memreverse8u64:
4019 return RValue::get(
4020 V: emitBuiltinWithOneOverloadedType<1>(CGF&: *this, E, IntrinsicID: Intrinsic::bswap));
4021
4022 case Builtin::BIstdc_memreverse8:
4023 case Builtin::BI__builtin_stdc_memreverse8: {
4024 Expr::EvalResult R;
4025 if (E->getArg(Arg: 0)->EvaluateAsInt(Result&: R, Ctx: getContext())) {
4026 uint64_t Size = R.Val.getInt().getZExtValue();
4027 if (Size <= 1) {
4028 EmitIgnoredExpr(E: E->getArg(Arg: 1));
4029 return RValue::get(V: nullptr);
4030 }
4031 if (Size == 2 || Size == 4 || Size == 8) {
4032 llvm::Type *IntTy = Builder.getIntNTy(N: Size * 8);
4033 Address PtrAddr = EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
4034 Address Addr = PtrAddr.withElementType(ElemTy: IntTy);
4035 Value *Val = Builder.CreateLoad(Addr);
4036 Function *F = CGM.getIntrinsic(IID: Intrinsic::bswap, Tys: IntTy);
4037 Value *Swapped = Builder.CreateCall(Callee: F, Args: Val);
4038 Builder.CreateStore(Val: Swapped, Addr);
4039 return RValue::get(V: nullptr);
4040 }
4041 }
4042
4043 // General case: fall back to the library function stdc_memreverse8.
4044 break;
4045 }
4046
4047 case Builtin::BI__builtin_constant_p: {
4048 llvm::Type *ResultType = ConvertType(T: E->getType());
4049
4050 const Expr *Arg = E->getArg(Arg: 0);
4051 QualType ArgType = Arg->getType();
4052 // FIXME: The allowance for Obj-C pointers and block pointers is historical
4053 // and likely a mistake.
4054 if (!ArgType->isIntegralOrEnumerationType() && !ArgType->isFloatingType() &&
4055 !ArgType->isObjCObjectPointerType() && !ArgType->isBlockPointerType())
4056 // Per the GCC documentation, only numeric constants are recognized after
4057 // inlining.
4058 return RValue::get(V: ConstantInt::get(Ty: ResultType, V: 0));
4059
4060 if (Arg->HasSideEffects(Ctx: getContext()))
4061 // The argument is unevaluated, so be conservative if it might have
4062 // side-effects.
4063 return RValue::get(V: ConstantInt::get(Ty: ResultType, V: 0));
4064
4065 Value *ArgValue = EmitScalarExpr(E: Arg);
4066 if (ArgType->isObjCObjectPointerType()) {
4067 // Convert Objective-C objects to id because we cannot distinguish between
4068 // LLVM types for Obj-C classes as they are opaque.
4069 ArgType = CGM.getContext().getObjCIdType();
4070 ArgValue = Builder.CreateBitCast(V: ArgValue, DestTy: ConvertType(T: ArgType));
4071 }
4072 Function *F =
4073 CGM.getIntrinsic(IID: Intrinsic::is_constant, Tys: ConvertType(T: ArgType));
4074 Value *Result = Builder.CreateCall(Callee: F, Args: ArgValue);
4075 if (Result->getType() != ResultType)
4076 Result = Builder.CreateIntCast(V: Result, DestTy: ResultType, /*isSigned*/false);
4077 return RValue::get(V: Result);
4078 }
4079 case Builtin::BI__builtin_dynamic_object_size:
4080 case Builtin::BI__builtin_object_size: {
4081 unsigned Type =
4082 E->getArg(Arg: 1)->EvaluateKnownConstInt(Ctx: getContext()).getZExtValue();
4083 auto *ResType = cast<llvm::IntegerType>(Val: ConvertType(T: E->getType()));
4084
4085 // We pass this builtin onto the optimizer so that it can figure out the
4086 // object size in more complex cases.
4087 bool IsDynamic = BuiltinID == Builtin::BI__builtin_dynamic_object_size;
4088 return RValue::get(V: emitBuiltinObjectSize(E: E->getArg(Arg: 0), Type, ResType,
4089 /*EmittedE=*/nullptr, IsDynamic));
4090 }
4091 case Builtin::BI__builtin_counted_by_ref: {
4092 // Default to returning '(void *) 0'.
4093 llvm::Value *Result = llvm::ConstantPointerNull::get(
4094 T: llvm::PointerType::getUnqual(C&: getLLVMContext()));
4095
4096 const Expr *Arg = E->getArg(Arg: 0)->IgnoreParenImpCasts();
4097
4098 if (auto *UO = dyn_cast<UnaryOperator>(Val: Arg);
4099 UO && UO->getOpcode() == UO_AddrOf) {
4100 Arg = UO->getSubExpr()->IgnoreParenImpCasts();
4101
4102 if (auto *ASE = dyn_cast<ArraySubscriptExpr>(Val: Arg))
4103 Arg = ASE->getBase()->IgnoreParenImpCasts();
4104 }
4105
4106 if (const MemberExpr *ME = dyn_cast_if_present<MemberExpr>(Val: Arg)) {
4107 if (auto *CATy =
4108 ME->getMemberDecl()->getType()->getAs<CountAttributedType>();
4109 CATy && CATy->getKind() == CountAttributedType::CountedBy) {
4110 const auto *MemberDecl = cast<FieldDecl>(Val: ME->getMemberDecl());
4111 if (const FieldDecl *CountFD = MemberDecl->findCountedByField())
4112 Result = GetCountedByFieldExprGEP(Base: Arg, FD: MemberDecl, CountDecl: CountFD);
4113 else
4114 llvm::report_fatal_error(reason: "Cannot find the counted_by 'count' field");
4115 }
4116 }
4117
4118 return RValue::get(V: Result);
4119 }
4120 case Builtin::BI__builtin_prefetch: {
4121 Value *Locality, *RW, *Address = EmitScalarExpr(E: E->getArg(Arg: 0));
4122 unsigned ICEArguments = (1 << 1) | (1 << 2);
4123 RW = (E->getNumArgs() > 1) ? EmitScalarOrConstFoldImmArg(ICEArguments, Idx: 1, E)
4124 : llvm::ConstantInt::get(Ty: Int32Ty, V: 0);
4125 RW = Builder.CreateZExtOrTrunc(V: RW, DestTy: Int32Ty);
4126 Locality = (E->getNumArgs() > 2)
4127 ? EmitScalarOrConstFoldImmArg(ICEArguments, Idx: 2, E)
4128 : llvm::ConstantInt::get(Ty: Int32Ty, V: 3);
4129 Locality = Builder.CreateZExtOrTrunc(V: Locality, DestTy: Int32Ty);
4130 Value *Data = llvm::ConstantInt::get(Ty: Int32Ty, V: 1);
4131 Function *F = CGM.getIntrinsic(IID: Intrinsic::prefetch, Tys: Address->getType());
4132 Builder.CreateCall(Callee: F, Args: {Address, RW, Locality, Data});
4133 return RValue::get(V: nullptr);
4134 }
4135 case Builtin::BI__builtin_readcyclecounter: {
4136 Function *F = CGM.getIntrinsic(IID: Intrinsic::readcyclecounter);
4137 return RValue::get(V: Builder.CreateCall(Callee: F));
4138 }
4139 case Builtin::BI__builtin_readsteadycounter: {
4140 Function *F = CGM.getIntrinsic(IID: Intrinsic::readsteadycounter);
4141 return RValue::get(V: Builder.CreateCall(Callee: F));
4142 }
4143 case Builtin::BI__builtin___clear_cache: {
4144 Value *Begin = EmitScalarExpr(E: E->getArg(Arg: 0));
4145 Value *End = EmitScalarExpr(E: E->getArg(Arg: 1));
4146 Function *F = CGM.getIntrinsic(IID: Intrinsic::clear_cache, Tys: {CGM.DefaultPtrTy});
4147 return RValue::get(V: Builder.CreateCall(Callee: F, Args: {Begin, End}));
4148 }
4149 case Builtin::BI__builtin_trap:
4150 EmitTrapCall(IntrID: Intrinsic::trap);
4151 return RValue::get(V: nullptr);
4152 case Builtin::BI__builtin_verbose_trap: {
4153 llvm::DebugLoc CallLocation = Builder.getCurrentDebugLocation();
4154 llvm::DILocation *TrapLocation = CallLocation;
4155 if (getDebugInfo()) {
4156 TrapLocation = getDebugInfo()->CreateTrapFailureMessageFor(
4157 TrapLocation, Category: *E->getArg(Arg: 0)->tryEvaluateString(Ctx&: getContext()),
4158 FailureMsg: *E->getArg(Arg: 1)->tryEvaluateString(Ctx&: getContext()));
4159 // Keep the trap on the builtin's source line. A line-zero location would
4160 // leave the trap attributed to the preceding line in the line table.
4161 TrapLocation = llvm::DILocation::get(
4162 Context&: getLLVMContext(), Line: CallLocation.getLine(), Column: CallLocation.getCol(),
4163 Scope: TrapLocation->getScope(), InlinedAt: TrapLocation->getInlinedAt());
4164 }
4165 ApplyDebugLocation ApplyTrapDI(*this, TrapLocation);
4166 // Currently no attempt is made to prevent traps from being merged.
4167 EmitTrapCall(IntrID: Intrinsic::trap);
4168 return RValue::get(V: nullptr);
4169 }
4170 case Builtin::BI__debugbreak:
4171 EmitTrapCall(IntrID: Intrinsic::debugtrap);
4172 return RValue::get(V: nullptr);
4173 case Builtin::BI__builtin_unreachable: {
4174 EmitUnreachable(Loc: E->getExprLoc());
4175
4176 // We do need to preserve an insertion point.
4177 EmitBlock(BB: createBasicBlock(name: "unreachable.cont"));
4178
4179 return RValue::get(V: nullptr);
4180 }
4181
4182 case Builtin::BI__builtin_powi:
4183 case Builtin::BI__builtin_powif:
4184 case Builtin::BI__builtin_powil: {
4185 llvm::Value *Src0 = EmitScalarExpr(E: E->getArg(Arg: 0));
4186 llvm::Value *Src1 = EmitScalarExpr(E: E->getArg(Arg: 1));
4187
4188 if (Builder.getIsFPConstrained()) {
4189 // FIXME: llvm.powi has 2 mangling types,
4190 // llvm.experimental.constrained.powi has one.
4191 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
4192 Function *F = CGM.getIntrinsic(IID: Intrinsic::experimental_constrained_powi,
4193 Tys: Src0->getType());
4194 return RValue::get(V: Builder.CreateConstrainedFPCall(Callee: F, Args: { Src0, Src1 }));
4195 }
4196
4197 Function *F = CGM.getIntrinsic(IID: Intrinsic::powi,
4198 Tys: { Src0->getType(), Src1->getType() });
4199 return RValue::get(V: Builder.CreateCall(Callee: F, Args: { Src0, Src1 }));
4200 }
4201 case Builtin::BI__builtin_frexpl: {
4202 // Linux PPC will not be adding additional PPCDoubleDouble support.
4203 // WIP to switch default to IEEE long double. Will emit libcall for
4204 // frexpl instead of legalizing this type in the BE.
4205 if (&getTarget().getLongDoubleFormat() == &llvm::APFloat::PPCDoubleDouble())
4206 break;
4207 [[fallthrough]];
4208 }
4209 case Builtin::BI__builtin_frexp:
4210 case Builtin::BI__builtin_frexpf:
4211 case Builtin::BI__builtin_frexpf128:
4212 case Builtin::BI__builtin_frexpf16:
4213 return RValue::get(V: emitFrexpBuiltin(CGF&: *this, E, IntrinsicID: Intrinsic::frexp));
4214 case Builtin::BImodf:
4215 case Builtin::BImodff:
4216 case Builtin::BImodfl:
4217 case Builtin::BI__builtin_modf:
4218 case Builtin::BI__builtin_modff:
4219 case Builtin::BI__builtin_modfl:
4220 if (Builder.getIsFPConstrained())
4221 break; // TODO: Emit constrained modf intrinsic once one exists.
4222 return RValue::get(V: emitModfBuiltin(CGF&: *this, E, IntrinsicID: Intrinsic::modf));
4223 case Builtin::BI__builtin_isgreater:
4224 case Builtin::BI__builtin_isgreaterequal:
4225 case Builtin::BI__builtin_isless:
4226 case Builtin::BI__builtin_islessequal:
4227 case Builtin::BI__builtin_islessgreater:
4228 case Builtin::BI__builtin_isunordered: {
4229 // Ordered comparisons: we know the arguments to these are matching scalar
4230 // floating point values.
4231 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
4232 Value *LHS = EmitScalarExpr(E: E->getArg(Arg: 0));
4233 Value *RHS = EmitScalarExpr(E: E->getArg(Arg: 1));
4234
4235 switch (BuiltinID) {
4236 default: llvm_unreachable("Unknown ordered comparison");
4237 case Builtin::BI__builtin_isgreater:
4238 LHS = Builder.CreateFCmpOGT(LHS, RHS, Name: "cmp");
4239 break;
4240 case Builtin::BI__builtin_isgreaterequal:
4241 LHS = Builder.CreateFCmpOGE(LHS, RHS, Name: "cmp");
4242 break;
4243 case Builtin::BI__builtin_isless:
4244 LHS = Builder.CreateFCmpOLT(LHS, RHS, Name: "cmp");
4245 break;
4246 case Builtin::BI__builtin_islessequal:
4247 LHS = Builder.CreateFCmpOLE(LHS, RHS, Name: "cmp");
4248 break;
4249 case Builtin::BI__builtin_islessgreater:
4250 LHS = Builder.CreateFCmpONE(LHS, RHS, Name: "cmp");
4251 break;
4252 case Builtin::BI__builtin_isunordered:
4253 LHS = Builder.CreateFCmpUNO(LHS, RHS, Name: "cmp");
4254 break;
4255 }
4256 // ZExt bool to int type.
4257 return RValue::get(V: Builder.CreateZExt(V: LHS, DestTy: ConvertType(T: E->getType())));
4258 }
4259
4260 case Builtin::BI__builtin_isnan: {
4261 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
4262 Value *V = EmitScalarExpr(E: E->getArg(Arg: 0));
4263 if (Value *Result = tryUseTestFPKind(CGF&: *this, BuiltinID, V))
4264 return RValue::get(V: Result);
4265 return RValue::get(
4266 V: Builder.CreateZExt(V: Builder.createIsFPClass(FPNum: V, Test: FPClassTest::fcNan),
4267 DestTy: ConvertType(T: E->getType())));
4268 }
4269
4270 case Builtin::BI__builtin_issignaling: {
4271 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
4272 Value *V = EmitScalarExpr(E: E->getArg(Arg: 0));
4273 return RValue::get(
4274 V: Builder.CreateZExt(V: Builder.createIsFPClass(FPNum: V, Test: FPClassTest::fcSNan),
4275 DestTy: ConvertType(T: E->getType())));
4276 }
4277
4278 case Builtin::BI__builtin_isinf: {
4279 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
4280 Value *V = EmitScalarExpr(E: E->getArg(Arg: 0));
4281 if (Value *Result = tryUseTestFPKind(CGF&: *this, BuiltinID, V))
4282 return RValue::get(V: Result);
4283 return RValue::get(
4284 V: Builder.CreateZExt(V: Builder.createIsFPClass(FPNum: V, Test: FPClassTest::fcInf),
4285 DestTy: ConvertType(T: E->getType())));
4286 }
4287
4288 case Builtin::BIfinite:
4289 case Builtin::BI__finite:
4290 case Builtin::BIfinitef:
4291 case Builtin::BI__finitef:
4292 case Builtin::BIfinitel:
4293 case Builtin::BI__finitel:
4294 case Builtin::BI__builtin_isfinite: {
4295 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
4296 Value *V = EmitScalarExpr(E: E->getArg(Arg: 0));
4297 if (Value *Result = tryUseTestFPKind(CGF&: *this, BuiltinID, V))
4298 return RValue::get(V: Result);
4299 return RValue::get(
4300 V: Builder.CreateZExt(V: Builder.createIsFPClass(FPNum: V, Test: FPClassTest::fcFinite),
4301 DestTy: ConvertType(T: E->getType())));
4302 }
4303
4304 case Builtin::BI__builtin_isnormal: {
4305 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
4306 Value *V = EmitScalarExpr(E: E->getArg(Arg: 0));
4307 return RValue::get(
4308 V: Builder.CreateZExt(V: Builder.createIsFPClass(FPNum: V, Test: FPClassTest::fcNormal),
4309 DestTy: ConvertType(T: E->getType())));
4310 }
4311
4312 case Builtin::BI__builtin_issubnormal: {
4313 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
4314 Value *V = EmitScalarExpr(E: E->getArg(Arg: 0));
4315 return RValue::get(
4316 V: Builder.CreateZExt(V: Builder.createIsFPClass(FPNum: V, Test: FPClassTest::fcSubnormal),
4317 DestTy: ConvertType(T: E->getType())));
4318 }
4319
4320 case Builtin::BI__builtin_iszero: {
4321 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
4322 Value *V = EmitScalarExpr(E: E->getArg(Arg: 0));
4323 return RValue::get(
4324 V: Builder.CreateZExt(V: Builder.createIsFPClass(FPNum: V, Test: FPClassTest::fcZero),
4325 DestTy: ConvertType(T: E->getType())));
4326 }
4327
4328 case Builtin::BI__builtin_isfpclass: {
4329 Expr::EvalResult Result;
4330 if (!E->getArg(Arg: 1)->EvaluateAsInt(Result, Ctx: CGM.getContext()))
4331 break;
4332 uint64_t Test = Result.Val.getInt().getLimitedValue();
4333 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
4334 Value *V = EmitScalarExpr(E: E->getArg(Arg: 0));
4335 return RValue::get(V: Builder.CreateZExt(V: Builder.createIsFPClass(FPNum: V, Test),
4336 DestTy: ConvertType(T: E->getType())));
4337 }
4338
4339 case Builtin::BI__builtin_nondeterministic_value: {
4340 llvm::Type *Ty = ConvertType(T: E->getArg(Arg: 0)->getType());
4341
4342 Value *Result = PoisonValue::get(T: Ty);
4343 Result = Builder.CreateFreeze(V: Result);
4344
4345 return RValue::get(V: Result);
4346 }
4347
4348 case Builtin::BI__builtin_elementwise_abs: {
4349 Value *Result;
4350 QualType QT = E->getArg(Arg: 0)->getType();
4351
4352 if (auto *VecTy = QT->getAs<VectorType>())
4353 QT = VecTy->getElementType();
4354 if (QT->isIntegerType())
4355 Result = Builder.CreateBinaryIntrinsic(
4356 ID: Intrinsic::abs, LHS: EmitScalarExpr(E: E->getArg(Arg: 0)), RHS: Builder.getFalse(),
4357 FMFSource: nullptr, Name: "elt.abs");
4358 else
4359 Result = emitBuiltinWithOneOverloadedType<1>(CGF&: *this, E, IntrinsicID: Intrinsic::fabs,
4360 Name: "elt.abs");
4361
4362 return RValue::get(V: Result);
4363 }
4364 case Builtin::BI__builtin_elementwise_bitreverse:
4365 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4366 CGF&: *this, E, IntrinsicID: Intrinsic::bitreverse, Name: "elt.bitreverse"));
4367 case Builtin::BI__builtin_elementwise_popcount:
4368 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4369 CGF&: *this, E, IntrinsicID: Intrinsic::ctpop, Name: "elt.ctpop"));
4370 case Builtin::BI__builtin_elementwise_canonicalize:
4371 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4372 CGF&: *this, E, IntrinsicID: Intrinsic::canonicalize, Name: "elt.canonicalize"));
4373 case Builtin::BI__builtin_elementwise_copysign:
4374 return RValue::get(
4375 V: emitBuiltinWithOneOverloadedType<2>(CGF&: *this, E, IntrinsicID: Intrinsic::copysign));
4376 case Builtin::BI__builtin_elementwise_fshl:
4377 return RValue::get(
4378 V: emitBuiltinWithOneOverloadedType<3>(CGF&: *this, E, IntrinsicID: Intrinsic::fshl));
4379 case Builtin::BI__builtin_elementwise_fshr:
4380 return RValue::get(
4381 V: emitBuiltinWithOneOverloadedType<3>(CGF&: *this, E, IntrinsicID: Intrinsic::fshr));
4382 case Builtin::BI__builtin_elementwise_clmul:
4383 return RValue::get(
4384 V: emitBuiltinWithOneOverloadedType<2>(CGF&: *this, E, IntrinsicID: Intrinsic::clmul));
4385 case Builtin::BI__builtin_elementwise_pext:
4386 return RValue::get(
4387 V: emitBuiltinWithOneOverloadedType<2>(CGF&: *this, E, IntrinsicID: Intrinsic::pext));
4388 case Builtin::BI__builtin_elementwise_pdep:
4389 return RValue::get(
4390 V: emitBuiltinWithOneOverloadedType<2>(CGF&: *this, E, IntrinsicID: Intrinsic::pdep));
4391
4392 case Builtin::BI__builtin_elementwise_add_sat:
4393 case Builtin::BI__builtin_elementwise_sub_sat: {
4394 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
4395 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
4396 Value *Result;
4397 assert(Op0->getType()->isIntOrIntVectorTy() && "integer type expected");
4398 QualType Ty = E->getArg(Arg: 0)->getType();
4399 if (auto *VecTy = Ty->getAs<VectorType>())
4400 Ty = VecTy->getElementType();
4401 bool IsSigned = Ty->isSignedIntegerType();
4402 unsigned Opc;
4403 if (BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_elementwise_add_sat)
4404 Opc = IsSigned ? Intrinsic::sadd_sat : Intrinsic::uadd_sat;
4405 else
4406 Opc = IsSigned ? Intrinsic::ssub_sat : Intrinsic::usub_sat;
4407 Result = Builder.CreateBinaryIntrinsic(ID: Opc, LHS: Op0, RHS: Op1, FMFSource: nullptr, Name: "elt.sat");
4408 return RValue::get(V: Result);
4409 }
4410
4411 case Builtin::BI__builtin_elementwise_max: {
4412 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
4413 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
4414 Value *Result;
4415 if (Op0->getType()->isIntOrIntVectorTy()) {
4416 QualType Ty = E->getArg(Arg: 0)->getType();
4417 Result = Builder.CreateBinaryIntrinsic(
4418 ID: Ty->hasSignedIntegerRepresentation() ? Intrinsic::smax
4419 : Intrinsic::umax,
4420 LHS: Op0, RHS: Op1, FMFSource: nullptr, Name: "elt.max");
4421 } else
4422 Result = Builder.CreateMaxNum(LHS: Op0, RHS: Op1, /*FMFSource=*/nullptr, Name: "elt.max");
4423 return RValue::get(V: Result);
4424 }
4425 case Builtin::BI__builtin_elementwise_min: {
4426 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
4427 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
4428 Value *Result;
4429 if (Op0->getType()->isIntOrIntVectorTy()) {
4430 QualType Ty = E->getArg(Arg: 0)->getType();
4431 Result = Builder.CreateBinaryIntrinsic(
4432 ID: Ty->hasSignedIntegerRepresentation() ? Intrinsic::smin
4433 : Intrinsic::umin,
4434 LHS: Op0, RHS: Op1, FMFSource: nullptr, Name: "elt.min");
4435 } else
4436 Result = Builder.CreateMinNum(LHS: Op0, RHS: Op1, /*FMFSource=*/nullptr, Name: "elt.min");
4437 return RValue::get(V: Result);
4438 }
4439
4440 case Builtin::BI__builtin_elementwise_maxnum: {
4441 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
4442 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
4443 Value *Result = Builder.CreateBinaryIntrinsic(ID: llvm::Intrinsic::maxnum, LHS: Op0,
4444 RHS: Op1, FMFSource: nullptr, Name: "elt.maxnum");
4445 return RValue::get(V: Result);
4446 }
4447
4448 case Builtin::BI__builtin_elementwise_minnum: {
4449 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
4450 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
4451 Value *Result = Builder.CreateBinaryIntrinsic(ID: llvm::Intrinsic::minnum, LHS: Op0,
4452 RHS: Op1, FMFSource: nullptr, Name: "elt.minnum");
4453 return RValue::get(V: Result);
4454 }
4455
4456 case Builtin::BI__builtin_elementwise_maximum: {
4457 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
4458 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
4459 Value *Result = Builder.CreateBinaryIntrinsic(ID: Intrinsic::maximum, LHS: Op0, RHS: Op1,
4460 FMFSource: nullptr, Name: "elt.maximum");
4461 return RValue::get(V: Result);
4462 }
4463
4464 case Builtin::BI__builtin_elementwise_minimum: {
4465 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
4466 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
4467 Value *Result = Builder.CreateBinaryIntrinsic(ID: Intrinsic::minimum, LHS: Op0, RHS: Op1,
4468 FMFSource: nullptr, Name: "elt.minimum");
4469 return RValue::get(V: Result);
4470 }
4471
4472 case Builtin::BI__builtin_elementwise_maximumnum: {
4473 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
4474 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
4475 Value *Result = Builder.CreateBinaryIntrinsic(
4476 ID: Intrinsic::maximumnum, LHS: Op0, RHS: Op1, FMFSource: nullptr, Name: "elt.maximumnum");
4477 return RValue::get(V: Result);
4478 }
4479
4480 case Builtin::BI__builtin_elementwise_minimumnum: {
4481 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
4482 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
4483 Value *Result = Builder.CreateBinaryIntrinsic(
4484 ID: Intrinsic::minimumnum, LHS: Op0, RHS: Op1, FMFSource: nullptr, Name: "elt.minimumnum");
4485 return RValue::get(V: Result);
4486 }
4487
4488 case Builtin::BI__builtin_reduce_max: {
4489 auto GetIntrinsicID = [this](QualType QT) {
4490 if (auto *VecTy = QT->getAs<VectorType>())
4491 QT = VecTy->getElementType();
4492 else if (QT->isSizelessVectorType())
4493 QT = QT->getSizelessVectorEltType(Ctx: CGM.getContext());
4494
4495 if (QT->isSignedIntegerType())
4496 return Intrinsic::vector_reduce_smax;
4497 if (QT->isUnsignedIntegerType())
4498 return Intrinsic::vector_reduce_umax;
4499 assert(QT->isFloatingType() && "must have a float here");
4500 return Intrinsic::vector_reduce_fmax;
4501 };
4502 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4503 CGF&: *this, E, IntrinsicID: GetIntrinsicID(E->getArg(Arg: 0)->getType()), Name: "rdx.min"));
4504 }
4505
4506 case Builtin::BI__builtin_reduce_min: {
4507 auto GetIntrinsicID = [this](QualType QT) {
4508 if (auto *VecTy = QT->getAs<VectorType>())
4509 QT = VecTy->getElementType();
4510 else if (QT->isSizelessVectorType())
4511 QT = QT->getSizelessVectorEltType(Ctx: CGM.getContext());
4512
4513 if (QT->isSignedIntegerType())
4514 return Intrinsic::vector_reduce_smin;
4515 if (QT->isUnsignedIntegerType())
4516 return Intrinsic::vector_reduce_umin;
4517 assert(QT->isFloatingType() && "must have a float here");
4518 return Intrinsic::vector_reduce_fmin;
4519 };
4520
4521 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4522 CGF&: *this, E, IntrinsicID: GetIntrinsicID(E->getArg(Arg: 0)->getType()), Name: "rdx.min"));
4523 }
4524
4525 case Builtin::BI__builtin_reduce_add:
4526 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4527 CGF&: *this, E, IntrinsicID: Intrinsic::vector_reduce_add, Name: "rdx.add"));
4528 case Builtin::BI__builtin_reduce_mul:
4529 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4530 CGF&: *this, E, IntrinsicID: Intrinsic::vector_reduce_mul, Name: "rdx.mul"));
4531 case Builtin::BI__builtin_reduce_xor:
4532 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4533 CGF&: *this, E, IntrinsicID: Intrinsic::vector_reduce_xor, Name: "rdx.xor"));
4534 case Builtin::BI__builtin_reduce_or:
4535 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4536 CGF&: *this, E, IntrinsicID: Intrinsic::vector_reduce_or, Name: "rdx.or"));
4537 case Builtin::BI__builtin_reduce_and:
4538 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4539 CGF&: *this, E, IntrinsicID: Intrinsic::vector_reduce_and, Name: "rdx.and"));
4540 case Builtin::BI__builtin_reduce_maximum:
4541 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4542 CGF&: *this, E, IntrinsicID: Intrinsic::vector_reduce_fmaximum, Name: "rdx.maximum"));
4543 case Builtin::BI__builtin_reduce_minimum:
4544 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4545 CGF&: *this, E, IntrinsicID: Intrinsic::vector_reduce_fminimum, Name: "rdx.minimum"));
4546 case Builtin::BI__builtin_reduce_assoc_fadd:
4547 case Builtin::BI__builtin_reduce_in_order_fadd: {
4548 llvm::Value *Vector = EmitScalarExpr(E: E->getArg(Arg: 0));
4549 llvm::Type *ScalarTy = Vector->getType()->getScalarType();
4550 llvm::Value *StartValue = nullptr;
4551 if (E->getNumArgs() == 2)
4552 StartValue = Builder.CreateFPCast(V: EmitScalarExpr(E: E->getArg(Arg: 1)), DestTy: ScalarTy);
4553 llvm::Value *Args[] = {/*start_value=*/StartValue
4554 ? StartValue
4555 : llvm::ConstantFP::get(Ty: ScalarTy, V: -0.0F),
4556 /*vector=*/Vector};
4557 llvm::Function *F =
4558 CGM.getIntrinsic(IID: Intrinsic::vector_reduce_fadd, Tys: Vector->getType());
4559 llvm::CallBase *Reduce = Builder.CreateCall(Callee: F, Args, Name: "rdx.addf");
4560 if (BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_reduce_assoc_fadd) {
4561 // `__builtin_reduce_assoc_fadd` is an associative reduction which
4562 // requires the reassoc FMF flag.
4563 llvm::FastMathFlags FMF;
4564 FMF.setAllowReassoc();
4565 cast<llvm::CallBase>(Val: Reduce)->setFastMathFlags(FMF);
4566 }
4567 return RValue::get(V: Reduce);
4568 }
4569
4570 case Builtin::BI__builtin_matrix_transpose: {
4571 auto *MatrixTy = E->getArg(Arg: 0)->getType()->castAs<ConstantMatrixType>();
4572 Value *MatValue = EmitScalarExpr(E: E->getArg(Arg: 0));
4573 MatrixBuilder MB(Builder);
4574 Value *Result = MB.CreateMatrixTranspose(Matrix: MatValue, Rows: MatrixTy->getNumRows(),
4575 Columns: MatrixTy->getNumColumns());
4576 return RValue::get(V: Result);
4577 }
4578
4579 case Builtin::BI__builtin_matrix_column_major_load: {
4580 MatrixBuilder MB(Builder);
4581 // Emit everything that isn't dependent on the first parameter type
4582 Value *Stride = EmitScalarExpr(E: E->getArg(Arg: 3));
4583 const auto *ResultTy = E->getType()->getAs<ConstantMatrixType>();
4584 auto *PtrTy = E->getArg(Arg: 0)->getType()->getAs<PointerType>();
4585 assert(PtrTy && "arg0 must be of pointer type");
4586 bool IsVolatile = PtrTy->getPointeeType().isVolatileQualified();
4587
4588 Address Src = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4589 EmitNonNullArgCheck(RV: RValue::get(V: Src.emitRawPointer(CGF&: *this)),
4590 ArgType: E->getArg(Arg: 0)->getType(), ArgLoc: E->getArg(Arg: 0)->getExprLoc(), AC: FD,
4591 ParmNum: 0);
4592 Value *Result = MB.CreateColumnMajorLoad(
4593 EltTy: Src.getElementType(), DataPtr: Src.emitRawPointer(CGF&: *this),
4594 Alignment: Align(Src.getAlignment().getQuantity()), Stride, IsVolatile,
4595 Rows: ResultTy->getNumRows(), Columns: ResultTy->getNumColumns(), Name: "matrix");
4596 return RValue::get(V: Result);
4597 }
4598
4599 case Builtin::BI__builtin_matrix_column_major_store: {
4600 MatrixBuilder MB(Builder);
4601 Value *Matrix = EmitScalarExpr(E: E->getArg(Arg: 0));
4602 Address Dst = EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
4603 Value *Stride = EmitScalarExpr(E: E->getArg(Arg: 2));
4604
4605 const auto *MatrixTy = E->getArg(Arg: 0)->getType()->getAs<ConstantMatrixType>();
4606 auto *PtrTy = E->getArg(Arg: 1)->getType()->getAs<PointerType>();
4607 assert(PtrTy && "arg1 must be of pointer type");
4608 bool IsVolatile = PtrTy->getPointeeType().isVolatileQualified();
4609
4610 EmitNonNullArgCheck(RV: RValue::get(V: Dst.emitRawPointer(CGF&: *this)),
4611 ArgType: E->getArg(Arg: 1)->getType(), ArgLoc: E->getArg(Arg: 1)->getExprLoc(), AC: FD,
4612 ParmNum: 0);
4613 Value *Result = MB.CreateColumnMajorStore(
4614 Matrix, Ptr: Dst.emitRawPointer(CGF&: *this),
4615 Alignment: Align(Dst.getAlignment().getQuantity()), Stride, IsVolatile,
4616 Rows: MatrixTy->getNumRows(), Columns: MatrixTy->getNumColumns());
4617 addInstToNewSourceAtom(KeyInstruction: cast<Instruction>(Val: Result), Backup: Matrix);
4618 return RValue::get(V: Result);
4619 }
4620
4621 case Builtin::BI__builtin_masked_load:
4622 case Builtin::BI__builtin_masked_expand_load: {
4623 llvm::Value *Mask = EmitScalarExpr(E: E->getArg(Arg: 0));
4624 llvm::Value *Ptr = EmitScalarExpr(E: E->getArg(Arg: 1));
4625
4626 llvm::Type *RetTy = CGM.getTypes().ConvertType(T: E->getType());
4627 llvm::Value *PassThru = llvm::PoisonValue::get(T: RetTy);
4628 if (E->getNumArgs() > 2)
4629 PassThru = EmitScalarExpr(E: E->getArg(Arg: 2));
4630
4631 CharUnits Align = CGM.getNaturalTypeAlignment(
4632 T: E->getType()->getAs<VectorType>()->getElementType(), BaseInfo: nullptr);
4633
4634 llvm::Value *Result;
4635 if (BuiltinID == Builtin::BI__builtin_masked_load)
4636 Result = Builder.CreateMaskedLoad(Ty: RetTy, Ptr, Alignment: Align.getAsAlign(), Mask,
4637 PassThru, Name: "masked_load");
4638 else
4639 Result = Builder.CreateMaskedExpandLoad(Ty: RetTy, Ptr, Align: MaybeAlign(), Mask,
4640 PassThru, Name: "masked_expand_load");
4641
4642 return RValue::get(V: Result);
4643 };
4644 case Builtin::BI__builtin_masked_gather: {
4645 llvm::Value *Mask = EmitScalarExpr(E: E->getArg(Arg: 0));
4646 llvm::Value *Idx = EmitScalarExpr(E: E->getArg(Arg: 1));
4647 llvm::Value *Ptr = EmitScalarExpr(E: E->getArg(Arg: 2));
4648
4649 llvm::Type *RetTy = CGM.getTypes().ConvertType(T: E->getType());
4650 CharUnits Align = CGM.getNaturalTypeAlignment(
4651 T: E->getType()->getAs<VectorType>()->getElementType(), BaseInfo: nullptr);
4652
4653 llvm::Value *PassThru = llvm::PoisonValue::get(T: RetTy);
4654 if (E->getNumArgs() > 3)
4655 PassThru = EmitScalarExpr(E: E->getArg(Arg: 3));
4656
4657 llvm::Type *ElemTy = CGM.getTypes().ConvertType(
4658 T: E->getType()->getAs<VectorType>()->getElementType());
4659 llvm::Value *PtrVec = Builder.CreateGEP(Ty: ElemTy, Ptr, IdxList: Idx);
4660
4661 llvm::Value *Result = Builder.CreateMaskedGather(
4662 Ty: RetTy, Ptrs: PtrVec, Alignment: Align.getAsAlign(), Mask, PassThru, Name: "masked_gather");
4663 return RValue::get(V: Result);
4664 }
4665 case Builtin::BI__builtin_masked_store:
4666 case Builtin::BI__builtin_masked_compress_store: {
4667 llvm::Value *Mask = EmitScalarExpr(E: E->getArg(Arg: 0));
4668 llvm::Value *Val = EmitScalarExpr(E: E->getArg(Arg: 1));
4669 llvm::Value *Ptr = EmitScalarExpr(E: E->getArg(Arg: 2));
4670
4671 CharUnits Align = CGM.getNaturalTypeAlignment(
4672 T: E->getArg(Arg: 1)->getType()->getAs<VectorType>()->getElementType(),
4673 BaseInfo: nullptr);
4674
4675 if (BuiltinID == Builtin::BI__builtin_masked_store)
4676 Builder.CreateMaskedStore(Val, Ptr, Alignment: Align.getAsAlign(), Mask);
4677 else
4678 Builder.CreateMaskedCompressStore(Val, Ptr, Align: MaybeAlign(), Mask);
4679
4680 return RValue::get(V: nullptr);
4681 }
4682 case Builtin::BI__builtin_masked_scatter: {
4683 llvm::Value *Mask = EmitScalarExpr(E: E->getArg(Arg: 0));
4684 llvm::Value *Idx = EmitScalarExpr(E: E->getArg(Arg: 1));
4685 llvm::Value *Val = EmitScalarExpr(E: E->getArg(Arg: 2));
4686 llvm::Value *Ptr = EmitScalarExpr(E: E->getArg(Arg: 3));
4687
4688 CharUnits Align = CGM.getNaturalTypeAlignment(
4689 T: E->getArg(Arg: 2)->getType()->getAs<VectorType>()->getElementType(),
4690 BaseInfo: nullptr);
4691
4692 llvm::Type *ElemTy = CGM.getTypes().ConvertType(
4693 T: E->getArg(Arg: 1)->getType()->getAs<VectorType>()->getElementType());
4694 llvm::Value *PtrVec = Builder.CreateGEP(Ty: ElemTy, Ptr, IdxList: Idx);
4695
4696 Builder.CreateMaskedScatter(Val, Ptrs: PtrVec, Alignment: Align.getAsAlign(), Mask);
4697 return RValue();
4698 }
4699 case Builtin::BI__builtin_isinf_sign: {
4700 // isinf_sign(x) -> fabs(x) == infinity ? (signbit(x) ? -1 : 1) : 0
4701 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
4702 // FIXME: for strictfp/IEEE-754 we need to not trap on SNaN here.
4703 Value *Arg = EmitScalarExpr(E: E->getArg(Arg: 0));
4704 Value *AbsArg = EmitFAbs(CGF&: *this, V: Arg);
4705 Value *IsInf = Builder.CreateFCmpOEQ(
4706 LHS: AbsArg, RHS: ConstantFP::getInfinity(Ty: Arg->getType()), Name: "isinf");
4707 Value *IsNeg = EmitSignBit(CGF&: *this, V: Arg);
4708
4709 llvm::Type *IntTy = ConvertType(T: E->getType());
4710 Value *Zero = Constant::getNullValue(Ty: IntTy);
4711 Value *One = ConstantInt::get(Ty: IntTy, V: 1);
4712 Value *NegativeOne = ConstantInt::getAllOnesValue(Ty: IntTy);
4713 Value *SignResult = Builder.CreateSelect(C: IsNeg, True: NegativeOne, False: One);
4714 Value *Result = Builder.CreateSelect(C: IsInf, True: SignResult, False: Zero);
4715 return RValue::get(V: Result);
4716 }
4717
4718 case Builtin::BI__builtin_flt_rounds: {
4719 Function *F = CGM.getIntrinsic(IID: Intrinsic::get_rounding);
4720
4721 llvm::Type *ResultType = ConvertType(T: E->getType());
4722 Value *Result = Builder.CreateCall(Callee: F);
4723 if (Result->getType() != ResultType)
4724 Result = Builder.CreateIntCast(V: Result, DestTy: ResultType, /*isSigned*/true,
4725 Name: "cast");
4726 return RValue::get(V: Result);
4727 }
4728
4729 case Builtin::BI__builtin_set_flt_rounds: {
4730 Function *F = CGM.getIntrinsic(IID: Intrinsic::set_rounding);
4731
4732 Value *V = EmitScalarExpr(E: E->getArg(Arg: 0));
4733 Builder.CreateCall(Callee: F, Args: V);
4734 return RValue::get(V: nullptr);
4735 }
4736
4737 case Builtin::BI__builtin_fpclassify: {
4738 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
4739 // FIXME: for strictfp/IEEE-754 we need to not trap on SNaN here.
4740 Value *V = EmitScalarExpr(E: E->getArg(Arg: 5));
4741 llvm::Type *Ty = ConvertType(T: E->getArg(Arg: 5)->getType());
4742
4743 // Create Result
4744 BasicBlock *Begin = Builder.GetInsertBlock();
4745 BasicBlock *End = createBasicBlock(name: "fpclassify_end", parent: this->CurFn);
4746 Builder.SetInsertPoint(End);
4747 PHINode *Result =
4748 Builder.CreatePHI(Ty: ConvertType(T: E->getArg(Arg: 0)->getType()), NumReservedValues: 4,
4749 Name: "fpclassify_result");
4750
4751 // if (V==0) return FP_ZERO
4752 Builder.SetInsertPoint(Begin);
4753 Value *IsZero = Builder.CreateFCmpOEQ(LHS: V, RHS: Constant::getNullValue(Ty),
4754 Name: "iszero");
4755 Value *ZeroLiteral = EmitScalarExpr(E: E->getArg(Arg: 4));
4756 BasicBlock *NotZero = createBasicBlock(name: "fpclassify_not_zero", parent: this->CurFn);
4757 Builder.CreateCondBr(Cond: IsZero, True: End, False: NotZero);
4758 Result->addIncoming(V: ZeroLiteral, BB: Begin);
4759
4760 // if (V != V) return FP_NAN
4761 Builder.SetInsertPoint(NotZero);
4762 Value *IsNan = Builder.CreateFCmpUNO(LHS: V, RHS: V, Name: "cmp");
4763 Value *NanLiteral = EmitScalarExpr(E: E->getArg(Arg: 0));
4764 BasicBlock *NotNan = createBasicBlock(name: "fpclassify_not_nan", parent: this->CurFn);
4765 Builder.CreateCondBr(Cond: IsNan, True: End, False: NotNan);
4766 Result->addIncoming(V: NanLiteral, BB: NotZero);
4767
4768 // if (fabs(V) == infinity) return FP_INFINITY
4769 Builder.SetInsertPoint(NotNan);
4770 Value *VAbs = EmitFAbs(CGF&: *this, V);
4771 Value *IsInf =
4772 Builder.CreateFCmpOEQ(LHS: VAbs, RHS: ConstantFP::getInfinity(Ty: V->getType()),
4773 Name: "isinf");
4774 Value *InfLiteral = EmitScalarExpr(E: E->getArg(Arg: 1));
4775 BasicBlock *NotInf = createBasicBlock(name: "fpclassify_not_inf", parent: this->CurFn);
4776 Builder.CreateCondBr(Cond: IsInf, True: End, False: NotInf);
4777 Result->addIncoming(V: InfLiteral, BB: NotNan);
4778
4779 // if (fabs(V) >= MIN_NORMAL) return FP_NORMAL else FP_SUBNORMAL
4780 Builder.SetInsertPoint(NotInf);
4781 APFloat Smallest = APFloat::getSmallestNormalized(
4782 Sem: getContext().getFloatTypeSemantics(T: E->getArg(Arg: 5)->getType()));
4783 Value *IsNormal =
4784 Builder.CreateFCmpUGE(LHS: VAbs, RHS: ConstantFP::get(Context&: V->getContext(), V: Smallest),
4785 Name: "isnormal");
4786 Value *NormalResult =
4787 Builder.CreateSelect(C: IsNormal, True: EmitScalarExpr(E: E->getArg(Arg: 2)),
4788 False: EmitScalarExpr(E: E->getArg(Arg: 3)));
4789 Builder.CreateBr(Dest: End);
4790 Result->addIncoming(V: NormalResult, BB: NotInf);
4791
4792 // return Result
4793 Builder.SetInsertPoint(End);
4794 return RValue::get(V: Result);
4795 }
4796
4797 // An alloca will always return a pointer to the alloca (stack) address
4798 // space. This address space need not be the same as the AST / Language
4799 // default (e.g. in C / C++ auto vars are in the generic address space). At
4800 // the AST level this is handled within CreateTempAlloca et al., but for the
4801 // builtin / dynamic alloca we have to handle it here. We use an explicit cast
4802 // instead of passing an AS to CreateAlloca so as to not inhibit optimisation.
4803 case Builtin::BIalloca:
4804 case Builtin::BI_alloca:
4805 case Builtin::BI__builtin_alloca_uninitialized:
4806 case Builtin::BI__builtin_alloca: {
4807 Value *Size = EmitScalarExpr(E: E->getArg(Arg: 0));
4808 const TargetInfo &TI = getContext().getTargetInfo();
4809 // The alignment of the alloca should correspond to __BIGGEST_ALIGNMENT__.
4810 const Align SuitableAlignmentInBytes =
4811 CGM.getContext()
4812 .toCharUnitsFromBits(BitSize: TI.getSuitableAlign())
4813 .getAsAlign();
4814 AllocaInst *AI = Builder.CreateAlloca(Ty: Builder.getInt8Ty(), ArraySize: Size);
4815 AI->setAlignment(SuitableAlignmentInBytes);
4816 if (BuiltinID != Builtin::BI__builtin_alloca_uninitialized)
4817 initializeAlloca(CGF&: *this, AI, Size, AlignmentInBytes: SuitableAlignmentInBytes);
4818 if (AI->getAddressSpace() !=
4819 CGM.getContext().getTargetAddressSpace(
4820 AS: E->getType()->getPointeeType().getAddressSpace())) {
4821 llvm::Type *Ty = CGM.getTypes().ConvertType(T: E->getType());
4822 return RValue::get(V: performAddrSpaceCast(Src: AI, DestTy: Ty));
4823 }
4824 return RValue::get(V: AI);
4825 }
4826
4827 case Builtin::BI__builtin_alloca_with_align_uninitialized:
4828 case Builtin::BI__builtin_alloca_with_align: {
4829 Value *Size = EmitScalarExpr(E: E->getArg(Arg: 0));
4830 Value *AlignmentInBitsValue = EmitScalarExpr(E: E->getArg(Arg: 1));
4831 auto *AlignmentInBitsCI = cast<ConstantInt>(Val: AlignmentInBitsValue);
4832 unsigned AlignmentInBits = AlignmentInBitsCI->getZExtValue();
4833 const Align AlignmentInBytes =
4834 CGM.getContext().toCharUnitsFromBits(BitSize: AlignmentInBits).getAsAlign();
4835 AllocaInst *AI = Builder.CreateAlloca(Ty: Builder.getInt8Ty(), ArraySize: Size);
4836 AI->setAlignment(AlignmentInBytes);
4837 if (BuiltinID != Builtin::BI__builtin_alloca_with_align_uninitialized)
4838 initializeAlloca(CGF&: *this, AI, Size, AlignmentInBytes);
4839 if (AI->getAddressSpace() !=
4840 CGM.getContext().getTargetAddressSpace(
4841 AS: E->getType()->getPointeeType().getAddressSpace())) {
4842 llvm::Type *Ty = CGM.getTypes().ConvertType(T: E->getType());
4843 return RValue::get(V: performAddrSpaceCast(Src: AI, DestTy: Ty));
4844 }
4845 return RValue::get(V: AI);
4846 }
4847
4848 case Builtin::BI__builtin_infer_alloc_token: {
4849 llvm::MDNode *MDN = buildAllocToken(E);
4850 llvm::Value *MDV = MetadataAsValue::get(Context&: getLLVMContext(), MD: MDN);
4851 llvm::Function *F =
4852 CGM.getIntrinsic(IID: llvm::Intrinsic::alloc_token_id, Tys: {IntPtrTy});
4853 llvm::CallBase *TokenID = Builder.CreateCall(Callee: F, Args: MDV);
4854 return RValue::get(V: TokenID);
4855 }
4856
4857 case Builtin::BIbzero:
4858 case Builtin::BI__builtin_bzero: {
4859 Address Dest = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4860 Value *SizeVal = EmitScalarExpr(E: E->getArg(Arg: 1));
4861 EmitNonNullArgCheck(Addr: Dest, ArgType: E->getArg(Arg: 0)->getType(),
4862 ArgLoc: E->getArg(Arg: 0)->getExprLoc(), AC: FD, ParmNum: 0);
4863 auto *I = Builder.CreateMemSet(Dest, Value: Builder.getInt8(C: 0), Size: SizeVal, IsVolatile: false);
4864 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
4865 return RValue::get(V: nullptr);
4866 }
4867
4868 case Builtin::BIbcopy:
4869 case Builtin::BI__builtin_bcopy: {
4870 Address Src = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4871 Address Dest = EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
4872 Value *SizeVal = EmitScalarExpr(E: E->getArg(Arg: 2));
4873 EmitNonNullArgCheck(RV: RValue::get(V: Src.emitRawPointer(CGF&: *this)),
4874 ArgType: E->getArg(Arg: 0)->getType(), ArgLoc: E->getArg(Arg: 0)->getExprLoc(), AC: FD,
4875 ParmNum: 0);
4876 EmitNonNullArgCheck(RV: RValue::get(V: Dest.emitRawPointer(CGF&: *this)),
4877 ArgType: E->getArg(Arg: 1)->getType(), ArgLoc: E->getArg(Arg: 1)->getExprLoc(), AC: FD,
4878 ParmNum: 0);
4879 auto *I = Builder.CreateMemMove(Dest, Src, Size: SizeVal, IsVolatile: false);
4880 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
4881 return RValue::get(V: nullptr);
4882 }
4883
4884 case Builtin::BImemcpy:
4885 case Builtin::BI__builtin_memcpy:
4886 case Builtin::BImempcpy:
4887 case Builtin::BI__builtin_mempcpy: {
4888 Address Dest = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4889 Address Src = EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
4890 Value *SizeVal = EmitScalarExpr(E: E->getArg(Arg: 2));
4891 EmitArgCheck(TCK_Store, Dest, E->getArg(Arg: 0), 0);
4892 EmitArgCheck(TCK_Load, Src, E->getArg(Arg: 1), 1);
4893 auto *I = Builder.CreateMemCpy(Dest, Src, Size: SizeVal, IsVolatile: false);
4894 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
4895 if (BuiltinID == Builtin::BImempcpy ||
4896 BuiltinID == Builtin::BI__builtin_mempcpy)
4897 return RValue::get(V: Builder.CreateInBoundsGEP(
4898 Ty: Dest.getElementType(), Ptr: Dest.emitRawPointer(CGF&: *this), IdxList: SizeVal));
4899 else
4900 return RValue::get(Addr: Dest, CGF&: *this);
4901 }
4902
4903 case Builtin::BI__builtin_memcpy_inline: {
4904 Address Dest = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4905 Address Src = EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
4906 uint64_t Size =
4907 E->getArg(Arg: 2)->EvaluateKnownConstInt(Ctx: getContext()).getZExtValue();
4908 EmitArgCheck(TCK_Store, Dest, E->getArg(Arg: 0), 0);
4909 EmitArgCheck(TCK_Load, Src, E->getArg(Arg: 1), 1);
4910 auto *I = Builder.CreateMemCpyInline(Dest, Src, Size);
4911 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
4912 return RValue::get(V: nullptr);
4913 }
4914
4915 case Builtin::BI__builtin_char_memchr:
4916 BuiltinID = Builtin::BI__builtin_memchr;
4917 break;
4918
4919 case Builtin::BI__builtin___memcpy_chk: {
4920 // fold __builtin_memcpy_chk(x, y, cst1, cst2) to memcpy iff cst1<=cst2.
4921 Expr::EvalResult SizeResult, DstSizeResult;
4922 if (!E->getArg(Arg: 2)->EvaluateAsInt(Result&: SizeResult, Ctx: CGM.getContext()) ||
4923 !E->getArg(Arg: 3)->EvaluateAsInt(Result&: DstSizeResult, Ctx: CGM.getContext()))
4924 break;
4925 llvm::APSInt Size = SizeResult.Val.getInt();
4926 llvm::APSInt DstSize = DstSizeResult.Val.getInt();
4927 if (Size.ugt(RHS: DstSize))
4928 break;
4929 Address Dest = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4930 Address Src = EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
4931 Value *SizeVal = llvm::ConstantInt::get(Context&: Builder.getContext(), V: Size);
4932 auto *I = Builder.CreateMemCpy(Dest, Src, Size: SizeVal, IsVolatile: false);
4933 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
4934 return RValue::get(Addr: Dest, CGF&: *this);
4935 }
4936
4937 case Builtin::BI__builtin_objc_memmove_collectable: {
4938 Address DestAddr = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4939 Address SrcAddr = EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
4940 Value *SizeVal = EmitScalarExpr(E: E->getArg(Arg: 2));
4941 CGM.getObjCRuntime().EmitGCMemmoveCollectable(CGF&: *this,
4942 DestPtr: DestAddr, SrcPtr: SrcAddr, Size: SizeVal);
4943 return RValue::get(Addr: DestAddr, CGF&: *this);
4944 }
4945
4946 case Builtin::BI__builtin___memmove_chk: {
4947 // fold __builtin_memmove_chk(x, y, cst1, cst2) to memmove iff cst1<=cst2.
4948 Expr::EvalResult SizeResult, DstSizeResult;
4949 if (!E->getArg(Arg: 2)->EvaluateAsInt(Result&: SizeResult, Ctx: CGM.getContext()) ||
4950 !E->getArg(Arg: 3)->EvaluateAsInt(Result&: DstSizeResult, Ctx: CGM.getContext()))
4951 break;
4952 llvm::APSInt Size = SizeResult.Val.getInt();
4953 llvm::APSInt DstSize = DstSizeResult.Val.getInt();
4954 if (Size.ugt(RHS: DstSize))
4955 break;
4956 Address Dest = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4957 Address Src = EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
4958 Value *SizeVal = llvm::ConstantInt::get(Context&: Builder.getContext(), V: Size);
4959 auto *I = Builder.CreateMemMove(Dest, Src, Size: SizeVal, IsVolatile: false);
4960 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
4961 return RValue::get(Addr: Dest, CGF&: *this);
4962 }
4963
4964 case Builtin::BI__builtin_trivially_relocate:
4965 case Builtin::BImemmove:
4966 case Builtin::BI__builtin_memmove: {
4967 Address Dest = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4968 Address Src = EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
4969 Value *SizeVal = EmitScalarExpr(E: E->getArg(Arg: 2));
4970 if (BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_trivially_relocate)
4971 SizeVal = Builder.CreateMul(
4972 LHS: SizeVal,
4973 RHS: ConstantInt::get(
4974 Ty: SizeVal->getType(),
4975 V: getContext()
4976 .getTypeSizeInChars(T: E->getArg(Arg: 0)->getType()->getPointeeType())
4977 .getQuantity()));
4978 EmitArgCheck(TCK_Store, Dest, E->getArg(Arg: 0), 0);
4979 EmitArgCheck(TCK_Load, Src, E->getArg(Arg: 1), 1);
4980 auto *I = Builder.CreateMemMove(Dest, Src, Size: SizeVal, IsVolatile: false);
4981 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
4982 return RValue::get(Addr: Dest, CGF&: *this);
4983 }
4984 case Builtin::BImemset:
4985 case Builtin::BI__builtin_memset: {
4986 Address Dest = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4987 Value *ByteVal = Builder.CreateTrunc(V: EmitScalarExpr(E: E->getArg(Arg: 1)),
4988 DestTy: Builder.getInt8Ty());
4989 Value *SizeVal = EmitScalarExpr(E: E->getArg(Arg: 2));
4990 EmitNonNullArgCheck(Addr: Dest, ArgType: E->getArg(Arg: 0)->getType(),
4991 ArgLoc: E->getArg(Arg: 0)->getExprLoc(), AC: FD, ParmNum: 0);
4992 auto *I = Builder.CreateMemSet(Dest, Value: ByteVal, Size: SizeVal, IsVolatile: false);
4993 addInstToNewSourceAtom(KeyInstruction: I, Backup: ByteVal);
4994 return RValue::get(Addr: Dest, CGF&: *this);
4995 }
4996 case Builtin::BI__builtin_memset_inline: {
4997 Address Dest = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4998 Value *ByteVal =
4999 Builder.CreateTrunc(V: EmitScalarExpr(E: E->getArg(Arg: 1)), DestTy: Builder.getInt8Ty());
5000 uint64_t Size =
5001 E->getArg(Arg: 2)->EvaluateKnownConstInt(Ctx: getContext()).getZExtValue();
5002 EmitNonNullArgCheck(RV: RValue::get(V: Dest.emitRawPointer(CGF&: *this)),
5003 ArgType: E->getArg(Arg: 0)->getType(), ArgLoc: E->getArg(Arg: 0)->getExprLoc(), AC: FD,
5004 ParmNum: 0);
5005 auto *I = Builder.CreateMemSetInline(Dest, Value: ByteVal, Size);
5006 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
5007 return RValue::get(V: nullptr);
5008 }
5009 case Builtin::BI__builtin___memset_chk: {
5010 // fold __builtin_memset_chk(x, y, cst1, cst2) to memset iff cst1<=cst2.
5011 Expr::EvalResult SizeResult, DstSizeResult;
5012 if (!E->getArg(Arg: 2)->EvaluateAsInt(Result&: SizeResult, Ctx: CGM.getContext()) ||
5013 !E->getArg(Arg: 3)->EvaluateAsInt(Result&: DstSizeResult, Ctx: CGM.getContext()))
5014 break;
5015 llvm::APSInt Size = SizeResult.Val.getInt();
5016 llvm::APSInt DstSize = DstSizeResult.Val.getInt();
5017 if (Size.ugt(RHS: DstSize))
5018 break;
5019 Address Dest = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
5020 Value *ByteVal = Builder.CreateTrunc(V: EmitScalarExpr(E: E->getArg(Arg: 1)),
5021 DestTy: Builder.getInt8Ty());
5022 Value *SizeVal = llvm::ConstantInt::get(Context&: Builder.getContext(), V: Size);
5023 auto *I = Builder.CreateMemSet(Dest, Value: ByteVal, Size: SizeVal, IsVolatile: false);
5024 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
5025 return RValue::get(Addr: Dest, CGF&: *this);
5026 }
5027 case Builtin::BI__builtin_wmemchr: {
5028 // The MSVC runtime library does not provide a definition of wmemchr, so we
5029 // need an inline implementation.
5030 if (!getTarget().getTriple().isOSMSVCRT())
5031 break;
5032
5033 llvm::Type *WCharTy = ConvertType(T: getContext().WCharTy);
5034 Value *Str = EmitScalarExpr(E: E->getArg(Arg: 0));
5035 Value *Chr = EmitScalarExpr(E: E->getArg(Arg: 1));
5036 Value *Size = EmitScalarExpr(E: E->getArg(Arg: 2));
5037
5038 BasicBlock *Entry = Builder.GetInsertBlock();
5039 BasicBlock *CmpEq = createBasicBlock(name: "wmemchr.eq");
5040 BasicBlock *Next = createBasicBlock(name: "wmemchr.next");
5041 BasicBlock *Exit = createBasicBlock(name: "wmemchr.exit");
5042 Value *SizeEq0 = Builder.CreateICmpEQ(LHS: Size, RHS: ConstantInt::get(Ty: SizeTy, V: 0));
5043 Builder.CreateCondBr(Cond: SizeEq0, True: Exit, False: CmpEq);
5044
5045 EmitBlock(BB: CmpEq);
5046 PHINode *StrPhi = Builder.CreatePHI(Ty: Str->getType(), NumReservedValues: 2);
5047 StrPhi->addIncoming(V: Str, BB: Entry);
5048 PHINode *SizePhi = Builder.CreatePHI(Ty: SizeTy, NumReservedValues: 2);
5049 SizePhi->addIncoming(V: Size, BB: Entry);
5050 CharUnits WCharAlign =
5051 getContext().getTypeAlignInChars(T: getContext().WCharTy);
5052 Value *StrCh = Builder.CreateAlignedLoad(Ty: WCharTy, Addr: StrPhi, Align: WCharAlign);
5053 Value *FoundChr = Builder.CreateConstInBoundsGEP1_32(Ty: WCharTy, Ptr: StrPhi, Idx0: 0);
5054 Value *StrEqChr = Builder.CreateICmpEQ(LHS: StrCh, RHS: Chr);
5055 Builder.CreateCondBr(Cond: StrEqChr, True: Exit, False: Next);
5056
5057 EmitBlock(BB: Next);
5058 Value *NextStr = Builder.CreateConstInBoundsGEP1_32(Ty: WCharTy, Ptr: StrPhi, Idx0: 1);
5059 Value *NextSize = Builder.CreateSub(LHS: SizePhi, RHS: ConstantInt::get(Ty: SizeTy, V: 1));
5060 Value *NextSizeEq0 =
5061 Builder.CreateICmpEQ(LHS: NextSize, RHS: ConstantInt::get(Ty: SizeTy, V: 0));
5062 Builder.CreateCondBr(Cond: NextSizeEq0, True: Exit, False: CmpEq);
5063 StrPhi->addIncoming(V: NextStr, BB: Next);
5064 SizePhi->addIncoming(V: NextSize, BB: Next);
5065
5066 EmitBlock(BB: Exit);
5067 PHINode *Ret = Builder.CreatePHI(Ty: Str->getType(), NumReservedValues: 3);
5068 Ret->addIncoming(V: llvm::Constant::getNullValue(Ty: Str->getType()), BB: Entry);
5069 Ret->addIncoming(V: llvm::Constant::getNullValue(Ty: Str->getType()), BB: Next);
5070 Ret->addIncoming(V: FoundChr, BB: CmpEq);
5071 return RValue::get(V: Ret);
5072 }
5073 case Builtin::BI__builtin_wmemcmp: {
5074 // The MSVC runtime library does not provide a definition of wmemcmp, so we
5075 // need an inline implementation.
5076 if (!getTarget().getTriple().isOSMSVCRT())
5077 break;
5078
5079 llvm::Type *WCharTy = ConvertType(T: getContext().WCharTy);
5080
5081 Value *Dst = EmitScalarExpr(E: E->getArg(Arg: 0));
5082 Value *Src = EmitScalarExpr(E: E->getArg(Arg: 1));
5083 Value *Size = EmitScalarExpr(E: E->getArg(Arg: 2));
5084
5085 BasicBlock *Entry = Builder.GetInsertBlock();
5086 BasicBlock *CmpGT = createBasicBlock(name: "wmemcmp.gt");
5087 BasicBlock *CmpLT = createBasicBlock(name: "wmemcmp.lt");
5088 BasicBlock *Next = createBasicBlock(name: "wmemcmp.next");
5089 BasicBlock *Exit = createBasicBlock(name: "wmemcmp.exit");
5090 Value *SizeEq0 = Builder.CreateICmpEQ(LHS: Size, RHS: ConstantInt::get(Ty: SizeTy, V: 0));
5091 Builder.CreateCondBr(Cond: SizeEq0, True: Exit, False: CmpGT);
5092
5093 EmitBlock(BB: CmpGT);
5094 PHINode *DstPhi = Builder.CreatePHI(Ty: Dst->getType(), NumReservedValues: 2);
5095 DstPhi->addIncoming(V: Dst, BB: Entry);
5096 PHINode *SrcPhi = Builder.CreatePHI(Ty: Src->getType(), NumReservedValues: 2);
5097 SrcPhi->addIncoming(V: Src, BB: Entry);
5098 PHINode *SizePhi = Builder.CreatePHI(Ty: SizeTy, NumReservedValues: 2);
5099 SizePhi->addIncoming(V: Size, BB: Entry);
5100 CharUnits WCharAlign =
5101 getContext().getTypeAlignInChars(T: getContext().WCharTy);
5102 Value *DstCh = Builder.CreateAlignedLoad(Ty: WCharTy, Addr: DstPhi, Align: WCharAlign);
5103 Value *SrcCh = Builder.CreateAlignedLoad(Ty: WCharTy, Addr: SrcPhi, Align: WCharAlign);
5104 Value *DstGtSrc = Builder.CreateICmpUGT(LHS: DstCh, RHS: SrcCh);
5105 Builder.CreateCondBr(Cond: DstGtSrc, True: Exit, False: CmpLT);
5106
5107 EmitBlock(BB: CmpLT);
5108 Value *DstLtSrc = Builder.CreateICmpULT(LHS: DstCh, RHS: SrcCh);
5109 Builder.CreateCondBr(Cond: DstLtSrc, True: Exit, False: Next);
5110
5111 EmitBlock(BB: Next);
5112 Value *NextDst = Builder.CreateConstInBoundsGEP1_32(Ty: WCharTy, Ptr: DstPhi, Idx0: 1);
5113 Value *NextSrc = Builder.CreateConstInBoundsGEP1_32(Ty: WCharTy, Ptr: SrcPhi, Idx0: 1);
5114 Value *NextSize = Builder.CreateSub(LHS: SizePhi, RHS: ConstantInt::get(Ty: SizeTy, V: 1));
5115 Value *NextSizeEq0 =
5116 Builder.CreateICmpEQ(LHS: NextSize, RHS: ConstantInt::get(Ty: SizeTy, V: 0));
5117 Builder.CreateCondBr(Cond: NextSizeEq0, True: Exit, False: CmpGT);
5118 DstPhi->addIncoming(V: NextDst, BB: Next);
5119 SrcPhi->addIncoming(V: NextSrc, BB: Next);
5120 SizePhi->addIncoming(V: NextSize, BB: Next);
5121
5122 EmitBlock(BB: Exit);
5123 PHINode *Ret = Builder.CreatePHI(Ty: IntTy, NumReservedValues: 4);
5124 Ret->addIncoming(V: ConstantInt::get(Ty: IntTy, V: 0), BB: Entry);
5125 Ret->addIncoming(V: ConstantInt::get(Ty: IntTy, V: 1), BB: CmpGT);
5126 Ret->addIncoming(V: ConstantInt::getAllOnesValue(Ty: IntTy), BB: CmpLT);
5127 Ret->addIncoming(V: ConstantInt::get(Ty: IntTy, V: 0), BB: Next);
5128 return RValue::get(V: Ret);
5129 }
5130 case Builtin::BI__builtin_dwarf_cfa: {
5131 // The offset in bytes from the first argument to the CFA.
5132 //
5133 // Why on earth is this in the frontend? Is there any reason at
5134 // all that the backend can't reasonably determine this while
5135 // lowering llvm.eh.dwarf.cfa()?
5136 //
5137 // TODO: If there's a satisfactory reason, add a target hook for
5138 // this instead of hard-coding 0, which is correct for most targets.
5139 int32_t Offset = 0;
5140
5141 Function *F = CGM.getIntrinsic(IID: Intrinsic::eh_dwarf_cfa);
5142 return RValue::get(V: Builder.CreateCall(Callee: F,
5143 Args: llvm::ConstantInt::get(Ty: Int32Ty, V: Offset)));
5144 }
5145 case Builtin::BI__builtin_return_address: {
5146 Value *Depth = ConstantEmitter(*this).emitAbstract(E: E->getArg(Arg: 0),
5147 T: getContext().UnsignedIntTy);
5148 Depth = Builder.CreateZExtOrTrunc(V: Depth, DestTy: Int32Ty);
5149 Function *F =
5150 CGM.getIntrinsic(IID: Intrinsic::returnaddress, Tys: {CGM.ProgramPtrTy});
5151 return RValue::get(V: Builder.CreateCall(Callee: F, Args: Depth));
5152 }
5153 case Builtin::BI_ReturnAddress: {
5154 Function *F =
5155 CGM.getIntrinsic(IID: Intrinsic::returnaddress, Tys: {CGM.ProgramPtrTy});
5156 return RValue::get(V: Builder.CreateCall(Callee: F, Args: Builder.getInt32(C: 0)));
5157 }
5158 case Builtin::BI__builtin_frame_address: {
5159 Value *Depth = ConstantEmitter(*this).emitAbstract(E: E->getArg(Arg: 0),
5160 T: getContext().UnsignedIntTy);
5161 Depth = Builder.CreateZExtOrTrunc(V: Depth, DestTy: Int32Ty);
5162 Function *F = CGM.getIntrinsic(IID: Intrinsic::frameaddress, Tys: AllocaInt8PtrTy);
5163 return RValue::get(V: Builder.CreateCall(Callee: F, Args: Depth));
5164 }
5165 case Builtin::BI__builtin_stack_address: {
5166 return RValue::get(V: Builder.CreateCall(
5167 Callee: CGM.getIntrinsic(IID: Intrinsic::stackaddress, Tys: AllocaInt8PtrTy)));
5168 }
5169 case Builtin::BI__builtin_extract_return_addr: {
5170 Value *Address = EmitScalarExpr(E: E->getArg(Arg: 0));
5171 Value *Result = getTargetHooks().decodeReturnAddress(CGF&: *this, Address);
5172 return RValue::get(V: Result);
5173 }
5174 case Builtin::BI__builtin_frob_return_addr: {
5175 Value *Address = EmitScalarExpr(E: E->getArg(Arg: 0));
5176 Value *Result = getTargetHooks().encodeReturnAddress(CGF&: *this, Address);
5177 return RValue::get(V: Result);
5178 }
5179 case Builtin::BI__builtin_dwarf_sp_column: {
5180 llvm::IntegerType *Ty
5181 = cast<llvm::IntegerType>(Val: ConvertType(T: E->getType()));
5182 int Column = getTargetHooks().getDwarfEHStackPointer(M&: CGM);
5183 if (Column == -1) {
5184 CGM.ErrorUnsupported(S: E, Type: "__builtin_dwarf_sp_column");
5185 return RValue::get(V: llvm::UndefValue::get(T: Ty));
5186 }
5187 return RValue::get(V: llvm::ConstantInt::get(Ty, V: Column, IsSigned: true));
5188 }
5189 case Builtin::BI__builtin_init_dwarf_reg_size_table: {
5190 Value *Address = EmitScalarExpr(E: E->getArg(Arg: 0));
5191 if (getTargetHooks().initDwarfEHRegSizeTable(CGF&: *this, Address))
5192 CGM.ErrorUnsupported(S: E, Type: "__builtin_init_dwarf_reg_size_table");
5193 return RValue::get(V: llvm::UndefValue::get(T: ConvertType(T: E->getType())));
5194 }
5195 case Builtin::BI__builtin_eh_return: {
5196 Value *Int = EmitScalarExpr(E: E->getArg(Arg: 0));
5197 Value *Ptr = EmitScalarExpr(E: E->getArg(Arg: 1));
5198
5199 llvm::IntegerType *IntTy = cast<llvm::IntegerType>(Val: Int->getType());
5200 assert((IntTy->getBitWidth() == 32 || IntTy->getBitWidth() == 64) &&
5201 "LLVM's __builtin_eh_return only supports 32- and 64-bit variants");
5202 Function *F =
5203 CGM.getIntrinsic(IID: IntTy->getBitWidth() == 32 ? Intrinsic::eh_return_i32
5204 : Intrinsic::eh_return_i64);
5205 Builder.CreateCall(Callee: F, Args: {Int, Ptr});
5206 Builder.CreateUnreachable();
5207
5208 // We do need to preserve an insertion point.
5209 EmitBlock(BB: createBasicBlock(name: "builtin_eh_return.cont"));
5210
5211 return RValue::get(V: nullptr);
5212 }
5213 case Builtin::BI__builtin_unwind_init: {
5214 Function *F = CGM.getIntrinsic(IID: Intrinsic::eh_unwind_init);
5215 Builder.CreateCall(Callee: F);
5216 return RValue::get(V: nullptr);
5217 }
5218 case Builtin::BI__builtin_extend_pointer: {
5219 // Extends a pointer to the size of an _Unwind_Word, which is
5220 // uint64_t on all platforms. Generally this gets poked into a
5221 // register and eventually used as an address, so if the
5222 // addressing registers are wider than pointers and the platform
5223 // doesn't implicitly ignore high-order bits when doing
5224 // addressing, we need to make sure we zext / sext based on
5225 // the platform's expectations.
5226 //
5227 // See: http://gcc.gnu.org/ml/gcc-bugs/2002-02/msg00237.html
5228
5229 // Cast the pointer to intptr_t.
5230 Value *Ptr = EmitScalarExpr(E: E->getArg(Arg: 0));
5231 Value *Result = Builder.CreatePtrToInt(V: Ptr, DestTy: IntPtrTy, Name: "extend.cast");
5232
5233 // If that's 64 bits, we're done.
5234 if (IntPtrTy->getBitWidth() == 64)
5235 return RValue::get(V: Result);
5236
5237 // Otherwise, ask the codegen data what to do.
5238 if (getTargetHooks().extendPointerWithSExt())
5239 return RValue::get(V: Builder.CreateSExt(V: Result, DestTy: Int64Ty, Name: "extend.sext"));
5240 else
5241 return RValue::get(V: Builder.CreateZExt(V: Result, DestTy: Int64Ty, Name: "extend.zext"));
5242 }
5243 case Builtin::BI__builtin_setjmp: {
5244 // Buffer is a void**.
5245 Address Buf = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
5246
5247 if (getTarget().getTriple().getArch() == llvm::Triple::systemz) {
5248 // On this target, the back end fills in the context buffer completely.
5249 // It doesn't really matter if the frontend stores to the buffer before
5250 // calling setjmp, the back-end is going to overwrite them anyway.
5251 Function *F = CGM.getIntrinsic(IID: Intrinsic::eh_sjlj_setjmp);
5252 return RValue::get(V: Builder.CreateCall(Callee: F, Args: Buf.emitRawPointer(CGF&: *this)));
5253 }
5254
5255 // Store the frame pointer to the setjmp buffer.
5256 Value *FrameAddr = Builder.CreateCall(
5257 Callee: CGM.getIntrinsic(IID: Intrinsic::frameaddress, Tys: AllocaInt8PtrTy),
5258 Args: ConstantInt::get(Ty: Int32Ty, V: 0));
5259 Builder.CreateStore(Val: FrameAddr, Addr: Buf);
5260
5261 // Store the stack pointer to the setjmp buffer.
5262 Value *StackAddr = Builder.CreateStackSave();
5263 assert(Buf.emitRawPointer(*this)->getType() == StackAddr->getType());
5264
5265 Address StackSaveSlot = Builder.CreateConstInBoundsGEP(Addr: Buf, Index: 2);
5266 Builder.CreateStore(Val: StackAddr, Addr: StackSaveSlot);
5267
5268 // Call LLVM's EH setjmp, which is lightweight.
5269 Function *F = CGM.getIntrinsic(IID: Intrinsic::eh_sjlj_setjmp);
5270 return RValue::get(V: Builder.CreateCall(Callee: F, Args: Buf.emitRawPointer(CGF&: *this)));
5271 }
5272 case Builtin::BI__builtin_longjmp: {
5273 Value *Buf = EmitScalarExpr(E: E->getArg(Arg: 0));
5274
5275 // Call LLVM's EH longjmp, which is lightweight.
5276 Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Intrinsic::eh_sjlj_longjmp), Args: Buf);
5277
5278 // longjmp doesn't return; mark this as unreachable.
5279 Builder.CreateUnreachable();
5280
5281 // We do need to preserve an insertion point.
5282 EmitBlock(BB: createBasicBlock(name: "longjmp.cont"));
5283
5284 return RValue::get(V: nullptr);
5285 }
5286 case Builtin::BI__builtin_launder: {
5287 const Expr *Arg = E->getArg(Arg: 0);
5288 QualType ArgTy = Arg->getType()->getPointeeType();
5289 Value *Ptr = EmitScalarExpr(E: Arg);
5290 if (TypeRequiresBuiltinLaunder(CGM, Ty: ArgTy))
5291 Ptr = Builder.CreateLaunderInvariantGroup(Ptr);
5292
5293 return RValue::get(V: Ptr);
5294 }
5295 case Builtin::BI__builtin_clear_padding: {
5296 Address Src = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
5297 auto PointeeTy = E->getArg(Arg: 0)->getType()->getPointeeType();
5298
5299 llvm::ArrayRef<ASTContext::BitInterval> Padding =
5300 getContext().getPaddingIntervals(Ty: PointeeTy);
5301 for (const auto &Interval : Padding)
5302 ClearPadding(CGF&: *this, Src, PaddingInterval: Interval);
5303
5304 return RValue::get(V: nullptr);
5305 }
5306 case Builtin::BI__sync_fetch_and_add:
5307 case Builtin::BI__sync_fetch_and_sub:
5308 case Builtin::BI__sync_fetch_and_or:
5309 case Builtin::BI__sync_fetch_and_and:
5310 case Builtin::BI__sync_fetch_and_xor:
5311 case Builtin::BI__sync_fetch_and_nand:
5312 case Builtin::BI__sync_add_and_fetch:
5313 case Builtin::BI__sync_sub_and_fetch:
5314 case Builtin::BI__sync_and_and_fetch:
5315 case Builtin::BI__sync_or_and_fetch:
5316 case Builtin::BI__sync_xor_and_fetch:
5317 case Builtin::BI__sync_nand_and_fetch:
5318 case Builtin::BI__sync_val_compare_and_swap:
5319 case Builtin::BI__sync_bool_compare_and_swap:
5320 case Builtin::BI__sync_lock_test_and_set:
5321 case Builtin::BI__sync_lock_release:
5322 case Builtin::BI__sync_swap:
5323 llvm_unreachable("Shouldn't make it through sema");
5324 case Builtin::BI__sync_fetch_and_add_1:
5325 case Builtin::BI__sync_fetch_and_add_2:
5326 case Builtin::BI__sync_fetch_and_add_4:
5327 case Builtin::BI__sync_fetch_and_add_8:
5328 case Builtin::BI__sync_fetch_and_add_16:
5329 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::Add, E);
5330 case Builtin::BI__sync_fetch_and_sub_1:
5331 case Builtin::BI__sync_fetch_and_sub_2:
5332 case Builtin::BI__sync_fetch_and_sub_4:
5333 case Builtin::BI__sync_fetch_and_sub_8:
5334 case Builtin::BI__sync_fetch_and_sub_16:
5335 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::Sub, E);
5336 case Builtin::BI__sync_fetch_and_or_1:
5337 case Builtin::BI__sync_fetch_and_or_2:
5338 case Builtin::BI__sync_fetch_and_or_4:
5339 case Builtin::BI__sync_fetch_and_or_8:
5340 case Builtin::BI__sync_fetch_and_or_16:
5341 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::Or, E);
5342 case Builtin::BI__sync_fetch_and_and_1:
5343 case Builtin::BI__sync_fetch_and_and_2:
5344 case Builtin::BI__sync_fetch_and_and_4:
5345 case Builtin::BI__sync_fetch_and_and_8:
5346 case Builtin::BI__sync_fetch_and_and_16:
5347 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::And, E);
5348 case Builtin::BI__sync_fetch_and_xor_1:
5349 case Builtin::BI__sync_fetch_and_xor_2:
5350 case Builtin::BI__sync_fetch_and_xor_4:
5351 case Builtin::BI__sync_fetch_and_xor_8:
5352 case Builtin::BI__sync_fetch_and_xor_16:
5353 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::Xor, E);
5354 case Builtin::BI__sync_fetch_and_nand_1:
5355 case Builtin::BI__sync_fetch_and_nand_2:
5356 case Builtin::BI__sync_fetch_and_nand_4:
5357 case Builtin::BI__sync_fetch_and_nand_8:
5358 case Builtin::BI__sync_fetch_and_nand_16:
5359 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::Nand, E);
5360
5361 // Clang extensions: not overloaded yet.
5362 case Builtin::BI__sync_fetch_and_min:
5363 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::Min, E);
5364 case Builtin::BI__sync_fetch_and_max:
5365 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::Max, E);
5366 case Builtin::BI__sync_fetch_and_umin:
5367 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::UMin, E);
5368 case Builtin::BI__sync_fetch_and_umax:
5369 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::UMax, E);
5370
5371 case Builtin::BI__sync_add_and_fetch_1:
5372 case Builtin::BI__sync_add_and_fetch_2:
5373 case Builtin::BI__sync_add_and_fetch_4:
5374 case Builtin::BI__sync_add_and_fetch_8:
5375 case Builtin::BI__sync_add_and_fetch_16:
5376 return EmitBinaryAtomicPost(CGF&: *this, Kind: llvm::AtomicRMWInst::Add, E,
5377 Op: llvm::Instruction::Add);
5378 case Builtin::BI__sync_sub_and_fetch_1:
5379 case Builtin::BI__sync_sub_and_fetch_2:
5380 case Builtin::BI__sync_sub_and_fetch_4:
5381 case Builtin::BI__sync_sub_and_fetch_8:
5382 case Builtin::BI__sync_sub_and_fetch_16:
5383 return EmitBinaryAtomicPost(CGF&: *this, Kind: llvm::AtomicRMWInst::Sub, E,
5384 Op: llvm::Instruction::Sub);
5385 case Builtin::BI__sync_and_and_fetch_1:
5386 case Builtin::BI__sync_and_and_fetch_2:
5387 case Builtin::BI__sync_and_and_fetch_4:
5388 case Builtin::BI__sync_and_and_fetch_8:
5389 case Builtin::BI__sync_and_and_fetch_16:
5390 return EmitBinaryAtomicPost(CGF&: *this, Kind: llvm::AtomicRMWInst::And, E,
5391 Op: llvm::Instruction::And);
5392 case Builtin::BI__sync_or_and_fetch_1:
5393 case Builtin::BI__sync_or_and_fetch_2:
5394 case Builtin::BI__sync_or_and_fetch_4:
5395 case Builtin::BI__sync_or_and_fetch_8:
5396 case Builtin::BI__sync_or_and_fetch_16:
5397 return EmitBinaryAtomicPost(CGF&: *this, Kind: llvm::AtomicRMWInst::Or, E,
5398 Op: llvm::Instruction::Or);
5399 case Builtin::BI__sync_xor_and_fetch_1:
5400 case Builtin::BI__sync_xor_and_fetch_2:
5401 case Builtin::BI__sync_xor_and_fetch_4:
5402 case Builtin::BI__sync_xor_and_fetch_8:
5403 case Builtin::BI__sync_xor_and_fetch_16:
5404 return EmitBinaryAtomicPost(CGF&: *this, Kind: llvm::AtomicRMWInst::Xor, E,
5405 Op: llvm::Instruction::Xor);
5406 case Builtin::BI__sync_nand_and_fetch_1:
5407 case Builtin::BI__sync_nand_and_fetch_2:
5408 case Builtin::BI__sync_nand_and_fetch_4:
5409 case Builtin::BI__sync_nand_and_fetch_8:
5410 case Builtin::BI__sync_nand_and_fetch_16:
5411 return EmitBinaryAtomicPost(CGF&: *this, Kind: llvm::AtomicRMWInst::Nand, E,
5412 Op: llvm::Instruction::And, Invert: true);
5413
5414 case Builtin::BI__sync_val_compare_and_swap_1:
5415 case Builtin::BI__sync_val_compare_and_swap_2:
5416 case Builtin::BI__sync_val_compare_and_swap_4:
5417 case Builtin::BI__sync_val_compare_and_swap_8:
5418 case Builtin::BI__sync_val_compare_and_swap_16:
5419 return RValue::get(V: MakeAtomicCmpXchgValue(
5420 CGF&: *this, E, ReturnBool: false, SuccessOrdering: AtomicOrdering::SequentiallyConsistent,
5421 FailureOrdering: AtomicOrdering::SequentiallyConsistent));
5422
5423 case Builtin::BI__sync_bool_compare_and_swap_1:
5424 case Builtin::BI__sync_bool_compare_and_swap_2:
5425 case Builtin::BI__sync_bool_compare_and_swap_4:
5426 case Builtin::BI__sync_bool_compare_and_swap_8:
5427 case Builtin::BI__sync_bool_compare_and_swap_16:
5428 return RValue::get(V: MakeAtomicCmpXchgValue(
5429 CGF&: *this, E, ReturnBool: true, SuccessOrdering: AtomicOrdering::SequentiallyConsistent,
5430 FailureOrdering: AtomicOrdering::SequentiallyConsistent));
5431
5432 case Builtin::BI__sync_swap_1:
5433 case Builtin::BI__sync_swap_2:
5434 case Builtin::BI__sync_swap_4:
5435 case Builtin::BI__sync_swap_8:
5436 case Builtin::BI__sync_swap_16:
5437 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::Xchg, E);
5438
5439 case Builtin::BI__sync_lock_test_and_set_1:
5440 case Builtin::BI__sync_lock_test_and_set_2:
5441 case Builtin::BI__sync_lock_test_and_set_4:
5442 case Builtin::BI__sync_lock_test_and_set_8:
5443 case Builtin::BI__sync_lock_test_and_set_16:
5444 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::Xchg, E);
5445
5446 case Builtin::BI__sync_lock_release_1:
5447 case Builtin::BI__sync_lock_release_2:
5448 case Builtin::BI__sync_lock_release_4:
5449 case Builtin::BI__sync_lock_release_8:
5450 case Builtin::BI__sync_lock_release_16: {
5451 Address Ptr = CheckAtomicAlignment(CGF&: *this, E);
5452 QualType ElTy = E->getArg(Arg: 0)->getType()->getPointeeType();
5453
5454 llvm::Type *ITy = llvm::IntegerType::get(C&: getLLVMContext(),
5455 NumBits: getContext().getTypeSize(T: ElTy));
5456 llvm::StoreInst *Store =
5457 Builder.CreateStore(Val: llvm::Constant::getNullValue(Ty: ITy), Addr: Ptr);
5458 Store->setAtomic(Ordering: llvm::AtomicOrdering::Release);
5459 return RValue::get(V: nullptr);
5460 }
5461
5462 case Builtin::BI__sync_synchronize: {
5463 // We assume this is supposed to correspond to a C++0x-style
5464 // sequentially-consistent fence (i.e. this is only usable for
5465 // synchronization, not device I/O or anything like that). This intrinsic
5466 // is really badly designed in the sense that in theory, there isn't
5467 // any way to safely use it... but in practice, it mostly works
5468 // to use it with non-atomic loads and stores to get acquire/release
5469 // semantics.
5470 Builder.CreateFence(Ordering: llvm::AtomicOrdering::SequentiallyConsistent);
5471 return RValue::get(V: nullptr);
5472 }
5473
5474 case Builtin::BI__builtin_nontemporal_load:
5475 return RValue::get(V: EmitNontemporalLoad(CGF&: *this, E));
5476 case Builtin::BI__builtin_nontemporal_store:
5477 return RValue::get(V: EmitNontemporalStore(CGF&: *this, E));
5478 case Builtin::BI__c11_atomic_is_lock_free:
5479 case Builtin::BI__atomic_is_lock_free: {
5480 // Call "bool __atomic_is_lock_free(size_t size, void *ptr)". For the
5481 // __c11 builtin, ptr is 0 (indicating a properly-aligned object), since
5482 // _Atomic(T) is always properly-aligned.
5483 const char *LibCallName = "__atomic_is_lock_free";
5484 CallArgList Args;
5485 Args.add(rvalue: RValue::get(V: EmitScalarExpr(E: E->getArg(Arg: 0))),
5486 type: getContext().getSizeType());
5487 if (BuiltinID == Builtin::BI__atomic_is_lock_free)
5488 Args.add(rvalue: RValue::get(V: EmitScalarExpr(E: E->getArg(Arg: 1))),
5489 type: getContext().VoidPtrTy);
5490 else
5491 Args.add(rvalue: RValue::get(V: llvm::Constant::getNullValue(Ty: VoidPtrTy)),
5492 type: getContext().VoidPtrTy);
5493 const CGFunctionInfo &FuncInfo =
5494 CGM.getTypes().arrangeBuiltinFunctionCall(resultType: E->getType(), args: Args);
5495 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(Info: FuncInfo);
5496 llvm::FunctionCallee Func = CGM.CreateRuntimeFunction(Ty: FTy, Name: LibCallName);
5497 return EmitCall(CallInfo: FuncInfo, Callee: CGCallee::forDirect(functionPtr: Func),
5498 ReturnValue: ReturnValueSlot(), Args);
5499 }
5500
5501 case Builtin::BI__atomic_thread_fence:
5502 case Builtin::BI__atomic_signal_fence:
5503 case Builtin::BI__c11_atomic_thread_fence:
5504 case Builtin::BI__c11_atomic_signal_fence: {
5505 llvm::SyncScope::ID SSID;
5506 if (BuiltinID == Builtin::BI__atomic_signal_fence ||
5507 BuiltinID == Builtin::BI__c11_atomic_signal_fence)
5508 SSID = llvm::SyncScope::SingleThread;
5509 else
5510 SSID = llvm::SyncScope::System;
5511 Value *Order = EmitScalarExpr(E: E->getArg(Arg: 0));
5512 if (isa<llvm::ConstantInt>(Val: Order)) {
5513 int ord = cast<llvm::ConstantInt>(Val: Order)->getZExtValue();
5514 switch (ord) {
5515 case 0: // memory_order_relaxed
5516 default: // invalid order
5517 break;
5518 case 1: // memory_order_consume
5519 case 2: // memory_order_acquire
5520 emitAtomicFence(Order: llvm::AtomicOrdering::Acquire, SSID);
5521 break;
5522 case 3: // memory_order_release
5523 emitAtomicFence(Order: llvm::AtomicOrdering::Release, SSID);
5524 break;
5525 case 4: // memory_order_acq_rel
5526 emitAtomicFence(Order: llvm::AtomicOrdering::AcquireRelease, SSID);
5527 break;
5528 case 5: // memory_order_seq_cst
5529 emitAtomicFence(Order: llvm::AtomicOrdering::SequentiallyConsistent, SSID);
5530 break;
5531 }
5532 return RValue::get(V: nullptr);
5533 }
5534
5535 llvm::BasicBlock *AcquireBB, *ReleaseBB, *AcqRelBB, *SeqCstBB;
5536 AcquireBB = createBasicBlock(name: "acquire", parent: CurFn);
5537 ReleaseBB = createBasicBlock(name: "release", parent: CurFn);
5538 AcqRelBB = createBasicBlock(name: "acqrel", parent: CurFn);
5539 SeqCstBB = createBasicBlock(name: "seqcst", parent: CurFn);
5540 llvm::BasicBlock *ContBB = createBasicBlock(name: "atomic.continue", parent: CurFn);
5541
5542 Order = Builder.CreateIntCast(V: Order, DestTy: Builder.getInt32Ty(), isSigned: false);
5543 llvm::SwitchInst *SI = Builder.CreateSwitch(V: Order, Dest: ContBB);
5544
5545 Builder.SetInsertPoint(AcquireBB);
5546 emitAtomicFence(Order: llvm::AtomicOrdering::Acquire, SSID);
5547 Builder.CreateBr(Dest: ContBB);
5548 SI->addCase(OnVal: Builder.getInt32(C: 1), Dest: AcquireBB);
5549 SI->addCase(OnVal: Builder.getInt32(C: 2), Dest: AcquireBB);
5550
5551 Builder.SetInsertPoint(ReleaseBB);
5552 emitAtomicFence(Order: llvm::AtomicOrdering::Release, SSID);
5553 Builder.CreateBr(Dest: ContBB);
5554 SI->addCase(OnVal: Builder.getInt32(C: 3), Dest: ReleaseBB);
5555
5556 Builder.SetInsertPoint(AcqRelBB);
5557 emitAtomicFence(Order: llvm::AtomicOrdering::AcquireRelease, SSID);
5558 Builder.CreateBr(Dest: ContBB);
5559 SI->addCase(OnVal: Builder.getInt32(C: 4), Dest: AcqRelBB);
5560
5561 Builder.SetInsertPoint(SeqCstBB);
5562 emitAtomicFence(Order: llvm::AtomicOrdering::SequentiallyConsistent, SSID);
5563 Builder.CreateBr(Dest: ContBB);
5564 SI->addCase(OnVal: Builder.getInt32(C: 5), Dest: SeqCstBB);
5565
5566 Builder.SetInsertPoint(ContBB);
5567 return RValue::get(V: nullptr);
5568 }
5569 case Builtin::BI__scoped_atomic_thread_fence: {
5570 auto ScopeModel = AtomicScopeModel::create(K: AtomicScopeModelKind::Generic);
5571
5572 Value *Order = EmitScalarExpr(E: E->getArg(Arg: 0));
5573 Value *Scope = EmitScalarExpr(E: E->getArg(Arg: 1));
5574 auto Ord = dyn_cast<llvm::ConstantInt>(Val: Order);
5575 auto Scp = dyn_cast<llvm::ConstantInt>(Val: Scope);
5576 if (Ord && Scp) {
5577 SyncScope SS = ScopeModel->isValid(S: Scp->getZExtValue())
5578 ? ScopeModel->map(S: Scp->getZExtValue())
5579 : ScopeModel->map(S: ScopeModel->getFallBackValue());
5580 switch (Ord->getZExtValue()) {
5581 case 0: // memory_order_relaxed
5582 default: // invalid order
5583 break;
5584 case 1: // memory_order_consume
5585 case 2: // memory_order_acquire
5586 emitAtomicFence(Order: llvm::AtomicOrdering::Acquire,
5587 SSID: getTargetHooks().getLLVMSyncScopeID(
5588 LangOpts: getLangOpts(), Scope: SS, Ordering: llvm::AtomicOrdering::Acquire,
5589 Ctx&: getLLVMContext()));
5590 break;
5591 case 3: // memory_order_release
5592 emitAtomicFence(Order: llvm::AtomicOrdering::Release,
5593 SSID: getTargetHooks().getLLVMSyncScopeID(
5594 LangOpts: getLangOpts(), Scope: SS, Ordering: llvm::AtomicOrdering::Release,
5595 Ctx&: getLLVMContext()));
5596 break;
5597 case 4: // memory_order_acq_rel
5598 emitAtomicFence(Order: llvm::AtomicOrdering::AcquireRelease,
5599 SSID: getTargetHooks().getLLVMSyncScopeID(
5600 LangOpts: getLangOpts(), Scope: SS,
5601 Ordering: llvm::AtomicOrdering::AcquireRelease,
5602 Ctx&: getLLVMContext()));
5603 break;
5604 case 5: // memory_order_seq_cst
5605 emitAtomicFence(Order: llvm::AtomicOrdering::SequentiallyConsistent,
5606 SSID: getTargetHooks().getLLVMSyncScopeID(
5607 LangOpts: getLangOpts(), Scope: SS,
5608 Ordering: llvm::AtomicOrdering::SequentiallyConsistent,
5609 Ctx&: getLLVMContext()));
5610 break;
5611 }
5612 return RValue::get(V: nullptr);
5613 }
5614
5615 llvm::BasicBlock *ContBB = createBasicBlock(name: "atomic.scope.continue", parent: CurFn);
5616
5617 llvm::SmallVector<std::pair<llvm::BasicBlock *, llvm::AtomicOrdering>>
5618 OrderBBs;
5619 if (Ord) {
5620 switch (Ord->getZExtValue()) {
5621 case 0: // memory_order_relaxed
5622 default: // invalid order
5623 ContBB->eraseFromParent();
5624 return RValue::get(V: nullptr);
5625 case 1: // memory_order_consume
5626 case 2: // memory_order_acquire
5627 OrderBBs.emplace_back(Args: Builder.GetInsertBlock(),
5628 Args: llvm::AtomicOrdering::Acquire);
5629 break;
5630 case 3: // memory_order_release
5631 OrderBBs.emplace_back(Args: Builder.GetInsertBlock(),
5632 Args: llvm::AtomicOrdering::Release);
5633 break;
5634 case 4: // memory_order_acq_rel
5635 OrderBBs.emplace_back(Args: Builder.GetInsertBlock(),
5636 Args: llvm::AtomicOrdering::AcquireRelease);
5637 break;
5638 case 5: // memory_order_seq_cst
5639 OrderBBs.emplace_back(Args: Builder.GetInsertBlock(),
5640 Args: llvm::AtomicOrdering::SequentiallyConsistent);
5641 break;
5642 }
5643 } else {
5644 llvm::BasicBlock *AcquireBB = createBasicBlock(name: "acquire", parent: CurFn);
5645 llvm::BasicBlock *ReleaseBB = createBasicBlock(name: "release", parent: CurFn);
5646 llvm::BasicBlock *AcqRelBB = createBasicBlock(name: "acqrel", parent: CurFn);
5647 llvm::BasicBlock *SeqCstBB = createBasicBlock(name: "seqcst", parent: CurFn);
5648
5649 Order = Builder.CreateIntCast(V: Order, DestTy: Builder.getInt32Ty(), isSigned: false);
5650 llvm::SwitchInst *SI = Builder.CreateSwitch(V: Order, Dest: ContBB);
5651 SI->addCase(OnVal: Builder.getInt32(C: 1), Dest: AcquireBB);
5652 SI->addCase(OnVal: Builder.getInt32(C: 2), Dest: AcquireBB);
5653 SI->addCase(OnVal: Builder.getInt32(C: 3), Dest: ReleaseBB);
5654 SI->addCase(OnVal: Builder.getInt32(C: 4), Dest: AcqRelBB);
5655 SI->addCase(OnVal: Builder.getInt32(C: 5), Dest: SeqCstBB);
5656
5657 OrderBBs.emplace_back(Args&: AcquireBB, Args: llvm::AtomicOrdering::Acquire);
5658 OrderBBs.emplace_back(Args&: ReleaseBB, Args: llvm::AtomicOrdering::Release);
5659 OrderBBs.emplace_back(Args&: AcqRelBB, Args: llvm::AtomicOrdering::AcquireRelease);
5660 OrderBBs.emplace_back(Args&: SeqCstBB,
5661 Args: llvm::AtomicOrdering::SequentiallyConsistent);
5662 }
5663
5664 for (auto &[OrderBB, Ordering] : OrderBBs) {
5665 Builder.SetInsertPoint(OrderBB);
5666 if (Scp) {
5667 SyncScope SS = ScopeModel->isValid(S: Scp->getZExtValue())
5668 ? ScopeModel->map(S: Scp->getZExtValue())
5669 : ScopeModel->map(S: ScopeModel->getFallBackValue());
5670 emitAtomicFence(Order: Ordering,
5671 SSID: getTargetHooks().getLLVMSyncScopeID(
5672 LangOpts: getLangOpts(), Scope: SS, Ordering, Ctx&: getLLVMContext()));
5673 Builder.CreateBr(Dest: ContBB);
5674 } else {
5675 llvm::DenseMap<unsigned, llvm::BasicBlock *> BBs;
5676 for (unsigned Scp : ScopeModel->getRuntimeValues())
5677 BBs[Scp] = createBasicBlock(name: getAsString(S: ScopeModel->map(S: Scp)), parent: CurFn);
5678
5679 auto *SC = Builder.CreateIntCast(V: Scope, DestTy: Builder.getInt32Ty(), isSigned: false);
5680 llvm::SwitchInst *SI = Builder.CreateSwitch(V: SC, Dest: ContBB);
5681 for (unsigned Scp : ScopeModel->getRuntimeValues()) {
5682 auto *B = BBs[Scp];
5683 SI->addCase(OnVal: Builder.getInt32(C: Scp), Dest: B);
5684
5685 Builder.SetInsertPoint(B);
5686 emitAtomicFence(Order: Ordering, SSID: getTargetHooks().getLLVMSyncScopeID(
5687 LangOpts: getLangOpts(), Scope: ScopeModel->map(S: Scp),
5688 Ordering, Ctx&: getLLVMContext()));
5689 Builder.CreateBr(Dest: ContBB);
5690 }
5691 }
5692 }
5693
5694 Builder.SetInsertPoint(ContBB);
5695 return RValue::get(V: nullptr);
5696 }
5697
5698 case Builtin::BI__builtin_signbit:
5699 case Builtin::BI__builtin_signbitf:
5700 case Builtin::BI__builtin_signbitl: {
5701 return RValue::get(
5702 V: Builder.CreateZExt(V: EmitSignBit(CGF&: *this, V: EmitScalarExpr(E: E->getArg(Arg: 0))),
5703 DestTy: ConvertType(T: E->getType())));
5704 }
5705 case Builtin::BI__warn_memset_zero_len:
5706 return RValue::getIgnored();
5707 case Builtin::BI__annotation: {
5708 // Re-encode each wide string to UTF8 and make an MDString.
5709 SmallVector<Metadata *, 1> Strings;
5710 for (const Expr *Arg : E->arguments()) {
5711 const auto *Str = cast<StringLiteral>(Val: Arg->IgnoreParenCasts());
5712 assert(Str->getCharByteWidth() == 2 || Str->getCharByteWidth() == 4);
5713 StringRef WideBytes = Str->getBytes();
5714 std::string StrUtf8;
5715 bool Converted =
5716 (Str->getCharByteWidth() == 2)
5717 ? convertUTF16ToUTF8String(
5718 SrcBytes: ArrayRef(WideBytes.data(), WideBytes.size()), Out&: StrUtf8)
5719 : convertUTF32ToUTF8String(
5720 SrcBytes: ArrayRef(WideBytes.data(), WideBytes.size()), Out&: StrUtf8);
5721 if (!Converted) {
5722 CGM.ErrorUnsupported(S: E, Type: "non-Unicode __annotation argument");
5723 continue;
5724 }
5725 Strings.push_back(Elt: llvm::MDString::get(Context&: getLLVMContext(), Str: StrUtf8));
5726 }
5727
5728 // Build and MDTuple of MDStrings and emit the intrinsic call.
5729 llvm::Function *F = CGM.getIntrinsic(IID: Intrinsic::codeview_annotation, Tys: {});
5730 MDTuple *StrTuple = MDTuple::get(Context&: getLLVMContext(), MDs: Strings);
5731 Builder.CreateCall(Callee: F, Args: MetadataAsValue::get(Context&: getLLVMContext(), MD: StrTuple));
5732 return RValue::getIgnored();
5733 }
5734 case Builtin::BI__builtin_annotation: {
5735 llvm::Value *AnnVal = EmitScalarExpr(E: E->getArg(Arg: 0));
5736 llvm::Function *F = CGM.getIntrinsic(
5737 IID: Intrinsic::annotation, Tys: {AnnVal->getType(), CGM.ConstGlobalsPtrTy});
5738
5739 // Get the annotation string, go through casts. Sema requires this to be a
5740 // non-wide string literal, potentially casted, so the cast<> is safe.
5741 const Expr *AnnotationStrExpr = E->getArg(Arg: 1)->IgnoreParenCasts();
5742 StringRef Str = cast<StringLiteral>(Val: AnnotationStrExpr)->getString();
5743 return RValue::get(
5744 V: EmitAnnotationCall(AnnotationFn: F, AnnotatedVal: AnnVal, AnnotationStr: Str, Location: E->getExprLoc(), Attr: nullptr));
5745 }
5746 case Builtin::BI__builtin_addcb:
5747 case Builtin::BI__builtin_addcs:
5748 case Builtin::BI__builtin_addc:
5749 case Builtin::BI__builtin_addcl:
5750 case Builtin::BI__builtin_addcll:
5751 case Builtin::BI__builtin_subcb:
5752 case Builtin::BI__builtin_subcs:
5753 case Builtin::BI__builtin_subc:
5754 case Builtin::BI__builtin_subcl:
5755 case Builtin::BI__builtin_subcll: {
5756
5757 // We translate all of these builtins from expressions of the form:
5758 // int x = ..., y = ..., carryin = ..., carryout, result;
5759 // result = __builtin_addc(x, y, carryin, &carryout);
5760 //
5761 // to LLVM IR of the form:
5762 //
5763 // %tmp1 = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %x, i32 %y)
5764 // %tmpsum1 = extractvalue {i32, i1} %tmp1, 0
5765 // %carry1 = extractvalue {i32, i1} %tmp1, 1
5766 // %tmp2 = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %tmpsum1,
5767 // i32 %carryin)
5768 // %result = extractvalue {i32, i1} %tmp2, 0
5769 // %carry2 = extractvalue {i32, i1} %tmp2, 1
5770 // %tmp3 = or i1 %carry1, %carry2
5771 // %tmp4 = zext i1 %tmp3 to i32
5772 // store i32 %tmp4, i32* %carryout
5773
5774 // Scalarize our inputs.
5775 llvm::Value *X = EmitScalarExpr(E: E->getArg(Arg: 0));
5776 llvm::Value *Y = EmitScalarExpr(E: E->getArg(Arg: 1));
5777 llvm::Value *Carryin = EmitScalarExpr(E: E->getArg(Arg: 2));
5778 Address CarryOutPtr = EmitPointerWithAlignment(Addr: E->getArg(Arg: 3));
5779
5780 // Decide if we are lowering to a uadd.with.overflow or usub.with.overflow.
5781 Intrinsic::ID IntrinsicId;
5782 switch (BuiltinID) {
5783 default: llvm_unreachable("Unknown multiprecision builtin id.");
5784 case Builtin::BI__builtin_addcb:
5785 case Builtin::BI__builtin_addcs:
5786 case Builtin::BI__builtin_addc:
5787 case Builtin::BI__builtin_addcl:
5788 case Builtin::BI__builtin_addcll:
5789 IntrinsicId = Intrinsic::uadd_with_overflow;
5790 break;
5791 case Builtin::BI__builtin_subcb:
5792 case Builtin::BI__builtin_subcs:
5793 case Builtin::BI__builtin_subc:
5794 case Builtin::BI__builtin_subcl:
5795 case Builtin::BI__builtin_subcll:
5796 IntrinsicId = Intrinsic::usub_with_overflow;
5797 break;
5798 }
5799
5800 // Construct our resulting LLVM IR expression.
5801 llvm::Value *Carry1;
5802 llvm::Value *Sum1 = EmitOverflowIntrinsic(CGF&: *this, IntrinsicID: IntrinsicId,
5803 X, Y, Carry&: Carry1);
5804 llvm::Value *Carry2;
5805 llvm::Value *Sum2 = EmitOverflowIntrinsic(CGF&: *this, IntrinsicID: IntrinsicId,
5806 X: Sum1, Y: Carryin, Carry&: Carry2);
5807 llvm::Value *CarryOut = Builder.CreateZExt(V: Builder.CreateOr(LHS: Carry1, RHS: Carry2),
5808 DestTy: X->getType());
5809 Builder.CreateStore(Val: CarryOut, Addr: CarryOutPtr);
5810 return RValue::get(V: Sum2);
5811 }
5812
5813 case Builtin::BI__builtin_add_overflow:
5814 case Builtin::BI__builtin_sub_overflow:
5815 case Builtin::BI__builtin_mul_overflow: {
5816 const clang::Expr *LeftArg = E->getArg(Arg: 0);
5817 const clang::Expr *RightArg = E->getArg(Arg: 1);
5818 const clang::Expr *ResultArg = E->getArg(Arg: 2);
5819
5820 clang::QualType ResultQTy =
5821 ResultArg->getType()->castAs<PointerType>()->getPointeeType();
5822
5823 WidthAndSignedness LeftInfo =
5824 getIntegerWidthAndSignedness(context: CGM.getContext(), Type: LeftArg->getType());
5825 WidthAndSignedness RightInfo =
5826 getIntegerWidthAndSignedness(context: CGM.getContext(), Type: RightArg->getType());
5827 WidthAndSignedness ResultInfo =
5828 getIntegerWidthAndSignedness(context: CGM.getContext(), Type: ResultQTy);
5829
5830 // Handle mixed-sign multiplication as a special case, because adding
5831 // runtime or backend support for our generic irgen would be too expensive.
5832 if (isSpecialMixedSignMultiply(BuiltinID, Op1Info: LeftInfo, Op2Info: RightInfo, ResultInfo))
5833 return EmitCheckedMixedSignMultiply(CGF&: *this, Op1: LeftArg, Op1Info: LeftInfo, Op2: RightArg,
5834 Op2Info: RightInfo, ResultArg, ResultQTy,
5835 ResultInfo);
5836
5837 if (isSpecialUnsignedMultiplySignedResult(BuiltinID, Op1Info: LeftInfo, Op2Info: RightInfo,
5838 ResultInfo))
5839 return EmitCheckedUnsignedMultiplySignedResult(
5840 CGF&: *this, Op1: LeftArg, Op1Info: LeftInfo, Op2: RightArg, Op2Info: RightInfo, ResultArg, ResultQTy,
5841 ResultInfo);
5842
5843 WidthAndSignedness EncompassingInfo =
5844 EncompassingIntegerType(Types: {LeftInfo, RightInfo, ResultInfo});
5845
5846 llvm::Type *EncompassingLLVMTy =
5847 llvm::IntegerType::get(C&: CGM.getLLVMContext(), NumBits: EncompassingInfo.Width);
5848
5849 llvm::Type *ResultLLVMTy = CGM.getTypes().ConvertType(T: ResultQTy);
5850
5851 Intrinsic::ID IntrinsicId;
5852 switch (BuiltinID) {
5853 default:
5854 llvm_unreachable("Unknown overflow builtin id.");
5855 case Builtin::BI__builtin_add_overflow:
5856 IntrinsicId = EncompassingInfo.Signed ? Intrinsic::sadd_with_overflow
5857 : Intrinsic::uadd_with_overflow;
5858 break;
5859 case Builtin::BI__builtin_sub_overflow:
5860 IntrinsicId = EncompassingInfo.Signed ? Intrinsic::ssub_with_overflow
5861 : Intrinsic::usub_with_overflow;
5862 break;
5863 case Builtin::BI__builtin_mul_overflow:
5864 IntrinsicId = EncompassingInfo.Signed ? Intrinsic::smul_with_overflow
5865 : Intrinsic::umul_with_overflow;
5866 break;
5867 }
5868
5869 llvm::Value *Left = EmitScalarExpr(E: LeftArg);
5870 llvm::Value *Right = EmitScalarExpr(E: RightArg);
5871 Address ResultPtr = EmitPointerWithAlignment(Addr: ResultArg);
5872
5873 // Extend each operand to the encompassing type.
5874 Left = Builder.CreateIntCast(V: Left, DestTy: EncompassingLLVMTy, isSigned: LeftInfo.Signed);
5875 Right = Builder.CreateIntCast(V: Right, DestTy: EncompassingLLVMTy, isSigned: RightInfo.Signed);
5876
5877 // Perform the operation on the extended values.
5878 llvm::Value *Overflow, *Result;
5879 Result = EmitOverflowIntrinsic(CGF&: *this, IntrinsicID: IntrinsicId, X: Left, Y: Right, Carry&: Overflow);
5880
5881 if (EncompassingInfo.Width > ResultInfo.Width) {
5882 // The encompassing type is wider than the result type, so we need to
5883 // truncate it.
5884 llvm::Value *ResultTrunc = Builder.CreateTrunc(V: Result, DestTy: ResultLLVMTy);
5885
5886 // To see if the truncation caused an overflow, we will extend
5887 // the result and then compare it to the original result.
5888 llvm::Value *ResultTruncExt = Builder.CreateIntCast(
5889 V: ResultTrunc, DestTy: EncompassingLLVMTy, isSigned: ResultInfo.Signed);
5890 llvm::Value *TruncationOverflow =
5891 Builder.CreateICmpNE(LHS: Result, RHS: ResultTruncExt);
5892
5893 Overflow = Builder.CreateOr(LHS: Overflow, RHS: TruncationOverflow);
5894 Result = ResultTrunc;
5895 }
5896
5897 // Finally, store the result using the pointer.
5898 bool isVolatile =
5899 ResultArg->getType()->getPointeeType().isVolatileQualified();
5900 Builder.CreateStore(Val: EmitToMemory(Value: Result, Ty: ResultQTy), Addr: ResultPtr, IsVolatile: isVolatile);
5901
5902 return RValue::get(V: Overflow);
5903 }
5904
5905 case Builtin::BI__builtin_uadd_overflow:
5906 case Builtin::BI__builtin_uaddl_overflow:
5907 case Builtin::BI__builtin_uaddll_overflow:
5908 case Builtin::BI__builtin_usub_overflow:
5909 case Builtin::BI__builtin_usubl_overflow:
5910 case Builtin::BI__builtin_usubll_overflow:
5911 case Builtin::BI__builtin_umul_overflow:
5912 case Builtin::BI__builtin_umull_overflow:
5913 case Builtin::BI__builtin_umulll_overflow:
5914 case Builtin::BI__builtin_sadd_overflow:
5915 case Builtin::BI__builtin_saddl_overflow:
5916 case Builtin::BI__builtin_saddll_overflow:
5917 case Builtin::BI__builtin_ssub_overflow:
5918 case Builtin::BI__builtin_ssubl_overflow:
5919 case Builtin::BI__builtin_ssubll_overflow:
5920 case Builtin::BI__builtin_smul_overflow:
5921 case Builtin::BI__builtin_smull_overflow:
5922 case Builtin::BI__builtin_smulll_overflow: {
5923
5924 // We translate all of these builtins directly to the relevant llvm IR node.
5925
5926 // Scalarize our inputs.
5927 llvm::Value *X = EmitScalarExpr(E: E->getArg(Arg: 0));
5928 llvm::Value *Y = EmitScalarExpr(E: E->getArg(Arg: 1));
5929 Address SumOutPtr = EmitPointerWithAlignment(Addr: E->getArg(Arg: 2));
5930
5931 // Decide which of the overflow intrinsics we are lowering to:
5932 Intrinsic::ID IntrinsicId;
5933 switch (BuiltinID) {
5934 default: llvm_unreachable("Unknown overflow builtin id.");
5935 case Builtin::BI__builtin_uadd_overflow:
5936 case Builtin::BI__builtin_uaddl_overflow:
5937 case Builtin::BI__builtin_uaddll_overflow:
5938 IntrinsicId = Intrinsic::uadd_with_overflow;
5939 break;
5940 case Builtin::BI__builtin_usub_overflow:
5941 case Builtin::BI__builtin_usubl_overflow:
5942 case Builtin::BI__builtin_usubll_overflow:
5943 IntrinsicId = Intrinsic::usub_with_overflow;
5944 break;
5945 case Builtin::BI__builtin_umul_overflow:
5946 case Builtin::BI__builtin_umull_overflow:
5947 case Builtin::BI__builtin_umulll_overflow:
5948 IntrinsicId = Intrinsic::umul_with_overflow;
5949 break;
5950 case Builtin::BI__builtin_sadd_overflow:
5951 case Builtin::BI__builtin_saddl_overflow:
5952 case Builtin::BI__builtin_saddll_overflow:
5953 IntrinsicId = Intrinsic::sadd_with_overflow;
5954 break;
5955 case Builtin::BI__builtin_ssub_overflow:
5956 case Builtin::BI__builtin_ssubl_overflow:
5957 case Builtin::BI__builtin_ssubll_overflow:
5958 IntrinsicId = Intrinsic::ssub_with_overflow;
5959 break;
5960 case Builtin::BI__builtin_smul_overflow:
5961 case Builtin::BI__builtin_smull_overflow:
5962 case Builtin::BI__builtin_smulll_overflow:
5963 IntrinsicId = Intrinsic::smul_with_overflow;
5964 break;
5965 }
5966
5967
5968 llvm::Value *Carry;
5969 llvm::Value *Sum = EmitOverflowIntrinsic(CGF&: *this, IntrinsicID: IntrinsicId, X, Y, Carry);
5970 Builder.CreateStore(Val: Sum, Addr: SumOutPtr);
5971
5972 return RValue::get(V: Carry);
5973 }
5974 case Builtin::BIaddressof:
5975 case Builtin::BI__addressof:
5976 case Builtin::BI__builtin_addressof:
5977 return RValue::get(V: EmitLValue(E: E->getArg(Arg: 0)).getPointer(CGF&: *this));
5978 case Builtin::BI__builtin_function_start:
5979 return RValue::get(V: CGM.GetFunctionStart(
5980 Decl: E->getArg(Arg: 0)->getAsBuiltinConstantDeclRef(Context: CGM.getContext())));
5981 case Builtin::BI__builtin_operator_new:
5982 return EmitBuiltinNewDeleteCall(
5983 Type: E->getCallee()->getType()->castAs<FunctionProtoType>(), TheCallExpr: E, IsDelete: false);
5984 case Builtin::BI__builtin_operator_delete:
5985 EmitBuiltinNewDeleteCall(
5986 Type: E->getCallee()->getType()->castAs<FunctionProtoType>(), TheCallExpr: E, IsDelete: true);
5987 return RValue::get(V: nullptr);
5988
5989 case Builtin::BI__builtin_is_aligned:
5990 return EmitBuiltinIsAligned(E);
5991 case Builtin::BI__builtin_align_up:
5992 return EmitBuiltinAlignTo(E, AlignUp: true);
5993 case Builtin::BI__builtin_align_down:
5994 return EmitBuiltinAlignTo(E, AlignUp: false);
5995
5996 case Builtin::BI__noop:
5997 // __noop always evaluates to an integer literal zero.
5998 return RValue::get(V: ConstantInt::get(Ty: IntTy, V: 0));
5999 case Builtin::BI__builtin_call_with_static_chain: {
6000 const CallExpr *Call = cast<CallExpr>(Val: E->getArg(Arg: 0));
6001 const Expr *Chain = E->getArg(Arg: 1);
6002 return EmitCall(FnType: Call->getCallee()->getType(),
6003 Callee: EmitCallee(E: Call->getCallee()), E: Call, ReturnValue,
6004 Chain: EmitScalarExpr(E: Chain));
6005 }
6006 case Builtin::BI_InterlockedExchange8:
6007 case Builtin::BI_InterlockedExchange16:
6008 case Builtin::BI_InterlockedExchange:
6009 case Builtin::BI_InterlockedExchangePointer:
6010 return RValue::get(
6011 V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::_InterlockedExchange, E));
6012 case Builtin::BI_InterlockedCompareExchangePointer:
6013 return RValue::get(
6014 V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::_InterlockedCompareExchange, E));
6015 case Builtin::BI_InterlockedCompareExchangePointer_nf:
6016 return RValue::get(
6017 V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::_InterlockedCompareExchange_nf, E));
6018 case Builtin::BI_InterlockedCompareExchange8:
6019 case Builtin::BI_InterlockedCompareExchange16:
6020 case Builtin::BI_InterlockedCompareExchange:
6021 case Builtin::BI_InterlockedCompareExchange64:
6022 return RValue::get(V: EmitAtomicCmpXchgForMSIntrin(CGF&: *this, E));
6023 case Builtin::BI_InterlockedIncrement16:
6024 case Builtin::BI_InterlockedIncrement:
6025 return RValue::get(
6026 V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::_InterlockedIncrement, E));
6027 case Builtin::BI_InterlockedDecrement16:
6028 case Builtin::BI_InterlockedDecrement:
6029 return RValue::get(
6030 V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::_InterlockedDecrement, E));
6031 case Builtin::BI_InterlockedAnd8:
6032 case Builtin::BI_InterlockedAnd16:
6033 case Builtin::BI_InterlockedAnd:
6034 return RValue::get(V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::_InterlockedAnd, E));
6035 case Builtin::BI_InterlockedExchangeAdd8:
6036 case Builtin::BI_InterlockedExchangeAdd16:
6037 case Builtin::BI_InterlockedExchangeAdd:
6038 return RValue::get(
6039 V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::_InterlockedExchangeAdd, E));
6040 case Builtin::BI_InterlockedExchangeSub8:
6041 case Builtin::BI_InterlockedExchangeSub16:
6042 case Builtin::BI_InterlockedExchangeSub:
6043 return RValue::get(
6044 V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::_InterlockedExchangeSub, E));
6045 case Builtin::BI_InterlockedOr8:
6046 case Builtin::BI_InterlockedOr16:
6047 case Builtin::BI_InterlockedOr:
6048 return RValue::get(V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::_InterlockedOr, E));
6049 case Builtin::BI_InterlockedXor8:
6050 case Builtin::BI_InterlockedXor16:
6051 case Builtin::BI_InterlockedXor:
6052 return RValue::get(V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::_InterlockedXor, E));
6053
6054 case Builtin::BI_bittest64:
6055 case Builtin::BI_bittest:
6056 case Builtin::BI_bittestandcomplement64:
6057 case Builtin::BI_bittestandcomplement:
6058 case Builtin::BI_bittestandreset64:
6059 case Builtin::BI_bittestandreset:
6060 case Builtin::BI_bittestandset64:
6061 case Builtin::BI_bittestandset:
6062 case Builtin::BI_interlockedbittestandreset:
6063 case Builtin::BI_interlockedbittestandreset64:
6064 case Builtin::BI_interlockedbittestandreset64_acq:
6065 case Builtin::BI_interlockedbittestandreset64_rel:
6066 case Builtin::BI_interlockedbittestandreset64_nf:
6067 case Builtin::BI_interlockedbittestandset64:
6068 case Builtin::BI_interlockedbittestandset64_acq:
6069 case Builtin::BI_interlockedbittestandset64_rel:
6070 case Builtin::BI_interlockedbittestandset64_nf:
6071 case Builtin::BI_interlockedbittestandset:
6072 case Builtin::BI_interlockedbittestandset_acq:
6073 case Builtin::BI_interlockedbittestandset_rel:
6074 case Builtin::BI_interlockedbittestandset_nf:
6075 case Builtin::BI_interlockedbittestandreset_acq:
6076 case Builtin::BI_interlockedbittestandreset_rel:
6077 case Builtin::BI_interlockedbittestandreset_nf:
6078 return RValue::get(V: EmitBitTestIntrinsic(CGF&: *this, BuiltinID, E));
6079
6080 // These builtins exist to emit regular volatile loads and stores not
6081 // affected by the -fms-volatile setting.
6082 case Builtin::BI__iso_volatile_load8:
6083 case Builtin::BI__iso_volatile_load16:
6084 case Builtin::BI__iso_volatile_load32:
6085 case Builtin::BI__iso_volatile_load64:
6086 return RValue::get(V: EmitISOVolatileLoad(CGF&: *this, E));
6087 case Builtin::BI__iso_volatile_store8:
6088 case Builtin::BI__iso_volatile_store16:
6089 case Builtin::BI__iso_volatile_store32:
6090 case Builtin::BI__iso_volatile_store64:
6091 return RValue::get(V: EmitISOVolatileStore(CGF&: *this, E));
6092
6093 case Builtin::BI__builtin_ptrauth_sign_constant:
6094 return RValue::get(V: ConstantEmitter(*this).emitAbstract(E, T: E->getType()));
6095
6096 case Builtin::BI__builtin_ptrauth_auth:
6097 case Builtin::BI__builtin_ptrauth_auth_and_resign:
6098 case Builtin::BI__builtin_ptrauth_auth_with_pc_and_resign:
6099 case Builtin::BI__builtin_ptrauth_auth_load_relative_and_sign:
6100 case Builtin::BI__builtin_ptrauth_blend_discriminator:
6101 case Builtin::BI__builtin_ptrauth_sign_generic_data:
6102 case Builtin::BI__builtin_ptrauth_sign_unauthenticated:
6103 case Builtin::BI__builtin_ptrauth_strip: {
6104 // Emit the arguments.
6105 SmallVector<llvm::Value *, 6> Args;
6106 for (auto argExpr : E->arguments())
6107 Args.push_back(Elt: EmitScalarExpr(E: argExpr));
6108
6109 // Cast the value to intptr_t, saving its original type.
6110 llvm::Type *OrigValueType = Args[0]->getType();
6111 if (OrigValueType->isPointerTy())
6112 Args[0] = Builder.CreatePtrToInt(V: Args[0], DestTy: IntPtrTy);
6113
6114 switch (BuiltinID) {
6115 case Builtin::BI__builtin_ptrauth_auth_with_pc_and_resign:
6116 // Convert oldDiscriminator (arg 2), oldPC (arg 3) and newDiscriminator
6117 // (arg 5) to intptr_t
6118 if (Args[2]->getType()->isPointerTy())
6119 Args[2] = Builder.CreatePtrToInt(V: Args[2], DestTy: IntPtrTy);
6120 if (Args[3]->getType()->isPointerTy())
6121 Args[3] = Builder.CreatePtrToInt(V: Args[3], DestTy: IntPtrTy);
6122 if (Args[5]->getType()->isPointerTy())
6123 Args[5] = Builder.CreatePtrToInt(V: Args[5], DestTy: IntPtrTy);
6124 break;
6125
6126 case Builtin::BI__builtin_ptrauth_auth_and_resign:
6127 case Builtin::BI__builtin_ptrauth_auth_load_relative_and_sign:
6128 if (Args[4]->getType()->isPointerTy())
6129 Args[4] = Builder.CreatePtrToInt(V: Args[4], DestTy: IntPtrTy);
6130 [[fallthrough]];
6131
6132 case Builtin::BI__builtin_ptrauth_auth:
6133 case Builtin::BI__builtin_ptrauth_sign_unauthenticated:
6134 if (Args[2]->getType()->isPointerTy())
6135 Args[2] = Builder.CreatePtrToInt(V: Args[2], DestTy: IntPtrTy);
6136 break;
6137
6138 case Builtin::BI__builtin_ptrauth_sign_generic_data:
6139 if (Args[1]->getType()->isPointerTy())
6140 Args[1] = Builder.CreatePtrToInt(V: Args[1], DestTy: IntPtrTy);
6141 break;
6142
6143 case Builtin::BI__builtin_ptrauth_blend_discriminator:
6144 case Builtin::BI__builtin_ptrauth_strip:
6145 break;
6146 }
6147
6148 // Call the intrinsic.
6149 auto IntrinsicID = [&]() -> unsigned {
6150 switch (BuiltinID) {
6151 case Builtin::BI__builtin_ptrauth_auth:
6152 return Intrinsic::ptrauth_auth;
6153 case Builtin::BI__builtin_ptrauth_auth_and_resign:
6154 return Intrinsic::ptrauth_resign;
6155 case Builtin::BI__builtin_ptrauth_auth_with_pc_and_resign:
6156 return Intrinsic::ptrauth_auth_with_pc_and_resign;
6157 case Builtin::BI__builtin_ptrauth_auth_load_relative_and_sign:
6158 return Intrinsic::ptrauth_resign_load_relative;
6159 case Builtin::BI__builtin_ptrauth_blend_discriminator:
6160 return Intrinsic::ptrauth_blend;
6161 case Builtin::BI__builtin_ptrauth_sign_generic_data:
6162 return Intrinsic::ptrauth_sign_generic;
6163 case Builtin::BI__builtin_ptrauth_sign_unauthenticated:
6164 return Intrinsic::ptrauth_sign;
6165 case Builtin::BI__builtin_ptrauth_strip:
6166 return Intrinsic::ptrauth_strip;
6167 }
6168 llvm_unreachable("bad ptrauth intrinsic");
6169 }();
6170 auto Intrinsic = CGM.getIntrinsic(IID: IntrinsicID);
6171 llvm::Value *Result = EmitRuntimeCall(callee: Intrinsic, args: Args);
6172
6173 if (BuiltinID != Builtin::BI__builtin_ptrauth_sign_generic_data &&
6174 BuiltinID != Builtin::BI__builtin_ptrauth_blend_discriminator &&
6175 OrigValueType->isPointerTy()) {
6176 Result = Builder.CreateIntToPtr(V: Result, DestTy: OrigValueType);
6177 }
6178 return RValue::get(V: Result);
6179 }
6180
6181 case Builtin::BI__builtin_get_vtable_pointer: {
6182 const Expr *Target = E->getArg(Arg: 0);
6183 QualType TargetType = Target->getType();
6184 const CXXRecordDecl *Decl = TargetType->getPointeeCXXRecordDecl();
6185 assert(Decl);
6186 auto ThisAddress = EmitPointerWithAlignment(Addr: Target);
6187 assert(ThisAddress.isValid());
6188 llvm::Value *VTablePointer =
6189 GetVTablePtr(This: ThisAddress, VTableTy: Int8PtrTy, VTableClass: Decl, AuthMode: VTableAuthMode::MustTrap);
6190 return RValue::get(V: VTablePointer);
6191 }
6192
6193 case Builtin::BI__exception_code:
6194 case Builtin::BI_exception_code:
6195 return RValue::get(V: EmitSEHExceptionCode());
6196 case Builtin::BI__exception_info:
6197 case Builtin::BI_exception_info:
6198 return RValue::get(V: EmitSEHExceptionInfo());
6199 case Builtin::BI__abnormal_termination:
6200 case Builtin::BI_abnormal_termination:
6201 return RValue::get(V: EmitSEHAbnormalTermination());
6202 case Builtin::BI_setjmpex:
6203 if (getTarget().getTriple().isOSMSVCRT() && E->getNumArgs() == 1 &&
6204 E->getArg(Arg: 0)->getType()->isPointerType())
6205 return EmitMSVCRTSetJmp(CGF&: *this, SJKind: MSVCSetJmpKind::_setjmpex, E);
6206 break;
6207 case Builtin::BI_setjmp:
6208 if (getTarget().getTriple().isOSMSVCRT() && E->getNumArgs() == 1 &&
6209 E->getArg(Arg: 0)->getType()->isPointerType()) {
6210 if (getTarget().getTriple().getArch() == llvm::Triple::x86)
6211 return EmitMSVCRTSetJmp(CGF&: *this, SJKind: MSVCSetJmpKind::_setjmp3, E);
6212 else if (getTarget().getTriple().getArch() == llvm::Triple::aarch64)
6213 return EmitMSVCRTSetJmp(CGF&: *this, SJKind: MSVCSetJmpKind::_setjmpex, E);
6214 return EmitMSVCRTSetJmp(CGF&: *this, SJKind: MSVCSetJmpKind::_setjmp, E);
6215 }
6216 break;
6217
6218 // C++ std:: builtins.
6219 case Builtin::BImove:
6220 case Builtin::BImove_if_noexcept:
6221 case Builtin::BIforward:
6222 case Builtin::BIforward_like:
6223 case Builtin::BIas_const:
6224 return RValue::get(V: EmitLValue(E: E->getArg(Arg: 0)).getPointer(CGF&: *this));
6225 case Builtin::BI__GetExceptionInfo: {
6226 if (llvm::GlobalVariable *GV =
6227 CGM.getCXXABI().getThrowInfo(T: FD->getParamDecl(i: 0)->getType()))
6228 return RValue::get(V: GV);
6229 break;
6230 }
6231
6232 case Builtin::BI__fastfail:
6233 return RValue::get(V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::__fastfail, E));
6234
6235 case Builtin::BI__builtin_coro_id:
6236 return EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_id);
6237 case Builtin::BI__builtin_coro_promise:
6238 return EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_promise);
6239 case Builtin::BI__builtin_coro_resume:
6240 EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_resume);
6241 return RValue::get(V: nullptr);
6242 case Builtin::BI__builtin_coro_frame:
6243 return EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_frame);
6244 case Builtin::BI__builtin_coro_noop:
6245 return EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_noop);
6246 case Builtin::BI__builtin_coro_free:
6247 return EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_free);
6248 case Builtin::BI__builtin_coro_destroy:
6249 EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_destroy);
6250 return RValue::get(V: nullptr);
6251 case Builtin::BI__builtin_coro_done:
6252 return EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_done);
6253 case Builtin::BI__builtin_coro_alloc:
6254 return EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_alloc);
6255 case Builtin::BI__builtin_coro_begin:
6256 return EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_begin);
6257 case Builtin::BI__builtin_coro_end:
6258 return EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_end);
6259 case Builtin::BI__builtin_coro_suspend:
6260 return EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_suspend);
6261 case Builtin::BI__builtin_coro_size:
6262 return EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_size);
6263 case Builtin::BI__builtin_coro_align:
6264 return EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_align);
6265
6266 // OpenCL v2.0 s6.13.16.2, Built-in pipe read and write functions
6267 case Builtin::BIread_pipe:
6268 case Builtin::BIwrite_pipe: {
6269 Value *Arg0 = EmitScalarExpr(E: E->getArg(Arg: 0)),
6270 *Arg1 = EmitScalarExpr(E: E->getArg(Arg: 1));
6271 CGOpenCLRuntime OpenCLRT(CGM);
6272 Value *PacketSize = OpenCLRT.getPipeElemSize(PipeArg: E->getArg(Arg: 0));
6273 Value *PacketAlign = OpenCLRT.getPipeElemAlign(PipeArg: E->getArg(Arg: 0));
6274
6275 // Type of the generic packet parameter.
6276 unsigned GenericAS =
6277 getContext().getTargetAddressSpace(AS: LangAS::opencl_generic);
6278 llvm::Type *I8PTy = llvm::PointerType::get(C&: getLLVMContext(), AddressSpace: GenericAS);
6279
6280 // Testing which overloaded version we should generate the call for.
6281 if (2U == E->getNumArgs()) {
6282 const char *Name = (BuiltinID == Builtin::BIread_pipe) ? "__read_pipe_2"
6283 : "__write_pipe_2";
6284 // Creating a generic function type to be able to call with any builtin or
6285 // user defined type.
6286 llvm::Type *ArgTys[] = {Arg0->getType(), I8PTy, Int32Ty, Int32Ty};
6287 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: Int32Ty, Params: ArgTys, isVarArg: false);
6288 Value *ACast = Builder.CreateAddrSpaceCast(V: Arg1, DestTy: I8PTy);
6289 return RValue::get(
6290 V: EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(Ty: FTy, Name),
6291 args: {Arg0, ACast, PacketSize, PacketAlign}));
6292 } else {
6293 assert(4 == E->getNumArgs() &&
6294 "Illegal number of parameters to pipe function");
6295 const char *Name = (BuiltinID == Builtin::BIread_pipe) ? "__read_pipe_4"
6296 : "__write_pipe_4";
6297
6298 llvm::Type *ArgTys[] = {Arg0->getType(), Arg1->getType(), Int32Ty, I8PTy,
6299 Int32Ty, Int32Ty};
6300 Value *Arg2 = EmitScalarExpr(E: E->getArg(Arg: 2)),
6301 *Arg3 = EmitScalarExpr(E: E->getArg(Arg: 3));
6302 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: Int32Ty, Params: ArgTys, isVarArg: false);
6303 Value *ACast = Builder.CreateAddrSpaceCast(V: Arg3, DestTy: I8PTy);
6304 // We know the third argument is an integer type, but we may need to cast
6305 // it to i32.
6306 if (Arg2->getType() != Int32Ty)
6307 Arg2 = Builder.CreateZExtOrTrunc(V: Arg2, DestTy: Int32Ty);
6308 return RValue::get(
6309 V: EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(Ty: FTy, Name),
6310 args: {Arg0, Arg1, Arg2, ACast, PacketSize, PacketAlign}));
6311 }
6312 }
6313 // OpenCL v2.0 s6.13.16 ,s9.17.3.5 - Built-in pipe reserve read and write
6314 // functions
6315 case Builtin::BIreserve_read_pipe:
6316 case Builtin::BIreserve_write_pipe:
6317 case Builtin::BIwork_group_reserve_read_pipe:
6318 case Builtin::BIwork_group_reserve_write_pipe:
6319 case Builtin::BIsub_group_reserve_read_pipe:
6320 case Builtin::BIsub_group_reserve_write_pipe: {
6321 // Composing the mangled name for the function.
6322 const char *Name;
6323 if (BuiltinID == Builtin::BIreserve_read_pipe)
6324 Name = "__reserve_read_pipe";
6325 else if (BuiltinID == Builtin::BIreserve_write_pipe)
6326 Name = "__reserve_write_pipe";
6327 else if (BuiltinID == Builtin::BIwork_group_reserve_read_pipe)
6328 Name = "__work_group_reserve_read_pipe";
6329 else if (BuiltinID == Builtin::BIwork_group_reserve_write_pipe)
6330 Name = "__work_group_reserve_write_pipe";
6331 else if (BuiltinID == Builtin::BIsub_group_reserve_read_pipe)
6332 Name = "__sub_group_reserve_read_pipe";
6333 else
6334 Name = "__sub_group_reserve_write_pipe";
6335
6336 Value *Arg0 = EmitScalarExpr(E: E->getArg(Arg: 0)),
6337 *Arg1 = EmitScalarExpr(E: E->getArg(Arg: 1));
6338 llvm::Type *ReservedIDTy = ConvertType(T: getContext().OCLReserveIDTy);
6339 CGOpenCLRuntime OpenCLRT(CGM);
6340 Value *PacketSize = OpenCLRT.getPipeElemSize(PipeArg: E->getArg(Arg: 0));
6341 Value *PacketAlign = OpenCLRT.getPipeElemAlign(PipeArg: E->getArg(Arg: 0));
6342
6343 // Building the generic function prototype.
6344 llvm::Type *ArgTys[] = {Arg0->getType(), Int32Ty, Int32Ty, Int32Ty};
6345 llvm::FunctionType *FTy =
6346 llvm::FunctionType::get(Result: ReservedIDTy, Params: ArgTys, isVarArg: false);
6347 // We know the second argument is an integer type, but we may need to cast
6348 // it to i32.
6349 if (Arg1->getType() != Int32Ty)
6350 Arg1 = Builder.CreateZExtOrTrunc(V: Arg1, DestTy: Int32Ty);
6351 return RValue::get(V: EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(Ty: FTy, Name),
6352 args: {Arg0, Arg1, PacketSize, PacketAlign}));
6353 }
6354 // OpenCL v2.0 s6.13.16, s9.17.3.5 - Built-in pipe commit read and write
6355 // functions
6356 case Builtin::BIcommit_read_pipe:
6357 case Builtin::BIcommit_write_pipe:
6358 case Builtin::BIwork_group_commit_read_pipe:
6359 case Builtin::BIwork_group_commit_write_pipe:
6360 case Builtin::BIsub_group_commit_read_pipe:
6361 case Builtin::BIsub_group_commit_write_pipe: {
6362 const char *Name;
6363 if (BuiltinID == Builtin::BIcommit_read_pipe)
6364 Name = "__commit_read_pipe";
6365 else if (BuiltinID == Builtin::BIcommit_write_pipe)
6366 Name = "__commit_write_pipe";
6367 else if (BuiltinID == Builtin::BIwork_group_commit_read_pipe)
6368 Name = "__work_group_commit_read_pipe";
6369 else if (BuiltinID == Builtin::BIwork_group_commit_write_pipe)
6370 Name = "__work_group_commit_write_pipe";
6371 else if (BuiltinID == Builtin::BIsub_group_commit_read_pipe)
6372 Name = "__sub_group_commit_read_pipe";
6373 else
6374 Name = "__sub_group_commit_write_pipe";
6375
6376 Value *Arg0 = EmitScalarExpr(E: E->getArg(Arg: 0)),
6377 *Arg1 = EmitScalarExpr(E: E->getArg(Arg: 1));
6378 CGOpenCLRuntime OpenCLRT(CGM);
6379 Value *PacketSize = OpenCLRT.getPipeElemSize(PipeArg: E->getArg(Arg: 0));
6380 Value *PacketAlign = OpenCLRT.getPipeElemAlign(PipeArg: E->getArg(Arg: 0));
6381
6382 // Building the generic function prototype.
6383 llvm::Type *ArgTys[] = {Arg0->getType(), Arg1->getType(), Int32Ty, Int32Ty};
6384 llvm::FunctionType *FTy = llvm::FunctionType::get(
6385 Result: llvm::Type::getVoidTy(C&: getLLVMContext()), Params: ArgTys, isVarArg: false);
6386
6387 return RValue::get(V: EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(Ty: FTy, Name),
6388 args: {Arg0, Arg1, PacketSize, PacketAlign}));
6389 }
6390 // OpenCL v2.0 s6.13.16.4 Built-in pipe query functions
6391 case Builtin::BIget_pipe_num_packets:
6392 case Builtin::BIget_pipe_max_packets: {
6393 const char *BaseName;
6394 const auto *PipeTy = E->getArg(Arg: 0)->getType()->castAs<PipeType>();
6395 if (BuiltinID == Builtin::BIget_pipe_num_packets)
6396 BaseName = "__get_pipe_num_packets";
6397 else
6398 BaseName = "__get_pipe_max_packets";
6399 std::string Name = std::string(BaseName) +
6400 std::string(PipeTy->isReadOnly() ? "_ro" : "_wo");
6401
6402 // Building the generic function prototype.
6403 Value *Arg0 = EmitScalarExpr(E: E->getArg(Arg: 0));
6404 CGOpenCLRuntime OpenCLRT(CGM);
6405 Value *PacketSize = OpenCLRT.getPipeElemSize(PipeArg: E->getArg(Arg: 0));
6406 Value *PacketAlign = OpenCLRT.getPipeElemAlign(PipeArg: E->getArg(Arg: 0));
6407 llvm::Type *ArgTys[] = {Arg0->getType(), Int32Ty, Int32Ty};
6408 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: Int32Ty, Params: ArgTys, isVarArg: false);
6409
6410 return RValue::get(V: EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(Ty: FTy, Name),
6411 args: {Arg0, PacketSize, PacketAlign}));
6412 }
6413
6414 // OpenCL v2.0 s6.13.9 - Address space qualifier functions.
6415 case Builtin::BIto_global:
6416 case Builtin::BIto_local:
6417 case Builtin::BIto_private: {
6418 auto Arg0 = EmitScalarExpr(E: E->getArg(Arg: 0));
6419 auto NewArgT = llvm::PointerType::get(
6420 C&: getLLVMContext(),
6421 AddressSpace: CGM.getContext().getTargetAddressSpace(AS: LangAS::opencl_generic));
6422 auto NewRetT = llvm::PointerType::get(
6423 C&: getLLVMContext(),
6424 AddressSpace: CGM.getContext().getTargetAddressSpace(
6425 AS: E->getType()->getPointeeType().getAddressSpace()));
6426 auto FTy = llvm::FunctionType::get(Result: NewRetT, Params: {NewArgT}, isVarArg: false);
6427 llvm::Value *NewArg;
6428 if (Arg0->getType()->getPointerAddressSpace() !=
6429 NewArgT->getPointerAddressSpace())
6430 NewArg = Builder.CreateAddrSpaceCast(V: Arg0, DestTy: NewArgT);
6431 else
6432 NewArg = Builder.CreateBitOrPointerCast(V: Arg0, DestTy: NewArgT);
6433 auto NewName = std::string("__") + E->getDirectCallee()->getName().str();
6434 auto NewCall =
6435 EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(Ty: FTy, Name: NewName), args: {NewArg});
6436 return RValue::get(V: Builder.CreateBitOrPointerCast(V: NewCall,
6437 DestTy: ConvertType(T: E->getType())));
6438 }
6439
6440 // OpenCL v2.0, s6.13.17 - Enqueue kernel function.
6441 // Table 6.13.17.1 specifies four overload forms of enqueue_kernel.
6442 // The code below expands the builtin call to a call to one of the following
6443 // functions that an OpenCL runtime library will have to provide:
6444 // __enqueue_kernel_basic
6445 // __enqueue_kernel_varargs
6446 // __enqueue_kernel_basic_events
6447 // __enqueue_kernel_events_varargs
6448 case Builtin::BIenqueue_kernel: {
6449 StringRef Name; // Generated function call name
6450 unsigned NumArgs = E->getNumArgs();
6451
6452 llvm::Type *QueueTy = ConvertType(T: getContext().OCLQueueTy);
6453 llvm::Type *GenericVoidPtrTy = Builder.getPtrTy(
6454 AddrSpace: getContext().getTargetAddressSpace(AS: LangAS::opencl_generic));
6455
6456 llvm::Value *Queue = EmitScalarExpr(E: E->getArg(Arg: 0));
6457 llvm::Value *Flags = EmitScalarExpr(E: E->getArg(Arg: 1));
6458 LValue NDRangeL = EmitAggExprToLValue(E: E->getArg(Arg: 2));
6459 llvm::Value *Range = NDRangeL.getAddress().emitRawPointer(CGF&: *this);
6460
6461 // FIXME: Look through the addrspacecast which may exist to the stack
6462 // temporary as a hack.
6463 //
6464 // This is hardcoding the assumed ABI of the target function. This assumes
6465 // direct passing for every argument except NDRange, which is assumed to be
6466 // byval or byref indirect passed.
6467 //
6468 // This should be fixed to query a signature from CGOpenCLRuntime, and go
6469 // through EmitCallArgs to get the correct target ABI.
6470 Range = Range->stripPointerCasts();
6471
6472 llvm::Type *RangePtrTy = Range->getType();
6473
6474 if (NumArgs == 4) {
6475 // The most basic form of the call with parameters:
6476 // queue_t, kernel_enqueue_flags_t, ndrange_t, block(void)
6477 Name = "__enqueue_kernel_basic";
6478 llvm::Type *ArgTys[] = {QueueTy, Int32Ty, RangePtrTy, GenericVoidPtrTy,
6479 GenericVoidPtrTy};
6480 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: Int32Ty, Params: ArgTys, isVarArg: false);
6481
6482 auto Info =
6483 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(CGF&: *this, E: E->getArg(Arg: 3));
6484 llvm::Value *Kernel =
6485 Builder.CreatePointerCast(V: Info.KernelHandle, DestTy: GenericVoidPtrTy);
6486 llvm::Value *Block =
6487 Builder.CreatePointerCast(V: Info.BlockArg, DestTy: GenericVoidPtrTy);
6488
6489 auto RTCall = EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(Ty: FTy, Name),
6490 args: {Queue, Flags, Range, Kernel, Block});
6491 return RValue::get(V: RTCall);
6492 }
6493 assert(NumArgs >= 5 && "Invalid enqueue_kernel signature");
6494
6495 // Create a temporary array to hold the sizes of local pointer arguments
6496 // for the block. \p First is the position of the first size argument.
6497 auto CreateArrayForSizeVar =
6498 [=](unsigned First) -> std::pair<llvm::Value *, llvm::Value *> {
6499 llvm::APInt ArraySize(32, NumArgs - First);
6500 QualType SizeArrayTy = getContext().getConstantArrayType(
6501 EltTy: getContext().getSizeType(), ArySize: ArraySize, SizeExpr: nullptr,
6502 ASM: ArraySizeModifier::Normal,
6503 /*IndexTypeQuals=*/0);
6504 auto Tmp = CreateMemTempWithoutCast(T: SizeArrayTy, Name: "block_sizes");
6505 llvm::Value *Alloca = Tmp.getPointer();
6506 llvm::Value *ElemPtr;
6507 EmitLifetimeStart(Addr: Alloca);
6508 // Each of the following arguments specifies the size of the corresponding
6509 // argument passed to the enqueued block.
6510 auto *Zero = llvm::ConstantInt::get(Ty: IntTy, V: 0);
6511 for (unsigned I = First; I < NumArgs; ++I) {
6512 auto *Index = llvm::ConstantInt::get(Ty: IntTy, V: I - First);
6513 auto *GEP =
6514 Builder.CreateGEP(Ty: Tmp.getElementType(), Ptr: Alloca, IdxList: {Zero, Index});
6515 if (I == First)
6516 ElemPtr = GEP;
6517 auto *V =
6518 Builder.CreateZExtOrTrunc(V: EmitScalarExpr(E: E->getArg(Arg: I)), DestTy: SizeTy);
6519 Builder.CreateAlignedStore(
6520 Val: V, Ptr: GEP, Align: CGM.getDataLayout().getPrefTypeAlign(Ty: SizeTy));
6521 }
6522 return {ElemPtr, Alloca};
6523 };
6524
6525 // Could have events and/or varargs.
6526 if (E->getArg(Arg: 3)->getType()->isBlockPointerType()) {
6527 // No events passed, but has variadic arguments.
6528 Name = "__enqueue_kernel_varargs";
6529 auto Info =
6530 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(CGF&: *this, E: E->getArg(Arg: 3));
6531 llvm::Value *Kernel =
6532 Builder.CreatePointerCast(V: Info.KernelHandle, DestTy: GenericVoidPtrTy);
6533 auto *Block = Builder.CreatePointerCast(V: Info.BlockArg, DestTy: GenericVoidPtrTy);
6534 auto [ElemPtr, TmpPtr] = CreateArrayForSizeVar(4);
6535
6536 // Create a vector of the arguments, as well as a constant value to
6537 // express to the runtime the number of variadic arguments.
6538 llvm::Value *const Args[] = {Queue, Flags,
6539 Range, Kernel,
6540 Block, ConstantInt::get(Ty: IntTy, V: NumArgs - 4),
6541 ElemPtr};
6542 llvm::Type *const ArgTys[] = {
6543 QueueTy, IntTy, RangePtrTy, GenericVoidPtrTy,
6544 GenericVoidPtrTy, IntTy, ElemPtr->getType()};
6545
6546 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: Int32Ty, Params: ArgTys, isVarArg: false);
6547 auto Call = RValue::get(
6548 V: EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(Ty: FTy, Name), args: Args));
6549 EmitLifetimeEnd(Addr: TmpPtr);
6550 return Call;
6551 }
6552 // Any calls now have event arguments passed.
6553 if (NumArgs >= 7) {
6554 llvm::PointerType *PtrTy = llvm::PointerType::get(
6555 C&: CGM.getLLVMContext(),
6556 AddressSpace: CGM.getContext().getTargetAddressSpace(AS: LangAS::opencl_generic));
6557
6558 llvm::Value *NumEvents =
6559 Builder.CreateZExtOrTrunc(V: EmitScalarExpr(E: E->getArg(Arg: 3)), DestTy: Int32Ty);
6560
6561 // Since SemaOpenCLBuiltinEnqueueKernel allows fifth and sixth arguments
6562 // to be a null pointer constant (including `0` literal), we can take it
6563 // into account and emit null pointer directly.
6564 llvm::Value *EventWaitList = nullptr;
6565 if (E->getArg(Arg: 4)->isNullPointerConstant(
6566 Ctx&: getContext(), NPC: Expr::NPC_ValueDependentIsNotNull)) {
6567 EventWaitList = llvm::ConstantPointerNull::get(T: PtrTy);
6568 } else {
6569 EventWaitList =
6570 E->getArg(Arg: 4)->getType()->isArrayType()
6571 ? EmitArrayToPointerDecay(Array: E->getArg(Arg: 4)).emitRawPointer(CGF&: *this)
6572 : EmitScalarExpr(E: E->getArg(Arg: 4));
6573 // Convert to generic address space.
6574 EventWaitList = Builder.CreatePointerCast(V: EventWaitList, DestTy: PtrTy);
6575 }
6576 llvm::Value *EventRet = nullptr;
6577 if (E->getArg(Arg: 5)->isNullPointerConstant(
6578 Ctx&: getContext(), NPC: Expr::NPC_ValueDependentIsNotNull)) {
6579 EventRet = llvm::ConstantPointerNull::get(T: PtrTy);
6580 } else {
6581 EventRet =
6582 Builder.CreatePointerCast(V: EmitScalarExpr(E: E->getArg(Arg: 5)), DestTy: PtrTy);
6583 }
6584
6585 auto Info =
6586 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(CGF&: *this, E: E->getArg(Arg: 6));
6587 llvm::Value *Kernel =
6588 Builder.CreatePointerCast(V: Info.KernelHandle, DestTy: GenericVoidPtrTy);
6589 llvm::Value *Block =
6590 Builder.CreatePointerCast(V: Info.BlockArg, DestTy: GenericVoidPtrTy);
6591
6592 std::vector<llvm::Type *> ArgTys = {
6593 QueueTy, Int32Ty, RangePtrTy, Int32Ty,
6594 PtrTy, PtrTy, GenericVoidPtrTy, GenericVoidPtrTy};
6595
6596 std::vector<llvm::Value *> Args = {Queue, Flags, Range,
6597 NumEvents, EventWaitList, EventRet,
6598 Kernel, Block};
6599
6600 if (NumArgs == 7) {
6601 // Has events but no variadics.
6602 Name = "__enqueue_kernel_basic_events";
6603 llvm::FunctionType *FTy =
6604 llvm::FunctionType::get(Result: Int32Ty, Params: ArgTys, isVarArg: false);
6605 return RValue::get(
6606 V: EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(Ty: FTy, Name), args: Args));
6607 }
6608 // Has event info and variadics
6609 // Pass the number of variadics to the runtime function too.
6610 Args.push_back(x: ConstantInt::get(Ty: Int32Ty, V: NumArgs - 7));
6611 ArgTys.push_back(x: Int32Ty);
6612 Name = "__enqueue_kernel_events_varargs";
6613
6614 auto [ElemPtr, TmpPtr] = CreateArrayForSizeVar(7);
6615 Args.push_back(x: ElemPtr);
6616 ArgTys.push_back(x: ElemPtr->getType());
6617
6618 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: Int32Ty, Params: ArgTys, isVarArg: false);
6619 auto Call = RValue::get(
6620 V: EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(Ty: FTy, Name), args: Args));
6621 EmitLifetimeEnd(Addr: TmpPtr);
6622 return Call;
6623 }
6624 llvm_unreachable("Unexpected enqueue_kernel signature");
6625 }
6626 // OpenCL v2.0 s6.13.17.6 - Kernel query functions need bitcast of block
6627 // parameter.
6628 case Builtin::BIget_kernel_work_group_size: {
6629 llvm::Type *GenericVoidPtrTy = Builder.getPtrTy(
6630 AddrSpace: getContext().getTargetAddressSpace(AS: LangAS::opencl_generic));
6631 auto Info =
6632 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(CGF&: *this, E: E->getArg(Arg: 0));
6633 Value *Kernel =
6634 Builder.CreatePointerCast(V: Info.KernelHandle, DestTy: GenericVoidPtrTy);
6635 Value *Arg = Builder.CreatePointerCast(V: Info.BlockArg, DestTy: GenericVoidPtrTy);
6636 return RValue::get(V: EmitRuntimeCall(
6637 callee: CGM.CreateRuntimeFunction(
6638 Ty: llvm::FunctionType::get(Result: IntTy, Params: {GenericVoidPtrTy, GenericVoidPtrTy},
6639 isVarArg: false),
6640 Name: "__get_kernel_work_group_size_impl"),
6641 args: {Kernel, Arg}));
6642 }
6643 case Builtin::BIget_kernel_preferred_work_group_size_multiple: {
6644 llvm::Type *GenericVoidPtrTy = Builder.getPtrTy(
6645 AddrSpace: getContext().getTargetAddressSpace(AS: LangAS::opencl_generic));
6646 auto Info =
6647 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(CGF&: *this, E: E->getArg(Arg: 0));
6648 Value *Kernel =
6649 Builder.CreatePointerCast(V: Info.KernelHandle, DestTy: GenericVoidPtrTy);
6650 Value *Arg = Builder.CreatePointerCast(V: Info.BlockArg, DestTy: GenericVoidPtrTy);
6651 return RValue::get(V: EmitRuntimeCall(
6652 callee: CGM.CreateRuntimeFunction(
6653 Ty: llvm::FunctionType::get(Result: IntTy, Params: {GenericVoidPtrTy, GenericVoidPtrTy},
6654 isVarArg: false),
6655 Name: "__get_kernel_preferred_work_group_size_multiple_impl"),
6656 args: {Kernel, Arg}));
6657 }
6658 case Builtin::BIget_kernel_max_sub_group_size_for_ndrange:
6659 case Builtin::BIget_kernel_sub_group_count_for_ndrange: {
6660 llvm::Type *GenericVoidPtrTy = Builder.getPtrTy(
6661 AddrSpace: getContext().getTargetAddressSpace(AS: LangAS::opencl_generic));
6662 LValue NDRangeL = EmitAggExprToLValue(E: E->getArg(Arg: 0));
6663 llvm::Value *NDRange = NDRangeL.getAddress().emitRawPointer(CGF&: *this);
6664 auto Info =
6665 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(CGF&: *this, E: E->getArg(Arg: 1));
6666 Value *Kernel =
6667 Builder.CreatePointerCast(V: Info.KernelHandle, DestTy: GenericVoidPtrTy);
6668 Value *Block = Builder.CreatePointerCast(V: Info.BlockArg, DestTy: GenericVoidPtrTy);
6669 const char *Name =
6670 BuiltinID == Builtin::BIget_kernel_max_sub_group_size_for_ndrange
6671 ? "__get_kernel_max_sub_group_size_for_ndrange_impl"
6672 : "__get_kernel_sub_group_count_for_ndrange_impl";
6673 return RValue::get(V: EmitRuntimeCall(
6674 callee: CGM.CreateRuntimeFunction(
6675 Ty: llvm::FunctionType::get(
6676 Result: IntTy, Params: {NDRange->getType(), GenericVoidPtrTy, GenericVoidPtrTy},
6677 isVarArg: false),
6678 Name),
6679 args: {NDRange, Kernel, Block}));
6680 }
6681 case Builtin::BI__builtin_store_half:
6682 case Builtin::BI__builtin_store_halff: {
6683 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
6684 Value *Val = EmitScalarExpr(E: E->getArg(Arg: 0));
6685 Address Address = EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
6686 Value *HalfVal = Builder.CreateFPTrunc(V: Val, DestTy: Builder.getHalfTy());
6687 Builder.CreateStore(Val: HalfVal, Addr: Address);
6688 return RValue::get(V: nullptr);
6689 }
6690 case Builtin::BI__builtin_load_half: {
6691 Address Address = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
6692 Value *HalfVal = Builder.CreateLoad(Addr: Address);
6693 return RValue::get(V: Builder.CreateFPExt(V: HalfVal, DestTy: Builder.getDoubleTy()));
6694 }
6695 case Builtin::BI__builtin_load_halff: {
6696 Address Address = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
6697 Value *HalfVal = Builder.CreateLoad(Addr: Address);
6698 return RValue::get(V: Builder.CreateFPExt(V: HalfVal, DestTy: Builder.getFloatTy()));
6699 }
6700 case Builtin::BI__builtin_printf:
6701 case Builtin::BIprintf:
6702 if (getTarget().getTriple().isNVPTX() ||
6703 getTarget().getTriple().isAMDGCN() ||
6704 (getTarget().getTriple().isSPIRV() &&
6705 getTarget().getTriple().getVendor() == Triple::VendorType::AMD)) {
6706 if (getTarget().getTriple().isNVPTX())
6707 return EmitNVPTXDevicePrintfCallExpr(E);
6708 if ((getTarget().getTriple().isAMDGCN() ||
6709 getTarget().getTriple().isSPIRV()) &&
6710 getLangOpts().HIP)
6711 return EmitAMDGPUDevicePrintfCallExpr(E);
6712 }
6713
6714 break;
6715 case Builtin::BI__builtin_canonicalize:
6716 case Builtin::BI__builtin_canonicalizef:
6717 case Builtin::BI__builtin_canonicalizef16:
6718 case Builtin::BI__builtin_canonicalizel:
6719 return RValue::get(
6720 V: emitBuiltinWithOneOverloadedType<1>(CGF&: *this, E, IntrinsicID: Intrinsic::canonicalize));
6721
6722 case Builtin::BI__builtin_thread_pointer: {
6723 if (!getContext().getTargetInfo().isTLSSupported())
6724 CGM.ErrorUnsupported(S: E, Type: "__builtin_thread_pointer");
6725
6726 return RValue::get(V: Builder.CreateIntrinsic(ID: llvm::Intrinsic::thread_pointer,
6727 OverloadTypes: {GlobalsInt8PtrTy}, Args: {}));
6728 }
6729 case Builtin::BI__builtin_os_log_format:
6730 return emitBuiltinOSLogFormat(E: *E);
6731
6732 case Builtin::BI__xray_customevent: {
6733 if (!ShouldXRayInstrumentFunction())
6734 return RValue::getIgnored();
6735
6736 if (!CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
6737 K: XRayInstrKind::Custom))
6738 return RValue::getIgnored();
6739
6740 if (const auto *XRayAttr = CurFuncDecl->getAttr<XRayInstrumentAttr>())
6741 if (XRayAttr->neverXRayInstrument() && !AlwaysEmitXRayCustomEvents())
6742 return RValue::getIgnored();
6743
6744 Function *F = CGM.getIntrinsic(IID: Intrinsic::xray_customevent);
6745 auto FTy = F->getFunctionType();
6746 auto Arg0 = E->getArg(Arg: 0);
6747 auto Arg0Val = EmitScalarExpr(E: Arg0);
6748 auto Arg0Ty = Arg0->getType();
6749 auto PTy0 = FTy->getParamType(i: 0);
6750 if (PTy0 != Arg0Val->getType()) {
6751 if (Arg0Ty->isArrayType())
6752 Arg0Val = EmitArrayToPointerDecay(Array: Arg0).emitRawPointer(CGF&: *this);
6753 else
6754 Arg0Val = Builder.CreatePointerCast(V: Arg0Val, DestTy: PTy0);
6755 }
6756 auto Arg1 = EmitScalarExpr(E: E->getArg(Arg: 1));
6757 auto PTy1 = FTy->getParamType(i: 1);
6758 if (PTy1 != Arg1->getType())
6759 Arg1 = Builder.CreateTruncOrBitCast(V: Arg1, DestTy: PTy1);
6760 return RValue::get(V: Builder.CreateCall(Callee: F, Args: {Arg0Val, Arg1}));
6761 }
6762
6763 case Builtin::BI__xray_typedevent: {
6764 // TODO: There should be a way to always emit events even if the current
6765 // function is not instrumented. Losing events in a stream can cripple
6766 // a trace.
6767 if (!ShouldXRayInstrumentFunction())
6768 return RValue::getIgnored();
6769
6770 if (!CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
6771 K: XRayInstrKind::Typed))
6772 return RValue::getIgnored();
6773
6774 if (const auto *XRayAttr = CurFuncDecl->getAttr<XRayInstrumentAttr>())
6775 if (XRayAttr->neverXRayInstrument() && !AlwaysEmitXRayTypedEvents())
6776 return RValue::getIgnored();
6777
6778 Function *F = CGM.getIntrinsic(IID: Intrinsic::xray_typedevent);
6779 auto FTy = F->getFunctionType();
6780 auto Arg0 = EmitScalarExpr(E: E->getArg(Arg: 0));
6781 auto PTy0 = FTy->getParamType(i: 0);
6782 if (PTy0 != Arg0->getType())
6783 Arg0 = Builder.CreateTruncOrBitCast(V: Arg0, DestTy: PTy0);
6784 auto Arg1 = E->getArg(Arg: 1);
6785 auto Arg1Val = EmitScalarExpr(E: Arg1);
6786 auto Arg1Ty = Arg1->getType();
6787 auto PTy1 = FTy->getParamType(i: 1);
6788 if (PTy1 != Arg1Val->getType()) {
6789 if (Arg1Ty->isArrayType())
6790 Arg1Val = EmitArrayToPointerDecay(Array: Arg1).emitRawPointer(CGF&: *this);
6791 else
6792 Arg1Val = Builder.CreatePointerCast(V: Arg1Val, DestTy: PTy1);
6793 }
6794 auto Arg2 = EmitScalarExpr(E: E->getArg(Arg: 2));
6795 auto PTy2 = FTy->getParamType(i: 2);
6796 if (PTy2 != Arg2->getType())
6797 Arg2 = Builder.CreateTruncOrBitCast(V: Arg2, DestTy: PTy2);
6798 return RValue::get(V: Builder.CreateCall(Callee: F, Args: {Arg0, Arg1Val, Arg2}));
6799 }
6800
6801 case Builtin::BI__builtin_ms_va_start:
6802 case Builtin::BI__builtin_ms_va_end:
6803 return RValue::get(
6804 V: EmitVAStartEnd(ArgValue: EmitMSVAListRef(E: E->getArg(Arg: 0)).emitRawPointer(CGF&: *this),
6805 IsStart: BuiltinID == Builtin::BI__builtin_ms_va_start));
6806
6807 case Builtin::BI__builtin_ms_va_copy: {
6808 // Lower this manually. We can't reliably determine whether or not any
6809 // given va_copy() is for a Win64 va_list from the calling convention
6810 // alone, because it's legal to do this from a System V ABI function.
6811 // With opaque pointer types, we won't have enough information in LLVM
6812 // IR to determine this from the argument types, either. Best to do it
6813 // now, while we have enough information.
6814 Address DestAddr = EmitMSVAListRef(E: E->getArg(Arg: 0));
6815 Address SrcAddr = EmitMSVAListRef(E: E->getArg(Arg: 1));
6816
6817 DestAddr = DestAddr.withElementType(ElemTy: Int8PtrTy);
6818 SrcAddr = SrcAddr.withElementType(ElemTy: Int8PtrTy);
6819
6820 Value *ArgPtr = Builder.CreateLoad(Addr: SrcAddr, Name: "ap.val");
6821 return RValue::get(V: Builder.CreateStore(Val: ArgPtr, Addr: DestAddr));
6822 }
6823
6824 case Builtin::BI__builtin_zos_va_start:
6825 case Builtin::BI__builtin_zos_va_end: {
6826 // The va_list is an array with 2 elements, called curr and next.
6827 // Element curr is set to 0. For builtin_zos_va_start, next is initialized
6828 // with a call to @llvm.va_start. Otherwise, next is passed to @llvm.va_end.
6829 Address VAList = EmitZOSVAListRef(E: E->getArg(Arg: 0));
6830 llvm::Type *VAListTy = ConvertType(T: getContext().getBuiltinZOSVaListType());
6831 VAList = VAList.withElementType(ElemTy: VAListTy);
6832 Address Curr = Builder.CreateConstArrayGEP(Addr: VAList, Index: 0, Name: "curr");
6833 Value *Zero = llvm::Constant::getNullValue(Ty: VoidPtrTy);
6834 Builder.CreateStore(Val: Zero, Addr: Curr);
6835 Address Next = Builder.CreateConstArrayGEP(Addr: VAList, Index: 1, Name: "next");
6836 return RValue::get(
6837 V: EmitVAStartEnd(ArgValue: Next.emitRawPointer(CGF&: *this),
6838 IsStart: BuiltinID == Builtin::BI__builtin_zos_va_start));
6839 }
6840 case Builtin::BI__builtin_zos_va_copy: {
6841 // Lower this manually because later can't reliably determine the type.
6842 Address Dest = EmitZOSVAListRef(E: E->getArg(Arg: 0));
6843 Address Src = EmitZOSVAListRef(E: E->getArg(Arg: 1));
6844 llvm::Type *VAListTy = ConvertType(T: getContext().getBuiltinZOSVaListType());
6845 uint64_t SizeBytes =
6846 CGM.getDataLayout().getTypeAllocSize(Ty: VAListTy).getFixedValue();
6847 Value *SizeVal = llvm::ConstantInt::get(Ty: Int64Ty, V: SizeBytes);
6848 Builder.CreateMemCpy(Dest, Src, Size: SizeVal, IsVolatile: false);
6849 return RValue::get(V: Dest.emitRawPointer(CGF&: *this));
6850 }
6851
6852 case Builtin::BI__builtin_get_device_side_mangled_name: {
6853 auto Name = CGM.getCUDARuntime().getDeviceSideName(
6854 ND: cast<DeclRefExpr>(Val: E->getArg(Arg: 0)->IgnoreImpCasts())->getDecl());
6855 auto Str = CGM.GetAddrOfConstantCString(Str: Name, GlobalName: "");
6856 return RValue::get(V: Str.getPointer());
6857 }
6858 }
6859
6860 // If this is an alias for a lib function (e.g. __builtin_sin), emit
6861 // the call using the normal call path, but using the unmangled
6862 // version of the function name.
6863 const auto &BI = getContext().BuiltinInfo;
6864 if (!shouldEmitBuiltinAsIR(BuiltinID, BI, CGF: *this) &&
6865 BI.isLibFunction(ID: BuiltinID))
6866 return emitLibraryCall(CGF&: *this, FD, E,
6867 calleeValue: CGM.getBuiltinLibFunction(FD, BuiltinID));
6868
6869 // If this is a predefined lib function (e.g. malloc), emit the call
6870 // using exactly the normal call path.
6871 if (BI.isPredefinedLibFunction(ID: BuiltinID))
6872 return emitLibraryCall(CGF&: *this, FD, E, calleeValue: CGM.getRawFunctionPointer(GD: FD));
6873
6874 // Check that a call to a target specific builtin has the correct target
6875 // features.
6876 // This is down here to avoid non-target specific builtins, however, if
6877 // generic builtins start to require generic target features then we
6878 // can move this up to the beginning of the function.
6879 checkTargetFeatures(E, TargetDecl: FD);
6880
6881 if (unsigned VectorWidth = getContext().BuiltinInfo.getRequiredVectorWidth(ID: BuiltinID))
6882 LargestVectorWidth = std::max(a: LargestVectorWidth, b: VectorWidth);
6883
6884 // See if we have a target specific intrinsic.
6885 std::string Name = getContext().BuiltinInfo.getName(ID: BuiltinID);
6886 Intrinsic::ID IntrinsicID = Intrinsic::not_intrinsic;
6887 StringRef Prefix =
6888 llvm::Triple::getArchTypePrefix(Kind: getTarget().getTriple().getArch());
6889 if (!Prefix.empty()) {
6890 IntrinsicID = Intrinsic::getIntrinsicForClangBuiltin(TargetPrefix: Prefix.data(), BuiltinName: Name);
6891 if (IntrinsicID == Intrinsic::not_intrinsic && Prefix == "spv" &&
6892 getTarget().getTriple().getOS() == llvm::Triple::OSType::AMDHSA)
6893 IntrinsicID = Intrinsic::getIntrinsicForClangBuiltin(TargetPrefix: "amdgcn", BuiltinName: Name);
6894 // NOTE we don't need to perform a compatibility flag check here since the
6895 // intrinsics are declared in Builtins*.def via LANGBUILTIN which filter the
6896 // MS builtins via ALL_MS_LANGUAGES and are filtered earlier.
6897 if (IntrinsicID == Intrinsic::not_intrinsic)
6898 IntrinsicID = Intrinsic::getIntrinsicForMSBuiltin(TargetPrefix: Prefix.data(), BuiltinName: Name);
6899 }
6900
6901 if (IntrinsicID != Intrinsic::not_intrinsic) {
6902 SmallVector<Value*, 16> Args;
6903
6904 // Find out if any arguments are required to be integer constant
6905 // expressions.
6906 unsigned ICEArguments = 0;
6907 ASTContext::GetBuiltinTypeError Error;
6908 getContext().GetBuiltinType(ID: BuiltinID, Error, IntegerConstantArgs: &ICEArguments);
6909 assert(Error == ASTContext::GE_None && "Should not codegen an error");
6910
6911 Function *F = CGM.getIntrinsic(IID: IntrinsicID);
6912 llvm::FunctionType *FTy = F->getFunctionType();
6913
6914 for (unsigned i = 0, e = E->getNumArgs(); i != e; ++i) {
6915 Value *ArgValue = EmitScalarOrConstFoldImmArg(ICEArguments, Idx: i, E);
6916 // If the intrinsic arg type is different from the builtin arg type
6917 // we need to do a bit cast.
6918 llvm::Type *PTy = FTy->getParamType(i);
6919 if (PTy != ArgValue->getType()) {
6920 // XXX - vector of pointers?
6921 if (auto *PtrTy = dyn_cast<llvm::PointerType>(Val: PTy)) {
6922 if (PtrTy->getAddressSpace() !=
6923 ArgValue->getType()->getPointerAddressSpace()) {
6924 ArgValue = Builder.CreateAddrSpaceCast(
6925 V: ArgValue, DestTy: llvm::PointerType::get(C&: getLLVMContext(),
6926 AddressSpace: PtrTy->getAddressSpace()));
6927 }
6928 }
6929
6930 // Cast vector type (e.g., v256i32) to x86_amx, this only happen
6931 // in amx intrinsics.
6932 if (PTy->isX86_AMXTy())
6933 ArgValue = Builder.CreateIntrinsic(ID: Intrinsic::x86_cast_vector_to_tile,
6934 OverloadTypes: {ArgValue->getType()}, Args: {ArgValue});
6935 else
6936 ArgValue = Builder.CreateBitCast(V: ArgValue, DestTy: PTy);
6937 }
6938
6939 Args.push_back(Elt: ArgValue);
6940 }
6941
6942 appendDefaultIntrinsicArgs(Args, F);
6943
6944 Value *V = Builder.CreateCall(Callee: F, Args);
6945 QualType BuiltinRetType = E->getType();
6946
6947 llvm::Type *RetTy = VoidTy;
6948 if (!BuiltinRetType->isVoidType())
6949 RetTy = ConvertType(T: BuiltinRetType);
6950
6951 if (RetTy != V->getType()) {
6952 // XXX - vector of pointers?
6953 if (auto *PtrTy = dyn_cast<llvm::PointerType>(Val: RetTy)) {
6954 if (PtrTy->getAddressSpace() != V->getType()->getPointerAddressSpace()) {
6955 V = Builder.CreateAddrSpaceCast(
6956 V, DestTy: llvm::PointerType::get(C&: getLLVMContext(),
6957 AddressSpace: PtrTy->getAddressSpace()));
6958 }
6959 }
6960
6961 // Cast x86_amx to vector type (e.g., v256i32), this only happen
6962 // in amx intrinsics.
6963 if (V->getType()->isX86_AMXTy())
6964 V = Builder.CreateIntrinsic(ID: Intrinsic::x86_cast_tile_to_vector, OverloadTypes: {RetTy},
6965 Args: {V});
6966 else
6967 V = Builder.CreateBitCast(V, DestTy: RetTy);
6968 }
6969
6970 if (RetTy->isVoidTy())
6971 return RValue::get(V: nullptr);
6972
6973 return RValue::get(V);
6974 }
6975
6976 // Some target-specific builtins can have aggregate return values, e.g.
6977 // __builtin_arm_mve_vld2q_u32. So if the result is an aggregate, force
6978 // ReturnValue to be non-null, so that the target-specific emission code can
6979 // always just emit into it.
6980 TypeEvaluationKind EvalKind = getEvaluationKind(T: E->getType());
6981 if (EvalKind == TEK_Aggregate && ReturnValue.isNull()) {
6982 Address DestPtr = CreateMemTemp(T: E->getType(), Name: "agg.tmp");
6983 ReturnValue = ReturnValueSlot(DestPtr, false);
6984 }
6985
6986 // Now see if we can emit a target-specific builtin.
6987 if (Value *V = EmitTargetBuiltinExpr(BuiltinID, E, ReturnValue)) {
6988 switch (EvalKind) {
6989 case TEK_Scalar:
6990 if (V->getType()->isVoidTy())
6991 return RValue::get(V: nullptr);
6992 return RValue::get(V);
6993 case TEK_Aggregate:
6994 return RValue::getAggregate(addr: ReturnValue.getAddress(),
6995 isVolatile: ReturnValue.isVolatile());
6996 case TEK_Complex:
6997 llvm_unreachable("No current target builtin returns complex");
6998 }
6999 llvm_unreachable("Bad evaluation kind in EmitBuiltinExpr");
7000 }
7001
7002 // EmitHLSLBuiltinExpr will check getLangOpts().HLSL
7003 if (Value *V = EmitHLSLBuiltinExpr(BuiltinID, E, ReturnValue)) {
7004 switch (EvalKind) {
7005 case TEK_Scalar:
7006 if (V->getType()->isVoidTy())
7007 return RValue::get(V: nullptr);
7008 return RValue::get(V);
7009 case TEK_Aggregate:
7010 return RValue::getAggregate(addr: ReturnValue.getAddress(),
7011 isVolatile: ReturnValue.isVolatile());
7012 case TEK_Complex:
7013 llvm_unreachable("No current hlsl builtin returns complex");
7014 }
7015 llvm_unreachable("Bad evaluation kind in EmitBuiltinExpr");
7016 }
7017
7018 if (getLangOpts().HIPStdPar && getLangOpts().CUDAIsDevice)
7019 return EmitHipStdParUnsupportedBuiltin(CGF: this, FD);
7020
7021 ErrorUnsupported(S: E, Type: "builtin function");
7022
7023 // Unknown builtin, for now just dump it out and return undef.
7024 return GetUndefRValue(Ty: E->getType());
7025}
7026
7027namespace {
7028struct BuiltinAlignArgs {
7029 llvm::Value *Src = nullptr;
7030 llvm::Type *SrcType = nullptr;
7031 llvm::Value *Alignment = nullptr;
7032 llvm::Value *Mask = nullptr;
7033 llvm::IntegerType *IntType = nullptr;
7034
7035 BuiltinAlignArgs(const CallExpr *E, CodeGenFunction &CGF) {
7036 QualType AstType = E->getArg(Arg: 0)->getType();
7037 if (AstType->isArrayType())
7038 Src = CGF.EmitArrayToPointerDecay(Array: E->getArg(Arg: 0)).emitRawPointer(CGF);
7039 else
7040 Src = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
7041 SrcType = Src->getType();
7042 if (SrcType->isPointerTy()) {
7043 IntType = IntegerType::get(
7044 C&: CGF.getLLVMContext(),
7045 NumBits: CGF.CGM.getDataLayout().getIndexTypeSizeInBits(Ty: SrcType));
7046 } else {
7047 assert(SrcType->isIntegerTy());
7048 IntType = cast<llvm::IntegerType>(Val: SrcType);
7049 }
7050 Alignment = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
7051 Alignment = CGF.Builder.CreateZExtOrTrunc(V: Alignment, DestTy: IntType, Name: "alignment");
7052 auto *One = llvm::ConstantInt::get(Ty: IntType, V: 1);
7053 Mask = CGF.Builder.CreateSub(LHS: Alignment, RHS: One, Name: "mask");
7054 }
7055};
7056} // namespace
7057
7058/// Generate (x & (y-1)) == 0.
7059RValue CodeGenFunction::EmitBuiltinIsAligned(const CallExpr *E) {
7060 BuiltinAlignArgs Args(E, *this);
7061 llvm::Value *SrcAddress = Args.Src;
7062 if (Args.SrcType->isPointerTy())
7063 SrcAddress =
7064 Builder.CreateBitOrPointerCast(V: Args.Src, DestTy: Args.IntType, Name: "src_addr");
7065 return RValue::get(V: Builder.CreateICmpEQ(
7066 LHS: Builder.CreateAnd(LHS: SrcAddress, RHS: Args.Mask, Name: "set_bits"),
7067 RHS: llvm::Constant::getNullValue(Ty: Args.IntType), Name: "is_aligned"));
7068}
7069
7070/// Generate (x & ~(y-1)) to align down or ((x+(y-1)) & ~(y-1)) to align up.
7071/// Note: For pointer types we can avoid ptrtoint/inttoptr pairs by using the
7072/// llvm.ptrmask intrinsic (with a GEP before in the align_up case).
7073RValue CodeGenFunction::EmitBuiltinAlignTo(const CallExpr *E, bool AlignUp) {
7074 BuiltinAlignArgs Args(E, *this);
7075 llvm::Value *SrcForMask = Args.Src;
7076 if (AlignUp) {
7077 // When aligning up we have to first add the mask to ensure we go over the
7078 // next alignment value and then align down to the next valid multiple.
7079 // By adding the mask, we ensure that align_up on an already aligned
7080 // value will not change the value.
7081 if (Args.Src->getType()->isPointerTy()) {
7082 if (getLangOpts().PointerOverflowDefined)
7083 SrcForMask =
7084 Builder.CreateGEP(Ty: Int8Ty, Ptr: SrcForMask, IdxList: Args.Mask, Name: "over_boundary");
7085 else
7086 SrcForMask = EmitCheckedInBoundsGEP(ElemTy: Int8Ty, Ptr: SrcForMask, IdxList: Args.Mask,
7087 /*SignedIndices=*/true,
7088 /*isSubtraction=*/IsSubtraction: false,
7089 Loc: E->getExprLoc(), Name: "over_boundary");
7090 } else {
7091 SrcForMask = Builder.CreateAdd(LHS: SrcForMask, RHS: Args.Mask, Name: "over_boundary");
7092 }
7093 }
7094 // Invert the mask to only clear the lower bits.
7095 llvm::Value *InvertedMask = Builder.CreateNot(V: Args.Mask, Name: "inverted_mask");
7096 llvm::Value *Result = nullptr;
7097 if (Args.Src->getType()->isPointerTy()) {
7098 Result = Builder.CreateIntrinsic(
7099 ID: Intrinsic::ptrmask, OverloadTypes: {Args.SrcType, Args.IntType},
7100 Args: {SrcForMask, InvertedMask}, FMFSource: nullptr, Name: "aligned_result");
7101 } else {
7102 Result = Builder.CreateAnd(LHS: SrcForMask, RHS: InvertedMask, Name: "aligned_result");
7103 }
7104 assert(Result->getType() == Args.SrcType);
7105 return RValue::get(V: Result);
7106}
7107