1//===---- CGBuiltin.cpp - Emit LLVM Code for builtins ---------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit Builtin calls as LLVM code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGBuiltin.h"
14#include "ABIInfo.h"
15#include "CGCUDARuntime.h"
16#include "CGCXXABI.h"
17#include "CGDebugInfo.h"
18#include "CGObjCRuntime.h"
19#include "CGOpenCLRuntime.h"
20#include "CGRecordLayout.h"
21#include "CGValue.h"
22#include "CodeGenFunction.h"
23#include "CodeGenModule.h"
24#include "ConstantEmitter.h"
25#include "PatternInit.h"
26#include "TargetInfo.h"
27#include "clang/AST/OSLog.h"
28#include "clang/AST/StmtVisitor.h"
29#include "clang/Basic/TargetInfo.h"
30#include "clang/Frontend/FrontendDiagnostic.h"
31#include "llvm/IR/InlineAsm.h"
32#include "llvm/IR/Instruction.h"
33#include "llvm/IR/Intrinsics.h"
34#include "llvm/IR/IntrinsicsX86.h"
35#include "llvm/IR/MatrixBuilder.h"
36#include "llvm/Support/ConvertUTF.h"
37#include "llvm/Support/ScopedPrinter.h"
38#include <optional>
39#include <utility>
40
41using namespace clang;
42using namespace CodeGen;
43using namespace llvm;
44
45/// Some builtins do not have library implementation on some targets and
46/// are instead emitted as LLVM IRs by some target builtin emitters.
47/// FIXME: Remove this when library support is added
48static bool shouldEmitBuiltinAsIR(unsigned BuiltinID,
49 const Builtin::Context &BI,
50 const CodeGenFunction &CGF) {
51 if (!CGF.CGM.getLangOpts().MathErrno &&
52 CGF.CurFPFeatures.getExceptionMode() ==
53 LangOptions::FPExceptionModeKind::FPE_Ignore &&
54 !CGF.CGM.getTargetCodeGenInfo().supportsLibCall()) {
55 switch (BuiltinID) {
56 default:
57 return false;
58 case Builtin::BIlogbf:
59 case Builtin::BI__builtin_logbf:
60 case Builtin::BIlogb:
61 case Builtin::BI__builtin_logb:
62 case Builtin::BIscalbnf:
63 case Builtin::BI__builtin_scalbnf:
64 case Builtin::BIscalbn:
65 case Builtin::BI__builtin_scalbn:
66 return true;
67 }
68 }
69 return false;
70}
71
72static Value *EmitTargetArchBuiltinExpr(CodeGenFunction *CGF,
73 unsigned BuiltinID, const CallExpr *E,
74 ReturnValueSlot ReturnValue,
75 llvm::Triple::ArchType Arch) {
76 // When compiling in HipStdPar mode we have to be conservative in rejecting
77 // target specific features in the FE, and defer the possible error to the
78 // AcceleratorCodeSelection pass, wherein iff an unsupported target builtin is
79 // referenced by an accelerator executable function, we emit an error.
80 // Returning nullptr here leads to the builtin being handled in
81 // EmitStdParUnsupportedBuiltin.
82 if (CGF->getLangOpts().HIPStdPar && CGF->getLangOpts().CUDAIsDevice &&
83 Arch != CGF->getTarget().getTriple().getArch())
84 return nullptr;
85
86 switch (Arch) {
87 case llvm::Triple::arm:
88 case llvm::Triple::armeb:
89 case llvm::Triple::thumb:
90 case llvm::Triple::thumbeb:
91 return CGF->EmitARMBuiltinExpr(BuiltinID, E, ReturnValue, Arch);
92 case llvm::Triple::aarch64:
93 case llvm::Triple::aarch64_32:
94 case llvm::Triple::aarch64_be:
95 return CGF->EmitAArch64BuiltinExpr(BuiltinID, E, Arch);
96 case llvm::Triple::bpfeb:
97 case llvm::Triple::bpfel:
98 return CGF->EmitBPFBuiltinExpr(BuiltinID, E);
99 case llvm::Triple::dxil:
100 return CGF->EmitDirectXBuiltinExpr(BuiltinID, E);
101 case llvm::Triple::x86:
102 case llvm::Triple::x86_64:
103 return CGF->EmitX86BuiltinExpr(BuiltinID, E);
104 case llvm::Triple::ppc:
105 case llvm::Triple::ppcle:
106 case llvm::Triple::ppc64:
107 case llvm::Triple::ppc64le:
108 return CGF->EmitPPCBuiltinExpr(BuiltinID, E);
109 case llvm::Triple::r600:
110 case llvm::Triple::amdgcn:
111 return CGF->EmitAMDGPUBuiltinExpr(BuiltinID, E);
112 case llvm::Triple::systemz:
113 return CGF->EmitSystemZBuiltinExpr(BuiltinID, E);
114 case llvm::Triple::nvptx:
115 case llvm::Triple::nvptx64:
116 return CGF->EmitNVPTXBuiltinExpr(BuiltinID, E);
117 case llvm::Triple::wasm32:
118 case llvm::Triple::wasm64:
119 return CGF->EmitWebAssemblyBuiltinExpr(BuiltinID, E);
120 case llvm::Triple::hexagon:
121 return CGF->EmitHexagonBuiltinExpr(BuiltinID, E);
122 case llvm::Triple::riscv32:
123 case llvm::Triple::riscv64:
124 return CGF->EmitRISCVBuiltinExpr(BuiltinID, E, ReturnValue);
125 case llvm::Triple::spirv32:
126 case llvm::Triple::spirv64:
127 if (CGF->getTarget().getTriple().getOS() == llvm::Triple::OSType::AMDHSA)
128 return CGF->EmitAMDGPUBuiltinExpr(BuiltinID, E);
129 [[fallthrough]];
130 case llvm::Triple::spirv:
131 return CGF->EmitSPIRVBuiltinExpr(BuiltinID, E);
132 default:
133 return nullptr;
134 }
135}
136
137Value *CodeGenFunction::EmitTargetBuiltinExpr(unsigned BuiltinID,
138 const CallExpr *E,
139 ReturnValueSlot ReturnValue) {
140 if (getContext().BuiltinInfo.isAuxBuiltinID(ID: BuiltinID)) {
141 assert(getContext().getAuxTargetInfo() && "Missing aux target info");
142 return EmitTargetArchBuiltinExpr(
143 CGF: this, BuiltinID: getContext().BuiltinInfo.getAuxBuiltinID(ID: BuiltinID), E,
144 ReturnValue, Arch: getContext().getAuxTargetInfo()->getTriple().getArch());
145 }
146
147 return EmitTargetArchBuiltinExpr(CGF: this, BuiltinID, E, ReturnValue,
148 Arch: getTarget().getTriple().getArch());
149}
150
151static void initializeAlloca(CodeGenFunction &CGF, AllocaInst *AI, Value *Size,
152 Align AlignmentInBytes) {
153 ConstantInt *Byte;
154 switch (CGF.getLangOpts().getTrivialAutoVarInit()) {
155 case LangOptions::TrivialAutoVarInitKind::Uninitialized:
156 // Nothing to initialize.
157 return;
158 case LangOptions::TrivialAutoVarInitKind::Zero:
159 Byte = CGF.Builder.getInt8(C: 0x00);
160 break;
161 case LangOptions::TrivialAutoVarInitKind::Pattern: {
162 llvm::Type *Int8 = llvm::IntegerType::getInt8Ty(C&: CGF.CGM.getLLVMContext());
163 Byte = llvm::dyn_cast<llvm::ConstantInt>(
164 Val: initializationPatternFor(CGF.CGM, Int8));
165 break;
166 }
167 }
168 if (CGF.CGM.stopAutoInit())
169 return;
170 auto *I = CGF.Builder.CreateMemSet(Ptr: AI, Val: Byte, Size, Align: AlignmentInBytes);
171 I->addAnnotationMetadata(Annotation: "auto-init");
172}
173
174/// getBuiltinLibFunction - Given a builtin id for a function like
175/// "__builtin_fabsf", return a Function* for "fabsf".
176llvm::Constant *CodeGenModule::getBuiltinLibFunction(const FunctionDecl *FD,
177 unsigned BuiltinID) {
178 assert(Context.BuiltinInfo.isLibFunction(BuiltinID));
179
180 // Get the name, skip over the __builtin_ prefix (if necessary). We may have
181 // to build this up so provide a small stack buffer to handle the vast
182 // majority of names.
183 llvm::SmallString<64> Name;
184 GlobalDecl D(FD);
185
186 // TODO: This list should be expanded or refactored after all GCC-compatible
187 // std libcall builtins are implemented.
188 static SmallDenseMap<unsigned, StringRef, 64> F128Builtins{
189 {Builtin::BI__builtin___fprintf_chk, "__fprintf_chkieee128"},
190 {Builtin::BI__builtin___printf_chk, "__printf_chkieee128"},
191 {Builtin::BI__builtin___snprintf_chk, "__snprintf_chkieee128"},
192 {Builtin::BI__builtin___sprintf_chk, "__sprintf_chkieee128"},
193 {Builtin::BI__builtin___vfprintf_chk, "__vfprintf_chkieee128"},
194 {Builtin::BI__builtin___vprintf_chk, "__vprintf_chkieee128"},
195 {Builtin::BI__builtin___vsnprintf_chk, "__vsnprintf_chkieee128"},
196 {Builtin::BI__builtin___vsprintf_chk, "__vsprintf_chkieee128"},
197 {Builtin::BI__builtin_fprintf, "__fprintfieee128"},
198 {Builtin::BI__builtin_printf, "__printfieee128"},
199 {Builtin::BI__builtin_snprintf, "__snprintfieee128"},
200 {Builtin::BI__builtin_sprintf, "__sprintfieee128"},
201 {Builtin::BI__builtin_vfprintf, "__vfprintfieee128"},
202 {Builtin::BI__builtin_vprintf, "__vprintfieee128"},
203 {Builtin::BI__builtin_vsnprintf, "__vsnprintfieee128"},
204 {Builtin::BI__builtin_vsprintf, "__vsprintfieee128"},
205 {Builtin::BI__builtin_fscanf, "__fscanfieee128"},
206 {Builtin::BI__builtin_scanf, "__scanfieee128"},
207 {Builtin::BI__builtin_sscanf, "__sscanfieee128"},
208 {Builtin::BI__builtin_vfscanf, "__vfscanfieee128"},
209 {Builtin::BI__builtin_vscanf, "__vscanfieee128"},
210 {Builtin::BI__builtin_vsscanf, "__vsscanfieee128"},
211 {Builtin::BI__builtin_nexttowardf128, "__nexttowardieee128"},
212 };
213
214 // The AIX library functions frexpl, ldexpl, and modfl are for 128-bit
215 // IBM 'long double' (i.e. __ibm128). Map to the 'double' versions
216 // if it is 64-bit 'long double' mode.
217 static SmallDenseMap<unsigned, StringRef, 4> AIXLongDouble64Builtins{
218 {Builtin::BI__builtin_frexpl, "frexp"},
219 {Builtin::BI__builtin_ldexpl, "ldexp"},
220 {Builtin::BI__builtin_modfl, "modf"},
221 };
222
223 // If the builtin has been declared explicitly with an assembler label,
224 // use the mangled name. This differs from the plain label on platforms
225 // that prefix labels.
226 if (FD->hasAttr<AsmLabelAttr>())
227 Name = getMangledName(GD: D);
228 else {
229 // TODO: This mutation should also be applied to other targets other than
230 // PPC, after backend supports IEEE 128-bit style libcalls.
231 if (getTriple().isPPC64() &&
232 &getTarget().getLongDoubleFormat() == &llvm::APFloat::IEEEquad() &&
233 F128Builtins.contains(Val: BuiltinID))
234 Name = F128Builtins[BuiltinID];
235 else if (getTriple().isOSAIX() &&
236 &getTarget().getLongDoubleFormat() ==
237 &llvm::APFloat::IEEEdouble() &&
238 AIXLongDouble64Builtins.contains(Val: BuiltinID))
239 Name = AIXLongDouble64Builtins[BuiltinID];
240 else
241 Name = Context.BuiltinInfo.getName(ID: BuiltinID).substr(pos: 10);
242 }
243
244 llvm::FunctionType *Ty =
245 cast<llvm::FunctionType>(Val: getTypes().ConvertType(T: FD->getType()));
246
247 return GetOrCreateLLVMFunction(MangledName: Name, Ty, D, /*ForVTable=*/false);
248}
249
250/// Emit the conversions required to turn the given value into an
251/// integer of the given size.
252Value *EmitToInt(CodeGenFunction &CGF, llvm::Value *V,
253 QualType T, llvm::IntegerType *IntType) {
254 V = CGF.EmitToMemory(Value: V, Ty: T);
255
256 if (V->getType()->isPointerTy())
257 return CGF.Builder.CreatePtrToInt(V, DestTy: IntType);
258
259 assert(V->getType() == IntType);
260 return V;
261}
262
263Value *EmitFromInt(CodeGenFunction &CGF, llvm::Value *V,
264 QualType T, llvm::Type *ResultType) {
265 V = CGF.EmitFromMemory(Value: V, Ty: T);
266
267 if (ResultType->isPointerTy())
268 return CGF.Builder.CreateIntToPtr(V, DestTy: ResultType);
269
270 assert(V->getType() == ResultType);
271 return V;
272}
273
274Address CheckAtomicAlignment(CodeGenFunction &CGF, const CallExpr *E) {
275 ASTContext &Ctx = CGF.getContext();
276 Address Ptr = CGF.EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
277 const llvm::DataLayout &DL = CGF.CGM.getDataLayout();
278 unsigned Bytes = Ptr.getElementType()->isPointerTy()
279 ? Ctx.getTypeSizeInChars(T: Ctx.VoidPtrTy).getQuantity()
280 : DL.getTypeStoreSize(Ty: Ptr.getElementType());
281 unsigned Align = Ptr.getAlignment().getQuantity();
282 if (Align % Bytes != 0) {
283 DiagnosticsEngine &Diags = CGF.CGM.getDiags();
284 Diags.Report(Loc: E->getBeginLoc(), DiagID: diag::warn_sync_op_misaligned);
285 // Force address to be at least naturally-aligned.
286 return Ptr.withAlignment(NewAlignment: CharUnits::fromQuantity(Quantity: Bytes));
287 }
288 return Ptr;
289}
290
291/// Utility to insert an atomic instruction based on Intrinsic::ID
292/// and the expression node.
293Value *MakeBinaryAtomicValue(
294 CodeGenFunction &CGF, llvm::AtomicRMWInst::BinOp Kind, const CallExpr *E,
295 AtomicOrdering Ordering) {
296
297 QualType T = E->getType();
298 assert(E->getArg(0)->getType()->isPointerType());
299 assert(CGF.getContext().hasSameUnqualifiedType(T,
300 E->getArg(0)->getType()->getPointeeType()));
301 assert(CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType()));
302
303 Address DestAddr = CheckAtomicAlignment(CGF, E);
304
305 llvm::IntegerType *IntType = llvm::IntegerType::get(
306 C&: CGF.getLLVMContext(), NumBits: CGF.getContext().getTypeSize(T));
307
308 llvm::Value *Val = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
309 llvm::Type *ValueType = Val->getType();
310 Val = EmitToInt(CGF, V: Val, T, IntType);
311
312 llvm::Value *Result =
313 CGF.Builder.CreateAtomicRMW(Op: Kind, Addr: DestAddr, Val, Ordering);
314 return EmitFromInt(CGF, V: Result, T, ResultType: ValueType);
315}
316
317static Value *EmitNontemporalStore(CodeGenFunction &CGF, const CallExpr *E) {
318 Value *Val = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
319 Address Addr = CGF.EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
320
321 Val = CGF.EmitToMemory(Value: Val, Ty: E->getArg(Arg: 0)->getType());
322 LValue LV = CGF.MakeAddrLValue(Addr, T: E->getArg(Arg: 0)->getType());
323 LV.setNontemporal(true);
324 CGF.EmitStoreOfScalar(value: Val, lvalue: LV, isInit: false);
325 return nullptr;
326}
327
328static Value *EmitNontemporalLoad(CodeGenFunction &CGF, const CallExpr *E) {
329 Address Addr = CGF.EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
330
331 LValue LV = CGF.MakeAddrLValue(Addr, T: E->getType());
332 LV.setNontemporal(true);
333 return CGF.EmitLoadOfScalar(lvalue: LV, Loc: E->getExprLoc());
334}
335
336static RValue EmitBinaryAtomic(CodeGenFunction &CGF,
337 llvm::AtomicRMWInst::BinOp Kind,
338 const CallExpr *E) {
339 return RValue::get(V: MakeBinaryAtomicValue(CGF, Kind, E));
340}
341
342/// Utility to insert an atomic instruction based Intrinsic::ID and
343/// the expression node, where the return value is the result of the
344/// operation.
345static RValue EmitBinaryAtomicPost(CodeGenFunction &CGF,
346 llvm::AtomicRMWInst::BinOp Kind,
347 const CallExpr *E,
348 Instruction::BinaryOps Op,
349 bool Invert = false) {
350 QualType T = E->getType();
351 assert(E->getArg(0)->getType()->isPointerType());
352 assert(CGF.getContext().hasSameUnqualifiedType(T,
353 E->getArg(0)->getType()->getPointeeType()));
354 assert(CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType()));
355
356 Address DestAddr = CheckAtomicAlignment(CGF, E);
357
358 llvm::IntegerType *IntType = llvm::IntegerType::get(
359 C&: CGF.getLLVMContext(), NumBits: CGF.getContext().getTypeSize(T));
360
361 llvm::Value *Val = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
362 llvm::Type *ValueType = Val->getType();
363 Val = EmitToInt(CGF, V: Val, T, IntType);
364
365 llvm::Value *Result = CGF.Builder.CreateAtomicRMW(
366 Op: Kind, Addr: DestAddr, Val, Ordering: llvm::AtomicOrdering::SequentiallyConsistent);
367 Result = CGF.Builder.CreateBinOp(Opc: Op, LHS: Result, RHS: Val);
368 if (Invert)
369 Result =
370 CGF.Builder.CreateBinOp(Opc: llvm::Instruction::Xor, LHS: Result,
371 RHS: llvm::ConstantInt::getAllOnesValue(Ty: IntType));
372 Result = EmitFromInt(CGF, V: Result, T, ResultType: ValueType);
373 return RValue::get(V: Result);
374}
375
376/// Utility to insert an atomic cmpxchg instruction.
377///
378/// @param CGF The current codegen function.
379/// @param E Builtin call expression to convert to cmpxchg.
380/// arg0 - address to operate on
381/// arg1 - value to compare with
382/// arg2 - new value
383/// @param ReturnBool Specifies whether to return success flag of
384/// cmpxchg result or the old value.
385///
386/// @returns result of cmpxchg, according to ReturnBool
387///
388/// Note: In order to lower Microsoft's _InterlockedCompareExchange* intrinsics
389/// invoke the function EmitAtomicCmpXchgForMSIntrin.
390Value *MakeAtomicCmpXchgValue(CodeGenFunction &CGF, const CallExpr *E,
391 bool ReturnBool) {
392 QualType T = ReturnBool ? E->getArg(Arg: 1)->getType() : E->getType();
393 Address DestAddr = CheckAtomicAlignment(CGF, E);
394
395 llvm::IntegerType *IntType = llvm::IntegerType::get(
396 C&: CGF.getLLVMContext(), NumBits: CGF.getContext().getTypeSize(T));
397
398 Value *Cmp = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
399 llvm::Type *ValueType = Cmp->getType();
400 Cmp = EmitToInt(CGF, V: Cmp, T, IntType);
401 Value *New = EmitToInt(CGF, V: CGF.EmitScalarExpr(E: E->getArg(Arg: 2)), T, IntType);
402
403 Value *Pair = CGF.Builder.CreateAtomicCmpXchg(
404 Addr: DestAddr, Cmp, New, SuccessOrdering: llvm::AtomicOrdering::SequentiallyConsistent,
405 FailureOrdering: llvm::AtomicOrdering::SequentiallyConsistent);
406 if (ReturnBool)
407 // Extract boolean success flag and zext it to int.
408 return CGF.Builder.CreateZExt(V: CGF.Builder.CreateExtractValue(Agg: Pair, Idxs: 1),
409 DestTy: CGF.ConvertType(T: E->getType()));
410 else
411 // Extract old value and emit it using the same type as compare value.
412 return EmitFromInt(CGF, V: CGF.Builder.CreateExtractValue(Agg: Pair, Idxs: 0), T,
413 ResultType: ValueType);
414}
415
416/// This function should be invoked to emit atomic cmpxchg for Microsoft's
417/// _InterlockedCompareExchange* intrinsics which have the following signature:
418/// T _InterlockedCompareExchange(T volatile *Destination,
419/// T Exchange,
420/// T Comparand);
421///
422/// Whereas the llvm 'cmpxchg' instruction has the following syntax:
423/// cmpxchg *Destination, Comparand, Exchange.
424/// So we need to swap Comparand and Exchange when invoking
425/// CreateAtomicCmpXchg. That is the reason we could not use the above utility
426/// function MakeAtomicCmpXchgValue since it expects the arguments to be
427/// already swapped.
428
429static
430Value *EmitAtomicCmpXchgForMSIntrin(CodeGenFunction &CGF, const CallExpr *E,
431 AtomicOrdering SuccessOrdering = AtomicOrdering::SequentiallyConsistent) {
432 assert(E->getArg(0)->getType()->isPointerType());
433 assert(CGF.getContext().hasSameUnqualifiedType(
434 E->getType(), E->getArg(0)->getType()->getPointeeType()));
435 assert(CGF.getContext().hasSameUnqualifiedType(E->getType(),
436 E->getArg(1)->getType()));
437 assert(CGF.getContext().hasSameUnqualifiedType(E->getType(),
438 E->getArg(2)->getType()));
439
440 Address DestAddr = CheckAtomicAlignment(CGF, E);
441
442 auto *Exchange = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
443 auto *RTy = Exchange->getType();
444
445 auto *Comparand = CGF.EmitScalarExpr(E: E->getArg(Arg: 2));
446
447 if (RTy->isPointerTy()) {
448 Exchange = CGF.Builder.CreatePtrToInt(V: Exchange, DestTy: CGF.IntPtrTy);
449 Comparand = CGF.Builder.CreatePtrToInt(V: Comparand, DestTy: CGF.IntPtrTy);
450 }
451
452 // For Release ordering, the failure ordering should be Monotonic.
453 auto FailureOrdering = SuccessOrdering == AtomicOrdering::Release ?
454 AtomicOrdering::Monotonic :
455 SuccessOrdering;
456
457 // The atomic instruction is marked volatile for consistency with MSVC. This
458 // blocks the few atomics optimizations that LLVM has. If we want to optimize
459 // _Interlocked* operations in the future, we will have to remove the volatile
460 // marker.
461 auto *CmpXchg = CGF.Builder.CreateAtomicCmpXchg(
462 Addr: DestAddr, Cmp: Comparand, New: Exchange, SuccessOrdering, FailureOrdering);
463 CmpXchg->setVolatile(true);
464
465 auto *Result = CGF.Builder.CreateExtractValue(Agg: CmpXchg, Idxs: 0);
466 if (RTy->isPointerTy()) {
467 Result = CGF.Builder.CreateIntToPtr(V: Result, DestTy: RTy);
468 }
469
470 return Result;
471}
472
473// 64-bit Microsoft platforms support 128 bit cmpxchg operations. They are
474// prototyped like this:
475//
476// unsigned char _InterlockedCompareExchange128...(
477// __int64 volatile * _Destination,
478// __int64 _ExchangeHigh,
479// __int64 _ExchangeLow,
480// __int64 * _ComparandResult);
481//
482// Note that Destination is assumed to be at least 16-byte aligned, despite
483// being typed int64.
484
485static Value *EmitAtomicCmpXchg128ForMSIntrin(CodeGenFunction &CGF,
486 const CallExpr *E,
487 AtomicOrdering SuccessOrdering) {
488 assert(E->getNumArgs() == 4);
489 llvm::Value *DestPtr = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
490 llvm::Value *ExchangeHigh = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
491 llvm::Value *ExchangeLow = CGF.EmitScalarExpr(E: E->getArg(Arg: 2));
492 Address ComparandAddr = CGF.EmitPointerWithAlignment(Addr: E->getArg(Arg: 3));
493
494 assert(DestPtr->getType()->isPointerTy());
495 assert(!ExchangeHigh->getType()->isPointerTy());
496 assert(!ExchangeLow->getType()->isPointerTy());
497
498 // For Release ordering, the failure ordering should be Monotonic.
499 auto FailureOrdering = SuccessOrdering == AtomicOrdering::Release
500 ? AtomicOrdering::Monotonic
501 : SuccessOrdering;
502
503 // Convert to i128 pointers and values. Alignment is also overridden for
504 // destination pointer.
505 llvm::Type *Int128Ty = llvm::IntegerType::get(C&: CGF.getLLVMContext(), NumBits: 128);
506 Address DestAddr(DestPtr, Int128Ty,
507 CGF.getContext().toCharUnitsFromBits(BitSize: 128));
508 ComparandAddr = ComparandAddr.withElementType(ElemTy: Int128Ty);
509
510 // (((i128)hi) << 64) | ((i128)lo)
511 ExchangeHigh = CGF.Builder.CreateZExt(V: ExchangeHigh, DestTy: Int128Ty);
512 ExchangeLow = CGF.Builder.CreateZExt(V: ExchangeLow, DestTy: Int128Ty);
513 ExchangeHigh =
514 CGF.Builder.CreateShl(LHS: ExchangeHigh, RHS: llvm::ConstantInt::get(Ty: Int128Ty, V: 64));
515 llvm::Value *Exchange = CGF.Builder.CreateOr(LHS: ExchangeHigh, RHS: ExchangeLow);
516
517 // Load the comparand for the instruction.
518 llvm::Value *Comparand = CGF.Builder.CreateLoad(Addr: ComparandAddr);
519
520 auto *CXI = CGF.Builder.CreateAtomicCmpXchg(Addr: DestAddr, Cmp: Comparand, New: Exchange,
521 SuccessOrdering, FailureOrdering);
522
523 // The atomic instruction is marked volatile for consistency with MSVC. This
524 // blocks the few atomics optimizations that LLVM has. If we want to optimize
525 // _Interlocked* operations in the future, we will have to remove the volatile
526 // marker.
527 CXI->setVolatile(true);
528
529 // Store the result as an outparameter.
530 CGF.Builder.CreateStore(Val: CGF.Builder.CreateExtractValue(Agg: CXI, Idxs: 0),
531 Addr: ComparandAddr);
532
533 // Get the success boolean and zero extend it to i8.
534 Value *Success = CGF.Builder.CreateExtractValue(Agg: CXI, Idxs: 1);
535 return CGF.Builder.CreateZExt(V: Success, DestTy: CGF.Int8Ty);
536}
537
538static Value *EmitAtomicIncrementValue(CodeGenFunction &CGF, const CallExpr *E,
539 AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
540 assert(E->getArg(0)->getType()->isPointerType());
541
542 auto *IntTy = CGF.ConvertType(T: E->getType());
543 Address DestAddr = CheckAtomicAlignment(CGF, E);
544 auto *Result = CGF.Builder.CreateAtomicRMW(
545 Op: AtomicRMWInst::Add, Addr: DestAddr, Val: ConstantInt::get(Ty: IntTy, V: 1), Ordering);
546 return CGF.Builder.CreateAdd(LHS: Result, RHS: ConstantInt::get(Ty: IntTy, V: 1));
547}
548
549static Value *EmitAtomicDecrementValue(
550 CodeGenFunction &CGF, const CallExpr *E,
551 AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
552 assert(E->getArg(0)->getType()->isPointerType());
553
554 auto *IntTy = CGF.ConvertType(T: E->getType());
555 Address DestAddr = CheckAtomicAlignment(CGF, E);
556 auto *Result = CGF.Builder.CreateAtomicRMW(
557 Op: AtomicRMWInst::Sub, Addr: DestAddr, Val: ConstantInt::get(Ty: IntTy, V: 1), Ordering);
558 return CGF.Builder.CreateSub(LHS: Result, RHS: ConstantInt::get(Ty: IntTy, V: 1));
559}
560
561// Build a plain volatile load.
562static Value *EmitISOVolatileLoad(CodeGenFunction &CGF, const CallExpr *E) {
563 Value *Ptr = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
564 QualType ElTy = E->getArg(Arg: 0)->getType()->getPointeeType();
565 CharUnits LoadSize = CGF.getContext().getTypeSizeInChars(T: ElTy);
566 llvm::Type *ITy =
567 llvm::IntegerType::get(C&: CGF.getLLVMContext(), NumBits: LoadSize.getQuantity() * 8);
568 llvm::LoadInst *Load = CGF.Builder.CreateAlignedLoad(Ty: ITy, Addr: Ptr, Align: LoadSize);
569 Load->setVolatile(true);
570 return Load;
571}
572
573// Build a plain volatile store.
574static Value *EmitISOVolatileStore(CodeGenFunction &CGF, const CallExpr *E) {
575 Value *Ptr = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
576 Value *Value = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
577 QualType ElTy = E->getArg(Arg: 0)->getType()->getPointeeType();
578 CharUnits StoreSize = CGF.getContext().getTypeSizeInChars(T: ElTy);
579 llvm::StoreInst *Store =
580 CGF.Builder.CreateAlignedStore(Val: Value, Addr: Ptr, Align: StoreSize);
581 Store->setVolatile(true);
582 return Store;
583}
584
585// Emit a simple mangled intrinsic that has 1 argument and a return type
586// matching the argument type. Depending on mode, this may be a constrained
587// floating-point intrinsic.
588Value *emitUnaryMaybeConstrainedFPBuiltin(CodeGenFunction &CGF,
589 const CallExpr *E, unsigned IntrinsicID,
590 unsigned ConstrainedIntrinsicID) {
591 llvm::Value *Src0 = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
592
593 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
594 if (CGF.Builder.getIsFPConstrained()) {
595 Function *F = CGF.CGM.getIntrinsic(IID: ConstrainedIntrinsicID, Tys: Src0->getType());
596 return CGF.Builder.CreateConstrainedFPCall(Callee: F, Args: { Src0 });
597 } else {
598 Function *F = CGF.CGM.getIntrinsic(IID: IntrinsicID, Tys: Src0->getType());
599 return CGF.Builder.CreateCall(Callee: F, Args: Src0);
600 }
601}
602
603// Emit an intrinsic that has 2 operands of the same type as its result.
604// Depending on mode, this may be a constrained floating-point intrinsic.
605static Value *emitBinaryMaybeConstrainedFPBuiltin(CodeGenFunction &CGF,
606 const CallExpr *E, unsigned IntrinsicID,
607 unsigned ConstrainedIntrinsicID) {
608 llvm::Value *Src0 = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
609 llvm::Value *Src1 = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
610
611 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
612 if (CGF.Builder.getIsFPConstrained()) {
613 Function *F = CGF.CGM.getIntrinsic(IID: ConstrainedIntrinsicID, Tys: Src0->getType());
614 return CGF.Builder.CreateConstrainedFPCall(Callee: F, Args: { Src0, Src1 });
615 } else {
616 Function *F = CGF.CGM.getIntrinsic(IID: IntrinsicID, Tys: Src0->getType());
617 return CGF.Builder.CreateCall(Callee: F, Args: { Src0, Src1 });
618 }
619}
620
621// Has second type mangled argument.
622static Value *
623emitBinaryExpMaybeConstrainedFPBuiltin(CodeGenFunction &CGF, const CallExpr *E,
624 Intrinsic::ID IntrinsicID,
625 Intrinsic::ID ConstrainedIntrinsicID) {
626 llvm::Value *Src0 = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
627 llvm::Value *Src1 = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
628
629 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
630 if (CGF.Builder.getIsFPConstrained()) {
631 Function *F = CGF.CGM.getIntrinsic(IID: ConstrainedIntrinsicID,
632 Tys: {Src0->getType(), Src1->getType()});
633 return CGF.Builder.CreateConstrainedFPCall(Callee: F, Args: {Src0, Src1});
634 }
635
636 Function *F =
637 CGF.CGM.getIntrinsic(IID: IntrinsicID, Tys: {Src0->getType(), Src1->getType()});
638 return CGF.Builder.CreateCall(Callee: F, Args: {Src0, Src1});
639}
640
641// Emit an intrinsic that has 3 operands of the same type as its result.
642// Depending on mode, this may be a constrained floating-point intrinsic.
643static Value *emitTernaryMaybeConstrainedFPBuiltin(CodeGenFunction &CGF,
644 const CallExpr *E, unsigned IntrinsicID,
645 unsigned ConstrainedIntrinsicID) {
646 llvm::Value *Src0 = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
647 llvm::Value *Src1 = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
648 llvm::Value *Src2 = CGF.EmitScalarExpr(E: E->getArg(Arg: 2));
649
650 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
651 if (CGF.Builder.getIsFPConstrained()) {
652 Function *F = CGF.CGM.getIntrinsic(IID: ConstrainedIntrinsicID, Tys: Src0->getType());
653 return CGF.Builder.CreateConstrainedFPCall(Callee: F, Args: { Src0, Src1, Src2 });
654 } else {
655 Function *F = CGF.CGM.getIntrinsic(IID: IntrinsicID, Tys: Src0->getType());
656 return CGF.Builder.CreateCall(Callee: F, Args: { Src0, Src1, Src2 });
657 }
658}
659
660// Emit an intrinsic that has overloaded integer result and fp operand.
661static Value *
662emitMaybeConstrainedFPToIntRoundBuiltin(CodeGenFunction &CGF, const CallExpr *E,
663 unsigned IntrinsicID,
664 unsigned ConstrainedIntrinsicID) {
665 llvm::Type *ResultType = CGF.ConvertType(T: E->getType());
666 llvm::Value *Src0 = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
667
668 if (CGF.Builder.getIsFPConstrained()) {
669 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
670 Function *F = CGF.CGM.getIntrinsic(IID: ConstrainedIntrinsicID,
671 Tys: {ResultType, Src0->getType()});
672 return CGF.Builder.CreateConstrainedFPCall(Callee: F, Args: {Src0});
673 } else {
674 Function *F =
675 CGF.CGM.getIntrinsic(IID: IntrinsicID, Tys: {ResultType, Src0->getType()});
676 return CGF.Builder.CreateCall(Callee: F, Args: Src0);
677 }
678}
679
680static Value *emitFrexpBuiltin(CodeGenFunction &CGF, const CallExpr *E,
681 Intrinsic::ID IntrinsicID) {
682 llvm::Value *Src0 = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
683 llvm::Value *Src1 = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
684
685 QualType IntPtrTy = E->getArg(Arg: 1)->getType()->getPointeeType();
686 llvm::Type *IntTy = CGF.ConvertType(T: IntPtrTy);
687 llvm::Function *F =
688 CGF.CGM.getIntrinsic(IID: IntrinsicID, Tys: {Src0->getType(), IntTy});
689 llvm::Value *Call = CGF.Builder.CreateCall(Callee: F, Args: Src0);
690
691 llvm::Value *Exp = CGF.Builder.CreateExtractValue(Agg: Call, Idxs: 1);
692 LValue LV = CGF.MakeNaturalAlignAddrLValue(V: Src1, T: IntPtrTy);
693 CGF.EmitStoreOfScalar(value: Exp, lvalue: LV);
694
695 return CGF.Builder.CreateExtractValue(Agg: Call, Idxs: 0);
696}
697
698static void emitSincosBuiltin(CodeGenFunction &CGF, const CallExpr *E,
699 Intrinsic::ID IntrinsicID) {
700 llvm::Value *Val = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
701 llvm::Value *Dest0 = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
702 llvm::Value *Dest1 = CGF.EmitScalarExpr(E: E->getArg(Arg: 2));
703
704 llvm::Function *F = CGF.CGM.getIntrinsic(IID: IntrinsicID, Tys: {Val->getType()});
705 llvm::Value *Call = CGF.Builder.CreateCall(Callee: F, Args: Val);
706
707 llvm::Value *SinResult = CGF.Builder.CreateExtractValue(Agg: Call, Idxs: 0);
708 llvm::Value *CosResult = CGF.Builder.CreateExtractValue(Agg: Call, Idxs: 1);
709
710 QualType DestPtrType = E->getArg(Arg: 1)->getType()->getPointeeType();
711 LValue SinLV = CGF.MakeNaturalAlignAddrLValue(V: Dest0, T: DestPtrType);
712 LValue CosLV = CGF.MakeNaturalAlignAddrLValue(V: Dest1, T: DestPtrType);
713
714 llvm::StoreInst *StoreSin =
715 CGF.Builder.CreateStore(Val: SinResult, Addr: SinLV.getAddress());
716 llvm::StoreInst *StoreCos =
717 CGF.Builder.CreateStore(Val: CosResult, Addr: CosLV.getAddress());
718
719 // Mark the two stores as non-aliasing with each other. The order of stores
720 // emitted by this builtin is arbitrary, enforcing a particular order will
721 // prevent optimizations later on.
722 llvm::MDBuilder MDHelper(CGF.getLLVMContext());
723 MDNode *Domain = MDHelper.createAnonymousAliasScopeDomain();
724 MDNode *AliasScope = MDHelper.createAnonymousAliasScope(Domain);
725 MDNode *AliasScopeList = MDNode::get(Context&: Call->getContext(), MDs: AliasScope);
726 StoreSin->setMetadata(KindID: LLVMContext::MD_alias_scope, Node: AliasScopeList);
727 StoreCos->setMetadata(KindID: LLVMContext::MD_noalias, Node: AliasScopeList);
728}
729
730static llvm::Value *emitModfBuiltin(CodeGenFunction &CGF, const CallExpr *E,
731 Intrinsic::ID IntrinsicID) {
732 llvm::Value *Val = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
733 llvm::Value *IntPartDest = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
734
735 llvm::Value *Call =
736 CGF.Builder.CreateIntrinsic(ID: IntrinsicID, Types: {Val->getType()}, Args: Val);
737
738 llvm::Value *FractionalResult = CGF.Builder.CreateExtractValue(Agg: Call, Idxs: 0);
739 llvm::Value *IntegralResult = CGF.Builder.CreateExtractValue(Agg: Call, Idxs: 1);
740
741 QualType DestPtrType = E->getArg(Arg: 1)->getType()->getPointeeType();
742 LValue IntegralLV = CGF.MakeNaturalAlignAddrLValue(V: IntPartDest, T: DestPtrType);
743 CGF.EmitStoreOfScalar(value: IntegralResult, lvalue: IntegralLV);
744
745 return FractionalResult;
746}
747
748/// EmitFAbs - Emit a call to @llvm.fabs().
749static Value *EmitFAbs(CodeGenFunction &CGF, Value *V) {
750 Function *F = CGF.CGM.getIntrinsic(IID: Intrinsic::fabs, Tys: V->getType());
751 llvm::CallInst *Call = CGF.Builder.CreateCall(Callee: F, Args: V);
752 Call->setDoesNotAccessMemory();
753 return Call;
754}
755
756/// Emit the computation of the sign bit for a floating point value. Returns
757/// the i1 sign bit value.
758static Value *EmitSignBit(CodeGenFunction &CGF, Value *V) {
759 LLVMContext &C = CGF.CGM.getLLVMContext();
760
761 llvm::Type *Ty = V->getType();
762 int Width = Ty->getPrimitiveSizeInBits();
763 llvm::Type *IntTy = llvm::IntegerType::get(C, NumBits: Width);
764 V = CGF.Builder.CreateBitCast(V, DestTy: IntTy);
765 if (Ty->isPPC_FP128Ty()) {
766 // We want the sign bit of the higher-order double. The bitcast we just
767 // did works as if the double-double was stored to memory and then
768 // read as an i128. The "store" will put the higher-order double in the
769 // lower address in both little- and big-Endian modes, but the "load"
770 // will treat those bits as a different part of the i128: the low bits in
771 // little-Endian, the high bits in big-Endian. Therefore, on big-Endian
772 // we need to shift the high bits down to the low before truncating.
773 Width >>= 1;
774 if (CGF.getTarget().isBigEndian()) {
775 Value *ShiftCst = llvm::ConstantInt::get(Ty: IntTy, V: Width);
776 V = CGF.Builder.CreateLShr(LHS: V, RHS: ShiftCst);
777 }
778 // We are truncating value in order to extract the higher-order
779 // double, which we will be using to extract the sign from.
780 IntTy = llvm::IntegerType::get(C, NumBits: Width);
781 V = CGF.Builder.CreateTrunc(V, DestTy: IntTy);
782 }
783 Value *Zero = llvm::Constant::getNullValue(Ty: IntTy);
784 return CGF.Builder.CreateICmpSLT(LHS: V, RHS: Zero);
785}
786
787/// Checks no arguments or results are passed indirectly in the ABI (i.e. via a
788/// hidden pointer). This is used to check annotating FP libcalls (that could
789/// set `errno`) with "int" TBAA metadata is safe. If any floating-point
790/// arguments are passed indirectly, setup for the call could be incorrectly
791/// optimized out.
792static bool HasNoIndirectArgumentsOrResults(CGFunctionInfo const &FnInfo) {
793 auto IsIndirect = [&](ABIArgInfo const &info) {
794 return info.isIndirect() || info.isIndirectAliased() || info.isInAlloca();
795 };
796 return !IsIndirect(FnInfo.getReturnInfo()) &&
797 llvm::none_of(Range: FnInfo.arguments(),
798 P: [&](CGFunctionInfoArgInfo const &ArgInfo) {
799 return IsIndirect(ArgInfo.info);
800 });
801}
802
803static RValue emitLibraryCall(CodeGenFunction &CGF, const FunctionDecl *FD,
804 const CallExpr *E, llvm::Constant *calleeValue) {
805 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
806 CGCallee callee = CGCallee::forDirect(functionPtr: calleeValue, abstractInfo: GlobalDecl(FD));
807 llvm::CallBase *callOrInvoke = nullptr;
808 CGFunctionInfo const *FnInfo = nullptr;
809 RValue Call =
810 CGF.EmitCall(FnType: E->getCallee()->getType(), Callee: callee, E, ReturnValue: ReturnValueSlot(),
811 /*Chain=*/nullptr, CallOrInvoke: &callOrInvoke, ResolvedFnInfo: &FnInfo);
812
813 if (unsigned BuiltinID = FD->getBuiltinID()) {
814 // Check whether a FP math builtin function, such as BI__builtin_expf
815 ASTContext &Context = CGF.getContext();
816 bool ConstWithoutErrnoAndExceptions =
817 Context.BuiltinInfo.isConstWithoutErrnoAndExceptions(ID: BuiltinID);
818 // Restrict to target with errno, for example, MacOS doesn't set errno.
819 // TODO: Support builtin function with complex type returned, eg: cacosh
820 if (ConstWithoutErrnoAndExceptions && CGF.CGM.getLangOpts().MathErrno &&
821 !CGF.Builder.getIsFPConstrained() && Call.isScalar() &&
822 HasNoIndirectArgumentsOrResults(FnInfo: *FnInfo)) {
823 // Emit "int" TBAA metadata on FP math libcalls.
824 clang::QualType IntTy = Context.IntTy;
825 TBAAAccessInfo TBAAInfo = CGF.CGM.getTBAAAccessInfo(AccessType: IntTy);
826 CGF.CGM.DecorateInstructionWithTBAA(Inst: callOrInvoke, TBAAInfo);
827 }
828 }
829 return Call;
830}
831
832/// Emit a call to llvm.{sadd,uadd,ssub,usub,smul,umul}.with.overflow.*
833/// depending on IntrinsicID.
834///
835/// \arg CGF The current codegen function.
836/// \arg IntrinsicID The ID for the Intrinsic we wish to generate.
837/// \arg X The first argument to the llvm.*.with.overflow.*.
838/// \arg Y The second argument to the llvm.*.with.overflow.*.
839/// \arg Carry The carry returned by the llvm.*.with.overflow.*.
840/// \returns The result (i.e. sum/product) returned by the intrinsic.
841llvm::Value *EmitOverflowIntrinsic(CodeGenFunction &CGF,
842 const Intrinsic::ID IntrinsicID,
843 llvm::Value *X, llvm::Value *Y,
844 llvm::Value *&Carry) {
845 // Make sure we have integers of the same width.
846 assert(X->getType() == Y->getType() &&
847 "Arguments must be the same type. (Did you forget to make sure both "
848 "arguments have the same integer width?)");
849
850 Function *Callee = CGF.CGM.getIntrinsic(IID: IntrinsicID, Tys: X->getType());
851 llvm::Value *Tmp = CGF.Builder.CreateCall(Callee, Args: {X, Y});
852 Carry = CGF.Builder.CreateExtractValue(Agg: Tmp, Idxs: 1);
853 return CGF.Builder.CreateExtractValue(Agg: Tmp, Idxs: 0);
854}
855
856namespace {
857 struct WidthAndSignedness {
858 unsigned Width;
859 bool Signed;
860 };
861}
862
863static WidthAndSignedness
864getIntegerWidthAndSignedness(const clang::ASTContext &context,
865 const clang::QualType Type) {
866 assert(Type->isIntegerType() && "Given type is not an integer.");
867 unsigned Width = context.getIntWidth(T: Type);
868 bool Signed = Type->isSignedIntegerType();
869 return {.Width: Width, .Signed: Signed};
870}
871
872// Given one or more integer types, this function produces an integer type that
873// encompasses them: any value in one of the given types could be expressed in
874// the encompassing type.
875static struct WidthAndSignedness
876EncompassingIntegerType(ArrayRef<struct WidthAndSignedness> Types) {
877 assert(Types.size() > 0 && "Empty list of types.");
878
879 // If any of the given types is signed, we must return a signed type.
880 bool Signed = false;
881 for (const auto &Type : Types) {
882 Signed |= Type.Signed;
883 }
884
885 // The encompassing type must have a width greater than or equal to the width
886 // of the specified types. Additionally, if the encompassing type is signed,
887 // its width must be strictly greater than the width of any unsigned types
888 // given.
889 unsigned Width = 0;
890 for (const auto &Type : Types) {
891 unsigned MinWidth = Type.Width + (Signed && !Type.Signed);
892 if (Width < MinWidth) {
893 Width = MinWidth;
894 }
895 }
896
897 return {.Width: Width, .Signed: Signed};
898}
899
900Value *CodeGenFunction::EmitVAStartEnd(Value *ArgValue, bool IsStart) {
901 Intrinsic::ID inst = IsStart ? Intrinsic::vastart : Intrinsic::vaend;
902 return Builder.CreateCall(Callee: CGM.getIntrinsic(IID: inst, Tys: {ArgValue->getType()}),
903 Args: ArgValue);
904}
905
906/// Checks if using the result of __builtin_object_size(p, @p From) in place of
907/// __builtin_object_size(p, @p To) is correct
908static bool areBOSTypesCompatible(int From, int To) {
909 // Note: Our __builtin_object_size implementation currently treats Type=0 and
910 // Type=2 identically. Encoding this implementation detail here may make
911 // improving __builtin_object_size difficult in the future, so it's omitted.
912 return From == To || (From == 0 && To == 1) || (From == 3 && To == 2);
913}
914
915static llvm::Value *
916getDefaultBuiltinObjectSizeResult(unsigned Type, llvm::IntegerType *ResType) {
917 return ConstantInt::get(Ty: ResType, V: (Type & 2) ? 0 : -1, /*isSigned=*/IsSigned: true);
918}
919
920llvm::Value *
921CodeGenFunction::evaluateOrEmitBuiltinObjectSize(const Expr *E, unsigned Type,
922 llvm::IntegerType *ResType,
923 llvm::Value *EmittedE,
924 bool IsDynamic) {
925 uint64_t ObjectSize;
926 if (!E->tryEvaluateObjectSize(Result&: ObjectSize, Ctx&: getContext(), Type))
927 return emitBuiltinObjectSize(E, Type, ResType, EmittedE, IsDynamic);
928 return ConstantInt::get(Ty: ResType, V: ObjectSize, /*isSigned=*/IsSigned: true);
929}
930
931namespace {
932
933/// StructFieldAccess is a simple visitor class to grab the first MemberExpr
934/// from an Expr. It records any ArraySubscriptExpr we meet along the way.
935class StructFieldAccess
936 : public ConstStmtVisitor<StructFieldAccess, const Expr *> {
937 bool AddrOfSeen = false;
938
939public:
940 const Expr *ArrayIndex = nullptr;
941 QualType ArrayElementTy;
942
943 const Expr *VisitMemberExpr(const MemberExpr *E) {
944 if (AddrOfSeen && E->getType()->isArrayType())
945 // Avoid forms like '&ptr->array'.
946 return nullptr;
947 return E;
948 }
949
950 const Expr *VisitArraySubscriptExpr(const ArraySubscriptExpr *E) {
951 if (ArrayIndex)
952 // We don't support multiple subscripts.
953 return nullptr;
954
955 AddrOfSeen = false; // '&ptr->array[idx]' is okay.
956 ArrayIndex = E->getIdx();
957 ArrayElementTy = E->getBase()->getType();
958 return Visit(S: E->getBase());
959 }
960 const Expr *VisitCastExpr(const CastExpr *E) {
961 if (E->getCastKind() == CK_LValueToRValue)
962 return E;
963 return Visit(S: E->getSubExpr());
964 }
965 const Expr *VisitParenExpr(const ParenExpr *E) {
966 return Visit(S: E->getSubExpr());
967 }
968 const Expr *VisitUnaryAddrOf(const clang::UnaryOperator *E) {
969 AddrOfSeen = true;
970 return Visit(S: E->getSubExpr());
971 }
972 const Expr *VisitUnaryDeref(const clang::UnaryOperator *E) {
973 AddrOfSeen = false;
974 return Visit(S: E->getSubExpr());
975 }
976};
977
978} // end anonymous namespace
979
980/// Find a struct's flexible array member. It may be embedded inside multiple
981/// sub-structs, but must still be the last field.
982static const FieldDecl *FindFlexibleArrayMemberField(CodeGenFunction &CGF,
983 ASTContext &Ctx,
984 const RecordDecl *RD) {
985 const LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel =
986 CGF.getLangOpts().getStrictFlexArraysLevel();
987
988 if (RD->isImplicit())
989 return nullptr;
990
991 for (const FieldDecl *FD : RD->fields()) {
992 if (Decl::isFlexibleArrayMemberLike(
993 Context: Ctx, D: FD, Ty: FD->getType(), StrictFlexArraysLevel,
994 /*IgnoreTemplateOrMacroSubstitution=*/true))
995 return FD;
996
997 if (auto RT = FD->getType()->getAs<RecordType>())
998 if (const FieldDecl *FD =
999 FindFlexibleArrayMemberField(CGF, Ctx, RD: RT->getAsRecordDecl()))
1000 return FD;
1001 }
1002
1003 return nullptr;
1004}
1005
1006/// Calculate the offset of a struct field. It may be embedded inside multiple
1007/// sub-structs.
1008static bool GetFieldOffset(ASTContext &Ctx, const RecordDecl *RD,
1009 const FieldDecl *FD, int64_t &Offset) {
1010 if (RD->isImplicit())
1011 return false;
1012
1013 // Keep track of the field number ourselves, because the other methods
1014 // (CGRecordLayout::getLLVMFieldNo) aren't always equivalent to how the AST
1015 // is laid out.
1016 uint32_t FieldNo = 0;
1017 const ASTRecordLayout &Layout = Ctx.getASTRecordLayout(D: RD);
1018
1019 for (const FieldDecl *Field : RD->fields()) {
1020 if (Field == FD) {
1021 Offset += Layout.getFieldOffset(FieldNo);
1022 return true;
1023 }
1024
1025 if (auto RT = Field->getType()->getAs<RecordType>()) {
1026 if (GetFieldOffset(Ctx, RD: RT->getAsRecordDecl(), FD, Offset)) {
1027 Offset += Layout.getFieldOffset(FieldNo);
1028 return true;
1029 }
1030 }
1031
1032 if (!RD->isUnion())
1033 ++FieldNo;
1034 }
1035
1036 return false;
1037}
1038
1039static std::optional<int64_t>
1040GetFieldOffset(ASTContext &Ctx, const RecordDecl *RD, const FieldDecl *FD) {
1041 int64_t Offset = 0;
1042
1043 if (GetFieldOffset(Ctx, RD, FD, Offset))
1044 return std::optional<int64_t>(Offset);
1045
1046 return std::nullopt;
1047}
1048
1049llvm::Value *CodeGenFunction::emitCountedBySize(const Expr *E,
1050 llvm::Value *EmittedE,
1051 unsigned Type,
1052 llvm::IntegerType *ResType) {
1053 // Note: If the whole struct is specificed in the __bdos (i.e. Visitor
1054 // returns a DeclRefExpr). The calculation of the whole size of the structure
1055 // with a flexible array member can be done in two ways:
1056 //
1057 // 1) sizeof(struct S) + count * sizeof(typeof(fam))
1058 // 2) offsetof(struct S, fam) + count * sizeof(typeof(fam))
1059 //
1060 // The first will add additional padding after the end of the array
1061 // allocation while the second method is more precise, but not quite expected
1062 // from programmers. See
1063 // https://lore.kernel.org/lkml/ZvV6X5FPBBW7CO1f@archlinux/ for a discussion
1064 // of the topic.
1065 //
1066 // GCC isn't (currently) able to calculate __bdos on a pointer to the whole
1067 // structure. Therefore, because of the above issue, we choose to match what
1068 // GCC does for consistency's sake.
1069
1070 StructFieldAccess Visitor;
1071 E = Visitor.Visit(S: E);
1072 if (!E)
1073 return nullptr;
1074
1075 const Expr *Idx = Visitor.ArrayIndex;
1076 if (Idx) {
1077 if (Idx->HasSideEffects(Ctx: getContext()))
1078 // We can't have side-effects.
1079 return getDefaultBuiltinObjectSizeResult(Type, ResType);
1080
1081 if (const auto *IL = dyn_cast<IntegerLiteral>(Val: Idx)) {
1082 int64_t Val = IL->getValue().getSExtValue();
1083 if (Val < 0)
1084 return getDefaultBuiltinObjectSizeResult(Type, ResType);
1085
1086 // The index is 0, so we don't need to take it into account.
1087 if (Val == 0)
1088 Idx = nullptr;
1089 }
1090 }
1091
1092 // __counted_by on either a flexible array member or a pointer into a struct
1093 // with a flexible array member.
1094 if (const auto *ME = dyn_cast<MemberExpr>(Val: E))
1095 return emitCountedByMemberSize(E: ME, Idx, EmittedE, CastedArrayElementTy: Visitor.ArrayElementTy,
1096 Type, ResType);
1097
1098 // __counted_by on a pointer in a struct.
1099 if (const auto *ICE = dyn_cast<ImplicitCastExpr>(Val: E);
1100 ICE && ICE->getCastKind() == CK_LValueToRValue)
1101 return emitCountedByPointerSize(E: ICE, Idx, EmittedE, CastedArrayElementTy: Visitor.ArrayElementTy,
1102 Type, ResType);
1103
1104 return nullptr;
1105}
1106
1107static llvm::Value *EmitPositiveResultOrZero(CodeGenFunction &CGF,
1108 llvm::Value *Res,
1109 llvm::Value *Index,
1110 llvm::IntegerType *ResType,
1111 bool IsSigned) {
1112 // cmp = (array_size >= 0)
1113 Value *Cmp = CGF.Builder.CreateIsNotNeg(Arg: Res);
1114 if (Index)
1115 // cmp = (cmp && index >= 0)
1116 Cmp = CGF.Builder.CreateAnd(LHS: CGF.Builder.CreateIsNotNeg(Arg: Index), RHS: Cmp);
1117
1118 // return cmp ? result : 0
1119 return CGF.Builder.CreateSelect(C: Cmp, True: Res,
1120 False: ConstantInt::get(Ty: ResType, V: 0, IsSigned));
1121}
1122
1123static std::pair<llvm::Value *, llvm::Value *>
1124GetCountFieldAndIndex(CodeGenFunction &CGF, const MemberExpr *ME,
1125 const FieldDecl *ArrayFD, const FieldDecl *CountFD,
1126 const Expr *Idx, llvm::IntegerType *ResType,
1127 bool IsSigned) {
1128 // count = ptr->count;
1129 Value *Count = CGF.EmitLoadOfCountedByField(Base: ME, FD: ArrayFD, CountDecl: CountFD);
1130 if (!Count)
1131 return std::make_pair<Value *>(x: nullptr, y: nullptr);
1132 Count = CGF.Builder.CreateIntCast(V: Count, DestTy: ResType, isSigned: IsSigned, Name: "count");
1133
1134 // index = ptr->index;
1135 Value *Index = nullptr;
1136 if (Idx) {
1137 bool IdxSigned = Idx->getType()->isSignedIntegerType();
1138 Index = CGF.EmitScalarExpr(E: Idx);
1139 Index = CGF.Builder.CreateIntCast(V: Index, DestTy: ResType, isSigned: IdxSigned, Name: "index");
1140 }
1141
1142 return std::make_pair(x&: Count, y&: Index);
1143}
1144
1145llvm::Value *CodeGenFunction::emitCountedByPointerSize(
1146 const ImplicitCastExpr *E, const Expr *Idx, llvm::Value *EmittedE,
1147 QualType CastedArrayElementTy, unsigned Type, llvm::IntegerType *ResType) {
1148 assert(E->getCastKind() == CK_LValueToRValue &&
1149 "must be an LValue to RValue cast");
1150
1151 const MemberExpr *ME = dyn_cast<MemberExpr>(Val: E->getSubExpr());
1152 if (!ME)
1153 return nullptr;
1154
1155 const auto *ArrayBaseFD = dyn_cast<FieldDecl>(Val: ME->getMemberDecl());
1156 if (!ArrayBaseFD || !ArrayBaseFD->getType()->isPointerType() ||
1157 !ArrayBaseFD->getType()->isCountAttributedType())
1158 return nullptr;
1159
1160 // Get the 'count' FieldDecl.
1161 const FieldDecl *CountFD = ArrayBaseFD->findCountedByField();
1162 if (!CountFD)
1163 // Can't find the field referenced by the "counted_by" attribute.
1164 return nullptr;
1165
1166 // Calculate the array's object size using these formulae. (Note: if the
1167 // calculation is negative, we return 0.):
1168 //
1169 // struct p;
1170 // struct s {
1171 // /* ... */
1172 // struct p **array __attribute__((counted_by(count)));
1173 // int count;
1174 // };
1175 //
1176 // 1) 'ptr->array':
1177 //
1178 // count = ptr->count;
1179 //
1180 // array_element_size = sizeof (*ptr->array);
1181 // array_size = count * array_element_size;
1182 //
1183 // result = array_size;
1184 //
1185 // cmp = (result >= 0)
1186 // return cmp ? result : 0;
1187 //
1188 // 2) '&((cast) ptr->array)[idx]':
1189 //
1190 // count = ptr->count;
1191 // index = idx;
1192 //
1193 // array_element_size = sizeof (*ptr->array);
1194 // array_size = count * array_element_size;
1195 //
1196 // casted_array_element_size = sizeof (*((cast) ptr->array));
1197 //
1198 // index_size = index * casted_array_element_size;
1199 // result = array_size - index_size;
1200 //
1201 // cmp = (result >= 0)
1202 // if (index)
1203 // cmp = (cmp && index > 0)
1204 // return cmp ? result : 0;
1205
1206 auto GetElementBaseSize = [&](QualType ElementTy) {
1207 CharUnits ElementSize =
1208 getContext().getTypeSizeInChars(T: ElementTy->getPointeeType());
1209
1210 if (ElementSize.isZero()) {
1211 // This might be a __sized_by on a 'void *', which counts bytes, not
1212 // elements.
1213 auto *CAT = ElementTy->getAs<CountAttributedType>();
1214 if (!CAT || (CAT->getKind() != CountAttributedType::SizedBy &&
1215 CAT->getKind() != CountAttributedType::SizedByOrNull))
1216 // Okay, not sure what it is now.
1217 // FIXME: Should this be an assert?
1218 return std::optional<CharUnits>();
1219
1220 ElementSize = CharUnits::One();
1221 }
1222
1223 return std::optional<CharUnits>(ElementSize);
1224 };
1225
1226 // Get the sizes of the original array element and the casted array element,
1227 // if different.
1228 std::optional<CharUnits> ArrayElementBaseSize =
1229 GetElementBaseSize(ArrayBaseFD->getType());
1230 if (!ArrayElementBaseSize)
1231 return nullptr;
1232
1233 std::optional<CharUnits> CastedArrayElementBaseSize = ArrayElementBaseSize;
1234 if (!CastedArrayElementTy.isNull() && CastedArrayElementTy->isPointerType()) {
1235 CastedArrayElementBaseSize = GetElementBaseSize(CastedArrayElementTy);
1236 if (!CastedArrayElementBaseSize)
1237 return nullptr;
1238 }
1239
1240 bool IsSigned = CountFD->getType()->isSignedIntegerType();
1241
1242 // count = ptr->count;
1243 // index = ptr->index;
1244 Value *Count, *Index;
1245 std::tie(args&: Count, args&: Index) = GetCountFieldAndIndex(
1246 CGF&: *this, ME, ArrayFD: ArrayBaseFD, CountFD, Idx, ResType, IsSigned);
1247 if (!Count)
1248 return nullptr;
1249
1250 // array_element_size = sizeof (*ptr->array)
1251 auto *ArrayElementSize = llvm::ConstantInt::get(
1252 Ty: ResType, V: ArrayElementBaseSize->getQuantity(), IsSigned);
1253
1254 // casted_array_element_size = sizeof (*((cast) ptr->array));
1255 auto *CastedArrayElementSize = llvm::ConstantInt::get(
1256 Ty: ResType, V: CastedArrayElementBaseSize->getQuantity(), IsSigned);
1257
1258 // array_size = count * array_element_size;
1259 Value *ArraySize = Builder.CreateMul(LHS: Count, RHS: ArrayElementSize, Name: "array_size",
1260 HasNUW: !IsSigned, HasNSW: IsSigned);
1261
1262 // Option (1) 'ptr->array'
1263 // result = array_size
1264 Value *Result = ArraySize;
1265
1266 if (Idx) { // Option (2) '&((cast) ptr->array)[idx]'
1267 // index_size = index * casted_array_element_size;
1268 Value *IndexSize = Builder.CreateMul(LHS: Index, RHS: CastedArrayElementSize,
1269 Name: "index_size", HasNUW: !IsSigned, HasNSW: IsSigned);
1270
1271 // result = result - index_size;
1272 Result =
1273 Builder.CreateSub(LHS: Result, RHS: IndexSize, Name: "result", HasNUW: !IsSigned, HasNSW: IsSigned);
1274 }
1275
1276 return EmitPositiveResultOrZero(CGF&: *this, Res: Result, Index, ResType, IsSigned);
1277}
1278
1279llvm::Value *CodeGenFunction::emitCountedByMemberSize(
1280 const MemberExpr *ME, const Expr *Idx, llvm::Value *EmittedE,
1281 QualType CastedArrayElementTy, unsigned Type, llvm::IntegerType *ResType) {
1282 const auto *FD = dyn_cast<FieldDecl>(Val: ME->getMemberDecl());
1283 if (!FD)
1284 return nullptr;
1285
1286 // Find the flexible array member and check that it has the __counted_by
1287 // attribute.
1288 ASTContext &Ctx = getContext();
1289 const RecordDecl *RD = FD->getDeclContext()->getOuterLexicalRecordContext();
1290 const FieldDecl *FlexibleArrayMemberFD = nullptr;
1291
1292 if (Decl::isFlexibleArrayMemberLike(
1293 Context: Ctx, D: FD, Ty: FD->getType(), StrictFlexArraysLevel: getLangOpts().getStrictFlexArraysLevel(),
1294 /*IgnoreTemplateOrMacroSubstitution=*/true))
1295 FlexibleArrayMemberFD = FD;
1296 else
1297 FlexibleArrayMemberFD = FindFlexibleArrayMemberField(CGF&: *this, Ctx, RD);
1298
1299 if (!FlexibleArrayMemberFD ||
1300 !FlexibleArrayMemberFD->getType()->isCountAttributedType())
1301 return nullptr;
1302
1303 // Get the 'count' FieldDecl.
1304 const FieldDecl *CountFD = FlexibleArrayMemberFD->findCountedByField();
1305 if (!CountFD)
1306 // Can't find the field referenced by the "counted_by" attribute.
1307 return nullptr;
1308
1309 // Calculate the flexible array member's object size using these formulae.
1310 // (Note: if the calculation is negative, we return 0.):
1311 //
1312 // struct p;
1313 // struct s {
1314 // /* ... */
1315 // int count;
1316 // struct p *array[] __attribute__((counted_by(count)));
1317 // };
1318 //
1319 // 1) 'ptr->array':
1320 //
1321 // count = ptr->count;
1322 //
1323 // flexible_array_member_element_size = sizeof (*ptr->array);
1324 // flexible_array_member_size =
1325 // count * flexible_array_member_element_size;
1326 //
1327 // result = flexible_array_member_size;
1328 //
1329 // cmp = (result >= 0)
1330 // return cmp ? result : 0;
1331 //
1332 // 2) '&((cast) ptr->array)[idx]':
1333 //
1334 // count = ptr->count;
1335 // index = idx;
1336 //
1337 // flexible_array_member_element_size = sizeof (*ptr->array);
1338 // flexible_array_member_size =
1339 // count * flexible_array_member_element_size;
1340 //
1341 // casted_flexible_array_member_element_size =
1342 // sizeof (*((cast) ptr->array));
1343 // index_size = index * casted_flexible_array_member_element_size;
1344 //
1345 // result = flexible_array_member_size - index_size;
1346 //
1347 // cmp = (result >= 0)
1348 // if (index != 0)
1349 // cmp = (cmp && index >= 0)
1350 // return cmp ? result : 0;
1351 //
1352 // 3) '&ptr->field':
1353 //
1354 // count = ptr->count;
1355 // sizeof_struct = sizeof (struct s);
1356 //
1357 // flexible_array_member_element_size = sizeof (*ptr->array);
1358 // flexible_array_member_size =
1359 // count * flexible_array_member_element_size;
1360 //
1361 // field_offset = offsetof (struct s, field);
1362 // offset_diff = sizeof_struct - field_offset;
1363 //
1364 // result = offset_diff + flexible_array_member_size;
1365 //
1366 // cmp = (result >= 0)
1367 // return cmp ? result : 0;
1368 //
1369 // 4) '&((cast) ptr->field_array)[idx]':
1370 //
1371 // count = ptr->count;
1372 // index = idx;
1373 // sizeof_struct = sizeof (struct s);
1374 //
1375 // flexible_array_member_element_size = sizeof (*ptr->array);
1376 // flexible_array_member_size =
1377 // count * flexible_array_member_element_size;
1378 //
1379 // casted_field_element_size = sizeof (*((cast) ptr->field_array));
1380 // field_offset = offsetof (struct s, field)
1381 // field_offset += index * casted_field_element_size;
1382 //
1383 // offset_diff = sizeof_struct - field_offset;
1384 //
1385 // result = offset_diff + flexible_array_member_size;
1386 //
1387 // cmp = (result >= 0)
1388 // if (index != 0)
1389 // cmp = (cmp && index >= 0)
1390 // return cmp ? result : 0;
1391
1392 bool IsSigned = CountFD->getType()->isSignedIntegerType();
1393
1394 QualType FlexibleArrayMemberTy = FlexibleArrayMemberFD->getType();
1395
1396 // Explicit cast because otherwise the CharWidth will promote an i32's into
1397 // u64's leading to overflows.
1398 int64_t CharWidth = static_cast<int64_t>(CGM.getContext().getCharWidth());
1399
1400 // field_offset = offsetof (struct s, field);
1401 Value *FieldOffset = nullptr;
1402 if (FlexibleArrayMemberFD != FD) {
1403 std::optional<int64_t> Offset = GetFieldOffset(Ctx, RD, FD);
1404 if (!Offset)
1405 return nullptr;
1406 FieldOffset =
1407 llvm::ConstantInt::get(Ty: ResType, V: *Offset / CharWidth, IsSigned);
1408 }
1409
1410 // count = ptr->count;
1411 // index = ptr->index;
1412 Value *Count, *Index;
1413 std::tie(args&: Count, args&: Index) = GetCountFieldAndIndex(
1414 CGF&: *this, ME, ArrayFD: FlexibleArrayMemberFD, CountFD, Idx, ResType, IsSigned);
1415 if (!Count)
1416 return nullptr;
1417
1418 // flexible_array_member_element_size = sizeof (*ptr->array);
1419 const ArrayType *ArrayTy = Ctx.getAsArrayType(T: FlexibleArrayMemberTy);
1420 CharUnits BaseSize = Ctx.getTypeSizeInChars(T: ArrayTy->getElementType());
1421 auto *FlexibleArrayMemberElementSize =
1422 llvm::ConstantInt::get(Ty: ResType, V: BaseSize.getQuantity(), IsSigned);
1423
1424 // flexible_array_member_size = count * flexible_array_member_element_size;
1425 Value *FlexibleArrayMemberSize =
1426 Builder.CreateMul(LHS: Count, RHS: FlexibleArrayMemberElementSize,
1427 Name: "flexible_array_member_size", HasNUW: !IsSigned, HasNSW: IsSigned);
1428
1429 Value *Result = nullptr;
1430 if (FlexibleArrayMemberFD == FD) {
1431 if (Idx) { // Option (2) '&((cast) ptr->array)[idx]'
1432 // casted_flexible_array_member_element_size =
1433 // sizeof (*((cast) ptr->array));
1434 llvm::ConstantInt *CastedFlexibleArrayMemberElementSize =
1435 FlexibleArrayMemberElementSize;
1436 if (!CastedArrayElementTy.isNull() &&
1437 CastedArrayElementTy->isPointerType()) {
1438 CharUnits BaseSize =
1439 Ctx.getTypeSizeInChars(T: CastedArrayElementTy->getPointeeType());
1440 CastedFlexibleArrayMemberElementSize =
1441 llvm::ConstantInt::get(Ty: ResType, V: BaseSize.getQuantity(), IsSigned);
1442 }
1443
1444 // index_size = index * casted_flexible_array_member_element_size;
1445 Value *IndexSize =
1446 Builder.CreateMul(LHS: Index, RHS: CastedFlexibleArrayMemberElementSize,
1447 Name: "index_size", HasNUW: !IsSigned, HasNSW: IsSigned);
1448
1449 // result = flexible_array_member_size - index_size;
1450 Result = Builder.CreateSub(LHS: FlexibleArrayMemberSize, RHS: IndexSize, Name: "result",
1451 HasNUW: !IsSigned, HasNSW: IsSigned);
1452 } else { // Option (1) 'ptr->array'
1453 // result = flexible_array_member_size;
1454 Result = FlexibleArrayMemberSize;
1455 }
1456 } else {
1457 // sizeof_struct = sizeof (struct s);
1458 llvm::StructType *StructTy = getTypes().getCGRecordLayout(RD).getLLVMType();
1459 const llvm::DataLayout &Layout = CGM.getDataLayout();
1460 TypeSize Size = Layout.getTypeSizeInBits(Ty: StructTy);
1461 Value *SizeofStruct =
1462 llvm::ConstantInt::get(Ty: ResType, V: Size.getKnownMinValue() / CharWidth);
1463
1464 if (Idx) { // Option (4) '&((cast) ptr->field_array)[idx]'
1465 // casted_field_element_size = sizeof (*((cast) ptr->field_array));
1466 CharUnits BaseSize;
1467 if (!CastedArrayElementTy.isNull() &&
1468 CastedArrayElementTy->isPointerType()) {
1469 BaseSize =
1470 Ctx.getTypeSizeInChars(T: CastedArrayElementTy->getPointeeType());
1471 } else {
1472 const ArrayType *ArrayTy = Ctx.getAsArrayType(T: FD->getType());
1473 BaseSize = Ctx.getTypeSizeInChars(T: ArrayTy->getElementType());
1474 }
1475
1476 llvm::ConstantInt *CastedFieldElementSize =
1477 llvm::ConstantInt::get(Ty: ResType, V: BaseSize.getQuantity(), IsSigned);
1478
1479 // field_offset += index * casted_field_element_size;
1480 Value *Mul = Builder.CreateMul(LHS: Index, RHS: CastedFieldElementSize,
1481 Name: "field_offset", HasNUW: !IsSigned, HasNSW: IsSigned);
1482 FieldOffset = Builder.CreateAdd(LHS: FieldOffset, RHS: Mul);
1483 }
1484 // Option (3) '&ptr->field', and Option (4) continuation.
1485 // offset_diff = flexible_array_member_offset - field_offset;
1486 Value *OffsetDiff = Builder.CreateSub(LHS: SizeofStruct, RHS: FieldOffset,
1487 Name: "offset_diff", HasNUW: !IsSigned, HasNSW: IsSigned);
1488
1489 // result = offset_diff + flexible_array_member_size;
1490 Result = Builder.CreateAdd(LHS: FlexibleArrayMemberSize, RHS: OffsetDiff, Name: "result");
1491 }
1492
1493 return EmitPositiveResultOrZero(CGF&: *this, Res: Result, Index, ResType, IsSigned);
1494}
1495
1496/// Returns a Value corresponding to the size of the given expression.
1497/// This Value may be either of the following:
1498/// - A llvm::Argument (if E is a param with the pass_object_size attribute on
1499/// it)
1500/// - A call to the @llvm.objectsize intrinsic
1501///
1502/// EmittedE is the result of emitting `E` as a scalar expr. If it's non-null
1503/// and we wouldn't otherwise try to reference a pass_object_size parameter,
1504/// we'll call @llvm.objectsize on EmittedE, rather than emitting E.
1505llvm::Value *
1506CodeGenFunction::emitBuiltinObjectSize(const Expr *E, unsigned Type,
1507 llvm::IntegerType *ResType,
1508 llvm::Value *EmittedE, bool IsDynamic) {
1509 // We need to reference an argument if the pointer is a parameter with the
1510 // pass_object_size attribute.
1511 if (auto *D = dyn_cast<DeclRefExpr>(Val: E->IgnoreParenImpCasts())) {
1512 auto *Param = dyn_cast<ParmVarDecl>(Val: D->getDecl());
1513 auto *PS = D->getDecl()->getAttr<PassObjectSizeAttr>();
1514 if (Param != nullptr && PS != nullptr &&
1515 areBOSTypesCompatible(From: PS->getType(), To: Type)) {
1516 auto Iter = SizeArguments.find(Val: Param);
1517 assert(Iter != SizeArguments.end());
1518
1519 const ImplicitParamDecl *D = Iter->second;
1520 auto DIter = LocalDeclMap.find(Val: D);
1521 assert(DIter != LocalDeclMap.end());
1522
1523 return EmitLoadOfScalar(Addr: DIter->second, /*Volatile=*/false,
1524 Ty: getContext().getSizeType(), Loc: E->getBeginLoc());
1525 }
1526 }
1527
1528 // LLVM can't handle Type=3 appropriately, and __builtin_object_size shouldn't
1529 // evaluate E for side-effects. In either case, we shouldn't lower to
1530 // @llvm.objectsize.
1531 if (Type == 3 || (!EmittedE && E->HasSideEffects(Ctx: getContext())))
1532 return getDefaultBuiltinObjectSizeResult(Type, ResType);
1533
1534 Value *Ptr = EmittedE ? EmittedE : EmitScalarExpr(E);
1535 assert(Ptr->getType()->isPointerTy() &&
1536 "Non-pointer passed to __builtin_object_size?");
1537
1538 if (IsDynamic)
1539 // Emit special code for a flexible array member with the "counted_by"
1540 // attribute.
1541 if (Value *V = emitCountedBySize(E, EmittedE: Ptr, Type, ResType))
1542 return V;
1543
1544 Function *F =
1545 CGM.getIntrinsic(IID: Intrinsic::objectsize, Tys: {ResType, Ptr->getType()});
1546
1547 // LLVM only supports 0 and 2, make sure that we pass along that as a boolean.
1548 Value *Min = Builder.getInt1(V: (Type & 2) != 0);
1549 // For GCC compatibility, __builtin_object_size treat NULL as unknown size.
1550 Value *NullIsUnknown = Builder.getTrue();
1551 Value *Dynamic = Builder.getInt1(V: IsDynamic);
1552 return Builder.CreateCall(Callee: F, Args: {Ptr, Min, NullIsUnknown, Dynamic});
1553}
1554
1555namespace {
1556/// A struct to generically describe a bit test intrinsic.
1557struct BitTest {
1558 enum ActionKind : uint8_t { TestOnly, Complement, Reset, Set };
1559 enum InterlockingKind : uint8_t {
1560 Unlocked,
1561 Sequential,
1562 Acquire,
1563 Release,
1564 NoFence
1565 };
1566
1567 ActionKind Action;
1568 InterlockingKind Interlocking;
1569 bool Is64Bit;
1570
1571 static BitTest decodeBitTestBuiltin(unsigned BuiltinID);
1572};
1573
1574} // namespace
1575
1576BitTest BitTest::decodeBitTestBuiltin(unsigned BuiltinID) {
1577 switch (BuiltinID) {
1578 // Main portable variants.
1579 case Builtin::BI_bittest:
1580 return {.Action: TestOnly, .Interlocking: Unlocked, .Is64Bit: false};
1581 case Builtin::BI_bittestandcomplement:
1582 return {.Action: Complement, .Interlocking: Unlocked, .Is64Bit: false};
1583 case Builtin::BI_bittestandreset:
1584 return {.Action: Reset, .Interlocking: Unlocked, .Is64Bit: false};
1585 case Builtin::BI_bittestandset:
1586 return {.Action: Set, .Interlocking: Unlocked, .Is64Bit: false};
1587 case Builtin::BI_interlockedbittestandreset:
1588 return {.Action: Reset, .Interlocking: Sequential, .Is64Bit: false};
1589 case Builtin::BI_interlockedbittestandset:
1590 return {.Action: Set, .Interlocking: Sequential, .Is64Bit: false};
1591
1592 // 64-bit variants.
1593 case Builtin::BI_bittest64:
1594 return {.Action: TestOnly, .Interlocking: Unlocked, .Is64Bit: true};
1595 case Builtin::BI_bittestandcomplement64:
1596 return {.Action: Complement, .Interlocking: Unlocked, .Is64Bit: true};
1597 case Builtin::BI_bittestandreset64:
1598 return {.Action: Reset, .Interlocking: Unlocked, .Is64Bit: true};
1599 case Builtin::BI_bittestandset64:
1600 return {.Action: Set, .Interlocking: Unlocked, .Is64Bit: true};
1601 case Builtin::BI_interlockedbittestandreset64:
1602 return {.Action: Reset, .Interlocking: Sequential, .Is64Bit: true};
1603 case Builtin::BI_interlockedbittestandset64:
1604 return {.Action: Set, .Interlocking: Sequential, .Is64Bit: true};
1605
1606 // ARM/AArch64-specific ordering variants.
1607 case Builtin::BI_interlockedbittestandset_acq:
1608 return {.Action: Set, .Interlocking: Acquire, .Is64Bit: false};
1609 case Builtin::BI_interlockedbittestandset_rel:
1610 return {.Action: Set, .Interlocking: Release, .Is64Bit: false};
1611 case Builtin::BI_interlockedbittestandset_nf:
1612 return {.Action: Set, .Interlocking: NoFence, .Is64Bit: false};
1613 case Builtin::BI_interlockedbittestandreset_acq:
1614 return {.Action: Reset, .Interlocking: Acquire, .Is64Bit: false};
1615 case Builtin::BI_interlockedbittestandreset_rel:
1616 return {.Action: Reset, .Interlocking: Release, .Is64Bit: false};
1617 case Builtin::BI_interlockedbittestandreset_nf:
1618 return {.Action: Reset, .Interlocking: NoFence, .Is64Bit: false};
1619 case Builtin::BI_interlockedbittestandreset64_acq:
1620 return {.Action: Reset, .Interlocking: Acquire, .Is64Bit: false};
1621 case Builtin::BI_interlockedbittestandreset64_rel:
1622 return {.Action: Reset, .Interlocking: Release, .Is64Bit: false};
1623 case Builtin::BI_interlockedbittestandreset64_nf:
1624 return {.Action: Reset, .Interlocking: NoFence, .Is64Bit: false};
1625 case Builtin::BI_interlockedbittestandset64_acq:
1626 return {.Action: Set, .Interlocking: Acquire, .Is64Bit: false};
1627 case Builtin::BI_interlockedbittestandset64_rel:
1628 return {.Action: Set, .Interlocking: Release, .Is64Bit: false};
1629 case Builtin::BI_interlockedbittestandset64_nf:
1630 return {.Action: Set, .Interlocking: NoFence, .Is64Bit: false};
1631 }
1632 llvm_unreachable("expected only bittest intrinsics");
1633}
1634
1635static char bitActionToX86BTCode(BitTest::ActionKind A) {
1636 switch (A) {
1637 case BitTest::TestOnly: return '\0';
1638 case BitTest::Complement: return 'c';
1639 case BitTest::Reset: return 'r';
1640 case BitTest::Set: return 's';
1641 }
1642 llvm_unreachable("invalid action");
1643}
1644
1645static llvm::Value *EmitX86BitTestIntrinsic(CodeGenFunction &CGF,
1646 BitTest BT,
1647 const CallExpr *E, Value *BitBase,
1648 Value *BitPos) {
1649 char Action = bitActionToX86BTCode(A: BT.Action);
1650 char SizeSuffix = BT.Is64Bit ? 'q' : 'l';
1651
1652 // Build the assembly.
1653 SmallString<64> Asm;
1654 raw_svector_ostream AsmOS(Asm);
1655 if (BT.Interlocking != BitTest::Unlocked)
1656 AsmOS << "lock ";
1657 AsmOS << "bt";
1658 if (Action)
1659 AsmOS << Action;
1660 AsmOS << SizeSuffix << " $2, ($1)";
1661
1662 // Build the constraints. FIXME: We should support immediates when possible.
1663 std::string Constraints = "={@ccc},r,r,~{cc},~{memory}";
1664 std::string_view MachineClobbers = CGF.getTarget().getClobbers();
1665 if (!MachineClobbers.empty()) {
1666 Constraints += ',';
1667 Constraints += MachineClobbers;
1668 }
1669 llvm::IntegerType *IntType = llvm::IntegerType::get(
1670 C&: CGF.getLLVMContext(),
1671 NumBits: CGF.getContext().getTypeSize(T: E->getArg(Arg: 1)->getType()));
1672 llvm::FunctionType *FTy =
1673 llvm::FunctionType::get(Result: CGF.Int8Ty, Params: {CGF.UnqualPtrTy, IntType}, isVarArg: false);
1674
1675 llvm::InlineAsm *IA =
1676 llvm::InlineAsm::get(Ty: FTy, AsmString: Asm, Constraints, /*hasSideEffects=*/true);
1677 return CGF.Builder.CreateCall(Callee: IA, Args: {BitBase, BitPos});
1678}
1679
1680static llvm::AtomicOrdering
1681getBitTestAtomicOrdering(BitTest::InterlockingKind I) {
1682 switch (I) {
1683 case BitTest::Unlocked: return llvm::AtomicOrdering::NotAtomic;
1684 case BitTest::Sequential: return llvm::AtomicOrdering::SequentiallyConsistent;
1685 case BitTest::Acquire: return llvm::AtomicOrdering::Acquire;
1686 case BitTest::Release: return llvm::AtomicOrdering::Release;
1687 case BitTest::NoFence: return llvm::AtomicOrdering::Monotonic;
1688 }
1689 llvm_unreachable("invalid interlocking");
1690}
1691
1692/// Emit a _bittest* intrinsic. These intrinsics take a pointer to an array of
1693/// bits and a bit position and read and optionally modify the bit at that
1694/// position. The position index can be arbitrarily large, i.e. it can be larger
1695/// than 31 or 63, so we need an indexed load in the general case.
1696static llvm::Value *EmitBitTestIntrinsic(CodeGenFunction &CGF,
1697 unsigned BuiltinID,
1698 const CallExpr *E) {
1699 Value *BitBase = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
1700 Value *BitPos = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
1701
1702 BitTest BT = BitTest::decodeBitTestBuiltin(BuiltinID);
1703
1704 // X86 has special BT, BTC, BTR, and BTS instructions that handle the array
1705 // indexing operation internally. Use them if possible.
1706 if (CGF.getTarget().getTriple().isX86())
1707 return EmitX86BitTestIntrinsic(CGF, BT, E, BitBase, BitPos);
1708
1709 // Otherwise, use generic code to load one byte and test the bit. Use all but
1710 // the bottom three bits as the array index, and the bottom three bits to form
1711 // a mask.
1712 // Bit = BitBaseI8[BitPos >> 3] & (1 << (BitPos & 0x7)) != 0;
1713 Value *ByteIndex = CGF.Builder.CreateAShr(
1714 LHS: BitPos, RHS: llvm::ConstantInt::get(Ty: BitPos->getType(), V: 3), Name: "bittest.byteidx");
1715 Address ByteAddr(CGF.Builder.CreateInBoundsGEP(Ty: CGF.Int8Ty, Ptr: BitBase, IdxList: ByteIndex,
1716 Name: "bittest.byteaddr"),
1717 CGF.Int8Ty, CharUnits::One());
1718 Value *PosLow =
1719 CGF.Builder.CreateAnd(LHS: CGF.Builder.CreateTrunc(V: BitPos, DestTy: CGF.Int8Ty),
1720 RHS: llvm::ConstantInt::get(Ty: CGF.Int8Ty, V: 0x7));
1721
1722 // The updating instructions will need a mask.
1723 Value *Mask = nullptr;
1724 if (BT.Action != BitTest::TestOnly) {
1725 Mask = CGF.Builder.CreateShl(LHS: llvm::ConstantInt::get(Ty: CGF.Int8Ty, V: 1), RHS: PosLow,
1726 Name: "bittest.mask");
1727 }
1728
1729 // Check the action and ordering of the interlocked intrinsics.
1730 llvm::AtomicOrdering Ordering = getBitTestAtomicOrdering(I: BT.Interlocking);
1731
1732 Value *OldByte = nullptr;
1733 if (Ordering != llvm::AtomicOrdering::NotAtomic) {
1734 // Emit a combined atomicrmw load/store operation for the interlocked
1735 // intrinsics.
1736 llvm::AtomicRMWInst::BinOp RMWOp = llvm::AtomicRMWInst::Or;
1737 if (BT.Action == BitTest::Reset) {
1738 Mask = CGF.Builder.CreateNot(V: Mask);
1739 RMWOp = llvm::AtomicRMWInst::And;
1740 }
1741 OldByte = CGF.Builder.CreateAtomicRMW(Op: RMWOp, Addr: ByteAddr, Val: Mask, Ordering);
1742 } else {
1743 // Emit a plain load for the non-interlocked intrinsics.
1744 OldByte = CGF.Builder.CreateLoad(Addr: ByteAddr, Name: "bittest.byte");
1745 Value *NewByte = nullptr;
1746 switch (BT.Action) {
1747 case BitTest::TestOnly:
1748 // Don't store anything.
1749 break;
1750 case BitTest::Complement:
1751 NewByte = CGF.Builder.CreateXor(LHS: OldByte, RHS: Mask);
1752 break;
1753 case BitTest::Reset:
1754 NewByte = CGF.Builder.CreateAnd(LHS: OldByte, RHS: CGF.Builder.CreateNot(V: Mask));
1755 break;
1756 case BitTest::Set:
1757 NewByte = CGF.Builder.CreateOr(LHS: OldByte, RHS: Mask);
1758 break;
1759 }
1760 if (NewByte)
1761 CGF.Builder.CreateStore(Val: NewByte, Addr: ByteAddr);
1762 }
1763
1764 // However we loaded the old byte, either by plain load or atomicrmw, shift
1765 // the bit into the low position and mask it to 0 or 1.
1766 Value *ShiftedByte = CGF.Builder.CreateLShr(LHS: OldByte, RHS: PosLow, Name: "bittest.shr");
1767 return CGF.Builder.CreateAnd(
1768 LHS: ShiftedByte, RHS: llvm::ConstantInt::get(Ty: CGF.Int8Ty, V: 1), Name: "bittest.res");
1769}
1770
1771namespace {
1772enum class MSVCSetJmpKind {
1773 _setjmpex,
1774 _setjmp3,
1775 _setjmp
1776};
1777}
1778
1779/// MSVC handles setjmp a bit differently on different platforms. On every
1780/// architecture except 32-bit x86, the frame address is passed. On x86, extra
1781/// parameters can be passed as variadic arguments, but we always pass none.
1782static RValue EmitMSVCRTSetJmp(CodeGenFunction &CGF, MSVCSetJmpKind SJKind,
1783 const CallExpr *E) {
1784 llvm::Value *Arg1 = nullptr;
1785 llvm::Type *Arg1Ty = nullptr;
1786 StringRef Name;
1787 bool IsVarArg = false;
1788 if (SJKind == MSVCSetJmpKind::_setjmp3) {
1789 Name = "_setjmp3";
1790 Arg1Ty = CGF.Int32Ty;
1791 Arg1 = llvm::ConstantInt::get(Ty: CGF.IntTy, V: 0);
1792 IsVarArg = true;
1793 } else {
1794 Name = SJKind == MSVCSetJmpKind::_setjmp ? "_setjmp" : "_setjmpex";
1795 Arg1Ty = CGF.Int8PtrTy;
1796 if (CGF.getTarget().getTriple().getArch() == llvm::Triple::aarch64) {
1797 Arg1 = CGF.Builder.CreateCall(
1798 Callee: CGF.CGM.getIntrinsic(IID: Intrinsic::sponentry, Tys: CGF.AllocaInt8PtrTy));
1799 } else
1800 Arg1 = CGF.Builder.CreateCall(
1801 Callee: CGF.CGM.getIntrinsic(IID: Intrinsic::frameaddress, Tys: CGF.AllocaInt8PtrTy),
1802 Args: llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: 0));
1803 }
1804
1805 // Mark the call site and declaration with ReturnsTwice.
1806 llvm::Type *ArgTypes[2] = {CGF.Int8PtrTy, Arg1Ty};
1807 llvm::AttributeList ReturnsTwiceAttr = llvm::AttributeList::get(
1808 C&: CGF.getLLVMContext(), Index: llvm::AttributeList::FunctionIndex,
1809 Kinds: llvm::Attribute::ReturnsTwice);
1810 llvm::FunctionCallee SetJmpFn = CGF.CGM.CreateRuntimeFunction(
1811 Ty: llvm::FunctionType::get(Result: CGF.IntTy, Params: ArgTypes, isVarArg: IsVarArg), Name,
1812 ExtraAttrs: ReturnsTwiceAttr, /*Local=*/true);
1813
1814 llvm::Value *Buf = CGF.Builder.CreateBitOrPointerCast(
1815 V: CGF.EmitScalarExpr(E: E->getArg(Arg: 0)), DestTy: CGF.Int8PtrTy);
1816 llvm::Value *Args[] = {Buf, Arg1};
1817 llvm::CallBase *CB = CGF.EmitRuntimeCallOrInvoke(callee: SetJmpFn, args: Args);
1818 CB->setAttributes(ReturnsTwiceAttr);
1819 return RValue::get(V: CB);
1820}
1821
1822// Emit an MSVC intrinsic. Assumes that arguments have *not* been evaluated.
1823Value *CodeGenFunction::EmitMSVCBuiltinExpr(MSVCIntrin BuiltinID,
1824 const CallExpr *E) {
1825 switch (BuiltinID) {
1826 case MSVCIntrin::_BitScanForward:
1827 case MSVCIntrin::_BitScanReverse: {
1828 Address IndexAddress(EmitPointerWithAlignment(Addr: E->getArg(Arg: 0)));
1829 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 1));
1830
1831 llvm::Type *ArgType = ArgValue->getType();
1832 llvm::Type *IndexType = IndexAddress.getElementType();
1833 llvm::Type *ResultType = ConvertType(T: E->getType());
1834
1835 Value *ArgZero = llvm::Constant::getNullValue(Ty: ArgType);
1836 Value *ResZero = llvm::Constant::getNullValue(Ty: ResultType);
1837 Value *ResOne = llvm::ConstantInt::get(Ty: ResultType, V: 1);
1838
1839 BasicBlock *Begin = Builder.GetInsertBlock();
1840 BasicBlock *End = createBasicBlock(name: "bitscan_end", parent: this->CurFn);
1841 Builder.SetInsertPoint(End);
1842 PHINode *Result = Builder.CreatePHI(Ty: ResultType, NumReservedValues: 2, Name: "bitscan_result");
1843
1844 Builder.SetInsertPoint(Begin);
1845 Value *IsZero = Builder.CreateICmpEQ(LHS: ArgValue, RHS: ArgZero);
1846 BasicBlock *NotZero = createBasicBlock(name: "bitscan_not_zero", parent: this->CurFn);
1847 Builder.CreateCondBr(Cond: IsZero, True: End, False: NotZero);
1848 Result->addIncoming(V: ResZero, BB: Begin);
1849
1850 Builder.SetInsertPoint(NotZero);
1851
1852 if (BuiltinID == MSVCIntrin::_BitScanForward) {
1853 Function *F = CGM.getIntrinsic(IID: Intrinsic::cttz, Tys: ArgType);
1854 Value *ZeroCount = Builder.CreateCall(Callee: F, Args: {ArgValue, Builder.getTrue()});
1855 ZeroCount = Builder.CreateIntCast(V: ZeroCount, DestTy: IndexType, isSigned: false);
1856 Builder.CreateStore(Val: ZeroCount, Addr: IndexAddress, IsVolatile: false);
1857 } else {
1858 unsigned ArgWidth = cast<llvm::IntegerType>(Val: ArgType)->getBitWidth();
1859 Value *ArgTypeLastIndex = llvm::ConstantInt::get(Ty: IndexType, V: ArgWidth - 1);
1860
1861 Function *F = CGM.getIntrinsic(IID: Intrinsic::ctlz, Tys: ArgType);
1862 Value *ZeroCount = Builder.CreateCall(Callee: F, Args: {ArgValue, Builder.getTrue()});
1863 ZeroCount = Builder.CreateIntCast(V: ZeroCount, DestTy: IndexType, isSigned: false);
1864 Value *Index = Builder.CreateNSWSub(LHS: ArgTypeLastIndex, RHS: ZeroCount);
1865 Builder.CreateStore(Val: Index, Addr: IndexAddress, IsVolatile: false);
1866 }
1867 Builder.CreateBr(Dest: End);
1868 Result->addIncoming(V: ResOne, BB: NotZero);
1869
1870 Builder.SetInsertPoint(End);
1871 return Result;
1872 }
1873 case MSVCIntrin::_InterlockedAnd:
1874 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::And, E);
1875 case MSVCIntrin::_InterlockedExchange:
1876 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Xchg, E);
1877 case MSVCIntrin::_InterlockedExchangeAdd:
1878 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Add, E);
1879 case MSVCIntrin::_InterlockedExchangeSub:
1880 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Sub, E);
1881 case MSVCIntrin::_InterlockedOr:
1882 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Or, E);
1883 case MSVCIntrin::_InterlockedXor:
1884 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Xor, E);
1885 case MSVCIntrin::_InterlockedExchangeAdd_acq:
1886 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Add, E,
1887 Ordering: AtomicOrdering::Acquire);
1888 case MSVCIntrin::_InterlockedExchangeAdd_rel:
1889 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Add, E,
1890 Ordering: AtomicOrdering::Release);
1891 case MSVCIntrin::_InterlockedExchangeAdd_nf:
1892 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Add, E,
1893 Ordering: AtomicOrdering::Monotonic);
1894 case MSVCIntrin::_InterlockedExchange_acq:
1895 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Xchg, E,
1896 Ordering: AtomicOrdering::Acquire);
1897 case MSVCIntrin::_InterlockedExchange_rel:
1898 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Xchg, E,
1899 Ordering: AtomicOrdering::Release);
1900 case MSVCIntrin::_InterlockedExchange_nf:
1901 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Xchg, E,
1902 Ordering: AtomicOrdering::Monotonic);
1903 case MSVCIntrin::_InterlockedCompareExchange:
1904 return EmitAtomicCmpXchgForMSIntrin(CGF&: *this, E);
1905 case MSVCIntrin::_InterlockedCompareExchange_acq:
1906 return EmitAtomicCmpXchgForMSIntrin(CGF&: *this, E, SuccessOrdering: AtomicOrdering::Acquire);
1907 case MSVCIntrin::_InterlockedCompareExchange_rel:
1908 return EmitAtomicCmpXchgForMSIntrin(CGF&: *this, E, SuccessOrdering: AtomicOrdering::Release);
1909 case MSVCIntrin::_InterlockedCompareExchange_nf:
1910 return EmitAtomicCmpXchgForMSIntrin(CGF&: *this, E, SuccessOrdering: AtomicOrdering::Monotonic);
1911 case MSVCIntrin::_InterlockedCompareExchange128:
1912 return EmitAtomicCmpXchg128ForMSIntrin(
1913 CGF&: *this, E, SuccessOrdering: AtomicOrdering::SequentiallyConsistent);
1914 case MSVCIntrin::_InterlockedCompareExchange128_acq:
1915 return EmitAtomicCmpXchg128ForMSIntrin(CGF&: *this, E, SuccessOrdering: AtomicOrdering::Acquire);
1916 case MSVCIntrin::_InterlockedCompareExchange128_rel:
1917 return EmitAtomicCmpXchg128ForMSIntrin(CGF&: *this, E, SuccessOrdering: AtomicOrdering::Release);
1918 case MSVCIntrin::_InterlockedCompareExchange128_nf:
1919 return EmitAtomicCmpXchg128ForMSIntrin(CGF&: *this, E, SuccessOrdering: AtomicOrdering::Monotonic);
1920 case MSVCIntrin::_InterlockedOr_acq:
1921 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Or, E,
1922 Ordering: AtomicOrdering::Acquire);
1923 case MSVCIntrin::_InterlockedOr_rel:
1924 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Or, E,
1925 Ordering: AtomicOrdering::Release);
1926 case MSVCIntrin::_InterlockedOr_nf:
1927 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Or, E,
1928 Ordering: AtomicOrdering::Monotonic);
1929 case MSVCIntrin::_InterlockedXor_acq:
1930 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Xor, E,
1931 Ordering: AtomicOrdering::Acquire);
1932 case MSVCIntrin::_InterlockedXor_rel:
1933 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Xor, E,
1934 Ordering: AtomicOrdering::Release);
1935 case MSVCIntrin::_InterlockedXor_nf:
1936 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Xor, E,
1937 Ordering: AtomicOrdering::Monotonic);
1938 case MSVCIntrin::_InterlockedAnd_acq:
1939 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::And, E,
1940 Ordering: AtomicOrdering::Acquire);
1941 case MSVCIntrin::_InterlockedAnd_rel:
1942 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::And, E,
1943 Ordering: AtomicOrdering::Release);
1944 case MSVCIntrin::_InterlockedAnd_nf:
1945 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::And, E,
1946 Ordering: AtomicOrdering::Monotonic);
1947 case MSVCIntrin::_InterlockedIncrement_acq:
1948 return EmitAtomicIncrementValue(CGF&: *this, E, Ordering: AtomicOrdering::Acquire);
1949 case MSVCIntrin::_InterlockedIncrement_rel:
1950 return EmitAtomicIncrementValue(CGF&: *this, E, Ordering: AtomicOrdering::Release);
1951 case MSVCIntrin::_InterlockedIncrement_nf:
1952 return EmitAtomicIncrementValue(CGF&: *this, E, Ordering: AtomicOrdering::Monotonic);
1953 case MSVCIntrin::_InterlockedDecrement_acq:
1954 return EmitAtomicDecrementValue(CGF&: *this, E, Ordering: AtomicOrdering::Acquire);
1955 case MSVCIntrin::_InterlockedDecrement_rel:
1956 return EmitAtomicDecrementValue(CGF&: *this, E, Ordering: AtomicOrdering::Release);
1957 case MSVCIntrin::_InterlockedDecrement_nf:
1958 return EmitAtomicDecrementValue(CGF&: *this, E, Ordering: AtomicOrdering::Monotonic);
1959
1960 case MSVCIntrin::_InterlockedDecrement:
1961 return EmitAtomicDecrementValue(CGF&: *this, E);
1962 case MSVCIntrin::_InterlockedIncrement:
1963 return EmitAtomicIncrementValue(CGF&: *this, E);
1964
1965 case MSVCIntrin::__fastfail: {
1966 // Request immediate process termination from the kernel. The instruction
1967 // sequences to do this are documented on MSDN:
1968 // https://msdn.microsoft.com/en-us/library/dn774154.aspx
1969 llvm::Triple::ArchType ISA = getTarget().getTriple().getArch();
1970 StringRef Asm, Constraints;
1971 switch (ISA) {
1972 default:
1973 ErrorUnsupported(S: E, Type: "__fastfail call for this architecture");
1974 break;
1975 case llvm::Triple::x86:
1976 case llvm::Triple::x86_64:
1977 Asm = "int $$0x29";
1978 Constraints = "{cx}";
1979 break;
1980 case llvm::Triple::thumb:
1981 Asm = "udf #251";
1982 Constraints = "{r0}";
1983 break;
1984 case llvm::Triple::aarch64:
1985 Asm = "brk #0xF003";
1986 Constraints = "{w0}";
1987 }
1988 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: VoidTy, Params: {Int32Ty}, isVarArg: false);
1989 llvm::InlineAsm *IA =
1990 llvm::InlineAsm::get(Ty: FTy, AsmString: Asm, Constraints, /*hasSideEffects=*/true);
1991 llvm::AttributeList NoReturnAttr = llvm::AttributeList::get(
1992 C&: getLLVMContext(), Index: llvm::AttributeList::FunctionIndex,
1993 Kinds: llvm::Attribute::NoReturn);
1994 llvm::CallInst *CI = Builder.CreateCall(Callee: IA, Args: EmitScalarExpr(E: E->getArg(Arg: 0)));
1995 CI->setAttributes(NoReturnAttr);
1996 return CI;
1997 }
1998 }
1999 llvm_unreachable("Incorrect MSVC intrinsic!");
2000}
2001
2002namespace {
2003// ARC cleanup for __builtin_os_log_format
2004struct CallObjCArcUse final : EHScopeStack::Cleanup {
2005 CallObjCArcUse(llvm::Value *object) : object(object) {}
2006 llvm::Value *object;
2007
2008 void Emit(CodeGenFunction &CGF, Flags flags) override {
2009 CGF.EmitARCIntrinsicUse(values: object);
2010 }
2011};
2012}
2013
2014Value *CodeGenFunction::EmitCheckedArgForBuiltin(const Expr *E,
2015 BuiltinCheckKind Kind) {
2016 assert((Kind == BCK_CLZPassedZero || Kind == BCK_CTZPassedZero) &&
2017 "Unsupported builtin check kind");
2018
2019 Value *ArgValue = EmitScalarExpr(E);
2020 if (!SanOpts.has(K: SanitizerKind::Builtin))
2021 return ArgValue;
2022
2023 auto CheckOrdinal = SanitizerKind::SO_Builtin;
2024 auto CheckHandler = SanitizerHandler::InvalidBuiltin;
2025 SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);
2026 Value *Cond = Builder.CreateICmpNE(
2027 LHS: ArgValue, RHS: llvm::Constant::getNullValue(Ty: ArgValue->getType()));
2028 EmitCheck(Checked: std::make_pair(x&: Cond, y&: CheckOrdinal), Check: CheckHandler,
2029 StaticArgs: {EmitCheckSourceLocation(Loc: E->getExprLoc()),
2030 llvm::ConstantInt::get(Ty: Builder.getInt8Ty(), V: Kind)},
2031 DynamicArgs: {});
2032 return ArgValue;
2033}
2034
2035Value *CodeGenFunction::EmitCheckedArgForAssume(const Expr *E) {
2036 Value *ArgValue = EvaluateExprAsBool(E);
2037 if (!SanOpts.has(K: SanitizerKind::Builtin))
2038 return ArgValue;
2039
2040 auto CheckOrdinal = SanitizerKind::SO_Builtin;
2041 auto CheckHandler = SanitizerHandler::InvalidBuiltin;
2042 SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);
2043 EmitCheck(
2044 Checked: std::make_pair(x&: ArgValue, y&: CheckOrdinal), Check: CheckHandler,
2045 StaticArgs: {EmitCheckSourceLocation(Loc: E->getExprLoc()),
2046 llvm::ConstantInt::get(Ty: Builder.getInt8Ty(), V: BCK_AssumePassedFalse)},
2047 DynamicArgs: {});
2048 return ArgValue;
2049}
2050
2051static Value *EmitAbs(CodeGenFunction &CGF, Value *ArgValue, bool HasNSW) {
2052 return CGF.Builder.CreateBinaryIntrinsic(
2053 ID: Intrinsic::abs, LHS: ArgValue,
2054 RHS: ConstantInt::get(Ty: CGF.Builder.getInt1Ty(), V: HasNSW));
2055}
2056
2057static Value *EmitOverflowCheckedAbs(CodeGenFunction &CGF, const CallExpr *E,
2058 bool SanitizeOverflow) {
2059 Value *ArgValue = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
2060
2061 // Try to eliminate overflow check.
2062 if (const auto *VCI = dyn_cast<llvm::ConstantInt>(Val: ArgValue)) {
2063 if (!VCI->isMinSignedValue())
2064 return EmitAbs(CGF, ArgValue, HasNSW: true);
2065 }
2066
2067 SmallVector<SanitizerKind::SanitizerOrdinal, 1> Ordinals;
2068 SanitizerHandler CheckHandler;
2069 if (SanitizeOverflow) {
2070 Ordinals.push_back(Elt: SanitizerKind::SO_SignedIntegerOverflow);
2071 CheckHandler = SanitizerHandler::NegateOverflow;
2072 } else
2073 CheckHandler = SanitizerHandler::SubOverflow;
2074
2075 SanitizerDebugLocation SanScope(&CGF, Ordinals, CheckHandler);
2076
2077 Constant *Zero = Constant::getNullValue(Ty: ArgValue->getType());
2078 Value *ResultAndOverflow = CGF.Builder.CreateBinaryIntrinsic(
2079 ID: Intrinsic::ssub_with_overflow, LHS: Zero, RHS: ArgValue);
2080 Value *Result = CGF.Builder.CreateExtractValue(Agg: ResultAndOverflow, Idxs: 0);
2081 Value *NotOverflow = CGF.Builder.CreateNot(
2082 V: CGF.Builder.CreateExtractValue(Agg: ResultAndOverflow, Idxs: 1));
2083
2084 // TODO: support -ftrapv-handler.
2085 if (SanitizeOverflow) {
2086 CGF.EmitCheck(Checked: {{NotOverflow, SanitizerKind::SO_SignedIntegerOverflow}},
2087 Check: CheckHandler,
2088 StaticArgs: {CGF.EmitCheckSourceLocation(Loc: E->getArg(Arg: 0)->getExprLoc()),
2089 CGF.EmitCheckTypeDescriptor(T: E->getType())},
2090 DynamicArgs: {ArgValue});
2091 } else
2092 CGF.EmitTrapCheck(Checked: NotOverflow, CheckHandlerID: CheckHandler);
2093
2094 Value *CmpResult = CGF.Builder.CreateICmpSLT(LHS: ArgValue, RHS: Zero, Name: "abscond");
2095 return CGF.Builder.CreateSelect(C: CmpResult, True: Result, False: ArgValue, Name: "abs");
2096}
2097
2098/// Get the argument type for arguments to os_log_helper.
2099static CanQualType getOSLogArgType(ASTContext &C, int Size) {
2100 QualType UnsignedTy = C.getIntTypeForBitwidth(DestWidth: Size * 8, /*Signed=*/false);
2101 return C.getCanonicalType(T: UnsignedTy);
2102}
2103
2104llvm::Function *CodeGenFunction::generateBuiltinOSLogHelperFunction(
2105 const analyze_os_log::OSLogBufferLayout &Layout,
2106 CharUnits BufferAlignment) {
2107 ASTContext &Ctx = getContext();
2108
2109 llvm::SmallString<64> Name;
2110 {
2111 raw_svector_ostream OS(Name);
2112 OS << "__os_log_helper";
2113 OS << "_" << BufferAlignment.getQuantity();
2114 OS << "_" << int(Layout.getSummaryByte());
2115 OS << "_" << int(Layout.getNumArgsByte());
2116 for (const auto &Item : Layout.Items)
2117 OS << "_" << int(Item.getSizeByte()) << "_"
2118 << int(Item.getDescriptorByte());
2119 }
2120
2121 if (llvm::Function *F = CGM.getModule().getFunction(Name))
2122 return F;
2123
2124 llvm::SmallVector<QualType, 4> ArgTys;
2125 FunctionArgList Args;
2126 Args.push_back(Elt: ImplicitParamDecl::Create(
2127 C&: Ctx, DC: nullptr, IdLoc: SourceLocation(), Id: &Ctx.Idents.get(Name: "buffer"), T: Ctx.VoidPtrTy,
2128 ParamKind: ImplicitParamKind::Other));
2129 ArgTys.emplace_back(Args&: Ctx.VoidPtrTy);
2130
2131 for (unsigned int I = 0, E = Layout.Items.size(); I < E; ++I) {
2132 char Size = Layout.Items[I].getSizeByte();
2133 if (!Size)
2134 continue;
2135
2136 QualType ArgTy = getOSLogArgType(C&: Ctx, Size);
2137 Args.push_back(Elt: ImplicitParamDecl::Create(
2138 C&: Ctx, DC: nullptr, IdLoc: SourceLocation(),
2139 Id: &Ctx.Idents.get(Name: std::string("arg") + llvm::to_string(Value: I)), T: ArgTy,
2140 ParamKind: ImplicitParamKind::Other));
2141 ArgTys.emplace_back(Args&: ArgTy);
2142 }
2143
2144 QualType ReturnTy = Ctx.VoidTy;
2145
2146 // The helper function has linkonce_odr linkage to enable the linker to merge
2147 // identical functions. To ensure the merging always happens, 'noinline' is
2148 // attached to the function when compiling with -Oz.
2149 const CGFunctionInfo &FI =
2150 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: ReturnTy, args: Args);
2151 llvm::FunctionType *FuncTy = CGM.getTypes().GetFunctionType(Info: FI);
2152 llvm::Function *Fn = llvm::Function::Create(
2153 Ty: FuncTy, Linkage: llvm::GlobalValue::LinkOnceODRLinkage, N: Name, M: &CGM.getModule());
2154 Fn->setVisibility(llvm::GlobalValue::HiddenVisibility);
2155 CGM.SetLLVMFunctionAttributes(GD: GlobalDecl(), Info: FI, F: Fn, /*IsThunk=*/false);
2156 CGM.SetLLVMFunctionAttributesForDefinition(D: nullptr, F: Fn);
2157 Fn->setDoesNotThrow();
2158
2159 // Attach 'noinline' at -Oz.
2160 if (CGM.getCodeGenOpts().OptimizeSize == 2)
2161 Fn->addFnAttr(Kind: llvm::Attribute::NoInline);
2162
2163 auto NL = ApplyDebugLocation::CreateEmpty(CGF&: *this);
2164 StartFunction(GD: GlobalDecl(), RetTy: ReturnTy, Fn, FnInfo: FI, Args);
2165
2166 // Create a scope with an artificial location for the body of this function.
2167 auto AL = ApplyDebugLocation::CreateArtificial(CGF&: *this);
2168
2169 CharUnits Offset;
2170 Address BufAddr = makeNaturalAddressForPointer(
2171 Ptr: Builder.CreateLoad(Addr: GetAddrOfLocalVar(VD: Args[0]), Name: "buf"), T: Ctx.VoidTy,
2172 Alignment: BufferAlignment);
2173 Builder.CreateStore(Val: Builder.getInt8(C: Layout.getSummaryByte()),
2174 Addr: Builder.CreateConstByteGEP(Addr: BufAddr, Offset: Offset++, Name: "summary"));
2175 Builder.CreateStore(Val: Builder.getInt8(C: Layout.getNumArgsByte()),
2176 Addr: Builder.CreateConstByteGEP(Addr: BufAddr, Offset: Offset++, Name: "numArgs"));
2177
2178 unsigned I = 1;
2179 for (const auto &Item : Layout.Items) {
2180 Builder.CreateStore(
2181 Val: Builder.getInt8(C: Item.getDescriptorByte()),
2182 Addr: Builder.CreateConstByteGEP(Addr: BufAddr, Offset: Offset++, Name: "argDescriptor"));
2183 Builder.CreateStore(
2184 Val: Builder.getInt8(C: Item.getSizeByte()),
2185 Addr: Builder.CreateConstByteGEP(Addr: BufAddr, Offset: Offset++, Name: "argSize"));
2186
2187 CharUnits Size = Item.size();
2188 if (!Size.getQuantity())
2189 continue;
2190
2191 Address Arg = GetAddrOfLocalVar(VD: Args[I]);
2192 Address Addr = Builder.CreateConstByteGEP(Addr: BufAddr, Offset, Name: "argData");
2193 Addr = Addr.withElementType(ElemTy: Arg.getElementType());
2194 Builder.CreateStore(Val: Builder.CreateLoad(Addr: Arg), Addr);
2195 Offset += Size;
2196 ++I;
2197 }
2198
2199 FinishFunction();
2200
2201 return Fn;
2202}
2203
2204RValue CodeGenFunction::emitBuiltinOSLogFormat(const CallExpr &E) {
2205 assert(E.getNumArgs() >= 2 &&
2206 "__builtin_os_log_format takes at least 2 arguments");
2207 ASTContext &Ctx = getContext();
2208 analyze_os_log::OSLogBufferLayout Layout;
2209 analyze_os_log::computeOSLogBufferLayout(Ctx, E: &E, layout&: Layout);
2210 Address BufAddr = EmitPointerWithAlignment(Addr: E.getArg(Arg: 0));
2211
2212 // Ignore argument 1, the format string. It is not currently used.
2213 CallArgList Args;
2214 Args.add(rvalue: RValue::get(V: BufAddr.emitRawPointer(CGF&: *this)), type: Ctx.VoidPtrTy);
2215
2216 for (const auto &Item : Layout.Items) {
2217 int Size = Item.getSizeByte();
2218 if (!Size)
2219 continue;
2220
2221 llvm::Value *ArgVal;
2222
2223 if (Item.getKind() == analyze_os_log::OSLogBufferItem::MaskKind) {
2224 uint64_t Val = 0;
2225 for (unsigned I = 0, E = Item.getMaskType().size(); I < E; ++I)
2226 Val |= ((uint64_t)Item.getMaskType()[I]) << I * 8;
2227 ArgVal = llvm::Constant::getIntegerValue(Ty: Int64Ty, V: llvm::APInt(64, Val));
2228 } else if (const Expr *TheExpr = Item.getExpr()) {
2229 ArgVal = EmitScalarExpr(E: TheExpr, /*Ignore*/ IgnoreResultAssign: false);
2230
2231 // If a temporary object that requires destruction after the full
2232 // expression is passed, push a lifetime-extended cleanup to extend its
2233 // lifetime to the end of the enclosing block scope.
2234 auto LifetimeExtendObject = [&](const Expr *E) {
2235 E = E->IgnoreParenCasts();
2236 // Extend lifetimes of objects returned by function calls and message
2237 // sends.
2238
2239 // FIXME: We should do this in other cases in which temporaries are
2240 // created including arguments of non-ARC types (e.g., C++
2241 // temporaries).
2242 if (isa<CallExpr>(Val: E) || isa<ObjCMessageExpr>(Val: E))
2243 return true;
2244 return false;
2245 };
2246
2247 if (TheExpr->getType()->isObjCRetainableType() &&
2248 getLangOpts().ObjCAutoRefCount && LifetimeExtendObject(TheExpr)) {
2249 assert(getEvaluationKind(TheExpr->getType()) == TEK_Scalar &&
2250 "Only scalar can be a ObjC retainable type");
2251 if (!isa<Constant>(Val: ArgVal)) {
2252 CleanupKind Cleanup = getARCCleanupKind();
2253 QualType Ty = TheExpr->getType();
2254 RawAddress Alloca = RawAddress::invalid();
2255 RawAddress Addr = CreateMemTemp(T: Ty, Name: "os.log.arg", Alloca: &Alloca);
2256 ArgVal = EmitARCRetain(type: Ty, value: ArgVal);
2257 Builder.CreateStore(Val: ArgVal, Addr);
2258 pushLifetimeExtendedDestroy(kind: Cleanup, addr: Alloca, type: Ty,
2259 destroyer: CodeGenFunction::destroyARCStrongPrecise,
2260 useEHCleanupForArray: Cleanup & EHCleanup);
2261
2262 // Push a clang.arc.use call to ensure ARC optimizer knows that the
2263 // argument has to be alive.
2264 if (CGM.getCodeGenOpts().OptimizationLevel != 0)
2265 pushCleanupAfterFullExpr<CallObjCArcUse>(Kind: Cleanup, A: ArgVal);
2266 }
2267 }
2268 } else {
2269 ArgVal = Builder.getInt32(C: Item.getConstValue().getQuantity());
2270 }
2271
2272 unsigned ArgValSize =
2273 CGM.getDataLayout().getTypeSizeInBits(Ty: ArgVal->getType());
2274 llvm::IntegerType *IntTy = llvm::Type::getIntNTy(C&: getLLVMContext(),
2275 N: ArgValSize);
2276 ArgVal = Builder.CreateBitOrPointerCast(V: ArgVal, DestTy: IntTy);
2277 CanQualType ArgTy = getOSLogArgType(C&: Ctx, Size);
2278 // If ArgVal has type x86_fp80, zero-extend ArgVal.
2279 ArgVal = Builder.CreateZExtOrBitCast(V: ArgVal, DestTy: ConvertType(T: ArgTy));
2280 Args.add(rvalue: RValue::get(V: ArgVal), type: ArgTy);
2281 }
2282
2283 const CGFunctionInfo &FI =
2284 CGM.getTypes().arrangeBuiltinFunctionCall(resultType: Ctx.VoidTy, args: Args);
2285 llvm::Function *F = CodeGenFunction(CGM).generateBuiltinOSLogHelperFunction(
2286 Layout, BufferAlignment: BufAddr.getAlignment());
2287 EmitCall(CallInfo: FI, Callee: CGCallee::forDirect(functionPtr: F), ReturnValue: ReturnValueSlot(), Args);
2288 return RValue::get(Addr: BufAddr, CGF&: *this);
2289}
2290
2291static bool isSpecialUnsignedMultiplySignedResult(
2292 unsigned BuiltinID, WidthAndSignedness Op1Info, WidthAndSignedness Op2Info,
2293 WidthAndSignedness ResultInfo) {
2294 return BuiltinID == Builtin::BI__builtin_mul_overflow &&
2295 Op1Info.Width == Op2Info.Width && Op2Info.Width == ResultInfo.Width &&
2296 !Op1Info.Signed && !Op2Info.Signed && ResultInfo.Signed;
2297}
2298
2299static RValue EmitCheckedUnsignedMultiplySignedResult(
2300 CodeGenFunction &CGF, const clang::Expr *Op1, WidthAndSignedness Op1Info,
2301 const clang::Expr *Op2, WidthAndSignedness Op2Info,
2302 const clang::Expr *ResultArg, QualType ResultQTy,
2303 WidthAndSignedness ResultInfo) {
2304 assert(isSpecialUnsignedMultiplySignedResult(
2305 Builtin::BI__builtin_mul_overflow, Op1Info, Op2Info, ResultInfo) &&
2306 "Cannot specialize this multiply");
2307
2308 llvm::Value *V1 = CGF.EmitScalarExpr(E: Op1);
2309 llvm::Value *V2 = CGF.EmitScalarExpr(E: Op2);
2310
2311 llvm::Value *HasOverflow;
2312 llvm::Value *Result = EmitOverflowIntrinsic(
2313 CGF, IntrinsicID: Intrinsic::umul_with_overflow, X: V1, Y: V2, Carry&: HasOverflow);
2314
2315 // The intrinsic call will detect overflow when the value is > UINT_MAX,
2316 // however, since the original builtin had a signed result, we need to report
2317 // an overflow when the result is greater than INT_MAX.
2318 auto IntMax = llvm::APInt::getSignedMaxValue(numBits: ResultInfo.Width);
2319 llvm::Value *IntMaxValue = llvm::ConstantInt::get(Ty: Result->getType(), V: IntMax);
2320
2321 llvm::Value *IntMaxOverflow = CGF.Builder.CreateICmpUGT(LHS: Result, RHS: IntMaxValue);
2322 HasOverflow = CGF.Builder.CreateOr(LHS: HasOverflow, RHS: IntMaxOverflow);
2323
2324 bool isVolatile =
2325 ResultArg->getType()->getPointeeType().isVolatileQualified();
2326 Address ResultPtr = CGF.EmitPointerWithAlignment(Addr: ResultArg);
2327 CGF.Builder.CreateStore(Val: CGF.EmitToMemory(Value: Result, Ty: ResultQTy), Addr: ResultPtr,
2328 IsVolatile: isVolatile);
2329 return RValue::get(V: HasOverflow);
2330}
2331
2332/// Determine if a binop is a checked mixed-sign multiply we can specialize.
2333static bool isSpecialMixedSignMultiply(unsigned BuiltinID,
2334 WidthAndSignedness Op1Info,
2335 WidthAndSignedness Op2Info,
2336 WidthAndSignedness ResultInfo) {
2337 return BuiltinID == Builtin::BI__builtin_mul_overflow &&
2338 std::max(a: Op1Info.Width, b: Op2Info.Width) >= ResultInfo.Width &&
2339 Op1Info.Signed != Op2Info.Signed;
2340}
2341
2342/// Emit a checked mixed-sign multiply. This is a cheaper specialization of
2343/// the generic checked-binop irgen.
2344static RValue
2345EmitCheckedMixedSignMultiply(CodeGenFunction &CGF, const clang::Expr *Op1,
2346 WidthAndSignedness Op1Info, const clang::Expr *Op2,
2347 WidthAndSignedness Op2Info,
2348 const clang::Expr *ResultArg, QualType ResultQTy,
2349 WidthAndSignedness ResultInfo) {
2350 assert(isSpecialMixedSignMultiply(Builtin::BI__builtin_mul_overflow, Op1Info,
2351 Op2Info, ResultInfo) &&
2352 "Not a mixed-sign multipliction we can specialize");
2353
2354 // Emit the signed and unsigned operands.
2355 const clang::Expr *SignedOp = Op1Info.Signed ? Op1 : Op2;
2356 const clang::Expr *UnsignedOp = Op1Info.Signed ? Op2 : Op1;
2357 llvm::Value *Signed = CGF.EmitScalarExpr(E: SignedOp);
2358 llvm::Value *Unsigned = CGF.EmitScalarExpr(E: UnsignedOp);
2359 unsigned SignedOpWidth = Op1Info.Signed ? Op1Info.Width : Op2Info.Width;
2360 unsigned UnsignedOpWidth = Op1Info.Signed ? Op2Info.Width : Op1Info.Width;
2361
2362 // One of the operands may be smaller than the other. If so, [s|z]ext it.
2363 if (SignedOpWidth < UnsignedOpWidth)
2364 Signed = CGF.Builder.CreateSExt(V: Signed, DestTy: Unsigned->getType(), Name: "op.sext");
2365 if (UnsignedOpWidth < SignedOpWidth)
2366 Unsigned = CGF.Builder.CreateZExt(V: Unsigned, DestTy: Signed->getType(), Name: "op.zext");
2367
2368 llvm::Type *OpTy = Signed->getType();
2369 llvm::Value *Zero = llvm::Constant::getNullValue(Ty: OpTy);
2370 Address ResultPtr = CGF.EmitPointerWithAlignment(Addr: ResultArg);
2371 llvm::Type *ResTy = CGF.getTypes().ConvertType(T: ResultQTy);
2372 unsigned OpWidth = std::max(a: Op1Info.Width, b: Op2Info.Width);
2373
2374 // Take the absolute value of the signed operand.
2375 llvm::Value *IsNegative = CGF.Builder.CreateICmpSLT(LHS: Signed, RHS: Zero);
2376 llvm::Value *AbsOfNegative = CGF.Builder.CreateSub(LHS: Zero, RHS: Signed);
2377 llvm::Value *AbsSigned =
2378 CGF.Builder.CreateSelect(C: IsNegative, True: AbsOfNegative, False: Signed);
2379
2380 // Perform a checked unsigned multiplication.
2381 llvm::Value *UnsignedOverflow;
2382 llvm::Value *UnsignedResult =
2383 EmitOverflowIntrinsic(CGF, IntrinsicID: Intrinsic::umul_with_overflow, X: AbsSigned,
2384 Y: Unsigned, Carry&: UnsignedOverflow);
2385
2386 llvm::Value *Overflow, *Result;
2387 if (ResultInfo.Signed) {
2388 // Signed overflow occurs if the result is greater than INT_MAX or lesser
2389 // than INT_MIN, i.e when |Result| > (INT_MAX + IsNegative).
2390 auto IntMax =
2391 llvm::APInt::getSignedMaxValue(numBits: ResultInfo.Width).zext(width: OpWidth);
2392 llvm::Value *MaxResult =
2393 CGF.Builder.CreateAdd(LHS: llvm::ConstantInt::get(Ty: OpTy, V: IntMax),
2394 RHS: CGF.Builder.CreateZExt(V: IsNegative, DestTy: OpTy));
2395 llvm::Value *SignedOverflow =
2396 CGF.Builder.CreateICmpUGT(LHS: UnsignedResult, RHS: MaxResult);
2397 Overflow = CGF.Builder.CreateOr(LHS: UnsignedOverflow, RHS: SignedOverflow);
2398
2399 // Prepare the signed result (possibly by negating it).
2400 llvm::Value *NegativeResult = CGF.Builder.CreateNeg(V: UnsignedResult);
2401 llvm::Value *SignedResult =
2402 CGF.Builder.CreateSelect(C: IsNegative, True: NegativeResult, False: UnsignedResult);
2403 Result = CGF.Builder.CreateTrunc(V: SignedResult, DestTy: ResTy);
2404 } else {
2405 // Unsigned overflow occurs if the result is < 0 or greater than UINT_MAX.
2406 llvm::Value *Underflow = CGF.Builder.CreateAnd(
2407 LHS: IsNegative, RHS: CGF.Builder.CreateIsNotNull(Arg: UnsignedResult));
2408 Overflow = CGF.Builder.CreateOr(LHS: UnsignedOverflow, RHS: Underflow);
2409 if (ResultInfo.Width < OpWidth) {
2410 auto IntMax =
2411 llvm::APInt::getMaxValue(numBits: ResultInfo.Width).zext(width: OpWidth);
2412 llvm::Value *TruncOverflow = CGF.Builder.CreateICmpUGT(
2413 LHS: UnsignedResult, RHS: llvm::ConstantInt::get(Ty: OpTy, V: IntMax));
2414 Overflow = CGF.Builder.CreateOr(LHS: Overflow, RHS: TruncOverflow);
2415 }
2416
2417 // Negate the product if it would be negative in infinite precision.
2418 Result = CGF.Builder.CreateSelect(
2419 C: IsNegative, True: CGF.Builder.CreateNeg(V: UnsignedResult), False: UnsignedResult);
2420
2421 Result = CGF.Builder.CreateTrunc(V: Result, DestTy: ResTy);
2422 }
2423 assert(Overflow && Result && "Missing overflow or result");
2424
2425 bool isVolatile =
2426 ResultArg->getType()->getPointeeType().isVolatileQualified();
2427 CGF.Builder.CreateStore(Val: CGF.EmitToMemory(Value: Result, Ty: ResultQTy), Addr: ResultPtr,
2428 IsVolatile: isVolatile);
2429 return RValue::get(V: Overflow);
2430}
2431
2432static bool
2433TypeRequiresBuiltinLaunderImp(const ASTContext &Ctx, QualType Ty,
2434 llvm::SmallPtrSetImpl<const Decl *> &Seen) {
2435 if (const auto *Arr = Ctx.getAsArrayType(T: Ty))
2436 Ty = Ctx.getBaseElementType(VAT: Arr);
2437
2438 const auto *Record = Ty->getAsCXXRecordDecl();
2439 if (!Record)
2440 return false;
2441
2442 // We've already checked this type, or are in the process of checking it.
2443 if (!Seen.insert(Ptr: Record).second)
2444 return false;
2445
2446 assert(Record->hasDefinition() &&
2447 "Incomplete types should already be diagnosed");
2448
2449 if (Record->isDynamicClass())
2450 return true;
2451
2452 for (FieldDecl *F : Record->fields()) {
2453 if (TypeRequiresBuiltinLaunderImp(Ctx, Ty: F->getType(), Seen))
2454 return true;
2455 }
2456 return false;
2457}
2458
2459/// Determine if the specified type requires laundering by checking if it is a
2460/// dynamic class type or contains a subobject which is a dynamic class type.
2461static bool TypeRequiresBuiltinLaunder(CodeGenModule &CGM, QualType Ty) {
2462 if (!CGM.getCodeGenOpts().StrictVTablePointers)
2463 return false;
2464 llvm::SmallPtrSet<const Decl *, 16> Seen;
2465 return TypeRequiresBuiltinLaunderImp(Ctx: CGM.getContext(), Ty, Seen);
2466}
2467
2468RValue CodeGenFunction::emitRotate(const CallExpr *E, bool IsRotateRight) {
2469 llvm::Value *Src = EmitScalarExpr(E: E->getArg(Arg: 0));
2470 llvm::Value *ShiftAmt = EmitScalarExpr(E: E->getArg(Arg: 1));
2471
2472 // The builtin's shift arg may have a different type than the source arg and
2473 // result, but the LLVM intrinsic uses the same type for all values.
2474 llvm::Type *Ty = Src->getType();
2475 ShiftAmt = Builder.CreateIntCast(V: ShiftAmt, DestTy: Ty, isSigned: false);
2476
2477 // Rotate is a special case of LLVM funnel shift - 1st 2 args are the same.
2478 unsigned IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;
2479 Function *F = CGM.getIntrinsic(IID, Tys: Ty);
2480 return RValue::get(V: Builder.CreateCall(Callee: F, Args: { Src, Src, ShiftAmt }));
2481}
2482
2483// Map math builtins for long-double to f128 version.
2484static unsigned mutateLongDoubleBuiltin(unsigned BuiltinID) {
2485 switch (BuiltinID) {
2486#define MUTATE_LDBL(func) \
2487 case Builtin::BI__builtin_##func##l: \
2488 return Builtin::BI__builtin_##func##f128;
2489 MUTATE_LDBL(sqrt)
2490 MUTATE_LDBL(cbrt)
2491 MUTATE_LDBL(fabs)
2492 MUTATE_LDBL(log)
2493 MUTATE_LDBL(log2)
2494 MUTATE_LDBL(log10)
2495 MUTATE_LDBL(log1p)
2496 MUTATE_LDBL(logb)
2497 MUTATE_LDBL(exp)
2498 MUTATE_LDBL(exp2)
2499 MUTATE_LDBL(expm1)
2500 MUTATE_LDBL(fdim)
2501 MUTATE_LDBL(hypot)
2502 MUTATE_LDBL(ilogb)
2503 MUTATE_LDBL(pow)
2504 MUTATE_LDBL(fmin)
2505 MUTATE_LDBL(fmax)
2506 MUTATE_LDBL(ceil)
2507 MUTATE_LDBL(trunc)
2508 MUTATE_LDBL(rint)
2509 MUTATE_LDBL(nearbyint)
2510 MUTATE_LDBL(round)
2511 MUTATE_LDBL(floor)
2512 MUTATE_LDBL(lround)
2513 MUTATE_LDBL(llround)
2514 MUTATE_LDBL(lrint)
2515 MUTATE_LDBL(llrint)
2516 MUTATE_LDBL(fmod)
2517 MUTATE_LDBL(modf)
2518 MUTATE_LDBL(nan)
2519 MUTATE_LDBL(nans)
2520 MUTATE_LDBL(inf)
2521 MUTATE_LDBL(fma)
2522 MUTATE_LDBL(sin)
2523 MUTATE_LDBL(cos)
2524 MUTATE_LDBL(tan)
2525 MUTATE_LDBL(sinh)
2526 MUTATE_LDBL(cosh)
2527 MUTATE_LDBL(tanh)
2528 MUTATE_LDBL(asin)
2529 MUTATE_LDBL(acos)
2530 MUTATE_LDBL(atan)
2531 MUTATE_LDBL(asinh)
2532 MUTATE_LDBL(acosh)
2533 MUTATE_LDBL(atanh)
2534 MUTATE_LDBL(atan2)
2535 MUTATE_LDBL(erf)
2536 MUTATE_LDBL(erfc)
2537 MUTATE_LDBL(ldexp)
2538 MUTATE_LDBL(frexp)
2539 MUTATE_LDBL(huge_val)
2540 MUTATE_LDBL(copysign)
2541 MUTATE_LDBL(nextafter)
2542 MUTATE_LDBL(nexttoward)
2543 MUTATE_LDBL(remainder)
2544 MUTATE_LDBL(remquo)
2545 MUTATE_LDBL(scalbln)
2546 MUTATE_LDBL(scalbn)
2547 MUTATE_LDBL(tgamma)
2548 MUTATE_LDBL(lgamma)
2549#undef MUTATE_LDBL
2550 default:
2551 return BuiltinID;
2552 }
2553}
2554
2555static Value *tryUseTestFPKind(CodeGenFunction &CGF, unsigned BuiltinID,
2556 Value *V) {
2557 if (CGF.Builder.getIsFPConstrained() &&
2558 CGF.Builder.getDefaultConstrainedExcept() != fp::ebIgnore) {
2559 if (Value *Result =
2560 CGF.getTargetHooks().testFPKind(V, BuiltinID, Builder&: CGF.Builder, CGM&: CGF.CGM))
2561 return Result;
2562 }
2563 return nullptr;
2564}
2565
2566static RValue EmitHipStdParUnsupportedBuiltin(CodeGenFunction *CGF,
2567 const FunctionDecl *FD) {
2568 auto Name = FD->getNameAsString() + "__hipstdpar_unsupported";
2569 auto FnTy = CGF->CGM.getTypes().GetFunctionType(GD: FD);
2570 auto UBF = CGF->CGM.getModule().getOrInsertFunction(Name, T: FnTy);
2571
2572 SmallVector<Value *, 16> Args;
2573 for (auto &&FormalTy : FnTy->params())
2574 Args.push_back(Elt: llvm::PoisonValue::get(T: FormalTy));
2575
2576 return RValue::get(V: CGF->Builder.CreateCall(Callee: UBF, Args));
2577}
2578
2579RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
2580 const CallExpr *E,
2581 ReturnValueSlot ReturnValue) {
2582 assert(!getContext().BuiltinInfo.isImmediate(BuiltinID) &&
2583 "Should not codegen for consteval builtins");
2584
2585 const FunctionDecl *FD = GD.getDecl()->getAsFunction();
2586 // See if we can constant fold this builtin. If so, don't emit it at all.
2587 // TODO: Extend this handling to all builtin calls that we can constant-fold.
2588 Expr::EvalResult Result;
2589 if (E->isPRValue() && E->EvaluateAsRValue(Result, Ctx: CGM.getContext()) &&
2590 !Result.hasSideEffects()) {
2591 if (Result.Val.isInt())
2592 return RValue::get(V: llvm::ConstantInt::get(Context&: getLLVMContext(),
2593 V: Result.Val.getInt()));
2594 if (Result.Val.isFloat())
2595 return RValue::get(V: llvm::ConstantFP::get(Context&: getLLVMContext(),
2596 V: Result.Val.getFloat()));
2597 }
2598
2599 // If current long-double semantics is IEEE 128-bit, replace math builtins
2600 // of long-double with f128 equivalent.
2601 // TODO: This mutation should also be applied to other targets other than PPC,
2602 // after backend supports IEEE 128-bit style libcalls.
2603 if (getTarget().getTriple().isPPC64() &&
2604 &getTarget().getLongDoubleFormat() == &llvm::APFloat::IEEEquad())
2605 BuiltinID = mutateLongDoubleBuiltin(BuiltinID);
2606
2607 // If the builtin has been declared explicitly with an assembler label,
2608 // disable the specialized emitting below. Ideally we should communicate the
2609 // rename in IR, or at least avoid generating the intrinsic calls that are
2610 // likely to get lowered to the renamed library functions.
2611 const unsigned BuiltinIDIfNoAsmLabel =
2612 FD->hasAttr<AsmLabelAttr>() ? 0 : BuiltinID;
2613
2614 std::optional<bool> ErrnoOverriden;
2615 // ErrnoOverriden is true if math-errno is overriden via the
2616 // '#pragma float_control(precise, on)'. This pragma disables fast-math,
2617 // which implies math-errno.
2618 if (E->hasStoredFPFeatures()) {
2619 FPOptionsOverride OP = E->getFPFeatures();
2620 if (OP.hasMathErrnoOverride())
2621 ErrnoOverriden = OP.getMathErrnoOverride();
2622 }
2623 // True if 'attribute__((optnone))' is used. This attribute overrides
2624 // fast-math which implies math-errno.
2625 bool OptNone = CurFuncDecl && CurFuncDecl->hasAttr<OptimizeNoneAttr>();
2626
2627 // True if we are compiling at -O2 and errno has been disabled
2628 // using the '#pragma float_control(precise, off)', and
2629 // attribute opt-none hasn't been seen.
2630 bool ErrnoOverridenToFalseWithOpt =
2631 ErrnoOverriden.has_value() && !ErrnoOverriden.value() && !OptNone &&
2632 CGM.getCodeGenOpts().OptimizationLevel != 0;
2633
2634 // There are LLVM math intrinsics/instructions corresponding to math library
2635 // functions except the LLVM op will never set errno while the math library
2636 // might. Also, math builtins have the same semantics as their math library
2637 // twins. Thus, we can transform math library and builtin calls to their
2638 // LLVM counterparts if the call is marked 'const' (known to never set errno).
2639 // In case FP exceptions are enabled, the experimental versions of the
2640 // intrinsics model those.
2641 bool ConstAlways =
2642 getContext().BuiltinInfo.isConst(ID: BuiltinID);
2643
2644 // There's a special case with the fma builtins where they are always const
2645 // if the target environment is GNU or the target is OS is Windows and we're
2646 // targeting the MSVCRT.dll environment.
2647 // FIXME: This list can be become outdated. Need to find a way to get it some
2648 // other way.
2649 switch (BuiltinID) {
2650 case Builtin::BI__builtin_fma:
2651 case Builtin::BI__builtin_fmaf:
2652 case Builtin::BI__builtin_fmal:
2653 case Builtin::BI__builtin_fmaf16:
2654 case Builtin::BIfma:
2655 case Builtin::BIfmaf:
2656 case Builtin::BIfmal: {
2657 auto &Trip = CGM.getTriple();
2658 if (Trip.isGNUEnvironment() || Trip.isOSMSVCRT())
2659 ConstAlways = true;
2660 break;
2661 }
2662 default:
2663 break;
2664 }
2665
2666 bool ConstWithoutErrnoAndExceptions =
2667 getContext().BuiltinInfo.isConstWithoutErrnoAndExceptions(ID: BuiltinID);
2668 bool ConstWithoutExceptions =
2669 getContext().BuiltinInfo.isConstWithoutExceptions(ID: BuiltinID);
2670
2671 // ConstAttr is enabled in fast-math mode. In fast-math mode, math-errno is
2672 // disabled.
2673 // Math intrinsics are generated only when math-errno is disabled. Any pragmas
2674 // or attributes that affect math-errno should prevent or allow math
2675 // intrinsics to be generated. Intrinsics are generated:
2676 // 1- In fast math mode, unless math-errno is overriden
2677 // via '#pragma float_control(precise, on)', or via an
2678 // 'attribute__((optnone))'.
2679 // 2- If math-errno was enabled on command line but overriden
2680 // to false via '#pragma float_control(precise, off))' and
2681 // 'attribute__((optnone))' hasn't been used.
2682 // 3- If we are compiling with optimization and errno has been disabled
2683 // via '#pragma float_control(precise, off)', and
2684 // 'attribute__((optnone))' hasn't been used.
2685
2686 bool ConstWithoutErrnoOrExceptions =
2687 ConstWithoutErrnoAndExceptions || ConstWithoutExceptions;
2688 bool GenerateIntrinsics =
2689 (ConstAlways && !OptNone) ||
2690 (!getLangOpts().MathErrno &&
2691 !(ErrnoOverriden.has_value() && ErrnoOverriden.value()) && !OptNone);
2692 if (!GenerateIntrinsics) {
2693 GenerateIntrinsics =
2694 ConstWithoutErrnoOrExceptions && !ConstWithoutErrnoAndExceptions;
2695 if (!GenerateIntrinsics)
2696 GenerateIntrinsics =
2697 ConstWithoutErrnoOrExceptions &&
2698 (!getLangOpts().MathErrno &&
2699 !(ErrnoOverriden.has_value() && ErrnoOverriden.value()) && !OptNone);
2700 if (!GenerateIntrinsics)
2701 GenerateIntrinsics =
2702 ConstWithoutErrnoOrExceptions && ErrnoOverridenToFalseWithOpt;
2703 }
2704 if (GenerateIntrinsics) {
2705 switch (BuiltinIDIfNoAsmLabel) {
2706 case Builtin::BIacos:
2707 case Builtin::BIacosf:
2708 case Builtin::BIacosl:
2709 case Builtin::BI__builtin_acos:
2710 case Builtin::BI__builtin_acosf:
2711 case Builtin::BI__builtin_acosf16:
2712 case Builtin::BI__builtin_acosl:
2713 case Builtin::BI__builtin_acosf128:
2714 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(
2715 CGF&: *this, E, IntrinsicID: Intrinsic::acos, ConstrainedIntrinsicID: Intrinsic::experimental_constrained_acos));
2716
2717 case Builtin::BIasin:
2718 case Builtin::BIasinf:
2719 case Builtin::BIasinl:
2720 case Builtin::BI__builtin_asin:
2721 case Builtin::BI__builtin_asinf:
2722 case Builtin::BI__builtin_asinf16:
2723 case Builtin::BI__builtin_asinl:
2724 case Builtin::BI__builtin_asinf128:
2725 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(
2726 CGF&: *this, E, IntrinsicID: Intrinsic::asin, ConstrainedIntrinsicID: Intrinsic::experimental_constrained_asin));
2727
2728 case Builtin::BIatan:
2729 case Builtin::BIatanf:
2730 case Builtin::BIatanl:
2731 case Builtin::BI__builtin_atan:
2732 case Builtin::BI__builtin_atanf:
2733 case Builtin::BI__builtin_atanf16:
2734 case Builtin::BI__builtin_atanl:
2735 case Builtin::BI__builtin_atanf128:
2736 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(
2737 CGF&: *this, E, IntrinsicID: Intrinsic::atan, ConstrainedIntrinsicID: Intrinsic::experimental_constrained_atan));
2738
2739 case Builtin::BIatan2:
2740 case Builtin::BIatan2f:
2741 case Builtin::BIatan2l:
2742 case Builtin::BI__builtin_atan2:
2743 case Builtin::BI__builtin_atan2f:
2744 case Builtin::BI__builtin_atan2f16:
2745 case Builtin::BI__builtin_atan2l:
2746 case Builtin::BI__builtin_atan2f128:
2747 return RValue::get(V: emitBinaryMaybeConstrainedFPBuiltin(
2748 CGF&: *this, E, IntrinsicID: Intrinsic::atan2,
2749 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_atan2));
2750
2751 case Builtin::BIceil:
2752 case Builtin::BIceilf:
2753 case Builtin::BIceill:
2754 case Builtin::BI__builtin_ceil:
2755 case Builtin::BI__builtin_ceilf:
2756 case Builtin::BI__builtin_ceilf16:
2757 case Builtin::BI__builtin_ceill:
2758 case Builtin::BI__builtin_ceilf128:
2759 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
2760 IntrinsicID: Intrinsic::ceil,
2761 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_ceil));
2762
2763 case Builtin::BIcopysign:
2764 case Builtin::BIcopysignf:
2765 case Builtin::BIcopysignl:
2766 case Builtin::BI__builtin_copysign:
2767 case Builtin::BI__builtin_copysignf:
2768 case Builtin::BI__builtin_copysignf16:
2769 case Builtin::BI__builtin_copysignl:
2770 case Builtin::BI__builtin_copysignf128:
2771 return RValue::get(
2772 V: emitBuiltinWithOneOverloadedType<2>(CGF&: *this, E, IntrinsicID: Intrinsic::copysign));
2773
2774 case Builtin::BIcos:
2775 case Builtin::BIcosf:
2776 case Builtin::BIcosl:
2777 case Builtin::BI__builtin_cos:
2778 case Builtin::BI__builtin_cosf:
2779 case Builtin::BI__builtin_cosf16:
2780 case Builtin::BI__builtin_cosl:
2781 case Builtin::BI__builtin_cosf128:
2782 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
2783 IntrinsicID: Intrinsic::cos,
2784 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_cos));
2785
2786 case Builtin::BIcosh:
2787 case Builtin::BIcoshf:
2788 case Builtin::BIcoshl:
2789 case Builtin::BI__builtin_cosh:
2790 case Builtin::BI__builtin_coshf:
2791 case Builtin::BI__builtin_coshf16:
2792 case Builtin::BI__builtin_coshl:
2793 case Builtin::BI__builtin_coshf128:
2794 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(
2795 CGF&: *this, E, IntrinsicID: Intrinsic::cosh, ConstrainedIntrinsicID: Intrinsic::experimental_constrained_cosh));
2796
2797 case Builtin::BIexp:
2798 case Builtin::BIexpf:
2799 case Builtin::BIexpl:
2800 case Builtin::BI__builtin_exp:
2801 case Builtin::BI__builtin_expf:
2802 case Builtin::BI__builtin_expf16:
2803 case Builtin::BI__builtin_expl:
2804 case Builtin::BI__builtin_expf128:
2805 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
2806 IntrinsicID: Intrinsic::exp,
2807 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_exp));
2808
2809 case Builtin::BIexp2:
2810 case Builtin::BIexp2f:
2811 case Builtin::BIexp2l:
2812 case Builtin::BI__builtin_exp2:
2813 case Builtin::BI__builtin_exp2f:
2814 case Builtin::BI__builtin_exp2f16:
2815 case Builtin::BI__builtin_exp2l:
2816 case Builtin::BI__builtin_exp2f128:
2817 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
2818 IntrinsicID: Intrinsic::exp2,
2819 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_exp2));
2820 case Builtin::BI__builtin_exp10:
2821 case Builtin::BI__builtin_exp10f:
2822 case Builtin::BI__builtin_exp10f16:
2823 case Builtin::BI__builtin_exp10l:
2824 case Builtin::BI__builtin_exp10f128: {
2825 // TODO: strictfp support
2826 if (Builder.getIsFPConstrained())
2827 break;
2828 return RValue::get(
2829 V: emitBuiltinWithOneOverloadedType<1>(CGF&: *this, E, IntrinsicID: Intrinsic::exp10));
2830 }
2831 case Builtin::BIfabs:
2832 case Builtin::BIfabsf:
2833 case Builtin::BIfabsl:
2834 case Builtin::BI__builtin_fabs:
2835 case Builtin::BI__builtin_fabsf:
2836 case Builtin::BI__builtin_fabsf16:
2837 case Builtin::BI__builtin_fabsl:
2838 case Builtin::BI__builtin_fabsf128:
2839 return RValue::get(
2840 V: emitBuiltinWithOneOverloadedType<1>(CGF&: *this, E, IntrinsicID: Intrinsic::fabs));
2841
2842 case Builtin::BIfloor:
2843 case Builtin::BIfloorf:
2844 case Builtin::BIfloorl:
2845 case Builtin::BI__builtin_floor:
2846 case Builtin::BI__builtin_floorf:
2847 case Builtin::BI__builtin_floorf16:
2848 case Builtin::BI__builtin_floorl:
2849 case Builtin::BI__builtin_floorf128:
2850 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
2851 IntrinsicID: Intrinsic::floor,
2852 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_floor));
2853
2854 case Builtin::BIfma:
2855 case Builtin::BIfmaf:
2856 case Builtin::BIfmal:
2857 case Builtin::BI__builtin_fma:
2858 case Builtin::BI__builtin_fmaf:
2859 case Builtin::BI__builtin_fmaf16:
2860 case Builtin::BI__builtin_fmal:
2861 case Builtin::BI__builtin_fmaf128:
2862 return RValue::get(V: emitTernaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
2863 IntrinsicID: Intrinsic::fma,
2864 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_fma));
2865
2866 case Builtin::BIfmax:
2867 case Builtin::BIfmaxf:
2868 case Builtin::BIfmaxl:
2869 case Builtin::BI__builtin_fmax:
2870 case Builtin::BI__builtin_fmaxf:
2871 case Builtin::BI__builtin_fmaxf16:
2872 case Builtin::BI__builtin_fmaxl:
2873 case Builtin::BI__builtin_fmaxf128:
2874 return RValue::get(V: emitBinaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
2875 IntrinsicID: Intrinsic::maxnum,
2876 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_maxnum));
2877
2878 case Builtin::BIfmin:
2879 case Builtin::BIfminf:
2880 case Builtin::BIfminl:
2881 case Builtin::BI__builtin_fmin:
2882 case Builtin::BI__builtin_fminf:
2883 case Builtin::BI__builtin_fminf16:
2884 case Builtin::BI__builtin_fminl:
2885 case Builtin::BI__builtin_fminf128:
2886 return RValue::get(V: emitBinaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
2887 IntrinsicID: Intrinsic::minnum,
2888 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_minnum));
2889
2890 case Builtin::BIfmaximum_num:
2891 case Builtin::BIfmaximum_numf:
2892 case Builtin::BIfmaximum_numl:
2893 case Builtin::BI__builtin_fmaximum_num:
2894 case Builtin::BI__builtin_fmaximum_numf:
2895 case Builtin::BI__builtin_fmaximum_numf16:
2896 case Builtin::BI__builtin_fmaximum_numl:
2897 case Builtin::BI__builtin_fmaximum_numf128:
2898 return RValue::get(
2899 V: emitBuiltinWithOneOverloadedType<2>(CGF&: *this, E, IntrinsicID: Intrinsic::maximumnum));
2900
2901 case Builtin::BIfminimum_num:
2902 case Builtin::BIfminimum_numf:
2903 case Builtin::BIfminimum_numl:
2904 case Builtin::BI__builtin_fminimum_num:
2905 case Builtin::BI__builtin_fminimum_numf:
2906 case Builtin::BI__builtin_fminimum_numf16:
2907 case Builtin::BI__builtin_fminimum_numl:
2908 case Builtin::BI__builtin_fminimum_numf128:
2909 return RValue::get(
2910 V: emitBuiltinWithOneOverloadedType<2>(CGF&: *this, E, IntrinsicID: Intrinsic::minimumnum));
2911
2912 // fmod() is a special-case. It maps to the frem instruction rather than an
2913 // LLVM intrinsic.
2914 case Builtin::BIfmod:
2915 case Builtin::BIfmodf:
2916 case Builtin::BIfmodl:
2917 case Builtin::BI__builtin_fmod:
2918 case Builtin::BI__builtin_fmodf:
2919 case Builtin::BI__builtin_fmodf16:
2920 case Builtin::BI__builtin_fmodl:
2921 case Builtin::BI__builtin_fmodf128:
2922 case Builtin::BI__builtin_elementwise_fmod: {
2923 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
2924 Value *Arg1 = EmitScalarExpr(E: E->getArg(Arg: 0));
2925 Value *Arg2 = EmitScalarExpr(E: E->getArg(Arg: 1));
2926 return RValue::get(V: Builder.CreateFRem(L: Arg1, R: Arg2, Name: "fmod"));
2927 }
2928
2929 case Builtin::BIlog:
2930 case Builtin::BIlogf:
2931 case Builtin::BIlogl:
2932 case Builtin::BI__builtin_log:
2933 case Builtin::BI__builtin_logf:
2934 case Builtin::BI__builtin_logf16:
2935 case Builtin::BI__builtin_logl:
2936 case Builtin::BI__builtin_logf128:
2937 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
2938 IntrinsicID: Intrinsic::log,
2939 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_log));
2940
2941 case Builtin::BIlog10:
2942 case Builtin::BIlog10f:
2943 case Builtin::BIlog10l:
2944 case Builtin::BI__builtin_log10:
2945 case Builtin::BI__builtin_log10f:
2946 case Builtin::BI__builtin_log10f16:
2947 case Builtin::BI__builtin_log10l:
2948 case Builtin::BI__builtin_log10f128:
2949 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
2950 IntrinsicID: Intrinsic::log10,
2951 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_log10));
2952
2953 case Builtin::BIlog2:
2954 case Builtin::BIlog2f:
2955 case Builtin::BIlog2l:
2956 case Builtin::BI__builtin_log2:
2957 case Builtin::BI__builtin_log2f:
2958 case Builtin::BI__builtin_log2f16:
2959 case Builtin::BI__builtin_log2l:
2960 case Builtin::BI__builtin_log2f128:
2961 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
2962 IntrinsicID: Intrinsic::log2,
2963 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_log2));
2964
2965 case Builtin::BInearbyint:
2966 case Builtin::BInearbyintf:
2967 case Builtin::BInearbyintl:
2968 case Builtin::BI__builtin_nearbyint:
2969 case Builtin::BI__builtin_nearbyintf:
2970 case Builtin::BI__builtin_nearbyintl:
2971 case Builtin::BI__builtin_nearbyintf128:
2972 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
2973 IntrinsicID: Intrinsic::nearbyint,
2974 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_nearbyint));
2975
2976 case Builtin::BIpow:
2977 case Builtin::BIpowf:
2978 case Builtin::BIpowl:
2979 case Builtin::BI__builtin_pow:
2980 case Builtin::BI__builtin_powf:
2981 case Builtin::BI__builtin_powf16:
2982 case Builtin::BI__builtin_powl:
2983 case Builtin::BI__builtin_powf128:
2984 return RValue::get(V: emitBinaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
2985 IntrinsicID: Intrinsic::pow,
2986 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_pow));
2987
2988 case Builtin::BIrint:
2989 case Builtin::BIrintf:
2990 case Builtin::BIrintl:
2991 case Builtin::BI__builtin_rint:
2992 case Builtin::BI__builtin_rintf:
2993 case Builtin::BI__builtin_rintf16:
2994 case Builtin::BI__builtin_rintl:
2995 case Builtin::BI__builtin_rintf128:
2996 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
2997 IntrinsicID: Intrinsic::rint,
2998 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_rint));
2999
3000 case Builtin::BIround:
3001 case Builtin::BIroundf:
3002 case Builtin::BIroundl:
3003 case Builtin::BI__builtin_round:
3004 case Builtin::BI__builtin_roundf:
3005 case Builtin::BI__builtin_roundf16:
3006 case Builtin::BI__builtin_roundl:
3007 case Builtin::BI__builtin_roundf128:
3008 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
3009 IntrinsicID: Intrinsic::round,
3010 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_round));
3011
3012 case Builtin::BIroundeven:
3013 case Builtin::BIroundevenf:
3014 case Builtin::BIroundevenl:
3015 case Builtin::BI__builtin_roundeven:
3016 case Builtin::BI__builtin_roundevenf:
3017 case Builtin::BI__builtin_roundevenf16:
3018 case Builtin::BI__builtin_roundevenl:
3019 case Builtin::BI__builtin_roundevenf128:
3020 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
3021 IntrinsicID: Intrinsic::roundeven,
3022 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_roundeven));
3023
3024 case Builtin::BIsin:
3025 case Builtin::BIsinf:
3026 case Builtin::BIsinl:
3027 case Builtin::BI__builtin_sin:
3028 case Builtin::BI__builtin_sinf:
3029 case Builtin::BI__builtin_sinf16:
3030 case Builtin::BI__builtin_sinl:
3031 case Builtin::BI__builtin_sinf128:
3032 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
3033 IntrinsicID: Intrinsic::sin,
3034 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_sin));
3035
3036 case Builtin::BIsinh:
3037 case Builtin::BIsinhf:
3038 case Builtin::BIsinhl:
3039 case Builtin::BI__builtin_sinh:
3040 case Builtin::BI__builtin_sinhf:
3041 case Builtin::BI__builtin_sinhf16:
3042 case Builtin::BI__builtin_sinhl:
3043 case Builtin::BI__builtin_sinhf128:
3044 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(
3045 CGF&: *this, E, IntrinsicID: Intrinsic::sinh, ConstrainedIntrinsicID: Intrinsic::experimental_constrained_sinh));
3046
3047 case Builtin::BI__builtin_sincospi:
3048 case Builtin::BI__builtin_sincospif:
3049 case Builtin::BI__builtin_sincospil:
3050 if (Builder.getIsFPConstrained())
3051 break; // TODO: Emit constrained sincospi intrinsic once one exists.
3052 emitSincosBuiltin(CGF&: *this, E, IntrinsicID: Intrinsic::sincospi);
3053 return RValue::get(V: nullptr);
3054
3055 case Builtin::BIsincos:
3056 case Builtin::BIsincosf:
3057 case Builtin::BIsincosl:
3058 case Builtin::BI__builtin_sincos:
3059 case Builtin::BI__builtin_sincosf:
3060 case Builtin::BI__builtin_sincosf16:
3061 case Builtin::BI__builtin_sincosl:
3062 case Builtin::BI__builtin_sincosf128:
3063 if (Builder.getIsFPConstrained())
3064 break; // TODO: Emit constrained sincos intrinsic once one exists.
3065 emitSincosBuiltin(CGF&: *this, E, IntrinsicID: Intrinsic::sincos);
3066 return RValue::get(V: nullptr);
3067
3068 case Builtin::BIsqrt:
3069 case Builtin::BIsqrtf:
3070 case Builtin::BIsqrtl:
3071 case Builtin::BI__builtin_sqrt:
3072 case Builtin::BI__builtin_sqrtf:
3073 case Builtin::BI__builtin_sqrtf16:
3074 case Builtin::BI__builtin_sqrtl:
3075 case Builtin::BI__builtin_sqrtf128:
3076 case Builtin::BI__builtin_elementwise_sqrt: {
3077 llvm::Value *Call = emitUnaryMaybeConstrainedFPBuiltin(
3078 CGF&: *this, E, IntrinsicID: Intrinsic::sqrt, ConstrainedIntrinsicID: Intrinsic::experimental_constrained_sqrt);
3079 SetSqrtFPAccuracy(Call);
3080 return RValue::get(V: Call);
3081 }
3082
3083 case Builtin::BItan:
3084 case Builtin::BItanf:
3085 case Builtin::BItanl:
3086 case Builtin::BI__builtin_tan:
3087 case Builtin::BI__builtin_tanf:
3088 case Builtin::BI__builtin_tanf16:
3089 case Builtin::BI__builtin_tanl:
3090 case Builtin::BI__builtin_tanf128:
3091 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(
3092 CGF&: *this, E, IntrinsicID: Intrinsic::tan, ConstrainedIntrinsicID: Intrinsic::experimental_constrained_tan));
3093
3094 case Builtin::BItanh:
3095 case Builtin::BItanhf:
3096 case Builtin::BItanhl:
3097 case Builtin::BI__builtin_tanh:
3098 case Builtin::BI__builtin_tanhf:
3099 case Builtin::BI__builtin_tanhf16:
3100 case Builtin::BI__builtin_tanhl:
3101 case Builtin::BI__builtin_tanhf128:
3102 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(
3103 CGF&: *this, E, IntrinsicID: Intrinsic::tanh, ConstrainedIntrinsicID: Intrinsic::experimental_constrained_tanh));
3104
3105 case Builtin::BItrunc:
3106 case Builtin::BItruncf:
3107 case Builtin::BItruncl:
3108 case Builtin::BI__builtin_trunc:
3109 case Builtin::BI__builtin_truncf:
3110 case Builtin::BI__builtin_truncf16:
3111 case Builtin::BI__builtin_truncl:
3112 case Builtin::BI__builtin_truncf128:
3113 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
3114 IntrinsicID: Intrinsic::trunc,
3115 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_trunc));
3116
3117 case Builtin::BIlround:
3118 case Builtin::BIlroundf:
3119 case Builtin::BIlroundl:
3120 case Builtin::BI__builtin_lround:
3121 case Builtin::BI__builtin_lroundf:
3122 case Builtin::BI__builtin_lroundl:
3123 case Builtin::BI__builtin_lroundf128:
3124 return RValue::get(V: emitMaybeConstrainedFPToIntRoundBuiltin(
3125 CGF&: *this, E, IntrinsicID: Intrinsic::lround,
3126 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_lround));
3127
3128 case Builtin::BIllround:
3129 case Builtin::BIllroundf:
3130 case Builtin::BIllroundl:
3131 case Builtin::BI__builtin_llround:
3132 case Builtin::BI__builtin_llroundf:
3133 case Builtin::BI__builtin_llroundl:
3134 case Builtin::BI__builtin_llroundf128:
3135 return RValue::get(V: emitMaybeConstrainedFPToIntRoundBuiltin(
3136 CGF&: *this, E, IntrinsicID: Intrinsic::llround,
3137 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_llround));
3138
3139 case Builtin::BIlrint:
3140 case Builtin::BIlrintf:
3141 case Builtin::BIlrintl:
3142 case Builtin::BI__builtin_lrint:
3143 case Builtin::BI__builtin_lrintf:
3144 case Builtin::BI__builtin_lrintl:
3145 case Builtin::BI__builtin_lrintf128:
3146 return RValue::get(V: emitMaybeConstrainedFPToIntRoundBuiltin(
3147 CGF&: *this, E, IntrinsicID: Intrinsic::lrint,
3148 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_lrint));
3149
3150 case Builtin::BIllrint:
3151 case Builtin::BIllrintf:
3152 case Builtin::BIllrintl:
3153 case Builtin::BI__builtin_llrint:
3154 case Builtin::BI__builtin_llrintf:
3155 case Builtin::BI__builtin_llrintl:
3156 case Builtin::BI__builtin_llrintf128:
3157 return RValue::get(V: emitMaybeConstrainedFPToIntRoundBuiltin(
3158 CGF&: *this, E, IntrinsicID: Intrinsic::llrint,
3159 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_llrint));
3160 case Builtin::BI__builtin_ldexp:
3161 case Builtin::BI__builtin_ldexpf:
3162 case Builtin::BI__builtin_ldexpl:
3163 case Builtin::BI__builtin_ldexpf16:
3164 case Builtin::BI__builtin_ldexpf128: {
3165 return RValue::get(V: emitBinaryExpMaybeConstrainedFPBuiltin(
3166 CGF&: *this, E, IntrinsicID: Intrinsic::ldexp,
3167 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_ldexp));
3168 }
3169 default:
3170 break;
3171 }
3172 }
3173
3174 // Check NonnullAttribute/NullabilityArg and Alignment.
3175 auto EmitArgCheck = [&](TypeCheckKind Kind, Address A, const Expr *Arg,
3176 unsigned ParmNum) {
3177 Value *Val = A.emitRawPointer(CGF&: *this);
3178 EmitNonNullArgCheck(RV: RValue::get(V: Val), ArgType: Arg->getType(), ArgLoc: Arg->getExprLoc(), AC: FD,
3179 ParmNum);
3180
3181 if (SanOpts.has(K: SanitizerKind::Alignment)) {
3182 SanitizerSet SkippedChecks;
3183 SkippedChecks.set(SanitizerKind::All);
3184 SkippedChecks.clear(K: SanitizerKind::Alignment);
3185 SourceLocation Loc = Arg->getExprLoc();
3186 // Strip an implicit cast.
3187 if (auto *CE = dyn_cast<ImplicitCastExpr>(Val: Arg))
3188 if (CE->getCastKind() == CK_BitCast)
3189 Arg = CE->getSubExpr();
3190 EmitTypeCheck(TCK: Kind, Loc, V: Val, Type: Arg->getType(), Alignment: A.getAlignment(),
3191 SkippedChecks);
3192 }
3193 };
3194
3195 switch (BuiltinIDIfNoAsmLabel) {
3196 default: break;
3197 case Builtin::BI__builtin___CFStringMakeConstantString:
3198 case Builtin::BI__builtin___NSStringMakeConstantString:
3199 return RValue::get(V: ConstantEmitter(*this).emitAbstract(E, T: E->getType()));
3200 case Builtin::BI__builtin_stdarg_start:
3201 case Builtin::BI__builtin_va_start:
3202 case Builtin::BI__va_start:
3203 case Builtin::BI__builtin_c23_va_start:
3204 case Builtin::BI__builtin_va_end:
3205 EmitVAStartEnd(ArgValue: BuiltinID == Builtin::BI__va_start
3206 ? EmitScalarExpr(E: E->getArg(Arg: 0))
3207 : EmitVAListRef(E: E->getArg(Arg: 0)).emitRawPointer(CGF&: *this),
3208 IsStart: BuiltinID != Builtin::BI__builtin_va_end);
3209 return RValue::get(V: nullptr);
3210 case Builtin::BI__builtin_va_copy: {
3211 Value *DstPtr = EmitVAListRef(E: E->getArg(Arg: 0)).emitRawPointer(CGF&: *this);
3212 Value *SrcPtr = EmitVAListRef(E: E->getArg(Arg: 1)).emitRawPointer(CGF&: *this);
3213 Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Intrinsic::vacopy, Tys: {DstPtr->getType()}),
3214 Args: {DstPtr, SrcPtr});
3215 return RValue::get(V: nullptr);
3216 }
3217 case Builtin::BIabs:
3218 case Builtin::BIlabs:
3219 case Builtin::BIllabs:
3220 case Builtin::BI__builtin_abs:
3221 case Builtin::BI__builtin_labs:
3222 case Builtin::BI__builtin_llabs: {
3223 bool SanitizeOverflow = SanOpts.has(K: SanitizerKind::SignedIntegerOverflow);
3224
3225 Value *Result;
3226 switch (getLangOpts().getSignedOverflowBehavior()) {
3227 case LangOptions::SOB_Defined:
3228 Result = EmitAbs(CGF&: *this, ArgValue: EmitScalarExpr(E: E->getArg(Arg: 0)), HasNSW: false);
3229 break;
3230 case LangOptions::SOB_Undefined:
3231 if (!SanitizeOverflow) {
3232 Result = EmitAbs(CGF&: *this, ArgValue: EmitScalarExpr(E: E->getArg(Arg: 0)), HasNSW: true);
3233 break;
3234 }
3235 [[fallthrough]];
3236 case LangOptions::SOB_Trapping:
3237 // TODO: Somehow handle the corner case when the address of abs is taken.
3238 Result = EmitOverflowCheckedAbs(CGF&: *this, E, SanitizeOverflow);
3239 break;
3240 }
3241 return RValue::get(V: Result);
3242 }
3243 case Builtin::BI__builtin_complex: {
3244 Value *Real = EmitScalarExpr(E: E->getArg(Arg: 0));
3245 Value *Imag = EmitScalarExpr(E: E->getArg(Arg: 1));
3246 return RValue::getComplex(C: {Real, Imag});
3247 }
3248 case Builtin::BI__builtin_conj:
3249 case Builtin::BI__builtin_conjf:
3250 case Builtin::BI__builtin_conjl:
3251 case Builtin::BIconj:
3252 case Builtin::BIconjf:
3253 case Builtin::BIconjl: {
3254 ComplexPairTy ComplexVal = EmitComplexExpr(E: E->getArg(Arg: 0));
3255 Value *Real = ComplexVal.first;
3256 Value *Imag = ComplexVal.second;
3257 Imag = Builder.CreateFNeg(V: Imag, Name: "neg");
3258 return RValue::getComplex(C: std::make_pair(x&: Real, y&: Imag));
3259 }
3260 case Builtin::BI__builtin_creal:
3261 case Builtin::BI__builtin_crealf:
3262 case Builtin::BI__builtin_creall:
3263 case Builtin::BIcreal:
3264 case Builtin::BIcrealf:
3265 case Builtin::BIcreall: {
3266 ComplexPairTy ComplexVal = EmitComplexExpr(E: E->getArg(Arg: 0));
3267 return RValue::get(V: ComplexVal.first);
3268 }
3269
3270 case Builtin::BI__builtin_preserve_access_index: {
3271 // Only enabled preserved access index region when debuginfo
3272 // is available as debuginfo is needed to preserve user-level
3273 // access pattern.
3274 if (!getDebugInfo()) {
3275 CGM.Error(loc: E->getExprLoc(), error: "using builtin_preserve_access_index() without -g");
3276 return RValue::get(V: EmitScalarExpr(E: E->getArg(Arg: 0)));
3277 }
3278
3279 // Nested builtin_preserve_access_index() not supported
3280 if (IsInPreservedAIRegion) {
3281 CGM.Error(loc: E->getExprLoc(), error: "nested builtin_preserve_access_index() not supported");
3282 return RValue::get(V: EmitScalarExpr(E: E->getArg(Arg: 0)));
3283 }
3284
3285 IsInPreservedAIRegion = true;
3286 Value *Res = EmitScalarExpr(E: E->getArg(Arg: 0));
3287 IsInPreservedAIRegion = false;
3288 return RValue::get(V: Res);
3289 }
3290
3291 case Builtin::BI__builtin_cimag:
3292 case Builtin::BI__builtin_cimagf:
3293 case Builtin::BI__builtin_cimagl:
3294 case Builtin::BIcimag:
3295 case Builtin::BIcimagf:
3296 case Builtin::BIcimagl: {
3297 ComplexPairTy ComplexVal = EmitComplexExpr(E: E->getArg(Arg: 0));
3298 return RValue::get(V: ComplexVal.second);
3299 }
3300
3301 case Builtin::BI__builtin_clrsb:
3302 case Builtin::BI__builtin_clrsbl:
3303 case Builtin::BI__builtin_clrsbll: {
3304 // clrsb(x) -> clz(x < 0 ? ~x : x) - 1 or
3305 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
3306
3307 llvm::Type *ArgType = ArgValue->getType();
3308 Function *F = CGM.getIntrinsic(IID: Intrinsic::ctlz, Tys: ArgType);
3309
3310 llvm::Type *ResultType = ConvertType(T: E->getType());
3311 Value *Zero = llvm::Constant::getNullValue(Ty: ArgType);
3312 Value *IsNeg = Builder.CreateICmpSLT(LHS: ArgValue, RHS: Zero, Name: "isneg");
3313 Value *Inverse = Builder.CreateNot(V: ArgValue, Name: "not");
3314 Value *Tmp = Builder.CreateSelect(C: IsNeg, True: Inverse, False: ArgValue);
3315 Value *Ctlz = Builder.CreateCall(Callee: F, Args: {Tmp, Builder.getFalse()});
3316 Value *Result = Builder.CreateSub(LHS: Ctlz, RHS: llvm::ConstantInt::get(Ty: ArgType, V: 1));
3317 Result = Builder.CreateIntCast(V: Result, DestTy: ResultType, /*isSigned*/true,
3318 Name: "cast");
3319 return RValue::get(V: Result);
3320 }
3321 case Builtin::BI__builtin_ctzs:
3322 case Builtin::BI__builtin_ctz:
3323 case Builtin::BI__builtin_ctzl:
3324 case Builtin::BI__builtin_ctzll:
3325 case Builtin::BI__builtin_ctzg: {
3326 bool HasFallback = BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_ctzg &&
3327 E->getNumArgs() > 1;
3328
3329 Value *ArgValue =
3330 HasFallback ? EmitScalarExpr(E: E->getArg(Arg: 0))
3331 : EmitCheckedArgForBuiltin(E: E->getArg(Arg: 0), Kind: BCK_CTZPassedZero);
3332
3333 llvm::Type *ArgType = ArgValue->getType();
3334 Function *F = CGM.getIntrinsic(IID: Intrinsic::cttz, Tys: ArgType);
3335
3336 llvm::Type *ResultType = ConvertType(T: E->getType());
3337 Value *ZeroUndef =
3338 Builder.getInt1(V: HasFallback || getTarget().isCLZForZeroUndef());
3339 Value *Result = Builder.CreateCall(Callee: F, Args: {ArgValue, ZeroUndef});
3340 if (Result->getType() != ResultType)
3341 Result =
3342 Builder.CreateIntCast(V: Result, DestTy: ResultType, /*isSigned*/ false, Name: "cast");
3343 if (!HasFallback)
3344 return RValue::get(V: Result);
3345
3346 Value *Zero = Constant::getNullValue(Ty: ArgType);
3347 Value *IsZero = Builder.CreateICmpEQ(LHS: ArgValue, RHS: Zero, Name: "iszero");
3348 Value *FallbackValue = EmitScalarExpr(E: E->getArg(Arg: 1));
3349 Value *ResultOrFallback =
3350 Builder.CreateSelect(C: IsZero, True: FallbackValue, False: Result, Name: "ctzg");
3351 return RValue::get(V: ResultOrFallback);
3352 }
3353 case Builtin::BI__builtin_clzs:
3354 case Builtin::BI__builtin_clz:
3355 case Builtin::BI__builtin_clzl:
3356 case Builtin::BI__builtin_clzll:
3357 case Builtin::BI__builtin_clzg: {
3358 bool HasFallback = BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_clzg &&
3359 E->getNumArgs() > 1;
3360
3361 Value *ArgValue =
3362 HasFallback ? EmitScalarExpr(E: E->getArg(Arg: 0))
3363 : EmitCheckedArgForBuiltin(E: E->getArg(Arg: 0), Kind: BCK_CLZPassedZero);
3364
3365 llvm::Type *ArgType = ArgValue->getType();
3366 Function *F = CGM.getIntrinsic(IID: Intrinsic::ctlz, Tys: ArgType);
3367
3368 llvm::Type *ResultType = ConvertType(T: E->getType());
3369 Value *ZeroUndef =
3370 Builder.getInt1(V: HasFallback || getTarget().isCLZForZeroUndef());
3371 Value *Result = Builder.CreateCall(Callee: F, Args: {ArgValue, ZeroUndef});
3372 if (Result->getType() != ResultType)
3373 Result =
3374 Builder.CreateIntCast(V: Result, DestTy: ResultType, /*isSigned*/ false, Name: "cast");
3375 if (!HasFallback)
3376 return RValue::get(V: Result);
3377
3378 Value *Zero = Constant::getNullValue(Ty: ArgType);
3379 Value *IsZero = Builder.CreateICmpEQ(LHS: ArgValue, RHS: Zero, Name: "iszero");
3380 Value *FallbackValue = EmitScalarExpr(E: E->getArg(Arg: 1));
3381 Value *ResultOrFallback =
3382 Builder.CreateSelect(C: IsZero, True: FallbackValue, False: Result, Name: "clzg");
3383 return RValue::get(V: ResultOrFallback);
3384 }
3385 case Builtin::BI__builtin_ffs:
3386 case Builtin::BI__builtin_ffsl:
3387 case Builtin::BI__builtin_ffsll: {
3388 // ffs(x) -> x ? cttz(x) + 1 : 0
3389 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
3390
3391 llvm::Type *ArgType = ArgValue->getType();
3392 Function *F = CGM.getIntrinsic(IID: Intrinsic::cttz, Tys: ArgType);
3393
3394 llvm::Type *ResultType = ConvertType(T: E->getType());
3395 Value *Tmp =
3396 Builder.CreateAdd(LHS: Builder.CreateCall(Callee: F, Args: {ArgValue, Builder.getTrue()}),
3397 RHS: llvm::ConstantInt::get(Ty: ArgType, V: 1));
3398 Value *Zero = llvm::Constant::getNullValue(Ty: ArgType);
3399 Value *IsZero = Builder.CreateICmpEQ(LHS: ArgValue, RHS: Zero, Name: "iszero");
3400 Value *Result = Builder.CreateSelect(C: IsZero, True: Zero, False: Tmp, Name: "ffs");
3401 if (Result->getType() != ResultType)
3402 Result = Builder.CreateIntCast(V: Result, DestTy: ResultType, /*isSigned*/true,
3403 Name: "cast");
3404 return RValue::get(V: Result);
3405 }
3406 case Builtin::BI__builtin_parity:
3407 case Builtin::BI__builtin_parityl:
3408 case Builtin::BI__builtin_parityll: {
3409 // parity(x) -> ctpop(x) & 1
3410 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
3411
3412 llvm::Type *ArgType = ArgValue->getType();
3413 Function *F = CGM.getIntrinsic(IID: Intrinsic::ctpop, Tys: ArgType);
3414
3415 llvm::Type *ResultType = ConvertType(T: E->getType());
3416 Value *Tmp = Builder.CreateCall(Callee: F, Args: ArgValue);
3417 Value *Result = Builder.CreateAnd(LHS: Tmp, RHS: llvm::ConstantInt::get(Ty: ArgType, V: 1));
3418 if (Result->getType() != ResultType)
3419 Result = Builder.CreateIntCast(V: Result, DestTy: ResultType, /*isSigned*/true,
3420 Name: "cast");
3421 return RValue::get(V: Result);
3422 }
3423 case Builtin::BI__lzcnt16:
3424 case Builtin::BI__lzcnt:
3425 case Builtin::BI__lzcnt64: {
3426 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
3427
3428 llvm::Type *ArgType = ArgValue->getType();
3429 Function *F = CGM.getIntrinsic(IID: Intrinsic::ctlz, Tys: ArgType);
3430
3431 llvm::Type *ResultType = ConvertType(T: E->getType());
3432 Value *Result = Builder.CreateCall(Callee: F, Args: {ArgValue, Builder.getFalse()});
3433 if (Result->getType() != ResultType)
3434 Result = Builder.CreateIntCast(V: Result, DestTy: ResultType, /*isSigned*/true,
3435 Name: "cast");
3436 return RValue::get(V: Result);
3437 }
3438 case Builtin::BI__popcnt16:
3439 case Builtin::BI__popcnt:
3440 case Builtin::BI__popcnt64:
3441 case Builtin::BI__builtin_popcount:
3442 case Builtin::BI__builtin_popcountl:
3443 case Builtin::BI__builtin_popcountll:
3444 case Builtin::BI__builtin_popcountg: {
3445 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
3446
3447 llvm::Type *ArgType = ArgValue->getType();
3448 Function *F = CGM.getIntrinsic(IID: Intrinsic::ctpop, Tys: ArgType);
3449
3450 llvm::Type *ResultType = ConvertType(T: E->getType());
3451 Value *Result = Builder.CreateCall(Callee: F, Args: ArgValue);
3452 if (Result->getType() != ResultType)
3453 Result =
3454 Builder.CreateIntCast(V: Result, DestTy: ResultType, /*isSigned*/ false, Name: "cast");
3455 return RValue::get(V: Result);
3456 }
3457 case Builtin::BI__builtin_unpredictable: {
3458 // Always return the argument of __builtin_unpredictable. LLVM does not
3459 // handle this builtin. Metadata for this builtin should be added directly
3460 // to instructions such as branches or switches that use it.
3461 return RValue::get(V: EmitScalarExpr(E: E->getArg(Arg: 0)));
3462 }
3463 case Builtin::BI__builtin_expect: {
3464 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
3465 llvm::Type *ArgType = ArgValue->getType();
3466
3467 Value *ExpectedValue = EmitScalarExpr(E: E->getArg(Arg: 1));
3468 // Don't generate llvm.expect on -O0 as the backend won't use it for
3469 // anything.
3470 // Note, we still IRGen ExpectedValue because it could have side-effects.
3471 if (CGM.getCodeGenOpts().OptimizationLevel == 0)
3472 return RValue::get(V: ArgValue);
3473
3474 Function *FnExpect = CGM.getIntrinsic(IID: Intrinsic::expect, Tys: ArgType);
3475 Value *Result =
3476 Builder.CreateCall(Callee: FnExpect, Args: {ArgValue, ExpectedValue}, Name: "expval");
3477 return RValue::get(V: Result);
3478 }
3479 case Builtin::BI__builtin_expect_with_probability: {
3480 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
3481 llvm::Type *ArgType = ArgValue->getType();
3482
3483 Value *ExpectedValue = EmitScalarExpr(E: E->getArg(Arg: 1));
3484 llvm::APFloat Probability(0.0);
3485 const Expr *ProbArg = E->getArg(Arg: 2);
3486 bool EvalSucceed = ProbArg->EvaluateAsFloat(Result&: Probability, Ctx: CGM.getContext());
3487 assert(EvalSucceed && "probability should be able to evaluate as float");
3488 (void)EvalSucceed;
3489 bool LoseInfo = false;
3490 Probability.convert(ToSemantics: llvm::APFloat::IEEEdouble(),
3491 RM: llvm::RoundingMode::Dynamic, losesInfo: &LoseInfo);
3492 llvm::Type *Ty = ConvertType(T: ProbArg->getType());
3493 Constant *Confidence = ConstantFP::get(Ty, V: Probability);
3494 // Don't generate llvm.expect.with.probability on -O0 as the backend
3495 // won't use it for anything.
3496 // Note, we still IRGen ExpectedValue because it could have side-effects.
3497 if (CGM.getCodeGenOpts().OptimizationLevel == 0)
3498 return RValue::get(V: ArgValue);
3499
3500 Function *FnExpect =
3501 CGM.getIntrinsic(IID: Intrinsic::expect_with_probability, Tys: ArgType);
3502 Value *Result = Builder.CreateCall(
3503 Callee: FnExpect, Args: {ArgValue, ExpectedValue, Confidence}, Name: "expval");
3504 return RValue::get(V: Result);
3505 }
3506 case Builtin::BI__builtin_assume_aligned: {
3507 const Expr *Ptr = E->getArg(Arg: 0);
3508 Value *PtrValue = EmitScalarExpr(E: Ptr);
3509 Value *OffsetValue =
3510 (E->getNumArgs() > 2) ? EmitScalarExpr(E: E->getArg(Arg: 2)) : nullptr;
3511
3512 Value *AlignmentValue = EmitScalarExpr(E: E->getArg(Arg: 1));
3513 ConstantInt *AlignmentCI = cast<ConstantInt>(Val: AlignmentValue);
3514 if (AlignmentCI->getValue().ugt(RHS: llvm::Value::MaximumAlignment))
3515 AlignmentCI = ConstantInt::get(Ty: AlignmentCI->getIntegerType(),
3516 V: llvm::Value::MaximumAlignment);
3517
3518 emitAlignmentAssumption(PtrValue, E: Ptr,
3519 /*The expr loc is sufficient.*/ AssumptionLoc: SourceLocation(),
3520 Alignment: AlignmentCI, OffsetValue);
3521 return RValue::get(V: PtrValue);
3522 }
3523 case Builtin::BI__builtin_assume_dereferenceable: {
3524 const Expr *Ptr = E->getArg(Arg: 0);
3525 const Expr *Size = E->getArg(Arg: 1);
3526 Value *PtrValue = EmitScalarExpr(E: Ptr);
3527 Value *SizeValue = EmitScalarExpr(E: Size);
3528 if (SizeValue->getType() != IntPtrTy)
3529 SizeValue =
3530 Builder.CreateIntCast(V: SizeValue, DestTy: IntPtrTy, isSigned: false, Name: "casted.size");
3531 Builder.CreateDereferenceableAssumption(PtrValue, SizeValue);
3532 return RValue::get(V: nullptr);
3533 }
3534 case Builtin::BI__assume:
3535 case Builtin::BI__builtin_assume: {
3536 if (E->getArg(Arg: 0)->HasSideEffects(Ctx: getContext()))
3537 return RValue::get(V: nullptr);
3538
3539 Value *ArgValue = EmitCheckedArgForAssume(E: E->getArg(Arg: 0));
3540 Function *FnAssume = CGM.getIntrinsic(IID: Intrinsic::assume);
3541 Builder.CreateCall(Callee: FnAssume, Args: ArgValue);
3542 return RValue::get(V: nullptr);
3543 }
3544 case Builtin::BI__builtin_assume_separate_storage: {
3545 const Expr *Arg0 = E->getArg(Arg: 0);
3546 const Expr *Arg1 = E->getArg(Arg: 1);
3547
3548 Value *Value0 = EmitScalarExpr(E: Arg0);
3549 Value *Value1 = EmitScalarExpr(E: Arg1);
3550
3551 Value *Values[] = {Value0, Value1};
3552 OperandBundleDefT<Value *> OBD("separate_storage", Values);
3553 Builder.CreateAssumption(Cond: ConstantInt::getTrue(Context&: getLLVMContext()), OpBundles: {OBD});
3554 return RValue::get(V: nullptr);
3555 }
3556 case Builtin::BI__builtin_allow_runtime_check: {
3557 StringRef Kind =
3558 cast<StringLiteral>(Val: E->getArg(Arg: 0)->IgnoreParenCasts())->getString();
3559 LLVMContext &Ctx = CGM.getLLVMContext();
3560 llvm::Value *Allow = Builder.CreateCall(
3561 Callee: CGM.getIntrinsic(IID: Intrinsic::allow_runtime_check),
3562 Args: llvm::MetadataAsValue::get(Context&: Ctx, MD: llvm::MDString::get(Context&: Ctx, Str: Kind)));
3563 return RValue::get(V: Allow);
3564 }
3565 case Builtin::BI__arithmetic_fence: {
3566 // Create the builtin call if FastMath is selected, and the target
3567 // supports the builtin, otherwise just return the argument.
3568 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
3569 llvm::FastMathFlags FMF = Builder.getFastMathFlags();
3570 bool isArithmeticFenceEnabled =
3571 FMF.allowReassoc() &&
3572 getContext().getTargetInfo().checkArithmeticFenceSupported();
3573 QualType ArgType = E->getArg(Arg: 0)->getType();
3574 if (ArgType->isComplexType()) {
3575 if (isArithmeticFenceEnabled) {
3576 QualType ElementType = ArgType->castAs<ComplexType>()->getElementType();
3577 ComplexPairTy ComplexVal = EmitComplexExpr(E: E->getArg(Arg: 0));
3578 Value *Real = Builder.CreateArithmeticFence(Val: ComplexVal.first,
3579 DstType: ConvertType(T: ElementType));
3580 Value *Imag = Builder.CreateArithmeticFence(Val: ComplexVal.second,
3581 DstType: ConvertType(T: ElementType));
3582 return RValue::getComplex(C: std::make_pair(x&: Real, y&: Imag));
3583 }
3584 ComplexPairTy ComplexVal = EmitComplexExpr(E: E->getArg(Arg: 0));
3585 Value *Real = ComplexVal.first;
3586 Value *Imag = ComplexVal.second;
3587 return RValue::getComplex(C: std::make_pair(x&: Real, y&: Imag));
3588 }
3589 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
3590 if (isArithmeticFenceEnabled)
3591 return RValue::get(
3592 V: Builder.CreateArithmeticFence(Val: ArgValue, DstType: ConvertType(T: ArgType)));
3593 return RValue::get(V: ArgValue);
3594 }
3595 case Builtin::BI__builtin_bswap16:
3596 case Builtin::BI__builtin_bswap32:
3597 case Builtin::BI__builtin_bswap64:
3598 case Builtin::BI_byteswap_ushort:
3599 case Builtin::BI_byteswap_ulong:
3600 case Builtin::BI_byteswap_uint64: {
3601 return RValue::get(
3602 V: emitBuiltinWithOneOverloadedType<1>(CGF&: *this, E, IntrinsicID: Intrinsic::bswap));
3603 }
3604 case Builtin::BI__builtin_bitreverse8:
3605 case Builtin::BI__builtin_bitreverse16:
3606 case Builtin::BI__builtin_bitreverse32:
3607 case Builtin::BI__builtin_bitreverse64: {
3608 return RValue::get(
3609 V: emitBuiltinWithOneOverloadedType<1>(CGF&: *this, E, IntrinsicID: Intrinsic::bitreverse));
3610 }
3611 case Builtin::BI__builtin_rotateleft8:
3612 case Builtin::BI__builtin_rotateleft16:
3613 case Builtin::BI__builtin_rotateleft32:
3614 case Builtin::BI__builtin_rotateleft64:
3615 case Builtin::BI_rotl8: // Microsoft variants of rotate left
3616 case Builtin::BI_rotl16:
3617 case Builtin::BI_rotl:
3618 case Builtin::BI_lrotl:
3619 case Builtin::BI_rotl64:
3620 return emitRotate(E, IsRotateRight: false);
3621
3622 case Builtin::BI__builtin_rotateright8:
3623 case Builtin::BI__builtin_rotateright16:
3624 case Builtin::BI__builtin_rotateright32:
3625 case Builtin::BI__builtin_rotateright64:
3626 case Builtin::BI_rotr8: // Microsoft variants of rotate right
3627 case Builtin::BI_rotr16:
3628 case Builtin::BI_rotr:
3629 case Builtin::BI_lrotr:
3630 case Builtin::BI_rotr64:
3631 return emitRotate(E, IsRotateRight: true);
3632
3633 case Builtin::BI__builtin_constant_p: {
3634 llvm::Type *ResultType = ConvertType(T: E->getType());
3635
3636 const Expr *Arg = E->getArg(Arg: 0);
3637 QualType ArgType = Arg->getType();
3638 // FIXME: The allowance for Obj-C pointers and block pointers is historical
3639 // and likely a mistake.
3640 if (!ArgType->isIntegralOrEnumerationType() && !ArgType->isFloatingType() &&
3641 !ArgType->isObjCObjectPointerType() && !ArgType->isBlockPointerType())
3642 // Per the GCC documentation, only numeric constants are recognized after
3643 // inlining.
3644 return RValue::get(V: ConstantInt::get(Ty: ResultType, V: 0));
3645
3646 if (Arg->HasSideEffects(Ctx: getContext()))
3647 // The argument is unevaluated, so be conservative if it might have
3648 // side-effects.
3649 return RValue::get(V: ConstantInt::get(Ty: ResultType, V: 0));
3650
3651 Value *ArgValue = EmitScalarExpr(E: Arg);
3652 if (ArgType->isObjCObjectPointerType()) {
3653 // Convert Objective-C objects to id because we cannot distinguish between
3654 // LLVM types for Obj-C classes as they are opaque.
3655 ArgType = CGM.getContext().getObjCIdType();
3656 ArgValue = Builder.CreateBitCast(V: ArgValue, DestTy: ConvertType(T: ArgType));
3657 }
3658 Function *F =
3659 CGM.getIntrinsic(IID: Intrinsic::is_constant, Tys: ConvertType(T: ArgType));
3660 Value *Result = Builder.CreateCall(Callee: F, Args: ArgValue);
3661 if (Result->getType() != ResultType)
3662 Result = Builder.CreateIntCast(V: Result, DestTy: ResultType, /*isSigned*/false);
3663 return RValue::get(V: Result);
3664 }
3665 case Builtin::BI__builtin_dynamic_object_size:
3666 case Builtin::BI__builtin_object_size: {
3667 unsigned Type =
3668 E->getArg(Arg: 1)->EvaluateKnownConstInt(Ctx: getContext()).getZExtValue();
3669 auto *ResType = cast<llvm::IntegerType>(Val: ConvertType(T: E->getType()));
3670
3671 // We pass this builtin onto the optimizer so that it can figure out the
3672 // object size in more complex cases.
3673 bool IsDynamic = BuiltinID == Builtin::BI__builtin_dynamic_object_size;
3674 return RValue::get(V: emitBuiltinObjectSize(E: E->getArg(Arg: 0), Type, ResType,
3675 /*EmittedE=*/nullptr, IsDynamic));
3676 }
3677 case Builtin::BI__builtin_counted_by_ref: {
3678 // Default to returning '(void *) 0'.
3679 llvm::Value *Result = llvm::ConstantPointerNull::get(
3680 T: llvm::PointerType::getUnqual(C&: getLLVMContext()));
3681
3682 const Expr *Arg = E->getArg(Arg: 0)->IgnoreParenImpCasts();
3683
3684 if (auto *UO = dyn_cast<UnaryOperator>(Val: Arg);
3685 UO && UO->getOpcode() == UO_AddrOf) {
3686 Arg = UO->getSubExpr()->IgnoreParenImpCasts();
3687
3688 if (auto *ASE = dyn_cast<ArraySubscriptExpr>(Val: Arg))
3689 Arg = ASE->getBase()->IgnoreParenImpCasts();
3690 }
3691
3692 if (const MemberExpr *ME = dyn_cast_if_present<MemberExpr>(Val: Arg)) {
3693 if (auto *CATy =
3694 ME->getMemberDecl()->getType()->getAs<CountAttributedType>();
3695 CATy && CATy->getKind() == CountAttributedType::CountedBy) {
3696 const auto *FAMDecl = cast<FieldDecl>(Val: ME->getMemberDecl());
3697 if (const FieldDecl *CountFD = FAMDecl->findCountedByField())
3698 Result = GetCountedByFieldExprGEP(Base: Arg, FD: FAMDecl, CountDecl: CountFD);
3699 else
3700 llvm::report_fatal_error(reason: "Cannot find the counted_by 'count' field");
3701 }
3702 }
3703
3704 return RValue::get(V: Result);
3705 }
3706 case Builtin::BI__builtin_prefetch: {
3707 Value *Locality, *RW, *Address = EmitScalarExpr(E: E->getArg(Arg: 0));
3708 // FIXME: Technically these constants should of type 'int', yes?
3709 RW = (E->getNumArgs() > 1) ? EmitScalarExpr(E: E->getArg(Arg: 1)) :
3710 llvm::ConstantInt::get(Ty: Int32Ty, V: 0);
3711 Locality = (E->getNumArgs() > 2) ? EmitScalarExpr(E: E->getArg(Arg: 2)) :
3712 llvm::ConstantInt::get(Ty: Int32Ty, V: 3);
3713 Value *Data = llvm::ConstantInt::get(Ty: Int32Ty, V: 1);
3714 Function *F = CGM.getIntrinsic(IID: Intrinsic::prefetch, Tys: Address->getType());
3715 Builder.CreateCall(Callee: F, Args: {Address, RW, Locality, Data});
3716 return RValue::get(V: nullptr);
3717 }
3718 case Builtin::BI__builtin_readcyclecounter: {
3719 Function *F = CGM.getIntrinsic(IID: Intrinsic::readcyclecounter);
3720 return RValue::get(V: Builder.CreateCall(Callee: F));
3721 }
3722 case Builtin::BI__builtin_readsteadycounter: {
3723 Function *F = CGM.getIntrinsic(IID: Intrinsic::readsteadycounter);
3724 return RValue::get(V: Builder.CreateCall(Callee: F));
3725 }
3726 case Builtin::BI__builtin___clear_cache: {
3727 Value *Begin = EmitScalarExpr(E: E->getArg(Arg: 0));
3728 Value *End = EmitScalarExpr(E: E->getArg(Arg: 1));
3729 Function *F = CGM.getIntrinsic(IID: Intrinsic::clear_cache);
3730 return RValue::get(V: Builder.CreateCall(Callee: F, Args: {Begin, End}));
3731 }
3732 case Builtin::BI__builtin_trap:
3733 EmitTrapCall(IntrID: Intrinsic::trap);
3734 return RValue::get(V: nullptr);
3735 case Builtin::BI__builtin_verbose_trap: {
3736 llvm::DILocation *TrapLocation = Builder.getCurrentDebugLocation();
3737 if (getDebugInfo()) {
3738 TrapLocation = getDebugInfo()->CreateTrapFailureMessageFor(
3739 TrapLocation, Category: *E->getArg(Arg: 0)->tryEvaluateString(Ctx&: getContext()),
3740 FailureMsg: *E->getArg(Arg: 1)->tryEvaluateString(Ctx&: getContext()));
3741 }
3742 ApplyDebugLocation ApplyTrapDI(*this, TrapLocation);
3743 // Currently no attempt is made to prevent traps from being merged.
3744 EmitTrapCall(IntrID: Intrinsic::trap);
3745 return RValue::get(V: nullptr);
3746 }
3747 case Builtin::BI__debugbreak:
3748 EmitTrapCall(IntrID: Intrinsic::debugtrap);
3749 return RValue::get(V: nullptr);
3750 case Builtin::BI__builtin_unreachable: {
3751 EmitUnreachable(Loc: E->getExprLoc());
3752
3753 // We do need to preserve an insertion point.
3754 EmitBlock(BB: createBasicBlock(name: "unreachable.cont"));
3755
3756 return RValue::get(V: nullptr);
3757 }
3758
3759 case Builtin::BI__builtin_powi:
3760 case Builtin::BI__builtin_powif:
3761 case Builtin::BI__builtin_powil: {
3762 llvm::Value *Src0 = EmitScalarExpr(E: E->getArg(Arg: 0));
3763 llvm::Value *Src1 = EmitScalarExpr(E: E->getArg(Arg: 1));
3764
3765 if (Builder.getIsFPConstrained()) {
3766 // FIXME: llvm.powi has 2 mangling types,
3767 // llvm.experimental.constrained.powi has one.
3768 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
3769 Function *F = CGM.getIntrinsic(IID: Intrinsic::experimental_constrained_powi,
3770 Tys: Src0->getType());
3771 return RValue::get(V: Builder.CreateConstrainedFPCall(Callee: F, Args: { Src0, Src1 }));
3772 }
3773
3774 Function *F = CGM.getIntrinsic(IID: Intrinsic::powi,
3775 Tys: { Src0->getType(), Src1->getType() });
3776 return RValue::get(V: Builder.CreateCall(Callee: F, Args: { Src0, Src1 }));
3777 }
3778 case Builtin::BI__builtin_frexpl: {
3779 // Linux PPC will not be adding additional PPCDoubleDouble support.
3780 // WIP to switch default to IEEE long double. Will emit libcall for
3781 // frexpl instead of legalizing this type in the BE.
3782 if (&getTarget().getLongDoubleFormat() == &llvm::APFloat::PPCDoubleDouble())
3783 break;
3784 [[fallthrough]];
3785 }
3786 case Builtin::BI__builtin_frexp:
3787 case Builtin::BI__builtin_frexpf:
3788 case Builtin::BI__builtin_frexpf128:
3789 case Builtin::BI__builtin_frexpf16:
3790 return RValue::get(V: emitFrexpBuiltin(CGF&: *this, E, IntrinsicID: Intrinsic::frexp));
3791 case Builtin::BImodf:
3792 case Builtin::BImodff:
3793 case Builtin::BImodfl:
3794 case Builtin::BI__builtin_modf:
3795 case Builtin::BI__builtin_modff:
3796 case Builtin::BI__builtin_modfl:
3797 if (Builder.getIsFPConstrained())
3798 break; // TODO: Emit constrained modf intrinsic once one exists.
3799 return RValue::get(V: emitModfBuiltin(CGF&: *this, E, IntrinsicID: Intrinsic::modf));
3800 case Builtin::BI__builtin_isgreater:
3801 case Builtin::BI__builtin_isgreaterequal:
3802 case Builtin::BI__builtin_isless:
3803 case Builtin::BI__builtin_islessequal:
3804 case Builtin::BI__builtin_islessgreater:
3805 case Builtin::BI__builtin_isunordered: {
3806 // Ordered comparisons: we know the arguments to these are matching scalar
3807 // floating point values.
3808 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
3809 Value *LHS = EmitScalarExpr(E: E->getArg(Arg: 0));
3810 Value *RHS = EmitScalarExpr(E: E->getArg(Arg: 1));
3811
3812 switch (BuiltinID) {
3813 default: llvm_unreachable("Unknown ordered comparison");
3814 case Builtin::BI__builtin_isgreater:
3815 LHS = Builder.CreateFCmpOGT(LHS, RHS, Name: "cmp");
3816 break;
3817 case Builtin::BI__builtin_isgreaterequal:
3818 LHS = Builder.CreateFCmpOGE(LHS, RHS, Name: "cmp");
3819 break;
3820 case Builtin::BI__builtin_isless:
3821 LHS = Builder.CreateFCmpOLT(LHS, RHS, Name: "cmp");
3822 break;
3823 case Builtin::BI__builtin_islessequal:
3824 LHS = Builder.CreateFCmpOLE(LHS, RHS, Name: "cmp");
3825 break;
3826 case Builtin::BI__builtin_islessgreater:
3827 LHS = Builder.CreateFCmpONE(LHS, RHS, Name: "cmp");
3828 break;
3829 case Builtin::BI__builtin_isunordered:
3830 LHS = Builder.CreateFCmpUNO(LHS, RHS, Name: "cmp");
3831 break;
3832 }
3833 // ZExt bool to int type.
3834 return RValue::get(V: Builder.CreateZExt(V: LHS, DestTy: ConvertType(T: E->getType())));
3835 }
3836
3837 case Builtin::BI__builtin_isnan: {
3838 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
3839 Value *V = EmitScalarExpr(E: E->getArg(Arg: 0));
3840 if (Value *Result = tryUseTestFPKind(CGF&: *this, BuiltinID, V))
3841 return RValue::get(V: Result);
3842 return RValue::get(
3843 V: Builder.CreateZExt(V: Builder.createIsFPClass(FPNum: V, Test: FPClassTest::fcNan),
3844 DestTy: ConvertType(T: E->getType())));
3845 }
3846
3847 case Builtin::BI__builtin_issignaling: {
3848 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
3849 Value *V = EmitScalarExpr(E: E->getArg(Arg: 0));
3850 return RValue::get(
3851 V: Builder.CreateZExt(V: Builder.createIsFPClass(FPNum: V, Test: FPClassTest::fcSNan),
3852 DestTy: ConvertType(T: E->getType())));
3853 }
3854
3855 case Builtin::BI__builtin_isinf: {
3856 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
3857 Value *V = EmitScalarExpr(E: E->getArg(Arg: 0));
3858 if (Value *Result = tryUseTestFPKind(CGF&: *this, BuiltinID, V))
3859 return RValue::get(V: Result);
3860 return RValue::get(
3861 V: Builder.CreateZExt(V: Builder.createIsFPClass(FPNum: V, Test: FPClassTest::fcInf),
3862 DestTy: ConvertType(T: E->getType())));
3863 }
3864
3865 case Builtin::BIfinite:
3866 case Builtin::BI__finite:
3867 case Builtin::BIfinitef:
3868 case Builtin::BI__finitef:
3869 case Builtin::BIfinitel:
3870 case Builtin::BI__finitel:
3871 case Builtin::BI__builtin_isfinite: {
3872 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
3873 Value *V = EmitScalarExpr(E: E->getArg(Arg: 0));
3874 if (Value *Result = tryUseTestFPKind(CGF&: *this, BuiltinID, V))
3875 return RValue::get(V: Result);
3876 return RValue::get(
3877 V: Builder.CreateZExt(V: Builder.createIsFPClass(FPNum: V, Test: FPClassTest::fcFinite),
3878 DestTy: ConvertType(T: E->getType())));
3879 }
3880
3881 case Builtin::BI__builtin_isnormal: {
3882 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
3883 Value *V = EmitScalarExpr(E: E->getArg(Arg: 0));
3884 return RValue::get(
3885 V: Builder.CreateZExt(V: Builder.createIsFPClass(FPNum: V, Test: FPClassTest::fcNormal),
3886 DestTy: ConvertType(T: E->getType())));
3887 }
3888
3889 case Builtin::BI__builtin_issubnormal: {
3890 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
3891 Value *V = EmitScalarExpr(E: E->getArg(Arg: 0));
3892 return RValue::get(
3893 V: Builder.CreateZExt(V: Builder.createIsFPClass(FPNum: V, Test: FPClassTest::fcSubnormal),
3894 DestTy: ConvertType(T: E->getType())));
3895 }
3896
3897 case Builtin::BI__builtin_iszero: {
3898 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
3899 Value *V = EmitScalarExpr(E: E->getArg(Arg: 0));
3900 return RValue::get(
3901 V: Builder.CreateZExt(V: Builder.createIsFPClass(FPNum: V, Test: FPClassTest::fcZero),
3902 DestTy: ConvertType(T: E->getType())));
3903 }
3904
3905 case Builtin::BI__builtin_isfpclass: {
3906 Expr::EvalResult Result;
3907 if (!E->getArg(Arg: 1)->EvaluateAsInt(Result, Ctx: CGM.getContext()))
3908 break;
3909 uint64_t Test = Result.Val.getInt().getLimitedValue();
3910 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
3911 Value *V = EmitScalarExpr(E: E->getArg(Arg: 0));
3912 return RValue::get(V: Builder.CreateZExt(V: Builder.createIsFPClass(FPNum: V, Test),
3913 DestTy: ConvertType(T: E->getType())));
3914 }
3915
3916 case Builtin::BI__builtin_nondeterministic_value: {
3917 llvm::Type *Ty = ConvertType(T: E->getArg(Arg: 0)->getType());
3918
3919 Value *Result = PoisonValue::get(T: Ty);
3920 Result = Builder.CreateFreeze(V: Result);
3921
3922 return RValue::get(V: Result);
3923 }
3924
3925 case Builtin::BI__builtin_elementwise_abs: {
3926 Value *Result;
3927 QualType QT = E->getArg(Arg: 0)->getType();
3928
3929 if (auto *VecTy = QT->getAs<VectorType>())
3930 QT = VecTy->getElementType();
3931 if (QT->isIntegerType())
3932 Result = Builder.CreateBinaryIntrinsic(
3933 ID: Intrinsic::abs, LHS: EmitScalarExpr(E: E->getArg(Arg: 0)), RHS: Builder.getFalse(),
3934 FMFSource: nullptr, Name: "elt.abs");
3935 else
3936 Result = emitBuiltinWithOneOverloadedType<1>(CGF&: *this, E, IntrinsicID: Intrinsic::fabs,
3937 Name: "elt.abs");
3938
3939 return RValue::get(V: Result);
3940 }
3941 case Builtin::BI__builtin_elementwise_acos:
3942 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
3943 CGF&: *this, E, IntrinsicID: Intrinsic::acos, Name: "elt.acos"));
3944 case Builtin::BI__builtin_elementwise_asin:
3945 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
3946 CGF&: *this, E, IntrinsicID: Intrinsic::asin, Name: "elt.asin"));
3947 case Builtin::BI__builtin_elementwise_atan:
3948 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
3949 CGF&: *this, E, IntrinsicID: Intrinsic::atan, Name: "elt.atan"));
3950 case Builtin::BI__builtin_elementwise_atan2:
3951 return RValue::get(V: emitBuiltinWithOneOverloadedType<2>(
3952 CGF&: *this, E, IntrinsicID: Intrinsic::atan2, Name: "elt.atan2"));
3953 case Builtin::BI__builtin_elementwise_ceil:
3954 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
3955 CGF&: *this, E, IntrinsicID: Intrinsic::ceil, Name: "elt.ceil"));
3956 case Builtin::BI__builtin_elementwise_exp:
3957 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
3958 CGF&: *this, E, IntrinsicID: Intrinsic::exp, Name: "elt.exp"));
3959 case Builtin::BI__builtin_elementwise_exp2:
3960 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
3961 CGF&: *this, E, IntrinsicID: Intrinsic::exp2, Name: "elt.exp2"));
3962 case Builtin::BI__builtin_elementwise_exp10:
3963 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
3964 CGF&: *this, E, IntrinsicID: Intrinsic::exp10, Name: "elt.exp10"));
3965 case Builtin::BI__builtin_elementwise_log:
3966 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
3967 CGF&: *this, E, IntrinsicID: Intrinsic::log, Name: "elt.log"));
3968 case Builtin::BI__builtin_elementwise_log2:
3969 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
3970 CGF&: *this, E, IntrinsicID: Intrinsic::log2, Name: "elt.log2"));
3971 case Builtin::BI__builtin_elementwise_log10:
3972 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
3973 CGF&: *this, E, IntrinsicID: Intrinsic::log10, Name: "elt.log10"));
3974 case Builtin::BI__builtin_elementwise_pow: {
3975 return RValue::get(
3976 V: emitBuiltinWithOneOverloadedType<2>(CGF&: *this, E, IntrinsicID: Intrinsic::pow));
3977 }
3978 case Builtin::BI__builtin_elementwise_bitreverse:
3979 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
3980 CGF&: *this, E, IntrinsicID: Intrinsic::bitreverse, Name: "elt.bitreverse"));
3981 case Builtin::BI__builtin_elementwise_cos:
3982 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
3983 CGF&: *this, E, IntrinsicID: Intrinsic::cos, Name: "elt.cos"));
3984 case Builtin::BI__builtin_elementwise_cosh:
3985 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
3986 CGF&: *this, E, IntrinsicID: Intrinsic::cosh, Name: "elt.cosh"));
3987 case Builtin::BI__builtin_elementwise_floor:
3988 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
3989 CGF&: *this, E, IntrinsicID: Intrinsic::floor, Name: "elt.floor"));
3990 case Builtin::BI__builtin_elementwise_popcount:
3991 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
3992 CGF&: *this, E, IntrinsicID: Intrinsic::ctpop, Name: "elt.ctpop"));
3993 case Builtin::BI__builtin_elementwise_roundeven:
3994 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
3995 CGF&: *this, E, IntrinsicID: Intrinsic::roundeven, Name: "elt.roundeven"));
3996 case Builtin::BI__builtin_elementwise_round:
3997 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
3998 CGF&: *this, E, IntrinsicID: Intrinsic::round, Name: "elt.round"));
3999 case Builtin::BI__builtin_elementwise_rint:
4000 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4001 CGF&: *this, E, IntrinsicID: Intrinsic::rint, Name: "elt.rint"));
4002 case Builtin::BI__builtin_elementwise_nearbyint:
4003 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4004 CGF&: *this, E, IntrinsicID: Intrinsic::nearbyint, Name: "elt.nearbyint"));
4005 case Builtin::BI__builtin_elementwise_sin:
4006 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4007 CGF&: *this, E, IntrinsicID: Intrinsic::sin, Name: "elt.sin"));
4008 case Builtin::BI__builtin_elementwise_sinh:
4009 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4010 CGF&: *this, E, IntrinsicID: Intrinsic::sinh, Name: "elt.sinh"));
4011 case Builtin::BI__builtin_elementwise_tan:
4012 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4013 CGF&: *this, E, IntrinsicID: Intrinsic::tan, Name: "elt.tan"));
4014 case Builtin::BI__builtin_elementwise_tanh:
4015 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4016 CGF&: *this, E, IntrinsicID: Intrinsic::tanh, Name: "elt.tanh"));
4017 case Builtin::BI__builtin_elementwise_trunc:
4018 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4019 CGF&: *this, E, IntrinsicID: Intrinsic::trunc, Name: "elt.trunc"));
4020 case Builtin::BI__builtin_elementwise_canonicalize:
4021 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4022 CGF&: *this, E, IntrinsicID: Intrinsic::canonicalize, Name: "elt.canonicalize"));
4023 case Builtin::BI__builtin_elementwise_copysign:
4024 return RValue::get(
4025 V: emitBuiltinWithOneOverloadedType<2>(CGF&: *this, E, IntrinsicID: Intrinsic::copysign));
4026 case Builtin::BI__builtin_elementwise_fma:
4027 return RValue::get(
4028 V: emitBuiltinWithOneOverloadedType<3>(CGF&: *this, E, IntrinsicID: Intrinsic::fma));
4029 case Builtin::BI__builtin_elementwise_add_sat:
4030 case Builtin::BI__builtin_elementwise_sub_sat: {
4031 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
4032 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
4033 Value *Result;
4034 assert(Op0->getType()->isIntOrIntVectorTy() && "integer type expected");
4035 QualType Ty = E->getArg(Arg: 0)->getType();
4036 if (auto *VecTy = Ty->getAs<VectorType>())
4037 Ty = VecTy->getElementType();
4038 bool IsSigned = Ty->isSignedIntegerType();
4039 unsigned Opc;
4040 if (BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_elementwise_add_sat)
4041 Opc = IsSigned ? Intrinsic::sadd_sat : Intrinsic::uadd_sat;
4042 else
4043 Opc = IsSigned ? Intrinsic::ssub_sat : Intrinsic::usub_sat;
4044 Result = Builder.CreateBinaryIntrinsic(ID: Opc, LHS: Op0, RHS: Op1, FMFSource: nullptr, Name: "elt.sat");
4045 return RValue::get(V: Result);
4046 }
4047
4048 case Builtin::BI__builtin_elementwise_max: {
4049 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
4050 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
4051 Value *Result;
4052 if (Op0->getType()->isIntOrIntVectorTy()) {
4053 QualType Ty = E->getArg(Arg: 0)->getType();
4054 if (auto *VecTy = Ty->getAs<VectorType>())
4055 Ty = VecTy->getElementType();
4056 Result = Builder.CreateBinaryIntrinsic(
4057 ID: Ty->isSignedIntegerType() ? Intrinsic::smax : Intrinsic::umax, LHS: Op0,
4058 RHS: Op1, FMFSource: nullptr, Name: "elt.max");
4059 } else
4060 Result = Builder.CreateMaxNum(LHS: Op0, RHS: Op1, /*FMFSource=*/nullptr, Name: "elt.max");
4061 return RValue::get(V: Result);
4062 }
4063 case Builtin::BI__builtin_elementwise_min: {
4064 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
4065 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
4066 Value *Result;
4067 if (Op0->getType()->isIntOrIntVectorTy()) {
4068 QualType Ty = E->getArg(Arg: 0)->getType();
4069 if (auto *VecTy = Ty->getAs<VectorType>())
4070 Ty = VecTy->getElementType();
4071 Result = Builder.CreateBinaryIntrinsic(
4072 ID: Ty->isSignedIntegerType() ? Intrinsic::smin : Intrinsic::umin, LHS: Op0,
4073 RHS: Op1, FMFSource: nullptr, Name: "elt.min");
4074 } else
4075 Result = Builder.CreateMinNum(LHS: Op0, RHS: Op1, /*FMFSource=*/nullptr, Name: "elt.min");
4076 return RValue::get(V: Result);
4077 }
4078
4079 case Builtin::BI__builtin_elementwise_maxnum: {
4080 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
4081 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
4082 Value *Result = Builder.CreateBinaryIntrinsic(ID: llvm::Intrinsic::maxnum, LHS: Op0,
4083 RHS: Op1, FMFSource: nullptr, Name: "elt.maxnum");
4084 return RValue::get(V: Result);
4085 }
4086
4087 case Builtin::BI__builtin_elementwise_minnum: {
4088 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
4089 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
4090 Value *Result = Builder.CreateBinaryIntrinsic(ID: llvm::Intrinsic::minnum, LHS: Op0,
4091 RHS: Op1, FMFSource: nullptr, Name: "elt.minnum");
4092 return RValue::get(V: Result);
4093 }
4094
4095 case Builtin::BI__builtin_elementwise_maximum: {
4096 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
4097 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
4098 Value *Result = Builder.CreateBinaryIntrinsic(ID: Intrinsic::maximum, LHS: Op0, RHS: Op1,
4099 FMFSource: nullptr, Name: "elt.maximum");
4100 return RValue::get(V: Result);
4101 }
4102
4103 case Builtin::BI__builtin_elementwise_minimum: {
4104 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
4105 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
4106 Value *Result = Builder.CreateBinaryIntrinsic(ID: Intrinsic::minimum, LHS: Op0, RHS: Op1,
4107 FMFSource: nullptr, Name: "elt.minimum");
4108 return RValue::get(V: Result);
4109 }
4110
4111 case Builtin::BI__builtin_reduce_max: {
4112 auto GetIntrinsicID = [this](QualType QT) {
4113 if (auto *VecTy = QT->getAs<VectorType>())
4114 QT = VecTy->getElementType();
4115 else if (QT->isSizelessVectorType())
4116 QT = QT->getSizelessVectorEltType(Ctx: CGM.getContext());
4117
4118 if (QT->isSignedIntegerType())
4119 return Intrinsic::vector_reduce_smax;
4120 if (QT->isUnsignedIntegerType())
4121 return Intrinsic::vector_reduce_umax;
4122 assert(QT->isFloatingType() && "must have a float here");
4123 return Intrinsic::vector_reduce_fmax;
4124 };
4125 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4126 CGF&: *this, E, IntrinsicID: GetIntrinsicID(E->getArg(Arg: 0)->getType()), Name: "rdx.min"));
4127 }
4128
4129 case Builtin::BI__builtin_reduce_min: {
4130 auto GetIntrinsicID = [this](QualType QT) {
4131 if (auto *VecTy = QT->getAs<VectorType>())
4132 QT = VecTy->getElementType();
4133 else if (QT->isSizelessVectorType())
4134 QT = QT->getSizelessVectorEltType(Ctx: CGM.getContext());
4135
4136 if (QT->isSignedIntegerType())
4137 return Intrinsic::vector_reduce_smin;
4138 if (QT->isUnsignedIntegerType())
4139 return Intrinsic::vector_reduce_umin;
4140 assert(QT->isFloatingType() && "must have a float here");
4141 return Intrinsic::vector_reduce_fmin;
4142 };
4143
4144 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4145 CGF&: *this, E, IntrinsicID: GetIntrinsicID(E->getArg(Arg: 0)->getType()), Name: "rdx.min"));
4146 }
4147
4148 case Builtin::BI__builtin_reduce_add:
4149 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4150 CGF&: *this, E, IntrinsicID: Intrinsic::vector_reduce_add, Name: "rdx.add"));
4151 case Builtin::BI__builtin_reduce_mul:
4152 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4153 CGF&: *this, E, IntrinsicID: Intrinsic::vector_reduce_mul, Name: "rdx.mul"));
4154 case Builtin::BI__builtin_reduce_xor:
4155 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4156 CGF&: *this, E, IntrinsicID: Intrinsic::vector_reduce_xor, Name: "rdx.xor"));
4157 case Builtin::BI__builtin_reduce_or:
4158 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4159 CGF&: *this, E, IntrinsicID: Intrinsic::vector_reduce_or, Name: "rdx.or"));
4160 case Builtin::BI__builtin_reduce_and:
4161 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4162 CGF&: *this, E, IntrinsicID: Intrinsic::vector_reduce_and, Name: "rdx.and"));
4163 case Builtin::BI__builtin_reduce_maximum:
4164 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4165 CGF&: *this, E, IntrinsicID: Intrinsic::vector_reduce_fmaximum, Name: "rdx.maximum"));
4166 case Builtin::BI__builtin_reduce_minimum:
4167 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4168 CGF&: *this, E, IntrinsicID: Intrinsic::vector_reduce_fminimum, Name: "rdx.minimum"));
4169
4170 case Builtin::BI__builtin_matrix_transpose: {
4171 auto *MatrixTy = E->getArg(Arg: 0)->getType()->castAs<ConstantMatrixType>();
4172 Value *MatValue = EmitScalarExpr(E: E->getArg(Arg: 0));
4173 MatrixBuilder MB(Builder);
4174 Value *Result = MB.CreateMatrixTranspose(Matrix: MatValue, Rows: MatrixTy->getNumRows(),
4175 Columns: MatrixTy->getNumColumns());
4176 return RValue::get(V: Result);
4177 }
4178
4179 case Builtin::BI__builtin_matrix_column_major_load: {
4180 MatrixBuilder MB(Builder);
4181 // Emit everything that isn't dependent on the first parameter type
4182 Value *Stride = EmitScalarExpr(E: E->getArg(Arg: 3));
4183 const auto *ResultTy = E->getType()->getAs<ConstantMatrixType>();
4184 auto *PtrTy = E->getArg(Arg: 0)->getType()->getAs<PointerType>();
4185 assert(PtrTy && "arg0 must be of pointer type");
4186 bool IsVolatile = PtrTy->getPointeeType().isVolatileQualified();
4187
4188 Address Src = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4189 EmitNonNullArgCheck(RV: RValue::get(V: Src.emitRawPointer(CGF&: *this)),
4190 ArgType: E->getArg(Arg: 0)->getType(), ArgLoc: E->getArg(Arg: 0)->getExprLoc(), AC: FD,
4191 ParmNum: 0);
4192 Value *Result = MB.CreateColumnMajorLoad(
4193 EltTy: Src.getElementType(), DataPtr: Src.emitRawPointer(CGF&: *this),
4194 Alignment: Align(Src.getAlignment().getQuantity()), Stride, IsVolatile,
4195 Rows: ResultTy->getNumRows(), Columns: ResultTy->getNumColumns(), Name: "matrix");
4196 return RValue::get(V: Result);
4197 }
4198
4199 case Builtin::BI__builtin_matrix_column_major_store: {
4200 MatrixBuilder MB(Builder);
4201 Value *Matrix = EmitScalarExpr(E: E->getArg(Arg: 0));
4202 Address Dst = EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
4203 Value *Stride = EmitScalarExpr(E: E->getArg(Arg: 2));
4204
4205 const auto *MatrixTy = E->getArg(Arg: 0)->getType()->getAs<ConstantMatrixType>();
4206 auto *PtrTy = E->getArg(Arg: 1)->getType()->getAs<PointerType>();
4207 assert(PtrTy && "arg1 must be of pointer type");
4208 bool IsVolatile = PtrTy->getPointeeType().isVolatileQualified();
4209
4210 EmitNonNullArgCheck(RV: RValue::get(V: Dst.emitRawPointer(CGF&: *this)),
4211 ArgType: E->getArg(Arg: 1)->getType(), ArgLoc: E->getArg(Arg: 1)->getExprLoc(), AC: FD,
4212 ParmNum: 0);
4213 Value *Result = MB.CreateColumnMajorStore(
4214 Matrix, Ptr: Dst.emitRawPointer(CGF&: *this),
4215 Alignment: Align(Dst.getAlignment().getQuantity()), Stride, IsVolatile,
4216 Rows: MatrixTy->getNumRows(), Columns: MatrixTy->getNumColumns());
4217 addInstToNewSourceAtom(KeyInstruction: cast<Instruction>(Val: Result), Backup: Matrix);
4218 return RValue::get(V: Result);
4219 }
4220
4221 case Builtin::BI__builtin_isinf_sign: {
4222 // isinf_sign(x) -> fabs(x) == infinity ? (signbit(x) ? -1 : 1) : 0
4223 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
4224 // FIXME: for strictfp/IEEE-754 we need to not trap on SNaN here.
4225 Value *Arg = EmitScalarExpr(E: E->getArg(Arg: 0));
4226 Value *AbsArg = EmitFAbs(CGF&: *this, V: Arg);
4227 Value *IsInf = Builder.CreateFCmpOEQ(
4228 LHS: AbsArg, RHS: ConstantFP::getInfinity(Ty: Arg->getType()), Name: "isinf");
4229 Value *IsNeg = EmitSignBit(CGF&: *this, V: Arg);
4230
4231 llvm::Type *IntTy = ConvertType(T: E->getType());
4232 Value *Zero = Constant::getNullValue(Ty: IntTy);
4233 Value *One = ConstantInt::get(Ty: IntTy, V: 1);
4234 Value *NegativeOne = ConstantInt::get(Ty: IntTy, V: -1);
4235 Value *SignResult = Builder.CreateSelect(C: IsNeg, True: NegativeOne, False: One);
4236 Value *Result = Builder.CreateSelect(C: IsInf, True: SignResult, False: Zero);
4237 return RValue::get(V: Result);
4238 }
4239
4240 case Builtin::BI__builtin_flt_rounds: {
4241 Function *F = CGM.getIntrinsic(IID: Intrinsic::get_rounding);
4242
4243 llvm::Type *ResultType = ConvertType(T: E->getType());
4244 Value *Result = Builder.CreateCall(Callee: F);
4245 if (Result->getType() != ResultType)
4246 Result = Builder.CreateIntCast(V: Result, DestTy: ResultType, /*isSigned*/true,
4247 Name: "cast");
4248 return RValue::get(V: Result);
4249 }
4250
4251 case Builtin::BI__builtin_set_flt_rounds: {
4252 Function *F = CGM.getIntrinsic(IID: Intrinsic::set_rounding);
4253
4254 Value *V = EmitScalarExpr(E: E->getArg(Arg: 0));
4255 Builder.CreateCall(Callee: F, Args: V);
4256 return RValue::get(V: nullptr);
4257 }
4258
4259 case Builtin::BI__builtin_fpclassify: {
4260 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
4261 // FIXME: for strictfp/IEEE-754 we need to not trap on SNaN here.
4262 Value *V = EmitScalarExpr(E: E->getArg(Arg: 5));
4263 llvm::Type *Ty = ConvertType(T: E->getArg(Arg: 5)->getType());
4264
4265 // Create Result
4266 BasicBlock *Begin = Builder.GetInsertBlock();
4267 BasicBlock *End = createBasicBlock(name: "fpclassify_end", parent: this->CurFn);
4268 Builder.SetInsertPoint(End);
4269 PHINode *Result =
4270 Builder.CreatePHI(Ty: ConvertType(T: E->getArg(Arg: 0)->getType()), NumReservedValues: 4,
4271 Name: "fpclassify_result");
4272
4273 // if (V==0) return FP_ZERO
4274 Builder.SetInsertPoint(Begin);
4275 Value *IsZero = Builder.CreateFCmpOEQ(LHS: V, RHS: Constant::getNullValue(Ty),
4276 Name: "iszero");
4277 Value *ZeroLiteral = EmitScalarExpr(E: E->getArg(Arg: 4));
4278 BasicBlock *NotZero = createBasicBlock(name: "fpclassify_not_zero", parent: this->CurFn);
4279 Builder.CreateCondBr(Cond: IsZero, True: End, False: NotZero);
4280 Result->addIncoming(V: ZeroLiteral, BB: Begin);
4281
4282 // if (V != V) return FP_NAN
4283 Builder.SetInsertPoint(NotZero);
4284 Value *IsNan = Builder.CreateFCmpUNO(LHS: V, RHS: V, Name: "cmp");
4285 Value *NanLiteral = EmitScalarExpr(E: E->getArg(Arg: 0));
4286 BasicBlock *NotNan = createBasicBlock(name: "fpclassify_not_nan", parent: this->CurFn);
4287 Builder.CreateCondBr(Cond: IsNan, True: End, False: NotNan);
4288 Result->addIncoming(V: NanLiteral, BB: NotZero);
4289
4290 // if (fabs(V) == infinity) return FP_INFINITY
4291 Builder.SetInsertPoint(NotNan);
4292 Value *VAbs = EmitFAbs(CGF&: *this, V);
4293 Value *IsInf =
4294 Builder.CreateFCmpOEQ(LHS: VAbs, RHS: ConstantFP::getInfinity(Ty: V->getType()),
4295 Name: "isinf");
4296 Value *InfLiteral = EmitScalarExpr(E: E->getArg(Arg: 1));
4297 BasicBlock *NotInf = createBasicBlock(name: "fpclassify_not_inf", parent: this->CurFn);
4298 Builder.CreateCondBr(Cond: IsInf, True: End, False: NotInf);
4299 Result->addIncoming(V: InfLiteral, BB: NotNan);
4300
4301 // if (fabs(V) >= MIN_NORMAL) return FP_NORMAL else FP_SUBNORMAL
4302 Builder.SetInsertPoint(NotInf);
4303 APFloat Smallest = APFloat::getSmallestNormalized(
4304 Sem: getContext().getFloatTypeSemantics(T: E->getArg(Arg: 5)->getType()));
4305 Value *IsNormal =
4306 Builder.CreateFCmpUGE(LHS: VAbs, RHS: ConstantFP::get(Context&: V->getContext(), V: Smallest),
4307 Name: "isnormal");
4308 Value *NormalResult =
4309 Builder.CreateSelect(C: IsNormal, True: EmitScalarExpr(E: E->getArg(Arg: 2)),
4310 False: EmitScalarExpr(E: E->getArg(Arg: 3)));
4311 Builder.CreateBr(Dest: End);
4312 Result->addIncoming(V: NormalResult, BB: NotInf);
4313
4314 // return Result
4315 Builder.SetInsertPoint(End);
4316 return RValue::get(V: Result);
4317 }
4318
4319 // An alloca will always return a pointer to the alloca (stack) address
4320 // space. This address space need not be the same as the AST / Language
4321 // default (e.g. in C / C++ auto vars are in the generic address space). At
4322 // the AST level this is handled within CreateTempAlloca et al., but for the
4323 // builtin / dynamic alloca we have to handle it here. We use an explicit cast
4324 // instead of passing an AS to CreateAlloca so as to not inhibit optimisation.
4325 case Builtin::BIalloca:
4326 case Builtin::BI_alloca:
4327 case Builtin::BI__builtin_alloca_uninitialized:
4328 case Builtin::BI__builtin_alloca: {
4329 Value *Size = EmitScalarExpr(E: E->getArg(Arg: 0));
4330 const TargetInfo &TI = getContext().getTargetInfo();
4331 // The alignment of the alloca should correspond to __BIGGEST_ALIGNMENT__.
4332 const Align SuitableAlignmentInBytes =
4333 CGM.getContext()
4334 .toCharUnitsFromBits(BitSize: TI.getSuitableAlign())
4335 .getAsAlign();
4336 AllocaInst *AI = Builder.CreateAlloca(Ty: Builder.getInt8Ty(), ArraySize: Size);
4337 AI->setAlignment(SuitableAlignmentInBytes);
4338 if (BuiltinID != Builtin::BI__builtin_alloca_uninitialized)
4339 initializeAlloca(CGF&: *this, AI, Size, AlignmentInBytes: SuitableAlignmentInBytes);
4340 LangAS AAS = getASTAllocaAddressSpace();
4341 LangAS EAS = E->getType()->getPointeeType().getAddressSpace();
4342 if (AAS != EAS) {
4343 llvm::Type *Ty = CGM.getTypes().ConvertType(T: E->getType());
4344 return RValue::get(
4345 V: getTargetHooks().performAddrSpaceCast(CGF&: *this, V: AI, SrcAddr: AAS, DestTy: Ty));
4346 }
4347 return RValue::get(V: AI);
4348 }
4349
4350 case Builtin::BI__builtin_alloca_with_align_uninitialized:
4351 case Builtin::BI__builtin_alloca_with_align: {
4352 Value *Size = EmitScalarExpr(E: E->getArg(Arg: 0));
4353 Value *AlignmentInBitsValue = EmitScalarExpr(E: E->getArg(Arg: 1));
4354 auto *AlignmentInBitsCI = cast<ConstantInt>(Val: AlignmentInBitsValue);
4355 unsigned AlignmentInBits = AlignmentInBitsCI->getZExtValue();
4356 const Align AlignmentInBytes =
4357 CGM.getContext().toCharUnitsFromBits(BitSize: AlignmentInBits).getAsAlign();
4358 AllocaInst *AI = Builder.CreateAlloca(Ty: Builder.getInt8Ty(), ArraySize: Size);
4359 AI->setAlignment(AlignmentInBytes);
4360 if (BuiltinID != Builtin::BI__builtin_alloca_with_align_uninitialized)
4361 initializeAlloca(CGF&: *this, AI, Size, AlignmentInBytes);
4362 LangAS AAS = getASTAllocaAddressSpace();
4363 LangAS EAS = E->getType()->getPointeeType().getAddressSpace();
4364 if (AAS != EAS) {
4365 llvm::Type *Ty = CGM.getTypes().ConvertType(T: E->getType());
4366 return RValue::get(
4367 V: getTargetHooks().performAddrSpaceCast(CGF&: *this, V: AI, SrcAddr: AAS, DestTy: Ty));
4368 }
4369 return RValue::get(V: AI);
4370 }
4371
4372 case Builtin::BIbzero:
4373 case Builtin::BI__builtin_bzero: {
4374 Address Dest = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4375 Value *SizeVal = EmitScalarExpr(E: E->getArg(Arg: 1));
4376 EmitNonNullArgCheck(Addr: Dest, ArgType: E->getArg(Arg: 0)->getType(),
4377 ArgLoc: E->getArg(Arg: 0)->getExprLoc(), AC: FD, ParmNum: 0);
4378 auto *I = Builder.CreateMemSet(Dest, Value: Builder.getInt8(C: 0), Size: SizeVal, IsVolatile: false);
4379 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
4380 return RValue::get(V: nullptr);
4381 }
4382
4383 case Builtin::BIbcopy:
4384 case Builtin::BI__builtin_bcopy: {
4385 Address Src = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4386 Address Dest = EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
4387 Value *SizeVal = EmitScalarExpr(E: E->getArg(Arg: 2));
4388 EmitNonNullArgCheck(RV: RValue::get(V: Src.emitRawPointer(CGF&: *this)),
4389 ArgType: E->getArg(Arg: 0)->getType(), ArgLoc: E->getArg(Arg: 0)->getExprLoc(), AC: FD,
4390 ParmNum: 0);
4391 EmitNonNullArgCheck(RV: RValue::get(V: Dest.emitRawPointer(CGF&: *this)),
4392 ArgType: E->getArg(Arg: 1)->getType(), ArgLoc: E->getArg(Arg: 1)->getExprLoc(), AC: FD,
4393 ParmNum: 0);
4394 auto *I = Builder.CreateMemMove(Dest, Src, Size: SizeVal, IsVolatile: false);
4395 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
4396 return RValue::get(V: nullptr);
4397 }
4398
4399 case Builtin::BImemcpy:
4400 case Builtin::BI__builtin_memcpy:
4401 case Builtin::BImempcpy:
4402 case Builtin::BI__builtin_mempcpy: {
4403 Address Dest = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4404 Address Src = EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
4405 Value *SizeVal = EmitScalarExpr(E: E->getArg(Arg: 2));
4406 EmitArgCheck(TCK_Store, Dest, E->getArg(Arg: 0), 0);
4407 EmitArgCheck(TCK_Load, Src, E->getArg(Arg: 1), 1);
4408 auto *I = Builder.CreateMemCpy(Dest, Src, Size: SizeVal, IsVolatile: false);
4409 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
4410 if (BuiltinID == Builtin::BImempcpy ||
4411 BuiltinID == Builtin::BI__builtin_mempcpy)
4412 return RValue::get(V: Builder.CreateInBoundsGEP(
4413 Ty: Dest.getElementType(), Ptr: Dest.emitRawPointer(CGF&: *this), IdxList: SizeVal));
4414 else
4415 return RValue::get(Addr: Dest, CGF&: *this);
4416 }
4417
4418 case Builtin::BI__builtin_memcpy_inline: {
4419 Address Dest = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4420 Address Src = EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
4421 uint64_t Size =
4422 E->getArg(Arg: 2)->EvaluateKnownConstInt(Ctx: getContext()).getZExtValue();
4423 EmitArgCheck(TCK_Store, Dest, E->getArg(Arg: 0), 0);
4424 EmitArgCheck(TCK_Load, Src, E->getArg(Arg: 1), 1);
4425 auto *I = Builder.CreateMemCpyInline(Dest, Src, Size);
4426 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
4427 return RValue::get(V: nullptr);
4428 }
4429
4430 case Builtin::BI__builtin_char_memchr:
4431 BuiltinID = Builtin::BI__builtin_memchr;
4432 break;
4433
4434 case Builtin::BI__builtin___memcpy_chk: {
4435 // fold __builtin_memcpy_chk(x, y, cst1, cst2) to memcpy iff cst1<=cst2.
4436 Expr::EvalResult SizeResult, DstSizeResult;
4437 if (!E->getArg(Arg: 2)->EvaluateAsInt(Result&: SizeResult, Ctx: CGM.getContext()) ||
4438 !E->getArg(Arg: 3)->EvaluateAsInt(Result&: DstSizeResult, Ctx: CGM.getContext()))
4439 break;
4440 llvm::APSInt Size = SizeResult.Val.getInt();
4441 llvm::APSInt DstSize = DstSizeResult.Val.getInt();
4442 if (Size.ugt(RHS: DstSize))
4443 break;
4444 Address Dest = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4445 Address Src = EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
4446 Value *SizeVal = llvm::ConstantInt::get(Context&: Builder.getContext(), V: Size);
4447 auto *I = Builder.CreateMemCpy(Dest, Src, Size: SizeVal, IsVolatile: false);
4448 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
4449 return RValue::get(Addr: Dest, CGF&: *this);
4450 }
4451
4452 case Builtin::BI__builtin_objc_memmove_collectable: {
4453 Address DestAddr = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4454 Address SrcAddr = EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
4455 Value *SizeVal = EmitScalarExpr(E: E->getArg(Arg: 2));
4456 CGM.getObjCRuntime().EmitGCMemmoveCollectable(CGF&: *this,
4457 DestPtr: DestAddr, SrcPtr: SrcAddr, Size: SizeVal);
4458 return RValue::get(Addr: DestAddr, CGF&: *this);
4459 }
4460
4461 case Builtin::BI__builtin___memmove_chk: {
4462 // fold __builtin_memmove_chk(x, y, cst1, cst2) to memmove iff cst1<=cst2.
4463 Expr::EvalResult SizeResult, DstSizeResult;
4464 if (!E->getArg(Arg: 2)->EvaluateAsInt(Result&: SizeResult, Ctx: CGM.getContext()) ||
4465 !E->getArg(Arg: 3)->EvaluateAsInt(Result&: DstSizeResult, Ctx: CGM.getContext()))
4466 break;
4467 llvm::APSInt Size = SizeResult.Val.getInt();
4468 llvm::APSInt DstSize = DstSizeResult.Val.getInt();
4469 if (Size.ugt(RHS: DstSize))
4470 break;
4471 Address Dest = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4472 Address Src = EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
4473 Value *SizeVal = llvm::ConstantInt::get(Context&: Builder.getContext(), V: Size);
4474 auto *I = Builder.CreateMemMove(Dest, Src, Size: SizeVal, IsVolatile: false);
4475 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
4476 return RValue::get(Addr: Dest, CGF&: *this);
4477 }
4478
4479 case Builtin::BI__builtin_trivially_relocate:
4480 case Builtin::BImemmove:
4481 case Builtin::BI__builtin_memmove: {
4482 Address Dest = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4483 Address Src = EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
4484 Value *SizeVal = EmitScalarExpr(E: E->getArg(Arg: 2));
4485 if (BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_trivially_relocate)
4486 SizeVal = Builder.CreateMul(
4487 LHS: SizeVal,
4488 RHS: ConstantInt::get(
4489 Ty: SizeVal->getType(),
4490 V: getContext()
4491 .getTypeSizeInChars(T: E->getArg(Arg: 0)->getType()->getPointeeType())
4492 .getQuantity()));
4493 EmitArgCheck(TCK_Store, Dest, E->getArg(Arg: 0), 0);
4494 EmitArgCheck(TCK_Load, Src, E->getArg(Arg: 1), 1);
4495 auto *I = Builder.CreateMemMove(Dest, Src, Size: SizeVal, IsVolatile: false);
4496 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
4497 return RValue::get(Addr: Dest, CGF&: *this);
4498 }
4499 case Builtin::BImemset:
4500 case Builtin::BI__builtin_memset: {
4501 Address Dest = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4502 Value *ByteVal = Builder.CreateTrunc(V: EmitScalarExpr(E: E->getArg(Arg: 1)),
4503 DestTy: Builder.getInt8Ty());
4504 Value *SizeVal = EmitScalarExpr(E: E->getArg(Arg: 2));
4505 EmitNonNullArgCheck(Addr: Dest, ArgType: E->getArg(Arg: 0)->getType(),
4506 ArgLoc: E->getArg(Arg: 0)->getExprLoc(), AC: FD, ParmNum: 0);
4507 auto *I = Builder.CreateMemSet(Dest, Value: ByteVal, Size: SizeVal, IsVolatile: false);
4508 addInstToNewSourceAtom(KeyInstruction: I, Backup: ByteVal);
4509 return RValue::get(Addr: Dest, CGF&: *this);
4510 }
4511 case Builtin::BI__builtin_memset_inline: {
4512 Address Dest = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4513 Value *ByteVal =
4514 Builder.CreateTrunc(V: EmitScalarExpr(E: E->getArg(Arg: 1)), DestTy: Builder.getInt8Ty());
4515 uint64_t Size =
4516 E->getArg(Arg: 2)->EvaluateKnownConstInt(Ctx: getContext()).getZExtValue();
4517 EmitNonNullArgCheck(RV: RValue::get(V: Dest.emitRawPointer(CGF&: *this)),
4518 ArgType: E->getArg(Arg: 0)->getType(), ArgLoc: E->getArg(Arg: 0)->getExprLoc(), AC: FD,
4519 ParmNum: 0);
4520 auto *I = Builder.CreateMemSetInline(Dest, Value: ByteVal, Size);
4521 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
4522 return RValue::get(V: nullptr);
4523 }
4524 case Builtin::BI__builtin___memset_chk: {
4525 // fold __builtin_memset_chk(x, y, cst1, cst2) to memset iff cst1<=cst2.
4526 Expr::EvalResult SizeResult, DstSizeResult;
4527 if (!E->getArg(Arg: 2)->EvaluateAsInt(Result&: SizeResult, Ctx: CGM.getContext()) ||
4528 !E->getArg(Arg: 3)->EvaluateAsInt(Result&: DstSizeResult, Ctx: CGM.getContext()))
4529 break;
4530 llvm::APSInt Size = SizeResult.Val.getInt();
4531 llvm::APSInt DstSize = DstSizeResult.Val.getInt();
4532 if (Size.ugt(RHS: DstSize))
4533 break;
4534 Address Dest = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4535 Value *ByteVal = Builder.CreateTrunc(V: EmitScalarExpr(E: E->getArg(Arg: 1)),
4536 DestTy: Builder.getInt8Ty());
4537 Value *SizeVal = llvm::ConstantInt::get(Context&: Builder.getContext(), V: Size);
4538 auto *I = Builder.CreateMemSet(Dest, Value: ByteVal, Size: SizeVal, IsVolatile: false);
4539 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
4540 return RValue::get(Addr: Dest, CGF&: *this);
4541 }
4542 case Builtin::BI__builtin_wmemchr: {
4543 // The MSVC runtime library does not provide a definition of wmemchr, so we
4544 // need an inline implementation.
4545 if (!getTarget().getTriple().isOSMSVCRT())
4546 break;
4547
4548 llvm::Type *WCharTy = ConvertType(T: getContext().WCharTy);
4549 Value *Str = EmitScalarExpr(E: E->getArg(Arg: 0));
4550 Value *Chr = EmitScalarExpr(E: E->getArg(Arg: 1));
4551 Value *Size = EmitScalarExpr(E: E->getArg(Arg: 2));
4552
4553 BasicBlock *Entry = Builder.GetInsertBlock();
4554 BasicBlock *CmpEq = createBasicBlock(name: "wmemchr.eq");
4555 BasicBlock *Next = createBasicBlock(name: "wmemchr.next");
4556 BasicBlock *Exit = createBasicBlock(name: "wmemchr.exit");
4557 Value *SizeEq0 = Builder.CreateICmpEQ(LHS: Size, RHS: ConstantInt::get(Ty: SizeTy, V: 0));
4558 Builder.CreateCondBr(Cond: SizeEq0, True: Exit, False: CmpEq);
4559
4560 EmitBlock(BB: CmpEq);
4561 PHINode *StrPhi = Builder.CreatePHI(Ty: Str->getType(), NumReservedValues: 2);
4562 StrPhi->addIncoming(V: Str, BB: Entry);
4563 PHINode *SizePhi = Builder.CreatePHI(Ty: SizeTy, NumReservedValues: 2);
4564 SizePhi->addIncoming(V: Size, BB: Entry);
4565 CharUnits WCharAlign =
4566 getContext().getTypeAlignInChars(T: getContext().WCharTy);
4567 Value *StrCh = Builder.CreateAlignedLoad(Ty: WCharTy, Addr: StrPhi, Align: WCharAlign);
4568 Value *FoundChr = Builder.CreateConstInBoundsGEP1_32(Ty: WCharTy, Ptr: StrPhi, Idx0: 0);
4569 Value *StrEqChr = Builder.CreateICmpEQ(LHS: StrCh, RHS: Chr);
4570 Builder.CreateCondBr(Cond: StrEqChr, True: Exit, False: Next);
4571
4572 EmitBlock(BB: Next);
4573 Value *NextStr = Builder.CreateConstInBoundsGEP1_32(Ty: WCharTy, Ptr: StrPhi, Idx0: 1);
4574 Value *NextSize = Builder.CreateSub(LHS: SizePhi, RHS: ConstantInt::get(Ty: SizeTy, V: 1));
4575 Value *NextSizeEq0 =
4576 Builder.CreateICmpEQ(LHS: NextSize, RHS: ConstantInt::get(Ty: SizeTy, V: 0));
4577 Builder.CreateCondBr(Cond: NextSizeEq0, True: Exit, False: CmpEq);
4578 StrPhi->addIncoming(V: NextStr, BB: Next);
4579 SizePhi->addIncoming(V: NextSize, BB: Next);
4580
4581 EmitBlock(BB: Exit);
4582 PHINode *Ret = Builder.CreatePHI(Ty: Str->getType(), NumReservedValues: 3);
4583 Ret->addIncoming(V: llvm::Constant::getNullValue(Ty: Str->getType()), BB: Entry);
4584 Ret->addIncoming(V: llvm::Constant::getNullValue(Ty: Str->getType()), BB: Next);
4585 Ret->addIncoming(V: FoundChr, BB: CmpEq);
4586 return RValue::get(V: Ret);
4587 }
4588 case Builtin::BI__builtin_wmemcmp: {
4589 // The MSVC runtime library does not provide a definition of wmemcmp, so we
4590 // need an inline implementation.
4591 if (!getTarget().getTriple().isOSMSVCRT())
4592 break;
4593
4594 llvm::Type *WCharTy = ConvertType(T: getContext().WCharTy);
4595
4596 Value *Dst = EmitScalarExpr(E: E->getArg(Arg: 0));
4597 Value *Src = EmitScalarExpr(E: E->getArg(Arg: 1));
4598 Value *Size = EmitScalarExpr(E: E->getArg(Arg: 2));
4599
4600 BasicBlock *Entry = Builder.GetInsertBlock();
4601 BasicBlock *CmpGT = createBasicBlock(name: "wmemcmp.gt");
4602 BasicBlock *CmpLT = createBasicBlock(name: "wmemcmp.lt");
4603 BasicBlock *Next = createBasicBlock(name: "wmemcmp.next");
4604 BasicBlock *Exit = createBasicBlock(name: "wmemcmp.exit");
4605 Value *SizeEq0 = Builder.CreateICmpEQ(LHS: Size, RHS: ConstantInt::get(Ty: SizeTy, V: 0));
4606 Builder.CreateCondBr(Cond: SizeEq0, True: Exit, False: CmpGT);
4607
4608 EmitBlock(BB: CmpGT);
4609 PHINode *DstPhi = Builder.CreatePHI(Ty: Dst->getType(), NumReservedValues: 2);
4610 DstPhi->addIncoming(V: Dst, BB: Entry);
4611 PHINode *SrcPhi = Builder.CreatePHI(Ty: Src->getType(), NumReservedValues: 2);
4612 SrcPhi->addIncoming(V: Src, BB: Entry);
4613 PHINode *SizePhi = Builder.CreatePHI(Ty: SizeTy, NumReservedValues: 2);
4614 SizePhi->addIncoming(V: Size, BB: Entry);
4615 CharUnits WCharAlign =
4616 getContext().getTypeAlignInChars(T: getContext().WCharTy);
4617 Value *DstCh = Builder.CreateAlignedLoad(Ty: WCharTy, Addr: DstPhi, Align: WCharAlign);
4618 Value *SrcCh = Builder.CreateAlignedLoad(Ty: WCharTy, Addr: SrcPhi, Align: WCharAlign);
4619 Value *DstGtSrc = Builder.CreateICmpUGT(LHS: DstCh, RHS: SrcCh);
4620 Builder.CreateCondBr(Cond: DstGtSrc, True: Exit, False: CmpLT);
4621
4622 EmitBlock(BB: CmpLT);
4623 Value *DstLtSrc = Builder.CreateICmpULT(LHS: DstCh, RHS: SrcCh);
4624 Builder.CreateCondBr(Cond: DstLtSrc, True: Exit, False: Next);
4625
4626 EmitBlock(BB: Next);
4627 Value *NextDst = Builder.CreateConstInBoundsGEP1_32(Ty: WCharTy, Ptr: DstPhi, Idx0: 1);
4628 Value *NextSrc = Builder.CreateConstInBoundsGEP1_32(Ty: WCharTy, Ptr: SrcPhi, Idx0: 1);
4629 Value *NextSize = Builder.CreateSub(LHS: SizePhi, RHS: ConstantInt::get(Ty: SizeTy, V: 1));
4630 Value *NextSizeEq0 =
4631 Builder.CreateICmpEQ(LHS: NextSize, RHS: ConstantInt::get(Ty: SizeTy, V: 0));
4632 Builder.CreateCondBr(Cond: NextSizeEq0, True: Exit, False: CmpGT);
4633 DstPhi->addIncoming(V: NextDst, BB: Next);
4634 SrcPhi->addIncoming(V: NextSrc, BB: Next);
4635 SizePhi->addIncoming(V: NextSize, BB: Next);
4636
4637 EmitBlock(BB: Exit);
4638 PHINode *Ret = Builder.CreatePHI(Ty: IntTy, NumReservedValues: 4);
4639 Ret->addIncoming(V: ConstantInt::get(Ty: IntTy, V: 0), BB: Entry);
4640 Ret->addIncoming(V: ConstantInt::get(Ty: IntTy, V: 1), BB: CmpGT);
4641 Ret->addIncoming(V: ConstantInt::get(Ty: IntTy, V: -1), BB: CmpLT);
4642 Ret->addIncoming(V: ConstantInt::get(Ty: IntTy, V: 0), BB: Next);
4643 return RValue::get(V: Ret);
4644 }
4645 case Builtin::BI__builtin_dwarf_cfa: {
4646 // The offset in bytes from the first argument to the CFA.
4647 //
4648 // Why on earth is this in the frontend? Is there any reason at
4649 // all that the backend can't reasonably determine this while
4650 // lowering llvm.eh.dwarf.cfa()?
4651 //
4652 // TODO: If there's a satisfactory reason, add a target hook for
4653 // this instead of hard-coding 0, which is correct for most targets.
4654 int32_t Offset = 0;
4655
4656 Function *F = CGM.getIntrinsic(IID: Intrinsic::eh_dwarf_cfa);
4657 return RValue::get(V: Builder.CreateCall(Callee: F,
4658 Args: llvm::ConstantInt::get(Ty: Int32Ty, V: Offset)));
4659 }
4660 case Builtin::BI__builtin_return_address: {
4661 Value *Depth = ConstantEmitter(*this).emitAbstract(E: E->getArg(Arg: 0),
4662 T: getContext().UnsignedIntTy);
4663 Function *F = CGM.getIntrinsic(IID: Intrinsic::returnaddress);
4664 return RValue::get(V: Builder.CreateCall(Callee: F, Args: Depth));
4665 }
4666 case Builtin::BI_ReturnAddress: {
4667 Function *F = CGM.getIntrinsic(IID: Intrinsic::returnaddress);
4668 return RValue::get(V: Builder.CreateCall(Callee: F, Args: Builder.getInt32(C: 0)));
4669 }
4670 case Builtin::BI__builtin_frame_address: {
4671 Value *Depth = ConstantEmitter(*this).emitAbstract(E: E->getArg(Arg: 0),
4672 T: getContext().UnsignedIntTy);
4673 Function *F = CGM.getIntrinsic(IID: Intrinsic::frameaddress, Tys: AllocaInt8PtrTy);
4674 return RValue::get(V: Builder.CreateCall(Callee: F, Args: Depth));
4675 }
4676 case Builtin::BI__builtin_extract_return_addr: {
4677 Value *Address = EmitScalarExpr(E: E->getArg(Arg: 0));
4678 Value *Result = getTargetHooks().decodeReturnAddress(CGF&: *this, Address);
4679 return RValue::get(V: Result);
4680 }
4681 case Builtin::BI__builtin_frob_return_addr: {
4682 Value *Address = EmitScalarExpr(E: E->getArg(Arg: 0));
4683 Value *Result = getTargetHooks().encodeReturnAddress(CGF&: *this, Address);
4684 return RValue::get(V: Result);
4685 }
4686 case Builtin::BI__builtin_dwarf_sp_column: {
4687 llvm::IntegerType *Ty
4688 = cast<llvm::IntegerType>(Val: ConvertType(T: E->getType()));
4689 int Column = getTargetHooks().getDwarfEHStackPointer(M&: CGM);
4690 if (Column == -1) {
4691 CGM.ErrorUnsupported(S: E, Type: "__builtin_dwarf_sp_column");
4692 return RValue::get(V: llvm::UndefValue::get(T: Ty));
4693 }
4694 return RValue::get(V: llvm::ConstantInt::get(Ty, V: Column, IsSigned: true));
4695 }
4696 case Builtin::BI__builtin_init_dwarf_reg_size_table: {
4697 Value *Address = EmitScalarExpr(E: E->getArg(Arg: 0));
4698 if (getTargetHooks().initDwarfEHRegSizeTable(CGF&: *this, Address))
4699 CGM.ErrorUnsupported(S: E, Type: "__builtin_init_dwarf_reg_size_table");
4700 return RValue::get(V: llvm::UndefValue::get(T: ConvertType(T: E->getType())));
4701 }
4702 case Builtin::BI__builtin_eh_return: {
4703 Value *Int = EmitScalarExpr(E: E->getArg(Arg: 0));
4704 Value *Ptr = EmitScalarExpr(E: E->getArg(Arg: 1));
4705
4706 llvm::IntegerType *IntTy = cast<llvm::IntegerType>(Val: Int->getType());
4707 assert((IntTy->getBitWidth() == 32 || IntTy->getBitWidth() == 64) &&
4708 "LLVM's __builtin_eh_return only supports 32- and 64-bit variants");
4709 Function *F =
4710 CGM.getIntrinsic(IID: IntTy->getBitWidth() == 32 ? Intrinsic::eh_return_i32
4711 : Intrinsic::eh_return_i64);
4712 Builder.CreateCall(Callee: F, Args: {Int, Ptr});
4713 Builder.CreateUnreachable();
4714
4715 // We do need to preserve an insertion point.
4716 EmitBlock(BB: createBasicBlock(name: "builtin_eh_return.cont"));
4717
4718 return RValue::get(V: nullptr);
4719 }
4720 case Builtin::BI__builtin_unwind_init: {
4721 Function *F = CGM.getIntrinsic(IID: Intrinsic::eh_unwind_init);
4722 Builder.CreateCall(Callee: F);
4723 return RValue::get(V: nullptr);
4724 }
4725 case Builtin::BI__builtin_extend_pointer: {
4726 // Extends a pointer to the size of an _Unwind_Word, which is
4727 // uint64_t on all platforms. Generally this gets poked into a
4728 // register and eventually used as an address, so if the
4729 // addressing registers are wider than pointers and the platform
4730 // doesn't implicitly ignore high-order bits when doing
4731 // addressing, we need to make sure we zext / sext based on
4732 // the platform's expectations.
4733 //
4734 // See: http://gcc.gnu.org/ml/gcc-bugs/2002-02/msg00237.html
4735
4736 // Cast the pointer to intptr_t.
4737 Value *Ptr = EmitScalarExpr(E: E->getArg(Arg: 0));
4738 Value *Result = Builder.CreatePtrToInt(V: Ptr, DestTy: IntPtrTy, Name: "extend.cast");
4739
4740 // If that's 64 bits, we're done.
4741 if (IntPtrTy->getBitWidth() == 64)
4742 return RValue::get(V: Result);
4743
4744 // Otherwise, ask the codegen data what to do.
4745 if (getTargetHooks().extendPointerWithSExt())
4746 return RValue::get(V: Builder.CreateSExt(V: Result, DestTy: Int64Ty, Name: "extend.sext"));
4747 else
4748 return RValue::get(V: Builder.CreateZExt(V: Result, DestTy: Int64Ty, Name: "extend.zext"));
4749 }
4750 case Builtin::BI__builtin_setjmp: {
4751 // Buffer is a void**.
4752 Address Buf = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4753
4754 if (getTarget().getTriple().getArch() == llvm::Triple::systemz) {
4755 // On this target, the back end fills in the context buffer completely.
4756 // It doesn't really matter if the frontend stores to the buffer before
4757 // calling setjmp, the back-end is going to overwrite them anyway.
4758 Function *F = CGM.getIntrinsic(IID: Intrinsic::eh_sjlj_setjmp);
4759 return RValue::get(V: Builder.CreateCall(Callee: F, Args: Buf.emitRawPointer(CGF&: *this)));
4760 }
4761
4762 // Store the frame pointer to the setjmp buffer.
4763 Value *FrameAddr = Builder.CreateCall(
4764 Callee: CGM.getIntrinsic(IID: Intrinsic::frameaddress, Tys: AllocaInt8PtrTy),
4765 Args: ConstantInt::get(Ty: Int32Ty, V: 0));
4766 Builder.CreateStore(Val: FrameAddr, Addr: Buf);
4767
4768 // Store the stack pointer to the setjmp buffer.
4769 Value *StackAddr = Builder.CreateStackSave();
4770 assert(Buf.emitRawPointer(*this)->getType() == StackAddr->getType());
4771
4772 Address StackSaveSlot = Builder.CreateConstInBoundsGEP(Addr: Buf, Index: 2);
4773 Builder.CreateStore(Val: StackAddr, Addr: StackSaveSlot);
4774
4775 // Call LLVM's EH setjmp, which is lightweight.
4776 Function *F = CGM.getIntrinsic(IID: Intrinsic::eh_sjlj_setjmp);
4777 return RValue::get(V: Builder.CreateCall(Callee: F, Args: Buf.emitRawPointer(CGF&: *this)));
4778 }
4779 case Builtin::BI__builtin_longjmp: {
4780 Value *Buf = EmitScalarExpr(E: E->getArg(Arg: 0));
4781
4782 // Call LLVM's EH longjmp, which is lightweight.
4783 Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Intrinsic::eh_sjlj_longjmp), Args: Buf);
4784
4785 // longjmp doesn't return; mark this as unreachable.
4786 Builder.CreateUnreachable();
4787
4788 // We do need to preserve an insertion point.
4789 EmitBlock(BB: createBasicBlock(name: "longjmp.cont"));
4790
4791 return RValue::get(V: nullptr);
4792 }
4793 case Builtin::BI__builtin_launder: {
4794 const Expr *Arg = E->getArg(Arg: 0);
4795 QualType ArgTy = Arg->getType()->getPointeeType();
4796 Value *Ptr = EmitScalarExpr(E: Arg);
4797 if (TypeRequiresBuiltinLaunder(CGM, Ty: ArgTy))
4798 Ptr = Builder.CreateLaunderInvariantGroup(Ptr);
4799
4800 return RValue::get(V: Ptr);
4801 }
4802 case Builtin::BI__sync_fetch_and_add:
4803 case Builtin::BI__sync_fetch_and_sub:
4804 case Builtin::BI__sync_fetch_and_or:
4805 case Builtin::BI__sync_fetch_and_and:
4806 case Builtin::BI__sync_fetch_and_xor:
4807 case Builtin::BI__sync_fetch_and_nand:
4808 case Builtin::BI__sync_add_and_fetch:
4809 case Builtin::BI__sync_sub_and_fetch:
4810 case Builtin::BI__sync_and_and_fetch:
4811 case Builtin::BI__sync_or_and_fetch:
4812 case Builtin::BI__sync_xor_and_fetch:
4813 case Builtin::BI__sync_nand_and_fetch:
4814 case Builtin::BI__sync_val_compare_and_swap:
4815 case Builtin::BI__sync_bool_compare_and_swap:
4816 case Builtin::BI__sync_lock_test_and_set:
4817 case Builtin::BI__sync_lock_release:
4818 case Builtin::BI__sync_swap:
4819 llvm_unreachable("Shouldn't make it through sema");
4820 case Builtin::BI__sync_fetch_and_add_1:
4821 case Builtin::BI__sync_fetch_and_add_2:
4822 case Builtin::BI__sync_fetch_and_add_4:
4823 case Builtin::BI__sync_fetch_and_add_8:
4824 case Builtin::BI__sync_fetch_and_add_16:
4825 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::Add, E);
4826 case Builtin::BI__sync_fetch_and_sub_1:
4827 case Builtin::BI__sync_fetch_and_sub_2:
4828 case Builtin::BI__sync_fetch_and_sub_4:
4829 case Builtin::BI__sync_fetch_and_sub_8:
4830 case Builtin::BI__sync_fetch_and_sub_16:
4831 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::Sub, E);
4832 case Builtin::BI__sync_fetch_and_or_1:
4833 case Builtin::BI__sync_fetch_and_or_2:
4834 case Builtin::BI__sync_fetch_and_or_4:
4835 case Builtin::BI__sync_fetch_and_or_8:
4836 case Builtin::BI__sync_fetch_and_or_16:
4837 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::Or, E);
4838 case Builtin::BI__sync_fetch_and_and_1:
4839 case Builtin::BI__sync_fetch_and_and_2:
4840 case Builtin::BI__sync_fetch_and_and_4:
4841 case Builtin::BI__sync_fetch_and_and_8:
4842 case Builtin::BI__sync_fetch_and_and_16:
4843 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::And, E);
4844 case Builtin::BI__sync_fetch_and_xor_1:
4845 case Builtin::BI__sync_fetch_and_xor_2:
4846 case Builtin::BI__sync_fetch_and_xor_4:
4847 case Builtin::BI__sync_fetch_and_xor_8:
4848 case Builtin::BI__sync_fetch_and_xor_16:
4849 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::Xor, E);
4850 case Builtin::BI__sync_fetch_and_nand_1:
4851 case Builtin::BI__sync_fetch_and_nand_2:
4852 case Builtin::BI__sync_fetch_and_nand_4:
4853 case Builtin::BI__sync_fetch_and_nand_8:
4854 case Builtin::BI__sync_fetch_and_nand_16:
4855 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::Nand, E);
4856
4857 // Clang extensions: not overloaded yet.
4858 case Builtin::BI__sync_fetch_and_min:
4859 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::Min, E);
4860 case Builtin::BI__sync_fetch_and_max:
4861 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::Max, E);
4862 case Builtin::BI__sync_fetch_and_umin:
4863 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::UMin, E);
4864 case Builtin::BI__sync_fetch_and_umax:
4865 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::UMax, E);
4866
4867 case Builtin::BI__sync_add_and_fetch_1:
4868 case Builtin::BI__sync_add_and_fetch_2:
4869 case Builtin::BI__sync_add_and_fetch_4:
4870 case Builtin::BI__sync_add_and_fetch_8:
4871 case Builtin::BI__sync_add_and_fetch_16:
4872 return EmitBinaryAtomicPost(CGF&: *this, Kind: llvm::AtomicRMWInst::Add, E,
4873 Op: llvm::Instruction::Add);
4874 case Builtin::BI__sync_sub_and_fetch_1:
4875 case Builtin::BI__sync_sub_and_fetch_2:
4876 case Builtin::BI__sync_sub_and_fetch_4:
4877 case Builtin::BI__sync_sub_and_fetch_8:
4878 case Builtin::BI__sync_sub_and_fetch_16:
4879 return EmitBinaryAtomicPost(CGF&: *this, Kind: llvm::AtomicRMWInst::Sub, E,
4880 Op: llvm::Instruction::Sub);
4881 case Builtin::BI__sync_and_and_fetch_1:
4882 case Builtin::BI__sync_and_and_fetch_2:
4883 case Builtin::BI__sync_and_and_fetch_4:
4884 case Builtin::BI__sync_and_and_fetch_8:
4885 case Builtin::BI__sync_and_and_fetch_16:
4886 return EmitBinaryAtomicPost(CGF&: *this, Kind: llvm::AtomicRMWInst::And, E,
4887 Op: llvm::Instruction::And);
4888 case Builtin::BI__sync_or_and_fetch_1:
4889 case Builtin::BI__sync_or_and_fetch_2:
4890 case Builtin::BI__sync_or_and_fetch_4:
4891 case Builtin::BI__sync_or_and_fetch_8:
4892 case Builtin::BI__sync_or_and_fetch_16:
4893 return EmitBinaryAtomicPost(CGF&: *this, Kind: llvm::AtomicRMWInst::Or, E,
4894 Op: llvm::Instruction::Or);
4895 case Builtin::BI__sync_xor_and_fetch_1:
4896 case Builtin::BI__sync_xor_and_fetch_2:
4897 case Builtin::BI__sync_xor_and_fetch_4:
4898 case Builtin::BI__sync_xor_and_fetch_8:
4899 case Builtin::BI__sync_xor_and_fetch_16:
4900 return EmitBinaryAtomicPost(CGF&: *this, Kind: llvm::AtomicRMWInst::Xor, E,
4901 Op: llvm::Instruction::Xor);
4902 case Builtin::BI__sync_nand_and_fetch_1:
4903 case Builtin::BI__sync_nand_and_fetch_2:
4904 case Builtin::BI__sync_nand_and_fetch_4:
4905 case Builtin::BI__sync_nand_and_fetch_8:
4906 case Builtin::BI__sync_nand_and_fetch_16:
4907 return EmitBinaryAtomicPost(CGF&: *this, Kind: llvm::AtomicRMWInst::Nand, E,
4908 Op: llvm::Instruction::And, Invert: true);
4909
4910 case Builtin::BI__sync_val_compare_and_swap_1:
4911 case Builtin::BI__sync_val_compare_and_swap_2:
4912 case Builtin::BI__sync_val_compare_and_swap_4:
4913 case Builtin::BI__sync_val_compare_and_swap_8:
4914 case Builtin::BI__sync_val_compare_and_swap_16:
4915 return RValue::get(V: MakeAtomicCmpXchgValue(CGF&: *this, E, ReturnBool: false));
4916
4917 case Builtin::BI__sync_bool_compare_and_swap_1:
4918 case Builtin::BI__sync_bool_compare_and_swap_2:
4919 case Builtin::BI__sync_bool_compare_and_swap_4:
4920 case Builtin::BI__sync_bool_compare_and_swap_8:
4921 case Builtin::BI__sync_bool_compare_and_swap_16:
4922 return RValue::get(V: MakeAtomicCmpXchgValue(CGF&: *this, E, ReturnBool: true));
4923
4924 case Builtin::BI__sync_swap_1:
4925 case Builtin::BI__sync_swap_2:
4926 case Builtin::BI__sync_swap_4:
4927 case Builtin::BI__sync_swap_8:
4928 case Builtin::BI__sync_swap_16:
4929 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::Xchg, E);
4930
4931 case Builtin::BI__sync_lock_test_and_set_1:
4932 case Builtin::BI__sync_lock_test_and_set_2:
4933 case Builtin::BI__sync_lock_test_and_set_4:
4934 case Builtin::BI__sync_lock_test_and_set_8:
4935 case Builtin::BI__sync_lock_test_and_set_16:
4936 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::Xchg, E);
4937
4938 case Builtin::BI__sync_lock_release_1:
4939 case Builtin::BI__sync_lock_release_2:
4940 case Builtin::BI__sync_lock_release_4:
4941 case Builtin::BI__sync_lock_release_8:
4942 case Builtin::BI__sync_lock_release_16: {
4943 Address Ptr = CheckAtomicAlignment(CGF&: *this, E);
4944 QualType ElTy = E->getArg(Arg: 0)->getType()->getPointeeType();
4945
4946 llvm::Type *ITy = llvm::IntegerType::get(C&: getLLVMContext(),
4947 NumBits: getContext().getTypeSize(T: ElTy));
4948 llvm::StoreInst *Store =
4949 Builder.CreateStore(Val: llvm::Constant::getNullValue(Ty: ITy), Addr: Ptr);
4950 Store->setAtomic(Ordering: llvm::AtomicOrdering::Release);
4951 return RValue::get(V: nullptr);
4952 }
4953
4954 case Builtin::BI__sync_synchronize: {
4955 // We assume this is supposed to correspond to a C++0x-style
4956 // sequentially-consistent fence (i.e. this is only usable for
4957 // synchronization, not device I/O or anything like that). This intrinsic
4958 // is really badly designed in the sense that in theory, there isn't
4959 // any way to safely use it... but in practice, it mostly works
4960 // to use it with non-atomic loads and stores to get acquire/release
4961 // semantics.
4962 Builder.CreateFence(Ordering: llvm::AtomicOrdering::SequentiallyConsistent);
4963 return RValue::get(V: nullptr);
4964 }
4965
4966 case Builtin::BI__builtin_nontemporal_load:
4967 return RValue::get(V: EmitNontemporalLoad(CGF&: *this, E));
4968 case Builtin::BI__builtin_nontemporal_store:
4969 return RValue::get(V: EmitNontemporalStore(CGF&: *this, E));
4970 case Builtin::BI__c11_atomic_is_lock_free:
4971 case Builtin::BI__atomic_is_lock_free: {
4972 // Call "bool __atomic_is_lock_free(size_t size, void *ptr)". For the
4973 // __c11 builtin, ptr is 0 (indicating a properly-aligned object), since
4974 // _Atomic(T) is always properly-aligned.
4975 const char *LibCallName = "__atomic_is_lock_free";
4976 CallArgList Args;
4977 Args.add(rvalue: RValue::get(V: EmitScalarExpr(E: E->getArg(Arg: 0))),
4978 type: getContext().getSizeType());
4979 if (BuiltinID == Builtin::BI__atomic_is_lock_free)
4980 Args.add(rvalue: RValue::get(V: EmitScalarExpr(E: E->getArg(Arg: 1))),
4981 type: getContext().VoidPtrTy);
4982 else
4983 Args.add(rvalue: RValue::get(V: llvm::Constant::getNullValue(Ty: VoidPtrTy)),
4984 type: getContext().VoidPtrTy);
4985 const CGFunctionInfo &FuncInfo =
4986 CGM.getTypes().arrangeBuiltinFunctionCall(resultType: E->getType(), args: Args);
4987 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(Info: FuncInfo);
4988 llvm::FunctionCallee Func = CGM.CreateRuntimeFunction(Ty: FTy, Name: LibCallName);
4989 return EmitCall(CallInfo: FuncInfo, Callee: CGCallee::forDirect(functionPtr: Func),
4990 ReturnValue: ReturnValueSlot(), Args);
4991 }
4992
4993 case Builtin::BI__atomic_thread_fence:
4994 case Builtin::BI__atomic_signal_fence:
4995 case Builtin::BI__c11_atomic_thread_fence:
4996 case Builtin::BI__c11_atomic_signal_fence: {
4997 llvm::SyncScope::ID SSID;
4998 if (BuiltinID == Builtin::BI__atomic_signal_fence ||
4999 BuiltinID == Builtin::BI__c11_atomic_signal_fence)
5000 SSID = llvm::SyncScope::SingleThread;
5001 else
5002 SSID = llvm::SyncScope::System;
5003 Value *Order = EmitScalarExpr(E: E->getArg(Arg: 0));
5004 if (isa<llvm::ConstantInt>(Val: Order)) {
5005 int ord = cast<llvm::ConstantInt>(Val: Order)->getZExtValue();
5006 switch (ord) {
5007 case 0: // memory_order_relaxed
5008 default: // invalid order
5009 break;
5010 case 1: // memory_order_consume
5011 case 2: // memory_order_acquire
5012 Builder.CreateFence(Ordering: llvm::AtomicOrdering::Acquire, SSID);
5013 break;
5014 case 3: // memory_order_release
5015 Builder.CreateFence(Ordering: llvm::AtomicOrdering::Release, SSID);
5016 break;
5017 case 4: // memory_order_acq_rel
5018 Builder.CreateFence(Ordering: llvm::AtomicOrdering::AcquireRelease, SSID);
5019 break;
5020 case 5: // memory_order_seq_cst
5021 Builder.CreateFence(Ordering: llvm::AtomicOrdering::SequentiallyConsistent, SSID);
5022 break;
5023 }
5024 return RValue::get(V: nullptr);
5025 }
5026
5027 llvm::BasicBlock *AcquireBB, *ReleaseBB, *AcqRelBB, *SeqCstBB;
5028 AcquireBB = createBasicBlock(name: "acquire", parent: CurFn);
5029 ReleaseBB = createBasicBlock(name: "release", parent: CurFn);
5030 AcqRelBB = createBasicBlock(name: "acqrel", parent: CurFn);
5031 SeqCstBB = createBasicBlock(name: "seqcst", parent: CurFn);
5032 llvm::BasicBlock *ContBB = createBasicBlock(name: "atomic.continue", parent: CurFn);
5033
5034 Order = Builder.CreateIntCast(V: Order, DestTy: Builder.getInt32Ty(), isSigned: false);
5035 llvm::SwitchInst *SI = Builder.CreateSwitch(V: Order, Dest: ContBB);
5036
5037 Builder.SetInsertPoint(AcquireBB);
5038 Builder.CreateFence(Ordering: llvm::AtomicOrdering::Acquire, SSID);
5039 Builder.CreateBr(Dest: ContBB);
5040 SI->addCase(OnVal: Builder.getInt32(C: 1), Dest: AcquireBB);
5041 SI->addCase(OnVal: Builder.getInt32(C: 2), Dest: AcquireBB);
5042
5043 Builder.SetInsertPoint(ReleaseBB);
5044 Builder.CreateFence(Ordering: llvm::AtomicOrdering::Release, SSID);
5045 Builder.CreateBr(Dest: ContBB);
5046 SI->addCase(OnVal: Builder.getInt32(C: 3), Dest: ReleaseBB);
5047
5048 Builder.SetInsertPoint(AcqRelBB);
5049 Builder.CreateFence(Ordering: llvm::AtomicOrdering::AcquireRelease, SSID);
5050 Builder.CreateBr(Dest: ContBB);
5051 SI->addCase(OnVal: Builder.getInt32(C: 4), Dest: AcqRelBB);
5052
5053 Builder.SetInsertPoint(SeqCstBB);
5054 Builder.CreateFence(Ordering: llvm::AtomicOrdering::SequentiallyConsistent, SSID);
5055 Builder.CreateBr(Dest: ContBB);
5056 SI->addCase(OnVal: Builder.getInt32(C: 5), Dest: SeqCstBB);
5057
5058 Builder.SetInsertPoint(ContBB);
5059 return RValue::get(V: nullptr);
5060 }
5061 case Builtin::BI__scoped_atomic_thread_fence: {
5062 auto ScopeModel = AtomicScopeModel::create(K: AtomicScopeModelKind::Generic);
5063
5064 Value *Order = EmitScalarExpr(E: E->getArg(Arg: 0));
5065 Value *Scope = EmitScalarExpr(E: E->getArg(Arg: 1));
5066 auto Ord = dyn_cast<llvm::ConstantInt>(Val: Order);
5067 auto Scp = dyn_cast<llvm::ConstantInt>(Val: Scope);
5068 if (Ord && Scp) {
5069 SyncScope SS = ScopeModel->isValid(S: Scp->getZExtValue())
5070 ? ScopeModel->map(S: Scp->getZExtValue())
5071 : ScopeModel->map(S: ScopeModel->getFallBackValue());
5072 switch (Ord->getZExtValue()) {
5073 case 0: // memory_order_relaxed
5074 default: // invalid order
5075 break;
5076 case 1: // memory_order_consume
5077 case 2: // memory_order_acquire
5078 Builder.CreateFence(
5079 Ordering: llvm::AtomicOrdering::Acquire,
5080 SSID: getTargetHooks().getLLVMSyncScopeID(LangOpts: getLangOpts(), Scope: SS,
5081 Ordering: llvm::AtomicOrdering::Acquire,
5082 Ctx&: getLLVMContext()));
5083 break;
5084 case 3: // memory_order_release
5085 Builder.CreateFence(
5086 Ordering: llvm::AtomicOrdering::Release,
5087 SSID: getTargetHooks().getLLVMSyncScopeID(LangOpts: getLangOpts(), Scope: SS,
5088 Ordering: llvm::AtomicOrdering::Release,
5089 Ctx&: getLLVMContext()));
5090 break;
5091 case 4: // memory_order_acq_rel
5092 Builder.CreateFence(Ordering: llvm::AtomicOrdering::AcquireRelease,
5093 SSID: getTargetHooks().getLLVMSyncScopeID(
5094 LangOpts: getLangOpts(), Scope: SS,
5095 Ordering: llvm::AtomicOrdering::AcquireRelease,
5096 Ctx&: getLLVMContext()));
5097 break;
5098 case 5: // memory_order_seq_cst
5099 Builder.CreateFence(Ordering: llvm::AtomicOrdering::SequentiallyConsistent,
5100 SSID: getTargetHooks().getLLVMSyncScopeID(
5101 LangOpts: getLangOpts(), Scope: SS,
5102 Ordering: llvm::AtomicOrdering::SequentiallyConsistent,
5103 Ctx&: getLLVMContext()));
5104 break;
5105 }
5106 return RValue::get(V: nullptr);
5107 }
5108
5109 llvm::BasicBlock *ContBB = createBasicBlock(name: "atomic.scope.continue", parent: CurFn);
5110
5111 llvm::SmallVector<std::pair<llvm::BasicBlock *, llvm::AtomicOrdering>>
5112 OrderBBs;
5113 if (Ord) {
5114 switch (Ord->getZExtValue()) {
5115 case 0: // memory_order_relaxed
5116 default: // invalid order
5117 ContBB->eraseFromParent();
5118 return RValue::get(V: nullptr);
5119 case 1: // memory_order_consume
5120 case 2: // memory_order_acquire
5121 OrderBBs.emplace_back(Args: Builder.GetInsertBlock(),
5122 Args: llvm::AtomicOrdering::Acquire);
5123 break;
5124 case 3: // memory_order_release
5125 OrderBBs.emplace_back(Args: Builder.GetInsertBlock(),
5126 Args: llvm::AtomicOrdering::Release);
5127 break;
5128 case 4: // memory_order_acq_rel
5129 OrderBBs.emplace_back(Args: Builder.GetInsertBlock(),
5130 Args: llvm::AtomicOrdering::AcquireRelease);
5131 break;
5132 case 5: // memory_order_seq_cst
5133 OrderBBs.emplace_back(Args: Builder.GetInsertBlock(),
5134 Args: llvm::AtomicOrdering::SequentiallyConsistent);
5135 break;
5136 }
5137 } else {
5138 llvm::BasicBlock *AcquireBB = createBasicBlock(name: "acquire", parent: CurFn);
5139 llvm::BasicBlock *ReleaseBB = createBasicBlock(name: "release", parent: CurFn);
5140 llvm::BasicBlock *AcqRelBB = createBasicBlock(name: "acqrel", parent: CurFn);
5141 llvm::BasicBlock *SeqCstBB = createBasicBlock(name: "seqcst", parent: CurFn);
5142
5143 Order = Builder.CreateIntCast(V: Order, DestTy: Builder.getInt32Ty(), isSigned: false);
5144 llvm::SwitchInst *SI = Builder.CreateSwitch(V: Order, Dest: ContBB);
5145 SI->addCase(OnVal: Builder.getInt32(C: 1), Dest: AcquireBB);
5146 SI->addCase(OnVal: Builder.getInt32(C: 2), Dest: AcquireBB);
5147 SI->addCase(OnVal: Builder.getInt32(C: 3), Dest: ReleaseBB);
5148 SI->addCase(OnVal: Builder.getInt32(C: 4), Dest: AcqRelBB);
5149 SI->addCase(OnVal: Builder.getInt32(C: 5), Dest: SeqCstBB);
5150
5151 OrderBBs.emplace_back(Args&: AcquireBB, Args: llvm::AtomicOrdering::Acquire);
5152 OrderBBs.emplace_back(Args&: ReleaseBB, Args: llvm::AtomicOrdering::Release);
5153 OrderBBs.emplace_back(Args&: AcqRelBB, Args: llvm::AtomicOrdering::AcquireRelease);
5154 OrderBBs.emplace_back(Args&: SeqCstBB,
5155 Args: llvm::AtomicOrdering::SequentiallyConsistent);
5156 }
5157
5158 for (auto &[OrderBB, Ordering] : OrderBBs) {
5159 Builder.SetInsertPoint(OrderBB);
5160 if (Scp) {
5161 SyncScope SS = ScopeModel->isValid(S: Scp->getZExtValue())
5162 ? ScopeModel->map(S: Scp->getZExtValue())
5163 : ScopeModel->map(S: ScopeModel->getFallBackValue());
5164 Builder.CreateFence(Ordering,
5165 SSID: getTargetHooks().getLLVMSyncScopeID(
5166 LangOpts: getLangOpts(), Scope: SS, Ordering, Ctx&: getLLVMContext()));
5167 Builder.CreateBr(Dest: ContBB);
5168 } else {
5169 llvm::DenseMap<unsigned, llvm::BasicBlock *> BBs;
5170 for (unsigned Scp : ScopeModel->getRuntimeValues())
5171 BBs[Scp] = createBasicBlock(name: getAsString(S: ScopeModel->map(S: Scp)), parent: CurFn);
5172
5173 auto *SC = Builder.CreateIntCast(V: Scope, DestTy: Builder.getInt32Ty(), isSigned: false);
5174 llvm::SwitchInst *SI = Builder.CreateSwitch(V: SC, Dest: ContBB);
5175 for (unsigned Scp : ScopeModel->getRuntimeValues()) {
5176 auto *B = BBs[Scp];
5177 SI->addCase(OnVal: Builder.getInt32(C: Scp), Dest: B);
5178
5179 Builder.SetInsertPoint(B);
5180 Builder.CreateFence(Ordering, SSID: getTargetHooks().getLLVMSyncScopeID(
5181 LangOpts: getLangOpts(), Scope: ScopeModel->map(S: Scp),
5182 Ordering, Ctx&: getLLVMContext()));
5183 Builder.CreateBr(Dest: ContBB);
5184 }
5185 }
5186 }
5187
5188 Builder.SetInsertPoint(ContBB);
5189 return RValue::get(V: nullptr);
5190 }
5191
5192 case Builtin::BI__builtin_signbit:
5193 case Builtin::BI__builtin_signbitf:
5194 case Builtin::BI__builtin_signbitl: {
5195 return RValue::get(
5196 V: Builder.CreateZExt(V: EmitSignBit(CGF&: *this, V: EmitScalarExpr(E: E->getArg(Arg: 0))),
5197 DestTy: ConvertType(T: E->getType())));
5198 }
5199 case Builtin::BI__warn_memset_zero_len:
5200 return RValue::getIgnored();
5201 case Builtin::BI__annotation: {
5202 // Re-encode each wide string to UTF8 and make an MDString.
5203 SmallVector<Metadata *, 1> Strings;
5204 for (const Expr *Arg : E->arguments()) {
5205 const auto *Str = cast<StringLiteral>(Val: Arg->IgnoreParenCasts());
5206 assert(Str->getCharByteWidth() == 2);
5207 StringRef WideBytes = Str->getBytes();
5208 std::string StrUtf8;
5209 if (!convertUTF16ToUTF8String(
5210 SrcBytes: ArrayRef(WideBytes.data(), WideBytes.size()), Out&: StrUtf8)) {
5211 CGM.ErrorUnsupported(S: E, Type: "non-UTF16 __annotation argument");
5212 continue;
5213 }
5214 Strings.push_back(Elt: llvm::MDString::get(Context&: getLLVMContext(), Str: StrUtf8));
5215 }
5216
5217 // Build and MDTuple of MDStrings and emit the intrinsic call.
5218 llvm::Function *F = CGM.getIntrinsic(IID: Intrinsic::codeview_annotation, Tys: {});
5219 MDTuple *StrTuple = MDTuple::get(Context&: getLLVMContext(), MDs: Strings);
5220 Builder.CreateCall(Callee: F, Args: MetadataAsValue::get(Context&: getLLVMContext(), MD: StrTuple));
5221 return RValue::getIgnored();
5222 }
5223 case Builtin::BI__builtin_annotation: {
5224 llvm::Value *AnnVal = EmitScalarExpr(E: E->getArg(Arg: 0));
5225 llvm::Function *F = CGM.getIntrinsic(
5226 IID: Intrinsic::annotation, Tys: {AnnVal->getType(), CGM.ConstGlobalsPtrTy});
5227
5228 // Get the annotation string, go through casts. Sema requires this to be a
5229 // non-wide string literal, potentially casted, so the cast<> is safe.
5230 const Expr *AnnotationStrExpr = E->getArg(Arg: 1)->IgnoreParenCasts();
5231 StringRef Str = cast<StringLiteral>(Val: AnnotationStrExpr)->getString();
5232 return RValue::get(
5233 V: EmitAnnotationCall(AnnotationFn: F, AnnotatedVal: AnnVal, AnnotationStr: Str, Location: E->getExprLoc(), Attr: nullptr));
5234 }
5235 case Builtin::BI__builtin_addcb:
5236 case Builtin::BI__builtin_addcs:
5237 case Builtin::BI__builtin_addc:
5238 case Builtin::BI__builtin_addcl:
5239 case Builtin::BI__builtin_addcll:
5240 case Builtin::BI__builtin_subcb:
5241 case Builtin::BI__builtin_subcs:
5242 case Builtin::BI__builtin_subc:
5243 case Builtin::BI__builtin_subcl:
5244 case Builtin::BI__builtin_subcll: {
5245
5246 // We translate all of these builtins from expressions of the form:
5247 // int x = ..., y = ..., carryin = ..., carryout, result;
5248 // result = __builtin_addc(x, y, carryin, &carryout);
5249 //
5250 // to LLVM IR of the form:
5251 //
5252 // %tmp1 = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %x, i32 %y)
5253 // %tmpsum1 = extractvalue {i32, i1} %tmp1, 0
5254 // %carry1 = extractvalue {i32, i1} %tmp1, 1
5255 // %tmp2 = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %tmpsum1,
5256 // i32 %carryin)
5257 // %result = extractvalue {i32, i1} %tmp2, 0
5258 // %carry2 = extractvalue {i32, i1} %tmp2, 1
5259 // %tmp3 = or i1 %carry1, %carry2
5260 // %tmp4 = zext i1 %tmp3 to i32
5261 // store i32 %tmp4, i32* %carryout
5262
5263 // Scalarize our inputs.
5264 llvm::Value *X = EmitScalarExpr(E: E->getArg(Arg: 0));
5265 llvm::Value *Y = EmitScalarExpr(E: E->getArg(Arg: 1));
5266 llvm::Value *Carryin = EmitScalarExpr(E: E->getArg(Arg: 2));
5267 Address CarryOutPtr = EmitPointerWithAlignment(Addr: E->getArg(Arg: 3));
5268
5269 // Decide if we are lowering to a uadd.with.overflow or usub.with.overflow.
5270 Intrinsic::ID IntrinsicId;
5271 switch (BuiltinID) {
5272 default: llvm_unreachable("Unknown multiprecision builtin id.");
5273 case Builtin::BI__builtin_addcb:
5274 case Builtin::BI__builtin_addcs:
5275 case Builtin::BI__builtin_addc:
5276 case Builtin::BI__builtin_addcl:
5277 case Builtin::BI__builtin_addcll:
5278 IntrinsicId = Intrinsic::uadd_with_overflow;
5279 break;
5280 case Builtin::BI__builtin_subcb:
5281 case Builtin::BI__builtin_subcs:
5282 case Builtin::BI__builtin_subc:
5283 case Builtin::BI__builtin_subcl:
5284 case Builtin::BI__builtin_subcll:
5285 IntrinsicId = Intrinsic::usub_with_overflow;
5286 break;
5287 }
5288
5289 // Construct our resulting LLVM IR expression.
5290 llvm::Value *Carry1;
5291 llvm::Value *Sum1 = EmitOverflowIntrinsic(CGF&: *this, IntrinsicID: IntrinsicId,
5292 X, Y, Carry&: Carry1);
5293 llvm::Value *Carry2;
5294 llvm::Value *Sum2 = EmitOverflowIntrinsic(CGF&: *this, IntrinsicID: IntrinsicId,
5295 X: Sum1, Y: Carryin, Carry&: Carry2);
5296 llvm::Value *CarryOut = Builder.CreateZExt(V: Builder.CreateOr(LHS: Carry1, RHS: Carry2),
5297 DestTy: X->getType());
5298 Builder.CreateStore(Val: CarryOut, Addr: CarryOutPtr);
5299 return RValue::get(V: Sum2);
5300 }
5301
5302 case Builtin::BI__builtin_add_overflow:
5303 case Builtin::BI__builtin_sub_overflow:
5304 case Builtin::BI__builtin_mul_overflow: {
5305 const clang::Expr *LeftArg = E->getArg(Arg: 0);
5306 const clang::Expr *RightArg = E->getArg(Arg: 1);
5307 const clang::Expr *ResultArg = E->getArg(Arg: 2);
5308
5309 clang::QualType ResultQTy =
5310 ResultArg->getType()->castAs<PointerType>()->getPointeeType();
5311
5312 WidthAndSignedness LeftInfo =
5313 getIntegerWidthAndSignedness(context: CGM.getContext(), Type: LeftArg->getType());
5314 WidthAndSignedness RightInfo =
5315 getIntegerWidthAndSignedness(context: CGM.getContext(), Type: RightArg->getType());
5316 WidthAndSignedness ResultInfo =
5317 getIntegerWidthAndSignedness(context: CGM.getContext(), Type: ResultQTy);
5318
5319 // Handle mixed-sign multiplication as a special case, because adding
5320 // runtime or backend support for our generic irgen would be too expensive.
5321 if (isSpecialMixedSignMultiply(BuiltinID, Op1Info: LeftInfo, Op2Info: RightInfo, ResultInfo))
5322 return EmitCheckedMixedSignMultiply(CGF&: *this, Op1: LeftArg, Op1Info: LeftInfo, Op2: RightArg,
5323 Op2Info: RightInfo, ResultArg, ResultQTy,
5324 ResultInfo);
5325
5326 if (isSpecialUnsignedMultiplySignedResult(BuiltinID, Op1Info: LeftInfo, Op2Info: RightInfo,
5327 ResultInfo))
5328 return EmitCheckedUnsignedMultiplySignedResult(
5329 CGF&: *this, Op1: LeftArg, Op1Info: LeftInfo, Op2: RightArg, Op2Info: RightInfo, ResultArg, ResultQTy,
5330 ResultInfo);
5331
5332 WidthAndSignedness EncompassingInfo =
5333 EncompassingIntegerType(Types: {LeftInfo, RightInfo, ResultInfo});
5334
5335 llvm::Type *EncompassingLLVMTy =
5336 llvm::IntegerType::get(C&: CGM.getLLVMContext(), NumBits: EncompassingInfo.Width);
5337
5338 llvm::Type *ResultLLVMTy = CGM.getTypes().ConvertType(T: ResultQTy);
5339
5340 Intrinsic::ID IntrinsicId;
5341 switch (BuiltinID) {
5342 default:
5343 llvm_unreachable("Unknown overflow builtin id.");
5344 case Builtin::BI__builtin_add_overflow:
5345 IntrinsicId = EncompassingInfo.Signed ? Intrinsic::sadd_with_overflow
5346 : Intrinsic::uadd_with_overflow;
5347 break;
5348 case Builtin::BI__builtin_sub_overflow:
5349 IntrinsicId = EncompassingInfo.Signed ? Intrinsic::ssub_with_overflow
5350 : Intrinsic::usub_with_overflow;
5351 break;
5352 case Builtin::BI__builtin_mul_overflow:
5353 IntrinsicId = EncompassingInfo.Signed ? Intrinsic::smul_with_overflow
5354 : Intrinsic::umul_with_overflow;
5355 break;
5356 }
5357
5358 llvm::Value *Left = EmitScalarExpr(E: LeftArg);
5359 llvm::Value *Right = EmitScalarExpr(E: RightArg);
5360 Address ResultPtr = EmitPointerWithAlignment(Addr: ResultArg);
5361
5362 // Extend each operand to the encompassing type.
5363 Left = Builder.CreateIntCast(V: Left, DestTy: EncompassingLLVMTy, isSigned: LeftInfo.Signed);
5364 Right = Builder.CreateIntCast(V: Right, DestTy: EncompassingLLVMTy, isSigned: RightInfo.Signed);
5365
5366 // Perform the operation on the extended values.
5367 llvm::Value *Overflow, *Result;
5368 Result = EmitOverflowIntrinsic(CGF&: *this, IntrinsicID: IntrinsicId, X: Left, Y: Right, Carry&: Overflow);
5369
5370 if (EncompassingInfo.Width > ResultInfo.Width) {
5371 // The encompassing type is wider than the result type, so we need to
5372 // truncate it.
5373 llvm::Value *ResultTrunc = Builder.CreateTrunc(V: Result, DestTy: ResultLLVMTy);
5374
5375 // To see if the truncation caused an overflow, we will extend
5376 // the result and then compare it to the original result.
5377 llvm::Value *ResultTruncExt = Builder.CreateIntCast(
5378 V: ResultTrunc, DestTy: EncompassingLLVMTy, isSigned: ResultInfo.Signed);
5379 llvm::Value *TruncationOverflow =
5380 Builder.CreateICmpNE(LHS: Result, RHS: ResultTruncExt);
5381
5382 Overflow = Builder.CreateOr(LHS: Overflow, RHS: TruncationOverflow);
5383 Result = ResultTrunc;
5384 }
5385
5386 // Finally, store the result using the pointer.
5387 bool isVolatile =
5388 ResultArg->getType()->getPointeeType().isVolatileQualified();
5389 Builder.CreateStore(Val: EmitToMemory(Value: Result, Ty: ResultQTy), Addr: ResultPtr, IsVolatile: isVolatile);
5390
5391 return RValue::get(V: Overflow);
5392 }
5393
5394 case Builtin::BI__builtin_uadd_overflow:
5395 case Builtin::BI__builtin_uaddl_overflow:
5396 case Builtin::BI__builtin_uaddll_overflow:
5397 case Builtin::BI__builtin_usub_overflow:
5398 case Builtin::BI__builtin_usubl_overflow:
5399 case Builtin::BI__builtin_usubll_overflow:
5400 case Builtin::BI__builtin_umul_overflow:
5401 case Builtin::BI__builtin_umull_overflow:
5402 case Builtin::BI__builtin_umulll_overflow:
5403 case Builtin::BI__builtin_sadd_overflow:
5404 case Builtin::BI__builtin_saddl_overflow:
5405 case Builtin::BI__builtin_saddll_overflow:
5406 case Builtin::BI__builtin_ssub_overflow:
5407 case Builtin::BI__builtin_ssubl_overflow:
5408 case Builtin::BI__builtin_ssubll_overflow:
5409 case Builtin::BI__builtin_smul_overflow:
5410 case Builtin::BI__builtin_smull_overflow:
5411 case Builtin::BI__builtin_smulll_overflow: {
5412
5413 // We translate all of these builtins directly to the relevant llvm IR node.
5414
5415 // Scalarize our inputs.
5416 llvm::Value *X = EmitScalarExpr(E: E->getArg(Arg: 0));
5417 llvm::Value *Y = EmitScalarExpr(E: E->getArg(Arg: 1));
5418 Address SumOutPtr = EmitPointerWithAlignment(Addr: E->getArg(Arg: 2));
5419
5420 // Decide which of the overflow intrinsics we are lowering to:
5421 Intrinsic::ID IntrinsicId;
5422 switch (BuiltinID) {
5423 default: llvm_unreachable("Unknown overflow builtin id.");
5424 case Builtin::BI__builtin_uadd_overflow:
5425 case Builtin::BI__builtin_uaddl_overflow:
5426 case Builtin::BI__builtin_uaddll_overflow:
5427 IntrinsicId = Intrinsic::uadd_with_overflow;
5428 break;
5429 case Builtin::BI__builtin_usub_overflow:
5430 case Builtin::BI__builtin_usubl_overflow:
5431 case Builtin::BI__builtin_usubll_overflow:
5432 IntrinsicId = Intrinsic::usub_with_overflow;
5433 break;
5434 case Builtin::BI__builtin_umul_overflow:
5435 case Builtin::BI__builtin_umull_overflow:
5436 case Builtin::BI__builtin_umulll_overflow:
5437 IntrinsicId = Intrinsic::umul_with_overflow;
5438 break;
5439 case Builtin::BI__builtin_sadd_overflow:
5440 case Builtin::BI__builtin_saddl_overflow:
5441 case Builtin::BI__builtin_saddll_overflow:
5442 IntrinsicId = Intrinsic::sadd_with_overflow;
5443 break;
5444 case Builtin::BI__builtin_ssub_overflow:
5445 case Builtin::BI__builtin_ssubl_overflow:
5446 case Builtin::BI__builtin_ssubll_overflow:
5447 IntrinsicId = Intrinsic::ssub_with_overflow;
5448 break;
5449 case Builtin::BI__builtin_smul_overflow:
5450 case Builtin::BI__builtin_smull_overflow:
5451 case Builtin::BI__builtin_smulll_overflow:
5452 IntrinsicId = Intrinsic::smul_with_overflow;
5453 break;
5454 }
5455
5456
5457 llvm::Value *Carry;
5458 llvm::Value *Sum = EmitOverflowIntrinsic(CGF&: *this, IntrinsicID: IntrinsicId, X, Y, Carry);
5459 Builder.CreateStore(Val: Sum, Addr: SumOutPtr);
5460
5461 return RValue::get(V: Carry);
5462 }
5463 case Builtin::BIaddressof:
5464 case Builtin::BI__addressof:
5465 case Builtin::BI__builtin_addressof:
5466 return RValue::get(V: EmitLValue(E: E->getArg(Arg: 0)).getPointer(CGF&: *this));
5467 case Builtin::BI__builtin_function_start:
5468 return RValue::get(V: CGM.GetFunctionStart(
5469 Decl: E->getArg(Arg: 0)->getAsBuiltinConstantDeclRef(Context: CGM.getContext())));
5470 case Builtin::BI__builtin_operator_new:
5471 return EmitBuiltinNewDeleteCall(
5472 Type: E->getCallee()->getType()->castAs<FunctionProtoType>(), TheCallExpr: E, IsDelete: false);
5473 case Builtin::BI__builtin_operator_delete:
5474 EmitBuiltinNewDeleteCall(
5475 Type: E->getCallee()->getType()->castAs<FunctionProtoType>(), TheCallExpr: E, IsDelete: true);
5476 return RValue::get(V: nullptr);
5477
5478 case Builtin::BI__builtin_is_aligned:
5479 return EmitBuiltinIsAligned(E);
5480 case Builtin::BI__builtin_align_up:
5481 return EmitBuiltinAlignTo(E, AlignUp: true);
5482 case Builtin::BI__builtin_align_down:
5483 return EmitBuiltinAlignTo(E, AlignUp: false);
5484
5485 case Builtin::BI__noop:
5486 // __noop always evaluates to an integer literal zero.
5487 return RValue::get(V: ConstantInt::get(Ty: IntTy, V: 0));
5488 case Builtin::BI__builtin_call_with_static_chain: {
5489 const CallExpr *Call = cast<CallExpr>(Val: E->getArg(Arg: 0));
5490 const Expr *Chain = E->getArg(Arg: 1);
5491 return EmitCall(FnType: Call->getCallee()->getType(),
5492 Callee: EmitCallee(E: Call->getCallee()), E: Call, ReturnValue,
5493 Chain: EmitScalarExpr(E: Chain));
5494 }
5495 case Builtin::BI_InterlockedExchange8:
5496 case Builtin::BI_InterlockedExchange16:
5497 case Builtin::BI_InterlockedExchange:
5498 case Builtin::BI_InterlockedExchangePointer:
5499 return RValue::get(
5500 V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::_InterlockedExchange, E));
5501 case Builtin::BI_InterlockedCompareExchangePointer:
5502 return RValue::get(
5503 V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::_InterlockedCompareExchange, E));
5504 case Builtin::BI_InterlockedCompareExchangePointer_nf:
5505 return RValue::get(
5506 V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::_InterlockedCompareExchange_nf, E));
5507 case Builtin::BI_InterlockedCompareExchange8:
5508 case Builtin::BI_InterlockedCompareExchange16:
5509 case Builtin::BI_InterlockedCompareExchange:
5510 case Builtin::BI_InterlockedCompareExchange64:
5511 return RValue::get(V: EmitAtomicCmpXchgForMSIntrin(CGF&: *this, E));
5512 case Builtin::BI_InterlockedIncrement16:
5513 case Builtin::BI_InterlockedIncrement:
5514 return RValue::get(
5515 V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::_InterlockedIncrement, E));
5516 case Builtin::BI_InterlockedDecrement16:
5517 case Builtin::BI_InterlockedDecrement:
5518 return RValue::get(
5519 V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::_InterlockedDecrement, E));
5520 case Builtin::BI_InterlockedAnd8:
5521 case Builtin::BI_InterlockedAnd16:
5522 case Builtin::BI_InterlockedAnd:
5523 return RValue::get(V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::_InterlockedAnd, E));
5524 case Builtin::BI_InterlockedExchangeAdd8:
5525 case Builtin::BI_InterlockedExchangeAdd16:
5526 case Builtin::BI_InterlockedExchangeAdd:
5527 return RValue::get(
5528 V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::_InterlockedExchangeAdd, E));
5529 case Builtin::BI_InterlockedExchangeSub8:
5530 case Builtin::BI_InterlockedExchangeSub16:
5531 case Builtin::BI_InterlockedExchangeSub:
5532 return RValue::get(
5533 V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::_InterlockedExchangeSub, E));
5534 case Builtin::BI_InterlockedOr8:
5535 case Builtin::BI_InterlockedOr16:
5536 case Builtin::BI_InterlockedOr:
5537 return RValue::get(V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::_InterlockedOr, E));
5538 case Builtin::BI_InterlockedXor8:
5539 case Builtin::BI_InterlockedXor16:
5540 case Builtin::BI_InterlockedXor:
5541 return RValue::get(V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::_InterlockedXor, E));
5542
5543 case Builtin::BI_bittest64:
5544 case Builtin::BI_bittest:
5545 case Builtin::BI_bittestandcomplement64:
5546 case Builtin::BI_bittestandcomplement:
5547 case Builtin::BI_bittestandreset64:
5548 case Builtin::BI_bittestandreset:
5549 case Builtin::BI_bittestandset64:
5550 case Builtin::BI_bittestandset:
5551 case Builtin::BI_interlockedbittestandreset:
5552 case Builtin::BI_interlockedbittestandreset64:
5553 case Builtin::BI_interlockedbittestandreset64_acq:
5554 case Builtin::BI_interlockedbittestandreset64_rel:
5555 case Builtin::BI_interlockedbittestandreset64_nf:
5556 case Builtin::BI_interlockedbittestandset64:
5557 case Builtin::BI_interlockedbittestandset64_acq:
5558 case Builtin::BI_interlockedbittestandset64_rel:
5559 case Builtin::BI_interlockedbittestandset64_nf:
5560 case Builtin::BI_interlockedbittestandset:
5561 case Builtin::BI_interlockedbittestandset_acq:
5562 case Builtin::BI_interlockedbittestandset_rel:
5563 case Builtin::BI_interlockedbittestandset_nf:
5564 case Builtin::BI_interlockedbittestandreset_acq:
5565 case Builtin::BI_interlockedbittestandreset_rel:
5566 case Builtin::BI_interlockedbittestandreset_nf:
5567 return RValue::get(V: EmitBitTestIntrinsic(CGF&: *this, BuiltinID, E));
5568
5569 // These builtins exist to emit regular volatile loads and stores not
5570 // affected by the -fms-volatile setting.
5571 case Builtin::BI__iso_volatile_load8:
5572 case Builtin::BI__iso_volatile_load16:
5573 case Builtin::BI__iso_volatile_load32:
5574 case Builtin::BI__iso_volatile_load64:
5575 return RValue::get(V: EmitISOVolatileLoad(CGF&: *this, E));
5576 case Builtin::BI__iso_volatile_store8:
5577 case Builtin::BI__iso_volatile_store16:
5578 case Builtin::BI__iso_volatile_store32:
5579 case Builtin::BI__iso_volatile_store64:
5580 return RValue::get(V: EmitISOVolatileStore(CGF&: *this, E));
5581
5582 case Builtin::BI__builtin_ptrauth_sign_constant:
5583 return RValue::get(V: ConstantEmitter(*this).emitAbstract(E, T: E->getType()));
5584
5585 case Builtin::BI__builtin_ptrauth_auth:
5586 case Builtin::BI__builtin_ptrauth_auth_and_resign:
5587 case Builtin::BI__builtin_ptrauth_blend_discriminator:
5588 case Builtin::BI__builtin_ptrauth_sign_generic_data:
5589 case Builtin::BI__builtin_ptrauth_sign_unauthenticated:
5590 case Builtin::BI__builtin_ptrauth_strip: {
5591 // Emit the arguments.
5592 SmallVector<llvm::Value *, 5> Args;
5593 for (auto argExpr : E->arguments())
5594 Args.push_back(Elt: EmitScalarExpr(E: argExpr));
5595
5596 // Cast the value to intptr_t, saving its original type.
5597 llvm::Type *OrigValueType = Args[0]->getType();
5598 if (OrigValueType->isPointerTy())
5599 Args[0] = Builder.CreatePtrToInt(V: Args[0], DestTy: IntPtrTy);
5600
5601 switch (BuiltinID) {
5602 case Builtin::BI__builtin_ptrauth_auth_and_resign:
5603 if (Args[4]->getType()->isPointerTy())
5604 Args[4] = Builder.CreatePtrToInt(V: Args[4], DestTy: IntPtrTy);
5605 [[fallthrough]];
5606
5607 case Builtin::BI__builtin_ptrauth_auth:
5608 case Builtin::BI__builtin_ptrauth_sign_unauthenticated:
5609 if (Args[2]->getType()->isPointerTy())
5610 Args[2] = Builder.CreatePtrToInt(V: Args[2], DestTy: IntPtrTy);
5611 break;
5612
5613 case Builtin::BI__builtin_ptrauth_sign_generic_data:
5614 if (Args[1]->getType()->isPointerTy())
5615 Args[1] = Builder.CreatePtrToInt(V: Args[1], DestTy: IntPtrTy);
5616 break;
5617
5618 case Builtin::BI__builtin_ptrauth_blend_discriminator:
5619 case Builtin::BI__builtin_ptrauth_strip:
5620 break;
5621 }
5622
5623 // Call the intrinsic.
5624 auto IntrinsicID = [&]() -> unsigned {
5625 switch (BuiltinID) {
5626 case Builtin::BI__builtin_ptrauth_auth:
5627 return Intrinsic::ptrauth_auth;
5628 case Builtin::BI__builtin_ptrauth_auth_and_resign:
5629 return Intrinsic::ptrauth_resign;
5630 case Builtin::BI__builtin_ptrauth_blend_discriminator:
5631 return Intrinsic::ptrauth_blend;
5632 case Builtin::BI__builtin_ptrauth_sign_generic_data:
5633 return Intrinsic::ptrauth_sign_generic;
5634 case Builtin::BI__builtin_ptrauth_sign_unauthenticated:
5635 return Intrinsic::ptrauth_sign;
5636 case Builtin::BI__builtin_ptrauth_strip:
5637 return Intrinsic::ptrauth_strip;
5638 }
5639 llvm_unreachable("bad ptrauth intrinsic");
5640 }();
5641 auto Intrinsic = CGM.getIntrinsic(IID: IntrinsicID);
5642 llvm::Value *Result = EmitRuntimeCall(callee: Intrinsic, args: Args);
5643
5644 if (BuiltinID != Builtin::BI__builtin_ptrauth_sign_generic_data &&
5645 BuiltinID != Builtin::BI__builtin_ptrauth_blend_discriminator &&
5646 OrigValueType->isPointerTy()) {
5647 Result = Builder.CreateIntToPtr(V: Result, DestTy: OrigValueType);
5648 }
5649 return RValue::get(V: Result);
5650 }
5651
5652 case Builtin::BI__builtin_get_vtable_pointer: {
5653 const Expr *Target = E->getArg(Arg: 0);
5654 QualType TargetType = Target->getType();
5655 const CXXRecordDecl *Decl = TargetType->getPointeeCXXRecordDecl();
5656 assert(Decl);
5657 auto ThisAddress = EmitPointerWithAlignment(Addr: Target);
5658 assert(ThisAddress.isValid());
5659 llvm::Value *VTablePointer =
5660 GetVTablePtr(This: ThisAddress, VTableTy: Int8PtrTy, VTableClass: Decl, AuthMode: VTableAuthMode::MustTrap);
5661 return RValue::get(V: VTablePointer);
5662 }
5663
5664 case Builtin::BI__exception_code:
5665 case Builtin::BI_exception_code:
5666 return RValue::get(V: EmitSEHExceptionCode());
5667 case Builtin::BI__exception_info:
5668 case Builtin::BI_exception_info:
5669 return RValue::get(V: EmitSEHExceptionInfo());
5670 case Builtin::BI__abnormal_termination:
5671 case Builtin::BI_abnormal_termination:
5672 return RValue::get(V: EmitSEHAbnormalTermination());
5673 case Builtin::BI_setjmpex:
5674 if (getTarget().getTriple().isOSMSVCRT() && E->getNumArgs() == 1 &&
5675 E->getArg(Arg: 0)->getType()->isPointerType())
5676 return EmitMSVCRTSetJmp(CGF&: *this, SJKind: MSVCSetJmpKind::_setjmpex, E);
5677 break;
5678 case Builtin::BI_setjmp:
5679 if (getTarget().getTriple().isOSMSVCRT() && E->getNumArgs() == 1 &&
5680 E->getArg(Arg: 0)->getType()->isPointerType()) {
5681 if (getTarget().getTriple().getArch() == llvm::Triple::x86)
5682 return EmitMSVCRTSetJmp(CGF&: *this, SJKind: MSVCSetJmpKind::_setjmp3, E);
5683 else if (getTarget().getTriple().getArch() == llvm::Triple::aarch64)
5684 return EmitMSVCRTSetJmp(CGF&: *this, SJKind: MSVCSetJmpKind::_setjmpex, E);
5685 return EmitMSVCRTSetJmp(CGF&: *this, SJKind: MSVCSetJmpKind::_setjmp, E);
5686 }
5687 break;
5688
5689 // C++ std:: builtins.
5690 case Builtin::BImove:
5691 case Builtin::BImove_if_noexcept:
5692 case Builtin::BIforward:
5693 case Builtin::BIforward_like:
5694 case Builtin::BIas_const:
5695 return RValue::get(V: EmitLValue(E: E->getArg(Arg: 0)).getPointer(CGF&: *this));
5696 case Builtin::BI__GetExceptionInfo: {
5697 if (llvm::GlobalVariable *GV =
5698 CGM.getCXXABI().getThrowInfo(T: FD->getParamDecl(i: 0)->getType()))
5699 return RValue::get(V: GV);
5700 break;
5701 }
5702
5703 case Builtin::BI__fastfail:
5704 return RValue::get(V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::__fastfail, E));
5705
5706 case Builtin::BI__builtin_coro_id:
5707 return EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_id);
5708 case Builtin::BI__builtin_coro_promise:
5709 return EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_promise);
5710 case Builtin::BI__builtin_coro_resume:
5711 EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_resume);
5712 return RValue::get(V: nullptr);
5713 case Builtin::BI__builtin_coro_frame:
5714 return EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_frame);
5715 case Builtin::BI__builtin_coro_noop:
5716 return EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_noop);
5717 case Builtin::BI__builtin_coro_free:
5718 return EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_free);
5719 case Builtin::BI__builtin_coro_destroy:
5720 EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_destroy);
5721 return RValue::get(V: nullptr);
5722 case Builtin::BI__builtin_coro_done:
5723 return EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_done);
5724 case Builtin::BI__builtin_coro_alloc:
5725 return EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_alloc);
5726 case Builtin::BI__builtin_coro_begin:
5727 return EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_begin);
5728 case Builtin::BI__builtin_coro_end:
5729 return EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_end);
5730 case Builtin::BI__builtin_coro_suspend:
5731 return EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_suspend);
5732 case Builtin::BI__builtin_coro_size:
5733 return EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_size);
5734 case Builtin::BI__builtin_coro_align:
5735 return EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_align);
5736
5737 // OpenCL v2.0 s6.13.16.2, Built-in pipe read and write functions
5738 case Builtin::BIread_pipe:
5739 case Builtin::BIwrite_pipe: {
5740 Value *Arg0 = EmitScalarExpr(E: E->getArg(Arg: 0)),
5741 *Arg1 = EmitScalarExpr(E: E->getArg(Arg: 1));
5742 CGOpenCLRuntime OpenCLRT(CGM);
5743 Value *PacketSize = OpenCLRT.getPipeElemSize(PipeArg: E->getArg(Arg: 0));
5744 Value *PacketAlign = OpenCLRT.getPipeElemAlign(PipeArg: E->getArg(Arg: 0));
5745
5746 // Type of the generic packet parameter.
5747 unsigned GenericAS =
5748 getContext().getTargetAddressSpace(AS: LangAS::opencl_generic);
5749 llvm::Type *I8PTy = llvm::PointerType::get(C&: getLLVMContext(), AddressSpace: GenericAS);
5750
5751 // Testing which overloaded version we should generate the call for.
5752 if (2U == E->getNumArgs()) {
5753 const char *Name = (BuiltinID == Builtin::BIread_pipe) ? "__read_pipe_2"
5754 : "__write_pipe_2";
5755 // Creating a generic function type to be able to call with any builtin or
5756 // user defined type.
5757 llvm::Type *ArgTys[] = {Arg0->getType(), I8PTy, Int32Ty, Int32Ty};
5758 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: Int32Ty, Params: ArgTys, isVarArg: false);
5759 Value *ACast = Builder.CreateAddrSpaceCast(V: Arg1, DestTy: I8PTy);
5760 return RValue::get(
5761 V: EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(Ty: FTy, Name),
5762 args: {Arg0, ACast, PacketSize, PacketAlign}));
5763 } else {
5764 assert(4 == E->getNumArgs() &&
5765 "Illegal number of parameters to pipe function");
5766 const char *Name = (BuiltinID == Builtin::BIread_pipe) ? "__read_pipe_4"
5767 : "__write_pipe_4";
5768
5769 llvm::Type *ArgTys[] = {Arg0->getType(), Arg1->getType(), Int32Ty, I8PTy,
5770 Int32Ty, Int32Ty};
5771 Value *Arg2 = EmitScalarExpr(E: E->getArg(Arg: 2)),
5772 *Arg3 = EmitScalarExpr(E: E->getArg(Arg: 3));
5773 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: Int32Ty, Params: ArgTys, isVarArg: false);
5774 Value *ACast = Builder.CreateAddrSpaceCast(V: Arg3, DestTy: I8PTy);
5775 // We know the third argument is an integer type, but we may need to cast
5776 // it to i32.
5777 if (Arg2->getType() != Int32Ty)
5778 Arg2 = Builder.CreateZExtOrTrunc(V: Arg2, DestTy: Int32Ty);
5779 return RValue::get(
5780 V: EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(Ty: FTy, Name),
5781 args: {Arg0, Arg1, Arg2, ACast, PacketSize, PacketAlign}));
5782 }
5783 }
5784 // OpenCL v2.0 s6.13.16 ,s9.17.3.5 - Built-in pipe reserve read and write
5785 // functions
5786 case Builtin::BIreserve_read_pipe:
5787 case Builtin::BIreserve_write_pipe:
5788 case Builtin::BIwork_group_reserve_read_pipe:
5789 case Builtin::BIwork_group_reserve_write_pipe:
5790 case Builtin::BIsub_group_reserve_read_pipe:
5791 case Builtin::BIsub_group_reserve_write_pipe: {
5792 // Composing the mangled name for the function.
5793 const char *Name;
5794 if (BuiltinID == Builtin::BIreserve_read_pipe)
5795 Name = "__reserve_read_pipe";
5796 else if (BuiltinID == Builtin::BIreserve_write_pipe)
5797 Name = "__reserve_write_pipe";
5798 else if (BuiltinID == Builtin::BIwork_group_reserve_read_pipe)
5799 Name = "__work_group_reserve_read_pipe";
5800 else if (BuiltinID == Builtin::BIwork_group_reserve_write_pipe)
5801 Name = "__work_group_reserve_write_pipe";
5802 else if (BuiltinID == Builtin::BIsub_group_reserve_read_pipe)
5803 Name = "__sub_group_reserve_read_pipe";
5804 else
5805 Name = "__sub_group_reserve_write_pipe";
5806
5807 Value *Arg0 = EmitScalarExpr(E: E->getArg(Arg: 0)),
5808 *Arg1 = EmitScalarExpr(E: E->getArg(Arg: 1));
5809 llvm::Type *ReservedIDTy = ConvertType(T: getContext().OCLReserveIDTy);
5810 CGOpenCLRuntime OpenCLRT(CGM);
5811 Value *PacketSize = OpenCLRT.getPipeElemSize(PipeArg: E->getArg(Arg: 0));
5812 Value *PacketAlign = OpenCLRT.getPipeElemAlign(PipeArg: E->getArg(Arg: 0));
5813
5814 // Building the generic function prototype.
5815 llvm::Type *ArgTys[] = {Arg0->getType(), Int32Ty, Int32Ty, Int32Ty};
5816 llvm::FunctionType *FTy =
5817 llvm::FunctionType::get(Result: ReservedIDTy, Params: ArgTys, isVarArg: false);
5818 // We know the second argument is an integer type, but we may need to cast
5819 // it to i32.
5820 if (Arg1->getType() != Int32Ty)
5821 Arg1 = Builder.CreateZExtOrTrunc(V: Arg1, DestTy: Int32Ty);
5822 return RValue::get(V: EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(Ty: FTy, Name),
5823 args: {Arg0, Arg1, PacketSize, PacketAlign}));
5824 }
5825 // OpenCL v2.0 s6.13.16, s9.17.3.5 - Built-in pipe commit read and write
5826 // functions
5827 case Builtin::BIcommit_read_pipe:
5828 case Builtin::BIcommit_write_pipe:
5829 case Builtin::BIwork_group_commit_read_pipe:
5830 case Builtin::BIwork_group_commit_write_pipe:
5831 case Builtin::BIsub_group_commit_read_pipe:
5832 case Builtin::BIsub_group_commit_write_pipe: {
5833 const char *Name;
5834 if (BuiltinID == Builtin::BIcommit_read_pipe)
5835 Name = "__commit_read_pipe";
5836 else if (BuiltinID == Builtin::BIcommit_write_pipe)
5837 Name = "__commit_write_pipe";
5838 else if (BuiltinID == Builtin::BIwork_group_commit_read_pipe)
5839 Name = "__work_group_commit_read_pipe";
5840 else if (BuiltinID == Builtin::BIwork_group_commit_write_pipe)
5841 Name = "__work_group_commit_write_pipe";
5842 else if (BuiltinID == Builtin::BIsub_group_commit_read_pipe)
5843 Name = "__sub_group_commit_read_pipe";
5844 else
5845 Name = "__sub_group_commit_write_pipe";
5846
5847 Value *Arg0 = EmitScalarExpr(E: E->getArg(Arg: 0)),
5848 *Arg1 = EmitScalarExpr(E: E->getArg(Arg: 1));
5849 CGOpenCLRuntime OpenCLRT(CGM);
5850 Value *PacketSize = OpenCLRT.getPipeElemSize(PipeArg: E->getArg(Arg: 0));
5851 Value *PacketAlign = OpenCLRT.getPipeElemAlign(PipeArg: E->getArg(Arg: 0));
5852
5853 // Building the generic function prototype.
5854 llvm::Type *ArgTys[] = {Arg0->getType(), Arg1->getType(), Int32Ty, Int32Ty};
5855 llvm::FunctionType *FTy = llvm::FunctionType::get(
5856 Result: llvm::Type::getVoidTy(C&: getLLVMContext()), Params: ArgTys, isVarArg: false);
5857
5858 return RValue::get(V: EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(Ty: FTy, Name),
5859 args: {Arg0, Arg1, PacketSize, PacketAlign}));
5860 }
5861 // OpenCL v2.0 s6.13.16.4 Built-in pipe query functions
5862 case Builtin::BIget_pipe_num_packets:
5863 case Builtin::BIget_pipe_max_packets: {
5864 const char *BaseName;
5865 const auto *PipeTy = E->getArg(Arg: 0)->getType()->castAs<PipeType>();
5866 if (BuiltinID == Builtin::BIget_pipe_num_packets)
5867 BaseName = "__get_pipe_num_packets";
5868 else
5869 BaseName = "__get_pipe_max_packets";
5870 std::string Name = std::string(BaseName) +
5871 std::string(PipeTy->isReadOnly() ? "_ro" : "_wo");
5872
5873 // Building the generic function prototype.
5874 Value *Arg0 = EmitScalarExpr(E: E->getArg(Arg: 0));
5875 CGOpenCLRuntime OpenCLRT(CGM);
5876 Value *PacketSize = OpenCLRT.getPipeElemSize(PipeArg: E->getArg(Arg: 0));
5877 Value *PacketAlign = OpenCLRT.getPipeElemAlign(PipeArg: E->getArg(Arg: 0));
5878 llvm::Type *ArgTys[] = {Arg0->getType(), Int32Ty, Int32Ty};
5879 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: Int32Ty, Params: ArgTys, isVarArg: false);
5880
5881 return RValue::get(V: EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(Ty: FTy, Name),
5882 args: {Arg0, PacketSize, PacketAlign}));
5883 }
5884
5885 // OpenCL v2.0 s6.13.9 - Address space qualifier functions.
5886 case Builtin::BIto_global:
5887 case Builtin::BIto_local:
5888 case Builtin::BIto_private: {
5889 auto Arg0 = EmitScalarExpr(E: E->getArg(Arg: 0));
5890 auto NewArgT = llvm::PointerType::get(
5891 C&: getLLVMContext(),
5892 AddressSpace: CGM.getContext().getTargetAddressSpace(AS: LangAS::opencl_generic));
5893 auto NewRetT = llvm::PointerType::get(
5894 C&: getLLVMContext(),
5895 AddressSpace: CGM.getContext().getTargetAddressSpace(
5896 AS: E->getType()->getPointeeType().getAddressSpace()));
5897 auto FTy = llvm::FunctionType::get(Result: NewRetT, Params: {NewArgT}, isVarArg: false);
5898 llvm::Value *NewArg;
5899 if (Arg0->getType()->getPointerAddressSpace() !=
5900 NewArgT->getPointerAddressSpace())
5901 NewArg = Builder.CreateAddrSpaceCast(V: Arg0, DestTy: NewArgT);
5902 else
5903 NewArg = Builder.CreateBitOrPointerCast(V: Arg0, DestTy: NewArgT);
5904 auto NewName = std::string("__") + E->getDirectCallee()->getName().str();
5905 auto NewCall =
5906 EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(Ty: FTy, Name: NewName), args: {NewArg});
5907 return RValue::get(V: Builder.CreateBitOrPointerCast(V: NewCall,
5908 DestTy: ConvertType(T: E->getType())));
5909 }
5910
5911 // OpenCL v2.0, s6.13.17 - Enqueue kernel function.
5912 // Table 6.13.17.1 specifies four overload forms of enqueue_kernel.
5913 // The code below expands the builtin call to a call to one of the following
5914 // functions that an OpenCL runtime library will have to provide:
5915 // __enqueue_kernel_basic
5916 // __enqueue_kernel_varargs
5917 // __enqueue_kernel_basic_events
5918 // __enqueue_kernel_events_varargs
5919 case Builtin::BIenqueue_kernel: {
5920 StringRef Name; // Generated function call name
5921 unsigned NumArgs = E->getNumArgs();
5922
5923 llvm::Type *QueueTy = ConvertType(T: getContext().OCLQueueTy);
5924 llvm::Type *GenericVoidPtrTy = Builder.getPtrTy(
5925 AddrSpace: getContext().getTargetAddressSpace(AS: LangAS::opencl_generic));
5926
5927 llvm::Value *Queue = EmitScalarExpr(E: E->getArg(Arg: 0));
5928 llvm::Value *Flags = EmitScalarExpr(E: E->getArg(Arg: 1));
5929 LValue NDRangeL = EmitAggExprToLValue(E: E->getArg(Arg: 2));
5930 llvm::Value *Range = NDRangeL.getAddress().emitRawPointer(CGF&: *this);
5931
5932 // FIXME: Look through the addrspacecast which may exist to the stack
5933 // temporary as a hack.
5934 //
5935 // This is hardcoding the assumed ABI of the target function. This assumes
5936 // direct passing for every argument except NDRange, which is assumed to be
5937 // byval or byref indirect passed.
5938 //
5939 // This should be fixed to query a signature from CGOpenCLRuntime, and go
5940 // through EmitCallArgs to get the correct target ABI.
5941 Range = Range->stripPointerCasts();
5942
5943 llvm::Type *RangePtrTy = Range->getType();
5944
5945 if (NumArgs == 4) {
5946 // The most basic form of the call with parameters:
5947 // queue_t, kernel_enqueue_flags_t, ndrange_t, block(void)
5948 Name = "__enqueue_kernel_basic";
5949 llvm::Type *ArgTys[] = {QueueTy, Int32Ty, RangePtrTy, GenericVoidPtrTy,
5950 GenericVoidPtrTy};
5951 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: Int32Ty, Params: ArgTys, isVarArg: false);
5952
5953 auto Info =
5954 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(CGF&: *this, E: E->getArg(Arg: 3));
5955 llvm::Value *Kernel =
5956 Builder.CreatePointerCast(V: Info.KernelHandle, DestTy: GenericVoidPtrTy);
5957 llvm::Value *Block =
5958 Builder.CreatePointerCast(V: Info.BlockArg, DestTy: GenericVoidPtrTy);
5959
5960 auto RTCall = EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(Ty: FTy, Name),
5961 args: {Queue, Flags, Range, Kernel, Block});
5962 return RValue::get(V: RTCall);
5963 }
5964 assert(NumArgs >= 5 && "Invalid enqueue_kernel signature");
5965
5966 // Create a temporary array to hold the sizes of local pointer arguments
5967 // for the block. \p First is the position of the first size argument.
5968 auto CreateArrayForSizeVar = [=](unsigned First)
5969 -> std::tuple<llvm::Value *, llvm::Value *, llvm::Value *> {
5970 llvm::APInt ArraySize(32, NumArgs - First);
5971 QualType SizeArrayTy = getContext().getConstantArrayType(
5972 EltTy: getContext().getSizeType(), ArySize: ArraySize, SizeExpr: nullptr,
5973 ASM: ArraySizeModifier::Normal,
5974 /*IndexTypeQuals=*/0);
5975 auto Tmp = CreateMemTemp(T: SizeArrayTy, Name: "block_sizes");
5976 llvm::Value *TmpPtr = Tmp.getPointer();
5977 // The EmitLifetime* pair expect a naked Alloca as their last argument,
5978 // however for cases where the default AS is not the Alloca AS, Tmp is
5979 // actually the Alloca ascasted to the default AS, hence the
5980 // stripPointerCasts()
5981 llvm::Value *Alloca = TmpPtr->stripPointerCasts();
5982 llvm::Value *TmpSize = EmitLifetimeStart(
5983 Size: CGM.getDataLayout().getTypeAllocSize(Ty: Tmp.getElementType()), Addr: Alloca);
5984 llvm::Value *ElemPtr;
5985 // Each of the following arguments specifies the size of the corresponding
5986 // argument passed to the enqueued block.
5987 auto *Zero = llvm::ConstantInt::get(Ty: IntTy, V: 0);
5988 for (unsigned I = First; I < NumArgs; ++I) {
5989 auto *Index = llvm::ConstantInt::get(Ty: IntTy, V: I - First);
5990 auto *GEP = Builder.CreateGEP(Ty: Tmp.getElementType(), Ptr: TmpPtr,
5991 IdxList: {Zero, Index});
5992 if (I == First)
5993 ElemPtr = GEP;
5994 auto *V =
5995 Builder.CreateZExtOrTrunc(V: EmitScalarExpr(E: E->getArg(Arg: I)), DestTy: SizeTy);
5996 Builder.CreateAlignedStore(
5997 Val: V, Ptr: GEP, Align: CGM.getDataLayout().getPrefTypeAlign(Ty: SizeTy));
5998 }
5999 // Return the Alloca itself rather than a potential ascast as this is only
6000 // used by the paired EmitLifetimeEnd.
6001 return {ElemPtr, TmpSize, Alloca};
6002 };
6003
6004 // Could have events and/or varargs.
6005 if (E->getArg(Arg: 3)->getType()->isBlockPointerType()) {
6006 // No events passed, but has variadic arguments.
6007 Name = "__enqueue_kernel_varargs";
6008 auto Info =
6009 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(CGF&: *this, E: E->getArg(Arg: 3));
6010 llvm::Value *Kernel =
6011 Builder.CreatePointerCast(V: Info.KernelHandle, DestTy: GenericVoidPtrTy);
6012 auto *Block = Builder.CreatePointerCast(V: Info.BlockArg, DestTy: GenericVoidPtrTy);
6013 auto [ElemPtr, TmpSize, TmpPtr] = CreateArrayForSizeVar(4);
6014
6015 // Create a vector of the arguments, as well as a constant value to
6016 // express to the runtime the number of variadic arguments.
6017 llvm::Value *const Args[] = {Queue, Flags,
6018 Range, Kernel,
6019 Block, ConstantInt::get(Ty: IntTy, V: NumArgs - 4),
6020 ElemPtr};
6021 llvm::Type *const ArgTys[] = {
6022 QueueTy, IntTy, RangePtrTy, GenericVoidPtrTy,
6023 GenericVoidPtrTy, IntTy, ElemPtr->getType()};
6024
6025 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: Int32Ty, Params: ArgTys, isVarArg: false);
6026 auto Call = RValue::get(
6027 V: EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(Ty: FTy, Name), args: Args));
6028 if (TmpSize)
6029 EmitLifetimeEnd(Size: TmpSize, Addr: TmpPtr);
6030 return Call;
6031 }
6032 // Any calls now have event arguments passed.
6033 if (NumArgs >= 7) {
6034 llvm::PointerType *PtrTy = llvm::PointerType::get(
6035 C&: CGM.getLLVMContext(),
6036 AddressSpace: CGM.getContext().getTargetAddressSpace(AS: LangAS::opencl_generic));
6037
6038 llvm::Value *NumEvents =
6039 Builder.CreateZExtOrTrunc(V: EmitScalarExpr(E: E->getArg(Arg: 3)), DestTy: Int32Ty);
6040
6041 // Since SemaOpenCLBuiltinEnqueueKernel allows fifth and sixth arguments
6042 // to be a null pointer constant (including `0` literal), we can take it
6043 // into account and emit null pointer directly.
6044 llvm::Value *EventWaitList = nullptr;
6045 if (E->getArg(Arg: 4)->isNullPointerConstant(
6046 Ctx&: getContext(), NPC: Expr::NPC_ValueDependentIsNotNull)) {
6047 EventWaitList = llvm::ConstantPointerNull::get(T: PtrTy);
6048 } else {
6049 EventWaitList =
6050 E->getArg(Arg: 4)->getType()->isArrayType()
6051 ? EmitArrayToPointerDecay(Array: E->getArg(Arg: 4)).emitRawPointer(CGF&: *this)
6052 : EmitScalarExpr(E: E->getArg(Arg: 4));
6053 // Convert to generic address space.
6054 EventWaitList = Builder.CreatePointerCast(V: EventWaitList, DestTy: PtrTy);
6055 }
6056 llvm::Value *EventRet = nullptr;
6057 if (E->getArg(Arg: 5)->isNullPointerConstant(
6058 Ctx&: getContext(), NPC: Expr::NPC_ValueDependentIsNotNull)) {
6059 EventRet = llvm::ConstantPointerNull::get(T: PtrTy);
6060 } else {
6061 EventRet =
6062 Builder.CreatePointerCast(V: EmitScalarExpr(E: E->getArg(Arg: 5)), DestTy: PtrTy);
6063 }
6064
6065 auto Info =
6066 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(CGF&: *this, E: E->getArg(Arg: 6));
6067 llvm::Value *Kernel =
6068 Builder.CreatePointerCast(V: Info.KernelHandle, DestTy: GenericVoidPtrTy);
6069 llvm::Value *Block =
6070 Builder.CreatePointerCast(V: Info.BlockArg, DestTy: GenericVoidPtrTy);
6071
6072 std::vector<llvm::Type *> ArgTys = {
6073 QueueTy, Int32Ty, RangePtrTy, Int32Ty,
6074 PtrTy, PtrTy, GenericVoidPtrTy, GenericVoidPtrTy};
6075
6076 std::vector<llvm::Value *> Args = {Queue, Flags, Range,
6077 NumEvents, EventWaitList, EventRet,
6078 Kernel, Block};
6079
6080 if (NumArgs == 7) {
6081 // Has events but no variadics.
6082 Name = "__enqueue_kernel_basic_events";
6083 llvm::FunctionType *FTy =
6084 llvm::FunctionType::get(Result: Int32Ty, Params: ArgTys, isVarArg: false);
6085 return RValue::get(
6086 V: EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(Ty: FTy, Name), args: Args));
6087 }
6088 // Has event info and variadics
6089 // Pass the number of variadics to the runtime function too.
6090 Args.push_back(x: ConstantInt::get(Ty: Int32Ty, V: NumArgs - 7));
6091 ArgTys.push_back(x: Int32Ty);
6092 Name = "__enqueue_kernel_events_varargs";
6093
6094 auto [ElemPtr, TmpSize, TmpPtr] = CreateArrayForSizeVar(7);
6095 Args.push_back(x: ElemPtr);
6096 ArgTys.push_back(x: ElemPtr->getType());
6097
6098 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: Int32Ty, Params: ArgTys, isVarArg: false);
6099 auto Call = RValue::get(
6100 V: EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(Ty: FTy, Name), args: Args));
6101 if (TmpSize)
6102 EmitLifetimeEnd(Size: TmpSize, Addr: TmpPtr);
6103 return Call;
6104 }
6105 llvm_unreachable("Unexpected enqueue_kernel signature");
6106 }
6107 // OpenCL v2.0 s6.13.17.6 - Kernel query functions need bitcast of block
6108 // parameter.
6109 case Builtin::BIget_kernel_work_group_size: {
6110 llvm::Type *GenericVoidPtrTy = Builder.getPtrTy(
6111 AddrSpace: getContext().getTargetAddressSpace(AS: LangAS::opencl_generic));
6112 auto Info =
6113 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(CGF&: *this, E: E->getArg(Arg: 0));
6114 Value *Kernel =
6115 Builder.CreatePointerCast(V: Info.KernelHandle, DestTy: GenericVoidPtrTy);
6116 Value *Arg = Builder.CreatePointerCast(V: Info.BlockArg, DestTy: GenericVoidPtrTy);
6117 return RValue::get(V: EmitRuntimeCall(
6118 callee: CGM.CreateRuntimeFunction(
6119 Ty: llvm::FunctionType::get(Result: IntTy, Params: {GenericVoidPtrTy, GenericVoidPtrTy},
6120 isVarArg: false),
6121 Name: "__get_kernel_work_group_size_impl"),
6122 args: {Kernel, Arg}));
6123 }
6124 case Builtin::BIget_kernel_preferred_work_group_size_multiple: {
6125 llvm::Type *GenericVoidPtrTy = Builder.getPtrTy(
6126 AddrSpace: getContext().getTargetAddressSpace(AS: LangAS::opencl_generic));
6127 auto Info =
6128 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(CGF&: *this, E: E->getArg(Arg: 0));
6129 Value *Kernel =
6130 Builder.CreatePointerCast(V: Info.KernelHandle, DestTy: GenericVoidPtrTy);
6131 Value *Arg = Builder.CreatePointerCast(V: Info.BlockArg, DestTy: GenericVoidPtrTy);
6132 return RValue::get(V: EmitRuntimeCall(
6133 callee: CGM.CreateRuntimeFunction(
6134 Ty: llvm::FunctionType::get(Result: IntTy, Params: {GenericVoidPtrTy, GenericVoidPtrTy},
6135 isVarArg: false),
6136 Name: "__get_kernel_preferred_work_group_size_multiple_impl"),
6137 args: {Kernel, Arg}));
6138 }
6139 case Builtin::BIget_kernel_max_sub_group_size_for_ndrange:
6140 case Builtin::BIget_kernel_sub_group_count_for_ndrange: {
6141 llvm::Type *GenericVoidPtrTy = Builder.getPtrTy(
6142 AddrSpace: getContext().getTargetAddressSpace(AS: LangAS::opencl_generic));
6143 LValue NDRangeL = EmitAggExprToLValue(E: E->getArg(Arg: 0));
6144 llvm::Value *NDRange = NDRangeL.getAddress().emitRawPointer(CGF&: *this);
6145 auto Info =
6146 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(CGF&: *this, E: E->getArg(Arg: 1));
6147 Value *Kernel =
6148 Builder.CreatePointerCast(V: Info.KernelHandle, DestTy: GenericVoidPtrTy);
6149 Value *Block = Builder.CreatePointerCast(V: Info.BlockArg, DestTy: GenericVoidPtrTy);
6150 const char *Name =
6151 BuiltinID == Builtin::BIget_kernel_max_sub_group_size_for_ndrange
6152 ? "__get_kernel_max_sub_group_size_for_ndrange_impl"
6153 : "__get_kernel_sub_group_count_for_ndrange_impl";
6154 return RValue::get(V: EmitRuntimeCall(
6155 callee: CGM.CreateRuntimeFunction(
6156 Ty: llvm::FunctionType::get(
6157 Result: IntTy, Params: {NDRange->getType(), GenericVoidPtrTy, GenericVoidPtrTy},
6158 isVarArg: false),
6159 Name),
6160 args: {NDRange, Kernel, Block}));
6161 }
6162 case Builtin::BI__builtin_store_half:
6163 case Builtin::BI__builtin_store_halff: {
6164 Value *Val = EmitScalarExpr(E: E->getArg(Arg: 0));
6165 Address Address = EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
6166 Value *HalfVal = Builder.CreateFPTrunc(V: Val, DestTy: Builder.getHalfTy());
6167 Builder.CreateStore(Val: HalfVal, Addr: Address);
6168 return RValue::get(V: nullptr);
6169 }
6170 case Builtin::BI__builtin_load_half: {
6171 Address Address = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
6172 Value *HalfVal = Builder.CreateLoad(Addr: Address);
6173 return RValue::get(V: Builder.CreateFPExt(V: HalfVal, DestTy: Builder.getDoubleTy()));
6174 }
6175 case Builtin::BI__builtin_load_halff: {
6176 Address Address = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
6177 Value *HalfVal = Builder.CreateLoad(Addr: Address);
6178 return RValue::get(V: Builder.CreateFPExt(V: HalfVal, DestTy: Builder.getFloatTy()));
6179 }
6180 case Builtin::BI__builtin_printf:
6181 case Builtin::BIprintf:
6182 if (getTarget().getTriple().isNVPTX() ||
6183 getTarget().getTriple().isAMDGCN() ||
6184 (getTarget().getTriple().isSPIRV() &&
6185 getTarget().getTriple().getVendor() == Triple::VendorType::AMD)) {
6186 if (getTarget().getTriple().isNVPTX())
6187 return EmitNVPTXDevicePrintfCallExpr(E);
6188 if ((getTarget().getTriple().isAMDGCN() ||
6189 getTarget().getTriple().isSPIRV()) &&
6190 getLangOpts().HIP)
6191 return EmitAMDGPUDevicePrintfCallExpr(E);
6192 }
6193
6194 break;
6195 case Builtin::BI__builtin_canonicalize:
6196 case Builtin::BI__builtin_canonicalizef:
6197 case Builtin::BI__builtin_canonicalizef16:
6198 case Builtin::BI__builtin_canonicalizel:
6199 return RValue::get(
6200 V: emitBuiltinWithOneOverloadedType<1>(CGF&: *this, E, IntrinsicID: Intrinsic::canonicalize));
6201
6202 case Builtin::BI__builtin_thread_pointer: {
6203 if (!getContext().getTargetInfo().isTLSSupported())
6204 CGM.ErrorUnsupported(S: E, Type: "__builtin_thread_pointer");
6205
6206 return RValue::get(V: Builder.CreateIntrinsic(ID: llvm::Intrinsic::thread_pointer,
6207 Types: {GlobalsInt8PtrTy}, Args: {}));
6208 }
6209 case Builtin::BI__builtin_os_log_format:
6210 return emitBuiltinOSLogFormat(E: *E);
6211
6212 case Builtin::BI__xray_customevent: {
6213 if (!ShouldXRayInstrumentFunction())
6214 return RValue::getIgnored();
6215
6216 if (!CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
6217 K: XRayInstrKind::Custom))
6218 return RValue::getIgnored();
6219
6220 if (const auto *XRayAttr = CurFuncDecl->getAttr<XRayInstrumentAttr>())
6221 if (XRayAttr->neverXRayInstrument() && !AlwaysEmitXRayCustomEvents())
6222 return RValue::getIgnored();
6223
6224 Function *F = CGM.getIntrinsic(IID: Intrinsic::xray_customevent);
6225 auto FTy = F->getFunctionType();
6226 auto Arg0 = E->getArg(Arg: 0);
6227 auto Arg0Val = EmitScalarExpr(E: Arg0);
6228 auto Arg0Ty = Arg0->getType();
6229 auto PTy0 = FTy->getParamType(i: 0);
6230 if (PTy0 != Arg0Val->getType()) {
6231 if (Arg0Ty->isArrayType())
6232 Arg0Val = EmitArrayToPointerDecay(Array: Arg0).emitRawPointer(CGF&: *this);
6233 else
6234 Arg0Val = Builder.CreatePointerCast(V: Arg0Val, DestTy: PTy0);
6235 }
6236 auto Arg1 = EmitScalarExpr(E: E->getArg(Arg: 1));
6237 auto PTy1 = FTy->getParamType(i: 1);
6238 if (PTy1 != Arg1->getType())
6239 Arg1 = Builder.CreateTruncOrBitCast(V: Arg1, DestTy: PTy1);
6240 return RValue::get(V: Builder.CreateCall(Callee: F, Args: {Arg0Val, Arg1}));
6241 }
6242
6243 case Builtin::BI__xray_typedevent: {
6244 // TODO: There should be a way to always emit events even if the current
6245 // function is not instrumented. Losing events in a stream can cripple
6246 // a trace.
6247 if (!ShouldXRayInstrumentFunction())
6248 return RValue::getIgnored();
6249
6250 if (!CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
6251 K: XRayInstrKind::Typed))
6252 return RValue::getIgnored();
6253
6254 if (const auto *XRayAttr = CurFuncDecl->getAttr<XRayInstrumentAttr>())
6255 if (XRayAttr->neverXRayInstrument() && !AlwaysEmitXRayTypedEvents())
6256 return RValue::getIgnored();
6257
6258 Function *F = CGM.getIntrinsic(IID: Intrinsic::xray_typedevent);
6259 auto FTy = F->getFunctionType();
6260 auto Arg0 = EmitScalarExpr(E: E->getArg(Arg: 0));
6261 auto PTy0 = FTy->getParamType(i: 0);
6262 if (PTy0 != Arg0->getType())
6263 Arg0 = Builder.CreateTruncOrBitCast(V: Arg0, DestTy: PTy0);
6264 auto Arg1 = E->getArg(Arg: 1);
6265 auto Arg1Val = EmitScalarExpr(E: Arg1);
6266 auto Arg1Ty = Arg1->getType();
6267 auto PTy1 = FTy->getParamType(i: 1);
6268 if (PTy1 != Arg1Val->getType()) {
6269 if (Arg1Ty->isArrayType())
6270 Arg1Val = EmitArrayToPointerDecay(Array: Arg1).emitRawPointer(CGF&: *this);
6271 else
6272 Arg1Val = Builder.CreatePointerCast(V: Arg1Val, DestTy: PTy1);
6273 }
6274 auto Arg2 = EmitScalarExpr(E: E->getArg(Arg: 2));
6275 auto PTy2 = FTy->getParamType(i: 2);
6276 if (PTy2 != Arg2->getType())
6277 Arg2 = Builder.CreateTruncOrBitCast(V: Arg2, DestTy: PTy2);
6278 return RValue::get(V: Builder.CreateCall(Callee: F, Args: {Arg0, Arg1Val, Arg2}));
6279 }
6280
6281 case Builtin::BI__builtin_ms_va_start:
6282 case Builtin::BI__builtin_ms_va_end:
6283 return RValue::get(
6284 V: EmitVAStartEnd(ArgValue: EmitMSVAListRef(E: E->getArg(Arg: 0)).emitRawPointer(CGF&: *this),
6285 IsStart: BuiltinID == Builtin::BI__builtin_ms_va_start));
6286
6287 case Builtin::BI__builtin_ms_va_copy: {
6288 // Lower this manually. We can't reliably determine whether or not any
6289 // given va_copy() is for a Win64 va_list from the calling convention
6290 // alone, because it's legal to do this from a System V ABI function.
6291 // With opaque pointer types, we won't have enough information in LLVM
6292 // IR to determine this from the argument types, either. Best to do it
6293 // now, while we have enough information.
6294 Address DestAddr = EmitMSVAListRef(E: E->getArg(Arg: 0));
6295 Address SrcAddr = EmitMSVAListRef(E: E->getArg(Arg: 1));
6296
6297 DestAddr = DestAddr.withElementType(ElemTy: Int8PtrTy);
6298 SrcAddr = SrcAddr.withElementType(ElemTy: Int8PtrTy);
6299
6300 Value *ArgPtr = Builder.CreateLoad(Addr: SrcAddr, Name: "ap.val");
6301 return RValue::get(V: Builder.CreateStore(Val: ArgPtr, Addr: DestAddr));
6302 }
6303
6304 case Builtin::BI__builtin_get_device_side_mangled_name: {
6305 auto Name = CGM.getCUDARuntime().getDeviceSideName(
6306 ND: cast<DeclRefExpr>(Val: E->getArg(Arg: 0)->IgnoreImpCasts())->getDecl());
6307 auto Str = CGM.GetAddrOfConstantCString(Str: Name, GlobalName: "");
6308 return RValue::get(V: Str.getPointer());
6309 }
6310 }
6311
6312 // If this is an alias for a lib function (e.g. __builtin_sin), emit
6313 // the call using the normal call path, but using the unmangled
6314 // version of the function name.
6315 const auto &BI = getContext().BuiltinInfo;
6316 if (!shouldEmitBuiltinAsIR(BuiltinID, BI, CGF: *this) &&
6317 BI.isLibFunction(ID: BuiltinID))
6318 return emitLibraryCall(CGF&: *this, FD, E,
6319 calleeValue: CGM.getBuiltinLibFunction(FD, BuiltinID));
6320
6321 // If this is a predefined lib function (e.g. malloc), emit the call
6322 // using exactly the normal call path.
6323 if (BI.isPredefinedLibFunction(ID: BuiltinID))
6324 return emitLibraryCall(CGF&: *this, FD, E, calleeValue: CGM.getRawFunctionPointer(GD: FD));
6325
6326 // Check that a call to a target specific builtin has the correct target
6327 // features.
6328 // This is down here to avoid non-target specific builtins, however, if
6329 // generic builtins start to require generic target features then we
6330 // can move this up to the beginning of the function.
6331 checkTargetFeatures(E, TargetDecl: FD);
6332
6333 if (unsigned VectorWidth = getContext().BuiltinInfo.getRequiredVectorWidth(ID: BuiltinID))
6334 LargestVectorWidth = std::max(a: LargestVectorWidth, b: VectorWidth);
6335
6336 // See if we have a target specific intrinsic.
6337 std::string Name = getContext().BuiltinInfo.getName(ID: BuiltinID);
6338 Intrinsic::ID IntrinsicID = Intrinsic::not_intrinsic;
6339 StringRef Prefix =
6340 llvm::Triple::getArchTypePrefix(Kind: getTarget().getTriple().getArch());
6341 if (!Prefix.empty()) {
6342 IntrinsicID = Intrinsic::getIntrinsicForClangBuiltin(TargetPrefix: Prefix.data(), BuiltinName: Name);
6343 if (IntrinsicID == Intrinsic::not_intrinsic && Prefix == "spv" &&
6344 getTarget().getTriple().getOS() == llvm::Triple::OSType::AMDHSA)
6345 IntrinsicID = Intrinsic::getIntrinsicForClangBuiltin(TargetPrefix: "amdgcn", BuiltinName: Name);
6346 // NOTE we don't need to perform a compatibility flag check here since the
6347 // intrinsics are declared in Builtins*.def via LANGBUILTIN which filter the
6348 // MS builtins via ALL_MS_LANGUAGES and are filtered earlier.
6349 if (IntrinsicID == Intrinsic::not_intrinsic)
6350 IntrinsicID = Intrinsic::getIntrinsicForMSBuiltin(TargetPrefix: Prefix.data(), BuiltinName: Name);
6351 }
6352
6353 if (IntrinsicID != Intrinsic::not_intrinsic) {
6354 SmallVector<Value*, 16> Args;
6355
6356 // Find out if any arguments are required to be integer constant
6357 // expressions.
6358 unsigned ICEArguments = 0;
6359 ASTContext::GetBuiltinTypeError Error;
6360 getContext().GetBuiltinType(ID: BuiltinID, Error, IntegerConstantArgs: &ICEArguments);
6361 assert(Error == ASTContext::GE_None && "Should not codegen an error");
6362
6363 Function *F = CGM.getIntrinsic(IID: IntrinsicID);
6364 llvm::FunctionType *FTy = F->getFunctionType();
6365
6366 for (unsigned i = 0, e = E->getNumArgs(); i != e; ++i) {
6367 Value *ArgValue = EmitScalarOrConstFoldImmArg(ICEArguments, Idx: i, E);
6368 // If the intrinsic arg type is different from the builtin arg type
6369 // we need to do a bit cast.
6370 llvm::Type *PTy = FTy->getParamType(i);
6371 if (PTy != ArgValue->getType()) {
6372 // XXX - vector of pointers?
6373 if (auto *PtrTy = dyn_cast<llvm::PointerType>(Val: PTy)) {
6374 if (PtrTy->getAddressSpace() !=
6375 ArgValue->getType()->getPointerAddressSpace()) {
6376 ArgValue = Builder.CreateAddrSpaceCast(
6377 V: ArgValue, DestTy: llvm::PointerType::get(C&: getLLVMContext(),
6378 AddressSpace: PtrTy->getAddressSpace()));
6379 }
6380 }
6381
6382 // Cast vector type (e.g., v256i32) to x86_amx, this only happen
6383 // in amx intrinsics.
6384 if (PTy->isX86_AMXTy())
6385 ArgValue = Builder.CreateIntrinsic(ID: Intrinsic::x86_cast_vector_to_tile,
6386 Types: {ArgValue->getType()}, Args: {ArgValue});
6387 else
6388 ArgValue = Builder.CreateBitCast(V: ArgValue, DestTy: PTy);
6389 }
6390
6391 Args.push_back(Elt: ArgValue);
6392 }
6393
6394 Value *V = Builder.CreateCall(Callee: F, Args);
6395 QualType BuiltinRetType = E->getType();
6396
6397 llvm::Type *RetTy = VoidTy;
6398 if (!BuiltinRetType->isVoidType())
6399 RetTy = ConvertType(T: BuiltinRetType);
6400
6401 if (RetTy != V->getType()) {
6402 // XXX - vector of pointers?
6403 if (auto *PtrTy = dyn_cast<llvm::PointerType>(Val: RetTy)) {
6404 if (PtrTy->getAddressSpace() != V->getType()->getPointerAddressSpace()) {
6405 V = Builder.CreateAddrSpaceCast(
6406 V, DestTy: llvm::PointerType::get(C&: getLLVMContext(),
6407 AddressSpace: PtrTy->getAddressSpace()));
6408 }
6409 }
6410
6411 // Cast x86_amx to vector type (e.g., v256i32), this only happen
6412 // in amx intrinsics.
6413 if (V->getType()->isX86_AMXTy())
6414 V = Builder.CreateIntrinsic(ID: Intrinsic::x86_cast_tile_to_vector, Types: {RetTy},
6415 Args: {V});
6416 else
6417 V = Builder.CreateBitCast(V, DestTy: RetTy);
6418 }
6419
6420 if (RetTy->isVoidTy())
6421 return RValue::get(V: nullptr);
6422
6423 return RValue::get(V);
6424 }
6425
6426 // Some target-specific builtins can have aggregate return values, e.g.
6427 // __builtin_arm_mve_vld2q_u32. So if the result is an aggregate, force
6428 // ReturnValue to be non-null, so that the target-specific emission code can
6429 // always just emit into it.
6430 TypeEvaluationKind EvalKind = getEvaluationKind(T: E->getType());
6431 if (EvalKind == TEK_Aggregate && ReturnValue.isNull()) {
6432 Address DestPtr = CreateMemTemp(T: E->getType(), Name: "agg.tmp");
6433 ReturnValue = ReturnValueSlot(DestPtr, false);
6434 }
6435
6436 // Now see if we can emit a target-specific builtin.
6437 if (Value *V = EmitTargetBuiltinExpr(BuiltinID, E, ReturnValue)) {
6438 switch (EvalKind) {
6439 case TEK_Scalar:
6440 if (V->getType()->isVoidTy())
6441 return RValue::get(V: nullptr);
6442 return RValue::get(V);
6443 case TEK_Aggregate:
6444 return RValue::getAggregate(addr: ReturnValue.getAddress(),
6445 isVolatile: ReturnValue.isVolatile());
6446 case TEK_Complex:
6447 llvm_unreachable("No current target builtin returns complex");
6448 }
6449 llvm_unreachable("Bad evaluation kind in EmitBuiltinExpr");
6450 }
6451
6452 // EmitHLSLBuiltinExpr will check getLangOpts().HLSL
6453 if (Value *V = EmitHLSLBuiltinExpr(BuiltinID, E, ReturnValue)) {
6454 switch (EvalKind) {
6455 case TEK_Scalar:
6456 if (V->getType()->isVoidTy())
6457 return RValue::get(V: nullptr);
6458 return RValue::get(V);
6459 case TEK_Aggregate:
6460 return RValue::getAggregate(addr: ReturnValue.getAddress(),
6461 isVolatile: ReturnValue.isVolatile());
6462 case TEK_Complex:
6463 llvm_unreachable("No current hlsl builtin returns complex");
6464 }
6465 llvm_unreachable("Bad evaluation kind in EmitBuiltinExpr");
6466 }
6467
6468 if (getLangOpts().HIPStdPar && getLangOpts().CUDAIsDevice)
6469 return EmitHipStdParUnsupportedBuiltin(CGF: this, FD);
6470
6471 ErrorUnsupported(S: E, Type: "builtin function");
6472
6473 // Unknown builtin, for now just dump it out and return undef.
6474 return GetUndefRValue(Ty: E->getType());
6475}
6476
6477namespace {
6478struct BuiltinAlignArgs {
6479 llvm::Value *Src = nullptr;
6480 llvm::Type *SrcType = nullptr;
6481 llvm::Value *Alignment = nullptr;
6482 llvm::Value *Mask = nullptr;
6483 llvm::IntegerType *IntType = nullptr;
6484
6485 BuiltinAlignArgs(const CallExpr *E, CodeGenFunction &CGF) {
6486 QualType AstType = E->getArg(Arg: 0)->getType();
6487 if (AstType->isArrayType())
6488 Src = CGF.EmitArrayToPointerDecay(Array: E->getArg(Arg: 0)).emitRawPointer(CGF);
6489 else
6490 Src = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
6491 SrcType = Src->getType();
6492 if (SrcType->isPointerTy()) {
6493 IntType = IntegerType::get(
6494 C&: CGF.getLLVMContext(),
6495 NumBits: CGF.CGM.getDataLayout().getIndexTypeSizeInBits(Ty: SrcType));
6496 } else {
6497 assert(SrcType->isIntegerTy());
6498 IntType = cast<llvm::IntegerType>(Val: SrcType);
6499 }
6500 Alignment = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
6501 Alignment = CGF.Builder.CreateZExtOrTrunc(V: Alignment, DestTy: IntType, Name: "alignment");
6502 auto *One = llvm::ConstantInt::get(Ty: IntType, V: 1);
6503 Mask = CGF.Builder.CreateSub(LHS: Alignment, RHS: One, Name: "mask");
6504 }
6505};
6506} // namespace
6507
6508/// Generate (x & (y-1)) == 0.
6509RValue CodeGenFunction::EmitBuiltinIsAligned(const CallExpr *E) {
6510 BuiltinAlignArgs Args(E, *this);
6511 llvm::Value *SrcAddress = Args.Src;
6512 if (Args.SrcType->isPointerTy())
6513 SrcAddress =
6514 Builder.CreateBitOrPointerCast(V: Args.Src, DestTy: Args.IntType, Name: "src_addr");
6515 return RValue::get(V: Builder.CreateICmpEQ(
6516 LHS: Builder.CreateAnd(LHS: SrcAddress, RHS: Args.Mask, Name: "set_bits"),
6517 RHS: llvm::Constant::getNullValue(Ty: Args.IntType), Name: "is_aligned"));
6518}
6519
6520/// Generate (x & ~(y-1)) to align down or ((x+(y-1)) & ~(y-1)) to align up.
6521/// Note: For pointer types we can avoid ptrtoint/inttoptr pairs by using the
6522/// llvm.ptrmask intrinsic (with a GEP before in the align_up case).
6523RValue CodeGenFunction::EmitBuiltinAlignTo(const CallExpr *E, bool AlignUp) {
6524 BuiltinAlignArgs Args(E, *this);
6525 llvm::Value *SrcForMask = Args.Src;
6526 if (AlignUp) {
6527 // When aligning up we have to first add the mask to ensure we go over the
6528 // next alignment value and then align down to the next valid multiple.
6529 // By adding the mask, we ensure that align_up on an already aligned
6530 // value will not change the value.
6531 if (Args.Src->getType()->isPointerTy()) {
6532 if (getLangOpts().PointerOverflowDefined)
6533 SrcForMask =
6534 Builder.CreateGEP(Ty: Int8Ty, Ptr: SrcForMask, IdxList: Args.Mask, Name: "over_boundary");
6535 else
6536 SrcForMask = EmitCheckedInBoundsGEP(ElemTy: Int8Ty, Ptr: SrcForMask, IdxList: Args.Mask,
6537 /*SignedIndices=*/true,
6538 /*isSubtraction=*/IsSubtraction: false,
6539 Loc: E->getExprLoc(), Name: "over_boundary");
6540 } else {
6541 SrcForMask = Builder.CreateAdd(LHS: SrcForMask, RHS: Args.Mask, Name: "over_boundary");
6542 }
6543 }
6544 // Invert the mask to only clear the lower bits.
6545 llvm::Value *InvertedMask = Builder.CreateNot(V: Args.Mask, Name: "inverted_mask");
6546 llvm::Value *Result = nullptr;
6547 if (Args.Src->getType()->isPointerTy()) {
6548 Result = Builder.CreateIntrinsic(
6549 ID: Intrinsic::ptrmask, Types: {Args.SrcType, Args.IntType},
6550 Args: {SrcForMask, InvertedMask}, FMFSource: nullptr, Name: "aligned_result");
6551 } else {
6552 Result = Builder.CreateAnd(LHS: SrcForMask, RHS: InvertedMask, Name: "aligned_result");
6553 }
6554 assert(Result->getType() == Args.SrcType);
6555 return RValue::get(V: Result);
6556}
6557