1//===---- CGBuiltin.cpp - Emit LLVM Code for builtins ---------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit Builtin calls as LLVM code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGBuiltin.h"
14#include "ABIInfo.h"
15#include "CGCUDARuntime.h"
16#include "CGCXXABI.h"
17#include "CGDebugInfo.h"
18#include "CGObjCRuntime.h"
19#include "CGOpenCLRuntime.h"
20#include "CGRecordLayout.h"
21#include "CGValue.h"
22#include "CodeGenFunction.h"
23#include "CodeGenModule.h"
24#include "ConstantEmitter.h"
25#include "PatternInit.h"
26#include "TargetInfo.h"
27#include "clang/AST/OSLog.h"
28#include "clang/AST/StmtVisitor.h"
29#include "clang/Basic/DiagnosticFrontend.h"
30#include "clang/Basic/TargetInfo.h"
31#include "llvm/ADT/APFloat.h"
32#include "llvm/IR/InlineAsm.h"
33#include "llvm/IR/Instruction.h"
34#include "llvm/IR/Intrinsics.h"
35#include "llvm/IR/IntrinsicsX86.h"
36#include "llvm/IR/MatrixBuilder.h"
37#include "llvm/Support/ConvertUTF.h"
38#include "llvm/Support/ScopedPrinter.h"
39#include <algorithm>
40#include <optional>
41#include <utility>
42
43using namespace clang;
44using namespace CodeGen;
45using namespace llvm;
46
47/// Some builtins do not have library implementation on some targets and
48/// are instead emitted as LLVM IRs by some target builtin emitters.
49/// FIXME: Remove this when library support is added
50static bool shouldEmitBuiltinAsIR(unsigned BuiltinID,
51 const Builtin::Context &BI,
52 const CodeGenFunction &CGF) {
53 if (!CGF.CGM.getLangOpts().MathErrno &&
54 CGF.CurFPFeatures.getExceptionMode() ==
55 LangOptions::FPExceptionModeKind::FPE_Ignore &&
56 !CGF.CGM.getTargetCodeGenInfo().supportsLibCall()) {
57 switch (BuiltinID) {
58 default:
59 return false;
60 case Builtin::BIlogbf:
61 case Builtin::BI__builtin_logbf:
62 case Builtin::BIlogb:
63 case Builtin::BI__builtin_logb:
64 case Builtin::BIscalbnf:
65 case Builtin::BI__builtin_scalbnf:
66 case Builtin::BIscalbn:
67 case Builtin::BI__builtin_scalbn:
68 return true;
69 }
70 }
71 return false;
72}
73
74static Value *EmitTargetArchBuiltinExpr(CodeGenFunction *CGF,
75 unsigned BuiltinID, const CallExpr *E,
76 ReturnValueSlot ReturnValue,
77 llvm::Triple::ArchType Arch) {
78 // When compiling in HipStdPar mode we have to be conservative in rejecting
79 // target specific features in the FE, and defer the possible error to the
80 // AcceleratorCodeSelection pass, wherein iff an unsupported target builtin is
81 // referenced by an accelerator executable function, we emit an error.
82 // Returning nullptr here leads to the builtin being handled in
83 // EmitStdParUnsupportedBuiltin.
84 if (CGF->getLangOpts().HIPStdPar && CGF->getLangOpts().CUDAIsDevice &&
85 Arch != CGF->getTarget().getTriple().getArch())
86 return nullptr;
87
88 switch (Arch) {
89 case llvm::Triple::arm:
90 case llvm::Triple::armeb:
91 case llvm::Triple::thumb:
92 case llvm::Triple::thumbeb:
93 return CGF->EmitARMBuiltinExpr(BuiltinID, E, ReturnValue, Arch);
94 case llvm::Triple::aarch64:
95 case llvm::Triple::aarch64_32:
96 case llvm::Triple::aarch64_be:
97 return CGF->EmitAArch64BuiltinExpr(BuiltinID, E, Arch);
98 case llvm::Triple::bpfeb:
99 case llvm::Triple::bpfel:
100 return CGF->EmitBPFBuiltinExpr(BuiltinID, E);
101 case llvm::Triple::dxil:
102 return CGF->EmitDirectXBuiltinExpr(BuiltinID, E);
103 case llvm::Triple::x86:
104 case llvm::Triple::x86_64:
105 return CGF->EmitX86BuiltinExpr(BuiltinID, E);
106 case llvm::Triple::ppc:
107 case llvm::Triple::ppcle:
108 case llvm::Triple::ppc64:
109 case llvm::Triple::ppc64le:
110 return CGF->EmitPPCBuiltinExpr(BuiltinID, E);
111 case llvm::Triple::amdgpu:
112 case llvm::Triple::r600:
113 return CGF->EmitAMDGPUBuiltinExpr(BuiltinID, E);
114 case llvm::Triple::systemz:
115 return CGF->EmitSystemZBuiltinExpr(BuiltinID, E);
116 case llvm::Triple::nvptx:
117 case llvm::Triple::nvptx64:
118 return CGF->EmitNVPTXBuiltinExpr(BuiltinID, E);
119 case llvm::Triple::wasm32:
120 case llvm::Triple::wasm64:
121 return CGF->EmitWebAssemblyBuiltinExpr(BuiltinID, E);
122 case llvm::Triple::hexagon:
123 return CGF->EmitHexagonBuiltinExpr(BuiltinID, E);
124 case llvm::Triple::riscv32:
125 case llvm::Triple::riscv64:
126 case llvm::Triple::riscv32be:
127 case llvm::Triple::riscv64be:
128 return CGF->EmitRISCVBuiltinExpr(BuiltinID, E, ReturnValue);
129 case llvm::Triple::spirv32:
130 case llvm::Triple::spirv64:
131 if (CGF->getTarget().getTriple().getOS() == llvm::Triple::OSType::AMDHSA)
132 return CGF->EmitAMDGPUBuiltinExpr(BuiltinID, E);
133 [[fallthrough]];
134 case llvm::Triple::spirv:
135 return CGF->EmitSPIRVBuiltinExpr(BuiltinID, E);
136 case llvm::Triple::avr:
137 return CGF->EmitAVRBuiltinExpr(BuiltinID, E);
138 default:
139 return nullptr;
140 }
141}
142
143Value *CodeGenFunction::EmitTargetBuiltinExpr(unsigned BuiltinID,
144 const CallExpr *E,
145 ReturnValueSlot ReturnValue) {
146 if (getContext().BuiltinInfo.isAuxBuiltinID(ID: BuiltinID)) {
147 assert(getContext().getAuxTargetInfo() && "Missing aux target info");
148 return EmitTargetArchBuiltinExpr(
149 CGF: this, BuiltinID: getContext().BuiltinInfo.getAuxBuiltinID(ID: BuiltinID), E,
150 ReturnValue, Arch: getContext().getAuxTargetInfo()->getTriple().getArch());
151 }
152
153 return EmitTargetArchBuiltinExpr(CGF: this, BuiltinID, E, ReturnValue,
154 Arch: getTarget().getTriple().getArch());
155}
156
157static void initializeAlloca(CodeGenFunction &CGF, AllocaInst *AI, Value *Size,
158 Align AlignmentInBytes) {
159 ConstantInt *Byte;
160 switch (CGF.getLangOpts().getTrivialAutoVarInit()) {
161 case LangOptions::TrivialAutoVarInitKind::Uninitialized:
162 // Nothing to initialize.
163 return;
164 case LangOptions::TrivialAutoVarInitKind::Zero:
165 Byte = CGF.Builder.getInt8(C: 0x00);
166 break;
167 case LangOptions::TrivialAutoVarInitKind::Pattern: {
168 llvm::Type *Int8 = llvm::IntegerType::getInt8Ty(C&: CGF.CGM.getLLVMContext());
169 Byte = llvm::dyn_cast<llvm::ConstantInt>(
170 Val: initializationPatternFor(CGF.CGM, Int8));
171 break;
172 }
173 }
174 if (CGF.CGM.stopAutoInit())
175 return;
176 auto *I = CGF.Builder.CreateMemSet(Ptr: AI, Val: Byte, Size, Align: AlignmentInBytes);
177 I->addAnnotationMetadata(Annotation: "auto-init");
178}
179
180/// getBuiltinLibFunction - Given a builtin id for a function like
181/// "__builtin_fabsf", return a Function* for "fabsf".
182llvm::Constant *CodeGenModule::getBuiltinLibFunction(const FunctionDecl *FD,
183 unsigned BuiltinID) {
184 assert(Context.BuiltinInfo.isLibFunction(BuiltinID));
185
186 // Get the name, skip over the __builtin_ prefix (if necessary). We may have
187 // to build this up so provide a small stack buffer to handle the vast
188 // majority of names.
189 llvm::SmallString<64> Name;
190 GlobalDecl D(FD);
191
192 // TODO: This list should be expanded or refactored after all GCC-compatible
193 // std libcall builtins are implemented.
194 static const SmallDenseMap<unsigned, StringRef, 64> F128Builtins{
195 {Builtin::BI__builtin___fprintf_chk, "__fprintf_chkieee128"},
196 {Builtin::BI__builtin___printf_chk, "__printf_chkieee128"},
197 {Builtin::BI__builtin___snprintf_chk, "__snprintf_chkieee128"},
198 {Builtin::BI__builtin___sprintf_chk, "__sprintf_chkieee128"},
199 {Builtin::BI__builtin___vfprintf_chk, "__vfprintf_chkieee128"},
200 {Builtin::BI__builtin___vprintf_chk, "__vprintf_chkieee128"},
201 {Builtin::BI__builtin___vsnprintf_chk, "__vsnprintf_chkieee128"},
202 {Builtin::BI__builtin___vsprintf_chk, "__vsprintf_chkieee128"},
203 {Builtin::BI__builtin_fprintf, "__fprintfieee128"},
204 {Builtin::BI__builtin_printf, "__printfieee128"},
205 {Builtin::BI__builtin_snprintf, "__snprintfieee128"},
206 {Builtin::BI__builtin_sprintf, "__sprintfieee128"},
207 {Builtin::BI__builtin_vfprintf, "__vfprintfieee128"},
208 {Builtin::BI__builtin_vprintf, "__vprintfieee128"},
209 {Builtin::BI__builtin_vsnprintf, "__vsnprintfieee128"},
210 {Builtin::BI__builtin_vsprintf, "__vsprintfieee128"},
211 {Builtin::BI__builtin_fscanf, "__fscanfieee128"},
212 {Builtin::BI__builtin_scanf, "__scanfieee128"},
213 {Builtin::BI__builtin_sscanf, "__sscanfieee128"},
214 {Builtin::BI__builtin_vfscanf, "__vfscanfieee128"},
215 {Builtin::BI__builtin_vscanf, "__vscanfieee128"},
216 {Builtin::BI__builtin_vsscanf, "__vsscanfieee128"},
217 {Builtin::BI__builtin_nexttowardf128, "__nexttowardieee128"},
218 };
219
220 // The AIX library functions frexpl, ldexpl, and modfl are for 128-bit
221 // IBM 'long double' (i.e. __ibm128). Map to the 'double' versions
222 // if it is 64-bit 'long double' mode.
223 static const SmallDenseMap<unsigned, StringRef, 4> AIXLongDouble64Builtins{
224 {Builtin::BI__builtin_frexpl, "frexp"},
225 {Builtin::BI__builtin_ldexpl, "ldexp"},
226 {Builtin::BI__builtin_modfl, "modf"},
227 };
228
229 // If the builtin has been declared explicitly with an assembler label,
230 // use the mangled name. This differs from the plain label on platforms
231 // that prefix labels.
232 if (FD->hasAttr<AsmLabelAttr>())
233 Name = getMangledName(GD: D);
234 else {
235 // TODO: This mutation should also be applied to other targets other than
236 // PPC, after backend supports IEEE 128-bit style libcalls.
237 if (getTriple().isPPC64() &&
238 &getTarget().getLongDoubleFormat() == &llvm::APFloat::IEEEquad() &&
239 F128Builtins.contains(Val: BuiltinID))
240 Name = F128Builtins.lookup(Val: BuiltinID);
241 else if (getTriple().isOSAIX() &&
242 &getTarget().getLongDoubleFormat() ==
243 &llvm::APFloat::IEEEdouble() &&
244 AIXLongDouble64Builtins.contains(Val: BuiltinID))
245 Name = AIXLongDouble64Builtins.lookup(Val: BuiltinID);
246 else
247 Name = Context.BuiltinInfo.getName(ID: BuiltinID).substr(pos: 10);
248 }
249
250 llvm::FunctionType *Ty =
251 cast<llvm::FunctionType>(Val: getTypes().ConvertType(T: FD->getType()));
252
253 return GetOrCreateLLVMFunction(MangledName: Name, Ty, D, /*ForVTable=*/false);
254}
255
256/// Emit the conversions required to turn the given value into an
257/// integer of the given size.
258Value *EmitToInt(CodeGenFunction &CGF, llvm::Value *V,
259 QualType T, llvm::IntegerType *IntType) {
260 V = CGF.EmitToMemory(Value: V, Ty: T);
261
262 if (V->getType()->isPointerTy())
263 return CGF.Builder.CreatePtrToInt(V, DestTy: IntType);
264
265 assert(V->getType() == IntType);
266 return V;
267}
268
269Value *EmitFromInt(CodeGenFunction &CGF, llvm::Value *V,
270 QualType T, llvm::Type *ResultType) {
271 V = CGF.EmitFromMemory(Value: V, Ty: T);
272
273 if (ResultType->isPointerTy())
274 return CGF.Builder.CreateIntToPtr(V, DestTy: ResultType);
275
276 assert(V->getType() == ResultType);
277 return V;
278}
279
280Address CheckAtomicAlignment(CodeGenFunction &CGF, const CallExpr *E) {
281 ASTContext &Ctx = CGF.getContext();
282 Address Ptr = CGF.EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
283 const llvm::DataLayout &DL = CGF.CGM.getDataLayout();
284 unsigned Bytes = Ptr.getElementType()->isPointerTy()
285 ? Ctx.getTypeSizeInChars(T: Ctx.VoidPtrTy).getQuantity()
286 : DL.getTypeStoreSize(Ty: Ptr.getElementType());
287 unsigned Align = Ptr.getAlignment().getQuantity();
288 if (Align % Bytes != 0) {
289 DiagnosticsEngine &Diags = CGF.CGM.getDiags();
290 Diags.Report(Loc: E->getBeginLoc(), DiagID: diag::warn_sync_op_misaligned);
291 // Force address to be at least naturally-aligned.
292 return Ptr.withAlignment(NewAlignment: CharUnits::fromQuantity(Quantity: Bytes));
293 }
294 return Ptr;
295}
296
297/// Utility to insert an atomic instruction based on Intrinsic::ID
298/// and the expression node.
299Value *MakeBinaryAtomicValue(
300 CodeGenFunction &CGF, llvm::AtomicRMWInst::BinOp Kind, const CallExpr *E,
301 AtomicOrdering Ordering) {
302
303 QualType T = E->getType();
304 assert(E->getArg(0)->getType()->isPointerType());
305 assert(CGF.getContext().hasSameUnqualifiedType(T,
306 E->getArg(0)->getType()->getPointeeType()));
307 assert(CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType()));
308
309 Address DestAddr = CheckAtomicAlignment(CGF, E);
310
311 llvm::IntegerType *IntType = llvm::IntegerType::get(
312 C&: CGF.getLLVMContext(), NumBits: CGF.getContext().getTypeSize(T));
313
314 llvm::Value *Val = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
315 llvm::Type *ValueType = Val->getType();
316 Val = EmitToInt(CGF, V: Val, T, IntType);
317
318 llvm::Value *Result =
319 CGF.Builder.CreateAtomicRMW(Op: Kind, Addr: DestAddr, Val, Ordering);
320 // Consider atomics to be volatile in MS kernel mode.
321 if (CGF.CGM.getLangOpts().Kernel)
322 cast<llvm::AtomicRMWInst>(Val: Result)->setVolatile(true);
323 return EmitFromInt(CGF, V: Result, T, ResultType: ValueType);
324}
325
326static Value *EmitNontemporalStore(CodeGenFunction &CGF, const CallExpr *E) {
327 Value *Val = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
328 Address Addr = CGF.EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
329
330 Val = CGF.EmitToMemory(Value: Val, Ty: E->getArg(Arg: 0)->getType());
331 LValue LV = CGF.MakeAddrLValue(Addr, T: E->getArg(Arg: 0)->getType());
332 LV.setNontemporal(true);
333 CGF.EmitStoreOfScalar(value: Val, lvalue: LV, isInit: false);
334 return nullptr;
335}
336
337static Value *EmitNontemporalLoad(CodeGenFunction &CGF, const CallExpr *E) {
338 Address Addr = CGF.EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
339
340 LValue LV = CGF.MakeAddrLValue(Addr, T: E->getType());
341 LV.setNontemporal(true);
342 return CGF.EmitLoadOfScalar(lvalue: LV, Loc: E->getExprLoc());
343}
344
345static RValue EmitBinaryAtomic(CodeGenFunction &CGF,
346 llvm::AtomicRMWInst::BinOp Kind,
347 const CallExpr *E) {
348 return RValue::get(V: MakeBinaryAtomicValue(CGF, Kind, E));
349}
350
351/// Utility to insert an atomic instruction based Intrinsic::ID and
352/// the expression node, where the return value is the result of the
353/// operation.
354static RValue EmitBinaryAtomicPost(CodeGenFunction &CGF,
355 llvm::AtomicRMWInst::BinOp Kind,
356 const CallExpr *E,
357 Instruction::BinaryOps Op,
358 bool Invert = false) {
359 QualType T = E->getType();
360 assert(E->getArg(0)->getType()->isPointerType());
361 assert(CGF.getContext().hasSameUnqualifiedType(T,
362 E->getArg(0)->getType()->getPointeeType()));
363 assert(CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType()));
364
365 Address DestAddr = CheckAtomicAlignment(CGF, E);
366
367 llvm::IntegerType *IntType = llvm::IntegerType::get(
368 C&: CGF.getLLVMContext(), NumBits: CGF.getContext().getTypeSize(T));
369
370 llvm::Value *Val = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
371 llvm::Type *ValueType = Val->getType();
372 Val = EmitToInt(CGF, V: Val, T, IntType);
373
374 llvm::Value *Result = CGF.Builder.CreateAtomicRMW(
375 Op: Kind, Addr: DestAddr, Val, Ordering: llvm::AtomicOrdering::SequentiallyConsistent);
376 Result = CGF.Builder.CreateBinOp(Opc: Op, LHS: Result, RHS: Val);
377 if (Invert)
378 Result =
379 CGF.Builder.CreateBinOp(Opc: llvm::Instruction::Xor, LHS: Result,
380 RHS: llvm::ConstantInt::getAllOnesValue(Ty: IntType));
381 Result = EmitFromInt(CGF, V: Result, T, ResultType: ValueType);
382 return RValue::get(V: Result);
383}
384
385/// Utility to insert an atomic cmpxchg instruction.
386///
387/// @param CGF The current codegen function.
388/// @param E Builtin call expression to convert to cmpxchg.
389/// arg0 - address to operate on
390/// arg1 - value to compare with
391/// arg2 - new value
392/// @param ReturnBool Specifies whether to return success flag of
393/// cmpxchg result or the old value.
394///
395/// @returns result of cmpxchg, according to ReturnBool
396///
397/// Note: In order to lower Microsoft's _InterlockedCompareExchange* intrinsics
398/// invoke the function EmitAtomicCmpXchgForMSIntrin.
399Value *MakeAtomicCmpXchgValue(CodeGenFunction &CGF, const CallExpr *E,
400 bool ReturnBool,
401 llvm::AtomicOrdering SuccessOrdering,
402 llvm::AtomicOrdering FailureOrdering) {
403 QualType T = ReturnBool ? E->getArg(Arg: 1)->getType() : E->getType();
404 Address DestAddr = CheckAtomicAlignment(CGF, E);
405
406 llvm::IntegerType *IntType = llvm::IntegerType::get(
407 C&: CGF.getLLVMContext(), NumBits: CGF.getContext().getTypeSize(T));
408
409 Value *Cmp = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
410 llvm::Type *ValueType = Cmp->getType();
411 Cmp = EmitToInt(CGF, V: Cmp, T, IntType);
412 Value *New = EmitToInt(CGF, V: CGF.EmitScalarExpr(E: E->getArg(Arg: 2)), T, IntType);
413
414 Value *Pair = CGF.Builder.CreateAtomicCmpXchg(
415 Addr: DestAddr, Cmp, New, SuccessOrdering, FailureOrdering);
416 if (ReturnBool)
417 // Extract boolean success flag and zext it to int.
418 return CGF.Builder.CreateZExt(V: CGF.Builder.CreateExtractValue(Agg: Pair, Idxs: 1),
419 DestTy: CGF.ConvertType(T: E->getType()));
420 else
421 // Extract old value and emit it using the same type as compare value.
422 return EmitFromInt(CGF, V: CGF.Builder.CreateExtractValue(Agg: Pair, Idxs: 0), T,
423 ResultType: ValueType);
424}
425
426/// This function should be invoked to emit atomic cmpxchg for Microsoft's
427/// _InterlockedCompareExchange* intrinsics which have the following signature:
428/// T _InterlockedCompareExchange(T volatile *Destination,
429/// T Exchange,
430/// T Comparand);
431///
432/// Whereas the llvm 'cmpxchg' instruction has the following syntax:
433/// cmpxchg *Destination, Comparand, Exchange.
434/// So we need to swap Comparand and Exchange when invoking
435/// CreateAtomicCmpXchg. That is the reason we could not use the above utility
436/// function MakeAtomicCmpXchgValue since it expects the arguments to be
437/// already swapped.
438
439static
440Value *EmitAtomicCmpXchgForMSIntrin(CodeGenFunction &CGF, const CallExpr *E,
441 AtomicOrdering SuccessOrdering = AtomicOrdering::SequentiallyConsistent) {
442 assert(E->getArg(0)->getType()->isPointerType());
443 assert(CGF.getContext().hasSameUnqualifiedType(
444 E->getType(), E->getArg(0)->getType()->getPointeeType()));
445 assert(CGF.getContext().hasSameUnqualifiedType(E->getType(),
446 E->getArg(1)->getType()));
447 assert(CGF.getContext().hasSameUnqualifiedType(E->getType(),
448 E->getArg(2)->getType()));
449
450 Address DestAddr = CheckAtomicAlignment(CGF, E);
451
452 auto *Exchange = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
453 auto *RTy = Exchange->getType();
454
455 auto *Comparand = CGF.EmitScalarExpr(E: E->getArg(Arg: 2));
456
457 if (RTy->isPointerTy()) {
458 Exchange = CGF.Builder.CreatePtrToInt(V: Exchange, DestTy: CGF.IntPtrTy);
459 Comparand = CGF.Builder.CreatePtrToInt(V: Comparand, DestTy: CGF.IntPtrTy);
460 }
461
462 // For Release ordering, the failure ordering should be Monotonic.
463 auto FailureOrdering = SuccessOrdering == AtomicOrdering::Release ?
464 AtomicOrdering::Monotonic :
465 SuccessOrdering;
466
467 // The atomic instruction is marked volatile for consistency with MSVC. This
468 // blocks the few atomics optimizations that LLVM has. If we want to optimize
469 // _Interlocked* operations in the future, we will have to remove the volatile
470 // marker.
471 auto *CmpXchg = CGF.Builder.CreateAtomicCmpXchg(
472 Addr: DestAddr, Cmp: Comparand, New: Exchange, SuccessOrdering, FailureOrdering);
473 CmpXchg->setVolatile(true);
474
475 auto *Result = CGF.Builder.CreateExtractValue(Agg: CmpXchg, Idxs: 0);
476 if (RTy->isPointerTy()) {
477 Result = CGF.Builder.CreateIntToPtr(V: Result, DestTy: RTy);
478 }
479
480 return Result;
481}
482
483// 64-bit Microsoft platforms support 128 bit cmpxchg operations. They are
484// prototyped like this:
485//
486// unsigned char _InterlockedCompareExchange128...(
487// __int64 volatile * _Destination,
488// __int64 _ExchangeHigh,
489// __int64 _ExchangeLow,
490// __int64 * _ComparandResult);
491//
492// Note that Destination is assumed to be at least 16-byte aligned, despite
493// being typed int64.
494
495static Value *EmitAtomicCmpXchg128ForMSIntrin(CodeGenFunction &CGF,
496 const CallExpr *E,
497 AtomicOrdering SuccessOrdering) {
498 assert(E->getNumArgs() == 4);
499 llvm::Value *DestPtr = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
500 llvm::Value *ExchangeHigh = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
501 llvm::Value *ExchangeLow = CGF.EmitScalarExpr(E: E->getArg(Arg: 2));
502 Address ComparandAddr = CGF.EmitPointerWithAlignment(Addr: E->getArg(Arg: 3));
503
504 assert(DestPtr->getType()->isPointerTy());
505 assert(!ExchangeHigh->getType()->isPointerTy());
506 assert(!ExchangeLow->getType()->isPointerTy());
507
508 // For Release ordering, the failure ordering should be Monotonic.
509 auto FailureOrdering = SuccessOrdering == AtomicOrdering::Release
510 ? AtomicOrdering::Monotonic
511 : SuccessOrdering;
512
513 // Convert to i128 pointers and values. Alignment is also overridden for
514 // destination pointer.
515 llvm::Type *Int128Ty = llvm::IntegerType::get(C&: CGF.getLLVMContext(), NumBits: 128);
516 Address DestAddr(DestPtr, Int128Ty,
517 CGF.getContext().toCharUnitsFromBits(BitSize: 128));
518 ComparandAddr = ComparandAddr.withElementType(ElemTy: Int128Ty);
519
520 // (((i128)hi) << 64) | ((i128)lo)
521 ExchangeHigh = CGF.Builder.CreateZExt(V: ExchangeHigh, DestTy: Int128Ty);
522 ExchangeLow = CGF.Builder.CreateZExt(V: ExchangeLow, DestTy: Int128Ty);
523 ExchangeHigh =
524 CGF.Builder.CreateShl(LHS: ExchangeHigh, RHS: llvm::ConstantInt::get(Ty: Int128Ty, V: 64));
525 llvm::Value *Exchange = CGF.Builder.CreateOr(LHS: ExchangeHigh, RHS: ExchangeLow);
526
527 // Load the comparand for the instruction.
528 llvm::Value *Comparand = CGF.Builder.CreateLoad(Addr: ComparandAddr);
529
530 auto *CXI = CGF.Builder.CreateAtomicCmpXchg(Addr: DestAddr, Cmp: Comparand, New: Exchange,
531 SuccessOrdering, FailureOrdering);
532
533 // The atomic instruction is marked volatile for consistency with MSVC. This
534 // blocks the few atomics optimizations that LLVM has. If we want to optimize
535 // _Interlocked* operations in the future, we will have to remove the volatile
536 // marker.
537 CXI->setVolatile(true);
538
539 // Store the result as an outparameter.
540 CGF.Builder.CreateStore(Val: CGF.Builder.CreateExtractValue(Agg: CXI, Idxs: 0),
541 Addr: ComparandAddr);
542
543 // Get the success boolean and zero extend it to i8.
544 Value *Success = CGF.Builder.CreateExtractValue(Agg: CXI, Idxs: 1);
545 return CGF.Builder.CreateZExt(V: Success, DestTy: CGF.Int8Ty);
546}
547
548static Value *EmitAtomicIncrementValue(CodeGenFunction &CGF, const CallExpr *E,
549 AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
550 assert(E->getArg(0)->getType()->isPointerType());
551
552 auto *IntTy = CGF.ConvertType(T: E->getType());
553 Address DestAddr = CheckAtomicAlignment(CGF, E);
554 auto *Result = CGF.Builder.CreateAtomicRMW(
555 Op: AtomicRMWInst::Add, Addr: DestAddr, Val: ConstantInt::get(Ty: IntTy, V: 1), Ordering);
556 return CGF.Builder.CreateAdd(LHS: Result, RHS: ConstantInt::get(Ty: IntTy, V: 1));
557}
558
559static Value *EmitAtomicDecrementValue(
560 CodeGenFunction &CGF, const CallExpr *E,
561 AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
562 assert(E->getArg(0)->getType()->isPointerType());
563
564 auto *IntTy = CGF.ConvertType(T: E->getType());
565 Address DestAddr = CheckAtomicAlignment(CGF, E);
566 auto *Result = CGF.Builder.CreateAtomicRMW(
567 Op: AtomicRMWInst::Sub, Addr: DestAddr, Val: ConstantInt::get(Ty: IntTy, V: 1), Ordering);
568 return CGF.Builder.CreateSub(LHS: Result, RHS: ConstantInt::get(Ty: IntTy, V: 1));
569}
570
571// Build a plain volatile load.
572static Value *EmitISOVolatileLoad(CodeGenFunction &CGF, const CallExpr *E) {
573 Value *Ptr = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
574 QualType ElTy = E->getArg(Arg: 0)->getType()->getPointeeType();
575 CharUnits LoadSize = CGF.getContext().getTypeSizeInChars(T: ElTy);
576 llvm::Type *ITy =
577 llvm::IntegerType::get(C&: CGF.getLLVMContext(), NumBits: LoadSize.getQuantity() * 8);
578 llvm::LoadInst *Load = CGF.Builder.CreateAlignedLoad(Ty: ITy, Addr: Ptr, Align: LoadSize);
579 Load->setAtomic(Ordering: llvm::AtomicOrdering::Monotonic);
580 Load->setVolatile(true);
581 return Load;
582}
583
584// Build a plain volatile store.
585static Value *EmitISOVolatileStore(CodeGenFunction &CGF, const CallExpr *E) {
586 Value *Ptr = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
587 Value *Value = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
588 QualType ElTy = E->getArg(Arg: 0)->getType()->getPointeeType();
589 CharUnits StoreSize = CGF.getContext().getTypeSizeInChars(T: ElTy);
590 llvm::StoreInst *Store =
591 CGF.Builder.CreateAlignedStore(Val: Value, Addr: Ptr, Align: StoreSize);
592 Store->setAtomic(Ordering: llvm::AtomicOrdering::Monotonic);
593 Store->setVolatile(true);
594 return Store;
595}
596
597// Emit a simple mangled intrinsic that has 1 argument and a return type
598// matching the argument type. Depending on mode, this may be a constrained
599// floating-point intrinsic.
600Value *emitUnaryMaybeConstrainedFPBuiltin(CodeGenFunction &CGF,
601 const CallExpr *E, unsigned IntrinsicID,
602 unsigned ConstrainedIntrinsicID) {
603 llvm::Value *Src0 = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
604
605 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
606 if (CGF.Builder.getIsFPConstrained()) {
607 Function *F = CGF.CGM.getIntrinsic(IID: ConstrainedIntrinsicID, Tys: Src0->getType());
608 return CGF.Builder.CreateConstrainedFPCall(Callee: F, Args: { Src0 });
609 } else {
610 Function *F = CGF.CGM.getIntrinsic(IID: IntrinsicID, Tys: Src0->getType());
611 return CGF.Builder.CreateCall(Callee: F, Args: Src0);
612 }
613}
614
615// Emit an intrinsic that has 2 operands of the same type as its result.
616// Depending on mode, this may be a constrained floating-point intrinsic.
617static Value *emitBinaryMaybeConstrainedFPBuiltin(CodeGenFunction &CGF,
618 const CallExpr *E, unsigned IntrinsicID,
619 unsigned ConstrainedIntrinsicID) {
620 llvm::Value *Src0 = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
621 llvm::Value *Src1 = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
622
623 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
624 if (CGF.Builder.getIsFPConstrained()) {
625 Function *F = CGF.CGM.getIntrinsic(IID: ConstrainedIntrinsicID, Tys: Src0->getType());
626 return CGF.Builder.CreateConstrainedFPCall(Callee: F, Args: { Src0, Src1 });
627 } else {
628 Function *F = CGF.CGM.getIntrinsic(IID: IntrinsicID, Tys: Src0->getType());
629 return CGF.Builder.CreateCall(Callee: F, Args: { Src0, Src1 });
630 }
631}
632
633// Has second type mangled argument.
634static Value *
635emitBinaryExpMaybeConstrainedFPBuiltin(CodeGenFunction &CGF, const CallExpr *E,
636 Intrinsic::ID IntrinsicID,
637 Intrinsic::ID ConstrainedIntrinsicID) {
638 llvm::Value *Src0 = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
639 llvm::Value *Src1 = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
640
641 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
642 if (CGF.Builder.getIsFPConstrained()) {
643 Function *F = CGF.CGM.getIntrinsic(IID: ConstrainedIntrinsicID,
644 Tys: {Src0->getType(), Src1->getType()});
645 return CGF.Builder.CreateConstrainedFPCall(Callee: F, Args: {Src0, Src1});
646 }
647
648 Function *F =
649 CGF.CGM.getIntrinsic(IID: IntrinsicID, Tys: {Src0->getType(), Src1->getType()});
650 return CGF.Builder.CreateCall(Callee: F, Args: {Src0, Src1});
651}
652
653// Emit an intrinsic that has 3 operands of the same type as its result.
654// Depending on mode, this may be a constrained floating-point intrinsic.
655static Value *emitTernaryMaybeConstrainedFPBuiltin(CodeGenFunction &CGF,
656 const CallExpr *E, unsigned IntrinsicID,
657 unsigned ConstrainedIntrinsicID) {
658 llvm::Value *Src0 = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
659 llvm::Value *Src1 = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
660 llvm::Value *Src2 = CGF.EmitScalarExpr(E: E->getArg(Arg: 2));
661
662 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
663 if (CGF.Builder.getIsFPConstrained()) {
664 Function *F = CGF.CGM.getIntrinsic(IID: ConstrainedIntrinsicID, Tys: Src0->getType());
665 return CGF.Builder.CreateConstrainedFPCall(Callee: F, Args: { Src0, Src1, Src2 });
666 } else {
667 Function *F = CGF.CGM.getIntrinsic(IID: IntrinsicID, Tys: Src0->getType());
668 return CGF.Builder.CreateCall(Callee: F, Args: { Src0, Src1, Src2 });
669 }
670}
671
672// Emit an intrinsic that has overloaded integer result and fp operand.
673static Value *
674emitMaybeConstrainedFPToIntRoundBuiltin(CodeGenFunction &CGF, const CallExpr *E,
675 unsigned IntrinsicID,
676 unsigned ConstrainedIntrinsicID) {
677 llvm::Type *ResultType = CGF.ConvertType(T: E->getType());
678 llvm::Value *Src0 = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
679
680 if (CGF.Builder.getIsFPConstrained()) {
681 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
682 Function *F = CGF.CGM.getIntrinsic(IID: ConstrainedIntrinsicID,
683 Tys: {ResultType, Src0->getType()});
684 return CGF.Builder.CreateConstrainedFPCall(Callee: F, Args: {Src0});
685 } else {
686 Function *F =
687 CGF.CGM.getIntrinsic(IID: IntrinsicID, Tys: {ResultType, Src0->getType()});
688 return CGF.Builder.CreateCall(Callee: F, Args: Src0);
689 }
690}
691
692static Value *emitFrexpBuiltin(CodeGenFunction &CGF, const CallExpr *E,
693 Intrinsic::ID IntrinsicID) {
694 llvm::Value *Src0 = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
695 llvm::Value *Src1 = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
696
697 QualType IntPtrTy = E->getArg(Arg: 1)->getType()->getPointeeType();
698 llvm::Type *IntTy = CGF.ConvertType(T: IntPtrTy);
699 llvm::Function *F =
700 CGF.CGM.getIntrinsic(IID: IntrinsicID, Tys: {Src0->getType(), IntTy});
701 llvm::Value *Call = CGF.Builder.CreateCall(Callee: F, Args: Src0);
702
703 llvm::Value *Exp = CGF.Builder.CreateExtractValue(Agg: Call, Idxs: 1);
704 LValue LV = CGF.MakeNaturalAlignAddrLValue(V: Src1, T: IntPtrTy);
705 CGF.EmitStoreOfScalar(value: Exp, lvalue: LV);
706
707 return CGF.Builder.CreateExtractValue(Agg: Call, Idxs: 0);
708}
709
710static void emitSincosBuiltin(CodeGenFunction &CGF, const CallExpr *E,
711 Intrinsic::ID IntrinsicID) {
712 llvm::Value *Val = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
713 llvm::Value *Dest0 = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
714 llvm::Value *Dest1 = CGF.EmitScalarExpr(E: E->getArg(Arg: 2));
715
716 llvm::Function *F = CGF.CGM.getIntrinsic(IID: IntrinsicID, Tys: {Val->getType()});
717 llvm::Value *Call = CGF.Builder.CreateCall(Callee: F, Args: Val);
718
719 llvm::Value *SinResult = CGF.Builder.CreateExtractValue(Agg: Call, Idxs: 0);
720 llvm::Value *CosResult = CGF.Builder.CreateExtractValue(Agg: Call, Idxs: 1);
721
722 QualType DestPtrType = E->getArg(Arg: 1)->getType()->getPointeeType();
723 LValue SinLV = CGF.MakeNaturalAlignAddrLValue(V: Dest0, T: DestPtrType);
724 LValue CosLV = CGF.MakeNaturalAlignAddrLValue(V: Dest1, T: DestPtrType);
725
726 llvm::StoreInst *StoreSin =
727 CGF.Builder.CreateStore(Val: SinResult, Addr: SinLV.getAddress());
728 llvm::StoreInst *StoreCos =
729 CGF.Builder.CreateStore(Val: CosResult, Addr: CosLV.getAddress());
730
731 // Mark the two stores as non-aliasing with each other. The order of stores
732 // emitted by this builtin is arbitrary, enforcing a particular order will
733 // prevent optimizations later on.
734 llvm::MDBuilder MDHelper(CGF.getLLVMContext());
735 MDNode *Domain = MDHelper.createAnonymousAliasScopeDomain();
736 MDNode *AliasScope = MDHelper.createAnonymousAliasScope(Domain);
737 MDNode *AliasScopeList = MDNode::get(Context&: Call->getContext(), MDs: AliasScope);
738 StoreSin->setMetadata(KindID: LLVMContext::MD_alias_scope, Node: AliasScopeList);
739 StoreCos->setMetadata(KindID: LLVMContext::MD_noalias, Node: AliasScopeList);
740}
741
742static llvm::Value *emitModfBuiltin(CodeGenFunction &CGF, const CallExpr *E,
743 Intrinsic::ID IntrinsicID) {
744 llvm::Value *Val = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
745 llvm::Value *IntPartDest = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
746
747 llvm::Value *Call =
748 CGF.Builder.CreateIntrinsic(ID: IntrinsicID, OverloadTypes: {Val->getType()}, Args: Val);
749
750 llvm::Value *FractionalResult = CGF.Builder.CreateExtractValue(Agg: Call, Idxs: 0);
751 llvm::Value *IntegralResult = CGF.Builder.CreateExtractValue(Agg: Call, Idxs: 1);
752
753 QualType DestPtrType = E->getArg(Arg: 1)->getType()->getPointeeType();
754 LValue IntegralLV = CGF.MakeNaturalAlignAddrLValue(V: IntPartDest, T: DestPtrType);
755 CGF.EmitStoreOfScalar(value: IntegralResult, lvalue: IntegralLV);
756
757 return FractionalResult;
758}
759
760/// EmitFAbs - Emit a call to @llvm.fabs().
761static Value *EmitFAbs(CodeGenFunction &CGF, Value *V) {
762 llvm::Value *Call = CGF.Builder.CreateFAbs(V);
763 if (auto *CallI = dyn_cast<llvm::CallInst>(Val: Call))
764 CallI->setDoesNotAccessMemory();
765 return Call;
766}
767
768/// Emit the computation of the sign bit for a floating point value. Returns
769/// the i1 sign bit value.
770static Value *EmitSignBit(CodeGenFunction &CGF, Value *V) {
771 LLVMContext &C = CGF.CGM.getLLVMContext();
772
773 llvm::Type *Ty = V->getType();
774 int Width = Ty->getPrimitiveSizeInBits();
775 llvm::Type *IntTy = llvm::IntegerType::get(C, NumBits: Width);
776 V = CGF.Builder.CreateBitCast(V, DestTy: IntTy);
777 if (Ty->isPPC_FP128Ty()) {
778 // We want the sign bit of the higher-order double. The bitcast we just
779 // did works as if the double-double was stored to memory and then
780 // read as an i128. The "store" will put the higher-order double in the
781 // lower address in both little- and big-Endian modes, but the "load"
782 // will treat those bits as a different part of the i128: the low bits in
783 // little-Endian, the high bits in big-Endian. Therefore, on big-Endian
784 // we need to shift the high bits down to the low before truncating.
785 Width >>= 1;
786 if (CGF.getTarget().isBigEndian()) {
787 Value *ShiftCst = llvm::ConstantInt::get(Ty: IntTy, V: Width);
788 V = CGF.Builder.CreateLShr(LHS: V, RHS: ShiftCst);
789 }
790 // We are truncating value in order to extract the higher-order
791 // double, which we will be using to extract the sign from.
792 IntTy = llvm::IntegerType::get(C, NumBits: Width);
793 V = CGF.Builder.CreateTrunc(V, DestTy: IntTy);
794 }
795 Value *Zero = llvm::Constant::getNullValue(Ty: IntTy);
796 return CGF.Builder.CreateICmpSLT(LHS: V, RHS: Zero);
797}
798
799static RValue emitLibraryCall(CodeGenFunction &CGF, const FunctionDecl *FD,
800 const CallExpr *E, llvm::Constant *calleeValue) {
801 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
802 CGCallee callee = CGCallee::forDirect(functionPtr: calleeValue, abstractInfo: GlobalDecl(FD));
803 llvm::CallBase *callOrInvoke = nullptr;
804 CGFunctionInfo const *FnInfo = nullptr;
805 return CGF.EmitCall(FnType: E->getCallee()->getType(), Callee: callee, E, ReturnValue: ReturnValueSlot(),
806 /*Chain=*/nullptr, CallOrInvoke: &callOrInvoke, ResolvedFnInfo: &FnInfo);
807}
808
809/// Emit a call to llvm.{sadd,uadd,ssub,usub,smul,umul}.with.overflow.*
810/// depending on IntrinsicID.
811///
812/// \arg CGF The current codegen function.
813/// \arg IntrinsicID The ID for the Intrinsic we wish to generate.
814/// \arg X The first argument to the llvm.*.with.overflow.*.
815/// \arg Y The second argument to the llvm.*.with.overflow.*.
816/// \arg Carry The carry returned by the llvm.*.with.overflow.*.
817/// \returns The result (i.e. sum/product) returned by the intrinsic.
818llvm::Value *EmitOverflowIntrinsic(CodeGenFunction &CGF,
819 const Intrinsic::ID IntrinsicID,
820 llvm::Value *X, llvm::Value *Y,
821 llvm::Value *&Carry) {
822 // Make sure we have integers of the same width.
823 assert(X->getType() == Y->getType() &&
824 "Arguments must be the same type. (Did you forget to make sure both "
825 "arguments have the same integer width?)");
826
827 Function *Callee = CGF.CGM.getIntrinsic(IID: IntrinsicID, Tys: X->getType());
828 llvm::Value *Tmp = CGF.Builder.CreateCall(Callee, Args: {X, Y});
829 Carry = CGF.Builder.CreateExtractValue(Agg: Tmp, Idxs: 1);
830 return CGF.Builder.CreateExtractValue(Agg: Tmp, Idxs: 0);
831}
832
833namespace {
834 struct WidthAndSignedness {
835 unsigned Width;
836 bool Signed;
837 };
838}
839
840static WidthAndSignedness
841getIntegerWidthAndSignedness(const clang::ASTContext &context,
842 const clang::QualType Type) {
843 assert(Type->isIntegerType() && "Given type is not an integer.");
844 unsigned Width = context.getIntWidth(T: Type);
845 bool Signed = Type->isSignedIntegerType();
846 return {.Width: Width, .Signed: Signed};
847}
848
849// Given one or more integer types, this function produces an integer type that
850// encompasses them: any value in one of the given types could be expressed in
851// the encompassing type.
852static struct WidthAndSignedness
853EncompassingIntegerType(ArrayRef<struct WidthAndSignedness> Types) {
854 assert(Types.size() > 0 && "Empty list of types.");
855
856 // If any of the given types is signed, we must return a signed type.
857 bool Signed = false;
858 for (const auto &Type : Types) {
859 Signed |= Type.Signed;
860 }
861
862 // The encompassing type must have a width greater than or equal to the width
863 // of the specified types. Additionally, if the encompassing type is signed,
864 // its width must be strictly greater than the width of any unsigned types
865 // given.
866 unsigned Width = 0;
867 for (const auto &Type : Types) {
868 unsigned MinWidth = Type.Width + (Signed && !Type.Signed);
869 if (Width < MinWidth) {
870 Width = MinWidth;
871 }
872 }
873
874 return {.Width: Width, .Signed: Signed};
875}
876
877Value *CodeGenFunction::EmitVAStartEnd(Value *ArgValue, bool IsStart) {
878 Intrinsic::ID inst = IsStart ? Intrinsic::vastart : Intrinsic::vaend;
879 return Builder.CreateCall(Callee: CGM.getIntrinsic(IID: inst, Tys: {ArgValue->getType()}),
880 Args: ArgValue);
881}
882
883/// Checks if using the result of __builtin_object_size(p, @p From) in place of
884/// __builtin_object_size(p, @p To) is correct
885static bool areBOSTypesCompatible(int From, int To) {
886 // Note: Our __builtin_object_size implementation currently treats Type=0 and
887 // Type=2 identically. Encoding this implementation detail here may make
888 // improving __builtin_object_size difficult in the future, so it's omitted.
889 return From == To || (From == 0 && To == 1) || (From == 3 && To == 2);
890}
891
892static llvm::Value *
893getDefaultBuiltinObjectSizeResult(unsigned Type, llvm::IntegerType *ResType) {
894 return ConstantInt::get(Ty: ResType, V: (Type & 2) ? 0 : -1, /*isSigned=*/IsSigned: true);
895}
896
897llvm::Value *
898CodeGenFunction::evaluateOrEmitBuiltinObjectSize(const Expr *E, unsigned Type,
899 llvm::IntegerType *ResType,
900 llvm::Value *EmittedE,
901 bool IsDynamic) {
902 if (std::optional<uint64_t> ObjectSize =
903 E->tryEvaluateObjectSize(Ctx: getContext(), Type))
904 return ConstantInt::get(Ty: ResType, V: *ObjectSize, /*isSigned=*/IsSigned: true);
905 return emitBuiltinObjectSize(E, Type, ResType, EmittedE, IsDynamic);
906}
907
908/// Find a struct's flexible array member. It may be embedded inside multiple
909/// sub-structs, but must still be the last field.
910static const FieldDecl *FindFlexibleArrayMemberField(CodeGenFunction &CGF,
911 ASTContext &Ctx,
912 const RecordDecl *RD) {
913 const LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel =
914 CGF.getLangOpts().getStrictFlexArraysLevel();
915
916 if (RD->isImplicit())
917 return nullptr;
918
919 for (const FieldDecl *FD : RD->fields()) {
920 if (Decl::isFlexibleArrayMemberLike(
921 Context: Ctx, D: FD, Ty: FD->getType(), StrictFlexArraysLevel,
922 /*IgnoreTemplateOrMacroSubstitution=*/true))
923 return FD;
924
925 if (const auto *RD = FD->getType()->getAsRecordDecl())
926 if (const FieldDecl *FD = FindFlexibleArrayMemberField(CGF, Ctx, RD))
927 return FD;
928 }
929
930 return nullptr;
931}
932
933/// Calculate the offset of a struct field. It may be embedded inside multiple
934/// sub-structs.
935static bool GetFieldOffset(ASTContext &Ctx, const RecordDecl *RD,
936 const FieldDecl *FD, int64_t &Offset) {
937 if (RD->isImplicit())
938 return false;
939
940 // Keep track of the field number ourselves, because the other methods
941 // (CGRecordLayout::getLLVMFieldNo) aren't always equivalent to how the AST
942 // is laid out.
943 uint32_t FieldNo = 0;
944 const ASTRecordLayout &Layout = Ctx.getASTRecordLayout(D: RD);
945
946 for (const FieldDecl *Field : RD->fields()) {
947 if (Field == FD) {
948 Offset += Layout.getFieldOffset(FieldNo);
949 return true;
950 }
951
952 if (const auto *RD = Field->getType()->getAsRecordDecl()) {
953 if (GetFieldOffset(Ctx, RD, FD, Offset)) {
954 Offset += Layout.getFieldOffset(FieldNo);
955 return true;
956 }
957 }
958
959 if (!RD->isUnion())
960 ++FieldNo;
961 }
962
963 return false;
964}
965
966static std::optional<int64_t>
967GetFieldOffset(ASTContext &Ctx, const RecordDecl *RD, const FieldDecl *FD) {
968 int64_t Offset = 0;
969
970 if (GetFieldOffset(Ctx, RD, FD, Offset))
971 return std::optional<int64_t>(Offset);
972
973 return std::nullopt;
974}
975
976llvm::Value *CodeGenFunction::emitCountedBySize(const Expr *E,
977 llvm::Value *EmittedE,
978 unsigned Type,
979 llvm::IntegerType *ResType) {
980 // Note: If the whole struct is specificed in the __bdos (i.e. Visitor
981 // returns a DeclRefExpr). The calculation of the whole size of the structure
982 // with a flexible array member can be done in two ways:
983 //
984 // 1) sizeof(struct S) + count * sizeof(typeof(fam))
985 // 2) offsetof(struct S, fam) + count * sizeof(typeof(fam))
986 //
987 // The first will add additional padding after the end of the array
988 // allocation while the second method is more precise, but not quite expected
989 // from programmers. See
990 // https://lore.kernel.org/lkml/ZvV6X5FPBBW7CO1f@archlinux/ for a discussion
991 // of the topic.
992 //
993 // GCC isn't (currently) able to calculate __bdos on a pointer to the whole
994 // structure. Therefore, because of the above issue, we choose to match what
995 // GCC does for consistency's sake.
996
997 const Expr *Idx = nullptr;
998 // FIXME: `ArrayElementTy` is misleadingly named. `findStructFieldAccess()`
999 // sets it to the type of the array-subscript base, i.e. the (possibly cast)
1000 // *pointer* being indexed (not an element type) or a null QualType when there
1001 // is no subscript.
1002 QualType ArrayElementTy;
1003 E = findStructFieldAccess(E, OutArrayIndex: &Idx, OutArrayElementTy: &ArrayElementTy);
1004 if (!E)
1005 return nullptr;
1006
1007 if (Idx) {
1008 if (Idx->HasSideEffects(Ctx: getContext()))
1009 // We can't have side-effects.
1010 return getDefaultBuiltinObjectSizeResult(Type, ResType);
1011
1012 if (const auto *IL = dyn_cast<IntegerLiteral>(Val: Idx)) {
1013 int64_t Val = IL->getValue().getSExtValue();
1014 if (Val < 0)
1015 return getDefaultBuiltinObjectSizeResult(Type, ResType);
1016
1017 // The index is 0, so we don't need to take it into account.
1018 if (Val == 0)
1019 Idx = nullptr;
1020 }
1021 }
1022
1023 // __counted_by on either a flexible array member or a pointer into a struct
1024 // with a flexible array member.
1025 if (const auto *ME = dyn_cast<MemberExpr>(Val: E))
1026 return emitCountedByMemberSize(E: ME, Idx, EmittedE, CastedArrayElementTy: ArrayElementTy, Type,
1027 ResType);
1028
1029 // __counted_by on a pointer in a struct.
1030 if (const auto *ICE = dyn_cast<ImplicitCastExpr>(Val: E);
1031 ICE && ICE->getCastKind() == CK_LValueToRValue)
1032 return emitCountedByPointerSize(E: ICE, Idx, EmittedE, CastedArrayElementTy: ArrayElementTy, Type,
1033 ResType);
1034
1035 return nullptr;
1036}
1037
1038static llvm::Value *EmitPositiveResultOrZero(CodeGenFunction &CGF,
1039 llvm::Value *Res,
1040 llvm::Value *Index,
1041 llvm::IntegerType *ResType,
1042 bool IsSigned) {
1043 // cmp = (array_size >= 0)
1044 Value *Cmp = CGF.Builder.CreateIsNotNeg(Arg: Res);
1045 if (Index)
1046 // cmp = (cmp && index >= 0)
1047 Cmp = CGF.Builder.CreateAnd(LHS: CGF.Builder.CreateIsNotNeg(Arg: Index), RHS: Cmp);
1048
1049 // return cmp ? result : 0
1050 return CGF.Builder.CreateSelect(C: Cmp, True: Res,
1051 False: ConstantInt::get(Ty: ResType, V: 0, IsSigned));
1052}
1053
1054static std::pair<llvm::Value *, llvm::Value *>
1055GetCountFieldAndIndex(CodeGenFunction &CGF, const MemberExpr *ME,
1056 const FieldDecl *ArrayFD, const FieldDecl *CountFD,
1057 const Expr *Idx, llvm::IntegerType *ResType,
1058 bool IsSigned) {
1059 // count = ptr->count;
1060 Value *Count = CGF.EmitLoadOfCountedByField(Base: ME, FD: ArrayFD, CountDecl: CountFD);
1061 if (!Count)
1062 return std::make_pair<Value *>(x: nullptr, y: nullptr);
1063 Count = CGF.Builder.CreateIntCast(V: Count, DestTy: ResType, isSigned: IsSigned, Name: "count");
1064
1065 // index = idx;
1066 Value *Index = nullptr;
1067 if (Idx) {
1068 bool IdxSigned = Idx->getType()->isSignedIntegerType();
1069 Index = CGF.EmitScalarExpr(E: Idx);
1070 Index = CGF.Builder.CreateIntCast(V: Index, DestTy: ResType, isSigned: IdxSigned, Name: "index");
1071 }
1072
1073 return std::make_pair(x&: Count, y&: Index);
1074}
1075
1076llvm::Value *CodeGenFunction::emitCountedByPointerSize(
1077 const ImplicitCastExpr *E, const Expr *Idx, llvm::Value *EmittedE,
1078 QualType CastedArrayElementTy, unsigned Type, llvm::IntegerType *ResType) {
1079 assert(E->getCastKind() == CK_LValueToRValue &&
1080 "must be an LValue to RValue cast");
1081 assert(EmittedE && "emitted must not be null");
1082
1083 const MemberExpr *ME =
1084 dyn_cast<MemberExpr>(Val: E->getSubExpr()->IgnoreParenNoopCasts(Ctx: getContext()));
1085 if (!ME)
1086 return nullptr;
1087
1088 const auto *ArrayBaseFD = dyn_cast<FieldDecl>(Val: ME->getMemberDecl());
1089 if (!ArrayBaseFD || !ArrayBaseFD->getType()->isPointerType() ||
1090 !ArrayBaseFD->getType()->isCountAttributedType())
1091 return nullptr;
1092
1093 // Get the 'count' FieldDecl.
1094 const FieldDecl *CountFD = ArrayBaseFD->findCountedByField();
1095 if (!CountFD)
1096 // Can't find the field referenced by the "counted_by" attribute.
1097 return nullptr;
1098
1099 // Calculate the array's object size using these formulae. (Note: if the
1100 // calculation is negative, we return 0.):
1101 //
1102 // struct p;
1103 // struct s {
1104 // /* ... */
1105 // struct p **array __attribute__((ATTR(count)));
1106 // int count;
1107 // };
1108 //
1109 // 1) 'ptr->array':
1110 //
1111 // #if ATTR is counted_by_or_null || ATTR is sized_by_or_null
1112 // count = ptr->array ? ptr->count : 0;
1113 // #else
1114 // count = ptr->count;
1115 // #endif
1116 //
1117 // #if ATTR is counted_by || ATTR is counted_by_or_null
1118 // array_element_size = sizeof (*ptr->array);
1119 // array_size = count * array_element_size;
1120 // #else
1121 // array_size = count;
1122 // #endif
1123 //
1124 // result = array_size;
1125 //
1126 // cmp = (result >= 0)
1127 // return cmp ? result : 0;
1128 //
1129 // 2) '&((cast) ptr->array)[idx]':
1130 //
1131 // #if ATTR is counted_by_or_null || ATTR is sized_by_or_null
1132 // count = ptr->array ? ptr->count : 0;
1133 // #else
1134 // count = ptr->count;
1135 // #endif
1136 // index = idx;
1137 //
1138 // #if ATTR is counted_by || ATTR is counted_by_or_null
1139 // array_element_size = sizeof (*ptr->array);
1140 // array_size = count * array_element_size;
1141 // #else
1142 // array_size = count;
1143 // #endif
1144 //
1145 // casted_array_element_size = sizeof (*((cast) ptr->array));
1146 //
1147 // index_size = index * casted_array_element_size;
1148 // result = array_size - index_size;
1149 //
1150 // cmp = (result >= 0)
1151 // if (index)
1152 // cmp = (cmp && index > 0)
1153 // return cmp ? result : 0;
1154
1155 auto GetPointeeSize = [&](QualType PtrTy) -> CharUnits {
1156 assert(!PtrTy.isNull());
1157 QualType PointeeTy = PtrTy->getPointeeType();
1158 assert(!PointeeTy.isNull() &&
1159 (PointeeTy->isVoidType() || !PointeeTy->isIncompleteType()) &&
1160 "pointee type must have a computable size");
1161
1162 CharUnits PointeeSize = getContext().getTypeSizeInChars(T: PointeeTy);
1163 if (PointeeSize.isZero()) {
1164 // Support GNU extension of treating `void` having size 1.
1165 PointeeSize = CharUnits::One();
1166 }
1167
1168 return PointeeSize;
1169 };
1170
1171 bool IsSigned = CountFD->getType()->isSignedIntegerType();
1172 const auto *CountAttributedTy =
1173 ArrayBaseFD->getType()->getAs<CountAttributedType>();
1174 assert(CountAttributedTy && "the field's type is not a CountAttributedType");
1175
1176 // count = ptr->count;
1177 // index = idx;
1178 Value *Count, *Index;
1179 std::tie(args&: Count, args&: Index) = GetCountFieldAndIndex(
1180 CGF&: *this, ME, ArrayFD: ArrayBaseFD, CountFD, Idx, ResType, IsSigned);
1181 if (!Count)
1182 return nullptr;
1183
1184 // For the _or_null variants, a null pointer describes no accessible memory:
1185 // count = ptr->array ? count : 0;
1186 if (CountAttributedTy->isOrNull()) {
1187 Value *Ptr = nullptr;
1188 if (!Idx) {
1189 // 1) 'ptr->array'
1190 // Reuse the already-emitted pointer value rather than re-loading `ME`.
1191 // Re-loading would produce a second, observable access for a volatile
1192 // pointer field
1193 Ptr = EmittedE;
1194 } else {
1195 // 2) '&((cast) ptr->array)[idx]'
1196 // FIXME: `EmittedE` is the element address, not `ptr->array`, so we fall
1197 // back to re-emitting `ME` and the pointer field is loaded twice. This is
1198 // normally harmless except when the pointer is `volatile`. Avoiding that
1199 // would require restructuring how the base pointer is emitted (it is
1200 // handled elsewhere in the callstack), so it is left as-is for now.
1201 Ptr = EmitScalarExpr(E: ME);
1202 }
1203 Value *IsNull = Builder.CreateIsNull(Arg: Ptr);
1204 Count = Builder.CreateSelect(C: IsNull, True: ConstantInt::get(Ty: ResType, V: 0, IsSigned),
1205 False: Count, Name: "count.or.null");
1206 }
1207
1208 // #if ATTR is counted_by || ATTR is counted_by_or_null
1209 // array_element_size = sizeof (*ptr->array);
1210 // array_size = count * array_element_size;
1211 // #else
1212 // array_size = count;
1213 // #endif
1214 Value *ArraySize;
1215 if (!CountAttributedTy->isCountInBytes()) {
1216 // `__counted_by`/`__counted_by_or_null` require a complete pointee at use
1217 // sites (enforced by Sema) so the element size is computable.
1218 CharUnits ArrayElementBaseSize = GetPointeeSize(ArrayBaseFD->getType());
1219
1220 // array_element_size = sizeof (*ptr->array)
1221 auto *ArrayElementSize = llvm::ConstantInt::get(
1222 Ty: ResType, V: ArrayElementBaseSize.getQuantity(), IsSigned);
1223
1224 // array_size = count * array_element_size;
1225 ArraySize = Builder.CreateMul(LHS: Count, RHS: ArrayElementSize, Name: "array_size",
1226 HasNUW: !IsSigned, HasNSW: IsSigned);
1227 } else {
1228 // array_size = count;
1229 ArraySize = Count;
1230 }
1231
1232 // Option (1) 'ptr->array'
1233 // result = array_size
1234 Value *Result = ArraySize;
1235
1236 if (Idx) { // Option (2) '&((cast) ptr->array)[idx]'
1237 // FIXME: CastedArrayElementTy is confusingly named. It's actually the base
1238 // expression of the ArraySubscriptExpr, not the element (pointee) type.
1239 CharUnits CastedArrayElementSizeInChars =
1240 GetPointeeSize(CastedArrayElementTy);
1241
1242 // casted_array_element_size = sizeof (*((cast) ptr->array));
1243 auto *CastedArrayElementSize = llvm::ConstantInt::get(
1244 Ty: ResType, V: CastedArrayElementSizeInChars.getQuantity(), IsSigned);
1245
1246 // index_size = index * casted_array_element_size;
1247 Value *IndexSize = Builder.CreateMul(LHS: Index, RHS: CastedArrayElementSize,
1248 Name: "index_size", HasNUW: !IsSigned, HasNSW: IsSigned);
1249
1250 // result = result - index_size;
1251 Result =
1252 Builder.CreateSub(LHS: Result, RHS: IndexSize, Name: "result", HasNUW: !IsSigned, HasNSW: IsSigned);
1253 }
1254
1255 return EmitPositiveResultOrZero(CGF&: *this, Res: Result, Index, ResType, IsSigned);
1256}
1257
1258llvm::Value *CodeGenFunction::emitCountedByMemberSize(
1259 const MemberExpr *ME, const Expr *Idx, llvm::Value *EmittedE,
1260 QualType CastedArrayElementTy, unsigned Type, llvm::IntegerType *ResType) {
1261 const auto *FD = dyn_cast<FieldDecl>(Val: ME->getMemberDecl());
1262 if (!FD)
1263 return nullptr;
1264
1265 // Find the flexible array member and check that it has the __counted_by
1266 // attribute.
1267 ASTContext &Ctx = getContext();
1268 const RecordDecl *RD = FD->getDeclContext()->getOuterLexicalRecordContext();
1269 const FieldDecl *FlexibleArrayMemberFD = nullptr;
1270
1271 if (Decl::isFlexibleArrayMemberLike(
1272 Context: Ctx, D: FD, Ty: FD->getType(), StrictFlexArraysLevel: getLangOpts().getStrictFlexArraysLevel(),
1273 /*IgnoreTemplateOrMacroSubstitution=*/true))
1274 FlexibleArrayMemberFD = FD;
1275 else
1276 FlexibleArrayMemberFD = FindFlexibleArrayMemberField(CGF&: *this, Ctx, RD);
1277
1278 if (!FlexibleArrayMemberFD ||
1279 !FlexibleArrayMemberFD->getType()->isCountAttributedType())
1280 return nullptr;
1281
1282 // Get the 'count' FieldDecl.
1283 const FieldDecl *CountFD = FlexibleArrayMemberFD->findCountedByField();
1284 if (!CountFD)
1285 // Can't find the field referenced by the "counted_by" attribute.
1286 return nullptr;
1287
1288 // Calculate the flexible array member's object size using these formulae.
1289 // (Note: if the calculation is negative, we return 0.):
1290 //
1291 // struct p;
1292 // struct s {
1293 // /* ... */
1294 // int count;
1295 // struct p *array[] __attribute__((counted_by(count)));
1296 // };
1297 //
1298 // 1) 'ptr->array':
1299 //
1300 // count = ptr->count;
1301 //
1302 // flexible_array_member_element_size = sizeof (*ptr->array);
1303 // flexible_array_member_size =
1304 // count * flexible_array_member_element_size;
1305 //
1306 // result = flexible_array_member_size;
1307 //
1308 // cmp = (result >= 0)
1309 // return cmp ? result : 0;
1310 //
1311 // 2) '&((cast) ptr->array)[idx]':
1312 //
1313 // count = ptr->count;
1314 // index = idx;
1315 //
1316 // flexible_array_member_element_size = sizeof (*ptr->array);
1317 // flexible_array_member_size =
1318 // count * flexible_array_member_element_size;
1319 //
1320 // casted_flexible_array_member_element_size =
1321 // sizeof (*((cast) ptr->array));
1322 // index_size = index * casted_flexible_array_member_element_size;
1323 //
1324 // result = flexible_array_member_size - index_size;
1325 //
1326 // cmp = (result >= 0)
1327 // if (index != 0)
1328 // cmp = (cmp && index >= 0)
1329 // return cmp ? result : 0;
1330 //
1331 // 3) '&ptr->field':
1332 //
1333 // count = ptr->count;
1334 // sizeof_struct = sizeof (struct s);
1335 //
1336 // flexible_array_member_element_size = sizeof (*ptr->array);
1337 // flexible_array_member_size =
1338 // count * flexible_array_member_element_size;
1339 //
1340 // field_offset = offsetof (struct s, field);
1341 // offset_diff = sizeof_struct - field_offset;
1342 //
1343 // result = offset_diff + flexible_array_member_size;
1344 //
1345 // cmp = (result >= 0)
1346 // return cmp ? result : 0;
1347 //
1348 // 4) '&((cast) ptr->field_array)[idx]':
1349 //
1350 // count = ptr->count;
1351 // index = idx;
1352 // sizeof_struct = sizeof (struct s);
1353 //
1354 // flexible_array_member_element_size = sizeof (*ptr->array);
1355 // flexible_array_member_size =
1356 // count * flexible_array_member_element_size;
1357 //
1358 // casted_field_element_size = sizeof (*((cast) ptr->field_array));
1359 // field_offset = offsetof (struct s, field)
1360 // field_offset += index * casted_field_element_size;
1361 //
1362 // offset_diff = sizeof_struct - field_offset;
1363 //
1364 // result = offset_diff + flexible_array_member_size;
1365 //
1366 // cmp = (result >= 0)
1367 // if (index != 0)
1368 // cmp = (cmp && index >= 0)
1369 // return cmp ? result : 0;
1370
1371 bool IsSigned = CountFD->getType()->isSignedIntegerType();
1372
1373 QualType FlexibleArrayMemberTy = FlexibleArrayMemberFD->getType();
1374
1375 // Explicit cast because otherwise the CharWidth will promote an i32's into
1376 // u64's leading to overflows.
1377 int64_t CharWidth = static_cast<int64_t>(CGM.getContext().getCharWidth());
1378
1379 // field_offset = offsetof (struct s, field);
1380 Value *FieldOffset = nullptr;
1381 if (FlexibleArrayMemberFD != FD) {
1382 std::optional<int64_t> Offset = GetFieldOffset(Ctx, RD, FD);
1383 if (!Offset)
1384 return nullptr;
1385 FieldOffset =
1386 llvm::ConstantInt::get(Ty: ResType, V: *Offset / CharWidth, IsSigned);
1387 }
1388
1389 // count = ptr->count;
1390 // index = ptr->index;
1391 Value *Count, *Index;
1392 std::tie(args&: Count, args&: Index) = GetCountFieldAndIndex(
1393 CGF&: *this, ME, ArrayFD: FlexibleArrayMemberFD, CountFD, Idx, ResType, IsSigned);
1394 if (!Count)
1395 return nullptr;
1396
1397 // flexible_array_member_element_size = sizeof (*ptr->array);
1398 const ArrayType *ArrayTy = Ctx.getAsArrayType(T: FlexibleArrayMemberTy);
1399 CharUnits BaseSize = Ctx.getTypeSizeInChars(T: ArrayTy->getElementType());
1400 auto *FlexibleArrayMemberElementSize =
1401 llvm::ConstantInt::get(Ty: ResType, V: BaseSize.getQuantity(), IsSigned);
1402
1403 // flexible_array_member_size = count * flexible_array_member_element_size;
1404 Value *FlexibleArrayMemberSize =
1405 Builder.CreateMul(LHS: Count, RHS: FlexibleArrayMemberElementSize,
1406 Name: "flexible_array_member_size", HasNUW: !IsSigned, HasNSW: IsSigned);
1407
1408 Value *Result = nullptr;
1409 if (FlexibleArrayMemberFD == FD) {
1410 if (Idx) { // Option (2) '&((cast) ptr->array)[idx]'
1411 // casted_flexible_array_member_element_size =
1412 // sizeof (*((cast) ptr->array));
1413 llvm::ConstantInt *CastedFlexibleArrayMemberElementSize =
1414 FlexibleArrayMemberElementSize;
1415 if (!CastedArrayElementTy.isNull() &&
1416 CastedArrayElementTy->isPointerType()) {
1417 CharUnits BaseSize =
1418 Ctx.getTypeSizeInChars(T: CastedArrayElementTy->getPointeeType());
1419 CastedFlexibleArrayMemberElementSize =
1420 llvm::ConstantInt::get(Ty: ResType, V: BaseSize.getQuantity(), IsSigned);
1421 }
1422
1423 // index_size = index * casted_flexible_array_member_element_size;
1424 Value *IndexSize =
1425 Builder.CreateMul(LHS: Index, RHS: CastedFlexibleArrayMemberElementSize,
1426 Name: "index_size", HasNUW: !IsSigned, HasNSW: IsSigned);
1427
1428 // result = flexible_array_member_size - index_size;
1429 Result = Builder.CreateSub(LHS: FlexibleArrayMemberSize, RHS: IndexSize, Name: "result",
1430 HasNUW: !IsSigned, HasNSW: IsSigned);
1431 } else { // Option (1) 'ptr->array'
1432 // result = flexible_array_member_size;
1433 Result = FlexibleArrayMemberSize;
1434 }
1435 } else {
1436 // sizeof_struct = sizeof (struct s);
1437 llvm::StructType *StructTy = getTypes().getCGRecordLayout(RD).getLLVMType();
1438 const llvm::DataLayout &Layout = CGM.getDataLayout();
1439 TypeSize Size = Layout.getTypeSizeInBits(Ty: StructTy);
1440 Value *SizeofStruct =
1441 llvm::ConstantInt::get(Ty: ResType, V: Size.getKnownMinValue() / CharWidth);
1442
1443 if (Idx) { // Option (4) '&((cast) ptr->field_array)[idx]'
1444 // casted_field_element_size = sizeof (*((cast) ptr->field_array));
1445 CharUnits BaseSize;
1446 if (!CastedArrayElementTy.isNull() &&
1447 CastedArrayElementTy->isPointerType()) {
1448 BaseSize =
1449 Ctx.getTypeSizeInChars(T: CastedArrayElementTy->getPointeeType());
1450 } else {
1451 const ArrayType *ArrayTy = Ctx.getAsArrayType(T: FD->getType());
1452 BaseSize = Ctx.getTypeSizeInChars(T: ArrayTy->getElementType());
1453 }
1454
1455 llvm::ConstantInt *CastedFieldElementSize =
1456 llvm::ConstantInt::get(Ty: ResType, V: BaseSize.getQuantity(), IsSigned);
1457
1458 // field_offset += index * casted_field_element_size;
1459 Value *Mul = Builder.CreateMul(LHS: Index, RHS: CastedFieldElementSize,
1460 Name: "field_offset", HasNUW: !IsSigned, HasNSW: IsSigned);
1461 FieldOffset = Builder.CreateAdd(LHS: FieldOffset, RHS: Mul);
1462 }
1463 // Option (3) '&ptr->field', and Option (4) continuation.
1464 // offset_diff = flexible_array_member_offset - field_offset;
1465 Value *OffsetDiff = Builder.CreateSub(LHS: SizeofStruct, RHS: FieldOffset,
1466 Name: "offset_diff", HasNUW: !IsSigned, HasNSW: IsSigned);
1467
1468 // result = offset_diff + flexible_array_member_size;
1469 Result = Builder.CreateAdd(LHS: FlexibleArrayMemberSize, RHS: OffsetDiff, Name: "result");
1470 }
1471
1472 return EmitPositiveResultOrZero(CGF&: *this, Res: Result, Index, ResType, IsSigned);
1473}
1474
1475/// Returns a Value corresponding to the size of the given expression.
1476/// This Value may be either of the following:
1477/// - A llvm::Argument (if E is a param with the pass_object_size attribute on
1478/// it)
1479/// - A call to the @llvm.objectsize intrinsic
1480///
1481/// EmittedE is the result of emitting `E` as a scalar expr. If it's non-null
1482/// and we wouldn't otherwise try to reference a pass_object_size parameter,
1483/// we'll call @llvm.objectsize on EmittedE, rather than emitting E.
1484llvm::Value *
1485CodeGenFunction::emitBuiltinObjectSize(const Expr *E, unsigned Type,
1486 llvm::IntegerType *ResType,
1487 llvm::Value *EmittedE, bool IsDynamic) {
1488 // We need to reference an argument if the pointer is a parameter with the
1489 // pass_object_size attribute.
1490 if (auto *D = dyn_cast<DeclRefExpr>(Val: E->IgnoreParenImpCasts())) {
1491 auto *Param = dyn_cast<ParmVarDecl>(Val: D->getDecl());
1492 auto *PS = D->getDecl()->getAttr<PassObjectSizeAttr>();
1493 if (Param != nullptr && PS != nullptr &&
1494 areBOSTypesCompatible(From: PS->getType(), To: Type)) {
1495 auto Iter = SizeArguments.find(Val: Param);
1496 assert(Iter != SizeArguments.end());
1497
1498 const ImplicitParamDecl *D = Iter->second;
1499 auto DIter = LocalDeclMap.find(Val: D);
1500 assert(DIter != LocalDeclMap.end());
1501
1502 return EmitLoadOfScalar(Addr: DIter->second, /*Volatile=*/false,
1503 Ty: getContext().getSizeType(), Loc: E->getBeginLoc());
1504 }
1505 }
1506
1507 // LLVM can't handle Type=3 appropriately, and __builtin_object_size shouldn't
1508 // evaluate E for side-effects. In either case, we shouldn't lower to
1509 // @llvm.objectsize.
1510 if (Type == 3 || (!EmittedE && E->HasSideEffects(Ctx: getContext())))
1511 return getDefaultBuiltinObjectSizeResult(Type, ResType);
1512
1513 Value *Ptr = EmittedE ? EmittedE : EmitScalarExpr(E);
1514 assert(Ptr->getType()->isPointerTy() &&
1515 "Non-pointer passed to __builtin_object_size?");
1516
1517 if (IsDynamic)
1518 // Emit special code for a flexible array member with the "counted_by"
1519 // attribute.
1520 if (Value *V = emitCountedBySize(E, EmittedE: Ptr, Type, ResType))
1521 return V;
1522
1523 Function *F =
1524 CGM.getIntrinsic(IID: Intrinsic::objectsize, Tys: {ResType, Ptr->getType()});
1525
1526 // LLVM only supports 0 and 2, make sure that we pass along that as a boolean.
1527 Value *Min = Builder.getInt1(V: (Type & 2) != 0);
1528 // For GCC compatibility, __builtin_object_size treat NULL as unknown size.
1529 Value *NullIsUnknown = Builder.getTrue();
1530 Value *Dynamic = Builder.getInt1(V: IsDynamic);
1531 return Builder.CreateCall(Callee: F, Args: {Ptr, Min, NullIsUnknown, Dynamic});
1532}
1533
1534namespace {
1535/// A struct to generically describe a bit test intrinsic.
1536struct BitTest {
1537 enum ActionKind : uint8_t { TestOnly, Complement, Reset, Set };
1538 enum InterlockingKind : uint8_t {
1539 Unlocked,
1540 Sequential,
1541 Acquire,
1542 Release,
1543 NoFence
1544 };
1545
1546 ActionKind Action;
1547 InterlockingKind Interlocking;
1548 bool Is64Bit;
1549
1550 static BitTest decodeBitTestBuiltin(unsigned BuiltinID);
1551};
1552
1553} // namespace
1554
1555BitTest BitTest::decodeBitTestBuiltin(unsigned BuiltinID) {
1556 switch (BuiltinID) {
1557 // Main portable variants.
1558 case Builtin::BI_bittest:
1559 return {.Action: TestOnly, .Interlocking: Unlocked, .Is64Bit: false};
1560 case Builtin::BI_bittestandcomplement:
1561 return {.Action: Complement, .Interlocking: Unlocked, .Is64Bit: false};
1562 case Builtin::BI_bittestandreset:
1563 return {.Action: Reset, .Interlocking: Unlocked, .Is64Bit: false};
1564 case Builtin::BI_bittestandset:
1565 return {.Action: Set, .Interlocking: Unlocked, .Is64Bit: false};
1566 case Builtin::BI_interlockedbittestandreset:
1567 return {.Action: Reset, .Interlocking: Sequential, .Is64Bit: false};
1568 case Builtin::BI_interlockedbittestandset:
1569 return {.Action: Set, .Interlocking: Sequential, .Is64Bit: false};
1570
1571 // 64-bit variants.
1572 case Builtin::BI_bittest64:
1573 return {.Action: TestOnly, .Interlocking: Unlocked, .Is64Bit: true};
1574 case Builtin::BI_bittestandcomplement64:
1575 return {.Action: Complement, .Interlocking: Unlocked, .Is64Bit: true};
1576 case Builtin::BI_bittestandreset64:
1577 return {.Action: Reset, .Interlocking: Unlocked, .Is64Bit: true};
1578 case Builtin::BI_bittestandset64:
1579 return {.Action: Set, .Interlocking: Unlocked, .Is64Bit: true};
1580 case Builtin::BI_interlockedbittestandreset64:
1581 return {.Action: Reset, .Interlocking: Sequential, .Is64Bit: true};
1582 case Builtin::BI_interlockedbittestandset64:
1583 return {.Action: Set, .Interlocking: Sequential, .Is64Bit: true};
1584
1585 // ARM/AArch64-specific ordering variants.
1586 case Builtin::BI_interlockedbittestandset_acq:
1587 return {.Action: Set, .Interlocking: Acquire, .Is64Bit: false};
1588 case Builtin::BI_interlockedbittestandset_rel:
1589 return {.Action: Set, .Interlocking: Release, .Is64Bit: false};
1590 case Builtin::BI_interlockedbittestandset_nf:
1591 return {.Action: Set, .Interlocking: NoFence, .Is64Bit: false};
1592 case Builtin::BI_interlockedbittestandreset_acq:
1593 return {.Action: Reset, .Interlocking: Acquire, .Is64Bit: false};
1594 case Builtin::BI_interlockedbittestandreset_rel:
1595 return {.Action: Reset, .Interlocking: Release, .Is64Bit: false};
1596 case Builtin::BI_interlockedbittestandreset_nf:
1597 return {.Action: Reset, .Interlocking: NoFence, .Is64Bit: false};
1598 case Builtin::BI_interlockedbittestandreset64_acq:
1599 return {.Action: Reset, .Interlocking: Acquire, .Is64Bit: false};
1600 case Builtin::BI_interlockedbittestandreset64_rel:
1601 return {.Action: Reset, .Interlocking: Release, .Is64Bit: false};
1602 case Builtin::BI_interlockedbittestandreset64_nf:
1603 return {.Action: Reset, .Interlocking: NoFence, .Is64Bit: false};
1604 case Builtin::BI_interlockedbittestandset64_acq:
1605 return {.Action: Set, .Interlocking: Acquire, .Is64Bit: false};
1606 case Builtin::BI_interlockedbittestandset64_rel:
1607 return {.Action: Set, .Interlocking: Release, .Is64Bit: false};
1608 case Builtin::BI_interlockedbittestandset64_nf:
1609 return {.Action: Set, .Interlocking: NoFence, .Is64Bit: false};
1610 }
1611 llvm_unreachable("expected only bittest intrinsics");
1612}
1613
1614static char bitActionToX86BTCode(BitTest::ActionKind A) {
1615 switch (A) {
1616 case BitTest::TestOnly: return '\0';
1617 case BitTest::Complement: return 'c';
1618 case BitTest::Reset: return 'r';
1619 case BitTest::Set: return 's';
1620 }
1621 llvm_unreachable("invalid action");
1622}
1623
1624static llvm::Value *EmitX86BitTestIntrinsic(CodeGenFunction &CGF,
1625 BitTest BT,
1626 const CallExpr *E, Value *BitBase,
1627 Value *BitPos) {
1628 char Action = bitActionToX86BTCode(A: BT.Action);
1629 char SizeSuffix = BT.Is64Bit ? 'q' : 'l';
1630
1631 // Build the assembly.
1632 SmallString<64> Asm;
1633 raw_svector_ostream AsmOS(Asm);
1634 if (BT.Interlocking != BitTest::Unlocked)
1635 AsmOS << "lock ";
1636 AsmOS << "bt";
1637 if (Action)
1638 AsmOS << Action;
1639 AsmOS << SizeSuffix << " $2, ($1)";
1640
1641 // Build the constraints. FIXME: We should support immediates when possible.
1642 std::string Constraints = "={@ccc},r,r,~{cc},~{memory}";
1643 std::string_view MachineClobbers = CGF.getTarget().getClobbers();
1644 if (!MachineClobbers.empty()) {
1645 Constraints += ',';
1646 Constraints += MachineClobbers;
1647 }
1648 llvm::IntegerType *IntType = llvm::IntegerType::get(
1649 C&: CGF.getLLVMContext(),
1650 NumBits: CGF.getContext().getTypeSize(T: E->getArg(Arg: 1)->getType()));
1651 llvm::FunctionType *FTy =
1652 llvm::FunctionType::get(Result: CGF.Int8Ty, Params: {CGF.DefaultPtrTy, IntType}, isVarArg: false);
1653
1654 llvm::InlineAsm *IA =
1655 llvm::InlineAsm::get(Ty: FTy, AsmString: Asm, Constraints, /*hasSideEffects=*/true);
1656 return CGF.Builder.CreateCall(Callee: IA, Args: {BitBase, BitPos});
1657}
1658
1659static llvm::AtomicOrdering
1660getBitTestAtomicOrdering(BitTest::InterlockingKind I) {
1661 switch (I) {
1662 case BitTest::Unlocked: return llvm::AtomicOrdering::NotAtomic;
1663 case BitTest::Sequential: return llvm::AtomicOrdering::SequentiallyConsistent;
1664 case BitTest::Acquire: return llvm::AtomicOrdering::Acquire;
1665 case BitTest::Release: return llvm::AtomicOrdering::Release;
1666 case BitTest::NoFence: return llvm::AtomicOrdering::Monotonic;
1667 }
1668 llvm_unreachable("invalid interlocking");
1669}
1670
1671static llvm::Value *EmitBitCountExpr(CodeGenFunction &CGF, const Expr *E) {
1672 llvm::Value *ArgValue = CGF.EmitScalarExpr(E);
1673 llvm::Type *ArgType = ArgValue->getType();
1674
1675 // Boolean vectors can be casted directly to its bitfield representation. We
1676 // intentionally do not round up to the next power of two size and let LLVM
1677 // handle the trailing bits.
1678 //
1679 // In big endian mode, the bitfield representation has a reversed bit order,
1680 // hence the need to add an operation to reverse it back to the expected
1681 // order.
1682 if (auto *VT = dyn_cast<llvm::FixedVectorType>(Val: ArgType);
1683 VT && VT->getElementType()->isIntegerTy(BitWidth: 1)) {
1684 llvm::Type *StorageType =
1685 llvm::Type::getIntNTy(C&: CGF.getLLVMContext(), N: VT->getNumElements());
1686 ArgValue = CGF.Builder.CreateBitCast(V: ArgValue, DestTy: StorageType);
1687
1688 if (CGF.getTarget().isBigEndian())
1689 ArgValue = CGF.Builder.CreateIntrinsic(ID: Intrinsic::bitreverse,
1690 OverloadTypes: {StorageType}, Args: ArgValue);
1691 }
1692
1693 return ArgValue;
1694}
1695
1696/// Emit a _bittest* intrinsic. These intrinsics take a pointer to an array of
1697/// bits and a bit position and read and optionally modify the bit at that
1698/// position. The position index can be arbitrarily large, i.e. it can be larger
1699/// than 31 or 63, so we need an indexed load in the general case.
1700static llvm::Value *EmitBitTestIntrinsic(CodeGenFunction &CGF,
1701 unsigned BuiltinID,
1702 const CallExpr *E) {
1703 Value *BitBase = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
1704 Value *BitPos = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
1705
1706 BitTest BT = BitTest::decodeBitTestBuiltin(BuiltinID);
1707
1708 // X86 has special BT, BTC, BTR, and BTS instructions that handle the array
1709 // indexing operation internally. Use them if possible.
1710 if (CGF.getTarget().getTriple().isX86())
1711 return EmitX86BitTestIntrinsic(CGF, BT, E, BitBase, BitPos);
1712
1713 // Otherwise, use generic code to load one byte and test the bit. Use all but
1714 // the bottom three bits as the array index, and the bottom three bits to form
1715 // a mask.
1716 // Bit = BitBaseI8[BitPos >> 3] & (1 << (BitPos & 0x7)) != 0;
1717 Value *ByteIndex = CGF.Builder.CreateAShr(
1718 LHS: BitPos, RHS: llvm::ConstantInt::get(Ty: BitPos->getType(), V: 3), Name: "bittest.byteidx");
1719 Address ByteAddr(CGF.Builder.CreateInBoundsGEP(Ty: CGF.Int8Ty, Ptr: BitBase, IdxList: ByteIndex,
1720 Name: "bittest.byteaddr"),
1721 CGF.Int8Ty, CharUnits::One());
1722 Value *PosLow =
1723 CGF.Builder.CreateAnd(LHS: CGF.Builder.CreateTrunc(V: BitPos, DestTy: CGF.Int8Ty),
1724 RHS: llvm::ConstantInt::get(Ty: CGF.Int8Ty, V: 0x7));
1725
1726 // The updating instructions will need a mask.
1727 Value *Mask = nullptr;
1728 if (BT.Action != BitTest::TestOnly) {
1729 Mask = CGF.Builder.CreateShl(LHS: llvm::ConstantInt::get(Ty: CGF.Int8Ty, V: 1), RHS: PosLow,
1730 Name: "bittest.mask");
1731 }
1732
1733 // Check the action and ordering of the interlocked intrinsics.
1734 llvm::AtomicOrdering Ordering = getBitTestAtomicOrdering(I: BT.Interlocking);
1735
1736 Value *OldByte = nullptr;
1737 if (Ordering != llvm::AtomicOrdering::NotAtomic) {
1738 // Emit a combined atomicrmw load/store operation for the interlocked
1739 // intrinsics.
1740 llvm::AtomicRMWInst::BinOp RMWOp = llvm::AtomicRMWInst::Or;
1741 if (BT.Action == BitTest::Reset) {
1742 Mask = CGF.Builder.CreateNot(V: Mask);
1743 RMWOp = llvm::AtomicRMWInst::And;
1744 }
1745 OldByte = CGF.Builder.CreateAtomicRMW(Op: RMWOp, Addr: ByteAddr, Val: Mask, Ordering);
1746 } else {
1747 // Emit a plain load for the non-interlocked intrinsics.
1748 OldByte = CGF.Builder.CreateLoad(Addr: ByteAddr, Name: "bittest.byte");
1749 Value *NewByte = nullptr;
1750 switch (BT.Action) {
1751 case BitTest::TestOnly:
1752 // Don't store anything.
1753 break;
1754 case BitTest::Complement:
1755 NewByte = CGF.Builder.CreateXor(LHS: OldByte, RHS: Mask);
1756 break;
1757 case BitTest::Reset:
1758 NewByte = CGF.Builder.CreateAnd(LHS: OldByte, RHS: CGF.Builder.CreateNot(V: Mask));
1759 break;
1760 case BitTest::Set:
1761 NewByte = CGF.Builder.CreateOr(LHS: OldByte, RHS: Mask);
1762 break;
1763 }
1764 if (NewByte)
1765 CGF.Builder.CreateStore(Val: NewByte, Addr: ByteAddr);
1766 }
1767
1768 // However we loaded the old byte, either by plain load or atomicrmw, shift
1769 // the bit into the low position and mask it to 0 or 1.
1770 Value *ShiftedByte = CGF.Builder.CreateLShr(LHS: OldByte, RHS: PosLow, Name: "bittest.shr");
1771 return CGF.Builder.CreateAnd(
1772 LHS: ShiftedByte, RHS: llvm::ConstantInt::get(Ty: CGF.Int8Ty, V: 1), Name: "bittest.res");
1773}
1774
1775namespace {
1776enum class MSVCSetJmpKind {
1777 _setjmpex,
1778 _setjmp3,
1779 _setjmp
1780};
1781}
1782
1783/// MSVC handles setjmp a bit differently on different platforms. On every
1784/// architecture except 32-bit x86, the frame address is passed. On x86, extra
1785/// parameters can be passed as variadic arguments, but we always pass none.
1786static RValue EmitMSVCRTSetJmp(CodeGenFunction &CGF, MSVCSetJmpKind SJKind,
1787 const CallExpr *E) {
1788 llvm::Value *Arg1 = nullptr;
1789 llvm::Type *Arg1Ty = nullptr;
1790 StringRef Name;
1791 bool IsVarArg = false;
1792 if (SJKind == MSVCSetJmpKind::_setjmp3) {
1793 Name = "_setjmp3";
1794 Arg1Ty = CGF.Int32Ty;
1795 Arg1 = llvm::ConstantInt::get(Ty: CGF.IntTy, V: 0);
1796 IsVarArg = true;
1797 } else {
1798 Name = SJKind == MSVCSetJmpKind::_setjmp ? "_setjmp" : "_setjmpex";
1799 Arg1Ty = CGF.Int8PtrTy;
1800 if (CGF.getTarget().getTriple().getArch() == llvm::Triple::aarch64) {
1801 Arg1 = CGF.Builder.CreateCall(
1802 Callee: CGF.CGM.getIntrinsic(IID: Intrinsic::sponentry, Tys: CGF.AllocaInt8PtrTy));
1803 } else
1804 Arg1 = CGF.Builder.CreateCall(
1805 Callee: CGF.CGM.getIntrinsic(IID: Intrinsic::frameaddress, Tys: CGF.AllocaInt8PtrTy),
1806 Args: llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: 0));
1807 }
1808
1809 // Mark the call site and declaration with ReturnsTwice.
1810 llvm::Type *ArgTypes[2] = {CGF.Int8PtrTy, Arg1Ty};
1811 llvm::AttributeList ReturnsTwiceAttr = llvm::AttributeList::get(
1812 C&: CGF.getLLVMContext(), Index: llvm::AttributeList::FunctionIndex,
1813 Kinds: llvm::Attribute::ReturnsTwice);
1814 llvm::FunctionCallee SetJmpFn = CGF.CGM.CreateRuntimeFunction(
1815 Ty: llvm::FunctionType::get(Result: CGF.IntTy, Params: ArgTypes, isVarArg: IsVarArg), Name,
1816 ExtraAttrs: ReturnsTwiceAttr, /*Local=*/true);
1817
1818 llvm::Value *Buf = CGF.Builder.CreateBitOrPointerCast(
1819 V: CGF.EmitScalarExpr(E: E->getArg(Arg: 0)), DestTy: CGF.Int8PtrTy);
1820 llvm::Value *Args[] = {Buf, Arg1};
1821 llvm::CallBase *CB = CGF.EmitRuntimeCallOrInvoke(callee: SetJmpFn, args: Args);
1822 CB->setAttributes(ReturnsTwiceAttr);
1823 return RValue::get(V: CB);
1824}
1825
1826// Emit an MSVC intrinsic. Assumes that arguments have *not* been evaluated.
1827Value *CodeGenFunction::EmitMSVCBuiltinExpr(MSVCIntrin BuiltinID,
1828 const CallExpr *E) {
1829 switch (BuiltinID) {
1830 case MSVCIntrin::_BitScanForward:
1831 case MSVCIntrin::_BitScanReverse: {
1832 Address IndexAddress(EmitPointerWithAlignment(Addr: E->getArg(Arg: 0)));
1833 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 1));
1834
1835 llvm::Type *ArgType = ArgValue->getType();
1836 llvm::Type *IndexType = IndexAddress.getElementType();
1837 llvm::Type *ResultType = ConvertType(T: E->getType());
1838
1839 Value *ArgZero = llvm::Constant::getNullValue(Ty: ArgType);
1840 Value *ResZero = llvm::Constant::getNullValue(Ty: ResultType);
1841 Value *ResOne = llvm::ConstantInt::get(Ty: ResultType, V: 1);
1842
1843 BasicBlock *Begin = Builder.GetInsertBlock();
1844 BasicBlock *End = createBasicBlock(name: "bitscan_end", parent: this->CurFn);
1845 Builder.SetInsertPoint(End);
1846 PHINode *Result = Builder.CreatePHI(Ty: ResultType, NumReservedValues: 2, Name: "bitscan_result");
1847
1848 Builder.SetInsertPoint(Begin);
1849 Value *IsZero = Builder.CreateICmpEQ(LHS: ArgValue, RHS: ArgZero);
1850 BasicBlock *NotZero = createBasicBlock(name: "bitscan_not_zero", parent: this->CurFn);
1851 Builder.CreateCondBr(Cond: IsZero, True: End, False: NotZero);
1852 Result->addIncoming(V: ResZero, BB: Begin);
1853
1854 Builder.SetInsertPoint(NotZero);
1855
1856 if (BuiltinID == MSVCIntrin::_BitScanForward) {
1857 Function *F = CGM.getIntrinsic(IID: Intrinsic::cttz, Tys: ArgType);
1858 Value *ZeroCount = Builder.CreateCall(Callee: F, Args: {ArgValue, Builder.getTrue()});
1859 ZeroCount = Builder.CreateIntCast(V: ZeroCount, DestTy: IndexType, isSigned: false);
1860 Builder.CreateStore(Val: ZeroCount, Addr: IndexAddress, IsVolatile: false);
1861 } else {
1862 unsigned ArgWidth = cast<llvm::IntegerType>(Val: ArgType)->getBitWidth();
1863 Value *ArgTypeLastIndex = llvm::ConstantInt::get(Ty: IndexType, V: ArgWidth - 1);
1864
1865 Function *F = CGM.getIntrinsic(IID: Intrinsic::ctlz, Tys: ArgType);
1866 Value *ZeroCount = Builder.CreateCall(Callee: F, Args: {ArgValue, Builder.getTrue()});
1867 ZeroCount = Builder.CreateIntCast(V: ZeroCount, DestTy: IndexType, isSigned: false);
1868 Value *Index = Builder.CreateNSWSub(LHS: ArgTypeLastIndex, RHS: ZeroCount);
1869 Builder.CreateStore(Val: Index, Addr: IndexAddress, IsVolatile: false);
1870 }
1871 Builder.CreateBr(Dest: End);
1872 Result->addIncoming(V: ResOne, BB: NotZero);
1873
1874 Builder.SetInsertPoint(End);
1875 return Result;
1876 }
1877 case MSVCIntrin::_InterlockedAnd:
1878 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::And, E);
1879 case MSVCIntrin::_InterlockedExchange:
1880 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Xchg, E);
1881 case MSVCIntrin::_InterlockedExchangeAdd:
1882 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Add, E);
1883 case MSVCIntrin::_InterlockedExchangeSub:
1884 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Sub, E);
1885 case MSVCIntrin::_InterlockedOr:
1886 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Or, E);
1887 case MSVCIntrin::_InterlockedXor:
1888 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Xor, E);
1889 case MSVCIntrin::_InterlockedExchangeAdd_acq:
1890 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Add, E,
1891 Ordering: AtomicOrdering::Acquire);
1892 case MSVCIntrin::_InterlockedExchangeAdd_rel:
1893 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Add, E,
1894 Ordering: AtomicOrdering::Release);
1895 case MSVCIntrin::_InterlockedExchangeAdd_nf:
1896 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Add, E,
1897 Ordering: AtomicOrdering::Monotonic);
1898 case MSVCIntrin::_InterlockedExchange_acq:
1899 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Xchg, E,
1900 Ordering: AtomicOrdering::Acquire);
1901 case MSVCIntrin::_InterlockedExchange_rel:
1902 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Xchg, E,
1903 Ordering: AtomicOrdering::Release);
1904 case MSVCIntrin::_InterlockedExchange_nf:
1905 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Xchg, E,
1906 Ordering: AtomicOrdering::Monotonic);
1907 case MSVCIntrin::_InterlockedCompareExchange:
1908 return EmitAtomicCmpXchgForMSIntrin(CGF&: *this, E);
1909 case MSVCIntrin::_InterlockedCompareExchange_acq:
1910 return EmitAtomicCmpXchgForMSIntrin(CGF&: *this, E, SuccessOrdering: AtomicOrdering::Acquire);
1911 case MSVCIntrin::_InterlockedCompareExchange_rel:
1912 return EmitAtomicCmpXchgForMSIntrin(CGF&: *this, E, SuccessOrdering: AtomicOrdering::Release);
1913 case MSVCIntrin::_InterlockedCompareExchange_nf:
1914 return EmitAtomicCmpXchgForMSIntrin(CGF&: *this, E, SuccessOrdering: AtomicOrdering::Monotonic);
1915 case MSVCIntrin::_InterlockedCompareExchange128:
1916 return EmitAtomicCmpXchg128ForMSIntrin(
1917 CGF&: *this, E, SuccessOrdering: AtomicOrdering::SequentiallyConsistent);
1918 case MSVCIntrin::_InterlockedCompareExchange128_acq:
1919 return EmitAtomicCmpXchg128ForMSIntrin(CGF&: *this, E, SuccessOrdering: AtomicOrdering::Acquire);
1920 case MSVCIntrin::_InterlockedCompareExchange128_rel:
1921 return EmitAtomicCmpXchg128ForMSIntrin(CGF&: *this, E, SuccessOrdering: AtomicOrdering::Release);
1922 case MSVCIntrin::_InterlockedCompareExchange128_nf:
1923 return EmitAtomicCmpXchg128ForMSIntrin(CGF&: *this, E, SuccessOrdering: AtomicOrdering::Monotonic);
1924 case MSVCIntrin::_InterlockedOr_acq:
1925 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Or, E,
1926 Ordering: AtomicOrdering::Acquire);
1927 case MSVCIntrin::_InterlockedOr_rel:
1928 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Or, E,
1929 Ordering: AtomicOrdering::Release);
1930 case MSVCIntrin::_InterlockedOr_nf:
1931 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Or, E,
1932 Ordering: AtomicOrdering::Monotonic);
1933 case MSVCIntrin::_InterlockedXor_acq:
1934 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Xor, E,
1935 Ordering: AtomicOrdering::Acquire);
1936 case MSVCIntrin::_InterlockedXor_rel:
1937 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Xor, E,
1938 Ordering: AtomicOrdering::Release);
1939 case MSVCIntrin::_InterlockedXor_nf:
1940 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Xor, E,
1941 Ordering: AtomicOrdering::Monotonic);
1942 case MSVCIntrin::_InterlockedAnd_acq:
1943 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::And, E,
1944 Ordering: AtomicOrdering::Acquire);
1945 case MSVCIntrin::_InterlockedAnd_rel:
1946 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::And, E,
1947 Ordering: AtomicOrdering::Release);
1948 case MSVCIntrin::_InterlockedAnd_nf:
1949 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::And, E,
1950 Ordering: AtomicOrdering::Monotonic);
1951 case MSVCIntrin::_InterlockedIncrement_acq:
1952 return EmitAtomicIncrementValue(CGF&: *this, E, Ordering: AtomicOrdering::Acquire);
1953 case MSVCIntrin::_InterlockedIncrement_rel:
1954 return EmitAtomicIncrementValue(CGF&: *this, E, Ordering: AtomicOrdering::Release);
1955 case MSVCIntrin::_InterlockedIncrement_nf:
1956 return EmitAtomicIncrementValue(CGF&: *this, E, Ordering: AtomicOrdering::Monotonic);
1957 case MSVCIntrin::_InterlockedDecrement_acq:
1958 return EmitAtomicDecrementValue(CGF&: *this, E, Ordering: AtomicOrdering::Acquire);
1959 case MSVCIntrin::_InterlockedDecrement_rel:
1960 return EmitAtomicDecrementValue(CGF&: *this, E, Ordering: AtomicOrdering::Release);
1961 case MSVCIntrin::_InterlockedDecrement_nf:
1962 return EmitAtomicDecrementValue(CGF&: *this, E, Ordering: AtomicOrdering::Monotonic);
1963
1964 case MSVCIntrin::_InterlockedDecrement:
1965 return EmitAtomicDecrementValue(CGF&: *this, E);
1966 case MSVCIntrin::_InterlockedIncrement:
1967 return EmitAtomicIncrementValue(CGF&: *this, E);
1968
1969 case MSVCIntrin::__fastfail: {
1970 // Request immediate process termination from the kernel. The instruction
1971 // sequences to do this are documented on MSDN:
1972 // https://msdn.microsoft.com/en-us/library/dn774154.aspx
1973 llvm::Triple::ArchType ISA = getTarget().getTriple().getArch();
1974 StringRef Asm, Constraints;
1975 switch (ISA) {
1976 default:
1977 ErrorUnsupported(S: E, Type: "__fastfail call for this architecture");
1978 break;
1979 case llvm::Triple::x86:
1980 case llvm::Triple::x86_64:
1981 Asm = "int $$0x29";
1982 Constraints = "{cx}";
1983 break;
1984 case llvm::Triple::thumb:
1985 Asm = "udf #251";
1986 Constraints = "{r0}";
1987 break;
1988 case llvm::Triple::aarch64:
1989 Asm = "brk #0xF003";
1990 Constraints = "{w0}";
1991 }
1992 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: VoidTy, Params: {Int32Ty}, isVarArg: false);
1993 llvm::InlineAsm *IA =
1994 llvm::InlineAsm::get(Ty: FTy, AsmString: Asm, Constraints, /*hasSideEffects=*/true);
1995 llvm::AttributeList NoReturnAttr = llvm::AttributeList::get(
1996 C&: getLLVMContext(), Index: llvm::AttributeList::FunctionIndex,
1997 Kinds: llvm::Attribute::NoReturn);
1998 llvm::CallInst *CI = Builder.CreateCall(Callee: IA, Args: EmitScalarExpr(E: E->getArg(Arg: 0)));
1999 CI->setAttributes(NoReturnAttr);
2000 return CI;
2001 }
2002 }
2003 llvm_unreachable("Incorrect MSVC intrinsic!");
2004}
2005
2006namespace {
2007// ARC cleanup for __builtin_os_log_format
2008struct CallObjCArcUse final : EHScopeStack::Cleanup {
2009 CallObjCArcUse(llvm::Value *object) : object(object) {}
2010 llvm::Value *object;
2011
2012 void Emit(CodeGenFunction &CGF, Flags flags) override {
2013 CGF.EmitARCIntrinsicUse(values: object);
2014 }
2015};
2016}
2017
2018Value *CodeGenFunction::EmitCheckedArgForBuiltin(const Expr *E,
2019 BuiltinCheckKind Kind) {
2020 assert((Kind == BCK_CLZPassedZero || Kind == BCK_CTZPassedZero) &&
2021 "Unsupported builtin check kind");
2022
2023 Value *ArgValue = EmitBitCountExpr(CGF&: *this, E);
2024 if (!SanOpts.has(K: SanitizerKind::Builtin))
2025 return ArgValue;
2026
2027 auto CheckOrdinal = SanitizerKind::SO_Builtin;
2028 auto CheckHandler = SanitizerHandler::InvalidBuiltin;
2029 SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);
2030 Value *Cond = Builder.CreateICmpNE(
2031 LHS: ArgValue, RHS: llvm::Constant::getNullValue(Ty: ArgValue->getType()));
2032 EmitCheck(Checked: std::make_pair(x&: Cond, y&: CheckOrdinal), Check: CheckHandler,
2033 StaticArgs: {EmitCheckSourceLocation(Loc: E->getExprLoc()),
2034 llvm::ConstantInt::get(Ty: Builder.getInt8Ty(), V: Kind)},
2035 DynamicArgs: {});
2036 return ArgValue;
2037}
2038
2039Value *CodeGenFunction::EmitCheckedArgForAssume(const Expr *E) {
2040 Value *ArgValue = EvaluateExprAsBool(E);
2041 if (!SanOpts.has(K: SanitizerKind::Builtin))
2042 return ArgValue;
2043
2044 auto CheckOrdinal = SanitizerKind::SO_Builtin;
2045 auto CheckHandler = SanitizerHandler::InvalidBuiltin;
2046 SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);
2047 EmitCheck(
2048 Checked: std::make_pair(x&: ArgValue, y&: CheckOrdinal), Check: CheckHandler,
2049 StaticArgs: {EmitCheckSourceLocation(Loc: E->getExprLoc()),
2050 llvm::ConstantInt::get(Ty: Builder.getInt8Ty(), V: BCK_AssumePassedFalse)},
2051 DynamicArgs: {});
2052 return ArgValue;
2053}
2054
2055static Value *EmitAbs(CodeGenFunction &CGF, Value *ArgValue, bool HasNSW) {
2056 return CGF.Builder.CreateBinaryIntrinsic(
2057 ID: Intrinsic::abs, LHS: ArgValue,
2058 RHS: ConstantInt::get(Ty: CGF.Builder.getInt1Ty(), V: HasNSW));
2059}
2060
2061static Value *EmitOverflowCheckedAbs(CodeGenFunction &CGF, const CallExpr *E,
2062 bool SanitizeOverflow) {
2063 Value *ArgValue = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
2064
2065 // Try to eliminate overflow check.
2066 if (const auto *VCI = dyn_cast<llvm::ConstantInt>(Val: ArgValue)) {
2067 if (!VCI->isMinSignedValue())
2068 return EmitAbs(CGF, ArgValue, HasNSW: true);
2069 }
2070
2071 SmallVector<SanitizerKind::SanitizerOrdinal, 1> Ordinals;
2072 SanitizerHandler CheckHandler;
2073 if (SanitizeOverflow) {
2074 Ordinals.push_back(Elt: SanitizerKind::SO_SignedIntegerOverflow);
2075 CheckHandler = SanitizerHandler::NegateOverflow;
2076 } else
2077 CheckHandler = SanitizerHandler::SubOverflow;
2078
2079 SanitizerDebugLocation SanScope(&CGF, Ordinals, CheckHandler);
2080
2081 Constant *Zero = Constant::getNullValue(Ty: ArgValue->getType());
2082 Value *ResultAndOverflow = CGF.Builder.CreateBinaryIntrinsic(
2083 ID: Intrinsic::ssub_with_overflow, LHS: Zero, RHS: ArgValue);
2084 Value *Result = CGF.Builder.CreateExtractValue(Agg: ResultAndOverflow, Idxs: 0);
2085 Value *NotOverflow = CGF.Builder.CreateNot(
2086 V: CGF.Builder.CreateExtractValue(Agg: ResultAndOverflow, Idxs: 1));
2087
2088 // TODO: support -ftrapv-handler.
2089 if (SanitizeOverflow) {
2090 CGF.EmitCheck(Checked: {{NotOverflow, SanitizerKind::SO_SignedIntegerOverflow}},
2091 Check: CheckHandler,
2092 StaticArgs: {CGF.EmitCheckSourceLocation(Loc: E->getArg(Arg: 0)->getExprLoc()),
2093 CGF.EmitCheckTypeDescriptor(T: E->getType())},
2094 DynamicArgs: {ArgValue});
2095 } else
2096 CGF.EmitTrapCheck(Checked: NotOverflow, CheckHandlerID: CheckHandler);
2097
2098 Value *CmpResult = CGF.Builder.CreateICmpSLT(LHS: ArgValue, RHS: Zero, Name: "abscond");
2099 return CGF.Builder.CreateSelect(C: CmpResult, True: Result, False: ArgValue, Name: "abs");
2100}
2101
2102/// Get the argument type for arguments to os_log_helper.
2103static CanQualType getOSLogArgType(ASTContext &C, int Size) {
2104 QualType UnsignedTy = C.getIntTypeForBitwidth(DestWidth: Size * 8, /*Signed=*/false);
2105 return C.getCanonicalType(T: UnsignedTy);
2106}
2107
2108llvm::Function *CodeGenFunction::generateBuiltinOSLogHelperFunction(
2109 const analyze_os_log::OSLogBufferLayout &Layout,
2110 CharUnits BufferAlignment) {
2111 ASTContext &Ctx = getContext();
2112
2113 llvm::SmallString<64> Name;
2114 {
2115 raw_svector_ostream OS(Name);
2116 OS << "__os_log_helper";
2117 OS << "_" << BufferAlignment.getQuantity();
2118 OS << "_" << int(Layout.getSummaryByte());
2119 OS << "_" << int(Layout.getNumArgsByte());
2120 for (const auto &Item : Layout.Items)
2121 OS << "_" << int(Item.getSizeByte()) << "_"
2122 << int(Item.getDescriptorByte());
2123 }
2124
2125 if (llvm::Function *F = CGM.getModule().getFunction(Name))
2126 return F;
2127
2128 llvm::SmallVector<QualType, 4> ArgTys;
2129 FunctionArgList Args;
2130 Args.push_back(Elt: ImplicitParamDecl::Create(
2131 C&: Ctx, DC: nullptr, IdLoc: SourceLocation(), Id: &Ctx.Idents.get(Name: "buffer"), T: Ctx.VoidPtrTy,
2132 ParamKind: ImplicitParamKind::Other));
2133 ArgTys.emplace_back(Args&: Ctx.VoidPtrTy);
2134
2135 for (unsigned int I = 0, E = Layout.Items.size(); I < E; ++I) {
2136 char Size = Layout.Items[I].getSizeByte();
2137 if (!Size)
2138 continue;
2139
2140 QualType ArgTy = getOSLogArgType(C&: Ctx, Size);
2141 Args.push_back(Elt: ImplicitParamDecl::Create(
2142 C&: Ctx, DC: nullptr, IdLoc: SourceLocation(),
2143 Id: &Ctx.Idents.get(Name: std::string("arg") + llvm::to_string(Value: I)), T: ArgTy,
2144 ParamKind: ImplicitParamKind::Other));
2145 ArgTys.emplace_back(Args&: ArgTy);
2146 }
2147
2148 QualType ReturnTy = Ctx.VoidTy;
2149
2150 // The helper function has linkonce_odr linkage to enable the linker to merge
2151 // identical functions. To ensure the merging always happens, 'noinline' is
2152 // attached to the function when compiling with -Oz.
2153 const CGFunctionInfo &FI =
2154 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: ReturnTy, args: Args);
2155 llvm::FunctionType *FuncTy = CGM.getTypes().GetFunctionType(Info: FI);
2156 llvm::Function *Fn = llvm::Function::Create(
2157 Ty: FuncTy, Linkage: llvm::GlobalValue::LinkOnceODRLinkage, N: Name, M: &CGM.getModule());
2158 Fn->setVisibility(llvm::GlobalValue::HiddenVisibility);
2159 CGM.SetLLVMFunctionAttributes(GD: GlobalDecl(), Info: FI, F: Fn, /*IsThunk=*/false);
2160 CGM.SetLLVMFunctionAttributesForDefinition(D: nullptr, F: Fn);
2161 Fn->setDoesNotThrow();
2162
2163 // Attach 'noinline' at -Oz.
2164 if (CGM.getCodeGenOpts().OptimizeSize == 2)
2165 Fn->addFnAttr(Kind: llvm::Attribute::NoInline);
2166
2167 auto NL = ApplyDebugLocation::CreateEmpty(CGF&: *this);
2168 StartFunction(GD: GlobalDecl(), RetTy: ReturnTy, Fn, FnInfo: FI, Args);
2169
2170 // Create a scope with an artificial location for the body of this function.
2171 auto AL = ApplyDebugLocation::CreateArtificial(CGF&: *this);
2172
2173 CharUnits Offset;
2174 Address BufAddr = makeNaturalAddressForPointer(
2175 Ptr: Builder.CreateLoad(Addr: GetAddrOfLocalVar(VD: Args[0]), Name: "buf"), T: Ctx.VoidTy,
2176 Alignment: BufferAlignment);
2177 Builder.CreateStore(Val: Builder.getInt8(C: Layout.getSummaryByte()),
2178 Addr: Builder.CreateConstByteGEP(Addr: BufAddr, Offset: Offset++, Name: "summary"));
2179 Builder.CreateStore(Val: Builder.getInt8(C: Layout.getNumArgsByte()),
2180 Addr: Builder.CreateConstByteGEP(Addr: BufAddr, Offset: Offset++, Name: "numArgs"));
2181
2182 unsigned I = 1;
2183 for (const auto &Item : Layout.Items) {
2184 Builder.CreateStore(
2185 Val: Builder.getInt8(C: Item.getDescriptorByte()),
2186 Addr: Builder.CreateConstByteGEP(Addr: BufAddr, Offset: Offset++, Name: "argDescriptor"));
2187 Builder.CreateStore(
2188 Val: Builder.getInt8(C: Item.getSizeByte()),
2189 Addr: Builder.CreateConstByteGEP(Addr: BufAddr, Offset: Offset++, Name: "argSize"));
2190
2191 CharUnits Size = Item.size();
2192 if (!Size.getQuantity())
2193 continue;
2194
2195 Address Arg = GetAddrOfLocalVar(VD: Args[I]);
2196 Address Addr = Builder.CreateConstByteGEP(Addr: BufAddr, Offset, Name: "argData");
2197 Addr = Addr.withElementType(ElemTy: Arg.getElementType());
2198 Builder.CreateStore(Val: Builder.CreateLoad(Addr: Arg), Addr);
2199 Offset += Size;
2200 ++I;
2201 }
2202
2203 FinishFunction();
2204
2205 return Fn;
2206}
2207
2208RValue CodeGenFunction::emitBuiltinOSLogFormat(const CallExpr &E) {
2209 assert(E.getNumArgs() >= 2 &&
2210 "__builtin_os_log_format takes at least 2 arguments");
2211 ASTContext &Ctx = getContext();
2212 analyze_os_log::OSLogBufferLayout Layout;
2213 analyze_os_log::computeOSLogBufferLayout(Ctx, E: &E, layout&: Layout);
2214 Address BufAddr = EmitPointerWithAlignment(Addr: E.getArg(Arg: 0));
2215
2216 // Ignore argument 1, the format string. It is not currently used.
2217 CallArgList Args;
2218 Args.add(rvalue: RValue::get(V: BufAddr.emitRawPointer(CGF&: *this)), type: Ctx.VoidPtrTy);
2219
2220 for (const auto &Item : Layout.Items) {
2221 int Size = Item.getSizeByte();
2222 if (!Size)
2223 continue;
2224
2225 llvm::Value *ArgVal;
2226
2227 if (Item.getKind() == analyze_os_log::OSLogBufferItem::MaskKind) {
2228 uint64_t Val = 0;
2229 for (unsigned I = 0, E = Item.getMaskType().size(); I < E; ++I)
2230 Val |= ((uint64_t)Item.getMaskType()[I]) << I * 8;
2231 ArgVal = llvm::Constant::getIntegerValue(Ty: Int64Ty, V: llvm::APInt(64, Val));
2232 } else if (const Expr *TheExpr = Item.getExpr()) {
2233 ArgVal = EmitScalarExpr(E: TheExpr, /*Ignore*/ IgnoreResultAssign: false);
2234
2235 // If a temporary object that requires destruction after the full
2236 // expression is passed, push a lifetime-extended cleanup to extend its
2237 // lifetime to the end of the enclosing block scope.
2238 auto LifetimeExtendObject = [&](const Expr *E) {
2239 E = E->IgnoreParenCasts();
2240 // Extend lifetimes of objects returned by function calls and message
2241 // sends.
2242
2243 // FIXME: We should do this in other cases in which temporaries are
2244 // created including arguments of non-ARC types (e.g., C++
2245 // temporaries).
2246 if (isa<CallExpr>(Val: E) || isa<ObjCMessageExpr>(Val: E))
2247 return true;
2248 return false;
2249 };
2250
2251 if (TheExpr->getType()->isObjCRetainableType() &&
2252 getLangOpts().ObjCAutoRefCount && LifetimeExtendObject(TheExpr)) {
2253 assert(getEvaluationKind(TheExpr->getType()) == TEK_Scalar &&
2254 "Only scalar can be a ObjC retainable type");
2255 if (!isa<Constant>(Val: ArgVal)) {
2256 CleanupKind Cleanup = getARCCleanupKind();
2257 QualType Ty = TheExpr->getType();
2258 RawAddress Alloca = CreateMemTempWithoutCast(T: Ty, Name: "os.log.arg");
2259 ArgVal = EmitARCRetain(type: Ty, value: ArgVal);
2260 Builder.CreateStore(Val: ArgVal, Addr: Alloca);
2261 pushLifetimeExtendedDestroy(kind: Cleanup, addr: Alloca, type: Ty,
2262 destroyer: CodeGenFunction::destroyARCStrongPrecise,
2263 useEHCleanupForArray: Cleanup & EHCleanup);
2264
2265 // Push a clang.arc.use call to ensure ARC optimizer knows that the
2266 // argument has to be alive.
2267 if (CGM.getCodeGenOpts().OptimizationLevel != 0)
2268 pushCleanupAfterFullExpr<CallObjCArcUse>(Kind: Cleanup, A: ArgVal);
2269 }
2270 }
2271 } else {
2272 ArgVal = Builder.getInt32(C: Item.getConstValue().getQuantity());
2273 }
2274
2275 unsigned ArgValSize =
2276 CGM.getDataLayout().getTypeSizeInBits(Ty: ArgVal->getType());
2277 llvm::IntegerType *IntTy = llvm::Type::getIntNTy(C&: getLLVMContext(),
2278 N: ArgValSize);
2279 ArgVal = Builder.CreateBitOrPointerCast(V: ArgVal, DestTy: IntTy);
2280 CanQualType ArgTy = getOSLogArgType(C&: Ctx, Size);
2281 // If ArgVal has type x86_fp80, zero-extend ArgVal.
2282 ArgVal = Builder.CreateZExtOrBitCast(V: ArgVal, DestTy: ConvertType(T: ArgTy));
2283 Args.add(rvalue: RValue::get(V: ArgVal), type: ArgTy);
2284 }
2285
2286 const CGFunctionInfo &FI =
2287 CGM.getTypes().arrangeBuiltinFunctionCall(resultType: Ctx.VoidTy, args: Args);
2288 llvm::Function *F = CodeGenFunction(CGM).generateBuiltinOSLogHelperFunction(
2289 Layout, BufferAlignment: BufAddr.getAlignment());
2290 EmitCall(CallInfo: FI, Callee: CGCallee::forDirect(functionPtr: F), ReturnValue: ReturnValueSlot(), Args);
2291 return RValue::get(Addr: BufAddr, CGF&: *this);
2292}
2293
2294static bool isSpecialUnsignedMultiplySignedResult(
2295 unsigned BuiltinID, WidthAndSignedness Op1Info, WidthAndSignedness Op2Info,
2296 WidthAndSignedness ResultInfo) {
2297 return BuiltinID == Builtin::BI__builtin_mul_overflow &&
2298 Op1Info.Width == Op2Info.Width && Op2Info.Width == ResultInfo.Width &&
2299 !Op1Info.Signed && !Op2Info.Signed && ResultInfo.Signed;
2300}
2301
2302static RValue EmitCheckedUnsignedMultiplySignedResult(
2303 CodeGenFunction &CGF, const clang::Expr *Op1, WidthAndSignedness Op1Info,
2304 const clang::Expr *Op2, WidthAndSignedness Op2Info,
2305 const clang::Expr *ResultArg, QualType ResultQTy,
2306 WidthAndSignedness ResultInfo) {
2307 assert(isSpecialUnsignedMultiplySignedResult(
2308 Builtin::BI__builtin_mul_overflow, Op1Info, Op2Info, ResultInfo) &&
2309 "Cannot specialize this multiply");
2310
2311 llvm::Value *V1 = CGF.EmitScalarExpr(E: Op1);
2312 llvm::Value *V2 = CGF.EmitScalarExpr(E: Op2);
2313
2314 llvm::Value *HasOverflow;
2315 llvm::Value *Result = EmitOverflowIntrinsic(
2316 CGF, IntrinsicID: Intrinsic::umul_with_overflow, X: V1, Y: V2, Carry&: HasOverflow);
2317
2318 // The intrinsic call will detect overflow when the value is > UINT_MAX,
2319 // however, since the original builtin had a signed result, we need to report
2320 // an overflow when the result is greater than INT_MAX.
2321 auto IntMax = llvm::APInt::getSignedMaxValue(numBits: ResultInfo.Width);
2322 llvm::Value *IntMaxValue = llvm::ConstantInt::get(Ty: Result->getType(), V: IntMax);
2323
2324 llvm::Value *IntMaxOverflow = CGF.Builder.CreateICmpUGT(LHS: Result, RHS: IntMaxValue);
2325 HasOverflow = CGF.Builder.CreateOr(LHS: HasOverflow, RHS: IntMaxOverflow);
2326
2327 bool isVolatile =
2328 ResultArg->getType()->getPointeeType().isVolatileQualified();
2329 Address ResultPtr = CGF.EmitPointerWithAlignment(Addr: ResultArg);
2330 CGF.Builder.CreateStore(Val: CGF.EmitToMemory(Value: Result, Ty: ResultQTy), Addr: ResultPtr,
2331 IsVolatile: isVolatile);
2332 return RValue::get(V: HasOverflow);
2333}
2334
2335/// Determine if a binop is a checked mixed-sign multiply we can specialize.
2336static bool isSpecialMixedSignMultiply(unsigned BuiltinID,
2337 WidthAndSignedness Op1Info,
2338 WidthAndSignedness Op2Info,
2339 WidthAndSignedness ResultInfo) {
2340 return BuiltinID == Builtin::BI__builtin_mul_overflow &&
2341 std::max(a: Op1Info.Width, b: Op2Info.Width) >= ResultInfo.Width &&
2342 Op1Info.Signed != Op2Info.Signed;
2343}
2344
2345/// Emit a checked mixed-sign multiply. This is a cheaper specialization of
2346/// the generic checked-binop irgen.
2347static RValue
2348EmitCheckedMixedSignMultiply(CodeGenFunction &CGF, const clang::Expr *Op1,
2349 WidthAndSignedness Op1Info, const clang::Expr *Op2,
2350 WidthAndSignedness Op2Info,
2351 const clang::Expr *ResultArg, QualType ResultQTy,
2352 WidthAndSignedness ResultInfo) {
2353 assert(isSpecialMixedSignMultiply(Builtin::BI__builtin_mul_overflow, Op1Info,
2354 Op2Info, ResultInfo) &&
2355 "Not a mixed-sign multipliction we can specialize");
2356
2357 // Emit the signed and unsigned operands.
2358 const clang::Expr *SignedOp = Op1Info.Signed ? Op1 : Op2;
2359 const clang::Expr *UnsignedOp = Op1Info.Signed ? Op2 : Op1;
2360 llvm::Value *Signed = CGF.EmitScalarExpr(E: SignedOp);
2361 llvm::Value *Unsigned = CGF.EmitScalarExpr(E: UnsignedOp);
2362 unsigned SignedOpWidth = Op1Info.Signed ? Op1Info.Width : Op2Info.Width;
2363 unsigned UnsignedOpWidth = Op1Info.Signed ? Op2Info.Width : Op1Info.Width;
2364
2365 // One of the operands may be smaller than the other. If so, [s|z]ext it.
2366 if (SignedOpWidth < UnsignedOpWidth)
2367 Signed = CGF.Builder.CreateSExt(V: Signed, DestTy: Unsigned->getType(), Name: "op.sext");
2368 if (UnsignedOpWidth < SignedOpWidth)
2369 Unsigned = CGF.Builder.CreateZExt(V: Unsigned, DestTy: Signed->getType(), Name: "op.zext");
2370
2371 llvm::Type *OpTy = Signed->getType();
2372 llvm::Value *Zero = llvm::Constant::getNullValue(Ty: OpTy);
2373 Address ResultPtr = CGF.EmitPointerWithAlignment(Addr: ResultArg);
2374 llvm::Type *ResTy = CGF.getTypes().ConvertType(T: ResultQTy);
2375 unsigned OpWidth = std::max(a: Op1Info.Width, b: Op2Info.Width);
2376
2377 // Take the absolute value of the signed operand.
2378 llvm::Value *IsNegative = CGF.Builder.CreateICmpSLT(LHS: Signed, RHS: Zero);
2379 llvm::Value *AbsOfNegative = CGF.Builder.CreateSub(LHS: Zero, RHS: Signed);
2380 llvm::Value *AbsSigned =
2381 CGF.Builder.CreateSelect(C: IsNegative, True: AbsOfNegative, False: Signed);
2382
2383 // Perform a checked unsigned multiplication.
2384 llvm::Value *UnsignedOverflow;
2385 llvm::Value *UnsignedResult =
2386 EmitOverflowIntrinsic(CGF, IntrinsicID: Intrinsic::umul_with_overflow, X: AbsSigned,
2387 Y: Unsigned, Carry&: UnsignedOverflow);
2388
2389 llvm::Value *Overflow, *Result;
2390 if (ResultInfo.Signed) {
2391 // Signed overflow occurs if the result is greater than INT_MAX or lesser
2392 // than INT_MIN, i.e when |Result| > (INT_MAX + IsNegative).
2393 auto IntMax =
2394 llvm::APInt::getSignedMaxValue(numBits: ResultInfo.Width).zext(width: OpWidth);
2395 llvm::Value *MaxResult =
2396 CGF.Builder.CreateAdd(LHS: llvm::ConstantInt::get(Ty: OpTy, V: IntMax),
2397 RHS: CGF.Builder.CreateZExt(V: IsNegative, DestTy: OpTy));
2398 llvm::Value *SignedOverflow =
2399 CGF.Builder.CreateICmpUGT(LHS: UnsignedResult, RHS: MaxResult);
2400 Overflow = CGF.Builder.CreateOr(LHS: UnsignedOverflow, RHS: SignedOverflow);
2401
2402 // Prepare the signed result (possibly by negating it).
2403 llvm::Value *NegativeResult = CGF.Builder.CreateNeg(V: UnsignedResult);
2404 llvm::Value *SignedResult =
2405 CGF.Builder.CreateSelect(C: IsNegative, True: NegativeResult, False: UnsignedResult);
2406 Result = CGF.Builder.CreateTrunc(V: SignedResult, DestTy: ResTy);
2407 } else {
2408 // Unsigned overflow occurs if the result is < 0 or greater than UINT_MAX.
2409 llvm::Value *Underflow = CGF.Builder.CreateAnd(
2410 LHS: IsNegative, RHS: CGF.Builder.CreateIsNotNull(Arg: UnsignedResult));
2411 Overflow = CGF.Builder.CreateOr(LHS: UnsignedOverflow, RHS: Underflow);
2412 if (ResultInfo.Width < OpWidth) {
2413 auto IntMax =
2414 llvm::APInt::getMaxValue(numBits: ResultInfo.Width).zext(width: OpWidth);
2415 llvm::Value *TruncOverflow = CGF.Builder.CreateICmpUGT(
2416 LHS: UnsignedResult, RHS: llvm::ConstantInt::get(Ty: OpTy, V: IntMax));
2417 Overflow = CGF.Builder.CreateOr(LHS: Overflow, RHS: TruncOverflow);
2418 }
2419
2420 // Negate the product if it would be negative in infinite precision.
2421 Result = CGF.Builder.CreateSelect(
2422 C: IsNegative, True: CGF.Builder.CreateNeg(V: UnsignedResult), False: UnsignedResult);
2423
2424 Result = CGF.Builder.CreateTrunc(V: Result, DestTy: ResTy);
2425 }
2426 assert(Overflow && Result && "Missing overflow or result");
2427
2428 bool isVolatile =
2429 ResultArg->getType()->getPointeeType().isVolatileQualified();
2430 CGF.Builder.CreateStore(Val: CGF.EmitToMemory(Value: Result, Ty: ResultQTy), Addr: ResultPtr,
2431 IsVolatile: isVolatile);
2432 return RValue::get(V: Overflow);
2433}
2434
2435/// Determine if the specified type requires laundering by checking if it is a
2436/// dynamic class type or contains a subobject which is a dynamic class type.
2437static bool TypeRequiresBuiltinLaunder(CodeGenModule &CGM, QualType Ty) {
2438 if (!CGM.getCodeGenOpts().StrictVTablePointers)
2439 return false;
2440 return Ty.requiresBuiltinLaunder(Context: CGM.getContext());
2441}
2442
2443RValue CodeGenFunction::emitRotate(const CallExpr *E, bool IsRotateRight) {
2444 llvm::Value *Src = EmitScalarExpr(E: E->getArg(Arg: 0));
2445 llvm::Value *ShiftAmt = EmitScalarExpr(E: E->getArg(Arg: 1));
2446
2447 // The builtin's shift arg may have a different type than the source arg and
2448 // result, but the LLVM intrinsic uses the same type for all values.
2449 llvm::Type *Ty = Src->getType();
2450 llvm::Type *ShiftTy = ShiftAmt->getType();
2451
2452 unsigned BitWidth = Ty->getIntegerBitWidth();
2453
2454 // Normalize shift amount to [0, BitWidth) range to match runtime behavior.
2455 // This matches the algorithm in ExprConstant.cpp for constant evaluation.
2456 if (BitWidth == 1) {
2457 // Rotating a 1-bit value is always a no-op
2458 ShiftAmt = ConstantInt::get(Ty: ShiftTy, V: 0);
2459 } else if (BitWidth == 2) {
2460 // For 2-bit values: rotation amount is 0 or 1 based on
2461 // whether the amount is even or odd. We can't use srem here because
2462 // the divisor (2) would be misinterpreted as -2 in 2-bit signed arithmetic.
2463 llvm::Value *One = ConstantInt::get(Ty: ShiftTy, V: 1);
2464 ShiftAmt = Builder.CreateAnd(LHS: ShiftAmt, RHS: One);
2465 } else {
2466 unsigned ShiftAmtBitWidth = ShiftTy->getIntegerBitWidth();
2467 bool ShiftAmtIsSigned = E->getArg(Arg: 1)->getType()->isSignedIntegerType();
2468
2469 // Choose the wider type for the divisor to avoid truncation
2470 llvm::Type *DivisorTy = ShiftAmtBitWidth > BitWidth ? ShiftTy : Ty;
2471 llvm::Value *Divisor = ConstantInt::get(Ty: DivisorTy, V: BitWidth);
2472
2473 // Extend ShiftAmt to match Divisor width if needed
2474 if (ShiftAmtBitWidth < DivisorTy->getIntegerBitWidth()) {
2475 ShiftAmt = Builder.CreateIntCast(V: ShiftAmt, DestTy: DivisorTy, isSigned: ShiftAmtIsSigned);
2476 }
2477
2478 // Normalize to [0, BitWidth)
2479 llvm::Value *RemResult;
2480 if (ShiftAmtIsSigned) {
2481 RemResult = Builder.CreateSRem(LHS: ShiftAmt, RHS: Divisor);
2482 // Signed remainder can be negative, convert to positive equivalent
2483 llvm::Value *Zero = ConstantInt::get(Ty: DivisorTy, V: 0);
2484 llvm::Value *IsNegative = Builder.CreateICmpSLT(LHS: RemResult, RHS: Zero);
2485 llvm::Value *PositiveShift = Builder.CreateAdd(LHS: RemResult, RHS: Divisor);
2486 ShiftAmt = Builder.CreateSelect(C: IsNegative, True: PositiveShift, False: RemResult);
2487 } else {
2488 ShiftAmt = Builder.CreateURem(LHS: ShiftAmt, RHS: Divisor);
2489 }
2490 }
2491
2492 // Convert to the source type if needed
2493 if (ShiftAmt->getType() != Ty) {
2494 ShiftAmt = Builder.CreateIntCast(V: ShiftAmt, DestTy: Ty, isSigned: false);
2495 }
2496
2497 // Rotate is a special case of LLVM funnel shift - 1st 2 args are the same.
2498 unsigned IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;
2499 Function *F = CGM.getIntrinsic(IID, Tys: Ty);
2500 return RValue::get(V: Builder.CreateCall(Callee: F, Args: {Src, Src, ShiftAmt}));
2501}
2502
2503// Map math builtins for long-double to f128 version.
2504static unsigned mutateLongDoubleBuiltin(unsigned BuiltinID) {
2505 switch (BuiltinID) {
2506#define MUTATE_LDBL(func) \
2507 case Builtin::BI__builtin_##func##l: \
2508 return Builtin::BI__builtin_##func##f128;
2509 MUTATE_LDBL(sqrt)
2510 MUTATE_LDBL(cbrt)
2511 MUTATE_LDBL(fabs)
2512 MUTATE_LDBL(log)
2513 MUTATE_LDBL(log2)
2514 MUTATE_LDBL(log10)
2515 MUTATE_LDBL(log1p)
2516 MUTATE_LDBL(logb)
2517 MUTATE_LDBL(exp)
2518 MUTATE_LDBL(exp2)
2519 MUTATE_LDBL(expm1)
2520 MUTATE_LDBL(fdim)
2521 MUTATE_LDBL(hypot)
2522 MUTATE_LDBL(ilogb)
2523 MUTATE_LDBL(pow)
2524 MUTATE_LDBL(fmin)
2525 MUTATE_LDBL(fmax)
2526 MUTATE_LDBL(ceil)
2527 MUTATE_LDBL(trunc)
2528 MUTATE_LDBL(rint)
2529 MUTATE_LDBL(nearbyint)
2530 MUTATE_LDBL(round)
2531 MUTATE_LDBL(floor)
2532 MUTATE_LDBL(lround)
2533 MUTATE_LDBL(llround)
2534 MUTATE_LDBL(lrint)
2535 MUTATE_LDBL(llrint)
2536 MUTATE_LDBL(fmod)
2537 MUTATE_LDBL(modf)
2538 MUTATE_LDBL(nan)
2539 MUTATE_LDBL(nans)
2540 MUTATE_LDBL(inf)
2541 MUTATE_LDBL(fma)
2542 MUTATE_LDBL(sin)
2543 MUTATE_LDBL(cos)
2544 MUTATE_LDBL(tan)
2545 MUTATE_LDBL(sinh)
2546 MUTATE_LDBL(cosh)
2547 MUTATE_LDBL(tanh)
2548 MUTATE_LDBL(asin)
2549 MUTATE_LDBL(acos)
2550 MUTATE_LDBL(atan)
2551 MUTATE_LDBL(asinh)
2552 MUTATE_LDBL(acosh)
2553 MUTATE_LDBL(atanh)
2554 MUTATE_LDBL(atan2)
2555 MUTATE_LDBL(erf)
2556 MUTATE_LDBL(erfc)
2557 MUTATE_LDBL(ldexp)
2558 MUTATE_LDBL(frexp)
2559 MUTATE_LDBL(huge_val)
2560 MUTATE_LDBL(copysign)
2561 MUTATE_LDBL(nextafter)
2562 MUTATE_LDBL(nexttoward)
2563 MUTATE_LDBL(remainder)
2564 MUTATE_LDBL(remquo)
2565 MUTATE_LDBL(scalbln)
2566 MUTATE_LDBL(scalbn)
2567 MUTATE_LDBL(tgamma)
2568 MUTATE_LDBL(lgamma)
2569#undef MUTATE_LDBL
2570 default:
2571 return BuiltinID;
2572 }
2573}
2574
2575static Value *tryUseTestFPKind(CodeGenFunction &CGF, unsigned BuiltinID,
2576 Value *V) {
2577 if (CGF.Builder.getIsFPConstrained() &&
2578 CGF.Builder.getDefaultConstrainedExcept() != fp::ebIgnore) {
2579 if (Value *Result =
2580 CGF.getTargetHooks().testFPKind(V, BuiltinID, Builder&: CGF.Builder, CGM&: CGF.CGM))
2581 return Result;
2582 }
2583 return nullptr;
2584}
2585
2586static RValue EmitHipStdParUnsupportedBuiltin(CodeGenFunction *CGF,
2587 const FunctionDecl *FD) {
2588 auto Name = FD->getNameAsString() + "__hipstdpar_unsupported";
2589 auto FnTy = CGF->CGM.getTypes().GetFunctionType(GD: FD);
2590 auto UBF = CGF->CGM.getModule().getOrInsertFunction(Name, T: FnTy);
2591
2592 SmallVector<Value *, 16> Args;
2593 for (auto &&FormalTy : FnTy->params())
2594 Args.push_back(Elt: llvm::PoisonValue::get(T: FormalTy));
2595
2596 return RValue::get(V: CGF->Builder.CreateCall(Callee: UBF, Args));
2597}
2598
2599// stdc_{leading,trailing}_{zeros,ones} and stdc_count_ones: counts bits using
2600// ctlz, cttz, or ctpop (IsPop). InvertArg flips the input to count the
2601// opposite bit value.
2602RValue CodeGenFunction::emitStdcCountIntrinsic(const CallExpr *E,
2603 Intrinsic::ID IntID,
2604 bool InvertArg, bool IsPop) {
2605 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
2606 llvm::Type *ArgType = ArgValue->getType();
2607 llvm::Type *ResultType = ConvertType(T: E->getType());
2608 Value *ActualArg = InvertArg ? Builder.CreateNot(V: ArgValue) : ArgValue;
2609 Function *F = CGM.getIntrinsic(IID: IntID, Tys: ArgType);
2610 Value *Result = IsPop
2611 ? Builder.CreateCall(Callee: F, Args: ActualArg)
2612 : Builder.CreateCall(Callee: F, Args: {ActualArg, Builder.getFalse()});
2613 if (Result->getType() != ResultType)
2614 Result = Builder.CreateIntCast(V: Result, DestTy: ResultType, isSigned: false);
2615 return RValue::get(V: Result);
2616}
2617
2618// stdc_count_zeros (BitWidth - ctpop) and stdc_bit_width (BitWidth - ctlz).
2619// IsPop selects ctpop; otherwise ctlz is used.
2620RValue CodeGenFunction::emitStdcBitWidthMinus(const CallExpr *E,
2621 Intrinsic::ID IntID, bool IsPop) {
2622 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
2623 llvm::Type *ArgType = ArgValue->getType();
2624 llvm::Type *ResultType = ConvertType(T: E->getType());
2625 unsigned BitWidth = ArgType->getIntegerBitWidth();
2626 Function *F = CGM.getIntrinsic(IID: IntID, Tys: ArgType);
2627 Value *Cnt = IsPop ? Builder.CreateCall(Callee: F, Args: ArgValue)
2628 : Builder.CreateCall(Callee: F, Args: {ArgValue, Builder.getFalse()});
2629 Value *Result = Builder.CreateSub(LHS: ConstantInt::get(Ty: ArgType, V: BitWidth), RHS: Cnt);
2630 if (Result->getType() != ResultType)
2631 Result = Builder.CreateIntCast(V: Result, DestTy: ResultType, isSigned: false);
2632 return RValue::get(V: Result);
2633}
2634
2635// stdc_first_{leading,trailing}_{zero,one}: returns the 1-based position of
2636// the first matching bit, or 0 if no such bit exists. InvertArg flips the
2637// input to search for zeros instead of ones.
2638RValue CodeGenFunction::emitStdcFirstBit(const CallExpr *E, Intrinsic::ID IntID,
2639 bool InvertArg) {
2640 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
2641 llvm::Type *ArgType = ArgValue->getType();
2642 llvm::Type *ResultType = ConvertType(T: E->getType());
2643 Value *Zero = ConstantInt::get(Ty: ArgType, V: 0);
2644 Value *One = ConstantInt::get(Ty: ArgType, V: 1);
2645 Value *ActualArg = InvertArg ? Builder.CreateNot(V: ArgValue) : ArgValue;
2646 Function *F = CGM.getIntrinsic(IID: IntID, Tys: ArgType);
2647 Value *Cnt = Builder.CreateCall(Callee: F, Args: {ActualArg, Builder.getFalse()});
2648 Value *Tmp = Builder.CreateAdd(LHS: Cnt, RHS: One);
2649 Value *IsZero = Builder.CreateICmpEQ(LHS: ActualArg, RHS: Zero);
2650 Value *Result = Builder.CreateSelect(C: IsZero, True: Zero, False: Tmp);
2651 if (Result->getType() != ResultType)
2652 Result = Builder.CreateIntCast(V: Result, DestTy: ResultType, isSigned: false);
2653 return RValue::get(V: Result);
2654}
2655
2656static void ClearPadding(CodeGenFunction &CGF, Address Src,
2657 const ASTContext::BitInterval &PaddingInterval) {
2658 uint64_t CharWidth = CGF.getContext().getCharWidth();
2659
2660 auto *I8Ptr = CGF.Builder.CreateBitCast(V: Src.getBasePointer(), DestTy: CGF.Int8PtrTy);
2661 auto *Zero = ConstantInt::get(Ty: CGF.Int8Ty, V: 0);
2662
2663 // Calculate byte indices and bit positions
2664 auto StartByte = PaddingInterval.First / CharWidth;
2665 auto StartBit = PaddingInterval.First % CharWidth;
2666 auto EndByte = PaddingInterval.Last / CharWidth;
2667 auto EndBit = PaddingInterval.Last % CharWidth;
2668
2669 if (StartByte == EndByte) {
2670 // Interval is within a single byte
2671 auto *Index = ConstantInt::get(Ty: CGF.IntTy, V: StartByte);
2672 auto *Element = CGF.Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: I8Ptr, IdxList: Index);
2673 Address ElementAddr(Element, CGF.Int8Ty,
2674 Src.getAlignment().alignmentAtOffset(
2675 offset: CharUnits::fromQuantity(Quantity: StartByte)));
2676
2677 auto *Value = CGF.Builder.CreateLoad(Addr: ElementAddr);
2678
2679 // Create mask to clear bits within the byte
2680 // We want to clear bits from StartBit to EndBit-1
2681 uint8_t bitsToClear = ((1 << EndBit) - 1) & ~((1 << StartBit) - 1);
2682 uint8_t bitsToKeep = ~bitsToClear;
2683 auto *MaskValue = ConstantInt::get(Ty: CGF.Int8Ty, V: bitsToKeep);
2684 auto *NewValue = CGF.Builder.CreateAnd(LHS: Value, RHS: MaskValue);
2685
2686 CGF.Builder.CreateStore(Val: NewValue, Addr: ElementAddr);
2687 } else {
2688 // Handle the start byte
2689 if (StartBit != 0) {
2690 auto *Index = ConstantInt::get(Ty: CGF.IntTy, V: StartByte);
2691 auto *Element = CGF.Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: I8Ptr, IdxList: Index);
2692 Address ElementAddr(Element, CGF.Int8Ty,
2693 Src.getAlignment().alignmentAtOffset(
2694 offset: CharUnits::fromQuantity(Quantity: StartByte)));
2695
2696 auto *Value = CGF.Builder.CreateLoad(Addr: ElementAddr);
2697
2698 uint8_t bitsToClear = ((1 << (CharWidth - StartBit)) - 1) << StartBit;
2699 uint8_t bitsToKeep = ~bitsToClear;
2700 auto *MaskValue = ConstantInt::get(Ty: CGF.Int8Ty, V: bitsToKeep);
2701 auto *NewValue = CGF.Builder.CreateAnd(LHS: Value, RHS: MaskValue);
2702
2703 CGF.Builder.CreateStore(Val: NewValue, Addr: ElementAddr);
2704 ++StartByte;
2705 }
2706
2707 // Handle full bytes in the middle
2708 for (auto Offset = StartByte; Offset < EndByte; ++Offset) {
2709 auto *Index = ConstantInt::get(Ty: CGF.IntTy, V: Offset);
2710 auto *Element = CGF.Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: I8Ptr, IdxList: Index);
2711 Address ElementAddr(Element, CGF.Int8Ty,
2712 Src.getAlignment().alignmentAtOffset(
2713 offset: CharUnits::fromQuantity(Quantity: Offset)));
2714
2715 CGF.Builder.CreateStore(Val: Zero, Addr: ElementAddr);
2716 }
2717
2718 // Handle the end byte
2719 if (EndBit != 0) {
2720 auto *Index = ConstantInt::get(Ty: CGF.IntTy, V: EndByte);
2721 auto *Element = CGF.Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: I8Ptr, IdxList: Index);
2722 Address ElementAddr(Element, CGF.Int8Ty,
2723 Src.getAlignment().alignmentAtOffset(
2724 offset: CharUnits::fromQuantity(Quantity: EndByte)));
2725
2726 auto *Value = CGF.Builder.CreateLoad(Addr: ElementAddr);
2727
2728 uint8_t bitsToClear = (1 << EndBit) - 1;
2729 uint8_t bitsToKeep = ~bitsToClear;
2730 auto *MaskValue = ConstantInt::get(Ty: CGF.Int8Ty, V: bitsToKeep);
2731 auto *NewValue = CGF.Builder.CreateAnd(LHS: Value, RHS: MaskValue);
2732
2733 CGF.Builder.CreateStore(Val: NewValue, Addr: ElementAddr);
2734 }
2735 }
2736}
2737
2738RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
2739 const CallExpr *E,
2740 ReturnValueSlot ReturnValue) {
2741 assert(!getContext().BuiltinInfo.isImmediate(BuiltinID) &&
2742 "Should not codegen for consteval builtins");
2743
2744 const FunctionDecl *FD = GD.getDecl()->getAsFunction();
2745 // See if we can constant fold this builtin. If so, don't emit it at all.
2746 // TODO: Extend this handling to all builtin calls that we can constant-fold.
2747 Expr::EvalResult Result;
2748 if (E->isPRValue() && E->EvaluateAsRValue(Result, Ctx: CGM.getContext()) &&
2749 !Result.hasSideEffects()) {
2750 if (Result.Val.isInt())
2751 return RValue::get(V: llvm::ConstantInt::get(Context&: getLLVMContext(),
2752 V: Result.Val.getInt()));
2753 if (Result.Val.isFloat())
2754 return RValue::get(V: llvm::ConstantFP::get(Context&: getLLVMContext(),
2755 V: Result.Val.getFloat()));
2756 }
2757
2758 // If current long-double semantics is IEEE 128-bit, replace math builtins
2759 // of long-double with f128 equivalent.
2760 // TODO: This mutation should also be applied to other targets other than PPC,
2761 // after backend supports IEEE 128-bit style libcalls.
2762 if (getTarget().getTriple().isPPC64() &&
2763 &getTarget().getLongDoubleFormat() == &llvm::APFloat::IEEEquad())
2764 BuiltinID = mutateLongDoubleBuiltin(BuiltinID);
2765
2766 // If the builtin has been declared explicitly with an assembler label,
2767 // disable the specialized emitting below. Ideally we should communicate the
2768 // rename in IR, or at least avoid generating the intrinsic calls that are
2769 // likely to get lowered to the renamed library functions.
2770 const unsigned BuiltinIDIfNoAsmLabel =
2771 FD->hasAttr<AsmLabelAttr>() ? 0 : BuiltinID;
2772
2773 std::optional<bool> ErrnoOverriden;
2774 // ErrnoOverriden is true if math-errno is overriden via the
2775 // '#pragma float_control(precise, on)'. This pragma disables fast-math,
2776 // which implies math-errno.
2777 if (E->hasStoredFPFeatures()) {
2778 FPOptionsOverride OP = E->getFPFeatures();
2779 if (OP.hasMathErrnoOverride())
2780 ErrnoOverriden = OP.getMathErrnoOverride();
2781 }
2782 // True if 'attribute__((optnone))' is used. This attribute overrides
2783 // fast-math which implies math-errno.
2784 bool OptNone = CurFuncDecl && CurFuncDecl->hasAttr<OptimizeNoneAttr>();
2785
2786 bool IsOptimizationEnabled = CGM.getCodeGenOpts().OptimizationLevel != 0;
2787
2788 bool GenerateFPMathIntrinsics =
2789 getContext().BuiltinInfo.shouldGenerateFPMathIntrinsic(
2790 BuiltinID, Trip: CGM.getTriple(), ErrnoOverwritten: ErrnoOverriden, MathErrnoEnabled: getLangOpts().MathErrno,
2791 HasOptNoneAttr: OptNone, IsOptimizationEnabled);
2792
2793 if (GenerateFPMathIntrinsics) {
2794 switch (BuiltinIDIfNoAsmLabel) {
2795 case Builtin::BIacos:
2796 case Builtin::BIacosf:
2797 case Builtin::BIacosl:
2798 case Builtin::BI__builtin_acos:
2799 case Builtin::BI__builtin_acosf:
2800 case Builtin::BI__builtin_acosf16:
2801 case Builtin::BI__builtin_acosl:
2802 case Builtin::BI__builtin_acosf128:
2803 case Builtin::BI__builtin_elementwise_acos:
2804 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(
2805 CGF&: *this, E, IntrinsicID: Intrinsic::acos, ConstrainedIntrinsicID: Intrinsic::experimental_constrained_acos));
2806
2807 case Builtin::BIasin:
2808 case Builtin::BIasinf:
2809 case Builtin::BIasinl:
2810 case Builtin::BI__builtin_asin:
2811 case Builtin::BI__builtin_asinf:
2812 case Builtin::BI__builtin_asinf16:
2813 case Builtin::BI__builtin_asinl:
2814 case Builtin::BI__builtin_asinf128:
2815 case Builtin::BI__builtin_elementwise_asin:
2816 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(
2817 CGF&: *this, E, IntrinsicID: Intrinsic::asin, ConstrainedIntrinsicID: Intrinsic::experimental_constrained_asin));
2818
2819 case Builtin::BIatan:
2820 case Builtin::BIatanf:
2821 case Builtin::BIatanl:
2822 case Builtin::BI__builtin_atan:
2823 case Builtin::BI__builtin_atanf:
2824 case Builtin::BI__builtin_atanf16:
2825 case Builtin::BI__builtin_atanl:
2826 case Builtin::BI__builtin_atanf128:
2827 case Builtin::BI__builtin_elementwise_atan:
2828 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(
2829 CGF&: *this, E, IntrinsicID: Intrinsic::atan, ConstrainedIntrinsicID: Intrinsic::experimental_constrained_atan));
2830
2831 case Builtin::BIatan2:
2832 case Builtin::BIatan2f:
2833 case Builtin::BIatan2l:
2834 case Builtin::BI__builtin_atan2:
2835 case Builtin::BI__builtin_atan2f:
2836 case Builtin::BI__builtin_atan2f16:
2837 case Builtin::BI__builtin_atan2l:
2838 case Builtin::BI__builtin_atan2f128:
2839 case Builtin::BI__builtin_elementwise_atan2:
2840 return RValue::get(V: emitBinaryMaybeConstrainedFPBuiltin(
2841 CGF&: *this, E, IntrinsicID: Intrinsic::atan2,
2842 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_atan2));
2843
2844 case Builtin::BIceil:
2845 case Builtin::BIceilf:
2846 case Builtin::BIceill:
2847 case Builtin::BI__builtin_ceil:
2848 case Builtin::BI__builtin_ceilf:
2849 case Builtin::BI__builtin_ceilf16:
2850 case Builtin::BI__builtin_ceill:
2851 case Builtin::BI__builtin_ceilf128:
2852 case Builtin::BI__builtin_elementwise_ceil:
2853 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
2854 IntrinsicID: Intrinsic::ceil,
2855 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_ceil));
2856
2857 case Builtin::BIcopysign:
2858 case Builtin::BIcopysignf:
2859 case Builtin::BIcopysignl:
2860 case Builtin::BI__builtin_copysign:
2861 case Builtin::BI__builtin_copysignf:
2862 case Builtin::BI__builtin_copysignf16:
2863 case Builtin::BI__builtin_copysignl:
2864 case Builtin::BI__builtin_copysignf128:
2865 return RValue::get(
2866 V: emitBuiltinWithOneOverloadedType<2>(CGF&: *this, E, IntrinsicID: Intrinsic::copysign));
2867
2868 case Builtin::BIcos:
2869 case Builtin::BIcosf:
2870 case Builtin::BIcosl:
2871 case Builtin::BI__builtin_cos:
2872 case Builtin::BI__builtin_cosf:
2873 case Builtin::BI__builtin_cosf16:
2874 case Builtin::BI__builtin_cosl:
2875 case Builtin::BI__builtin_cosf128:
2876 case Builtin::BI__builtin_elementwise_cos:
2877 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
2878 IntrinsicID: Intrinsic::cos,
2879 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_cos));
2880
2881 case Builtin::BIcosh:
2882 case Builtin::BIcoshf:
2883 case Builtin::BIcoshl:
2884 case Builtin::BI__builtin_cosh:
2885 case Builtin::BI__builtin_coshf:
2886 case Builtin::BI__builtin_coshf16:
2887 case Builtin::BI__builtin_coshl:
2888 case Builtin::BI__builtin_coshf128:
2889 case Builtin::BI__builtin_elementwise_cosh:
2890 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(
2891 CGF&: *this, E, IntrinsicID: Intrinsic::cosh, ConstrainedIntrinsicID: Intrinsic::experimental_constrained_cosh));
2892
2893 case Builtin::BIexp:
2894 case Builtin::BIexpf:
2895 case Builtin::BIexpl:
2896 case Builtin::BI__builtin_exp:
2897 case Builtin::BI__builtin_expf:
2898 case Builtin::BI__builtin_expf16:
2899 case Builtin::BI__builtin_expl:
2900 case Builtin::BI__builtin_expf128:
2901 case Builtin::BI__builtin_elementwise_exp:
2902 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
2903 IntrinsicID: Intrinsic::exp,
2904 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_exp));
2905
2906 case Builtin::BIexp2:
2907 case Builtin::BIexp2f:
2908 case Builtin::BIexp2l:
2909 case Builtin::BI__builtin_exp2:
2910 case Builtin::BI__builtin_exp2f:
2911 case Builtin::BI__builtin_exp2f16:
2912 case Builtin::BI__builtin_exp2l:
2913 case Builtin::BI__builtin_exp2f128:
2914 case Builtin::BI__builtin_elementwise_exp2:
2915 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
2916 IntrinsicID: Intrinsic::exp2,
2917 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_exp2));
2918 case Builtin::BI__builtin_exp10:
2919 case Builtin::BI__builtin_exp10f:
2920 case Builtin::BI__builtin_exp10f16:
2921 case Builtin::BI__builtin_exp10l:
2922 case Builtin::BI__builtin_exp10f128:
2923 case Builtin::BI__builtin_elementwise_exp10: {
2924 // TODO: strictfp support
2925 if (Builder.getIsFPConstrained())
2926 break;
2927 return RValue::get(
2928 V: emitBuiltinWithOneOverloadedType<1>(CGF&: *this, E, IntrinsicID: Intrinsic::exp10));
2929 }
2930 case Builtin::BIfabs:
2931 case Builtin::BIfabsf:
2932 case Builtin::BIfabsl:
2933 case Builtin::BI__builtin_fabs:
2934 case Builtin::BI__builtin_fabsf:
2935 case Builtin::BI__builtin_fabsf16:
2936 case Builtin::BI__builtin_fabsl:
2937 case Builtin::BI__builtin_fabsf128:
2938 return RValue::get(
2939 V: emitBuiltinWithOneOverloadedType<1>(CGF&: *this, E, IntrinsicID: Intrinsic::fabs));
2940
2941 case Builtin::BIfloor:
2942 case Builtin::BIfloorf:
2943 case Builtin::BIfloorl:
2944 case Builtin::BI__builtin_floor:
2945 case Builtin::BI__builtin_floorf:
2946 case Builtin::BI__builtin_floorf16:
2947 case Builtin::BI__builtin_floorl:
2948 case Builtin::BI__builtin_floorf128:
2949 case Builtin::BI__builtin_elementwise_floor:
2950 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
2951 IntrinsicID: Intrinsic::floor,
2952 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_floor));
2953
2954 case Builtin::BIfma:
2955 case Builtin::BIfmaf:
2956 case Builtin::BIfmal:
2957 case Builtin::BI__builtin_fma:
2958 case Builtin::BI__builtin_fmaf:
2959 case Builtin::BI__builtin_fmaf16:
2960 case Builtin::BI__builtin_fmal:
2961 case Builtin::BI__builtin_fmaf128:
2962 case Builtin::BI__builtin_elementwise_fma:
2963 return RValue::get(V: emitTernaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
2964 IntrinsicID: Intrinsic::fma,
2965 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_fma));
2966
2967 case Builtin::BIfmax:
2968 case Builtin::BIfmaxf:
2969 case Builtin::BIfmaxl:
2970 case Builtin::BI__builtin_fmax:
2971 case Builtin::BI__builtin_fmaxf:
2972 case Builtin::BI__builtin_fmaxf16:
2973 case Builtin::BI__builtin_fmaxl:
2974 case Builtin::BI__builtin_fmaxf128: {
2975 IRBuilder<>::FastMathFlagGuard FMFGuard(Builder);
2976 Builder.getFastMathFlags().setNoSignedZeros();
2977 return RValue::get(V: emitBinaryMaybeConstrainedFPBuiltin(
2978 CGF&: *this, E, IntrinsicID: Intrinsic::maxnum,
2979 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_maxnum));
2980 }
2981
2982 case Builtin::BIfmin:
2983 case Builtin::BIfminf:
2984 case Builtin::BIfminl:
2985 case Builtin::BI__builtin_fmin:
2986 case Builtin::BI__builtin_fminf:
2987 case Builtin::BI__builtin_fminf16:
2988 case Builtin::BI__builtin_fminl:
2989 case Builtin::BI__builtin_fminf128: {
2990 IRBuilder<>::FastMathFlagGuard FMFGuard(Builder);
2991 Builder.getFastMathFlags().setNoSignedZeros();
2992 return RValue::get(V: emitBinaryMaybeConstrainedFPBuiltin(
2993 CGF&: *this, E, IntrinsicID: Intrinsic::minnum,
2994 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_minnum));
2995 }
2996
2997 case Builtin::BIfmaximum_num:
2998 case Builtin::BIfmaximum_numf:
2999 case Builtin::BIfmaximum_numl:
3000 case Builtin::BI__builtin_fmaximum_num:
3001 case Builtin::BI__builtin_fmaximum_numf:
3002 case Builtin::BI__builtin_fmaximum_numf16:
3003 case Builtin::BI__builtin_fmaximum_numl:
3004 case Builtin::BI__builtin_fmaximum_numf128:
3005 return RValue::get(
3006 V: emitBuiltinWithOneOverloadedType<2>(CGF&: *this, E, IntrinsicID: Intrinsic::maximumnum));
3007
3008 case Builtin::BIfminimum_num:
3009 case Builtin::BIfminimum_numf:
3010 case Builtin::BIfminimum_numl:
3011 case Builtin::BI__builtin_fminimum_num:
3012 case Builtin::BI__builtin_fminimum_numf:
3013 case Builtin::BI__builtin_fminimum_numf16:
3014 case Builtin::BI__builtin_fminimum_numl:
3015 case Builtin::BI__builtin_fminimum_numf128:
3016 return RValue::get(
3017 V: emitBuiltinWithOneOverloadedType<2>(CGF&: *this, E, IntrinsicID: Intrinsic::minimumnum));
3018
3019 // fmod() is a special-case. It maps to the frem instruction rather than an
3020 // LLVM intrinsic.
3021 case Builtin::BIfmod:
3022 case Builtin::BIfmodf:
3023 case Builtin::BIfmodl:
3024 case Builtin::BI__builtin_fmod:
3025 case Builtin::BI__builtin_fmodf:
3026 case Builtin::BI__builtin_fmodf16:
3027 case Builtin::BI__builtin_fmodl:
3028 case Builtin::BI__builtin_fmodf128:
3029 case Builtin::BI__builtin_elementwise_fmod: {
3030 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
3031 Value *Arg1 = EmitScalarExpr(E: E->getArg(Arg: 0));
3032 Value *Arg2 = EmitScalarExpr(E: E->getArg(Arg: 1));
3033 if (Builder.getIsFPConstrained()) {
3034 Function *F = CGM.getIntrinsic(IID: Intrinsic::experimental_constrained_frem,
3035 Tys: Arg1->getType());
3036 return RValue::get(V: Builder.CreateConstrainedFPCall(Callee: F, Args: {Arg1, Arg2}));
3037 } else {
3038 return RValue::get(V: Builder.CreateFRem(L: Arg1, R: Arg2, Name: "fmod"));
3039 }
3040 }
3041
3042 case Builtin::BIlog:
3043 case Builtin::BIlogf:
3044 case Builtin::BIlogl:
3045 case Builtin::BI__builtin_log:
3046 case Builtin::BI__builtin_logf:
3047 case Builtin::BI__builtin_logf16:
3048 case Builtin::BI__builtin_logl:
3049 case Builtin::BI__builtin_logf128:
3050 case Builtin::BI__builtin_elementwise_log:
3051 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
3052 IntrinsicID: Intrinsic::log,
3053 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_log));
3054
3055 case Builtin::BIlog10:
3056 case Builtin::BIlog10f:
3057 case Builtin::BIlog10l:
3058 case Builtin::BI__builtin_log10:
3059 case Builtin::BI__builtin_log10f:
3060 case Builtin::BI__builtin_log10f16:
3061 case Builtin::BI__builtin_log10l:
3062 case Builtin::BI__builtin_log10f128:
3063 case Builtin::BI__builtin_elementwise_log10:
3064 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
3065 IntrinsicID: Intrinsic::log10,
3066 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_log10));
3067
3068 case Builtin::BIlog2:
3069 case Builtin::BIlog2f:
3070 case Builtin::BIlog2l:
3071 case Builtin::BI__builtin_log2:
3072 case Builtin::BI__builtin_log2f:
3073 case Builtin::BI__builtin_log2f16:
3074 case Builtin::BI__builtin_log2l:
3075 case Builtin::BI__builtin_log2f128:
3076 case Builtin::BI__builtin_elementwise_log2:
3077 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
3078 IntrinsicID: Intrinsic::log2,
3079 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_log2));
3080
3081 case Builtin::BInearbyint:
3082 case Builtin::BInearbyintf:
3083 case Builtin::BInearbyintl:
3084 case Builtin::BI__builtin_nearbyint:
3085 case Builtin::BI__builtin_nearbyintf:
3086 case Builtin::BI__builtin_nearbyintl:
3087 case Builtin::BI__builtin_nearbyintf128:
3088 case Builtin::BI__builtin_elementwise_nearbyint:
3089 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
3090 IntrinsicID: Intrinsic::nearbyint,
3091 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_nearbyint));
3092
3093 case Builtin::BIpow:
3094 case Builtin::BIpowf:
3095 case Builtin::BIpowl:
3096 case Builtin::BI__builtin_pow:
3097 case Builtin::BI__builtin_powf:
3098 case Builtin::BI__builtin_powf16:
3099 case Builtin::BI__builtin_powl:
3100 case Builtin::BI__builtin_powf128:
3101 case Builtin::BI__builtin_elementwise_pow:
3102 return RValue::get(V: emitBinaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
3103 IntrinsicID: Intrinsic::pow,
3104 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_pow));
3105
3106 case Builtin::BIrint:
3107 case Builtin::BIrintf:
3108 case Builtin::BIrintl:
3109 case Builtin::BI__builtin_rint:
3110 case Builtin::BI__builtin_rintf:
3111 case Builtin::BI__builtin_rintf16:
3112 case Builtin::BI__builtin_rintl:
3113 case Builtin::BI__builtin_rintf128:
3114 case Builtin::BI__builtin_elementwise_rint:
3115 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
3116 IntrinsicID: Intrinsic::rint,
3117 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_rint));
3118
3119 case Builtin::BIround:
3120 case Builtin::BIroundf:
3121 case Builtin::BIroundl:
3122 case Builtin::BI__builtin_round:
3123 case Builtin::BI__builtin_roundf:
3124 case Builtin::BI__builtin_roundf16:
3125 case Builtin::BI__builtin_roundl:
3126 case Builtin::BI__builtin_roundf128:
3127 case Builtin::BI__builtin_elementwise_round:
3128 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
3129 IntrinsicID: Intrinsic::round,
3130 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_round));
3131
3132 case Builtin::BIroundeven:
3133 case Builtin::BIroundevenf:
3134 case Builtin::BIroundevenl:
3135 case Builtin::BI__builtin_roundeven:
3136 case Builtin::BI__builtin_roundevenf:
3137 case Builtin::BI__builtin_roundevenf16:
3138 case Builtin::BI__builtin_roundevenl:
3139 case Builtin::BI__builtin_roundevenf128:
3140 case Builtin::BI__builtin_elementwise_roundeven:
3141 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
3142 IntrinsicID: Intrinsic::roundeven,
3143 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_roundeven));
3144
3145 case Builtin::BIsin:
3146 case Builtin::BIsinf:
3147 case Builtin::BIsinl:
3148 case Builtin::BI__builtin_sin:
3149 case Builtin::BI__builtin_sinf:
3150 case Builtin::BI__builtin_sinf16:
3151 case Builtin::BI__builtin_sinl:
3152 case Builtin::BI__builtin_sinf128:
3153 case Builtin::BI__builtin_elementwise_sin:
3154 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
3155 IntrinsicID: Intrinsic::sin,
3156 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_sin));
3157
3158 case Builtin::BIsinh:
3159 case Builtin::BIsinhf:
3160 case Builtin::BIsinhl:
3161 case Builtin::BI__builtin_sinh:
3162 case Builtin::BI__builtin_sinhf:
3163 case Builtin::BI__builtin_sinhf16:
3164 case Builtin::BI__builtin_sinhl:
3165 case Builtin::BI__builtin_sinhf128:
3166 case Builtin::BI__builtin_elementwise_sinh:
3167 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(
3168 CGF&: *this, E, IntrinsicID: Intrinsic::sinh, ConstrainedIntrinsicID: Intrinsic::experimental_constrained_sinh));
3169
3170 case Builtin::BI__builtin_sincospi:
3171 case Builtin::BI__builtin_sincospif:
3172 case Builtin::BI__builtin_sincospil:
3173 if (Builder.getIsFPConstrained())
3174 break; // TODO: Emit constrained sincospi intrinsic once one exists.
3175 emitSincosBuiltin(CGF&: *this, E, IntrinsicID: Intrinsic::sincospi);
3176 return RValue::get(V: nullptr);
3177
3178 case Builtin::BIsincos:
3179 case Builtin::BIsincosf:
3180 case Builtin::BIsincosl:
3181 case Builtin::BI__builtin_sincos:
3182 case Builtin::BI__builtin_sincosf:
3183 case Builtin::BI__builtin_sincosf16:
3184 case Builtin::BI__builtin_sincosl:
3185 case Builtin::BI__builtin_sincosf128:
3186 if (Builder.getIsFPConstrained())
3187 break; // TODO: Emit constrained sincos intrinsic once one exists.
3188 emitSincosBuiltin(CGF&: *this, E, IntrinsicID: Intrinsic::sincos);
3189 return RValue::get(V: nullptr);
3190
3191 case Builtin::BIsqrt:
3192 case Builtin::BIsqrtf:
3193 case Builtin::BIsqrtl:
3194 case Builtin::BI__builtin_sqrt:
3195 case Builtin::BI__builtin_sqrtf:
3196 case Builtin::BI__builtin_sqrtf16:
3197 case Builtin::BI__builtin_sqrtl:
3198 case Builtin::BI__builtin_sqrtf128:
3199 case Builtin::BI__builtin_elementwise_sqrt: {
3200 llvm::Value *Call = emitUnaryMaybeConstrainedFPBuiltin(
3201 CGF&: *this, E, IntrinsicID: Intrinsic::sqrt, ConstrainedIntrinsicID: Intrinsic::experimental_constrained_sqrt);
3202 SetSqrtFPAccuracy(Call);
3203 return RValue::get(V: Call);
3204 }
3205
3206 case Builtin::BItan:
3207 case Builtin::BItanf:
3208 case Builtin::BItanl:
3209 case Builtin::BI__builtin_tan:
3210 case Builtin::BI__builtin_tanf:
3211 case Builtin::BI__builtin_tanf16:
3212 case Builtin::BI__builtin_tanl:
3213 case Builtin::BI__builtin_tanf128:
3214 case Builtin::BI__builtin_elementwise_tan:
3215 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(
3216 CGF&: *this, E, IntrinsicID: Intrinsic::tan, ConstrainedIntrinsicID: Intrinsic::experimental_constrained_tan));
3217
3218 case Builtin::BItanh:
3219 case Builtin::BItanhf:
3220 case Builtin::BItanhl:
3221 case Builtin::BI__builtin_tanh:
3222 case Builtin::BI__builtin_tanhf:
3223 case Builtin::BI__builtin_tanhf16:
3224 case Builtin::BI__builtin_tanhl:
3225 case Builtin::BI__builtin_tanhf128:
3226 case Builtin::BI__builtin_elementwise_tanh:
3227 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(
3228 CGF&: *this, E, IntrinsicID: Intrinsic::tanh, ConstrainedIntrinsicID: Intrinsic::experimental_constrained_tanh));
3229
3230 case Builtin::BItrunc:
3231 case Builtin::BItruncf:
3232 case Builtin::BItruncl:
3233 case Builtin::BI__builtin_trunc:
3234 case Builtin::BI__builtin_truncf:
3235 case Builtin::BI__builtin_truncf16:
3236 case Builtin::BI__builtin_truncl:
3237 case Builtin::BI__builtin_truncf128:
3238 case Builtin::BI__builtin_elementwise_trunc:
3239 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
3240 IntrinsicID: Intrinsic::trunc,
3241 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_trunc));
3242
3243 case Builtin::BIlround:
3244 case Builtin::BIlroundf:
3245 case Builtin::BIlroundl:
3246 case Builtin::BI__builtin_lround:
3247 case Builtin::BI__builtin_lroundf:
3248 case Builtin::BI__builtin_lroundl:
3249 case Builtin::BI__builtin_lroundf128:
3250 return RValue::get(V: emitMaybeConstrainedFPToIntRoundBuiltin(
3251 CGF&: *this, E, IntrinsicID: Intrinsic::lround,
3252 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_lround));
3253
3254 case Builtin::BIllround:
3255 case Builtin::BIllroundf:
3256 case Builtin::BIllroundl:
3257 case Builtin::BI__builtin_llround:
3258 case Builtin::BI__builtin_llroundf:
3259 case Builtin::BI__builtin_llroundl:
3260 case Builtin::BI__builtin_llroundf128:
3261 return RValue::get(V: emitMaybeConstrainedFPToIntRoundBuiltin(
3262 CGF&: *this, E, IntrinsicID: Intrinsic::llround,
3263 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_llround));
3264
3265 case Builtin::BIlrint:
3266 case Builtin::BIlrintf:
3267 case Builtin::BIlrintl:
3268 case Builtin::BI__builtin_lrint:
3269 case Builtin::BI__builtin_lrintf:
3270 case Builtin::BI__builtin_lrintl:
3271 case Builtin::BI__builtin_lrintf128:
3272 return RValue::get(V: emitMaybeConstrainedFPToIntRoundBuiltin(
3273 CGF&: *this, E, IntrinsicID: Intrinsic::lrint,
3274 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_lrint));
3275
3276 case Builtin::BIllrint:
3277 case Builtin::BIllrintf:
3278 case Builtin::BIllrintl:
3279 case Builtin::BI__builtin_llrint:
3280 case Builtin::BI__builtin_llrintf:
3281 case Builtin::BI__builtin_llrintl:
3282 case Builtin::BI__builtin_llrintf128:
3283 return RValue::get(V: emitMaybeConstrainedFPToIntRoundBuiltin(
3284 CGF&: *this, E, IntrinsicID: Intrinsic::llrint,
3285 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_llrint));
3286 case Builtin::BI__builtin_ldexp:
3287 case Builtin::BI__builtin_ldexpf:
3288 case Builtin::BI__builtin_ldexpl:
3289 case Builtin::BI__builtin_ldexpf16:
3290 case Builtin::BI__builtin_ldexpf128:
3291 case Builtin::BI__builtin_elementwise_ldexp:
3292 return RValue::get(V: emitBinaryExpMaybeConstrainedFPBuiltin(
3293 CGF&: *this, E, IntrinsicID: Intrinsic::ldexp,
3294 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_ldexp));
3295 default:
3296 break;
3297 }
3298 }
3299
3300 // Check NonnullAttribute/NullabilityArg and Alignment.
3301 auto EmitArgCheck = [&](TypeCheckKind Kind, Address A, const Expr *Arg,
3302 unsigned ParmNum) {
3303 Value *Val = A.emitRawPointer(CGF&: *this);
3304 EmitNonNullArgCheck(RV: RValue::get(V: Val), ArgType: Arg->getType(), ArgLoc: Arg->getExprLoc(), AC: FD,
3305 ParmNum);
3306
3307 if (SanOpts.has(K: SanitizerKind::Alignment)) {
3308 SanitizerSet SkippedChecks;
3309 SkippedChecks.set(SanitizerKind::All);
3310 SkippedChecks.clear(K: SanitizerKind::Alignment);
3311 SourceLocation Loc = Arg->getExprLoc();
3312 // Strip an implicit cast.
3313 if (auto *CE = dyn_cast<ImplicitCastExpr>(Val: Arg))
3314 if (CE->getCastKind() == CK_BitCast)
3315 Arg = CE->getSubExpr();
3316 EmitTypeCheck(TCK: Kind, Loc, V: Val, Type: Arg->getType(), Alignment: A.getAlignment(),
3317 SkippedChecks);
3318 }
3319 };
3320
3321 switch (BuiltinIDIfNoAsmLabel) {
3322 default: break;
3323 case Builtin::BI__builtin___CFStringMakeConstantString:
3324 case Builtin::BI__builtin___NSStringMakeConstantString:
3325 return RValue::get(V: ConstantEmitter(*this).emitAbstract(E, T: E->getType()));
3326 case Builtin::BI__builtin_stdarg_start:
3327 case Builtin::BI__builtin_va_start:
3328 case Builtin::BI__va_start:
3329 case Builtin::BI__builtin_c23_va_start:
3330 case Builtin::BI__builtin_va_end:
3331 EmitVAStartEnd(ArgValue: BuiltinID == Builtin::BI__va_start
3332 ? EmitScalarExpr(E: E->getArg(Arg: 0))
3333 : EmitVAListRef(E: E->getArg(Arg: 0)).emitRawPointer(CGF&: *this),
3334 IsStart: BuiltinID != Builtin::BI__builtin_va_end);
3335 return RValue::get(V: nullptr);
3336 case Builtin::BI__builtin_va_copy: {
3337 Value *DstPtr = EmitVAListRef(E: E->getArg(Arg: 0)).emitRawPointer(CGF&: *this);
3338 Value *SrcPtr = EmitVAListRef(E: E->getArg(Arg: 1)).emitRawPointer(CGF&: *this);
3339 Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Intrinsic::vacopy, Tys: {DstPtr->getType()}),
3340 Args: {DstPtr, SrcPtr});
3341 return RValue::get(V: nullptr);
3342 }
3343 case Builtin::BIabs:
3344 case Builtin::BIlabs:
3345 case Builtin::BIllabs:
3346 case Builtin::BI__builtin_abs:
3347 case Builtin::BI__builtin_labs:
3348 case Builtin::BI__builtin_llabs: {
3349 bool SanitizeOverflow = SanOpts.has(K: SanitizerKind::SignedIntegerOverflow);
3350
3351 Value *Result;
3352 switch (getLangOpts().getSignedOverflowBehavior()) {
3353 case LangOptions::SOB_Defined:
3354 Result = EmitAbs(CGF&: *this, ArgValue: EmitScalarExpr(E: E->getArg(Arg: 0)), HasNSW: false);
3355 break;
3356 case LangOptions::SOB_Undefined:
3357 if (!SanitizeOverflow) {
3358 Result = EmitAbs(CGF&: *this, ArgValue: EmitScalarExpr(E: E->getArg(Arg: 0)), HasNSW: true);
3359 break;
3360 }
3361 [[fallthrough]];
3362 case LangOptions::SOB_Trapping:
3363 // TODO: Somehow handle the corner case when the address of abs is taken.
3364 Result = EmitOverflowCheckedAbs(CGF&: *this, E, SanitizeOverflow);
3365 break;
3366 }
3367 return RValue::get(V: Result);
3368 }
3369 case Builtin::BI__builtin_complex: {
3370 Value *Real = EmitScalarExpr(E: E->getArg(Arg: 0));
3371 Value *Imag = EmitScalarExpr(E: E->getArg(Arg: 1));
3372 return RValue::getComplex(C: {Real, Imag});
3373 }
3374 case Builtin::BI__builtin_conj:
3375 case Builtin::BI__builtin_conjf:
3376 case Builtin::BI__builtin_conjl:
3377 case Builtin::BIconj:
3378 case Builtin::BIconjf:
3379 case Builtin::BIconjl: {
3380 ComplexPairTy ComplexVal = EmitComplexExpr(E: E->getArg(Arg: 0));
3381 Value *Real = ComplexVal.first;
3382 Value *Imag = ComplexVal.second;
3383 Imag = Builder.CreateFNeg(V: Imag, Name: "neg");
3384 return RValue::getComplex(C: std::make_pair(x&: Real, y&: Imag));
3385 }
3386 case Builtin::BI__builtin_creal:
3387 case Builtin::BI__builtin_crealf:
3388 case Builtin::BI__builtin_creall:
3389 case Builtin::BIcreal:
3390 case Builtin::BIcrealf:
3391 case Builtin::BIcreall: {
3392 ComplexPairTy ComplexVal = EmitComplexExpr(E: E->getArg(Arg: 0));
3393 return RValue::get(V: ComplexVal.first);
3394 }
3395
3396 case Builtin::BI__builtin_preserve_access_index: {
3397 // Only enabled preserved access index region when debuginfo
3398 // is available as debuginfo is needed to preserve user-level
3399 // access pattern.
3400 if (!getDebugInfo()) {
3401 CGM.Error(loc: E->getExprLoc(), error: "using builtin_preserve_access_index() without -g");
3402 return RValue::get(V: EmitScalarExpr(E: E->getArg(Arg: 0)));
3403 }
3404
3405 // Nested builtin_preserve_access_index() not supported
3406 if (IsInPreservedAIRegion) {
3407 CGM.Error(loc: E->getExprLoc(), error: "nested builtin_preserve_access_index() not supported");
3408 return RValue::get(V: EmitScalarExpr(E: E->getArg(Arg: 0)));
3409 }
3410
3411 IsInPreservedAIRegion = true;
3412 Value *Res = EmitScalarExpr(E: E->getArg(Arg: 0));
3413 IsInPreservedAIRegion = false;
3414 return RValue::get(V: Res);
3415 }
3416
3417 case Builtin::BI__builtin_cimag:
3418 case Builtin::BI__builtin_cimagf:
3419 case Builtin::BI__builtin_cimagl:
3420 case Builtin::BIcimag:
3421 case Builtin::BIcimagf:
3422 case Builtin::BIcimagl: {
3423 ComplexPairTy ComplexVal = EmitComplexExpr(E: E->getArg(Arg: 0));
3424 return RValue::get(V: ComplexVal.second);
3425 }
3426
3427 case Builtin::BI__builtin_clrsb:
3428 case Builtin::BI__builtin_clrsbl:
3429 case Builtin::BI__builtin_clrsbll: {
3430 // clrsb(x) -> clz(x < 0 ? ~x : x) - 1 or
3431 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
3432
3433 llvm::Type *ArgType = ArgValue->getType();
3434 Function *F = CGM.getIntrinsic(IID: Intrinsic::ctlz, Tys: ArgType);
3435
3436 llvm::Type *ResultType = ConvertType(T: E->getType());
3437 Value *Zero = llvm::Constant::getNullValue(Ty: ArgType);
3438 Value *IsNeg = Builder.CreateICmpSLT(LHS: ArgValue, RHS: Zero, Name: "isneg");
3439 Value *Inverse = Builder.CreateNot(V: ArgValue, Name: "not");
3440 Value *Tmp = Builder.CreateSelect(C: IsNeg, True: Inverse, False: ArgValue);
3441 Value *Ctlz = Builder.CreateCall(Callee: F, Args: {Tmp, Builder.getFalse()});
3442 Value *Result =
3443 Builder.CreateNUWSub(LHS: Ctlz, RHS: llvm::ConstantInt::get(Ty: ArgType, V: 1));
3444 Result = Builder.CreateIntCast(V: Result, DestTy: ResultType, /*isSigned*/true,
3445 Name: "cast");
3446 return RValue::get(V: Result);
3447 }
3448 case Builtin::BI__builtin_ctzs:
3449 case Builtin::BI__builtin_ctz:
3450 case Builtin::BI__builtin_ctzl:
3451 case Builtin::BI__builtin_ctzll:
3452 case Builtin::BI__builtin_ctzg:
3453 case Builtin::BI__builtin_elementwise_ctzg: {
3454 bool HasFallback =
3455 (BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_ctzg ||
3456 BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_elementwise_ctzg) &&
3457 E->getNumArgs() > 1;
3458
3459 Value *ArgValue =
3460 HasFallback ? EmitBitCountExpr(CGF&: *this, E: E->getArg(Arg: 0))
3461 : EmitCheckedArgForBuiltin(E: E->getArg(Arg: 0), Kind: BCK_CTZPassedZero);
3462
3463 llvm::Type *ArgType = ArgValue->getType();
3464 Function *F = CGM.getIntrinsic(IID: Intrinsic::cttz, Tys: ArgType);
3465
3466 llvm::Type *ResultType = ConvertType(T: E->getType());
3467 // The elementwise builtins always exhibit zero-is-undef behaviour
3468 Value *ZeroUndef = Builder.getInt1(
3469 V: HasFallback || getTarget().isCLZForZeroUndef() ||
3470 BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_elementwise_ctzg);
3471 Value *Result = Builder.CreateCall(Callee: F, Args: {ArgValue, ZeroUndef});
3472 if (Result->getType() != ResultType)
3473 Result =
3474 Builder.CreateIntCast(V: Result, DestTy: ResultType, /*isSigned*/ false, Name: "cast");
3475 if (!HasFallback)
3476 return RValue::get(V: Result);
3477
3478 Value *Zero = Constant::getNullValue(Ty: ArgType);
3479 Value *IsZero = Builder.CreateICmpEQ(LHS: ArgValue, RHS: Zero, Name: "iszero");
3480 Value *FallbackValue = EmitScalarExpr(E: E->getArg(Arg: 1));
3481 Value *ResultOrFallback =
3482 Builder.CreateSelect(C: IsZero, True: FallbackValue, False: Result, Name: "ctzg");
3483 return RValue::get(V: ResultOrFallback);
3484 }
3485 case Builtin::BI__builtin_clzs:
3486 case Builtin::BI__builtin_clz:
3487 case Builtin::BI__builtin_clzl:
3488 case Builtin::BI__builtin_clzll:
3489 case Builtin::BI__builtin_clzg:
3490 case Builtin::BI__builtin_elementwise_clzg: {
3491 bool HasFallback =
3492 (BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_clzg ||
3493 BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_elementwise_clzg) &&
3494 E->getNumArgs() > 1;
3495
3496 Value *ArgValue =
3497 HasFallback ? EmitBitCountExpr(CGF&: *this, E: E->getArg(Arg: 0))
3498 : EmitCheckedArgForBuiltin(E: E->getArg(Arg: 0), Kind: BCK_CLZPassedZero);
3499
3500 llvm::Type *ArgType = ArgValue->getType();
3501 Function *F = CGM.getIntrinsic(IID: Intrinsic::ctlz, Tys: ArgType);
3502
3503 llvm::Type *ResultType = ConvertType(T: E->getType());
3504 // The elementwise builtins always exhibit zero-is-undef behaviour
3505 Value *ZeroUndef = Builder.getInt1(
3506 V: HasFallback || getTarget().isCLZForZeroUndef() ||
3507 BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_elementwise_clzg);
3508 Value *Result = Builder.CreateCall(Callee: F, Args: {ArgValue, ZeroUndef});
3509 if (Result->getType() != ResultType)
3510 Result =
3511 Builder.CreateIntCast(V: Result, DestTy: ResultType, /*isSigned*/ false, Name: "cast");
3512 if (!HasFallback)
3513 return RValue::get(V: Result);
3514
3515 Value *Zero = Constant::getNullValue(Ty: ArgType);
3516 Value *IsZero = Builder.CreateICmpEQ(LHS: ArgValue, RHS: Zero, Name: "iszero");
3517 Value *FallbackValue = EmitScalarExpr(E: E->getArg(Arg: 1));
3518 Value *ResultOrFallback =
3519 Builder.CreateSelect(C: IsZero, True: FallbackValue, False: Result, Name: "clzg");
3520 return RValue::get(V: ResultOrFallback);
3521 }
3522 case Builtin::BI__builtin_ffs:
3523 case Builtin::BI__builtin_ffsl:
3524 case Builtin::BI__builtin_ffsll: {
3525 // ffs(x) -> x ? cttz(x) + 1 : 0
3526 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
3527
3528 llvm::Type *ArgType = ArgValue->getType();
3529 Function *F = CGM.getIntrinsic(IID: Intrinsic::cttz, Tys: ArgType);
3530
3531 llvm::Type *ResultType = ConvertType(T: E->getType());
3532 Value *Tmp =
3533 Builder.CreateAdd(LHS: Builder.CreateCall(Callee: F, Args: {ArgValue, Builder.getTrue()}),
3534 RHS: llvm::ConstantInt::get(Ty: ArgType, V: 1));
3535 Value *Zero = llvm::Constant::getNullValue(Ty: ArgType);
3536 Value *IsZero = Builder.CreateICmpEQ(LHS: ArgValue, RHS: Zero, Name: "iszero");
3537 Value *Result = Builder.CreateSelect(C: IsZero, True: Zero, False: Tmp, Name: "ffs");
3538 if (Result->getType() != ResultType)
3539 Result = Builder.CreateIntCast(V: Result, DestTy: ResultType, /*isSigned*/true,
3540 Name: "cast");
3541 return RValue::get(V: Result);
3542 }
3543 case Builtin::BI__builtin_parity:
3544 case Builtin::BI__builtin_parityl:
3545 case Builtin::BI__builtin_parityll: {
3546 // parity(x) -> ctpop(x) & 1
3547 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
3548
3549 llvm::Type *ArgType = ArgValue->getType();
3550 Function *F = CGM.getIntrinsic(IID: Intrinsic::ctpop, Tys: ArgType);
3551
3552 llvm::Type *ResultType = ConvertType(T: E->getType());
3553 Value *Tmp = Builder.CreateCall(Callee: F, Args: ArgValue);
3554 Value *Result = Builder.CreateAnd(LHS: Tmp, RHS: llvm::ConstantInt::get(Ty: ArgType, V: 1));
3555 if (Result->getType() != ResultType)
3556 Result = Builder.CreateIntCast(V: Result, DestTy: ResultType, /*isSigned*/true,
3557 Name: "cast");
3558 return RValue::get(V: Result);
3559 }
3560 case Builtin::BI__lzcnt16:
3561 case Builtin::BI__lzcnt:
3562 case Builtin::BI__lzcnt64: {
3563 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
3564
3565 llvm::Type *ArgType = ArgValue->getType();
3566 Function *F = CGM.getIntrinsic(IID: Intrinsic::ctlz, Tys: ArgType);
3567
3568 llvm::Type *ResultType = ConvertType(T: E->getType());
3569 Value *Result = Builder.CreateCall(Callee: F, Args: {ArgValue, Builder.getFalse()});
3570 if (Result->getType() != ResultType)
3571 Result = Builder.CreateIntCast(V: Result, DestTy: ResultType, /*isSigned*/true,
3572 Name: "cast");
3573 return RValue::get(V: Result);
3574 }
3575 case Builtin::BI__popcnt16:
3576 case Builtin::BI__popcnt:
3577 case Builtin::BI__popcnt64:
3578 case Builtin::BI__builtin_popcount:
3579 case Builtin::BI__builtin_popcountl:
3580 case Builtin::BI__builtin_popcountll:
3581 case Builtin::BI__builtin_popcountg: {
3582 Value *ArgValue = EmitBitCountExpr(CGF&: *this, E: E->getArg(Arg: 0));
3583
3584 llvm::Type *ArgType = ArgValue->getType();
3585 Function *F = CGM.getIntrinsic(IID: Intrinsic::ctpop, Tys: ArgType);
3586
3587 llvm::Type *ResultType = ConvertType(T: E->getType());
3588 Value *Result = Builder.CreateCall(Callee: F, Args: ArgValue);
3589 if (Result->getType() != ResultType)
3590 Result =
3591 Builder.CreateIntCast(V: Result, DestTy: ResultType, /*isSigned*/ false, Name: "cast");
3592 return RValue::get(V: Result);
3593 }
3594 case Builtin::BI__builtin_unpredictable: {
3595 // Always return the argument of __builtin_unpredictable. LLVM does not
3596 // handle this builtin. Metadata for this builtin should be added directly
3597 // to instructions such as branches or switches that use it.
3598 return RValue::get(V: EmitScalarExpr(E: E->getArg(Arg: 0)));
3599 }
3600 case Builtin::BI__builtin_expect: {
3601 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
3602 llvm::Type *ArgType = ArgValue->getType();
3603
3604 Value *ExpectedValue = EmitScalarExpr(E: E->getArg(Arg: 1));
3605 // Don't generate llvm.expect on -O0 as the backend won't use it for
3606 // anything.
3607 // Note, we still IRGen ExpectedValue because it could have side-effects.
3608 if (CGM.getCodeGenOpts().OptimizationLevel == 0)
3609 return RValue::get(V: ArgValue);
3610
3611 Function *FnExpect = CGM.getIntrinsic(IID: Intrinsic::expect, Tys: ArgType);
3612 Value *Result =
3613 Builder.CreateCall(Callee: FnExpect, Args: {ArgValue, ExpectedValue}, Name: "expval");
3614 return RValue::get(V: Result);
3615 }
3616 case Builtin::BI__builtin_expect_with_probability: {
3617 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
3618 llvm::Type *ArgType = ArgValue->getType();
3619
3620 Value *ExpectedValue = EmitScalarExpr(E: E->getArg(Arg: 1));
3621 llvm::APFloat Probability(0.0);
3622 const Expr *ProbArg = E->getArg(Arg: 2);
3623 bool EvalSucceed = ProbArg->EvaluateAsFloat(Result&: Probability, Ctx: CGM.getContext());
3624 assert(EvalSucceed && "probability should be able to evaluate as float");
3625 (void)EvalSucceed;
3626 bool LoseInfo = false;
3627 Probability.convert(ToSemantics: llvm::APFloat::IEEEdouble(),
3628 RM: llvm::RoundingMode::Dynamic, losesInfo: &LoseInfo);
3629 llvm::Type *Ty = ConvertType(T: ProbArg->getType());
3630 Constant *Confidence = ConstantFP::get(Ty, V: Probability);
3631 // Don't generate llvm.expect.with.probability on -O0 as the backend
3632 // won't use it for anything.
3633 // Note, we still IRGen ExpectedValue because it could have side-effects.
3634 if (CGM.getCodeGenOpts().OptimizationLevel == 0)
3635 return RValue::get(V: ArgValue);
3636
3637 Function *FnExpect =
3638 CGM.getIntrinsic(IID: Intrinsic::expect_with_probability, Tys: ArgType);
3639 Value *Result = Builder.CreateCall(
3640 Callee: FnExpect, Args: {ArgValue, ExpectedValue, Confidence}, Name: "expval");
3641 return RValue::get(V: Result);
3642 }
3643 case Builtin::BI__builtin_assume_aligned: {
3644 const Expr *Ptr = E->getArg(Arg: 0);
3645 Value *PtrValue = EmitScalarExpr(E: Ptr);
3646 Value *OffsetValue =
3647 (E->getNumArgs() > 2) ? EmitScalarExpr(E: E->getArg(Arg: 2)) : nullptr;
3648
3649 Value *AlignmentValue = EmitScalarExpr(E: E->getArg(Arg: 1));
3650 ConstantInt *AlignmentCI = cast<ConstantInt>(Val: AlignmentValue);
3651 if (AlignmentCI->getValue().ugt(RHS: llvm::Value::MaximumAlignment))
3652 AlignmentCI = ConstantInt::get(Ty: AlignmentCI->getIntegerType(),
3653 V: llvm::Value::MaximumAlignment);
3654
3655 emitAlignmentAssumption(PtrValue, E: Ptr,
3656 /*The expr loc is sufficient.*/ AssumptionLoc: SourceLocation(),
3657 Alignment: AlignmentCI, OffsetValue);
3658 return RValue::get(V: PtrValue);
3659 }
3660 case Builtin::BI__builtin_assume_dereferenceable: {
3661 const Expr *Ptr = E->getArg(Arg: 0);
3662 const Expr *Size = E->getArg(Arg: 1);
3663 Value *PtrValue = EmitScalarExpr(E: Ptr);
3664 Value *SizeValue = EmitScalarExpr(E: Size);
3665 if (SizeValue->getType() != IntPtrTy)
3666 SizeValue =
3667 Builder.CreateIntCast(V: SizeValue, DestTy: IntPtrTy, isSigned: false, Name: "casted.size");
3668 Builder.CreateDereferenceableAssumption(PtrValue, SizeValue);
3669 return RValue::get(V: nullptr);
3670 }
3671 case Builtin::BI__assume:
3672 case Builtin::BI__builtin_assume: {
3673 if (E->getArg(Arg: 0)->HasSideEffects(Ctx: getContext()))
3674 return RValue::get(V: nullptr);
3675
3676 Value *ArgValue = EmitCheckedArgForAssume(E: E->getArg(Arg: 0));
3677 Function *FnAssume = CGM.getIntrinsic(IID: Intrinsic::assume);
3678 Builder.CreateCall(Callee: FnAssume, Args: ArgValue);
3679 return RValue::get(V: nullptr);
3680 }
3681 case Builtin::BI__builtin_assume_separate_storage: {
3682 const Expr *Arg0 = E->getArg(Arg: 0);
3683 const Expr *Arg1 = E->getArg(Arg: 1);
3684
3685 Value *Value0 = EmitScalarExpr(E: Arg0);
3686 Value *Value1 = EmitScalarExpr(E: Arg1);
3687
3688 Value *Values[] = {Value0, Value1};
3689 OperandBundleDefT<Value *> OBD("separate_storage", Values);
3690 Builder.CreateAssumption(OpBundles: {OBD});
3691 return RValue::get(V: nullptr);
3692 }
3693 case Builtin::BI__builtin_allow_runtime_check: {
3694 StringRef Kind =
3695 cast<StringLiteral>(Val: E->getArg(Arg: 0)->IgnoreParenCasts())->getString();
3696 LLVMContext &Ctx = CGM.getLLVMContext();
3697 llvm::Value *Allow = Builder.CreateCall(
3698 Callee: CGM.getIntrinsic(IID: Intrinsic::allow_runtime_check),
3699 Args: llvm::MetadataAsValue::get(Context&: Ctx, MD: llvm::MDString::get(Context&: Ctx, Str: Kind)));
3700 return RValue::get(V: Allow);
3701 }
3702 case Builtin::BI__builtin_allow_sanitize_check: {
3703 Intrinsic::ID IntrID = Intrinsic::not_intrinsic;
3704 StringRef Name =
3705 cast<StringLiteral>(Val: E->getArg(Arg: 0)->IgnoreParenCasts())->getString();
3706
3707 // We deliberately allow the use of kernel- and non-kernel names
3708 // interchangably, even when one or the other is enabled. This is consistent
3709 // with the no_sanitize-attribute, which allows either kernel- or non-kernel
3710 // name to disable instrumentation (see CodeGenFunction::StartFunction).
3711 if (getLangOpts().Sanitize.hasOneOf(K: SanitizerKind::Address |
3712 SanitizerKind::KernelAddress) &&
3713 (Name == "address" || Name == "kernel-address")) {
3714 IntrID = Intrinsic::allow_sanitize_address;
3715 } else if (getLangOpts().Sanitize.has(K: SanitizerKind::Thread) &&
3716 Name == "thread") {
3717 IntrID = Intrinsic::allow_sanitize_thread;
3718 } else if (getLangOpts().Sanitize.hasOneOf(K: SanitizerKind::Memory |
3719 SanitizerKind::KernelMemory) &&
3720 (Name == "memory" || Name == "kernel-memory")) {
3721 IntrID = Intrinsic::allow_sanitize_memory;
3722 } else if (getLangOpts().Sanitize.hasOneOf(
3723 K: SanitizerKind::HWAddress | SanitizerKind::KernelHWAddress) &&
3724 (Name == "hwaddress" || Name == "kernel-hwaddress")) {
3725 IntrID = Intrinsic::allow_sanitize_hwaddress;
3726 }
3727
3728 if (IntrID != Intrinsic::not_intrinsic) {
3729 llvm::Value *Allow = Builder.CreateCall(Callee: CGM.getIntrinsic(IID: IntrID));
3730 return RValue::get(V: Allow);
3731 }
3732 // If the checked sanitizer is not enabled, we can safely lower to false
3733 // right away. This is also more efficient, since the LowerAllowCheckPass
3734 // must not always be enabled if none of the above sanitizers are enabled.
3735 return RValue::get(V: Builder.getFalse());
3736 }
3737 case Builtin::BI__arithmetic_fence: {
3738 // Create the builtin call if FastMath is selected, and the target
3739 // supports the builtin, otherwise just return the argument.
3740 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
3741 llvm::FastMathFlags FMF = Builder.getFastMathFlags();
3742 bool isArithmeticFenceEnabled =
3743 FMF.allowReassoc() &&
3744 getContext().getTargetInfo().checkArithmeticFenceSupported();
3745 QualType ArgType = E->getArg(Arg: 0)->getType();
3746 if (ArgType->isComplexType()) {
3747 if (isArithmeticFenceEnabled) {
3748 QualType ElementType = ArgType->castAs<ComplexType>()->getElementType();
3749 ComplexPairTy ComplexVal = EmitComplexExpr(E: E->getArg(Arg: 0));
3750 Value *Real = Builder.CreateArithmeticFence(Val: ComplexVal.first,
3751 DstType: ConvertType(T: ElementType));
3752 Value *Imag = Builder.CreateArithmeticFence(Val: ComplexVal.second,
3753 DstType: ConvertType(T: ElementType));
3754 return RValue::getComplex(C: std::make_pair(x&: Real, y&: Imag));
3755 }
3756 ComplexPairTy ComplexVal = EmitComplexExpr(E: E->getArg(Arg: 0));
3757 Value *Real = ComplexVal.first;
3758 Value *Imag = ComplexVal.second;
3759 return RValue::getComplex(C: std::make_pair(x&: Real, y&: Imag));
3760 }
3761 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
3762 if (isArithmeticFenceEnabled)
3763 return RValue::get(
3764 V: Builder.CreateArithmeticFence(Val: ArgValue, DstType: ConvertType(T: ArgType)));
3765 return RValue::get(V: ArgValue);
3766 }
3767 case Builtin::BI__builtin_bswapg: {
3768 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
3769 llvm::IntegerType *IntTy = cast<llvm::IntegerType>(Val: ArgValue->getType());
3770 assert(IntTy && "LLVM's __builtin_bswapg only supports integer variants");
3771 if (IntTy->getBitWidth() == 1 || IntTy->getBitWidth() == 8)
3772 return RValue::get(V: ArgValue);
3773 assert(((IntTy->getBitWidth() % 16 == 0 && IntTy->getBitWidth() != 0)) &&
3774 "LLVM's __builtin_bswapg only supports integer variants that has a "
3775 "multiple of 16 bits as well as a single byte");
3776 return RValue::get(
3777 V: emitBuiltinWithOneOverloadedType<1>(CGF&: *this, E, IntrinsicID: Intrinsic::bswap));
3778 }
3779 case Builtin::BI__builtin_bswap16:
3780 case Builtin::BI__builtin_bswap32:
3781 case Builtin::BI__builtin_bswap64:
3782 case Builtin::BI_byteswap_ushort:
3783 case Builtin::BI_byteswap_ulong:
3784 case Builtin::BI_byteswap_uint64: {
3785 return RValue::get(
3786 V: emitBuiltinWithOneOverloadedType<1>(CGF&: *this, E, IntrinsicID: Intrinsic::bswap));
3787 }
3788 case Builtin::BI__builtin_bitreverseg: {
3789 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
3790 llvm::IntegerType *IntTy = cast<llvm::IntegerType>(Val: ArgValue->getType());
3791 assert(IntTy &&
3792 "LLVM's __builtin_bitreverseg only support integer variants");
3793 if (IntTy->getBitWidth() == 1)
3794 return RValue::get(V: ArgValue);
3795 return RValue::get(
3796 V: emitBuiltinWithOneOverloadedType<1>(CGF&: *this, E, IntrinsicID: Intrinsic::bitreverse));
3797 }
3798 case Builtin::BI__builtin_bitreverse8:
3799 case Builtin::BI__builtin_bitreverse16:
3800 case Builtin::BI__builtin_bitreverse32:
3801 case Builtin::BI__builtin_bitreverse64: {
3802 return RValue::get(
3803 V: emitBuiltinWithOneOverloadedType<1>(CGF&: *this, E, IntrinsicID: Intrinsic::bitreverse));
3804 }
3805 case Builtin::BI__builtin_rotateleft8:
3806 case Builtin::BI__builtin_rotateleft16:
3807 case Builtin::BI__builtin_rotateleft32:
3808 case Builtin::BI__builtin_rotateleft64:
3809 case Builtin::BI__builtin_stdc_rotate_left:
3810 case Builtin::BIstdc_rotate_left_uc:
3811 case Builtin::BIstdc_rotate_left_us:
3812 case Builtin::BIstdc_rotate_left_ui:
3813 case Builtin::BIstdc_rotate_left_ul:
3814 case Builtin::BIstdc_rotate_left_ull:
3815 case Builtin::BI_rotl8: // Microsoft variants of rotate left
3816 case Builtin::BI_rotl16:
3817 case Builtin::BI_rotl:
3818 case Builtin::BI_lrotl:
3819 case Builtin::BI_rotl64:
3820 return emitRotate(E, IsRotateRight: false);
3821
3822 case Builtin::BI__builtin_rotateright8:
3823 case Builtin::BI__builtin_rotateright16:
3824 case Builtin::BI__builtin_rotateright32:
3825 case Builtin::BI__builtin_rotateright64:
3826 case Builtin::BI__builtin_stdc_rotate_right:
3827 case Builtin::BIstdc_rotate_right_uc:
3828 case Builtin::BIstdc_rotate_right_us:
3829 case Builtin::BIstdc_rotate_right_ui:
3830 case Builtin::BIstdc_rotate_right_ul:
3831 case Builtin::BIstdc_rotate_right_ull:
3832 case Builtin::BI_rotr8: // Microsoft variants of rotate right
3833 case Builtin::BI_rotr16:
3834 case Builtin::BI_rotr:
3835 case Builtin::BI_lrotr:
3836 case Builtin::BI_rotr64:
3837 return emitRotate(E, IsRotateRight: true);
3838
3839 case Builtin::BIstdc_leading_zeros_uc:
3840 case Builtin::BIstdc_leading_zeros_us:
3841 case Builtin::BIstdc_leading_zeros_ui:
3842 case Builtin::BIstdc_leading_zeros_ul:
3843 case Builtin::BIstdc_leading_zeros_ull:
3844 case Builtin::BI__builtin_stdc_leading_zeros:
3845 return emitStdcCountIntrinsic(E, IntID: Intrinsic::ctlz, /*InvertArg=*/false);
3846 case Builtin::BIstdc_leading_ones_uc:
3847 case Builtin::BIstdc_leading_ones_us:
3848 case Builtin::BIstdc_leading_ones_ui:
3849 case Builtin::BIstdc_leading_ones_ul:
3850 case Builtin::BIstdc_leading_ones_ull:
3851 case Builtin::BI__builtin_stdc_leading_ones:
3852 return emitStdcCountIntrinsic(E, IntID: Intrinsic::ctlz, /*InvertArg=*/true);
3853 case Builtin::BIstdc_trailing_zeros_uc:
3854 case Builtin::BIstdc_trailing_zeros_us:
3855 case Builtin::BIstdc_trailing_zeros_ui:
3856 case Builtin::BIstdc_trailing_zeros_ul:
3857 case Builtin::BIstdc_trailing_zeros_ull:
3858 case Builtin::BI__builtin_stdc_trailing_zeros:
3859 return emitStdcCountIntrinsic(E, IntID: Intrinsic::cttz, /*InvertArg=*/false);
3860 case Builtin::BIstdc_trailing_ones_uc:
3861 case Builtin::BIstdc_trailing_ones_us:
3862 case Builtin::BIstdc_trailing_ones_ui:
3863 case Builtin::BIstdc_trailing_ones_ul:
3864 case Builtin::BIstdc_trailing_ones_ull:
3865 case Builtin::BI__builtin_stdc_trailing_ones:
3866 return emitStdcCountIntrinsic(E, IntID: Intrinsic::cttz, /*InvertArg=*/true);
3867 case Builtin::BIstdc_first_leading_zero_uc:
3868 case Builtin::BIstdc_first_leading_zero_us:
3869 case Builtin::BIstdc_first_leading_zero_ui:
3870 case Builtin::BIstdc_first_leading_zero_ul:
3871 case Builtin::BIstdc_first_leading_zero_ull:
3872 case Builtin::BI__builtin_stdc_first_leading_zero:
3873 return emitStdcFirstBit(E, IntID: Intrinsic::ctlz, /*InvertArg=*/true);
3874 case Builtin::BIstdc_first_leading_one_uc:
3875 case Builtin::BIstdc_first_leading_one_us:
3876 case Builtin::BIstdc_first_leading_one_ui:
3877 case Builtin::BIstdc_first_leading_one_ul:
3878 case Builtin::BIstdc_first_leading_one_ull:
3879 case Builtin::BI__builtin_stdc_first_leading_one:
3880 return emitStdcFirstBit(E, IntID: Intrinsic::ctlz, /*InvertArg=*/false);
3881 case Builtin::BIstdc_first_trailing_zero_uc:
3882 case Builtin::BIstdc_first_trailing_zero_us:
3883 case Builtin::BIstdc_first_trailing_zero_ui:
3884 case Builtin::BIstdc_first_trailing_zero_ul:
3885 case Builtin::BIstdc_first_trailing_zero_ull:
3886 case Builtin::BI__builtin_stdc_first_trailing_zero:
3887 return emitStdcFirstBit(E, IntID: Intrinsic::cttz, /*InvertArg=*/true);
3888 case Builtin::BIstdc_first_trailing_one_uc:
3889 case Builtin::BIstdc_first_trailing_one_us:
3890 case Builtin::BIstdc_first_trailing_one_ui:
3891 case Builtin::BIstdc_first_trailing_one_ul:
3892 case Builtin::BIstdc_first_trailing_one_ull:
3893 case Builtin::BI__builtin_stdc_first_trailing_one:
3894 return emitStdcFirstBit(E, IntID: Intrinsic::cttz, /*InvertArg=*/false);
3895 case Builtin::BIstdc_count_zeros_uc:
3896 case Builtin::BIstdc_count_zeros_us:
3897 case Builtin::BIstdc_count_zeros_ui:
3898 case Builtin::BIstdc_count_zeros_ul:
3899 case Builtin::BIstdc_count_zeros_ull:
3900 case Builtin::BI__builtin_stdc_count_zeros:
3901 return emitStdcBitWidthMinus(E, IntID: Intrinsic::ctpop, /*IsPop=*/true);
3902 case Builtin::BIstdc_count_ones_uc:
3903 case Builtin::BIstdc_count_ones_us:
3904 case Builtin::BIstdc_count_ones_ui:
3905 case Builtin::BIstdc_count_ones_ul:
3906 case Builtin::BIstdc_count_ones_ull:
3907 case Builtin::BI__builtin_stdc_count_ones:
3908 return emitStdcCountIntrinsic(E, IntID: Intrinsic::ctpop, /*InvertArg=*/false,
3909 /*IsPop=*/true);
3910 case Builtin::BIstdc_has_single_bit_uc:
3911 case Builtin::BIstdc_has_single_bit_us:
3912 case Builtin::BIstdc_has_single_bit_ui:
3913 case Builtin::BIstdc_has_single_bit_ul:
3914 case Builtin::BIstdc_has_single_bit_ull:
3915 case Builtin::BI__builtin_stdc_has_single_bit: {
3916 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
3917 llvm::Type *ArgType = ArgValue->getType();
3918 Value *One = ConstantInt::get(Ty: ArgType, V: 1);
3919 Function *F = CGM.getIntrinsic(IID: Intrinsic::ctpop, Tys: ArgType);
3920 Value *PopCnt = Builder.CreateCall(Callee: F, Args: ArgValue);
3921 return RValue::get(V: Builder.CreateICmpEQ(LHS: PopCnt, RHS: One));
3922 }
3923 case Builtin::BIstdc_bit_width_uc:
3924 case Builtin::BIstdc_bit_width_us:
3925 case Builtin::BIstdc_bit_width_ui:
3926 case Builtin::BIstdc_bit_width_ul:
3927 case Builtin::BIstdc_bit_width_ull:
3928 case Builtin::BI__builtin_stdc_bit_width:
3929 return emitStdcBitWidthMinus(E, IntID: Intrinsic::ctlz, /*IsPop=*/false);
3930 case Builtin::BIstdc_bit_floor_uc:
3931 case Builtin::BIstdc_bit_floor_us:
3932 case Builtin::BIstdc_bit_floor_ui:
3933 case Builtin::BIstdc_bit_floor_ul:
3934 case Builtin::BIstdc_bit_floor_ull:
3935 case Builtin::BI__builtin_stdc_bit_floor: {
3936 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
3937 llvm::Type *ArgType = ArgValue->getType();
3938 unsigned BitWidth = ArgType->getIntegerBitWidth();
3939 Value *Zero = ConstantInt::get(Ty: ArgType, V: 0);
3940 Value *One = ConstantInt::get(Ty: ArgType, V: 1);
3941 Function *F = CGM.getIntrinsic(IID: Intrinsic::ctlz, Tys: ArgType);
3942 Value *LZ = Builder.CreateCall(Callee: F, Args: {ArgValue, Builder.getTrue()});
3943 Value *ShiftAmt =
3944 Builder.CreateSub(LHS: ConstantInt::get(Ty: ArgType, V: BitWidth - 1), RHS: LZ);
3945 Value *Shifted = Builder.CreateShl(LHS: One, RHS: ShiftAmt);
3946 Value *IsZero = Builder.CreateICmpEQ(LHS: ArgValue, RHS: Zero);
3947 Value *Result = Builder.CreateSelect(C: IsZero, True: Zero, False: Shifted);
3948 return RValue::get(V: Result);
3949 }
3950 case Builtin::BIstdc_bit_ceil_uc:
3951 case Builtin::BIstdc_bit_ceil_us:
3952 case Builtin::BIstdc_bit_ceil_ui:
3953 case Builtin::BIstdc_bit_ceil_ul:
3954 case Builtin::BIstdc_bit_ceil_ull:
3955 case Builtin::BI__builtin_stdc_bit_ceil: {
3956 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
3957 llvm::Type *ArgType = ArgValue->getType();
3958 Value *One = ConstantInt::get(Ty: ArgType, V: 1);
3959 Value *IsLEOne = Builder.CreateICmpULE(LHS: ArgValue, RHS: One, Name: "isleone");
3960
3961 BasicBlock *EntryBB = Builder.GetInsertBlock();
3962 BasicBlock *CalcBB = createBasicBlock(name: "bitceil.calc", parent: CurFn);
3963 BasicBlock *MergeBB = createBasicBlock(name: "bitceil.merge", parent: CurFn);
3964
3965 Builder.CreateCondBr(Cond: IsLEOne, True: MergeBB, False: CalcBB);
3966
3967 Builder.SetInsertPoint(CalcBB);
3968 Function *F = CGM.getIntrinsic(IID: Intrinsic::ctlz, Tys: ArgType);
3969 Value *ArgMinusOne = Builder.CreateSub(LHS: ArgValue, RHS: One);
3970 Value *LZ = Builder.CreateCall(Callee: F, Args: {ArgMinusOne, Builder.getFalse()});
3971 // 2<<(BitWidth-1-LZ) to get the next power of two. The shift
3972 // amount is always in [0, BitWidth-1], so when LZ==0 (argument has its MSB
3973 // set), the result wraps to 0
3974 unsigned BitWidth = ArgType->getIntegerBitWidth();
3975 Value *ShiftAmt =
3976 Builder.CreateSub(LHS: ConstantInt::get(Ty: ArgType, V: BitWidth - 1), RHS: LZ);
3977 Value *Two = Builder.CreateShl(LHS: One, RHS: One);
3978 Value *Tmp = Builder.CreateShl(LHS: Two, RHS: ShiftAmt);
3979 Builder.CreateBr(Dest: MergeBB);
3980
3981 Builder.SetInsertPoint(MergeBB);
3982 PHINode *Phi = Builder.CreatePHI(Ty: ArgType, NumReservedValues: 2);
3983 Phi->addIncoming(V: One, BB: EntryBB);
3984 Phi->addIncoming(V: Tmp, BB: CalcBB);
3985 return RValue::get(V: Phi);
3986 }
3987
3988 // stdc_memreverse8u8 is a no-op (single byte, nothing to swap).
3989 case Builtin::BIstdc_memreverse8u8:
3990 return RValue::get(V: EmitScalarExpr(E: E->getArg(Arg: 0)));
3991
3992 case Builtin::BIstdc_memreverse8u16:
3993 case Builtin::BIstdc_memreverse8u32:
3994 case Builtin::BIstdc_memreverse8u64:
3995 return RValue::get(
3996 V: emitBuiltinWithOneOverloadedType<1>(CGF&: *this, E, IntrinsicID: Intrinsic::bswap));
3997
3998 case Builtin::BIstdc_memreverse8:
3999 case Builtin::BI__builtin_stdc_memreverse8: {
4000 Expr::EvalResult R;
4001 if (E->getArg(Arg: 0)->EvaluateAsInt(Result&: R, Ctx: getContext())) {
4002 uint64_t Size = R.Val.getInt().getZExtValue();
4003 if (Size <= 1) {
4004 EmitIgnoredExpr(E: E->getArg(Arg: 1));
4005 return RValue::get(V: nullptr);
4006 }
4007 if (Size == 2 || Size == 4 || Size == 8) {
4008 llvm::Type *IntTy = Builder.getIntNTy(N: Size * 8);
4009 Address PtrAddr = EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
4010 Address Addr = PtrAddr.withElementType(ElemTy: IntTy);
4011 Value *Val = Builder.CreateLoad(Addr);
4012 Function *F = CGM.getIntrinsic(IID: Intrinsic::bswap, Tys: IntTy);
4013 Value *Swapped = Builder.CreateCall(Callee: F, Args: Val);
4014 Builder.CreateStore(Val: Swapped, Addr);
4015 return RValue::get(V: nullptr);
4016 }
4017 }
4018
4019 // General case: fall back to the library function stdc_memreverse8.
4020 break;
4021 }
4022
4023 case Builtin::BI__builtin_constant_p: {
4024 llvm::Type *ResultType = ConvertType(T: E->getType());
4025
4026 const Expr *Arg = E->getArg(Arg: 0);
4027 QualType ArgType = Arg->getType();
4028 // FIXME: The allowance for Obj-C pointers and block pointers is historical
4029 // and likely a mistake.
4030 if (!ArgType->isIntegralOrEnumerationType() && !ArgType->isFloatingType() &&
4031 !ArgType->isObjCObjectPointerType() && !ArgType->isBlockPointerType())
4032 // Per the GCC documentation, only numeric constants are recognized after
4033 // inlining.
4034 return RValue::get(V: ConstantInt::get(Ty: ResultType, V: 0));
4035
4036 if (Arg->HasSideEffects(Ctx: getContext()))
4037 // The argument is unevaluated, so be conservative if it might have
4038 // side-effects.
4039 return RValue::get(V: ConstantInt::get(Ty: ResultType, V: 0));
4040
4041 Value *ArgValue = EmitScalarExpr(E: Arg);
4042 if (ArgType->isObjCObjectPointerType()) {
4043 // Convert Objective-C objects to id because we cannot distinguish between
4044 // LLVM types for Obj-C classes as they are opaque.
4045 ArgType = CGM.getContext().getObjCIdType();
4046 ArgValue = Builder.CreateBitCast(V: ArgValue, DestTy: ConvertType(T: ArgType));
4047 }
4048 Function *F =
4049 CGM.getIntrinsic(IID: Intrinsic::is_constant, Tys: ConvertType(T: ArgType));
4050 Value *Result = Builder.CreateCall(Callee: F, Args: ArgValue);
4051 if (Result->getType() != ResultType)
4052 Result = Builder.CreateIntCast(V: Result, DestTy: ResultType, /*isSigned*/false);
4053 return RValue::get(V: Result);
4054 }
4055 case Builtin::BI__builtin_dynamic_object_size:
4056 case Builtin::BI__builtin_object_size: {
4057 unsigned Type =
4058 E->getArg(Arg: 1)->EvaluateKnownConstInt(Ctx: getContext()).getZExtValue();
4059 auto *ResType = cast<llvm::IntegerType>(Val: ConvertType(T: E->getType()));
4060
4061 // We pass this builtin onto the optimizer so that it can figure out the
4062 // object size in more complex cases.
4063 bool IsDynamic = BuiltinID == Builtin::BI__builtin_dynamic_object_size;
4064 return RValue::get(V: emitBuiltinObjectSize(E: E->getArg(Arg: 0), Type, ResType,
4065 /*EmittedE=*/nullptr, IsDynamic));
4066 }
4067 case Builtin::BI__builtin_counted_by_ref: {
4068 // Default to returning '(void *) 0'.
4069 llvm::Value *Result = llvm::ConstantPointerNull::get(
4070 T: llvm::PointerType::getUnqual(C&: getLLVMContext()));
4071
4072 const Expr *Arg = E->getArg(Arg: 0)->IgnoreParenImpCasts();
4073
4074 if (auto *UO = dyn_cast<UnaryOperator>(Val: Arg);
4075 UO && UO->getOpcode() == UO_AddrOf) {
4076 Arg = UO->getSubExpr()->IgnoreParenImpCasts();
4077
4078 if (auto *ASE = dyn_cast<ArraySubscriptExpr>(Val: Arg))
4079 Arg = ASE->getBase()->IgnoreParenImpCasts();
4080 }
4081
4082 if (const MemberExpr *ME = dyn_cast_if_present<MemberExpr>(Val: Arg)) {
4083 if (auto *CATy =
4084 ME->getMemberDecl()->getType()->getAs<CountAttributedType>();
4085 CATy && CATy->getKind() == CountAttributedType::CountedBy) {
4086 const auto *MemberDecl = cast<FieldDecl>(Val: ME->getMemberDecl());
4087 if (const FieldDecl *CountFD = MemberDecl->findCountedByField())
4088 Result = GetCountedByFieldExprGEP(Base: Arg, FD: MemberDecl, CountDecl: CountFD);
4089 else
4090 llvm::report_fatal_error(reason: "Cannot find the counted_by 'count' field");
4091 }
4092 }
4093
4094 return RValue::get(V: Result);
4095 }
4096 case Builtin::BI__builtin_prefetch: {
4097 Value *Locality, *RW, *Address = EmitScalarExpr(E: E->getArg(Arg: 0));
4098 unsigned ICEArguments = (1 << 1) | (1 << 2);
4099 // FIXME: Technically these constants should of type 'int', yes?
4100 RW = (E->getNumArgs() > 1) ? EmitScalarOrConstFoldImmArg(ICEArguments, Idx: 1, E)
4101 : llvm::ConstantInt::get(Ty: Int32Ty, V: 0);
4102 Locality = (E->getNumArgs() > 2)
4103 ? EmitScalarOrConstFoldImmArg(ICEArguments, Idx: 2, E)
4104 : llvm::ConstantInt::get(Ty: Int32Ty, V: 3);
4105 Value *Data = llvm::ConstantInt::get(Ty: Int32Ty, V: 1);
4106 Function *F = CGM.getIntrinsic(IID: Intrinsic::prefetch, Tys: Address->getType());
4107 Builder.CreateCall(Callee: F, Args: {Address, RW, Locality, Data});
4108 return RValue::get(V: nullptr);
4109 }
4110 case Builtin::BI__builtin_readcyclecounter: {
4111 Function *F = CGM.getIntrinsic(IID: Intrinsic::readcyclecounter);
4112 return RValue::get(V: Builder.CreateCall(Callee: F));
4113 }
4114 case Builtin::BI__builtin_readsteadycounter: {
4115 Function *F = CGM.getIntrinsic(IID: Intrinsic::readsteadycounter);
4116 return RValue::get(V: Builder.CreateCall(Callee: F));
4117 }
4118 case Builtin::BI__builtin___clear_cache: {
4119 Value *Begin = EmitScalarExpr(E: E->getArg(Arg: 0));
4120 Value *End = EmitScalarExpr(E: E->getArg(Arg: 1));
4121 Function *F = CGM.getIntrinsic(IID: Intrinsic::clear_cache, Tys: {CGM.DefaultPtrTy});
4122 return RValue::get(V: Builder.CreateCall(Callee: F, Args: {Begin, End}));
4123 }
4124 case Builtin::BI__builtin_trap:
4125 EmitTrapCall(IntrID: Intrinsic::trap);
4126 return RValue::get(V: nullptr);
4127 case Builtin::BI__builtin_verbose_trap: {
4128 llvm::DILocation *TrapLocation = Builder.getCurrentDebugLocation();
4129 if (getDebugInfo()) {
4130 TrapLocation = getDebugInfo()->CreateTrapFailureMessageFor(
4131 TrapLocation, Category: *E->getArg(Arg: 0)->tryEvaluateString(Ctx&: getContext()),
4132 FailureMsg: *E->getArg(Arg: 1)->tryEvaluateString(Ctx&: getContext()));
4133 }
4134 ApplyDebugLocation ApplyTrapDI(*this, TrapLocation);
4135 // Currently no attempt is made to prevent traps from being merged.
4136 EmitTrapCall(IntrID: Intrinsic::trap);
4137 return RValue::get(V: nullptr);
4138 }
4139 case Builtin::BI__debugbreak:
4140 EmitTrapCall(IntrID: Intrinsic::debugtrap);
4141 return RValue::get(V: nullptr);
4142 case Builtin::BI__builtin_unreachable: {
4143 EmitUnreachable(Loc: E->getExprLoc());
4144
4145 // We do need to preserve an insertion point.
4146 EmitBlock(BB: createBasicBlock(name: "unreachable.cont"));
4147
4148 return RValue::get(V: nullptr);
4149 }
4150
4151 case Builtin::BI__builtin_powi:
4152 case Builtin::BI__builtin_powif:
4153 case Builtin::BI__builtin_powil: {
4154 llvm::Value *Src0 = EmitScalarExpr(E: E->getArg(Arg: 0));
4155 llvm::Value *Src1 = EmitScalarExpr(E: E->getArg(Arg: 1));
4156
4157 if (Builder.getIsFPConstrained()) {
4158 // FIXME: llvm.powi has 2 mangling types,
4159 // llvm.experimental.constrained.powi has one.
4160 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
4161 Function *F = CGM.getIntrinsic(IID: Intrinsic::experimental_constrained_powi,
4162 Tys: Src0->getType());
4163 return RValue::get(V: Builder.CreateConstrainedFPCall(Callee: F, Args: { Src0, Src1 }));
4164 }
4165
4166 Function *F = CGM.getIntrinsic(IID: Intrinsic::powi,
4167 Tys: { Src0->getType(), Src1->getType() });
4168 return RValue::get(V: Builder.CreateCall(Callee: F, Args: { Src0, Src1 }));
4169 }
4170 case Builtin::BI__builtin_frexpl: {
4171 // Linux PPC will not be adding additional PPCDoubleDouble support.
4172 // WIP to switch default to IEEE long double. Will emit libcall for
4173 // frexpl instead of legalizing this type in the BE.
4174 if (&getTarget().getLongDoubleFormat() == &llvm::APFloat::PPCDoubleDouble())
4175 break;
4176 [[fallthrough]];
4177 }
4178 case Builtin::BI__builtin_frexp:
4179 case Builtin::BI__builtin_frexpf:
4180 case Builtin::BI__builtin_frexpf128:
4181 case Builtin::BI__builtin_frexpf16:
4182 return RValue::get(V: emitFrexpBuiltin(CGF&: *this, E, IntrinsicID: Intrinsic::frexp));
4183 case Builtin::BImodf:
4184 case Builtin::BImodff:
4185 case Builtin::BImodfl:
4186 case Builtin::BI__builtin_modf:
4187 case Builtin::BI__builtin_modff:
4188 case Builtin::BI__builtin_modfl:
4189 if (Builder.getIsFPConstrained())
4190 break; // TODO: Emit constrained modf intrinsic once one exists.
4191 return RValue::get(V: emitModfBuiltin(CGF&: *this, E, IntrinsicID: Intrinsic::modf));
4192 case Builtin::BI__builtin_isgreater:
4193 case Builtin::BI__builtin_isgreaterequal:
4194 case Builtin::BI__builtin_isless:
4195 case Builtin::BI__builtin_islessequal:
4196 case Builtin::BI__builtin_islessgreater:
4197 case Builtin::BI__builtin_isunordered: {
4198 // Ordered comparisons: we know the arguments to these are matching scalar
4199 // floating point values.
4200 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
4201 Value *LHS = EmitScalarExpr(E: E->getArg(Arg: 0));
4202 Value *RHS = EmitScalarExpr(E: E->getArg(Arg: 1));
4203
4204 switch (BuiltinID) {
4205 default: llvm_unreachable("Unknown ordered comparison");
4206 case Builtin::BI__builtin_isgreater:
4207 LHS = Builder.CreateFCmpOGT(LHS, RHS, Name: "cmp");
4208 break;
4209 case Builtin::BI__builtin_isgreaterequal:
4210 LHS = Builder.CreateFCmpOGE(LHS, RHS, Name: "cmp");
4211 break;
4212 case Builtin::BI__builtin_isless:
4213 LHS = Builder.CreateFCmpOLT(LHS, RHS, Name: "cmp");
4214 break;
4215 case Builtin::BI__builtin_islessequal:
4216 LHS = Builder.CreateFCmpOLE(LHS, RHS, Name: "cmp");
4217 break;
4218 case Builtin::BI__builtin_islessgreater:
4219 LHS = Builder.CreateFCmpONE(LHS, RHS, Name: "cmp");
4220 break;
4221 case Builtin::BI__builtin_isunordered:
4222 LHS = Builder.CreateFCmpUNO(LHS, RHS, Name: "cmp");
4223 break;
4224 }
4225 // ZExt bool to int type.
4226 return RValue::get(V: Builder.CreateZExt(V: LHS, DestTy: ConvertType(T: E->getType())));
4227 }
4228
4229 case Builtin::BI__builtin_isnan: {
4230 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
4231 Value *V = EmitScalarExpr(E: E->getArg(Arg: 0));
4232 if (Value *Result = tryUseTestFPKind(CGF&: *this, BuiltinID, V))
4233 return RValue::get(V: Result);
4234 return RValue::get(
4235 V: Builder.CreateZExt(V: Builder.createIsFPClass(FPNum: V, Test: FPClassTest::fcNan),
4236 DestTy: ConvertType(T: E->getType())));
4237 }
4238
4239 case Builtin::BI__builtin_issignaling: {
4240 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
4241 Value *V = EmitScalarExpr(E: E->getArg(Arg: 0));
4242 return RValue::get(
4243 V: Builder.CreateZExt(V: Builder.createIsFPClass(FPNum: V, Test: FPClassTest::fcSNan),
4244 DestTy: ConvertType(T: E->getType())));
4245 }
4246
4247 case Builtin::BI__builtin_isinf: {
4248 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
4249 Value *V = EmitScalarExpr(E: E->getArg(Arg: 0));
4250 if (Value *Result = tryUseTestFPKind(CGF&: *this, BuiltinID, V))
4251 return RValue::get(V: Result);
4252 return RValue::get(
4253 V: Builder.CreateZExt(V: Builder.createIsFPClass(FPNum: V, Test: FPClassTest::fcInf),
4254 DestTy: ConvertType(T: E->getType())));
4255 }
4256
4257 case Builtin::BIfinite:
4258 case Builtin::BI__finite:
4259 case Builtin::BIfinitef:
4260 case Builtin::BI__finitef:
4261 case Builtin::BIfinitel:
4262 case Builtin::BI__finitel:
4263 case Builtin::BI__builtin_isfinite: {
4264 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
4265 Value *V = EmitScalarExpr(E: E->getArg(Arg: 0));
4266 if (Value *Result = tryUseTestFPKind(CGF&: *this, BuiltinID, V))
4267 return RValue::get(V: Result);
4268 return RValue::get(
4269 V: Builder.CreateZExt(V: Builder.createIsFPClass(FPNum: V, Test: FPClassTest::fcFinite),
4270 DestTy: ConvertType(T: E->getType())));
4271 }
4272
4273 case Builtin::BI__builtin_isnormal: {
4274 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
4275 Value *V = EmitScalarExpr(E: E->getArg(Arg: 0));
4276 return RValue::get(
4277 V: Builder.CreateZExt(V: Builder.createIsFPClass(FPNum: V, Test: FPClassTest::fcNormal),
4278 DestTy: ConvertType(T: E->getType())));
4279 }
4280
4281 case Builtin::BI__builtin_issubnormal: {
4282 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
4283 Value *V = EmitScalarExpr(E: E->getArg(Arg: 0));
4284 return RValue::get(
4285 V: Builder.CreateZExt(V: Builder.createIsFPClass(FPNum: V, Test: FPClassTest::fcSubnormal),
4286 DestTy: ConvertType(T: E->getType())));
4287 }
4288
4289 case Builtin::BI__builtin_iszero: {
4290 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
4291 Value *V = EmitScalarExpr(E: E->getArg(Arg: 0));
4292 return RValue::get(
4293 V: Builder.CreateZExt(V: Builder.createIsFPClass(FPNum: V, Test: FPClassTest::fcZero),
4294 DestTy: ConvertType(T: E->getType())));
4295 }
4296
4297 case Builtin::BI__builtin_isfpclass: {
4298 Expr::EvalResult Result;
4299 if (!E->getArg(Arg: 1)->EvaluateAsInt(Result, Ctx: CGM.getContext()))
4300 break;
4301 uint64_t Test = Result.Val.getInt().getLimitedValue();
4302 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
4303 Value *V = EmitScalarExpr(E: E->getArg(Arg: 0));
4304 return RValue::get(V: Builder.CreateZExt(V: Builder.createIsFPClass(FPNum: V, Test),
4305 DestTy: ConvertType(T: E->getType())));
4306 }
4307
4308 case Builtin::BI__builtin_nondeterministic_value: {
4309 llvm::Type *Ty = ConvertType(T: E->getArg(Arg: 0)->getType());
4310
4311 Value *Result = PoisonValue::get(T: Ty);
4312 Result = Builder.CreateFreeze(V: Result);
4313
4314 return RValue::get(V: Result);
4315 }
4316
4317 case Builtin::BI__builtin_elementwise_abs: {
4318 Value *Result;
4319 QualType QT = E->getArg(Arg: 0)->getType();
4320
4321 if (auto *VecTy = QT->getAs<VectorType>())
4322 QT = VecTy->getElementType();
4323 if (QT->isIntegerType())
4324 Result = Builder.CreateBinaryIntrinsic(
4325 ID: Intrinsic::abs, LHS: EmitScalarExpr(E: E->getArg(Arg: 0)), RHS: Builder.getFalse(),
4326 FMFSource: nullptr, Name: "elt.abs");
4327 else
4328 Result = emitBuiltinWithOneOverloadedType<1>(CGF&: *this, E, IntrinsicID: Intrinsic::fabs,
4329 Name: "elt.abs");
4330
4331 return RValue::get(V: Result);
4332 }
4333 case Builtin::BI__builtin_elementwise_bitreverse:
4334 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4335 CGF&: *this, E, IntrinsicID: Intrinsic::bitreverse, Name: "elt.bitreverse"));
4336 case Builtin::BI__builtin_elementwise_popcount:
4337 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4338 CGF&: *this, E, IntrinsicID: Intrinsic::ctpop, Name: "elt.ctpop"));
4339 case Builtin::BI__builtin_elementwise_canonicalize:
4340 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4341 CGF&: *this, E, IntrinsicID: Intrinsic::canonicalize, Name: "elt.canonicalize"));
4342 case Builtin::BI__builtin_elementwise_copysign:
4343 return RValue::get(
4344 V: emitBuiltinWithOneOverloadedType<2>(CGF&: *this, E, IntrinsicID: Intrinsic::copysign));
4345 case Builtin::BI__builtin_elementwise_fshl:
4346 return RValue::get(
4347 V: emitBuiltinWithOneOverloadedType<3>(CGF&: *this, E, IntrinsicID: Intrinsic::fshl));
4348 case Builtin::BI__builtin_elementwise_fshr:
4349 return RValue::get(
4350 V: emitBuiltinWithOneOverloadedType<3>(CGF&: *this, E, IntrinsicID: Intrinsic::fshr));
4351 case Builtin::BI__builtin_elementwise_clmul:
4352 return RValue::get(
4353 V: emitBuiltinWithOneOverloadedType<2>(CGF&: *this, E, IntrinsicID: Intrinsic::clmul));
4354 case Builtin::BI__builtin_elementwise_pext:
4355 return RValue::get(
4356 V: emitBuiltinWithOneOverloadedType<2>(CGF&: *this, E, IntrinsicID: Intrinsic::pext));
4357 case Builtin::BI__builtin_elementwise_pdep:
4358 return RValue::get(
4359 V: emitBuiltinWithOneOverloadedType<2>(CGF&: *this, E, IntrinsicID: Intrinsic::pdep));
4360
4361 case Builtin::BI__builtin_elementwise_add_sat:
4362 case Builtin::BI__builtin_elementwise_sub_sat: {
4363 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
4364 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
4365 Value *Result;
4366 assert(Op0->getType()->isIntOrIntVectorTy() && "integer type expected");
4367 QualType Ty = E->getArg(Arg: 0)->getType();
4368 if (auto *VecTy = Ty->getAs<VectorType>())
4369 Ty = VecTy->getElementType();
4370 bool IsSigned = Ty->isSignedIntegerType();
4371 unsigned Opc;
4372 if (BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_elementwise_add_sat)
4373 Opc = IsSigned ? Intrinsic::sadd_sat : Intrinsic::uadd_sat;
4374 else
4375 Opc = IsSigned ? Intrinsic::ssub_sat : Intrinsic::usub_sat;
4376 Result = Builder.CreateBinaryIntrinsic(ID: Opc, LHS: Op0, RHS: Op1, FMFSource: nullptr, Name: "elt.sat");
4377 return RValue::get(V: Result);
4378 }
4379
4380 case Builtin::BI__builtin_elementwise_max: {
4381 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
4382 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
4383 Value *Result;
4384 if (Op0->getType()->isIntOrIntVectorTy()) {
4385 QualType Ty = E->getArg(Arg: 0)->getType();
4386 if (auto *VecTy = Ty->getAs<VectorType>())
4387 Ty = VecTy->getElementType();
4388 Result = Builder.CreateBinaryIntrinsic(
4389 ID: Ty->isSignedIntegerType() ? Intrinsic::smax : Intrinsic::umax, LHS: Op0,
4390 RHS: Op1, FMFSource: nullptr, Name: "elt.max");
4391 } else
4392 Result = Builder.CreateMaxNum(LHS: Op0, RHS: Op1, /*FMFSource=*/nullptr, Name: "elt.max");
4393 return RValue::get(V: Result);
4394 }
4395 case Builtin::BI__builtin_elementwise_min: {
4396 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
4397 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
4398 Value *Result;
4399 if (Op0->getType()->isIntOrIntVectorTy()) {
4400 QualType Ty = E->getArg(Arg: 0)->getType();
4401 if (auto *VecTy = Ty->getAs<VectorType>())
4402 Ty = VecTy->getElementType();
4403 Result = Builder.CreateBinaryIntrinsic(
4404 ID: Ty->isSignedIntegerType() ? Intrinsic::smin : Intrinsic::umin, LHS: Op0,
4405 RHS: Op1, FMFSource: nullptr, Name: "elt.min");
4406 } else
4407 Result = Builder.CreateMinNum(LHS: Op0, RHS: Op1, /*FMFSource=*/nullptr, Name: "elt.min");
4408 return RValue::get(V: Result);
4409 }
4410
4411 case Builtin::BI__builtin_elementwise_maxnum: {
4412 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
4413 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
4414 Value *Result = Builder.CreateBinaryIntrinsic(ID: llvm::Intrinsic::maxnum, LHS: Op0,
4415 RHS: Op1, FMFSource: nullptr, Name: "elt.maxnum");
4416 return RValue::get(V: Result);
4417 }
4418
4419 case Builtin::BI__builtin_elementwise_minnum: {
4420 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
4421 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
4422 Value *Result = Builder.CreateBinaryIntrinsic(ID: llvm::Intrinsic::minnum, LHS: Op0,
4423 RHS: Op1, FMFSource: nullptr, Name: "elt.minnum");
4424 return RValue::get(V: Result);
4425 }
4426
4427 case Builtin::BI__builtin_elementwise_maximum: {
4428 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
4429 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
4430 Value *Result = Builder.CreateBinaryIntrinsic(ID: Intrinsic::maximum, LHS: Op0, RHS: Op1,
4431 FMFSource: nullptr, Name: "elt.maximum");
4432 return RValue::get(V: Result);
4433 }
4434
4435 case Builtin::BI__builtin_elementwise_minimum: {
4436 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
4437 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
4438 Value *Result = Builder.CreateBinaryIntrinsic(ID: Intrinsic::minimum, LHS: Op0, RHS: Op1,
4439 FMFSource: nullptr, Name: "elt.minimum");
4440 return RValue::get(V: Result);
4441 }
4442
4443 case Builtin::BI__builtin_elementwise_maximumnum: {
4444 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
4445 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
4446 Value *Result = Builder.CreateBinaryIntrinsic(
4447 ID: Intrinsic::maximumnum, LHS: Op0, RHS: Op1, FMFSource: nullptr, Name: "elt.maximumnum");
4448 return RValue::get(V: Result);
4449 }
4450
4451 case Builtin::BI__builtin_elementwise_minimumnum: {
4452 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
4453 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
4454 Value *Result = Builder.CreateBinaryIntrinsic(
4455 ID: Intrinsic::minimumnum, LHS: Op0, RHS: Op1, FMFSource: nullptr, Name: "elt.minimumnum");
4456 return RValue::get(V: Result);
4457 }
4458
4459 case Builtin::BI__builtin_reduce_max: {
4460 auto GetIntrinsicID = [this](QualType QT) {
4461 if (auto *VecTy = QT->getAs<VectorType>())
4462 QT = VecTy->getElementType();
4463 else if (QT->isSizelessVectorType())
4464 QT = QT->getSizelessVectorEltType(Ctx: CGM.getContext());
4465
4466 if (QT->isSignedIntegerType())
4467 return Intrinsic::vector_reduce_smax;
4468 if (QT->isUnsignedIntegerType())
4469 return Intrinsic::vector_reduce_umax;
4470 assert(QT->isFloatingType() && "must have a float here");
4471 return Intrinsic::vector_reduce_fmax;
4472 };
4473 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4474 CGF&: *this, E, IntrinsicID: GetIntrinsicID(E->getArg(Arg: 0)->getType()), Name: "rdx.min"));
4475 }
4476
4477 case Builtin::BI__builtin_reduce_min: {
4478 auto GetIntrinsicID = [this](QualType QT) {
4479 if (auto *VecTy = QT->getAs<VectorType>())
4480 QT = VecTy->getElementType();
4481 else if (QT->isSizelessVectorType())
4482 QT = QT->getSizelessVectorEltType(Ctx: CGM.getContext());
4483
4484 if (QT->isSignedIntegerType())
4485 return Intrinsic::vector_reduce_smin;
4486 if (QT->isUnsignedIntegerType())
4487 return Intrinsic::vector_reduce_umin;
4488 assert(QT->isFloatingType() && "must have a float here");
4489 return Intrinsic::vector_reduce_fmin;
4490 };
4491
4492 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4493 CGF&: *this, E, IntrinsicID: GetIntrinsicID(E->getArg(Arg: 0)->getType()), Name: "rdx.min"));
4494 }
4495
4496 case Builtin::BI__builtin_reduce_add:
4497 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4498 CGF&: *this, E, IntrinsicID: Intrinsic::vector_reduce_add, Name: "rdx.add"));
4499 case Builtin::BI__builtin_reduce_mul:
4500 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4501 CGF&: *this, E, IntrinsicID: Intrinsic::vector_reduce_mul, Name: "rdx.mul"));
4502 case Builtin::BI__builtin_reduce_xor:
4503 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4504 CGF&: *this, E, IntrinsicID: Intrinsic::vector_reduce_xor, Name: "rdx.xor"));
4505 case Builtin::BI__builtin_reduce_or:
4506 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4507 CGF&: *this, E, IntrinsicID: Intrinsic::vector_reduce_or, Name: "rdx.or"));
4508 case Builtin::BI__builtin_reduce_and:
4509 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4510 CGF&: *this, E, IntrinsicID: Intrinsic::vector_reduce_and, Name: "rdx.and"));
4511 case Builtin::BI__builtin_reduce_maximum:
4512 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4513 CGF&: *this, E, IntrinsicID: Intrinsic::vector_reduce_fmaximum, Name: "rdx.maximum"));
4514 case Builtin::BI__builtin_reduce_minimum:
4515 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4516 CGF&: *this, E, IntrinsicID: Intrinsic::vector_reduce_fminimum, Name: "rdx.minimum"));
4517 case Builtin::BI__builtin_reduce_assoc_fadd:
4518 case Builtin::BI__builtin_reduce_in_order_fadd: {
4519 llvm::Value *Vector = EmitScalarExpr(E: E->getArg(Arg: 0));
4520 llvm::Type *ScalarTy = Vector->getType()->getScalarType();
4521 llvm::Value *StartValue = nullptr;
4522 if (E->getNumArgs() == 2)
4523 StartValue = Builder.CreateFPCast(V: EmitScalarExpr(E: E->getArg(Arg: 1)), DestTy: ScalarTy);
4524 llvm::Value *Args[] = {/*start_value=*/StartValue
4525 ? StartValue
4526 : llvm::ConstantFP::get(Ty: ScalarTy, V: -0.0F),
4527 /*vector=*/Vector};
4528 llvm::Function *F =
4529 CGM.getIntrinsic(IID: Intrinsic::vector_reduce_fadd, Tys: Vector->getType());
4530 llvm::CallBase *Reduce = Builder.CreateCall(Callee: F, Args, Name: "rdx.addf");
4531 if (BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_reduce_assoc_fadd) {
4532 // `__builtin_reduce_assoc_fadd` is an associative reduction which
4533 // requires the reassoc FMF flag.
4534 llvm::FastMathFlags FMF;
4535 FMF.setAllowReassoc();
4536 cast<llvm::CallBase>(Val: Reduce)->setFastMathFlags(FMF);
4537 }
4538 return RValue::get(V: Reduce);
4539 }
4540
4541 case Builtin::BI__builtin_matrix_transpose: {
4542 auto *MatrixTy = E->getArg(Arg: 0)->getType()->castAs<ConstantMatrixType>();
4543 Value *MatValue = EmitScalarExpr(E: E->getArg(Arg: 0));
4544 MatrixBuilder MB(Builder);
4545 Value *Result = MB.CreateMatrixTranspose(Matrix: MatValue, Rows: MatrixTy->getNumRows(),
4546 Columns: MatrixTy->getNumColumns());
4547 return RValue::get(V: Result);
4548 }
4549
4550 case Builtin::BI__builtin_matrix_column_major_load: {
4551 MatrixBuilder MB(Builder);
4552 // Emit everything that isn't dependent on the first parameter type
4553 Value *Stride = EmitScalarExpr(E: E->getArg(Arg: 3));
4554 const auto *ResultTy = E->getType()->getAs<ConstantMatrixType>();
4555 auto *PtrTy = E->getArg(Arg: 0)->getType()->getAs<PointerType>();
4556 assert(PtrTy && "arg0 must be of pointer type");
4557 bool IsVolatile = PtrTy->getPointeeType().isVolatileQualified();
4558
4559 Address Src = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4560 EmitNonNullArgCheck(RV: RValue::get(V: Src.emitRawPointer(CGF&: *this)),
4561 ArgType: E->getArg(Arg: 0)->getType(), ArgLoc: E->getArg(Arg: 0)->getExprLoc(), AC: FD,
4562 ParmNum: 0);
4563 Value *Result = MB.CreateColumnMajorLoad(
4564 EltTy: Src.getElementType(), DataPtr: Src.emitRawPointer(CGF&: *this),
4565 Alignment: Align(Src.getAlignment().getQuantity()), Stride, IsVolatile,
4566 Rows: ResultTy->getNumRows(), Columns: ResultTy->getNumColumns(), Name: "matrix");
4567 return RValue::get(V: Result);
4568 }
4569
4570 case Builtin::BI__builtin_matrix_column_major_store: {
4571 MatrixBuilder MB(Builder);
4572 Value *Matrix = EmitScalarExpr(E: E->getArg(Arg: 0));
4573 Address Dst = EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
4574 Value *Stride = EmitScalarExpr(E: E->getArg(Arg: 2));
4575
4576 const auto *MatrixTy = E->getArg(Arg: 0)->getType()->getAs<ConstantMatrixType>();
4577 auto *PtrTy = E->getArg(Arg: 1)->getType()->getAs<PointerType>();
4578 assert(PtrTy && "arg1 must be of pointer type");
4579 bool IsVolatile = PtrTy->getPointeeType().isVolatileQualified();
4580
4581 EmitNonNullArgCheck(RV: RValue::get(V: Dst.emitRawPointer(CGF&: *this)),
4582 ArgType: E->getArg(Arg: 1)->getType(), ArgLoc: E->getArg(Arg: 1)->getExprLoc(), AC: FD,
4583 ParmNum: 0);
4584 Value *Result = MB.CreateColumnMajorStore(
4585 Matrix, Ptr: Dst.emitRawPointer(CGF&: *this),
4586 Alignment: Align(Dst.getAlignment().getQuantity()), Stride, IsVolatile,
4587 Rows: MatrixTy->getNumRows(), Columns: MatrixTy->getNumColumns());
4588 addInstToNewSourceAtom(KeyInstruction: cast<Instruction>(Val: Result), Backup: Matrix);
4589 return RValue::get(V: Result);
4590 }
4591
4592 case Builtin::BI__builtin_masked_load:
4593 case Builtin::BI__builtin_masked_expand_load: {
4594 llvm::Value *Mask = EmitScalarExpr(E: E->getArg(Arg: 0));
4595 llvm::Value *Ptr = EmitScalarExpr(E: E->getArg(Arg: 1));
4596
4597 llvm::Type *RetTy = CGM.getTypes().ConvertType(T: E->getType());
4598 llvm::Value *PassThru = llvm::PoisonValue::get(T: RetTy);
4599 if (E->getNumArgs() > 2)
4600 PassThru = EmitScalarExpr(E: E->getArg(Arg: 2));
4601
4602 CharUnits Align = CGM.getNaturalTypeAlignment(
4603 T: E->getType()->getAs<VectorType>()->getElementType(), BaseInfo: nullptr);
4604
4605 llvm::Value *Result;
4606 if (BuiltinID == Builtin::BI__builtin_masked_load)
4607 Result = Builder.CreateMaskedLoad(Ty: RetTy, Ptr, Alignment: Align.getAsAlign(), Mask,
4608 PassThru, Name: "masked_load");
4609 else
4610 Result = Builder.CreateMaskedExpandLoad(Ty: RetTy, Ptr, Align: MaybeAlign(), Mask,
4611 PassThru, Name: "masked_expand_load");
4612
4613 return RValue::get(V: Result);
4614 };
4615 case Builtin::BI__builtin_masked_gather: {
4616 llvm::Value *Mask = EmitScalarExpr(E: E->getArg(Arg: 0));
4617 llvm::Value *Idx = EmitScalarExpr(E: E->getArg(Arg: 1));
4618 llvm::Value *Ptr = EmitScalarExpr(E: E->getArg(Arg: 2));
4619
4620 llvm::Type *RetTy = CGM.getTypes().ConvertType(T: E->getType());
4621 CharUnits Align = CGM.getNaturalTypeAlignment(
4622 T: E->getType()->getAs<VectorType>()->getElementType(), BaseInfo: nullptr);
4623
4624 llvm::Value *PassThru = llvm::PoisonValue::get(T: RetTy);
4625 if (E->getNumArgs() > 3)
4626 PassThru = EmitScalarExpr(E: E->getArg(Arg: 3));
4627
4628 llvm::Type *ElemTy = CGM.getTypes().ConvertType(
4629 T: E->getType()->getAs<VectorType>()->getElementType());
4630 llvm::Value *PtrVec = Builder.CreateGEP(Ty: ElemTy, Ptr, IdxList: Idx);
4631
4632 llvm::Value *Result = Builder.CreateMaskedGather(
4633 Ty: RetTy, Ptrs: PtrVec, Alignment: Align.getAsAlign(), Mask, PassThru, Name: "masked_gather");
4634 return RValue::get(V: Result);
4635 }
4636 case Builtin::BI__builtin_masked_store:
4637 case Builtin::BI__builtin_masked_compress_store: {
4638 llvm::Value *Mask = EmitScalarExpr(E: E->getArg(Arg: 0));
4639 llvm::Value *Val = EmitScalarExpr(E: E->getArg(Arg: 1));
4640 llvm::Value *Ptr = EmitScalarExpr(E: E->getArg(Arg: 2));
4641
4642 CharUnits Align = CGM.getNaturalTypeAlignment(
4643 T: E->getArg(Arg: 1)->getType()->getAs<VectorType>()->getElementType(),
4644 BaseInfo: nullptr);
4645
4646 if (BuiltinID == Builtin::BI__builtin_masked_store)
4647 Builder.CreateMaskedStore(Val, Ptr, Alignment: Align.getAsAlign(), Mask);
4648 else
4649 Builder.CreateMaskedCompressStore(Val, Ptr, Align: MaybeAlign(), Mask);
4650
4651 return RValue::get(V: nullptr);
4652 }
4653 case Builtin::BI__builtin_masked_scatter: {
4654 llvm::Value *Mask = EmitScalarExpr(E: E->getArg(Arg: 0));
4655 llvm::Value *Idx = EmitScalarExpr(E: E->getArg(Arg: 1));
4656 llvm::Value *Val = EmitScalarExpr(E: E->getArg(Arg: 2));
4657 llvm::Value *Ptr = EmitScalarExpr(E: E->getArg(Arg: 3));
4658
4659 CharUnits Align = CGM.getNaturalTypeAlignment(
4660 T: E->getArg(Arg: 2)->getType()->getAs<VectorType>()->getElementType(),
4661 BaseInfo: nullptr);
4662
4663 llvm::Type *ElemTy = CGM.getTypes().ConvertType(
4664 T: E->getArg(Arg: 1)->getType()->getAs<VectorType>()->getElementType());
4665 llvm::Value *PtrVec = Builder.CreateGEP(Ty: ElemTy, Ptr, IdxList: Idx);
4666
4667 Builder.CreateMaskedScatter(Val, Ptrs: PtrVec, Alignment: Align.getAsAlign(), Mask);
4668 return RValue();
4669 }
4670 case Builtin::BI__builtin_isinf_sign: {
4671 // isinf_sign(x) -> fabs(x) == infinity ? (signbit(x) ? -1 : 1) : 0
4672 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
4673 // FIXME: for strictfp/IEEE-754 we need to not trap on SNaN here.
4674 Value *Arg = EmitScalarExpr(E: E->getArg(Arg: 0));
4675 Value *AbsArg = EmitFAbs(CGF&: *this, V: Arg);
4676 Value *IsInf = Builder.CreateFCmpOEQ(
4677 LHS: AbsArg, RHS: ConstantFP::getInfinity(Ty: Arg->getType()), Name: "isinf");
4678 Value *IsNeg = EmitSignBit(CGF&: *this, V: Arg);
4679
4680 llvm::Type *IntTy = ConvertType(T: E->getType());
4681 Value *Zero = Constant::getNullValue(Ty: IntTy);
4682 Value *One = ConstantInt::get(Ty: IntTy, V: 1);
4683 Value *NegativeOne = ConstantInt::getAllOnesValue(Ty: IntTy);
4684 Value *SignResult = Builder.CreateSelect(C: IsNeg, True: NegativeOne, False: One);
4685 Value *Result = Builder.CreateSelect(C: IsInf, True: SignResult, False: Zero);
4686 return RValue::get(V: Result);
4687 }
4688
4689 case Builtin::BI__builtin_flt_rounds: {
4690 Function *F = CGM.getIntrinsic(IID: Intrinsic::get_rounding);
4691
4692 llvm::Type *ResultType = ConvertType(T: E->getType());
4693 Value *Result = Builder.CreateCall(Callee: F);
4694 if (Result->getType() != ResultType)
4695 Result = Builder.CreateIntCast(V: Result, DestTy: ResultType, /*isSigned*/true,
4696 Name: "cast");
4697 return RValue::get(V: Result);
4698 }
4699
4700 case Builtin::BI__builtin_set_flt_rounds: {
4701 Function *F = CGM.getIntrinsic(IID: Intrinsic::set_rounding);
4702
4703 Value *V = EmitScalarExpr(E: E->getArg(Arg: 0));
4704 Builder.CreateCall(Callee: F, Args: V);
4705 return RValue::get(V: nullptr);
4706 }
4707
4708 case Builtin::BI__builtin_fpclassify: {
4709 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
4710 // FIXME: for strictfp/IEEE-754 we need to not trap on SNaN here.
4711 Value *V = EmitScalarExpr(E: E->getArg(Arg: 5));
4712 llvm::Type *Ty = ConvertType(T: E->getArg(Arg: 5)->getType());
4713
4714 // Create Result
4715 BasicBlock *Begin = Builder.GetInsertBlock();
4716 BasicBlock *End = createBasicBlock(name: "fpclassify_end", parent: this->CurFn);
4717 Builder.SetInsertPoint(End);
4718 PHINode *Result =
4719 Builder.CreatePHI(Ty: ConvertType(T: E->getArg(Arg: 0)->getType()), NumReservedValues: 4,
4720 Name: "fpclassify_result");
4721
4722 // if (V==0) return FP_ZERO
4723 Builder.SetInsertPoint(Begin);
4724 Value *IsZero = Builder.CreateFCmpOEQ(LHS: V, RHS: Constant::getNullValue(Ty),
4725 Name: "iszero");
4726 Value *ZeroLiteral = EmitScalarExpr(E: E->getArg(Arg: 4));
4727 BasicBlock *NotZero = createBasicBlock(name: "fpclassify_not_zero", parent: this->CurFn);
4728 Builder.CreateCondBr(Cond: IsZero, True: End, False: NotZero);
4729 Result->addIncoming(V: ZeroLiteral, BB: Begin);
4730
4731 // if (V != V) return FP_NAN
4732 Builder.SetInsertPoint(NotZero);
4733 Value *IsNan = Builder.CreateFCmpUNO(LHS: V, RHS: V, Name: "cmp");
4734 Value *NanLiteral = EmitScalarExpr(E: E->getArg(Arg: 0));
4735 BasicBlock *NotNan = createBasicBlock(name: "fpclassify_not_nan", parent: this->CurFn);
4736 Builder.CreateCondBr(Cond: IsNan, True: End, False: NotNan);
4737 Result->addIncoming(V: NanLiteral, BB: NotZero);
4738
4739 // if (fabs(V) == infinity) return FP_INFINITY
4740 Builder.SetInsertPoint(NotNan);
4741 Value *VAbs = EmitFAbs(CGF&: *this, V);
4742 Value *IsInf =
4743 Builder.CreateFCmpOEQ(LHS: VAbs, RHS: ConstantFP::getInfinity(Ty: V->getType()),
4744 Name: "isinf");
4745 Value *InfLiteral = EmitScalarExpr(E: E->getArg(Arg: 1));
4746 BasicBlock *NotInf = createBasicBlock(name: "fpclassify_not_inf", parent: this->CurFn);
4747 Builder.CreateCondBr(Cond: IsInf, True: End, False: NotInf);
4748 Result->addIncoming(V: InfLiteral, BB: NotNan);
4749
4750 // if (fabs(V) >= MIN_NORMAL) return FP_NORMAL else FP_SUBNORMAL
4751 Builder.SetInsertPoint(NotInf);
4752 APFloat Smallest = APFloat::getSmallestNormalized(
4753 Sem: getContext().getFloatTypeSemantics(T: E->getArg(Arg: 5)->getType()));
4754 Value *IsNormal =
4755 Builder.CreateFCmpUGE(LHS: VAbs, RHS: ConstantFP::get(Context&: V->getContext(), V: Smallest),
4756 Name: "isnormal");
4757 Value *NormalResult =
4758 Builder.CreateSelect(C: IsNormal, True: EmitScalarExpr(E: E->getArg(Arg: 2)),
4759 False: EmitScalarExpr(E: E->getArg(Arg: 3)));
4760 Builder.CreateBr(Dest: End);
4761 Result->addIncoming(V: NormalResult, BB: NotInf);
4762
4763 // return Result
4764 Builder.SetInsertPoint(End);
4765 return RValue::get(V: Result);
4766 }
4767
4768 // An alloca will always return a pointer to the alloca (stack) address
4769 // space. This address space need not be the same as the AST / Language
4770 // default (e.g. in C / C++ auto vars are in the generic address space). At
4771 // the AST level this is handled within CreateTempAlloca et al., but for the
4772 // builtin / dynamic alloca we have to handle it here. We use an explicit cast
4773 // instead of passing an AS to CreateAlloca so as to not inhibit optimisation.
4774 case Builtin::BIalloca:
4775 case Builtin::BI_alloca:
4776 case Builtin::BI__builtin_alloca_uninitialized:
4777 case Builtin::BI__builtin_alloca: {
4778 Value *Size = EmitScalarExpr(E: E->getArg(Arg: 0));
4779 const TargetInfo &TI = getContext().getTargetInfo();
4780 // The alignment of the alloca should correspond to __BIGGEST_ALIGNMENT__.
4781 const Align SuitableAlignmentInBytes =
4782 CGM.getContext()
4783 .toCharUnitsFromBits(BitSize: TI.getSuitableAlign())
4784 .getAsAlign();
4785 AllocaInst *AI = Builder.CreateAlloca(Ty: Builder.getInt8Ty(), ArraySize: Size);
4786 AI->setAlignment(SuitableAlignmentInBytes);
4787 if (BuiltinID != Builtin::BI__builtin_alloca_uninitialized)
4788 initializeAlloca(CGF&: *this, AI, Size, AlignmentInBytes: SuitableAlignmentInBytes);
4789 if (AI->getAddressSpace() !=
4790 CGM.getContext().getTargetAddressSpace(
4791 AS: E->getType()->getPointeeType().getAddressSpace())) {
4792 llvm::Type *Ty = CGM.getTypes().ConvertType(T: E->getType());
4793 return RValue::get(V: performAddrSpaceCast(Src: AI, DestTy: Ty));
4794 }
4795 return RValue::get(V: AI);
4796 }
4797
4798 case Builtin::BI__builtin_alloca_with_align_uninitialized:
4799 case Builtin::BI__builtin_alloca_with_align: {
4800 Value *Size = EmitScalarExpr(E: E->getArg(Arg: 0));
4801 Value *AlignmentInBitsValue = EmitScalarExpr(E: E->getArg(Arg: 1));
4802 auto *AlignmentInBitsCI = cast<ConstantInt>(Val: AlignmentInBitsValue);
4803 unsigned AlignmentInBits = AlignmentInBitsCI->getZExtValue();
4804 const Align AlignmentInBytes =
4805 CGM.getContext().toCharUnitsFromBits(BitSize: AlignmentInBits).getAsAlign();
4806 AllocaInst *AI = Builder.CreateAlloca(Ty: Builder.getInt8Ty(), ArraySize: Size);
4807 AI->setAlignment(AlignmentInBytes);
4808 if (BuiltinID != Builtin::BI__builtin_alloca_with_align_uninitialized)
4809 initializeAlloca(CGF&: *this, AI, Size, AlignmentInBytes);
4810 if (AI->getAddressSpace() !=
4811 CGM.getContext().getTargetAddressSpace(
4812 AS: E->getType()->getPointeeType().getAddressSpace())) {
4813 llvm::Type *Ty = CGM.getTypes().ConvertType(T: E->getType());
4814 return RValue::get(V: performAddrSpaceCast(Src: AI, DestTy: Ty));
4815 }
4816 return RValue::get(V: AI);
4817 }
4818
4819 case Builtin::BI__builtin_infer_alloc_token: {
4820 llvm::MDNode *MDN = buildAllocToken(E);
4821 llvm::Value *MDV = MetadataAsValue::get(Context&: getLLVMContext(), MD: MDN);
4822 llvm::Function *F =
4823 CGM.getIntrinsic(IID: llvm::Intrinsic::alloc_token_id, Tys: {IntPtrTy});
4824 llvm::CallBase *TokenID = Builder.CreateCall(Callee: F, Args: MDV);
4825 return RValue::get(V: TokenID);
4826 }
4827
4828 case Builtin::BIbzero:
4829 case Builtin::BI__builtin_bzero: {
4830 Address Dest = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4831 Value *SizeVal = EmitScalarExpr(E: E->getArg(Arg: 1));
4832 EmitNonNullArgCheck(Addr: Dest, ArgType: E->getArg(Arg: 0)->getType(),
4833 ArgLoc: E->getArg(Arg: 0)->getExprLoc(), AC: FD, ParmNum: 0);
4834 auto *I = Builder.CreateMemSet(Dest, Value: Builder.getInt8(C: 0), Size: SizeVal, IsVolatile: false);
4835 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
4836 return RValue::get(V: nullptr);
4837 }
4838
4839 case Builtin::BIbcopy:
4840 case Builtin::BI__builtin_bcopy: {
4841 Address Src = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4842 Address Dest = EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
4843 Value *SizeVal = EmitScalarExpr(E: E->getArg(Arg: 2));
4844 EmitNonNullArgCheck(RV: RValue::get(V: Src.emitRawPointer(CGF&: *this)),
4845 ArgType: E->getArg(Arg: 0)->getType(), ArgLoc: E->getArg(Arg: 0)->getExprLoc(), AC: FD,
4846 ParmNum: 0);
4847 EmitNonNullArgCheck(RV: RValue::get(V: Dest.emitRawPointer(CGF&: *this)),
4848 ArgType: E->getArg(Arg: 1)->getType(), ArgLoc: E->getArg(Arg: 1)->getExprLoc(), AC: FD,
4849 ParmNum: 0);
4850 auto *I = Builder.CreateMemMove(Dest, Src, Size: SizeVal, IsVolatile: false);
4851 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
4852 return RValue::get(V: nullptr);
4853 }
4854
4855 case Builtin::BImemcpy:
4856 case Builtin::BI__builtin_memcpy:
4857 case Builtin::BImempcpy:
4858 case Builtin::BI__builtin_mempcpy: {
4859 Address Dest = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4860 Address Src = EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
4861 Value *SizeVal = EmitScalarExpr(E: E->getArg(Arg: 2));
4862 EmitArgCheck(TCK_Store, Dest, E->getArg(Arg: 0), 0);
4863 EmitArgCheck(TCK_Load, Src, E->getArg(Arg: 1), 1);
4864 auto *I = Builder.CreateMemCpy(Dest, Src, Size: SizeVal, IsVolatile: false);
4865 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
4866 if (BuiltinID == Builtin::BImempcpy ||
4867 BuiltinID == Builtin::BI__builtin_mempcpy)
4868 return RValue::get(V: Builder.CreateInBoundsGEP(
4869 Ty: Dest.getElementType(), Ptr: Dest.emitRawPointer(CGF&: *this), IdxList: SizeVal));
4870 else
4871 return RValue::get(Addr: Dest, CGF&: *this);
4872 }
4873
4874 case Builtin::BI__builtin_memcpy_inline: {
4875 Address Dest = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4876 Address Src = EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
4877 uint64_t Size =
4878 E->getArg(Arg: 2)->EvaluateKnownConstInt(Ctx: getContext()).getZExtValue();
4879 EmitArgCheck(TCK_Store, Dest, E->getArg(Arg: 0), 0);
4880 EmitArgCheck(TCK_Load, Src, E->getArg(Arg: 1), 1);
4881 auto *I = Builder.CreateMemCpyInline(Dest, Src, Size);
4882 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
4883 return RValue::get(V: nullptr);
4884 }
4885
4886 case Builtin::BI__builtin_char_memchr:
4887 BuiltinID = Builtin::BI__builtin_memchr;
4888 break;
4889
4890 case Builtin::BI__builtin___memcpy_chk: {
4891 // fold __builtin_memcpy_chk(x, y, cst1, cst2) to memcpy iff cst1<=cst2.
4892 Expr::EvalResult SizeResult, DstSizeResult;
4893 if (!E->getArg(Arg: 2)->EvaluateAsInt(Result&: SizeResult, Ctx: CGM.getContext()) ||
4894 !E->getArg(Arg: 3)->EvaluateAsInt(Result&: DstSizeResult, Ctx: CGM.getContext()))
4895 break;
4896 llvm::APSInt Size = SizeResult.Val.getInt();
4897 llvm::APSInt DstSize = DstSizeResult.Val.getInt();
4898 if (Size.ugt(RHS: DstSize))
4899 break;
4900 Address Dest = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4901 Address Src = EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
4902 Value *SizeVal = llvm::ConstantInt::get(Context&: Builder.getContext(), V: Size);
4903 auto *I = Builder.CreateMemCpy(Dest, Src, Size: SizeVal, IsVolatile: false);
4904 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
4905 return RValue::get(Addr: Dest, CGF&: *this);
4906 }
4907
4908 case Builtin::BI__builtin_objc_memmove_collectable: {
4909 Address DestAddr = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4910 Address SrcAddr = EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
4911 Value *SizeVal = EmitScalarExpr(E: E->getArg(Arg: 2));
4912 CGM.getObjCRuntime().EmitGCMemmoveCollectable(CGF&: *this,
4913 DestPtr: DestAddr, SrcPtr: SrcAddr, Size: SizeVal);
4914 return RValue::get(Addr: DestAddr, CGF&: *this);
4915 }
4916
4917 case Builtin::BI__builtin___memmove_chk: {
4918 // fold __builtin_memmove_chk(x, y, cst1, cst2) to memmove iff cst1<=cst2.
4919 Expr::EvalResult SizeResult, DstSizeResult;
4920 if (!E->getArg(Arg: 2)->EvaluateAsInt(Result&: SizeResult, Ctx: CGM.getContext()) ||
4921 !E->getArg(Arg: 3)->EvaluateAsInt(Result&: DstSizeResult, Ctx: CGM.getContext()))
4922 break;
4923 llvm::APSInt Size = SizeResult.Val.getInt();
4924 llvm::APSInt DstSize = DstSizeResult.Val.getInt();
4925 if (Size.ugt(RHS: DstSize))
4926 break;
4927 Address Dest = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4928 Address Src = EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
4929 Value *SizeVal = llvm::ConstantInt::get(Context&: Builder.getContext(), V: Size);
4930 auto *I = Builder.CreateMemMove(Dest, Src, Size: SizeVal, IsVolatile: false);
4931 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
4932 return RValue::get(Addr: Dest, CGF&: *this);
4933 }
4934
4935 case Builtin::BI__builtin_trivially_relocate:
4936 case Builtin::BImemmove:
4937 case Builtin::BI__builtin_memmove: {
4938 Address Dest = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4939 Address Src = EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
4940 Value *SizeVal = EmitScalarExpr(E: E->getArg(Arg: 2));
4941 if (BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_trivially_relocate)
4942 SizeVal = Builder.CreateMul(
4943 LHS: SizeVal,
4944 RHS: ConstantInt::get(
4945 Ty: SizeVal->getType(),
4946 V: getContext()
4947 .getTypeSizeInChars(T: E->getArg(Arg: 0)->getType()->getPointeeType())
4948 .getQuantity()));
4949 EmitArgCheck(TCK_Store, Dest, E->getArg(Arg: 0), 0);
4950 EmitArgCheck(TCK_Load, Src, E->getArg(Arg: 1), 1);
4951 auto *I = Builder.CreateMemMove(Dest, Src, Size: SizeVal, IsVolatile: false);
4952 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
4953 return RValue::get(Addr: Dest, CGF&: *this);
4954 }
4955 case Builtin::BImemset:
4956 case Builtin::BI__builtin_memset: {
4957 Address Dest = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4958 Value *ByteVal = Builder.CreateTrunc(V: EmitScalarExpr(E: E->getArg(Arg: 1)),
4959 DestTy: Builder.getInt8Ty());
4960 Value *SizeVal = EmitScalarExpr(E: E->getArg(Arg: 2));
4961 EmitNonNullArgCheck(Addr: Dest, ArgType: E->getArg(Arg: 0)->getType(),
4962 ArgLoc: E->getArg(Arg: 0)->getExprLoc(), AC: FD, ParmNum: 0);
4963 auto *I = Builder.CreateMemSet(Dest, Value: ByteVal, Size: SizeVal, IsVolatile: false);
4964 addInstToNewSourceAtom(KeyInstruction: I, Backup: ByteVal);
4965 return RValue::get(Addr: Dest, CGF&: *this);
4966 }
4967 case Builtin::BI__builtin_memset_inline: {
4968 Address Dest = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4969 Value *ByteVal =
4970 Builder.CreateTrunc(V: EmitScalarExpr(E: E->getArg(Arg: 1)), DestTy: Builder.getInt8Ty());
4971 uint64_t Size =
4972 E->getArg(Arg: 2)->EvaluateKnownConstInt(Ctx: getContext()).getZExtValue();
4973 EmitNonNullArgCheck(RV: RValue::get(V: Dest.emitRawPointer(CGF&: *this)),
4974 ArgType: E->getArg(Arg: 0)->getType(), ArgLoc: E->getArg(Arg: 0)->getExprLoc(), AC: FD,
4975 ParmNum: 0);
4976 auto *I = Builder.CreateMemSetInline(Dest, Value: ByteVal, Size);
4977 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
4978 return RValue::get(V: nullptr);
4979 }
4980 case Builtin::BI__builtin___memset_chk: {
4981 // fold __builtin_memset_chk(x, y, cst1, cst2) to memset iff cst1<=cst2.
4982 Expr::EvalResult SizeResult, DstSizeResult;
4983 if (!E->getArg(Arg: 2)->EvaluateAsInt(Result&: SizeResult, Ctx: CGM.getContext()) ||
4984 !E->getArg(Arg: 3)->EvaluateAsInt(Result&: DstSizeResult, Ctx: CGM.getContext()))
4985 break;
4986 llvm::APSInt Size = SizeResult.Val.getInt();
4987 llvm::APSInt DstSize = DstSizeResult.Val.getInt();
4988 if (Size.ugt(RHS: DstSize))
4989 break;
4990 Address Dest = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4991 Value *ByteVal = Builder.CreateTrunc(V: EmitScalarExpr(E: E->getArg(Arg: 1)),
4992 DestTy: Builder.getInt8Ty());
4993 Value *SizeVal = llvm::ConstantInt::get(Context&: Builder.getContext(), V: Size);
4994 auto *I = Builder.CreateMemSet(Dest, Value: ByteVal, Size: SizeVal, IsVolatile: false);
4995 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
4996 return RValue::get(Addr: Dest, CGF&: *this);
4997 }
4998 case Builtin::BI__builtin_wmemchr: {
4999 // The MSVC runtime library does not provide a definition of wmemchr, so we
5000 // need an inline implementation.
5001 if (!getTarget().getTriple().isOSMSVCRT())
5002 break;
5003
5004 llvm::Type *WCharTy = ConvertType(T: getContext().WCharTy);
5005 Value *Str = EmitScalarExpr(E: E->getArg(Arg: 0));
5006 Value *Chr = EmitScalarExpr(E: E->getArg(Arg: 1));
5007 Value *Size = EmitScalarExpr(E: E->getArg(Arg: 2));
5008
5009 BasicBlock *Entry = Builder.GetInsertBlock();
5010 BasicBlock *CmpEq = createBasicBlock(name: "wmemchr.eq");
5011 BasicBlock *Next = createBasicBlock(name: "wmemchr.next");
5012 BasicBlock *Exit = createBasicBlock(name: "wmemchr.exit");
5013 Value *SizeEq0 = Builder.CreateICmpEQ(LHS: Size, RHS: ConstantInt::get(Ty: SizeTy, V: 0));
5014 Builder.CreateCondBr(Cond: SizeEq0, True: Exit, False: CmpEq);
5015
5016 EmitBlock(BB: CmpEq);
5017 PHINode *StrPhi = Builder.CreatePHI(Ty: Str->getType(), NumReservedValues: 2);
5018 StrPhi->addIncoming(V: Str, BB: Entry);
5019 PHINode *SizePhi = Builder.CreatePHI(Ty: SizeTy, NumReservedValues: 2);
5020 SizePhi->addIncoming(V: Size, BB: Entry);
5021 CharUnits WCharAlign =
5022 getContext().getTypeAlignInChars(T: getContext().WCharTy);
5023 Value *StrCh = Builder.CreateAlignedLoad(Ty: WCharTy, Addr: StrPhi, Align: WCharAlign);
5024 Value *FoundChr = Builder.CreateConstInBoundsGEP1_32(Ty: WCharTy, Ptr: StrPhi, Idx0: 0);
5025 Value *StrEqChr = Builder.CreateICmpEQ(LHS: StrCh, RHS: Chr);
5026 Builder.CreateCondBr(Cond: StrEqChr, True: Exit, False: Next);
5027
5028 EmitBlock(BB: Next);
5029 Value *NextStr = Builder.CreateConstInBoundsGEP1_32(Ty: WCharTy, Ptr: StrPhi, Idx0: 1);
5030 Value *NextSize = Builder.CreateSub(LHS: SizePhi, RHS: ConstantInt::get(Ty: SizeTy, V: 1));
5031 Value *NextSizeEq0 =
5032 Builder.CreateICmpEQ(LHS: NextSize, RHS: ConstantInt::get(Ty: SizeTy, V: 0));
5033 Builder.CreateCondBr(Cond: NextSizeEq0, True: Exit, False: CmpEq);
5034 StrPhi->addIncoming(V: NextStr, BB: Next);
5035 SizePhi->addIncoming(V: NextSize, BB: Next);
5036
5037 EmitBlock(BB: Exit);
5038 PHINode *Ret = Builder.CreatePHI(Ty: Str->getType(), NumReservedValues: 3);
5039 Ret->addIncoming(V: llvm::Constant::getNullValue(Ty: Str->getType()), BB: Entry);
5040 Ret->addIncoming(V: llvm::Constant::getNullValue(Ty: Str->getType()), BB: Next);
5041 Ret->addIncoming(V: FoundChr, BB: CmpEq);
5042 return RValue::get(V: Ret);
5043 }
5044 case Builtin::BI__builtin_wmemcmp: {
5045 // The MSVC runtime library does not provide a definition of wmemcmp, so we
5046 // need an inline implementation.
5047 if (!getTarget().getTriple().isOSMSVCRT())
5048 break;
5049
5050 llvm::Type *WCharTy = ConvertType(T: getContext().WCharTy);
5051
5052 Value *Dst = EmitScalarExpr(E: E->getArg(Arg: 0));
5053 Value *Src = EmitScalarExpr(E: E->getArg(Arg: 1));
5054 Value *Size = EmitScalarExpr(E: E->getArg(Arg: 2));
5055
5056 BasicBlock *Entry = Builder.GetInsertBlock();
5057 BasicBlock *CmpGT = createBasicBlock(name: "wmemcmp.gt");
5058 BasicBlock *CmpLT = createBasicBlock(name: "wmemcmp.lt");
5059 BasicBlock *Next = createBasicBlock(name: "wmemcmp.next");
5060 BasicBlock *Exit = createBasicBlock(name: "wmemcmp.exit");
5061 Value *SizeEq0 = Builder.CreateICmpEQ(LHS: Size, RHS: ConstantInt::get(Ty: SizeTy, V: 0));
5062 Builder.CreateCondBr(Cond: SizeEq0, True: Exit, False: CmpGT);
5063
5064 EmitBlock(BB: CmpGT);
5065 PHINode *DstPhi = Builder.CreatePHI(Ty: Dst->getType(), NumReservedValues: 2);
5066 DstPhi->addIncoming(V: Dst, BB: Entry);
5067 PHINode *SrcPhi = Builder.CreatePHI(Ty: Src->getType(), NumReservedValues: 2);
5068 SrcPhi->addIncoming(V: Src, BB: Entry);
5069 PHINode *SizePhi = Builder.CreatePHI(Ty: SizeTy, NumReservedValues: 2);
5070 SizePhi->addIncoming(V: Size, BB: Entry);
5071 CharUnits WCharAlign =
5072 getContext().getTypeAlignInChars(T: getContext().WCharTy);
5073 Value *DstCh = Builder.CreateAlignedLoad(Ty: WCharTy, Addr: DstPhi, Align: WCharAlign);
5074 Value *SrcCh = Builder.CreateAlignedLoad(Ty: WCharTy, Addr: SrcPhi, Align: WCharAlign);
5075 Value *DstGtSrc = Builder.CreateICmpUGT(LHS: DstCh, RHS: SrcCh);
5076 Builder.CreateCondBr(Cond: DstGtSrc, True: Exit, False: CmpLT);
5077
5078 EmitBlock(BB: CmpLT);
5079 Value *DstLtSrc = Builder.CreateICmpULT(LHS: DstCh, RHS: SrcCh);
5080 Builder.CreateCondBr(Cond: DstLtSrc, True: Exit, False: Next);
5081
5082 EmitBlock(BB: Next);
5083 Value *NextDst = Builder.CreateConstInBoundsGEP1_32(Ty: WCharTy, Ptr: DstPhi, Idx0: 1);
5084 Value *NextSrc = Builder.CreateConstInBoundsGEP1_32(Ty: WCharTy, Ptr: SrcPhi, Idx0: 1);
5085 Value *NextSize = Builder.CreateSub(LHS: SizePhi, RHS: ConstantInt::get(Ty: SizeTy, V: 1));
5086 Value *NextSizeEq0 =
5087 Builder.CreateICmpEQ(LHS: NextSize, RHS: ConstantInt::get(Ty: SizeTy, V: 0));
5088 Builder.CreateCondBr(Cond: NextSizeEq0, True: Exit, False: CmpGT);
5089 DstPhi->addIncoming(V: NextDst, BB: Next);
5090 SrcPhi->addIncoming(V: NextSrc, BB: Next);
5091 SizePhi->addIncoming(V: NextSize, BB: Next);
5092
5093 EmitBlock(BB: Exit);
5094 PHINode *Ret = Builder.CreatePHI(Ty: IntTy, NumReservedValues: 4);
5095 Ret->addIncoming(V: ConstantInt::get(Ty: IntTy, V: 0), BB: Entry);
5096 Ret->addIncoming(V: ConstantInt::get(Ty: IntTy, V: 1), BB: CmpGT);
5097 Ret->addIncoming(V: ConstantInt::getAllOnesValue(Ty: IntTy), BB: CmpLT);
5098 Ret->addIncoming(V: ConstantInt::get(Ty: IntTy, V: 0), BB: Next);
5099 return RValue::get(V: Ret);
5100 }
5101 case Builtin::BI__builtin_dwarf_cfa: {
5102 // The offset in bytes from the first argument to the CFA.
5103 //
5104 // Why on earth is this in the frontend? Is there any reason at
5105 // all that the backend can't reasonably determine this while
5106 // lowering llvm.eh.dwarf.cfa()?
5107 //
5108 // TODO: If there's a satisfactory reason, add a target hook for
5109 // this instead of hard-coding 0, which is correct for most targets.
5110 int32_t Offset = 0;
5111
5112 Function *F = CGM.getIntrinsic(IID: Intrinsic::eh_dwarf_cfa);
5113 return RValue::get(V: Builder.CreateCall(Callee: F,
5114 Args: llvm::ConstantInt::get(Ty: Int32Ty, V: Offset)));
5115 }
5116 case Builtin::BI__builtin_return_address: {
5117 Value *Depth = ConstantEmitter(*this).emitAbstract(E: E->getArg(Arg: 0),
5118 T: getContext().UnsignedIntTy);
5119 Function *F =
5120 CGM.getIntrinsic(IID: Intrinsic::returnaddress, Tys: {CGM.ProgramPtrTy});
5121 return RValue::get(V: Builder.CreateCall(Callee: F, Args: Depth));
5122 }
5123 case Builtin::BI_ReturnAddress: {
5124 Function *F =
5125 CGM.getIntrinsic(IID: Intrinsic::returnaddress, Tys: {CGM.ProgramPtrTy});
5126 return RValue::get(V: Builder.CreateCall(Callee: F, Args: Builder.getInt32(C: 0)));
5127 }
5128 case Builtin::BI__builtin_frame_address: {
5129 Value *Depth = ConstantEmitter(*this).emitAbstract(E: E->getArg(Arg: 0),
5130 T: getContext().UnsignedIntTy);
5131 Function *F = CGM.getIntrinsic(IID: Intrinsic::frameaddress, Tys: AllocaInt8PtrTy);
5132 return RValue::get(V: Builder.CreateCall(Callee: F, Args: Depth));
5133 }
5134 case Builtin::BI__builtin_stack_address: {
5135 return RValue::get(V: Builder.CreateCall(
5136 Callee: CGM.getIntrinsic(IID: Intrinsic::stackaddress, Tys: AllocaInt8PtrTy)));
5137 }
5138 case Builtin::BI__builtin_extract_return_addr: {
5139 Value *Address = EmitScalarExpr(E: E->getArg(Arg: 0));
5140 Value *Result = getTargetHooks().decodeReturnAddress(CGF&: *this, Address);
5141 return RValue::get(V: Result);
5142 }
5143 case Builtin::BI__builtin_frob_return_addr: {
5144 Value *Address = EmitScalarExpr(E: E->getArg(Arg: 0));
5145 Value *Result = getTargetHooks().encodeReturnAddress(CGF&: *this, Address);
5146 return RValue::get(V: Result);
5147 }
5148 case Builtin::BI__builtin_dwarf_sp_column: {
5149 llvm::IntegerType *Ty
5150 = cast<llvm::IntegerType>(Val: ConvertType(T: E->getType()));
5151 int Column = getTargetHooks().getDwarfEHStackPointer(M&: CGM);
5152 if (Column == -1) {
5153 CGM.ErrorUnsupported(S: E, Type: "__builtin_dwarf_sp_column");
5154 return RValue::get(V: llvm::UndefValue::get(T: Ty));
5155 }
5156 return RValue::get(V: llvm::ConstantInt::get(Ty, V: Column, IsSigned: true));
5157 }
5158 case Builtin::BI__builtin_init_dwarf_reg_size_table: {
5159 Value *Address = EmitScalarExpr(E: E->getArg(Arg: 0));
5160 if (getTargetHooks().initDwarfEHRegSizeTable(CGF&: *this, Address))
5161 CGM.ErrorUnsupported(S: E, Type: "__builtin_init_dwarf_reg_size_table");
5162 return RValue::get(V: llvm::UndefValue::get(T: ConvertType(T: E->getType())));
5163 }
5164 case Builtin::BI__builtin_eh_return: {
5165 Value *Int = EmitScalarExpr(E: E->getArg(Arg: 0));
5166 Value *Ptr = EmitScalarExpr(E: E->getArg(Arg: 1));
5167
5168 llvm::IntegerType *IntTy = cast<llvm::IntegerType>(Val: Int->getType());
5169 assert((IntTy->getBitWidth() == 32 || IntTy->getBitWidth() == 64) &&
5170 "LLVM's __builtin_eh_return only supports 32- and 64-bit variants");
5171 Function *F =
5172 CGM.getIntrinsic(IID: IntTy->getBitWidth() == 32 ? Intrinsic::eh_return_i32
5173 : Intrinsic::eh_return_i64);
5174 Builder.CreateCall(Callee: F, Args: {Int, Ptr});
5175 Builder.CreateUnreachable();
5176
5177 // We do need to preserve an insertion point.
5178 EmitBlock(BB: createBasicBlock(name: "builtin_eh_return.cont"));
5179
5180 return RValue::get(V: nullptr);
5181 }
5182 case Builtin::BI__builtin_unwind_init: {
5183 Function *F = CGM.getIntrinsic(IID: Intrinsic::eh_unwind_init);
5184 Builder.CreateCall(Callee: F);
5185 return RValue::get(V: nullptr);
5186 }
5187 case Builtin::BI__builtin_extend_pointer: {
5188 // Extends a pointer to the size of an _Unwind_Word, which is
5189 // uint64_t on all platforms. Generally this gets poked into a
5190 // register and eventually used as an address, so if the
5191 // addressing registers are wider than pointers and the platform
5192 // doesn't implicitly ignore high-order bits when doing
5193 // addressing, we need to make sure we zext / sext based on
5194 // the platform's expectations.
5195 //
5196 // See: http://gcc.gnu.org/ml/gcc-bugs/2002-02/msg00237.html
5197
5198 // Cast the pointer to intptr_t.
5199 Value *Ptr = EmitScalarExpr(E: E->getArg(Arg: 0));
5200 Value *Result = Builder.CreatePtrToInt(V: Ptr, DestTy: IntPtrTy, Name: "extend.cast");
5201
5202 // If that's 64 bits, we're done.
5203 if (IntPtrTy->getBitWidth() == 64)
5204 return RValue::get(V: Result);
5205
5206 // Otherwise, ask the codegen data what to do.
5207 if (getTargetHooks().extendPointerWithSExt())
5208 return RValue::get(V: Builder.CreateSExt(V: Result, DestTy: Int64Ty, Name: "extend.sext"));
5209 else
5210 return RValue::get(V: Builder.CreateZExt(V: Result, DestTy: Int64Ty, Name: "extend.zext"));
5211 }
5212 case Builtin::BI__builtin_setjmp: {
5213 // Buffer is a void**.
5214 Address Buf = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
5215
5216 if (getTarget().getTriple().getArch() == llvm::Triple::systemz) {
5217 // On this target, the back end fills in the context buffer completely.
5218 // It doesn't really matter if the frontend stores to the buffer before
5219 // calling setjmp, the back-end is going to overwrite them anyway.
5220 Function *F = CGM.getIntrinsic(IID: Intrinsic::eh_sjlj_setjmp);
5221 return RValue::get(V: Builder.CreateCall(Callee: F, Args: Buf.emitRawPointer(CGF&: *this)));
5222 }
5223
5224 // Store the frame pointer to the setjmp buffer.
5225 Value *FrameAddr = Builder.CreateCall(
5226 Callee: CGM.getIntrinsic(IID: Intrinsic::frameaddress, Tys: AllocaInt8PtrTy),
5227 Args: ConstantInt::get(Ty: Int32Ty, V: 0));
5228 Builder.CreateStore(Val: FrameAddr, Addr: Buf);
5229
5230 // Store the stack pointer to the setjmp buffer.
5231 Value *StackAddr = Builder.CreateStackSave();
5232 assert(Buf.emitRawPointer(*this)->getType() == StackAddr->getType());
5233
5234 Address StackSaveSlot = Builder.CreateConstInBoundsGEP(Addr: Buf, Index: 2);
5235 Builder.CreateStore(Val: StackAddr, Addr: StackSaveSlot);
5236
5237 // Call LLVM's EH setjmp, which is lightweight.
5238 Function *F = CGM.getIntrinsic(IID: Intrinsic::eh_sjlj_setjmp);
5239 return RValue::get(V: Builder.CreateCall(Callee: F, Args: Buf.emitRawPointer(CGF&: *this)));
5240 }
5241 case Builtin::BI__builtin_longjmp: {
5242 Value *Buf = EmitScalarExpr(E: E->getArg(Arg: 0));
5243
5244 // Call LLVM's EH longjmp, which is lightweight.
5245 Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Intrinsic::eh_sjlj_longjmp), Args: Buf);
5246
5247 // longjmp doesn't return; mark this as unreachable.
5248 Builder.CreateUnreachable();
5249
5250 // We do need to preserve an insertion point.
5251 EmitBlock(BB: createBasicBlock(name: "longjmp.cont"));
5252
5253 return RValue::get(V: nullptr);
5254 }
5255 case Builtin::BI__builtin_launder: {
5256 const Expr *Arg = E->getArg(Arg: 0);
5257 QualType ArgTy = Arg->getType()->getPointeeType();
5258 Value *Ptr = EmitScalarExpr(E: Arg);
5259 if (TypeRequiresBuiltinLaunder(CGM, Ty: ArgTy))
5260 Ptr = Builder.CreateLaunderInvariantGroup(Ptr);
5261
5262 return RValue::get(V: Ptr);
5263 }
5264 case Builtin::BI__builtin_clear_padding: {
5265 Address Src = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
5266 auto PointeeTy = E->getArg(Arg: 0)->getType()->getPointeeType();
5267
5268 llvm::ArrayRef<ASTContext::BitInterval> Padding =
5269 getContext().getPaddingIntervals(Ty: PointeeTy);
5270 for (const auto &Interval : Padding)
5271 ClearPadding(CGF&: *this, Src, PaddingInterval: Interval);
5272
5273 return RValue::get(V: nullptr);
5274 }
5275 case Builtin::BI__sync_fetch_and_add:
5276 case Builtin::BI__sync_fetch_and_sub:
5277 case Builtin::BI__sync_fetch_and_or:
5278 case Builtin::BI__sync_fetch_and_and:
5279 case Builtin::BI__sync_fetch_and_xor:
5280 case Builtin::BI__sync_fetch_and_nand:
5281 case Builtin::BI__sync_add_and_fetch:
5282 case Builtin::BI__sync_sub_and_fetch:
5283 case Builtin::BI__sync_and_and_fetch:
5284 case Builtin::BI__sync_or_and_fetch:
5285 case Builtin::BI__sync_xor_and_fetch:
5286 case Builtin::BI__sync_nand_and_fetch:
5287 case Builtin::BI__sync_val_compare_and_swap:
5288 case Builtin::BI__sync_bool_compare_and_swap:
5289 case Builtin::BI__sync_lock_test_and_set:
5290 case Builtin::BI__sync_lock_release:
5291 case Builtin::BI__sync_swap:
5292 llvm_unreachable("Shouldn't make it through sema");
5293 case Builtin::BI__sync_fetch_and_add_1:
5294 case Builtin::BI__sync_fetch_and_add_2:
5295 case Builtin::BI__sync_fetch_and_add_4:
5296 case Builtin::BI__sync_fetch_and_add_8:
5297 case Builtin::BI__sync_fetch_and_add_16:
5298 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::Add, E);
5299 case Builtin::BI__sync_fetch_and_sub_1:
5300 case Builtin::BI__sync_fetch_and_sub_2:
5301 case Builtin::BI__sync_fetch_and_sub_4:
5302 case Builtin::BI__sync_fetch_and_sub_8:
5303 case Builtin::BI__sync_fetch_and_sub_16:
5304 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::Sub, E);
5305 case Builtin::BI__sync_fetch_and_or_1:
5306 case Builtin::BI__sync_fetch_and_or_2:
5307 case Builtin::BI__sync_fetch_and_or_4:
5308 case Builtin::BI__sync_fetch_and_or_8:
5309 case Builtin::BI__sync_fetch_and_or_16:
5310 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::Or, E);
5311 case Builtin::BI__sync_fetch_and_and_1:
5312 case Builtin::BI__sync_fetch_and_and_2:
5313 case Builtin::BI__sync_fetch_and_and_4:
5314 case Builtin::BI__sync_fetch_and_and_8:
5315 case Builtin::BI__sync_fetch_and_and_16:
5316 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::And, E);
5317 case Builtin::BI__sync_fetch_and_xor_1:
5318 case Builtin::BI__sync_fetch_and_xor_2:
5319 case Builtin::BI__sync_fetch_and_xor_4:
5320 case Builtin::BI__sync_fetch_and_xor_8:
5321 case Builtin::BI__sync_fetch_and_xor_16:
5322 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::Xor, E);
5323 case Builtin::BI__sync_fetch_and_nand_1:
5324 case Builtin::BI__sync_fetch_and_nand_2:
5325 case Builtin::BI__sync_fetch_and_nand_4:
5326 case Builtin::BI__sync_fetch_and_nand_8:
5327 case Builtin::BI__sync_fetch_and_nand_16:
5328 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::Nand, E);
5329
5330 // Clang extensions: not overloaded yet.
5331 case Builtin::BI__sync_fetch_and_min:
5332 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::Min, E);
5333 case Builtin::BI__sync_fetch_and_max:
5334 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::Max, E);
5335 case Builtin::BI__sync_fetch_and_umin:
5336 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::UMin, E);
5337 case Builtin::BI__sync_fetch_and_umax:
5338 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::UMax, E);
5339
5340 case Builtin::BI__sync_add_and_fetch_1:
5341 case Builtin::BI__sync_add_and_fetch_2:
5342 case Builtin::BI__sync_add_and_fetch_4:
5343 case Builtin::BI__sync_add_and_fetch_8:
5344 case Builtin::BI__sync_add_and_fetch_16:
5345 return EmitBinaryAtomicPost(CGF&: *this, Kind: llvm::AtomicRMWInst::Add, E,
5346 Op: llvm::Instruction::Add);
5347 case Builtin::BI__sync_sub_and_fetch_1:
5348 case Builtin::BI__sync_sub_and_fetch_2:
5349 case Builtin::BI__sync_sub_and_fetch_4:
5350 case Builtin::BI__sync_sub_and_fetch_8:
5351 case Builtin::BI__sync_sub_and_fetch_16:
5352 return EmitBinaryAtomicPost(CGF&: *this, Kind: llvm::AtomicRMWInst::Sub, E,
5353 Op: llvm::Instruction::Sub);
5354 case Builtin::BI__sync_and_and_fetch_1:
5355 case Builtin::BI__sync_and_and_fetch_2:
5356 case Builtin::BI__sync_and_and_fetch_4:
5357 case Builtin::BI__sync_and_and_fetch_8:
5358 case Builtin::BI__sync_and_and_fetch_16:
5359 return EmitBinaryAtomicPost(CGF&: *this, Kind: llvm::AtomicRMWInst::And, E,
5360 Op: llvm::Instruction::And);
5361 case Builtin::BI__sync_or_and_fetch_1:
5362 case Builtin::BI__sync_or_and_fetch_2:
5363 case Builtin::BI__sync_or_and_fetch_4:
5364 case Builtin::BI__sync_or_and_fetch_8:
5365 case Builtin::BI__sync_or_and_fetch_16:
5366 return EmitBinaryAtomicPost(CGF&: *this, Kind: llvm::AtomicRMWInst::Or, E,
5367 Op: llvm::Instruction::Or);
5368 case Builtin::BI__sync_xor_and_fetch_1:
5369 case Builtin::BI__sync_xor_and_fetch_2:
5370 case Builtin::BI__sync_xor_and_fetch_4:
5371 case Builtin::BI__sync_xor_and_fetch_8:
5372 case Builtin::BI__sync_xor_and_fetch_16:
5373 return EmitBinaryAtomicPost(CGF&: *this, Kind: llvm::AtomicRMWInst::Xor, E,
5374 Op: llvm::Instruction::Xor);
5375 case Builtin::BI__sync_nand_and_fetch_1:
5376 case Builtin::BI__sync_nand_and_fetch_2:
5377 case Builtin::BI__sync_nand_and_fetch_4:
5378 case Builtin::BI__sync_nand_and_fetch_8:
5379 case Builtin::BI__sync_nand_and_fetch_16:
5380 return EmitBinaryAtomicPost(CGF&: *this, Kind: llvm::AtomicRMWInst::Nand, E,
5381 Op: llvm::Instruction::And, Invert: true);
5382
5383 case Builtin::BI__sync_val_compare_and_swap_1:
5384 case Builtin::BI__sync_val_compare_and_swap_2:
5385 case Builtin::BI__sync_val_compare_and_swap_4:
5386 case Builtin::BI__sync_val_compare_and_swap_8:
5387 case Builtin::BI__sync_val_compare_and_swap_16:
5388 return RValue::get(V: MakeAtomicCmpXchgValue(
5389 CGF&: *this, E, ReturnBool: false, SuccessOrdering: AtomicOrdering::SequentiallyConsistent,
5390 FailureOrdering: AtomicOrdering::SequentiallyConsistent));
5391
5392 case Builtin::BI__sync_bool_compare_and_swap_1:
5393 case Builtin::BI__sync_bool_compare_and_swap_2:
5394 case Builtin::BI__sync_bool_compare_and_swap_4:
5395 case Builtin::BI__sync_bool_compare_and_swap_8:
5396 case Builtin::BI__sync_bool_compare_and_swap_16:
5397 return RValue::get(V: MakeAtomicCmpXchgValue(
5398 CGF&: *this, E, ReturnBool: true, SuccessOrdering: AtomicOrdering::SequentiallyConsistent,
5399 FailureOrdering: AtomicOrdering::SequentiallyConsistent));
5400
5401 case Builtin::BI__sync_swap_1:
5402 case Builtin::BI__sync_swap_2:
5403 case Builtin::BI__sync_swap_4:
5404 case Builtin::BI__sync_swap_8:
5405 case Builtin::BI__sync_swap_16:
5406 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::Xchg, E);
5407
5408 case Builtin::BI__sync_lock_test_and_set_1:
5409 case Builtin::BI__sync_lock_test_and_set_2:
5410 case Builtin::BI__sync_lock_test_and_set_4:
5411 case Builtin::BI__sync_lock_test_and_set_8:
5412 case Builtin::BI__sync_lock_test_and_set_16:
5413 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::Xchg, E);
5414
5415 case Builtin::BI__sync_lock_release_1:
5416 case Builtin::BI__sync_lock_release_2:
5417 case Builtin::BI__sync_lock_release_4:
5418 case Builtin::BI__sync_lock_release_8:
5419 case Builtin::BI__sync_lock_release_16: {
5420 Address Ptr = CheckAtomicAlignment(CGF&: *this, E);
5421 QualType ElTy = E->getArg(Arg: 0)->getType()->getPointeeType();
5422
5423 llvm::Type *ITy = llvm::IntegerType::get(C&: getLLVMContext(),
5424 NumBits: getContext().getTypeSize(T: ElTy));
5425 llvm::StoreInst *Store =
5426 Builder.CreateStore(Val: llvm::Constant::getNullValue(Ty: ITy), Addr: Ptr);
5427 Store->setAtomic(Ordering: llvm::AtomicOrdering::Release);
5428 return RValue::get(V: nullptr);
5429 }
5430
5431 case Builtin::BI__sync_synchronize: {
5432 // We assume this is supposed to correspond to a C++0x-style
5433 // sequentially-consistent fence (i.e. this is only usable for
5434 // synchronization, not device I/O or anything like that). This intrinsic
5435 // is really badly designed in the sense that in theory, there isn't
5436 // any way to safely use it... but in practice, it mostly works
5437 // to use it with non-atomic loads and stores to get acquire/release
5438 // semantics.
5439 Builder.CreateFence(Ordering: llvm::AtomicOrdering::SequentiallyConsistent);
5440 return RValue::get(V: nullptr);
5441 }
5442
5443 case Builtin::BI__builtin_nontemporal_load:
5444 return RValue::get(V: EmitNontemporalLoad(CGF&: *this, E));
5445 case Builtin::BI__builtin_nontemporal_store:
5446 return RValue::get(V: EmitNontemporalStore(CGF&: *this, E));
5447 case Builtin::BI__c11_atomic_is_lock_free:
5448 case Builtin::BI__atomic_is_lock_free: {
5449 // Call "bool __atomic_is_lock_free(size_t size, void *ptr)". For the
5450 // __c11 builtin, ptr is 0 (indicating a properly-aligned object), since
5451 // _Atomic(T) is always properly-aligned.
5452 const char *LibCallName = "__atomic_is_lock_free";
5453 CallArgList Args;
5454 Args.add(rvalue: RValue::get(V: EmitScalarExpr(E: E->getArg(Arg: 0))),
5455 type: getContext().getSizeType());
5456 if (BuiltinID == Builtin::BI__atomic_is_lock_free)
5457 Args.add(rvalue: RValue::get(V: EmitScalarExpr(E: E->getArg(Arg: 1))),
5458 type: getContext().VoidPtrTy);
5459 else
5460 Args.add(rvalue: RValue::get(V: llvm::Constant::getNullValue(Ty: VoidPtrTy)),
5461 type: getContext().VoidPtrTy);
5462 const CGFunctionInfo &FuncInfo =
5463 CGM.getTypes().arrangeBuiltinFunctionCall(resultType: E->getType(), args: Args);
5464 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(Info: FuncInfo);
5465 llvm::FunctionCallee Func = CGM.CreateRuntimeFunction(Ty: FTy, Name: LibCallName);
5466 return EmitCall(CallInfo: FuncInfo, Callee: CGCallee::forDirect(functionPtr: Func),
5467 ReturnValue: ReturnValueSlot(), Args);
5468 }
5469
5470 case Builtin::BI__atomic_thread_fence:
5471 case Builtin::BI__atomic_signal_fence:
5472 case Builtin::BI__c11_atomic_thread_fence:
5473 case Builtin::BI__c11_atomic_signal_fence: {
5474 llvm::SyncScope::ID SSID;
5475 if (BuiltinID == Builtin::BI__atomic_signal_fence ||
5476 BuiltinID == Builtin::BI__c11_atomic_signal_fence)
5477 SSID = llvm::SyncScope::SingleThread;
5478 else
5479 SSID = llvm::SyncScope::System;
5480 Value *Order = EmitScalarExpr(E: E->getArg(Arg: 0));
5481 if (isa<llvm::ConstantInt>(Val: Order)) {
5482 int ord = cast<llvm::ConstantInt>(Val: Order)->getZExtValue();
5483 switch (ord) {
5484 case 0: // memory_order_relaxed
5485 default: // invalid order
5486 break;
5487 case 1: // memory_order_consume
5488 case 2: // memory_order_acquire
5489 emitAtomicFence(Order: llvm::AtomicOrdering::Acquire, SSID);
5490 break;
5491 case 3: // memory_order_release
5492 emitAtomicFence(Order: llvm::AtomicOrdering::Release, SSID);
5493 break;
5494 case 4: // memory_order_acq_rel
5495 emitAtomicFence(Order: llvm::AtomicOrdering::AcquireRelease, SSID);
5496 break;
5497 case 5: // memory_order_seq_cst
5498 emitAtomicFence(Order: llvm::AtomicOrdering::SequentiallyConsistent, SSID);
5499 break;
5500 }
5501 return RValue::get(V: nullptr);
5502 }
5503
5504 llvm::BasicBlock *AcquireBB, *ReleaseBB, *AcqRelBB, *SeqCstBB;
5505 AcquireBB = createBasicBlock(name: "acquire", parent: CurFn);
5506 ReleaseBB = createBasicBlock(name: "release", parent: CurFn);
5507 AcqRelBB = createBasicBlock(name: "acqrel", parent: CurFn);
5508 SeqCstBB = createBasicBlock(name: "seqcst", parent: CurFn);
5509 llvm::BasicBlock *ContBB = createBasicBlock(name: "atomic.continue", parent: CurFn);
5510
5511 Order = Builder.CreateIntCast(V: Order, DestTy: Builder.getInt32Ty(), isSigned: false);
5512 llvm::SwitchInst *SI = Builder.CreateSwitch(V: Order, Dest: ContBB);
5513
5514 Builder.SetInsertPoint(AcquireBB);
5515 emitAtomicFence(Order: llvm::AtomicOrdering::Acquire, SSID);
5516 Builder.CreateBr(Dest: ContBB);
5517 SI->addCase(OnVal: Builder.getInt32(C: 1), Dest: AcquireBB);
5518 SI->addCase(OnVal: Builder.getInt32(C: 2), Dest: AcquireBB);
5519
5520 Builder.SetInsertPoint(ReleaseBB);
5521 emitAtomicFence(Order: llvm::AtomicOrdering::Release, SSID);
5522 Builder.CreateBr(Dest: ContBB);
5523 SI->addCase(OnVal: Builder.getInt32(C: 3), Dest: ReleaseBB);
5524
5525 Builder.SetInsertPoint(AcqRelBB);
5526 emitAtomicFence(Order: llvm::AtomicOrdering::AcquireRelease, SSID);
5527 Builder.CreateBr(Dest: ContBB);
5528 SI->addCase(OnVal: Builder.getInt32(C: 4), Dest: AcqRelBB);
5529
5530 Builder.SetInsertPoint(SeqCstBB);
5531 emitAtomicFence(Order: llvm::AtomicOrdering::SequentiallyConsistent, SSID);
5532 Builder.CreateBr(Dest: ContBB);
5533 SI->addCase(OnVal: Builder.getInt32(C: 5), Dest: SeqCstBB);
5534
5535 Builder.SetInsertPoint(ContBB);
5536 return RValue::get(V: nullptr);
5537 }
5538 case Builtin::BI__scoped_atomic_thread_fence: {
5539 auto ScopeModel = AtomicScopeModel::create(K: AtomicScopeModelKind::Generic);
5540
5541 Value *Order = EmitScalarExpr(E: E->getArg(Arg: 0));
5542 Value *Scope = EmitScalarExpr(E: E->getArg(Arg: 1));
5543 auto Ord = dyn_cast<llvm::ConstantInt>(Val: Order);
5544 auto Scp = dyn_cast<llvm::ConstantInt>(Val: Scope);
5545 if (Ord && Scp) {
5546 SyncScope SS = ScopeModel->isValid(S: Scp->getZExtValue())
5547 ? ScopeModel->map(S: Scp->getZExtValue())
5548 : ScopeModel->map(S: ScopeModel->getFallBackValue());
5549 switch (Ord->getZExtValue()) {
5550 case 0: // memory_order_relaxed
5551 default: // invalid order
5552 break;
5553 case 1: // memory_order_consume
5554 case 2: // memory_order_acquire
5555 emitAtomicFence(Order: llvm::AtomicOrdering::Acquire,
5556 SSID: getTargetHooks().getLLVMSyncScopeID(
5557 LangOpts: getLangOpts(), Scope: SS, Ordering: llvm::AtomicOrdering::Acquire,
5558 Ctx&: getLLVMContext()));
5559 break;
5560 case 3: // memory_order_release
5561 emitAtomicFence(Order: llvm::AtomicOrdering::Release,
5562 SSID: getTargetHooks().getLLVMSyncScopeID(
5563 LangOpts: getLangOpts(), Scope: SS, Ordering: llvm::AtomicOrdering::Release,
5564 Ctx&: getLLVMContext()));
5565 break;
5566 case 4: // memory_order_acq_rel
5567 emitAtomicFence(Order: llvm::AtomicOrdering::AcquireRelease,
5568 SSID: getTargetHooks().getLLVMSyncScopeID(
5569 LangOpts: getLangOpts(), Scope: SS,
5570 Ordering: llvm::AtomicOrdering::AcquireRelease,
5571 Ctx&: getLLVMContext()));
5572 break;
5573 case 5: // memory_order_seq_cst
5574 emitAtomicFence(Order: llvm::AtomicOrdering::SequentiallyConsistent,
5575 SSID: getTargetHooks().getLLVMSyncScopeID(
5576 LangOpts: getLangOpts(), Scope: SS,
5577 Ordering: llvm::AtomicOrdering::SequentiallyConsistent,
5578 Ctx&: getLLVMContext()));
5579 break;
5580 }
5581 return RValue::get(V: nullptr);
5582 }
5583
5584 llvm::BasicBlock *ContBB = createBasicBlock(name: "atomic.scope.continue", parent: CurFn);
5585
5586 llvm::SmallVector<std::pair<llvm::BasicBlock *, llvm::AtomicOrdering>>
5587 OrderBBs;
5588 if (Ord) {
5589 switch (Ord->getZExtValue()) {
5590 case 0: // memory_order_relaxed
5591 default: // invalid order
5592 ContBB->eraseFromParent();
5593 return RValue::get(V: nullptr);
5594 case 1: // memory_order_consume
5595 case 2: // memory_order_acquire
5596 OrderBBs.emplace_back(Args: Builder.GetInsertBlock(),
5597 Args: llvm::AtomicOrdering::Acquire);
5598 break;
5599 case 3: // memory_order_release
5600 OrderBBs.emplace_back(Args: Builder.GetInsertBlock(),
5601 Args: llvm::AtomicOrdering::Release);
5602 break;
5603 case 4: // memory_order_acq_rel
5604 OrderBBs.emplace_back(Args: Builder.GetInsertBlock(),
5605 Args: llvm::AtomicOrdering::AcquireRelease);
5606 break;
5607 case 5: // memory_order_seq_cst
5608 OrderBBs.emplace_back(Args: Builder.GetInsertBlock(),
5609 Args: llvm::AtomicOrdering::SequentiallyConsistent);
5610 break;
5611 }
5612 } else {
5613 llvm::BasicBlock *AcquireBB = createBasicBlock(name: "acquire", parent: CurFn);
5614 llvm::BasicBlock *ReleaseBB = createBasicBlock(name: "release", parent: CurFn);
5615 llvm::BasicBlock *AcqRelBB = createBasicBlock(name: "acqrel", parent: CurFn);
5616 llvm::BasicBlock *SeqCstBB = createBasicBlock(name: "seqcst", parent: CurFn);
5617
5618 Order = Builder.CreateIntCast(V: Order, DestTy: Builder.getInt32Ty(), isSigned: false);
5619 llvm::SwitchInst *SI = Builder.CreateSwitch(V: Order, Dest: ContBB);
5620 SI->addCase(OnVal: Builder.getInt32(C: 1), Dest: AcquireBB);
5621 SI->addCase(OnVal: Builder.getInt32(C: 2), Dest: AcquireBB);
5622 SI->addCase(OnVal: Builder.getInt32(C: 3), Dest: ReleaseBB);
5623 SI->addCase(OnVal: Builder.getInt32(C: 4), Dest: AcqRelBB);
5624 SI->addCase(OnVal: Builder.getInt32(C: 5), Dest: SeqCstBB);
5625
5626 OrderBBs.emplace_back(Args&: AcquireBB, Args: llvm::AtomicOrdering::Acquire);
5627 OrderBBs.emplace_back(Args&: ReleaseBB, Args: llvm::AtomicOrdering::Release);
5628 OrderBBs.emplace_back(Args&: AcqRelBB, Args: llvm::AtomicOrdering::AcquireRelease);
5629 OrderBBs.emplace_back(Args&: SeqCstBB,
5630 Args: llvm::AtomicOrdering::SequentiallyConsistent);
5631 }
5632
5633 for (auto &[OrderBB, Ordering] : OrderBBs) {
5634 Builder.SetInsertPoint(OrderBB);
5635 if (Scp) {
5636 SyncScope SS = ScopeModel->isValid(S: Scp->getZExtValue())
5637 ? ScopeModel->map(S: Scp->getZExtValue())
5638 : ScopeModel->map(S: ScopeModel->getFallBackValue());
5639 emitAtomicFence(Order: Ordering,
5640 SSID: getTargetHooks().getLLVMSyncScopeID(
5641 LangOpts: getLangOpts(), Scope: SS, Ordering, Ctx&: getLLVMContext()));
5642 Builder.CreateBr(Dest: ContBB);
5643 } else {
5644 llvm::DenseMap<unsigned, llvm::BasicBlock *> BBs;
5645 for (unsigned Scp : ScopeModel->getRuntimeValues())
5646 BBs[Scp] = createBasicBlock(name: getAsString(S: ScopeModel->map(S: Scp)), parent: CurFn);
5647
5648 auto *SC = Builder.CreateIntCast(V: Scope, DestTy: Builder.getInt32Ty(), isSigned: false);
5649 llvm::SwitchInst *SI = Builder.CreateSwitch(V: SC, Dest: ContBB);
5650 for (unsigned Scp : ScopeModel->getRuntimeValues()) {
5651 auto *B = BBs[Scp];
5652 SI->addCase(OnVal: Builder.getInt32(C: Scp), Dest: B);
5653
5654 Builder.SetInsertPoint(B);
5655 emitAtomicFence(Order: Ordering, SSID: getTargetHooks().getLLVMSyncScopeID(
5656 LangOpts: getLangOpts(), Scope: ScopeModel->map(S: Scp),
5657 Ordering, Ctx&: getLLVMContext()));
5658 Builder.CreateBr(Dest: ContBB);
5659 }
5660 }
5661 }
5662
5663 Builder.SetInsertPoint(ContBB);
5664 return RValue::get(V: nullptr);
5665 }
5666
5667 case Builtin::BI__builtin_signbit:
5668 case Builtin::BI__builtin_signbitf:
5669 case Builtin::BI__builtin_signbitl: {
5670 return RValue::get(
5671 V: Builder.CreateZExt(V: EmitSignBit(CGF&: *this, V: EmitScalarExpr(E: E->getArg(Arg: 0))),
5672 DestTy: ConvertType(T: E->getType())));
5673 }
5674 case Builtin::BI__warn_memset_zero_len:
5675 return RValue::getIgnored();
5676 case Builtin::BI__annotation: {
5677 // Re-encode each wide string to UTF8 and make an MDString.
5678 SmallVector<Metadata *, 1> Strings;
5679 for (const Expr *Arg : E->arguments()) {
5680 const auto *Str = cast<StringLiteral>(Val: Arg->IgnoreParenCasts());
5681 assert(Str->getCharByteWidth() == 2 || Str->getCharByteWidth() == 4);
5682 StringRef WideBytes = Str->getBytes();
5683 std::string StrUtf8;
5684 bool Converted =
5685 (Str->getCharByteWidth() == 2)
5686 ? convertUTF16ToUTF8String(
5687 SrcBytes: ArrayRef(WideBytes.data(), WideBytes.size()), Out&: StrUtf8)
5688 : convertUTF32ToUTF8String(
5689 SrcBytes: ArrayRef(WideBytes.data(), WideBytes.size()), Out&: StrUtf8);
5690 if (!Converted) {
5691 CGM.ErrorUnsupported(S: E, Type: "non-Unicode __annotation argument");
5692 continue;
5693 }
5694 Strings.push_back(Elt: llvm::MDString::get(Context&: getLLVMContext(), Str: StrUtf8));
5695 }
5696
5697 // Build and MDTuple of MDStrings and emit the intrinsic call.
5698 llvm::Function *F = CGM.getIntrinsic(IID: Intrinsic::codeview_annotation, Tys: {});
5699 MDTuple *StrTuple = MDTuple::get(Context&: getLLVMContext(), MDs: Strings);
5700 Builder.CreateCall(Callee: F, Args: MetadataAsValue::get(Context&: getLLVMContext(), MD: StrTuple));
5701 return RValue::getIgnored();
5702 }
5703 case Builtin::BI__builtin_annotation: {
5704 llvm::Value *AnnVal = EmitScalarExpr(E: E->getArg(Arg: 0));
5705 llvm::Function *F = CGM.getIntrinsic(
5706 IID: Intrinsic::annotation, Tys: {AnnVal->getType(), CGM.ConstGlobalsPtrTy});
5707
5708 // Get the annotation string, go through casts. Sema requires this to be a
5709 // non-wide string literal, potentially casted, so the cast<> is safe.
5710 const Expr *AnnotationStrExpr = E->getArg(Arg: 1)->IgnoreParenCasts();
5711 StringRef Str = cast<StringLiteral>(Val: AnnotationStrExpr)->getString();
5712 return RValue::get(
5713 V: EmitAnnotationCall(AnnotationFn: F, AnnotatedVal: AnnVal, AnnotationStr: Str, Location: E->getExprLoc(), Attr: nullptr));
5714 }
5715 case Builtin::BI__builtin_addcb:
5716 case Builtin::BI__builtin_addcs:
5717 case Builtin::BI__builtin_addc:
5718 case Builtin::BI__builtin_addcl:
5719 case Builtin::BI__builtin_addcll:
5720 case Builtin::BI__builtin_subcb:
5721 case Builtin::BI__builtin_subcs:
5722 case Builtin::BI__builtin_subc:
5723 case Builtin::BI__builtin_subcl:
5724 case Builtin::BI__builtin_subcll: {
5725
5726 // We translate all of these builtins from expressions of the form:
5727 // int x = ..., y = ..., carryin = ..., carryout, result;
5728 // result = __builtin_addc(x, y, carryin, &carryout);
5729 //
5730 // to LLVM IR of the form:
5731 //
5732 // %tmp1 = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %x, i32 %y)
5733 // %tmpsum1 = extractvalue {i32, i1} %tmp1, 0
5734 // %carry1 = extractvalue {i32, i1} %tmp1, 1
5735 // %tmp2 = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %tmpsum1,
5736 // i32 %carryin)
5737 // %result = extractvalue {i32, i1} %tmp2, 0
5738 // %carry2 = extractvalue {i32, i1} %tmp2, 1
5739 // %tmp3 = or i1 %carry1, %carry2
5740 // %tmp4 = zext i1 %tmp3 to i32
5741 // store i32 %tmp4, i32* %carryout
5742
5743 // Scalarize our inputs.
5744 llvm::Value *X = EmitScalarExpr(E: E->getArg(Arg: 0));
5745 llvm::Value *Y = EmitScalarExpr(E: E->getArg(Arg: 1));
5746 llvm::Value *Carryin = EmitScalarExpr(E: E->getArg(Arg: 2));
5747 Address CarryOutPtr = EmitPointerWithAlignment(Addr: E->getArg(Arg: 3));
5748
5749 // Decide if we are lowering to a uadd.with.overflow or usub.with.overflow.
5750 Intrinsic::ID IntrinsicId;
5751 switch (BuiltinID) {
5752 default: llvm_unreachable("Unknown multiprecision builtin id.");
5753 case Builtin::BI__builtin_addcb:
5754 case Builtin::BI__builtin_addcs:
5755 case Builtin::BI__builtin_addc:
5756 case Builtin::BI__builtin_addcl:
5757 case Builtin::BI__builtin_addcll:
5758 IntrinsicId = Intrinsic::uadd_with_overflow;
5759 break;
5760 case Builtin::BI__builtin_subcb:
5761 case Builtin::BI__builtin_subcs:
5762 case Builtin::BI__builtin_subc:
5763 case Builtin::BI__builtin_subcl:
5764 case Builtin::BI__builtin_subcll:
5765 IntrinsicId = Intrinsic::usub_with_overflow;
5766 break;
5767 }
5768
5769 // Construct our resulting LLVM IR expression.
5770 llvm::Value *Carry1;
5771 llvm::Value *Sum1 = EmitOverflowIntrinsic(CGF&: *this, IntrinsicID: IntrinsicId,
5772 X, Y, Carry&: Carry1);
5773 llvm::Value *Carry2;
5774 llvm::Value *Sum2 = EmitOverflowIntrinsic(CGF&: *this, IntrinsicID: IntrinsicId,
5775 X: Sum1, Y: Carryin, Carry&: Carry2);
5776 llvm::Value *CarryOut = Builder.CreateZExt(V: Builder.CreateOr(LHS: Carry1, RHS: Carry2),
5777 DestTy: X->getType());
5778 Builder.CreateStore(Val: CarryOut, Addr: CarryOutPtr);
5779 return RValue::get(V: Sum2);
5780 }
5781
5782 case Builtin::BI__builtin_add_overflow:
5783 case Builtin::BI__builtin_sub_overflow:
5784 case Builtin::BI__builtin_mul_overflow: {
5785 const clang::Expr *LeftArg = E->getArg(Arg: 0);
5786 const clang::Expr *RightArg = E->getArg(Arg: 1);
5787 const clang::Expr *ResultArg = E->getArg(Arg: 2);
5788
5789 clang::QualType ResultQTy =
5790 ResultArg->getType()->castAs<PointerType>()->getPointeeType();
5791
5792 WidthAndSignedness LeftInfo =
5793 getIntegerWidthAndSignedness(context: CGM.getContext(), Type: LeftArg->getType());
5794 WidthAndSignedness RightInfo =
5795 getIntegerWidthAndSignedness(context: CGM.getContext(), Type: RightArg->getType());
5796 WidthAndSignedness ResultInfo =
5797 getIntegerWidthAndSignedness(context: CGM.getContext(), Type: ResultQTy);
5798
5799 // Handle mixed-sign multiplication as a special case, because adding
5800 // runtime or backend support for our generic irgen would be too expensive.
5801 if (isSpecialMixedSignMultiply(BuiltinID, Op1Info: LeftInfo, Op2Info: RightInfo, ResultInfo))
5802 return EmitCheckedMixedSignMultiply(CGF&: *this, Op1: LeftArg, Op1Info: LeftInfo, Op2: RightArg,
5803 Op2Info: RightInfo, ResultArg, ResultQTy,
5804 ResultInfo);
5805
5806 if (isSpecialUnsignedMultiplySignedResult(BuiltinID, Op1Info: LeftInfo, Op2Info: RightInfo,
5807 ResultInfo))
5808 return EmitCheckedUnsignedMultiplySignedResult(
5809 CGF&: *this, Op1: LeftArg, Op1Info: LeftInfo, Op2: RightArg, Op2Info: RightInfo, ResultArg, ResultQTy,
5810 ResultInfo);
5811
5812 WidthAndSignedness EncompassingInfo =
5813 EncompassingIntegerType(Types: {LeftInfo, RightInfo, ResultInfo});
5814
5815 llvm::Type *EncompassingLLVMTy =
5816 llvm::IntegerType::get(C&: CGM.getLLVMContext(), NumBits: EncompassingInfo.Width);
5817
5818 llvm::Type *ResultLLVMTy = CGM.getTypes().ConvertType(T: ResultQTy);
5819
5820 Intrinsic::ID IntrinsicId;
5821 switch (BuiltinID) {
5822 default:
5823 llvm_unreachable("Unknown overflow builtin id.");
5824 case Builtin::BI__builtin_add_overflow:
5825 IntrinsicId = EncompassingInfo.Signed ? Intrinsic::sadd_with_overflow
5826 : Intrinsic::uadd_with_overflow;
5827 break;
5828 case Builtin::BI__builtin_sub_overflow:
5829 IntrinsicId = EncompassingInfo.Signed ? Intrinsic::ssub_with_overflow
5830 : Intrinsic::usub_with_overflow;
5831 break;
5832 case Builtin::BI__builtin_mul_overflow:
5833 IntrinsicId = EncompassingInfo.Signed ? Intrinsic::smul_with_overflow
5834 : Intrinsic::umul_with_overflow;
5835 break;
5836 }
5837
5838 llvm::Value *Left = EmitScalarExpr(E: LeftArg);
5839 llvm::Value *Right = EmitScalarExpr(E: RightArg);
5840 Address ResultPtr = EmitPointerWithAlignment(Addr: ResultArg);
5841
5842 // Extend each operand to the encompassing type.
5843 Left = Builder.CreateIntCast(V: Left, DestTy: EncompassingLLVMTy, isSigned: LeftInfo.Signed);
5844 Right = Builder.CreateIntCast(V: Right, DestTy: EncompassingLLVMTy, isSigned: RightInfo.Signed);
5845
5846 // Perform the operation on the extended values.
5847 llvm::Value *Overflow, *Result;
5848 Result = EmitOverflowIntrinsic(CGF&: *this, IntrinsicID: IntrinsicId, X: Left, Y: Right, Carry&: Overflow);
5849
5850 if (EncompassingInfo.Width > ResultInfo.Width) {
5851 // The encompassing type is wider than the result type, so we need to
5852 // truncate it.
5853 llvm::Value *ResultTrunc = Builder.CreateTrunc(V: Result, DestTy: ResultLLVMTy);
5854
5855 // To see if the truncation caused an overflow, we will extend
5856 // the result and then compare it to the original result.
5857 llvm::Value *ResultTruncExt = Builder.CreateIntCast(
5858 V: ResultTrunc, DestTy: EncompassingLLVMTy, isSigned: ResultInfo.Signed);
5859 llvm::Value *TruncationOverflow =
5860 Builder.CreateICmpNE(LHS: Result, RHS: ResultTruncExt);
5861
5862 Overflow = Builder.CreateOr(LHS: Overflow, RHS: TruncationOverflow);
5863 Result = ResultTrunc;
5864 }
5865
5866 // Finally, store the result using the pointer.
5867 bool isVolatile =
5868 ResultArg->getType()->getPointeeType().isVolatileQualified();
5869 Builder.CreateStore(Val: EmitToMemory(Value: Result, Ty: ResultQTy), Addr: ResultPtr, IsVolatile: isVolatile);
5870
5871 return RValue::get(V: Overflow);
5872 }
5873
5874 case Builtin::BI__builtin_uadd_overflow:
5875 case Builtin::BI__builtin_uaddl_overflow:
5876 case Builtin::BI__builtin_uaddll_overflow:
5877 case Builtin::BI__builtin_usub_overflow:
5878 case Builtin::BI__builtin_usubl_overflow:
5879 case Builtin::BI__builtin_usubll_overflow:
5880 case Builtin::BI__builtin_umul_overflow:
5881 case Builtin::BI__builtin_umull_overflow:
5882 case Builtin::BI__builtin_umulll_overflow:
5883 case Builtin::BI__builtin_sadd_overflow:
5884 case Builtin::BI__builtin_saddl_overflow:
5885 case Builtin::BI__builtin_saddll_overflow:
5886 case Builtin::BI__builtin_ssub_overflow:
5887 case Builtin::BI__builtin_ssubl_overflow:
5888 case Builtin::BI__builtin_ssubll_overflow:
5889 case Builtin::BI__builtin_smul_overflow:
5890 case Builtin::BI__builtin_smull_overflow:
5891 case Builtin::BI__builtin_smulll_overflow: {
5892
5893 // We translate all of these builtins directly to the relevant llvm IR node.
5894
5895 // Scalarize our inputs.
5896 llvm::Value *X = EmitScalarExpr(E: E->getArg(Arg: 0));
5897 llvm::Value *Y = EmitScalarExpr(E: E->getArg(Arg: 1));
5898 Address SumOutPtr = EmitPointerWithAlignment(Addr: E->getArg(Arg: 2));
5899
5900 // Decide which of the overflow intrinsics we are lowering to:
5901 Intrinsic::ID IntrinsicId;
5902 switch (BuiltinID) {
5903 default: llvm_unreachable("Unknown overflow builtin id.");
5904 case Builtin::BI__builtin_uadd_overflow:
5905 case Builtin::BI__builtin_uaddl_overflow:
5906 case Builtin::BI__builtin_uaddll_overflow:
5907 IntrinsicId = Intrinsic::uadd_with_overflow;
5908 break;
5909 case Builtin::BI__builtin_usub_overflow:
5910 case Builtin::BI__builtin_usubl_overflow:
5911 case Builtin::BI__builtin_usubll_overflow:
5912 IntrinsicId = Intrinsic::usub_with_overflow;
5913 break;
5914 case Builtin::BI__builtin_umul_overflow:
5915 case Builtin::BI__builtin_umull_overflow:
5916 case Builtin::BI__builtin_umulll_overflow:
5917 IntrinsicId = Intrinsic::umul_with_overflow;
5918 break;
5919 case Builtin::BI__builtin_sadd_overflow:
5920 case Builtin::BI__builtin_saddl_overflow:
5921 case Builtin::BI__builtin_saddll_overflow:
5922 IntrinsicId = Intrinsic::sadd_with_overflow;
5923 break;
5924 case Builtin::BI__builtin_ssub_overflow:
5925 case Builtin::BI__builtin_ssubl_overflow:
5926 case Builtin::BI__builtin_ssubll_overflow:
5927 IntrinsicId = Intrinsic::ssub_with_overflow;
5928 break;
5929 case Builtin::BI__builtin_smul_overflow:
5930 case Builtin::BI__builtin_smull_overflow:
5931 case Builtin::BI__builtin_smulll_overflow:
5932 IntrinsicId = Intrinsic::smul_with_overflow;
5933 break;
5934 }
5935
5936
5937 llvm::Value *Carry;
5938 llvm::Value *Sum = EmitOverflowIntrinsic(CGF&: *this, IntrinsicID: IntrinsicId, X, Y, Carry);
5939 Builder.CreateStore(Val: Sum, Addr: SumOutPtr);
5940
5941 return RValue::get(V: Carry);
5942 }
5943 case Builtin::BIaddressof:
5944 case Builtin::BI__addressof:
5945 case Builtin::BI__builtin_addressof:
5946 return RValue::get(V: EmitLValue(E: E->getArg(Arg: 0)).getPointer(CGF&: *this));
5947 case Builtin::BI__builtin_function_start:
5948 return RValue::get(V: CGM.GetFunctionStart(
5949 Decl: E->getArg(Arg: 0)->getAsBuiltinConstantDeclRef(Context: CGM.getContext())));
5950 case Builtin::BI__builtin_operator_new:
5951 return EmitBuiltinNewDeleteCall(
5952 Type: E->getCallee()->getType()->castAs<FunctionProtoType>(), TheCallExpr: E, IsDelete: false);
5953 case Builtin::BI__builtin_operator_delete:
5954 EmitBuiltinNewDeleteCall(
5955 Type: E->getCallee()->getType()->castAs<FunctionProtoType>(), TheCallExpr: E, IsDelete: true);
5956 return RValue::get(V: nullptr);
5957
5958 case Builtin::BI__builtin_is_aligned:
5959 return EmitBuiltinIsAligned(E);
5960 case Builtin::BI__builtin_align_up:
5961 return EmitBuiltinAlignTo(E, AlignUp: true);
5962 case Builtin::BI__builtin_align_down:
5963 return EmitBuiltinAlignTo(E, AlignUp: false);
5964
5965 case Builtin::BI__noop:
5966 // __noop always evaluates to an integer literal zero.
5967 return RValue::get(V: ConstantInt::get(Ty: IntTy, V: 0));
5968 case Builtin::BI__builtin_call_with_static_chain: {
5969 const CallExpr *Call = cast<CallExpr>(Val: E->getArg(Arg: 0));
5970 const Expr *Chain = E->getArg(Arg: 1);
5971 return EmitCall(FnType: Call->getCallee()->getType(),
5972 Callee: EmitCallee(E: Call->getCallee()), E: Call, ReturnValue,
5973 Chain: EmitScalarExpr(E: Chain));
5974 }
5975 case Builtin::BI_InterlockedExchange8:
5976 case Builtin::BI_InterlockedExchange16:
5977 case Builtin::BI_InterlockedExchange:
5978 case Builtin::BI_InterlockedExchangePointer:
5979 return RValue::get(
5980 V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::_InterlockedExchange, E));
5981 case Builtin::BI_InterlockedCompareExchangePointer:
5982 return RValue::get(
5983 V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::_InterlockedCompareExchange, E));
5984 case Builtin::BI_InterlockedCompareExchangePointer_nf:
5985 return RValue::get(
5986 V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::_InterlockedCompareExchange_nf, E));
5987 case Builtin::BI_InterlockedCompareExchange8:
5988 case Builtin::BI_InterlockedCompareExchange16:
5989 case Builtin::BI_InterlockedCompareExchange:
5990 case Builtin::BI_InterlockedCompareExchange64:
5991 return RValue::get(V: EmitAtomicCmpXchgForMSIntrin(CGF&: *this, E));
5992 case Builtin::BI_InterlockedIncrement16:
5993 case Builtin::BI_InterlockedIncrement:
5994 return RValue::get(
5995 V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::_InterlockedIncrement, E));
5996 case Builtin::BI_InterlockedDecrement16:
5997 case Builtin::BI_InterlockedDecrement:
5998 return RValue::get(
5999 V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::_InterlockedDecrement, E));
6000 case Builtin::BI_InterlockedAnd8:
6001 case Builtin::BI_InterlockedAnd16:
6002 case Builtin::BI_InterlockedAnd:
6003 return RValue::get(V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::_InterlockedAnd, E));
6004 case Builtin::BI_InterlockedExchangeAdd8:
6005 case Builtin::BI_InterlockedExchangeAdd16:
6006 case Builtin::BI_InterlockedExchangeAdd:
6007 return RValue::get(
6008 V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::_InterlockedExchangeAdd, E));
6009 case Builtin::BI_InterlockedExchangeSub8:
6010 case Builtin::BI_InterlockedExchangeSub16:
6011 case Builtin::BI_InterlockedExchangeSub:
6012 return RValue::get(
6013 V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::_InterlockedExchangeSub, E));
6014 case Builtin::BI_InterlockedOr8:
6015 case Builtin::BI_InterlockedOr16:
6016 case Builtin::BI_InterlockedOr:
6017 return RValue::get(V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::_InterlockedOr, E));
6018 case Builtin::BI_InterlockedXor8:
6019 case Builtin::BI_InterlockedXor16:
6020 case Builtin::BI_InterlockedXor:
6021 return RValue::get(V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::_InterlockedXor, E));
6022
6023 case Builtin::BI_bittest64:
6024 case Builtin::BI_bittest:
6025 case Builtin::BI_bittestandcomplement64:
6026 case Builtin::BI_bittestandcomplement:
6027 case Builtin::BI_bittestandreset64:
6028 case Builtin::BI_bittestandreset:
6029 case Builtin::BI_bittestandset64:
6030 case Builtin::BI_bittestandset:
6031 case Builtin::BI_interlockedbittestandreset:
6032 case Builtin::BI_interlockedbittestandreset64:
6033 case Builtin::BI_interlockedbittestandreset64_acq:
6034 case Builtin::BI_interlockedbittestandreset64_rel:
6035 case Builtin::BI_interlockedbittestandreset64_nf:
6036 case Builtin::BI_interlockedbittestandset64:
6037 case Builtin::BI_interlockedbittestandset64_acq:
6038 case Builtin::BI_interlockedbittestandset64_rel:
6039 case Builtin::BI_interlockedbittestandset64_nf:
6040 case Builtin::BI_interlockedbittestandset:
6041 case Builtin::BI_interlockedbittestandset_acq:
6042 case Builtin::BI_interlockedbittestandset_rel:
6043 case Builtin::BI_interlockedbittestandset_nf:
6044 case Builtin::BI_interlockedbittestandreset_acq:
6045 case Builtin::BI_interlockedbittestandreset_rel:
6046 case Builtin::BI_interlockedbittestandreset_nf:
6047 return RValue::get(V: EmitBitTestIntrinsic(CGF&: *this, BuiltinID, E));
6048
6049 // These builtins exist to emit regular volatile loads and stores not
6050 // affected by the -fms-volatile setting.
6051 case Builtin::BI__iso_volatile_load8:
6052 case Builtin::BI__iso_volatile_load16:
6053 case Builtin::BI__iso_volatile_load32:
6054 case Builtin::BI__iso_volatile_load64:
6055 return RValue::get(V: EmitISOVolatileLoad(CGF&: *this, E));
6056 case Builtin::BI__iso_volatile_store8:
6057 case Builtin::BI__iso_volatile_store16:
6058 case Builtin::BI__iso_volatile_store32:
6059 case Builtin::BI__iso_volatile_store64:
6060 return RValue::get(V: EmitISOVolatileStore(CGF&: *this, E));
6061
6062 case Builtin::BI__builtin_ptrauth_sign_constant:
6063 return RValue::get(V: ConstantEmitter(*this).emitAbstract(E, T: E->getType()));
6064
6065 case Builtin::BI__builtin_ptrauth_auth:
6066 case Builtin::BI__builtin_ptrauth_auth_and_resign:
6067 case Builtin::BI__builtin_ptrauth_auth_with_pc_and_resign:
6068 case Builtin::BI__builtin_ptrauth_auth_load_relative_and_sign:
6069 case Builtin::BI__builtin_ptrauth_blend_discriminator:
6070 case Builtin::BI__builtin_ptrauth_sign_generic_data:
6071 case Builtin::BI__builtin_ptrauth_sign_unauthenticated:
6072 case Builtin::BI__builtin_ptrauth_strip: {
6073 // Emit the arguments.
6074 SmallVector<llvm::Value *, 6> Args;
6075 for (auto argExpr : E->arguments())
6076 Args.push_back(Elt: EmitScalarExpr(E: argExpr));
6077
6078 // Cast the value to intptr_t, saving its original type.
6079 llvm::Type *OrigValueType = Args[0]->getType();
6080 if (OrigValueType->isPointerTy())
6081 Args[0] = Builder.CreatePtrToInt(V: Args[0], DestTy: IntPtrTy);
6082
6083 switch (BuiltinID) {
6084 case Builtin::BI__builtin_ptrauth_auth_with_pc_and_resign:
6085 // Convert oldDiscriminator (arg 2), oldPC (arg 3) and newDiscriminator
6086 // (arg 5) to intptr_t
6087 if (Args[2]->getType()->isPointerTy())
6088 Args[2] = Builder.CreatePtrToInt(V: Args[2], DestTy: IntPtrTy);
6089 if (Args[3]->getType()->isPointerTy())
6090 Args[3] = Builder.CreatePtrToInt(V: Args[3], DestTy: IntPtrTy);
6091 if (Args[5]->getType()->isPointerTy())
6092 Args[5] = Builder.CreatePtrToInt(V: Args[5], DestTy: IntPtrTy);
6093 break;
6094
6095 case Builtin::BI__builtin_ptrauth_auth_and_resign:
6096 case Builtin::BI__builtin_ptrauth_auth_load_relative_and_sign:
6097 if (Args[4]->getType()->isPointerTy())
6098 Args[4] = Builder.CreatePtrToInt(V: Args[4], DestTy: IntPtrTy);
6099 [[fallthrough]];
6100
6101 case Builtin::BI__builtin_ptrauth_auth:
6102 case Builtin::BI__builtin_ptrauth_sign_unauthenticated:
6103 if (Args[2]->getType()->isPointerTy())
6104 Args[2] = Builder.CreatePtrToInt(V: Args[2], DestTy: IntPtrTy);
6105 break;
6106
6107 case Builtin::BI__builtin_ptrauth_sign_generic_data:
6108 if (Args[1]->getType()->isPointerTy())
6109 Args[1] = Builder.CreatePtrToInt(V: Args[1], DestTy: IntPtrTy);
6110 break;
6111
6112 case Builtin::BI__builtin_ptrauth_blend_discriminator:
6113 case Builtin::BI__builtin_ptrauth_strip:
6114 break;
6115 }
6116
6117 // Call the intrinsic.
6118 auto IntrinsicID = [&]() -> unsigned {
6119 switch (BuiltinID) {
6120 case Builtin::BI__builtin_ptrauth_auth:
6121 return Intrinsic::ptrauth_auth;
6122 case Builtin::BI__builtin_ptrauth_auth_and_resign:
6123 return Intrinsic::ptrauth_resign;
6124 case Builtin::BI__builtin_ptrauth_auth_with_pc_and_resign:
6125 return Intrinsic::ptrauth_auth_with_pc_and_resign;
6126 case Builtin::BI__builtin_ptrauth_auth_load_relative_and_sign:
6127 return Intrinsic::ptrauth_resign_load_relative;
6128 case Builtin::BI__builtin_ptrauth_blend_discriminator:
6129 return Intrinsic::ptrauth_blend;
6130 case Builtin::BI__builtin_ptrauth_sign_generic_data:
6131 return Intrinsic::ptrauth_sign_generic;
6132 case Builtin::BI__builtin_ptrauth_sign_unauthenticated:
6133 return Intrinsic::ptrauth_sign;
6134 case Builtin::BI__builtin_ptrauth_strip:
6135 return Intrinsic::ptrauth_strip;
6136 }
6137 llvm_unreachable("bad ptrauth intrinsic");
6138 }();
6139 auto Intrinsic = CGM.getIntrinsic(IID: IntrinsicID);
6140 llvm::Value *Result = EmitRuntimeCall(callee: Intrinsic, args: Args);
6141
6142 if (BuiltinID != Builtin::BI__builtin_ptrauth_sign_generic_data &&
6143 BuiltinID != Builtin::BI__builtin_ptrauth_blend_discriminator &&
6144 OrigValueType->isPointerTy()) {
6145 Result = Builder.CreateIntToPtr(V: Result, DestTy: OrigValueType);
6146 }
6147 return RValue::get(V: Result);
6148 }
6149
6150 case Builtin::BI__builtin_get_vtable_pointer: {
6151 const Expr *Target = E->getArg(Arg: 0);
6152 QualType TargetType = Target->getType();
6153 const CXXRecordDecl *Decl = TargetType->getPointeeCXXRecordDecl();
6154 assert(Decl);
6155 auto ThisAddress = EmitPointerWithAlignment(Addr: Target);
6156 assert(ThisAddress.isValid());
6157 llvm::Value *VTablePointer =
6158 GetVTablePtr(This: ThisAddress, VTableTy: Int8PtrTy, VTableClass: Decl, AuthMode: VTableAuthMode::MustTrap);
6159 return RValue::get(V: VTablePointer);
6160 }
6161
6162 case Builtin::BI__exception_code:
6163 case Builtin::BI_exception_code:
6164 return RValue::get(V: EmitSEHExceptionCode());
6165 case Builtin::BI__exception_info:
6166 case Builtin::BI_exception_info:
6167 return RValue::get(V: EmitSEHExceptionInfo());
6168 case Builtin::BI__abnormal_termination:
6169 case Builtin::BI_abnormal_termination:
6170 return RValue::get(V: EmitSEHAbnormalTermination());
6171 case Builtin::BI_setjmpex:
6172 if (getTarget().getTriple().isOSMSVCRT() && E->getNumArgs() == 1 &&
6173 E->getArg(Arg: 0)->getType()->isPointerType())
6174 return EmitMSVCRTSetJmp(CGF&: *this, SJKind: MSVCSetJmpKind::_setjmpex, E);
6175 break;
6176 case Builtin::BI_setjmp:
6177 if (getTarget().getTriple().isOSMSVCRT() && E->getNumArgs() == 1 &&
6178 E->getArg(Arg: 0)->getType()->isPointerType()) {
6179 if (getTarget().getTriple().getArch() == llvm::Triple::x86)
6180 return EmitMSVCRTSetJmp(CGF&: *this, SJKind: MSVCSetJmpKind::_setjmp3, E);
6181 else if (getTarget().getTriple().getArch() == llvm::Triple::aarch64)
6182 return EmitMSVCRTSetJmp(CGF&: *this, SJKind: MSVCSetJmpKind::_setjmpex, E);
6183 return EmitMSVCRTSetJmp(CGF&: *this, SJKind: MSVCSetJmpKind::_setjmp, E);
6184 }
6185 break;
6186
6187 // C++ std:: builtins.
6188 case Builtin::BImove:
6189 case Builtin::BImove_if_noexcept:
6190 case Builtin::BIforward:
6191 case Builtin::BIforward_like:
6192 case Builtin::BIas_const:
6193 return RValue::get(V: EmitLValue(E: E->getArg(Arg: 0)).getPointer(CGF&: *this));
6194 case Builtin::BI__GetExceptionInfo: {
6195 if (llvm::GlobalVariable *GV =
6196 CGM.getCXXABI().getThrowInfo(T: FD->getParamDecl(i: 0)->getType()))
6197 return RValue::get(V: GV);
6198 break;
6199 }
6200
6201 case Builtin::BI__fastfail:
6202 return RValue::get(V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::__fastfail, E));
6203
6204 case Builtin::BI__builtin_coro_id:
6205 return EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_id);
6206 case Builtin::BI__builtin_coro_promise:
6207 return EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_promise);
6208 case Builtin::BI__builtin_coro_resume:
6209 EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_resume);
6210 return RValue::get(V: nullptr);
6211 case Builtin::BI__builtin_coro_frame:
6212 return EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_frame);
6213 case Builtin::BI__builtin_coro_noop:
6214 return EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_noop);
6215 case Builtin::BI__builtin_coro_free:
6216 return EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_free);
6217 case Builtin::BI__builtin_coro_destroy:
6218 EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_destroy);
6219 return RValue::get(V: nullptr);
6220 case Builtin::BI__builtin_coro_done:
6221 return EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_done);
6222 case Builtin::BI__builtin_coro_alloc:
6223 return EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_alloc);
6224 case Builtin::BI__builtin_coro_begin:
6225 return EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_begin);
6226 case Builtin::BI__builtin_coro_end:
6227 return EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_end);
6228 case Builtin::BI__builtin_coro_suspend:
6229 return EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_suspend);
6230 case Builtin::BI__builtin_coro_size:
6231 return EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_size);
6232 case Builtin::BI__builtin_coro_align:
6233 return EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_align);
6234
6235 // OpenCL v2.0 s6.13.16.2, Built-in pipe read and write functions
6236 case Builtin::BIread_pipe:
6237 case Builtin::BIwrite_pipe: {
6238 Value *Arg0 = EmitScalarExpr(E: E->getArg(Arg: 0)),
6239 *Arg1 = EmitScalarExpr(E: E->getArg(Arg: 1));
6240 CGOpenCLRuntime OpenCLRT(CGM);
6241 Value *PacketSize = OpenCLRT.getPipeElemSize(PipeArg: E->getArg(Arg: 0));
6242 Value *PacketAlign = OpenCLRT.getPipeElemAlign(PipeArg: E->getArg(Arg: 0));
6243
6244 // Type of the generic packet parameter.
6245 unsigned GenericAS =
6246 getContext().getTargetAddressSpace(AS: LangAS::opencl_generic);
6247 llvm::Type *I8PTy = llvm::PointerType::get(C&: getLLVMContext(), AddressSpace: GenericAS);
6248
6249 // Testing which overloaded version we should generate the call for.
6250 if (2U == E->getNumArgs()) {
6251 const char *Name = (BuiltinID == Builtin::BIread_pipe) ? "__read_pipe_2"
6252 : "__write_pipe_2";
6253 // Creating a generic function type to be able to call with any builtin or
6254 // user defined type.
6255 llvm::Type *ArgTys[] = {Arg0->getType(), I8PTy, Int32Ty, Int32Ty};
6256 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: Int32Ty, Params: ArgTys, isVarArg: false);
6257 Value *ACast = Builder.CreateAddrSpaceCast(V: Arg1, DestTy: I8PTy);
6258 return RValue::get(
6259 V: EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(Ty: FTy, Name),
6260 args: {Arg0, ACast, PacketSize, PacketAlign}));
6261 } else {
6262 assert(4 == E->getNumArgs() &&
6263 "Illegal number of parameters to pipe function");
6264 const char *Name = (BuiltinID == Builtin::BIread_pipe) ? "__read_pipe_4"
6265 : "__write_pipe_4";
6266
6267 llvm::Type *ArgTys[] = {Arg0->getType(), Arg1->getType(), Int32Ty, I8PTy,
6268 Int32Ty, Int32Ty};
6269 Value *Arg2 = EmitScalarExpr(E: E->getArg(Arg: 2)),
6270 *Arg3 = EmitScalarExpr(E: E->getArg(Arg: 3));
6271 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: Int32Ty, Params: ArgTys, isVarArg: false);
6272 Value *ACast = Builder.CreateAddrSpaceCast(V: Arg3, DestTy: I8PTy);
6273 // We know the third argument is an integer type, but we may need to cast
6274 // it to i32.
6275 if (Arg2->getType() != Int32Ty)
6276 Arg2 = Builder.CreateZExtOrTrunc(V: Arg2, DestTy: Int32Ty);
6277 return RValue::get(
6278 V: EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(Ty: FTy, Name),
6279 args: {Arg0, Arg1, Arg2, ACast, PacketSize, PacketAlign}));
6280 }
6281 }
6282 // OpenCL v2.0 s6.13.16 ,s9.17.3.5 - Built-in pipe reserve read and write
6283 // functions
6284 case Builtin::BIreserve_read_pipe:
6285 case Builtin::BIreserve_write_pipe:
6286 case Builtin::BIwork_group_reserve_read_pipe:
6287 case Builtin::BIwork_group_reserve_write_pipe:
6288 case Builtin::BIsub_group_reserve_read_pipe:
6289 case Builtin::BIsub_group_reserve_write_pipe: {
6290 // Composing the mangled name for the function.
6291 const char *Name;
6292 if (BuiltinID == Builtin::BIreserve_read_pipe)
6293 Name = "__reserve_read_pipe";
6294 else if (BuiltinID == Builtin::BIreserve_write_pipe)
6295 Name = "__reserve_write_pipe";
6296 else if (BuiltinID == Builtin::BIwork_group_reserve_read_pipe)
6297 Name = "__work_group_reserve_read_pipe";
6298 else if (BuiltinID == Builtin::BIwork_group_reserve_write_pipe)
6299 Name = "__work_group_reserve_write_pipe";
6300 else if (BuiltinID == Builtin::BIsub_group_reserve_read_pipe)
6301 Name = "__sub_group_reserve_read_pipe";
6302 else
6303 Name = "__sub_group_reserve_write_pipe";
6304
6305 Value *Arg0 = EmitScalarExpr(E: E->getArg(Arg: 0)),
6306 *Arg1 = EmitScalarExpr(E: E->getArg(Arg: 1));
6307 llvm::Type *ReservedIDTy = ConvertType(T: getContext().OCLReserveIDTy);
6308 CGOpenCLRuntime OpenCLRT(CGM);
6309 Value *PacketSize = OpenCLRT.getPipeElemSize(PipeArg: E->getArg(Arg: 0));
6310 Value *PacketAlign = OpenCLRT.getPipeElemAlign(PipeArg: E->getArg(Arg: 0));
6311
6312 // Building the generic function prototype.
6313 llvm::Type *ArgTys[] = {Arg0->getType(), Int32Ty, Int32Ty, Int32Ty};
6314 llvm::FunctionType *FTy =
6315 llvm::FunctionType::get(Result: ReservedIDTy, Params: ArgTys, isVarArg: false);
6316 // We know the second argument is an integer type, but we may need to cast
6317 // it to i32.
6318 if (Arg1->getType() != Int32Ty)
6319 Arg1 = Builder.CreateZExtOrTrunc(V: Arg1, DestTy: Int32Ty);
6320 return RValue::get(V: EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(Ty: FTy, Name),
6321 args: {Arg0, Arg1, PacketSize, PacketAlign}));
6322 }
6323 // OpenCL v2.0 s6.13.16, s9.17.3.5 - Built-in pipe commit read and write
6324 // functions
6325 case Builtin::BIcommit_read_pipe:
6326 case Builtin::BIcommit_write_pipe:
6327 case Builtin::BIwork_group_commit_read_pipe:
6328 case Builtin::BIwork_group_commit_write_pipe:
6329 case Builtin::BIsub_group_commit_read_pipe:
6330 case Builtin::BIsub_group_commit_write_pipe: {
6331 const char *Name;
6332 if (BuiltinID == Builtin::BIcommit_read_pipe)
6333 Name = "__commit_read_pipe";
6334 else if (BuiltinID == Builtin::BIcommit_write_pipe)
6335 Name = "__commit_write_pipe";
6336 else if (BuiltinID == Builtin::BIwork_group_commit_read_pipe)
6337 Name = "__work_group_commit_read_pipe";
6338 else if (BuiltinID == Builtin::BIwork_group_commit_write_pipe)
6339 Name = "__work_group_commit_write_pipe";
6340 else if (BuiltinID == Builtin::BIsub_group_commit_read_pipe)
6341 Name = "__sub_group_commit_read_pipe";
6342 else
6343 Name = "__sub_group_commit_write_pipe";
6344
6345 Value *Arg0 = EmitScalarExpr(E: E->getArg(Arg: 0)),
6346 *Arg1 = EmitScalarExpr(E: E->getArg(Arg: 1));
6347 CGOpenCLRuntime OpenCLRT(CGM);
6348 Value *PacketSize = OpenCLRT.getPipeElemSize(PipeArg: E->getArg(Arg: 0));
6349 Value *PacketAlign = OpenCLRT.getPipeElemAlign(PipeArg: E->getArg(Arg: 0));
6350
6351 // Building the generic function prototype.
6352 llvm::Type *ArgTys[] = {Arg0->getType(), Arg1->getType(), Int32Ty, Int32Ty};
6353 llvm::FunctionType *FTy = llvm::FunctionType::get(
6354 Result: llvm::Type::getVoidTy(C&: getLLVMContext()), Params: ArgTys, isVarArg: false);
6355
6356 return RValue::get(V: EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(Ty: FTy, Name),
6357 args: {Arg0, Arg1, PacketSize, PacketAlign}));
6358 }
6359 // OpenCL v2.0 s6.13.16.4 Built-in pipe query functions
6360 case Builtin::BIget_pipe_num_packets:
6361 case Builtin::BIget_pipe_max_packets: {
6362 const char *BaseName;
6363 const auto *PipeTy = E->getArg(Arg: 0)->getType()->castAs<PipeType>();
6364 if (BuiltinID == Builtin::BIget_pipe_num_packets)
6365 BaseName = "__get_pipe_num_packets";
6366 else
6367 BaseName = "__get_pipe_max_packets";
6368 std::string Name = std::string(BaseName) +
6369 std::string(PipeTy->isReadOnly() ? "_ro" : "_wo");
6370
6371 // Building the generic function prototype.
6372 Value *Arg0 = EmitScalarExpr(E: E->getArg(Arg: 0));
6373 CGOpenCLRuntime OpenCLRT(CGM);
6374 Value *PacketSize = OpenCLRT.getPipeElemSize(PipeArg: E->getArg(Arg: 0));
6375 Value *PacketAlign = OpenCLRT.getPipeElemAlign(PipeArg: E->getArg(Arg: 0));
6376 llvm::Type *ArgTys[] = {Arg0->getType(), Int32Ty, Int32Ty};
6377 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: Int32Ty, Params: ArgTys, isVarArg: false);
6378
6379 return RValue::get(V: EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(Ty: FTy, Name),
6380 args: {Arg0, PacketSize, PacketAlign}));
6381 }
6382
6383 // OpenCL v2.0 s6.13.9 - Address space qualifier functions.
6384 case Builtin::BIto_global:
6385 case Builtin::BIto_local:
6386 case Builtin::BIto_private: {
6387 auto Arg0 = EmitScalarExpr(E: E->getArg(Arg: 0));
6388 auto NewArgT = llvm::PointerType::get(
6389 C&: getLLVMContext(),
6390 AddressSpace: CGM.getContext().getTargetAddressSpace(AS: LangAS::opencl_generic));
6391 auto NewRetT = llvm::PointerType::get(
6392 C&: getLLVMContext(),
6393 AddressSpace: CGM.getContext().getTargetAddressSpace(
6394 AS: E->getType()->getPointeeType().getAddressSpace()));
6395 auto FTy = llvm::FunctionType::get(Result: NewRetT, Params: {NewArgT}, isVarArg: false);
6396 llvm::Value *NewArg;
6397 if (Arg0->getType()->getPointerAddressSpace() !=
6398 NewArgT->getPointerAddressSpace())
6399 NewArg = Builder.CreateAddrSpaceCast(V: Arg0, DestTy: NewArgT);
6400 else
6401 NewArg = Builder.CreateBitOrPointerCast(V: Arg0, DestTy: NewArgT);
6402 auto NewName = std::string("__") + E->getDirectCallee()->getName().str();
6403 auto NewCall =
6404 EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(Ty: FTy, Name: NewName), args: {NewArg});
6405 return RValue::get(V: Builder.CreateBitOrPointerCast(V: NewCall,
6406 DestTy: ConvertType(T: E->getType())));
6407 }
6408
6409 // OpenCL v2.0, s6.13.17 - Enqueue kernel function.
6410 // Table 6.13.17.1 specifies four overload forms of enqueue_kernel.
6411 // The code below expands the builtin call to a call to one of the following
6412 // functions that an OpenCL runtime library will have to provide:
6413 // __enqueue_kernel_basic
6414 // __enqueue_kernel_varargs
6415 // __enqueue_kernel_basic_events
6416 // __enqueue_kernel_events_varargs
6417 case Builtin::BIenqueue_kernel: {
6418 StringRef Name; // Generated function call name
6419 unsigned NumArgs = E->getNumArgs();
6420
6421 llvm::Type *QueueTy = ConvertType(T: getContext().OCLQueueTy);
6422 llvm::Type *GenericVoidPtrTy = Builder.getPtrTy(
6423 AddrSpace: getContext().getTargetAddressSpace(AS: LangAS::opencl_generic));
6424
6425 llvm::Value *Queue = EmitScalarExpr(E: E->getArg(Arg: 0));
6426 llvm::Value *Flags = EmitScalarExpr(E: E->getArg(Arg: 1));
6427 LValue NDRangeL = EmitAggExprToLValue(E: E->getArg(Arg: 2));
6428 llvm::Value *Range = NDRangeL.getAddress().emitRawPointer(CGF&: *this);
6429
6430 // FIXME: Look through the addrspacecast which may exist to the stack
6431 // temporary as a hack.
6432 //
6433 // This is hardcoding the assumed ABI of the target function. This assumes
6434 // direct passing for every argument except NDRange, which is assumed to be
6435 // byval or byref indirect passed.
6436 //
6437 // This should be fixed to query a signature from CGOpenCLRuntime, and go
6438 // through EmitCallArgs to get the correct target ABI.
6439 Range = Range->stripPointerCasts();
6440
6441 llvm::Type *RangePtrTy = Range->getType();
6442
6443 if (NumArgs == 4) {
6444 // The most basic form of the call with parameters:
6445 // queue_t, kernel_enqueue_flags_t, ndrange_t, block(void)
6446 Name = "__enqueue_kernel_basic";
6447 llvm::Type *ArgTys[] = {QueueTy, Int32Ty, RangePtrTy, GenericVoidPtrTy,
6448 GenericVoidPtrTy};
6449 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: Int32Ty, Params: ArgTys, isVarArg: false);
6450
6451 auto Info =
6452 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(CGF&: *this, E: E->getArg(Arg: 3));
6453 llvm::Value *Kernel =
6454 Builder.CreatePointerCast(V: Info.KernelHandle, DestTy: GenericVoidPtrTy);
6455 llvm::Value *Block =
6456 Builder.CreatePointerCast(V: Info.BlockArg, DestTy: GenericVoidPtrTy);
6457
6458 auto RTCall = EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(Ty: FTy, Name),
6459 args: {Queue, Flags, Range, Kernel, Block});
6460 return RValue::get(V: RTCall);
6461 }
6462 assert(NumArgs >= 5 && "Invalid enqueue_kernel signature");
6463
6464 // Create a temporary array to hold the sizes of local pointer arguments
6465 // for the block. \p First is the position of the first size argument.
6466 auto CreateArrayForSizeVar =
6467 [=](unsigned First) -> std::pair<llvm::Value *, llvm::Value *> {
6468 llvm::APInt ArraySize(32, NumArgs - First);
6469 QualType SizeArrayTy = getContext().getConstantArrayType(
6470 EltTy: getContext().getSizeType(), ArySize: ArraySize, SizeExpr: nullptr,
6471 ASM: ArraySizeModifier::Normal,
6472 /*IndexTypeQuals=*/0);
6473 auto Tmp = CreateMemTempWithoutCast(T: SizeArrayTy, Name: "block_sizes");
6474 llvm::Value *Alloca = Tmp.getPointer();
6475 llvm::Value *ElemPtr;
6476 EmitLifetimeStart(Addr: Alloca);
6477 // Each of the following arguments specifies the size of the corresponding
6478 // argument passed to the enqueued block.
6479 auto *Zero = llvm::ConstantInt::get(Ty: IntTy, V: 0);
6480 for (unsigned I = First; I < NumArgs; ++I) {
6481 auto *Index = llvm::ConstantInt::get(Ty: IntTy, V: I - First);
6482 auto *GEP =
6483 Builder.CreateGEP(Ty: Tmp.getElementType(), Ptr: Alloca, IdxList: {Zero, Index});
6484 if (I == First)
6485 ElemPtr = GEP;
6486 auto *V =
6487 Builder.CreateZExtOrTrunc(V: EmitScalarExpr(E: E->getArg(Arg: I)), DestTy: SizeTy);
6488 Builder.CreateAlignedStore(
6489 Val: V, Ptr: GEP, Align: CGM.getDataLayout().getPrefTypeAlign(Ty: SizeTy));
6490 }
6491 return {ElemPtr, Alloca};
6492 };
6493
6494 // Could have events and/or varargs.
6495 if (E->getArg(Arg: 3)->getType()->isBlockPointerType()) {
6496 // No events passed, but has variadic arguments.
6497 Name = "__enqueue_kernel_varargs";
6498 auto Info =
6499 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(CGF&: *this, E: E->getArg(Arg: 3));
6500 llvm::Value *Kernel =
6501 Builder.CreatePointerCast(V: Info.KernelHandle, DestTy: GenericVoidPtrTy);
6502 auto *Block = Builder.CreatePointerCast(V: Info.BlockArg, DestTy: GenericVoidPtrTy);
6503 auto [ElemPtr, TmpPtr] = CreateArrayForSizeVar(4);
6504
6505 // Create a vector of the arguments, as well as a constant value to
6506 // express to the runtime the number of variadic arguments.
6507 llvm::Value *const Args[] = {Queue, Flags,
6508 Range, Kernel,
6509 Block, ConstantInt::get(Ty: IntTy, V: NumArgs - 4),
6510 ElemPtr};
6511 llvm::Type *const ArgTys[] = {
6512 QueueTy, IntTy, RangePtrTy, GenericVoidPtrTy,
6513 GenericVoidPtrTy, IntTy, ElemPtr->getType()};
6514
6515 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: Int32Ty, Params: ArgTys, isVarArg: false);
6516 auto Call = RValue::get(
6517 V: EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(Ty: FTy, Name), args: Args));
6518 EmitLifetimeEnd(Addr: TmpPtr);
6519 return Call;
6520 }
6521 // Any calls now have event arguments passed.
6522 if (NumArgs >= 7) {
6523 llvm::PointerType *PtrTy = llvm::PointerType::get(
6524 C&: CGM.getLLVMContext(),
6525 AddressSpace: CGM.getContext().getTargetAddressSpace(AS: LangAS::opencl_generic));
6526
6527 llvm::Value *NumEvents =
6528 Builder.CreateZExtOrTrunc(V: EmitScalarExpr(E: E->getArg(Arg: 3)), DestTy: Int32Ty);
6529
6530 // Since SemaOpenCLBuiltinEnqueueKernel allows fifth and sixth arguments
6531 // to be a null pointer constant (including `0` literal), we can take it
6532 // into account and emit null pointer directly.
6533 llvm::Value *EventWaitList = nullptr;
6534 if (E->getArg(Arg: 4)->isNullPointerConstant(
6535 Ctx&: getContext(), NPC: Expr::NPC_ValueDependentIsNotNull)) {
6536 EventWaitList = llvm::ConstantPointerNull::get(T: PtrTy);
6537 } else {
6538 EventWaitList =
6539 E->getArg(Arg: 4)->getType()->isArrayType()
6540 ? EmitArrayToPointerDecay(Array: E->getArg(Arg: 4)).emitRawPointer(CGF&: *this)
6541 : EmitScalarExpr(E: E->getArg(Arg: 4));
6542 // Convert to generic address space.
6543 EventWaitList = Builder.CreatePointerCast(V: EventWaitList, DestTy: PtrTy);
6544 }
6545 llvm::Value *EventRet = nullptr;
6546 if (E->getArg(Arg: 5)->isNullPointerConstant(
6547 Ctx&: getContext(), NPC: Expr::NPC_ValueDependentIsNotNull)) {
6548 EventRet = llvm::ConstantPointerNull::get(T: PtrTy);
6549 } else {
6550 EventRet =
6551 Builder.CreatePointerCast(V: EmitScalarExpr(E: E->getArg(Arg: 5)), DestTy: PtrTy);
6552 }
6553
6554 auto Info =
6555 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(CGF&: *this, E: E->getArg(Arg: 6));
6556 llvm::Value *Kernel =
6557 Builder.CreatePointerCast(V: Info.KernelHandle, DestTy: GenericVoidPtrTy);
6558 llvm::Value *Block =
6559 Builder.CreatePointerCast(V: Info.BlockArg, DestTy: GenericVoidPtrTy);
6560
6561 std::vector<llvm::Type *> ArgTys = {
6562 QueueTy, Int32Ty, RangePtrTy, Int32Ty,
6563 PtrTy, PtrTy, GenericVoidPtrTy, GenericVoidPtrTy};
6564
6565 std::vector<llvm::Value *> Args = {Queue, Flags, Range,
6566 NumEvents, EventWaitList, EventRet,
6567 Kernel, Block};
6568
6569 if (NumArgs == 7) {
6570 // Has events but no variadics.
6571 Name = "__enqueue_kernel_basic_events";
6572 llvm::FunctionType *FTy =
6573 llvm::FunctionType::get(Result: Int32Ty, Params: ArgTys, isVarArg: false);
6574 return RValue::get(
6575 V: EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(Ty: FTy, Name), args: Args));
6576 }
6577 // Has event info and variadics
6578 // Pass the number of variadics to the runtime function too.
6579 Args.push_back(x: ConstantInt::get(Ty: Int32Ty, V: NumArgs - 7));
6580 ArgTys.push_back(x: Int32Ty);
6581 Name = "__enqueue_kernel_events_varargs";
6582
6583 auto [ElemPtr, TmpPtr] = CreateArrayForSizeVar(7);
6584 Args.push_back(x: ElemPtr);
6585 ArgTys.push_back(x: ElemPtr->getType());
6586
6587 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: Int32Ty, Params: ArgTys, isVarArg: false);
6588 auto Call = RValue::get(
6589 V: EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(Ty: FTy, Name), args: Args));
6590 EmitLifetimeEnd(Addr: TmpPtr);
6591 return Call;
6592 }
6593 llvm_unreachable("Unexpected enqueue_kernel signature");
6594 }
6595 // OpenCL v2.0 s6.13.17.6 - Kernel query functions need bitcast of block
6596 // parameter.
6597 case Builtin::BIget_kernel_work_group_size: {
6598 llvm::Type *GenericVoidPtrTy = Builder.getPtrTy(
6599 AddrSpace: getContext().getTargetAddressSpace(AS: LangAS::opencl_generic));
6600 auto Info =
6601 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(CGF&: *this, E: E->getArg(Arg: 0));
6602 Value *Kernel =
6603 Builder.CreatePointerCast(V: Info.KernelHandle, DestTy: GenericVoidPtrTy);
6604 Value *Arg = Builder.CreatePointerCast(V: Info.BlockArg, DestTy: GenericVoidPtrTy);
6605 return RValue::get(V: EmitRuntimeCall(
6606 callee: CGM.CreateRuntimeFunction(
6607 Ty: llvm::FunctionType::get(Result: IntTy, Params: {GenericVoidPtrTy, GenericVoidPtrTy},
6608 isVarArg: false),
6609 Name: "__get_kernel_work_group_size_impl"),
6610 args: {Kernel, Arg}));
6611 }
6612 case Builtin::BIget_kernel_preferred_work_group_size_multiple: {
6613 llvm::Type *GenericVoidPtrTy = Builder.getPtrTy(
6614 AddrSpace: getContext().getTargetAddressSpace(AS: LangAS::opencl_generic));
6615 auto Info =
6616 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(CGF&: *this, E: E->getArg(Arg: 0));
6617 Value *Kernel =
6618 Builder.CreatePointerCast(V: Info.KernelHandle, DestTy: GenericVoidPtrTy);
6619 Value *Arg = Builder.CreatePointerCast(V: Info.BlockArg, DestTy: GenericVoidPtrTy);
6620 return RValue::get(V: EmitRuntimeCall(
6621 callee: CGM.CreateRuntimeFunction(
6622 Ty: llvm::FunctionType::get(Result: IntTy, Params: {GenericVoidPtrTy, GenericVoidPtrTy},
6623 isVarArg: false),
6624 Name: "__get_kernel_preferred_work_group_size_multiple_impl"),
6625 args: {Kernel, Arg}));
6626 }
6627 case Builtin::BIget_kernel_max_sub_group_size_for_ndrange:
6628 case Builtin::BIget_kernel_sub_group_count_for_ndrange: {
6629 llvm::Type *GenericVoidPtrTy = Builder.getPtrTy(
6630 AddrSpace: getContext().getTargetAddressSpace(AS: LangAS::opencl_generic));
6631 LValue NDRangeL = EmitAggExprToLValue(E: E->getArg(Arg: 0));
6632 llvm::Value *NDRange = NDRangeL.getAddress().emitRawPointer(CGF&: *this);
6633 auto Info =
6634 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(CGF&: *this, E: E->getArg(Arg: 1));
6635 Value *Kernel =
6636 Builder.CreatePointerCast(V: Info.KernelHandle, DestTy: GenericVoidPtrTy);
6637 Value *Block = Builder.CreatePointerCast(V: Info.BlockArg, DestTy: GenericVoidPtrTy);
6638 const char *Name =
6639 BuiltinID == Builtin::BIget_kernel_max_sub_group_size_for_ndrange
6640 ? "__get_kernel_max_sub_group_size_for_ndrange_impl"
6641 : "__get_kernel_sub_group_count_for_ndrange_impl";
6642 return RValue::get(V: EmitRuntimeCall(
6643 callee: CGM.CreateRuntimeFunction(
6644 Ty: llvm::FunctionType::get(
6645 Result: IntTy, Params: {NDRange->getType(), GenericVoidPtrTy, GenericVoidPtrTy},
6646 isVarArg: false),
6647 Name),
6648 args: {NDRange, Kernel, Block}));
6649 }
6650 case Builtin::BI__builtin_store_half:
6651 case Builtin::BI__builtin_store_halff: {
6652 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
6653 Value *Val = EmitScalarExpr(E: E->getArg(Arg: 0));
6654 Address Address = EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
6655 Value *HalfVal = Builder.CreateFPTrunc(V: Val, DestTy: Builder.getHalfTy());
6656 Builder.CreateStore(Val: HalfVal, Addr: Address);
6657 return RValue::get(V: nullptr);
6658 }
6659 case Builtin::BI__builtin_load_half: {
6660 Address Address = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
6661 Value *HalfVal = Builder.CreateLoad(Addr: Address);
6662 return RValue::get(V: Builder.CreateFPExt(V: HalfVal, DestTy: Builder.getDoubleTy()));
6663 }
6664 case Builtin::BI__builtin_load_halff: {
6665 Address Address = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
6666 Value *HalfVal = Builder.CreateLoad(Addr: Address);
6667 return RValue::get(V: Builder.CreateFPExt(V: HalfVal, DestTy: Builder.getFloatTy()));
6668 }
6669 case Builtin::BI__builtin_printf:
6670 case Builtin::BIprintf:
6671 if (getTarget().getTriple().isNVPTX() ||
6672 getTarget().getTriple().isAMDGCN() ||
6673 (getTarget().getTriple().isSPIRV() &&
6674 getTarget().getTriple().getVendor() == Triple::VendorType::AMD)) {
6675 if (getTarget().getTriple().isNVPTX())
6676 return EmitNVPTXDevicePrintfCallExpr(E);
6677 if ((getTarget().getTriple().isAMDGCN() ||
6678 getTarget().getTriple().isSPIRV()) &&
6679 getLangOpts().HIP)
6680 return EmitAMDGPUDevicePrintfCallExpr(E);
6681 }
6682
6683 break;
6684 case Builtin::BI__builtin_canonicalize:
6685 case Builtin::BI__builtin_canonicalizef:
6686 case Builtin::BI__builtin_canonicalizef16:
6687 case Builtin::BI__builtin_canonicalizel:
6688 return RValue::get(
6689 V: emitBuiltinWithOneOverloadedType<1>(CGF&: *this, E, IntrinsicID: Intrinsic::canonicalize));
6690
6691 case Builtin::BI__builtin_thread_pointer: {
6692 if (!getContext().getTargetInfo().isTLSSupported())
6693 CGM.ErrorUnsupported(S: E, Type: "__builtin_thread_pointer");
6694
6695 return RValue::get(V: Builder.CreateIntrinsic(ID: llvm::Intrinsic::thread_pointer,
6696 OverloadTypes: {GlobalsInt8PtrTy}, Args: {}));
6697 }
6698 case Builtin::BI__builtin_os_log_format:
6699 return emitBuiltinOSLogFormat(E: *E);
6700
6701 case Builtin::BI__xray_customevent: {
6702 if (!ShouldXRayInstrumentFunction())
6703 return RValue::getIgnored();
6704
6705 if (!CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
6706 K: XRayInstrKind::Custom))
6707 return RValue::getIgnored();
6708
6709 if (const auto *XRayAttr = CurFuncDecl->getAttr<XRayInstrumentAttr>())
6710 if (XRayAttr->neverXRayInstrument() && !AlwaysEmitXRayCustomEvents())
6711 return RValue::getIgnored();
6712
6713 Function *F = CGM.getIntrinsic(IID: Intrinsic::xray_customevent);
6714 auto FTy = F->getFunctionType();
6715 auto Arg0 = E->getArg(Arg: 0);
6716 auto Arg0Val = EmitScalarExpr(E: Arg0);
6717 auto Arg0Ty = Arg0->getType();
6718 auto PTy0 = FTy->getParamType(i: 0);
6719 if (PTy0 != Arg0Val->getType()) {
6720 if (Arg0Ty->isArrayType())
6721 Arg0Val = EmitArrayToPointerDecay(Array: Arg0).emitRawPointer(CGF&: *this);
6722 else
6723 Arg0Val = Builder.CreatePointerCast(V: Arg0Val, DestTy: PTy0);
6724 }
6725 auto Arg1 = EmitScalarExpr(E: E->getArg(Arg: 1));
6726 auto PTy1 = FTy->getParamType(i: 1);
6727 if (PTy1 != Arg1->getType())
6728 Arg1 = Builder.CreateTruncOrBitCast(V: Arg1, DestTy: PTy1);
6729 return RValue::get(V: Builder.CreateCall(Callee: F, Args: {Arg0Val, Arg1}));
6730 }
6731
6732 case Builtin::BI__xray_typedevent: {
6733 // TODO: There should be a way to always emit events even if the current
6734 // function is not instrumented. Losing events in a stream can cripple
6735 // a trace.
6736 if (!ShouldXRayInstrumentFunction())
6737 return RValue::getIgnored();
6738
6739 if (!CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
6740 K: XRayInstrKind::Typed))
6741 return RValue::getIgnored();
6742
6743 if (const auto *XRayAttr = CurFuncDecl->getAttr<XRayInstrumentAttr>())
6744 if (XRayAttr->neverXRayInstrument() && !AlwaysEmitXRayTypedEvents())
6745 return RValue::getIgnored();
6746
6747 Function *F = CGM.getIntrinsic(IID: Intrinsic::xray_typedevent);
6748 auto FTy = F->getFunctionType();
6749 auto Arg0 = EmitScalarExpr(E: E->getArg(Arg: 0));
6750 auto PTy0 = FTy->getParamType(i: 0);
6751 if (PTy0 != Arg0->getType())
6752 Arg0 = Builder.CreateTruncOrBitCast(V: Arg0, DestTy: PTy0);
6753 auto Arg1 = E->getArg(Arg: 1);
6754 auto Arg1Val = EmitScalarExpr(E: Arg1);
6755 auto Arg1Ty = Arg1->getType();
6756 auto PTy1 = FTy->getParamType(i: 1);
6757 if (PTy1 != Arg1Val->getType()) {
6758 if (Arg1Ty->isArrayType())
6759 Arg1Val = EmitArrayToPointerDecay(Array: Arg1).emitRawPointer(CGF&: *this);
6760 else
6761 Arg1Val = Builder.CreatePointerCast(V: Arg1Val, DestTy: PTy1);
6762 }
6763 auto Arg2 = EmitScalarExpr(E: E->getArg(Arg: 2));
6764 auto PTy2 = FTy->getParamType(i: 2);
6765 if (PTy2 != Arg2->getType())
6766 Arg2 = Builder.CreateTruncOrBitCast(V: Arg2, DestTy: PTy2);
6767 return RValue::get(V: Builder.CreateCall(Callee: F, Args: {Arg0, Arg1Val, Arg2}));
6768 }
6769
6770 case Builtin::BI__builtin_ms_va_start:
6771 case Builtin::BI__builtin_ms_va_end:
6772 return RValue::get(
6773 V: EmitVAStartEnd(ArgValue: EmitMSVAListRef(E: E->getArg(Arg: 0)).emitRawPointer(CGF&: *this),
6774 IsStart: BuiltinID == Builtin::BI__builtin_ms_va_start));
6775
6776 case Builtin::BI__builtin_ms_va_copy: {
6777 // Lower this manually. We can't reliably determine whether or not any
6778 // given va_copy() is for a Win64 va_list from the calling convention
6779 // alone, because it's legal to do this from a System V ABI function.
6780 // With opaque pointer types, we won't have enough information in LLVM
6781 // IR to determine this from the argument types, either. Best to do it
6782 // now, while we have enough information.
6783 Address DestAddr = EmitMSVAListRef(E: E->getArg(Arg: 0));
6784 Address SrcAddr = EmitMSVAListRef(E: E->getArg(Arg: 1));
6785
6786 DestAddr = DestAddr.withElementType(ElemTy: Int8PtrTy);
6787 SrcAddr = SrcAddr.withElementType(ElemTy: Int8PtrTy);
6788
6789 Value *ArgPtr = Builder.CreateLoad(Addr: SrcAddr, Name: "ap.val");
6790 return RValue::get(V: Builder.CreateStore(Val: ArgPtr, Addr: DestAddr));
6791 }
6792
6793 case Builtin::BI__builtin_zos_va_start:
6794 case Builtin::BI__builtin_zos_va_end: {
6795 // The va_list is an array with 2 elements, called curr and next.
6796 // Element curr is set to 0. For builtin_zos_va_start, next is initialized
6797 // with a call to @llvm.va_start. Otherwise, next is passed to @llvm.va_end.
6798 Address VAList = EmitZOSVAListRef(E: E->getArg(Arg: 0));
6799 llvm::Type *VAListTy = ConvertType(T: getContext().getBuiltinZOSVaListType());
6800 VAList = VAList.withElementType(ElemTy: VAListTy);
6801 Address Curr = Builder.CreateConstArrayGEP(Addr: VAList, Index: 0, Name: "curr");
6802 Value *Zero = llvm::Constant::getNullValue(Ty: VoidPtrTy);
6803 Builder.CreateStore(Val: Zero, Addr: Curr);
6804 Address Next = Builder.CreateConstArrayGEP(Addr: VAList, Index: 1, Name: "next");
6805 return RValue::get(
6806 V: EmitVAStartEnd(ArgValue: Next.emitRawPointer(CGF&: *this),
6807 IsStart: BuiltinID == Builtin::BI__builtin_zos_va_start));
6808 }
6809 case Builtin::BI__builtin_zos_va_copy: {
6810 // Lower this manually because later can't reliably determine the type.
6811 Address Dest = EmitZOSVAListRef(E: E->getArg(Arg: 0));
6812 Address Src = EmitZOSVAListRef(E: E->getArg(Arg: 1));
6813 llvm::Type *VAListTy = ConvertType(T: getContext().getBuiltinZOSVaListType());
6814 uint64_t SizeBytes =
6815 CGM.getDataLayout().getTypeAllocSize(Ty: VAListTy).getFixedValue();
6816 Value *SizeVal = llvm::ConstantInt::get(Ty: Int64Ty, V: SizeBytes);
6817 Builder.CreateMemCpy(Dest, Src, Size: SizeVal, IsVolatile: false);
6818 return RValue::get(V: Dest.emitRawPointer(CGF&: *this));
6819 }
6820
6821 case Builtin::BI__builtin_get_device_side_mangled_name: {
6822 auto Name = CGM.getCUDARuntime().getDeviceSideName(
6823 ND: cast<DeclRefExpr>(Val: E->getArg(Arg: 0)->IgnoreImpCasts())->getDecl());
6824 auto Str = CGM.GetAddrOfConstantCString(Str: Name, GlobalName: "");
6825 return RValue::get(V: Str.getPointer());
6826 }
6827 }
6828
6829 // If this is an alias for a lib function (e.g. __builtin_sin), emit
6830 // the call using the normal call path, but using the unmangled
6831 // version of the function name.
6832 const auto &BI = getContext().BuiltinInfo;
6833 if (!shouldEmitBuiltinAsIR(BuiltinID, BI, CGF: *this) &&
6834 BI.isLibFunction(ID: BuiltinID))
6835 return emitLibraryCall(CGF&: *this, FD, E,
6836 calleeValue: CGM.getBuiltinLibFunction(FD, BuiltinID));
6837
6838 // If this is a predefined lib function (e.g. malloc), emit the call
6839 // using exactly the normal call path.
6840 if (BI.isPredefinedLibFunction(ID: BuiltinID))
6841 return emitLibraryCall(CGF&: *this, FD, E, calleeValue: CGM.getRawFunctionPointer(GD: FD));
6842
6843 // Check that a call to a target specific builtin has the correct target
6844 // features.
6845 // This is down here to avoid non-target specific builtins, however, if
6846 // generic builtins start to require generic target features then we
6847 // can move this up to the beginning of the function.
6848 checkTargetFeatures(E, TargetDecl: FD);
6849
6850 if (unsigned VectorWidth = getContext().BuiltinInfo.getRequiredVectorWidth(ID: BuiltinID))
6851 LargestVectorWidth = std::max(a: LargestVectorWidth, b: VectorWidth);
6852
6853 // See if we have a target specific intrinsic.
6854 std::string Name = getContext().BuiltinInfo.getName(ID: BuiltinID);
6855 Intrinsic::ID IntrinsicID = Intrinsic::not_intrinsic;
6856 StringRef Prefix =
6857 llvm::Triple::getArchTypePrefix(Kind: getTarget().getTriple().getArch());
6858 if (!Prefix.empty()) {
6859 IntrinsicID = Intrinsic::getIntrinsicForClangBuiltin(TargetPrefix: Prefix.data(), BuiltinName: Name);
6860 if (IntrinsicID == Intrinsic::not_intrinsic && Prefix == "spv" &&
6861 getTarget().getTriple().getOS() == llvm::Triple::OSType::AMDHSA)
6862 IntrinsicID = Intrinsic::getIntrinsicForClangBuiltin(TargetPrefix: "amdgcn", BuiltinName: Name);
6863 // NOTE we don't need to perform a compatibility flag check here since the
6864 // intrinsics are declared in Builtins*.def via LANGBUILTIN which filter the
6865 // MS builtins via ALL_MS_LANGUAGES and are filtered earlier.
6866 if (IntrinsicID == Intrinsic::not_intrinsic)
6867 IntrinsicID = Intrinsic::getIntrinsicForMSBuiltin(TargetPrefix: Prefix.data(), BuiltinName: Name);
6868 }
6869
6870 if (IntrinsicID != Intrinsic::not_intrinsic) {
6871 SmallVector<Value*, 16> Args;
6872
6873 // Find out if any arguments are required to be integer constant
6874 // expressions.
6875 unsigned ICEArguments = 0;
6876 ASTContext::GetBuiltinTypeError Error;
6877 getContext().GetBuiltinType(ID: BuiltinID, Error, IntegerConstantArgs: &ICEArguments);
6878 assert(Error == ASTContext::GE_None && "Should not codegen an error");
6879
6880 Function *F = CGM.getIntrinsic(IID: IntrinsicID);
6881 llvm::FunctionType *FTy = F->getFunctionType();
6882
6883 for (unsigned i = 0, e = E->getNumArgs(); i != e; ++i) {
6884 Value *ArgValue = EmitScalarOrConstFoldImmArg(ICEArguments, Idx: i, E);
6885 // If the intrinsic arg type is different from the builtin arg type
6886 // we need to do a bit cast.
6887 llvm::Type *PTy = FTy->getParamType(i);
6888 if (PTy != ArgValue->getType()) {
6889 // XXX - vector of pointers?
6890 if (auto *PtrTy = dyn_cast<llvm::PointerType>(Val: PTy)) {
6891 if (PtrTy->getAddressSpace() !=
6892 ArgValue->getType()->getPointerAddressSpace()) {
6893 ArgValue = Builder.CreateAddrSpaceCast(
6894 V: ArgValue, DestTy: llvm::PointerType::get(C&: getLLVMContext(),
6895 AddressSpace: PtrTy->getAddressSpace()));
6896 }
6897 }
6898
6899 // Cast vector type (e.g., v256i32) to x86_amx, this only happen
6900 // in amx intrinsics.
6901 if (PTy->isX86_AMXTy())
6902 ArgValue = Builder.CreateIntrinsic(ID: Intrinsic::x86_cast_vector_to_tile,
6903 OverloadTypes: {ArgValue->getType()}, Args: {ArgValue});
6904 else
6905 ArgValue = Builder.CreateBitCast(V: ArgValue, DestTy: PTy);
6906 }
6907
6908 Args.push_back(Elt: ArgValue);
6909 }
6910
6911 Value *V = Builder.CreateCall(Callee: F, Args);
6912 QualType BuiltinRetType = E->getType();
6913
6914 llvm::Type *RetTy = VoidTy;
6915 if (!BuiltinRetType->isVoidType())
6916 RetTy = ConvertType(T: BuiltinRetType);
6917
6918 if (RetTy != V->getType()) {
6919 // XXX - vector of pointers?
6920 if (auto *PtrTy = dyn_cast<llvm::PointerType>(Val: RetTy)) {
6921 if (PtrTy->getAddressSpace() != V->getType()->getPointerAddressSpace()) {
6922 V = Builder.CreateAddrSpaceCast(
6923 V, DestTy: llvm::PointerType::get(C&: getLLVMContext(),
6924 AddressSpace: PtrTy->getAddressSpace()));
6925 }
6926 }
6927
6928 // Cast x86_amx to vector type (e.g., v256i32), this only happen
6929 // in amx intrinsics.
6930 if (V->getType()->isX86_AMXTy())
6931 V = Builder.CreateIntrinsic(ID: Intrinsic::x86_cast_tile_to_vector, OverloadTypes: {RetTy},
6932 Args: {V});
6933 else
6934 V = Builder.CreateBitCast(V, DestTy: RetTy);
6935 }
6936
6937 if (RetTy->isVoidTy())
6938 return RValue::get(V: nullptr);
6939
6940 return RValue::get(V);
6941 }
6942
6943 // Some target-specific builtins can have aggregate return values, e.g.
6944 // __builtin_arm_mve_vld2q_u32. So if the result is an aggregate, force
6945 // ReturnValue to be non-null, so that the target-specific emission code can
6946 // always just emit into it.
6947 TypeEvaluationKind EvalKind = getEvaluationKind(T: E->getType());
6948 if (EvalKind == TEK_Aggregate && ReturnValue.isNull()) {
6949 Address DestPtr = CreateMemTemp(T: E->getType(), Name: "agg.tmp");
6950 ReturnValue = ReturnValueSlot(DestPtr, false);
6951 }
6952
6953 // Now see if we can emit a target-specific builtin.
6954 if (Value *V = EmitTargetBuiltinExpr(BuiltinID, E, ReturnValue)) {
6955 switch (EvalKind) {
6956 case TEK_Scalar:
6957 if (V->getType()->isVoidTy())
6958 return RValue::get(V: nullptr);
6959 return RValue::get(V);
6960 case TEK_Aggregate:
6961 return RValue::getAggregate(addr: ReturnValue.getAddress(),
6962 isVolatile: ReturnValue.isVolatile());
6963 case TEK_Complex:
6964 llvm_unreachable("No current target builtin returns complex");
6965 }
6966 llvm_unreachable("Bad evaluation kind in EmitBuiltinExpr");
6967 }
6968
6969 // EmitHLSLBuiltinExpr will check getLangOpts().HLSL
6970 if (Value *V = EmitHLSLBuiltinExpr(BuiltinID, E, ReturnValue)) {
6971 switch (EvalKind) {
6972 case TEK_Scalar:
6973 if (V->getType()->isVoidTy())
6974 return RValue::get(V: nullptr);
6975 return RValue::get(V);
6976 case TEK_Aggregate:
6977 return RValue::getAggregate(addr: ReturnValue.getAddress(),
6978 isVolatile: ReturnValue.isVolatile());
6979 case TEK_Complex:
6980 llvm_unreachable("No current hlsl builtin returns complex");
6981 }
6982 llvm_unreachable("Bad evaluation kind in EmitBuiltinExpr");
6983 }
6984
6985 if (getLangOpts().HIPStdPar && getLangOpts().CUDAIsDevice)
6986 return EmitHipStdParUnsupportedBuiltin(CGF: this, FD);
6987
6988 ErrorUnsupported(S: E, Type: "builtin function");
6989
6990 // Unknown builtin, for now just dump it out and return undef.
6991 return GetUndefRValue(Ty: E->getType());
6992}
6993
6994namespace {
6995struct BuiltinAlignArgs {
6996 llvm::Value *Src = nullptr;
6997 llvm::Type *SrcType = nullptr;
6998 llvm::Value *Alignment = nullptr;
6999 llvm::Value *Mask = nullptr;
7000 llvm::IntegerType *IntType = nullptr;
7001
7002 BuiltinAlignArgs(const CallExpr *E, CodeGenFunction &CGF) {
7003 QualType AstType = E->getArg(Arg: 0)->getType();
7004 if (AstType->isArrayType())
7005 Src = CGF.EmitArrayToPointerDecay(Array: E->getArg(Arg: 0)).emitRawPointer(CGF);
7006 else
7007 Src = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
7008 SrcType = Src->getType();
7009 if (SrcType->isPointerTy()) {
7010 IntType = IntegerType::get(
7011 C&: CGF.getLLVMContext(),
7012 NumBits: CGF.CGM.getDataLayout().getIndexTypeSizeInBits(Ty: SrcType));
7013 } else {
7014 assert(SrcType->isIntegerTy());
7015 IntType = cast<llvm::IntegerType>(Val: SrcType);
7016 }
7017 Alignment = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
7018 Alignment = CGF.Builder.CreateZExtOrTrunc(V: Alignment, DestTy: IntType, Name: "alignment");
7019 auto *One = llvm::ConstantInt::get(Ty: IntType, V: 1);
7020 Mask = CGF.Builder.CreateSub(LHS: Alignment, RHS: One, Name: "mask");
7021 }
7022};
7023} // namespace
7024
7025/// Generate (x & (y-1)) == 0.
7026RValue CodeGenFunction::EmitBuiltinIsAligned(const CallExpr *E) {
7027 BuiltinAlignArgs Args(E, *this);
7028 llvm::Value *SrcAddress = Args.Src;
7029 if (Args.SrcType->isPointerTy())
7030 SrcAddress =
7031 Builder.CreateBitOrPointerCast(V: Args.Src, DestTy: Args.IntType, Name: "src_addr");
7032 return RValue::get(V: Builder.CreateICmpEQ(
7033 LHS: Builder.CreateAnd(LHS: SrcAddress, RHS: Args.Mask, Name: "set_bits"),
7034 RHS: llvm::Constant::getNullValue(Ty: Args.IntType), Name: "is_aligned"));
7035}
7036
7037/// Generate (x & ~(y-1)) to align down or ((x+(y-1)) & ~(y-1)) to align up.
7038/// Note: For pointer types we can avoid ptrtoint/inttoptr pairs by using the
7039/// llvm.ptrmask intrinsic (with a GEP before in the align_up case).
7040RValue CodeGenFunction::EmitBuiltinAlignTo(const CallExpr *E, bool AlignUp) {
7041 BuiltinAlignArgs Args(E, *this);
7042 llvm::Value *SrcForMask = Args.Src;
7043 if (AlignUp) {
7044 // When aligning up we have to first add the mask to ensure we go over the
7045 // next alignment value and then align down to the next valid multiple.
7046 // By adding the mask, we ensure that align_up on an already aligned
7047 // value will not change the value.
7048 if (Args.Src->getType()->isPointerTy()) {
7049 if (getLangOpts().PointerOverflowDefined)
7050 SrcForMask =
7051 Builder.CreateGEP(Ty: Int8Ty, Ptr: SrcForMask, IdxList: Args.Mask, Name: "over_boundary");
7052 else
7053 SrcForMask = EmitCheckedInBoundsGEP(ElemTy: Int8Ty, Ptr: SrcForMask, IdxList: Args.Mask,
7054 /*SignedIndices=*/true,
7055 /*isSubtraction=*/IsSubtraction: false,
7056 Loc: E->getExprLoc(), Name: "over_boundary");
7057 } else {
7058 SrcForMask = Builder.CreateAdd(LHS: SrcForMask, RHS: Args.Mask, Name: "over_boundary");
7059 }
7060 }
7061 // Invert the mask to only clear the lower bits.
7062 llvm::Value *InvertedMask = Builder.CreateNot(V: Args.Mask, Name: "inverted_mask");
7063 llvm::Value *Result = nullptr;
7064 if (Args.Src->getType()->isPointerTy()) {
7065 Result = Builder.CreateIntrinsic(
7066 ID: Intrinsic::ptrmask, OverloadTypes: {Args.SrcType, Args.IntType},
7067 Args: {SrcForMask, InvertedMask}, FMFSource: nullptr, Name: "aligned_result");
7068 } else {
7069 Result = Builder.CreateAnd(LHS: SrcForMask, RHS: InvertedMask, Name: "aligned_result");
7070 }
7071 assert(Result->getType() == Args.SrcType);
7072 return RValue::get(V: Result);
7073}
7074