1//===---- CGBuiltin.cpp - Emit LLVM Code for builtins ---------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit Builtin calls as LLVM code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGBuiltin.h"
14#include "ABIInfo.h"
15#include "CGCUDARuntime.h"
16#include "CGCXXABI.h"
17#include "CGDebugInfo.h"
18#include "CGObjCRuntime.h"
19#include "CGOpenCLRuntime.h"
20#include "CGRecordLayout.h"
21#include "CGValue.h"
22#include "CodeGenFunction.h"
23#include "CodeGenModule.h"
24#include "ConstantEmitter.h"
25#include "PatternInit.h"
26#include "TargetInfo.h"
27#include "clang/AST/InferAlloc.h"
28#include "clang/AST/OSLog.h"
29#include "clang/AST/StmtVisitor.h"
30#include "clang/Basic/DiagnosticFrontend.h"
31#include "clang/Basic/TargetInfo.h"
32#include "clang/CodeGenUtils/TargetUtils.h"
33#include "llvm/ADT/APFloat.h"
34#include "llvm/IR/InlineAsm.h"
35#include "llvm/IR/Instruction.h"
36#include "llvm/IR/Intrinsics.h"
37#include "llvm/IR/IntrinsicsX86.h"
38#include "llvm/IR/MatrixBuilder.h"
39#include "llvm/Support/ConvertUTF.h"
40#include "llvm/Support/ScopedPrinter.h"
41#include <algorithm>
42#include <optional>
43#include <utility>
44
45using namespace clang;
46using namespace CodeGen;
47using namespace llvm;
48
49/// Some builtins do not have library implementation on some targets and
50/// are instead emitted as LLVM IRs by some target builtin emitters.
51/// FIXME: Remove this when library support is added
52static bool shouldEmitBuiltinAsIR(unsigned BuiltinID,
53 const Builtin::Context &BI,
54 const CodeGenFunction &CGF) {
55 if (!CGF.CGM.getLangOpts().MathErrno &&
56 CGF.CurFPFeatures.getExceptionMode() ==
57 LangOptions::FPExceptionModeKind::FPE_Ignore &&
58 !CGF.CGM.getTargetCodeGenInfo().supportsLibCall()) {
59 switch (BuiltinID) {
60 default:
61 return false;
62 case Builtin::BIlogbf:
63 case Builtin::BI__builtin_logbf:
64 case Builtin::BIlogb:
65 case Builtin::BI__builtin_logb:
66 case Builtin::BIscalbnf:
67 case Builtin::BI__builtin_scalbnf:
68 case Builtin::BIscalbn:
69 case Builtin::BI__builtin_scalbn:
70 return true;
71 }
72 }
73 return false;
74}
75
76static Value *EmitTargetArchBuiltinExpr(CodeGenFunction *CGF,
77 unsigned BuiltinID, const CallExpr *E,
78 ReturnValueSlot ReturnValue,
79 llvm::Triple::ArchType Arch) {
80 // When compiling in HipStdPar mode we have to be conservative in rejecting
81 // target specific features in the FE, and defer the possible error to the
82 // AcceleratorCodeSelection pass, wherein iff an unsupported target builtin is
83 // referenced by an accelerator executable function, we emit an error.
84 // Returning nullptr here leads to the builtin being handled in
85 // EmitStdParUnsupportedBuiltin.
86 if (CGF->getLangOpts().HIPStdPar && CGF->getLangOpts().CUDAIsDevice &&
87 Arch != CGF->getTarget().getTriple().getArch())
88 return nullptr;
89
90 switch (Arch) {
91 case llvm::Triple::arm:
92 case llvm::Triple::armeb:
93 case llvm::Triple::thumb:
94 case llvm::Triple::thumbeb:
95 return CGF->EmitARMBuiltinExpr(BuiltinID, E, ReturnValue, Arch);
96 case llvm::Triple::aarch64:
97 case llvm::Triple::aarch64_32:
98 case llvm::Triple::aarch64_be:
99 return CGF->EmitAArch64BuiltinExpr(BuiltinID, E, Arch);
100 case llvm::Triple::bpfeb:
101 case llvm::Triple::bpfel:
102 return CGF->EmitBPFBuiltinExpr(BuiltinID, E);
103 case llvm::Triple::dxil:
104 return CGF->EmitDirectXBuiltinExpr(BuiltinID, E);
105 case llvm::Triple::x86:
106 case llvm::Triple::x86_64:
107 return CGF->EmitX86BuiltinExpr(BuiltinID, E);
108 case llvm::Triple::ppc:
109 case llvm::Triple::ppcle:
110 case llvm::Triple::ppc64:
111 case llvm::Triple::ppc64le:
112 return CGF->EmitPPCBuiltinExpr(BuiltinID, E);
113 case llvm::Triple::amdgpu:
114 case llvm::Triple::r600:
115 return CGF->EmitAMDGPUBuiltinExpr(BuiltinID, E);
116 case llvm::Triple::systemz:
117 return CGF->EmitSystemZBuiltinExpr(BuiltinID, E);
118 case llvm::Triple::nvptx:
119 case llvm::Triple::nvptx64:
120 return CGF->EmitNVPTXBuiltinExpr(BuiltinID, E);
121 case llvm::Triple::wasm32:
122 case llvm::Triple::wasm64:
123 return CGF->EmitWebAssemblyBuiltinExpr(BuiltinID, E);
124 case llvm::Triple::hexagon:
125 return CGF->EmitHexagonBuiltinExpr(BuiltinID, E);
126 case llvm::Triple::riscv32:
127 case llvm::Triple::riscv64:
128 case llvm::Triple::riscv32be:
129 case llvm::Triple::riscv64be:
130 return CGF->EmitRISCVBuiltinExpr(BuiltinID, E, ReturnValue);
131 case llvm::Triple::spirv32:
132 case llvm::Triple::spirv64:
133 if (CGF->getTarget().getTriple().getOS() == llvm::Triple::OSType::AMDHSA)
134 return CGF->EmitAMDGPUBuiltinExpr(BuiltinID, E);
135 [[fallthrough]];
136 case llvm::Triple::spirv:
137 return CGF->EmitSPIRVBuiltinExpr(BuiltinID, E);
138 case llvm::Triple::avr:
139 return CGF->EmitAVRBuiltinExpr(BuiltinID, E);
140 default:
141 return nullptr;
142 }
143}
144
145Value *CodeGenFunction::EmitTargetBuiltinExpr(unsigned BuiltinID,
146 const CallExpr *E,
147 ReturnValueSlot ReturnValue) {
148 if (getContext().BuiltinInfo.isAuxBuiltinID(ID: BuiltinID)) {
149 assert(getContext().getAuxTargetInfo() && "Missing aux target info");
150 return EmitTargetArchBuiltinExpr(
151 CGF: this, BuiltinID: getContext().BuiltinInfo.getAuxBuiltinID(ID: BuiltinID), E,
152 ReturnValue, Arch: getContext().getAuxTargetInfo()->getTriple().getArch());
153 }
154
155 return EmitTargetArchBuiltinExpr(CGF: this, BuiltinID, E, ReturnValue,
156 Arch: getTarget().getTriple().getArch());
157}
158
159static void initializeAlloca(CodeGenFunction &CGF, AllocaInst *AI, Value *Size,
160 Align AlignmentInBytes) {
161 ConstantInt *Byte;
162 switch (CGF.getLangOpts().getTrivialAutoVarInit()) {
163 case LangOptions::TrivialAutoVarInitKind::Uninitialized:
164 // Nothing to initialize.
165 return;
166 case LangOptions::TrivialAutoVarInitKind::Zero:
167 Byte = CGF.Builder.getInt8(C: 0x00);
168 break;
169 case LangOptions::TrivialAutoVarInitKind::Pattern: {
170 llvm::Type *Int8 = llvm::IntegerType::getInt8Ty(C&: CGF.CGM.getLLVMContext());
171 Byte = llvm::dyn_cast<llvm::ConstantInt>(
172 Val: initializationPatternFor(CGF.CGM, Int8));
173 break;
174 }
175 }
176 if (CGF.CGM.stopAutoInit())
177 return;
178 auto *I = CGF.Builder.CreateMemSet(Ptr: AI, Val: Byte, Size, Align: AlignmentInBytes);
179 I->addAnnotationMetadata(Annotation: "auto-init");
180}
181
182/// getBuiltinLibFunction - Given a builtin id for a function like
183/// "__builtin_fabsf", return a Function* for "fabsf".
184llvm::Constant *CodeGenModule::getBuiltinLibFunction(const FunctionDecl *FD,
185 unsigned BuiltinID) {
186 assert(Context.BuiltinInfo.isLibFunction(BuiltinID));
187
188 // Get the name, skip over the __builtin_ prefix (if necessary). We may have
189 // to build this up so provide a small stack buffer to handle the vast
190 // majority of names.
191 llvm::SmallString<64> Name;
192 GlobalDecl D(FD);
193
194 // If the builtin has been declared explicitly with an assembler label,
195 // use the mangled name. This differs from the plain label on platforms
196 // that prefix labels.
197 if (FD->hasAttr<AsmLabelAttr>())
198 Name = getMangledName(GD: D);
199 else if (StringRef TargetName = CodeGenUtils::getTargetBuiltinLibFunctionName(
200 BuiltinID, Target: getTarget());
201 !TargetName.empty())
202 Name = TargetName;
203 else
204 Name = Context.BuiltinInfo.getName(ID: BuiltinID).substr(pos: 10);
205
206 llvm::FunctionType *Ty =
207 cast<llvm::FunctionType>(Val: getTypes().ConvertType(T: FD->getType()));
208
209 return GetOrCreateLLVMFunction(MangledName: Name, Ty, D, /*ForVTable=*/false);
210}
211
212void appendDefaultIntrinsicArgs(SmallVectorImpl<llvm::Value *> &Args,
213 llvm::Function *F) {
214 llvm::FunctionType *FTy = F->getFunctionType();
215 unsigned NumParams = FTy->getNumParams();
216 if (Args.size() >= NumParams)
217 return;
218
219 auto [FirstDefault, Defaults] =
220 Intrinsic::getAllDefaultArgValues(IID: F->getIntrinsicID());
221 assert(Args.size() >= FirstDefault &&
222 "builtin passes fewer arguments than the intrinsic requires");
223
224 for (unsigned I = Args.size(); I != NumParams; ++I) {
225 llvm::Type *ParamTy = FTy->getParamType(i: I);
226 unsigned DefaultIdx = I - FirstDefault;
227 assert(ParamTy->isIntegerTy() &&
228 "intrinsic default arguments must be integer-typed");
229 Args.push_back(Elt: llvm::ConstantInt::get(Ty: ParamTy, V: Defaults[DefaultIdx]));
230 }
231}
232
233/// Emit the conversions required to turn the given value into an
234/// integer of the given size.
235Value *EmitToInt(CodeGenFunction &CGF, llvm::Value *V,
236 QualType T, llvm::IntegerType *IntType) {
237 V = CGF.EmitToMemory(Value: V, Ty: T);
238
239 if (V->getType()->isPointerTy())
240 return CGF.Builder.CreatePtrToInt(V, DestTy: IntType);
241
242 assert(V->getType() == IntType);
243 return V;
244}
245
246Value *EmitFromInt(CodeGenFunction &CGF, llvm::Value *V,
247 QualType T, llvm::Type *ResultType) {
248 V = CGF.EmitFromMemory(Value: V, Ty: T);
249
250 if (ResultType->isPointerTy())
251 return CGF.Builder.CreateIntToPtr(V, DestTy: ResultType);
252
253 assert(V->getType() == ResultType);
254 return V;
255}
256
257Address CheckAtomicAlignment(CodeGenFunction &CGF, const CallExpr *E) {
258 ASTContext &Ctx = CGF.getContext();
259 Address Ptr = CGF.EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
260 const llvm::DataLayout &DL = CGF.CGM.getDataLayout();
261 unsigned Bytes = Ptr.getElementType()->isPointerTy()
262 ? Ctx.getTypeSizeInChars(T: Ctx.VoidPtrTy).getQuantity()
263 : DL.getTypeStoreSize(Ty: Ptr.getElementType());
264 unsigned Align = Ptr.getAlignment().getQuantity();
265 if (Align % Bytes != 0) {
266 DiagnosticsEngine &Diags = CGF.CGM.getDiags();
267 Diags.Report(Loc: E->getBeginLoc(), DiagID: diag::warn_sync_op_misaligned);
268 // Force address to be at least naturally-aligned.
269 return Ptr.withAlignment(NewAlignment: CharUnits::fromQuantity(Quantity: Bytes));
270 }
271 return Ptr;
272}
273
274/// Utility to insert an atomic instruction based on Intrinsic::ID
275/// and the expression node.
276Value *MakeBinaryAtomicValue(CodeGenFunction &CGF,
277 llvm::AtomicRMWInst::BinOp Kind, const CallExpr *E,
278 AtomicOrdering Ordering,
279 llvm::SyncScope::ID SSID) {
280
281 QualType T = E->getType();
282 assert(E->getArg(0)->getType()->isPointerType());
283 assert(CGF.getContext().hasSameUnqualifiedType(T,
284 E->getArg(0)->getType()->getPointeeType()));
285 assert(CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType()));
286
287 Address DestAddr = CheckAtomicAlignment(CGF, E);
288
289 llvm::IntegerType *IntType = llvm::IntegerType::get(
290 C&: CGF.getLLVMContext(), NumBits: CGF.getContext().getTypeSize(T));
291
292 llvm::Value *Val = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
293 llvm::Type *ValueType = Val->getType();
294 Val = EmitToInt(CGF, V: Val, T, IntType);
295
296 llvm::Value *Result =
297 CGF.Builder.CreateAtomicRMW(Op: Kind, Addr: DestAddr, Val, Ordering, SSID);
298 // Consider atomics to be volatile in MS kernel mode.
299 if (CGF.CGM.getLangOpts().Kernel)
300 cast<llvm::AtomicRMWInst>(Val: Result)->setVolatile(true);
301 return EmitFromInt(CGF, V: Result, T, ResultType: ValueType);
302}
303
304static Value *EmitNontemporalStore(CodeGenFunction &CGF, const CallExpr *E) {
305 Value *Val = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
306 Address Addr = CGF.EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
307
308 Val = CGF.EmitToMemory(Value: Val, Ty: E->getArg(Arg: 0)->getType());
309 LValue LV = CGF.MakeAddrLValue(Addr, T: E->getArg(Arg: 0)->getType());
310 LV.setNontemporal(true);
311 CGF.EmitStoreOfScalar(value: Val, lvalue: LV, isInit: false);
312 return nullptr;
313}
314
315static Value *EmitNontemporalLoad(CodeGenFunction &CGF, const CallExpr *E) {
316 Address Addr = CGF.EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
317
318 LValue LV = CGF.MakeAddrLValue(Addr, T: E->getType());
319 LV.setNontemporal(true);
320 return CGF.EmitLoadOfScalar(lvalue: LV, Loc: E->getExprLoc());
321}
322
323static RValue EmitBinaryAtomic(CodeGenFunction &CGF,
324 llvm::AtomicRMWInst::BinOp Kind,
325 const CallExpr *E) {
326 return RValue::get(V: MakeBinaryAtomicValue(CGF, Kind, E));
327}
328
329/// Utility to insert an atomic instruction based Intrinsic::ID and
330/// the expression node, where the return value is the result of the
331/// operation.
332static RValue EmitBinaryAtomicPost(CodeGenFunction &CGF,
333 llvm::AtomicRMWInst::BinOp Kind,
334 const CallExpr *E,
335 Instruction::BinaryOps Op,
336 bool Invert = false) {
337 QualType T = E->getType();
338 assert(E->getArg(0)->getType()->isPointerType());
339 assert(CGF.getContext().hasSameUnqualifiedType(T,
340 E->getArg(0)->getType()->getPointeeType()));
341 assert(CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType()));
342
343 Address DestAddr = CheckAtomicAlignment(CGF, E);
344
345 llvm::IntegerType *IntType = llvm::IntegerType::get(
346 C&: CGF.getLLVMContext(), NumBits: CGF.getContext().getTypeSize(T));
347
348 llvm::Value *Val = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
349 llvm::Type *ValueType = Val->getType();
350 Val = EmitToInt(CGF, V: Val, T, IntType);
351
352 llvm::Value *Result = CGF.Builder.CreateAtomicRMW(
353 Op: Kind, Addr: DestAddr, Val, Ordering: llvm::AtomicOrdering::SequentiallyConsistent);
354 Result = CGF.Builder.CreateBinOp(Opc: Op, LHS: Result, RHS: Val);
355 if (Invert)
356 Result =
357 CGF.Builder.CreateBinOp(Opc: llvm::Instruction::Xor, LHS: Result,
358 RHS: llvm::ConstantInt::getAllOnesValue(Ty: IntType));
359 Result = EmitFromInt(CGF, V: Result, T, ResultType: ValueType);
360 return RValue::get(V: Result);
361}
362
363/// Utility to insert an atomic cmpxchg instruction.
364///
365/// @param CGF The current codegen function.
366/// @param E Builtin call expression to convert to cmpxchg.
367/// arg0 - address to operate on
368/// arg1 - value to compare with
369/// arg2 - new value
370/// @param ReturnBool Specifies whether to return success flag of
371/// cmpxchg result or the old value.
372///
373/// @returns result of cmpxchg, according to ReturnBool
374///
375/// Note: In order to lower Microsoft's _InterlockedCompareExchange* intrinsics
376/// invoke the function EmitAtomicCmpXchgForMSIntrin.
377Value *MakeAtomicCmpXchgValue(CodeGenFunction &CGF, const CallExpr *E,
378 bool ReturnBool,
379 llvm::AtomicOrdering SuccessOrdering,
380 llvm::AtomicOrdering FailureOrdering,
381 llvm::SyncScope::ID SSID) {
382 QualType T = ReturnBool ? E->getArg(Arg: 1)->getType() : E->getType();
383 Address DestAddr = CheckAtomicAlignment(CGF, E);
384
385 llvm::IntegerType *IntType = llvm::IntegerType::get(
386 C&: CGF.getLLVMContext(), NumBits: CGF.getContext().getTypeSize(T));
387
388 Value *Cmp = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
389 llvm::Type *ValueType = Cmp->getType();
390 Cmp = EmitToInt(CGF, V: Cmp, T, IntType);
391 Value *New = EmitToInt(CGF, V: CGF.EmitScalarExpr(E: E->getArg(Arg: 2)), T, IntType);
392
393 Value *Pair = CGF.Builder.CreateAtomicCmpXchg(
394 Addr: DestAddr, Cmp, New, SuccessOrdering, FailureOrdering, SSID);
395 if (ReturnBool)
396 // Extract boolean success flag and zext it to int.
397 return CGF.Builder.CreateZExt(V: CGF.Builder.CreateExtractValue(Agg: Pair, Idxs: 1),
398 DestTy: CGF.ConvertType(T: E->getType()));
399 else
400 // Extract old value and emit it using the same type as compare value.
401 return EmitFromInt(CGF, V: CGF.Builder.CreateExtractValue(Agg: Pair, Idxs: 0), T,
402 ResultType: ValueType);
403}
404
405/// This function should be invoked to emit atomic cmpxchg for Microsoft's
406/// _InterlockedCompareExchange* intrinsics which have the following signature:
407/// T _InterlockedCompareExchange(T volatile *Destination,
408/// T Exchange,
409/// T Comparand);
410///
411/// Whereas the llvm 'cmpxchg' instruction has the following syntax:
412/// cmpxchg *Destination, Comparand, Exchange.
413/// So we need to swap Comparand and Exchange when invoking
414/// CreateAtomicCmpXchg. That is the reason we could not use the above utility
415/// function MakeAtomicCmpXchgValue since it expects the arguments to be
416/// already swapped.
417
418static
419Value *EmitAtomicCmpXchgForMSIntrin(CodeGenFunction &CGF, const CallExpr *E,
420 AtomicOrdering SuccessOrdering = AtomicOrdering::SequentiallyConsistent) {
421 assert(E->getArg(0)->getType()->isPointerType());
422 assert(CGF.getContext().hasSameUnqualifiedType(
423 E->getType(), E->getArg(0)->getType()->getPointeeType()));
424 assert(CGF.getContext().hasSameUnqualifiedType(E->getType(),
425 E->getArg(1)->getType()));
426 assert(CGF.getContext().hasSameUnqualifiedType(E->getType(),
427 E->getArg(2)->getType()));
428
429 Address DestAddr = CheckAtomicAlignment(CGF, E);
430
431 auto *Exchange = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
432 auto *RTy = Exchange->getType();
433
434 auto *Comparand = CGF.EmitScalarExpr(E: E->getArg(Arg: 2));
435
436 if (RTy->isPointerTy()) {
437 Exchange = CGF.Builder.CreatePtrToInt(V: Exchange, DestTy: CGF.IntPtrTy);
438 Comparand = CGF.Builder.CreatePtrToInt(V: Comparand, DestTy: CGF.IntPtrTy);
439 }
440
441 // For Release ordering, the failure ordering should be Monotonic.
442 auto FailureOrdering = SuccessOrdering == AtomicOrdering::Release ?
443 AtomicOrdering::Monotonic :
444 SuccessOrdering;
445
446 // The atomic instruction is marked volatile for consistency with MSVC. This
447 // blocks the few atomics optimizations that LLVM has. If we want to optimize
448 // _Interlocked* operations in the future, we will have to remove the volatile
449 // marker.
450 auto *CmpXchg = CGF.Builder.CreateAtomicCmpXchg(
451 Addr: DestAddr, Cmp: Comparand, New: Exchange, SuccessOrdering, FailureOrdering);
452 CmpXchg->setVolatile(true);
453
454 auto *Result = CGF.Builder.CreateExtractValue(Agg: CmpXchg, Idxs: 0);
455 if (RTy->isPointerTy()) {
456 Result = CGF.Builder.CreateIntToPtr(V: Result, DestTy: RTy);
457 }
458
459 return Result;
460}
461
462// 64-bit Microsoft platforms support 128 bit cmpxchg operations. They are
463// prototyped like this:
464//
465// unsigned char _InterlockedCompareExchange128...(
466// __int64 volatile * _Destination,
467// __int64 _ExchangeHigh,
468// __int64 _ExchangeLow,
469// __int64 * _ComparandResult);
470//
471// Note that Destination is assumed to be at least 16-byte aligned, despite
472// being typed int64.
473
474static Value *EmitAtomicCmpXchg128ForMSIntrin(CodeGenFunction &CGF,
475 const CallExpr *E,
476 AtomicOrdering SuccessOrdering) {
477 assert(E->getNumArgs() == 4);
478 llvm::Value *DestPtr = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
479 llvm::Value *ExchangeHigh = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
480 llvm::Value *ExchangeLow = CGF.EmitScalarExpr(E: E->getArg(Arg: 2));
481 Address ComparandAddr = CGF.EmitPointerWithAlignment(Addr: E->getArg(Arg: 3));
482
483 assert(DestPtr->getType()->isPointerTy());
484 assert(!ExchangeHigh->getType()->isPointerTy());
485 assert(!ExchangeLow->getType()->isPointerTy());
486
487 // For Release ordering, the failure ordering should be Monotonic.
488 auto FailureOrdering = SuccessOrdering == AtomicOrdering::Release
489 ? AtomicOrdering::Monotonic
490 : SuccessOrdering;
491
492 // Convert to i128 pointers and values. Alignment is also overridden for
493 // destination pointer.
494 llvm::Type *Int128Ty = llvm::IntegerType::get(C&: CGF.getLLVMContext(), NumBits: 128);
495 Address DestAddr(DestPtr, Int128Ty,
496 CGF.getContext().toCharUnitsFromBits(BitSize: 128));
497 ComparandAddr = ComparandAddr.withElementType(ElemTy: Int128Ty);
498
499 // (((i128)hi) << 64) | ((i128)lo)
500 ExchangeHigh = CGF.Builder.CreateZExt(V: ExchangeHigh, DestTy: Int128Ty);
501 ExchangeLow = CGF.Builder.CreateZExt(V: ExchangeLow, DestTy: Int128Ty);
502 ExchangeHigh =
503 CGF.Builder.CreateShl(LHS: ExchangeHigh, RHS: llvm::ConstantInt::get(Ty: Int128Ty, V: 64));
504 llvm::Value *Exchange = CGF.Builder.CreateOr(LHS: ExchangeHigh, RHS: ExchangeLow);
505
506 // Load the comparand for the instruction.
507 llvm::Value *Comparand = CGF.Builder.CreateLoad(Addr: ComparandAddr);
508
509 auto *CXI = CGF.Builder.CreateAtomicCmpXchg(Addr: DestAddr, Cmp: Comparand, New: Exchange,
510 SuccessOrdering, FailureOrdering);
511
512 // The atomic instruction is marked volatile for consistency with MSVC. This
513 // blocks the few atomics optimizations that LLVM has. If we want to optimize
514 // _Interlocked* operations in the future, we will have to remove the volatile
515 // marker.
516 CXI->setVolatile(true);
517
518 // Store the result as an outparameter.
519 CGF.Builder.CreateStore(Val: CGF.Builder.CreateExtractValue(Agg: CXI, Idxs: 0),
520 Addr: ComparandAddr);
521
522 // Get the success boolean and zero extend it to i8.
523 Value *Success = CGF.Builder.CreateExtractValue(Agg: CXI, Idxs: 1);
524 return CGF.Builder.CreateZExt(V: Success, DestTy: CGF.Int8Ty);
525}
526
527static Value *EmitAtomicIncrementValue(CodeGenFunction &CGF, const CallExpr *E,
528 AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
529 assert(E->getArg(0)->getType()->isPointerType());
530
531 auto *IntTy = CGF.ConvertType(T: E->getType());
532 Address DestAddr = CheckAtomicAlignment(CGF, E);
533 auto *Result = CGF.Builder.CreateAtomicRMW(
534 Op: AtomicRMWInst::Add, Addr: DestAddr, Val: ConstantInt::get(Ty: IntTy, V: 1), Ordering);
535 return CGF.Builder.CreateAdd(LHS: Result, RHS: ConstantInt::get(Ty: IntTy, V: 1));
536}
537
538static Value *EmitAtomicDecrementValue(
539 CodeGenFunction &CGF, const CallExpr *E,
540 AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
541 assert(E->getArg(0)->getType()->isPointerType());
542
543 auto *IntTy = CGF.ConvertType(T: E->getType());
544 Address DestAddr = CheckAtomicAlignment(CGF, E);
545 auto *Result = CGF.Builder.CreateAtomicRMW(
546 Op: AtomicRMWInst::Sub, Addr: DestAddr, Val: ConstantInt::get(Ty: IntTy, V: 1), Ordering);
547 return CGF.Builder.CreateSub(LHS: Result, RHS: ConstantInt::get(Ty: IntTy, V: 1));
548}
549
550// Build a plain volatile load.
551static Value *EmitISOVolatileLoad(CodeGenFunction &CGF, const CallExpr *E) {
552 Value *Ptr = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
553 QualType ElTy = E->getArg(Arg: 0)->getType()->getPointeeType();
554 CharUnits LoadSize = CGF.getContext().getTypeSizeInChars(T: ElTy);
555 llvm::Type *ITy =
556 llvm::IntegerType::get(C&: CGF.getLLVMContext(), NumBits: LoadSize.getQuantity() * 8);
557 llvm::LoadInst *Load = CGF.Builder.CreateAlignedLoad(Ty: ITy, Addr: Ptr, Align: LoadSize);
558 Load->setAtomic(Ordering: llvm::AtomicOrdering::Monotonic);
559 Load->setVolatile(true);
560 return Load;
561}
562
563// Build a plain volatile store.
564static Value *EmitISOVolatileStore(CodeGenFunction &CGF, const CallExpr *E) {
565 Value *Ptr = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
566 Value *Value = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
567 QualType ElTy = E->getArg(Arg: 0)->getType()->getPointeeType();
568 CharUnits StoreSize = CGF.getContext().getTypeSizeInChars(T: ElTy);
569 llvm::StoreInst *Store =
570 CGF.Builder.CreateAlignedStore(Val: Value, Addr: Ptr, Align: StoreSize);
571 Store->setAtomic(Ordering: llvm::AtomicOrdering::Monotonic);
572 Store->setVolatile(true);
573 return Store;
574}
575
576// Emit a simple mangled intrinsic that has 1 argument and a return type
577// matching the argument type. Depending on mode, this may be a constrained
578// floating-point intrinsic.
579Value *emitUnaryMaybeConstrainedFPBuiltin(CodeGenFunction &CGF,
580 const CallExpr *E, unsigned IntrinsicID,
581 unsigned ConstrainedIntrinsicID) {
582 llvm::Value *Src0 = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
583
584 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
585 if (CGF.Builder.getIsFPConstrained()) {
586 Function *F = CGF.CGM.getIntrinsic(IID: ConstrainedIntrinsicID, Tys: Src0->getType());
587 return CGF.Builder.CreateConstrainedFPCall(Callee: F, Args: { Src0 });
588 } else {
589 Function *F = CGF.CGM.getIntrinsic(IID: IntrinsicID, Tys: Src0->getType());
590 return CGF.Builder.CreateCall(Callee: F, Args: Src0);
591 }
592}
593
594// Emit an intrinsic that has 2 operands of the same type as its result.
595// Depending on mode, this may be a constrained floating-point intrinsic.
596static Value *emitBinaryMaybeConstrainedFPBuiltin(CodeGenFunction &CGF,
597 const CallExpr *E, unsigned IntrinsicID,
598 unsigned ConstrainedIntrinsicID) {
599 llvm::Value *Src0 = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
600 llvm::Value *Src1 = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
601
602 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
603 if (CGF.Builder.getIsFPConstrained()) {
604 Function *F = CGF.CGM.getIntrinsic(IID: ConstrainedIntrinsicID, Tys: Src0->getType());
605 return CGF.Builder.CreateConstrainedFPCall(Callee: F, Args: { Src0, Src1 });
606 } else {
607 Function *F = CGF.CGM.getIntrinsic(IID: IntrinsicID, Tys: Src0->getType());
608 return CGF.Builder.CreateCall(Callee: F, Args: { Src0, Src1 });
609 }
610}
611
612// Has second type mangled argument.
613static Value *
614emitBinaryExpMaybeConstrainedFPBuiltin(CodeGenFunction &CGF, const CallExpr *E,
615 Intrinsic::ID IntrinsicID,
616 Intrinsic::ID ConstrainedIntrinsicID) {
617 llvm::Value *Src0 = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
618 llvm::Value *Src1 = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
619
620 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
621 if (CGF.Builder.getIsFPConstrained()) {
622 Function *F = CGF.CGM.getIntrinsic(IID: ConstrainedIntrinsicID,
623 Tys: {Src0->getType(), Src1->getType()});
624 return CGF.Builder.CreateConstrainedFPCall(Callee: F, Args: {Src0, Src1});
625 }
626
627 Function *F =
628 CGF.CGM.getIntrinsic(IID: IntrinsicID, Tys: {Src0->getType(), Src1->getType()});
629 return CGF.Builder.CreateCall(Callee: F, Args: {Src0, Src1});
630}
631
632// Emit an intrinsic that has 3 operands of the same type as its result.
633// Depending on mode, this may be a constrained floating-point intrinsic.
634static Value *emitTernaryMaybeConstrainedFPBuiltin(CodeGenFunction &CGF,
635 const CallExpr *E, unsigned IntrinsicID,
636 unsigned ConstrainedIntrinsicID) {
637 llvm::Value *Src0 = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
638 llvm::Value *Src1 = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
639 llvm::Value *Src2 = CGF.EmitScalarExpr(E: E->getArg(Arg: 2));
640
641 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
642 if (CGF.Builder.getIsFPConstrained()) {
643 Function *F = CGF.CGM.getIntrinsic(IID: ConstrainedIntrinsicID, Tys: Src0->getType());
644 return CGF.Builder.CreateConstrainedFPCall(Callee: F, Args: { Src0, Src1, Src2 });
645 } else {
646 Function *F = CGF.CGM.getIntrinsic(IID: IntrinsicID, Tys: Src0->getType());
647 return CGF.Builder.CreateCall(Callee: F, Args: { Src0, Src1, Src2 });
648 }
649}
650
651// Emit an intrinsic that has overloaded integer result and fp operand.
652static Value *
653emitMaybeConstrainedFPToIntRoundBuiltin(CodeGenFunction &CGF, const CallExpr *E,
654 unsigned IntrinsicID,
655 unsigned ConstrainedIntrinsicID) {
656 llvm::Type *ResultType = CGF.ConvertType(T: E->getType());
657 llvm::Value *Src0 = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
658
659 if (CGF.Builder.getIsFPConstrained()) {
660 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
661 Function *F = CGF.CGM.getIntrinsic(IID: ConstrainedIntrinsicID,
662 Tys: {ResultType, Src0->getType()});
663 return CGF.Builder.CreateConstrainedFPCall(Callee: F, Args: {Src0});
664 } else {
665 Function *F =
666 CGF.CGM.getIntrinsic(IID: IntrinsicID, Tys: {ResultType, Src0->getType()});
667 return CGF.Builder.CreateCall(Callee: F, Args: Src0);
668 }
669}
670
671static Value *emitFrexpBuiltin(CodeGenFunction &CGF, const CallExpr *E,
672 Intrinsic::ID IntrinsicID) {
673 llvm::Value *Src0 = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
674 llvm::Value *Src1 = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
675
676 QualType IntPtrTy = E->getArg(Arg: 1)->getType()->getPointeeType();
677 llvm::Type *IntTy = CGF.ConvertType(T: IntPtrTy);
678 llvm::Function *F =
679 CGF.CGM.getIntrinsic(IID: IntrinsicID, Tys: {Src0->getType(), IntTy});
680 llvm::Value *Call = CGF.Builder.CreateCall(Callee: F, Args: Src0);
681
682 llvm::Value *Exp = CGF.Builder.CreateExtractValue(Agg: Call, Idxs: 1);
683 LValue LV = CGF.MakeNaturalAlignAddrLValue(V: Src1, T: IntPtrTy);
684 CGF.EmitStoreOfScalar(value: Exp, lvalue: LV);
685
686 return CGF.Builder.CreateExtractValue(Agg: Call, Idxs: 0);
687}
688
689static void emitSincosBuiltin(CodeGenFunction &CGF, const CallExpr *E,
690 Intrinsic::ID IntrinsicID) {
691 llvm::Value *Val = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
692 llvm::Value *Dest0 = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
693 llvm::Value *Dest1 = CGF.EmitScalarExpr(E: E->getArg(Arg: 2));
694
695 llvm::Function *F = CGF.CGM.getIntrinsic(IID: IntrinsicID, Tys: {Val->getType()});
696 llvm::Value *Call = CGF.Builder.CreateCall(Callee: F, Args: Val);
697
698 llvm::Value *SinResult = CGF.Builder.CreateExtractValue(Agg: Call, Idxs: 0);
699 llvm::Value *CosResult = CGF.Builder.CreateExtractValue(Agg: Call, Idxs: 1);
700
701 QualType DestPtrType = E->getArg(Arg: 1)->getType()->getPointeeType();
702 LValue SinLV = CGF.MakeNaturalAlignAddrLValue(V: Dest0, T: DestPtrType);
703 LValue CosLV = CGF.MakeNaturalAlignAddrLValue(V: Dest1, T: DestPtrType);
704
705 llvm::StoreInst *StoreSin =
706 CGF.Builder.CreateStore(Val: SinResult, Addr: SinLV.getAddress());
707 llvm::StoreInst *StoreCos =
708 CGF.Builder.CreateStore(Val: CosResult, Addr: CosLV.getAddress());
709
710 // Mark the two stores as non-aliasing with each other. The order of stores
711 // emitted by this builtin is arbitrary, enforcing a particular order will
712 // prevent optimizations later on.
713 llvm::MDBuilder MDHelper(CGF.getLLVMContext());
714 MDNode *Domain = MDHelper.createAnonymousAliasScopeDomain();
715 MDNode *AliasScope = MDHelper.createAnonymousAliasScope(Domain);
716 MDNode *AliasScopeList = MDNode::get(Context&: Call->getContext(), MDs: AliasScope);
717 StoreSin->setMetadata(KindID: LLVMContext::MD_alias_scope, Node: AliasScopeList);
718 StoreCos->setMetadata(KindID: LLVMContext::MD_noalias, Node: AliasScopeList);
719}
720
721static llvm::Value *emitModfBuiltin(CodeGenFunction &CGF, const CallExpr *E,
722 Intrinsic::ID IntrinsicID) {
723 llvm::Value *Val = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
724 llvm::Value *IntPartDest = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
725
726 llvm::Value *Call =
727 CGF.Builder.CreateIntrinsic(ID: IntrinsicID, OverloadTypes: {Val->getType()}, Args: Val);
728
729 llvm::Value *FractionalResult = CGF.Builder.CreateExtractValue(Agg: Call, Idxs: 0);
730 llvm::Value *IntegralResult = CGF.Builder.CreateExtractValue(Agg: Call, Idxs: 1);
731
732 QualType DestPtrType = E->getArg(Arg: 1)->getType()->getPointeeType();
733 LValue IntegralLV = CGF.MakeNaturalAlignAddrLValue(V: IntPartDest, T: DestPtrType);
734 CGF.EmitStoreOfScalar(value: IntegralResult, lvalue: IntegralLV);
735
736 return FractionalResult;
737}
738
739/// EmitFAbs - Emit a call to @llvm.fabs().
740static Value *EmitFAbs(CodeGenFunction &CGF, Value *V) {
741 return CGF.Builder.CreateFAbs(V);
742}
743
744/// Emit the computation of the sign bit for a floating point value. Returns
745/// the i1 sign bit value.
746static Value *EmitSignBit(CodeGenFunction &CGF, Value *V) {
747 LLVMContext &C = CGF.CGM.getLLVMContext();
748
749 llvm::Type *Ty = V->getType();
750 int Width = Ty->getPrimitiveSizeInBits();
751 llvm::Type *IntTy = llvm::IntegerType::get(C, NumBits: Width);
752 V = CGF.Builder.CreateBitCast(V, DestTy: IntTy);
753 if (Ty->isPPC_FP128Ty()) {
754 // We want the sign bit of the higher-order double. The bitcast we just
755 // did works as if the double-double was stored to memory and then
756 // read as an i128. The "store" will put the higher-order double in the
757 // lower address in both little- and big-Endian modes, but the "load"
758 // will treat those bits as a different part of the i128: the low bits in
759 // little-Endian, the high bits in big-Endian. Therefore, on big-Endian
760 // we need to shift the high bits down to the low before truncating.
761 Width >>= 1;
762 if (CGF.getTarget().isBigEndian()) {
763 Value *ShiftCst = llvm::ConstantInt::get(Ty: IntTy, V: Width);
764 V = CGF.Builder.CreateLShr(LHS: V, RHS: ShiftCst);
765 }
766 // We are truncating value in order to extract the higher-order
767 // double, which we will be using to extract the sign from.
768 IntTy = llvm::IntegerType::get(C, NumBits: Width);
769 V = CGF.Builder.CreateTrunc(V, DestTy: IntTy);
770 }
771 Value *Zero = llvm::Constant::getNullValue(Ty: IntTy);
772 return CGF.Builder.CreateICmpSLT(LHS: V, RHS: Zero);
773}
774
775static RValue emitLibraryCall(CodeGenFunction &CGF, const FunctionDecl *FD,
776 const CallExpr *E, llvm::Constant *calleeValue) {
777 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
778 CGCallee callee = CGCallee::forDirect(functionPtr: calleeValue, abstractInfo: GlobalDecl(FD));
779 llvm::CallBase *callOrInvoke = nullptr;
780 CGFunctionInfo const *FnInfo = nullptr;
781 return CGF.EmitCall(FnType: E->getCallee()->getType(), Callee: callee, E, ReturnValue: ReturnValueSlot(),
782 /*Chain=*/nullptr, CallOrInvoke: &callOrInvoke, ResolvedFnInfo: &FnInfo);
783}
784
785/// Emit a call to llvm.{sadd,uadd,ssub,usub,smul,umul}.with.overflow.*
786/// depending on IntrinsicID.
787///
788/// \arg CGF The current codegen function.
789/// \arg IntrinsicID The ID for the Intrinsic we wish to generate.
790/// \arg X The first argument to the llvm.*.with.overflow.*.
791/// \arg Y The second argument to the llvm.*.with.overflow.*.
792/// \arg Carry The carry returned by the llvm.*.with.overflow.*.
793/// \returns The result (i.e. sum/product) returned by the intrinsic.
794llvm::Value *EmitOverflowIntrinsic(CodeGenFunction &CGF,
795 const Intrinsic::ID IntrinsicID,
796 llvm::Value *X, llvm::Value *Y,
797 llvm::Value *&Carry) {
798 // Make sure we have integers of the same width.
799 assert(X->getType() == Y->getType() &&
800 "Arguments must be the same type. (Did you forget to make sure both "
801 "arguments have the same integer width?)");
802
803 Function *Callee = CGF.CGM.getIntrinsic(IID: IntrinsicID, Tys: X->getType());
804 llvm::Value *Tmp = CGF.Builder.CreateCall(Callee, Args: {X, Y});
805 Carry = CGF.Builder.CreateExtractValue(Agg: Tmp, Idxs: 1);
806 return CGF.Builder.CreateExtractValue(Agg: Tmp, Idxs: 0);
807}
808
809namespace {
810 struct WidthAndSignedness {
811 unsigned Width;
812 bool Signed;
813 };
814}
815
816static WidthAndSignedness
817getIntegerWidthAndSignedness(const clang::ASTContext &context,
818 const clang::QualType Type) {
819 assert(Type->isIntegerType() && "Given type is not an integer.");
820 unsigned Width = context.getIntWidth(T: Type);
821 bool Signed = Type->isSignedIntegerType();
822 return {.Width: Width, .Signed: Signed};
823}
824
825// Given one or more integer types, this function produces an integer type that
826// encompasses them: any value in one of the given types could be expressed in
827// the encompassing type.
828static struct WidthAndSignedness
829EncompassingIntegerType(ArrayRef<struct WidthAndSignedness> Types) {
830 assert(Types.size() > 0 && "Empty list of types.");
831
832 // If any of the given types is signed, we must return a signed type.
833 bool Signed = false;
834 for (const auto &Type : Types) {
835 Signed |= Type.Signed;
836 }
837
838 // The encompassing type must have a width greater than or equal to the width
839 // of the specified types. Additionally, if the encompassing type is signed,
840 // its width must be strictly greater than the width of any unsigned types
841 // given.
842 unsigned Width = 0;
843 for (const auto &Type : Types) {
844 unsigned MinWidth = Type.Width + (Signed && !Type.Signed);
845 if (Width < MinWidth) {
846 Width = MinWidth;
847 }
848 }
849
850 return {.Width: Width, .Signed: Signed};
851}
852
853Value *CodeGenFunction::EmitVAStartEnd(Value *ArgValue, bool IsStart) {
854 Intrinsic::ID inst = IsStart ? Intrinsic::vastart : Intrinsic::vaend;
855 return Builder.CreateCall(Callee: CGM.getIntrinsic(IID: inst, Tys: {ArgValue->getType()}),
856 Args: ArgValue);
857}
858
859/// Checks if using the result of __builtin_object_size(p, @p From) in place of
860/// __builtin_object_size(p, @p To) is correct
861static bool areBOSTypesCompatible(int From, int To) {
862 // Note: Our __builtin_object_size implementation currently treats Type=0 and
863 // Type=2 identically. Encoding this implementation detail here may make
864 // improving __builtin_object_size difficult in the future, so it's omitted.
865 return From == To || (From == 0 && To == 1) || (From == 3 && To == 2);
866}
867
868static llvm::Value *
869getDefaultBuiltinObjectSizeResult(unsigned Type, llvm::IntegerType *ResType) {
870 return ConstantInt::get(Ty: ResType, V: (Type & 2) ? 0 : -1, /*isSigned=*/IsSigned: true);
871}
872
873llvm::Value *
874CodeGenFunction::evaluateOrEmitBuiltinObjectSize(const Expr *E, unsigned Type,
875 llvm::IntegerType *ResType,
876 llvm::Value *EmittedE,
877 bool IsDynamic) {
878 if (std::optional<uint64_t> ObjectSize =
879 E->tryEvaluateObjectSize(Ctx: getContext(), Type))
880 return ConstantInt::get(Ty: ResType, V: *ObjectSize, /*isSigned=*/IsSigned: true);
881 return emitBuiltinObjectSize(E, Type, ResType, EmittedE, IsDynamic);
882}
883
884/// Find a struct's flexible array member. It may be embedded inside multiple
885/// sub-structs, but must still be the last field.
886static const FieldDecl *FindFlexibleArrayMemberField(CodeGenFunction &CGF,
887 ASTContext &Ctx,
888 const RecordDecl *RD) {
889 const LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel =
890 CGF.getLangOpts().getStrictFlexArraysLevel();
891
892 if (RD->isImplicit())
893 return nullptr;
894
895 for (const FieldDecl *FD : RD->fields()) {
896 if (Decl::isFlexibleArrayMemberLike(
897 Context: Ctx, D: FD, Ty: FD->getType(), StrictFlexArraysLevel,
898 /*IgnoreTemplateOrMacroSubstitution=*/true))
899 return FD;
900
901 if (const auto *RD = FD->getType()->getAsRecordDecl())
902 if (const FieldDecl *FD = FindFlexibleArrayMemberField(CGF, Ctx, RD))
903 return FD;
904 }
905
906 return nullptr;
907}
908
909/// Calculate the offset of a struct field. It may be embedded inside multiple
910/// sub-structs.
911static bool GetFieldOffset(ASTContext &Ctx, const RecordDecl *RD,
912 const FieldDecl *FD, int64_t &Offset) {
913 if (RD->isImplicit())
914 return false;
915
916 // Keep track of the field number ourselves, because the other methods
917 // (CGRecordLayout::getLLVMFieldNo) aren't always equivalent to how the AST
918 // is laid out.
919 uint32_t FieldNo = 0;
920 const ASTRecordLayout &Layout = Ctx.getASTRecordLayout(D: RD);
921
922 for (const FieldDecl *Field : RD->fields()) {
923 if (Field == FD) {
924 Offset += Layout.getFieldOffset(FieldNo);
925 return true;
926 }
927
928 if (const auto *RD = Field->getType()->getAsRecordDecl()) {
929 if (GetFieldOffset(Ctx, RD, FD, Offset)) {
930 Offset += Layout.getFieldOffset(FieldNo);
931 return true;
932 }
933 }
934
935 if (!RD->isUnion())
936 ++FieldNo;
937 }
938
939 return false;
940}
941
942static std::optional<int64_t>
943GetFieldOffset(ASTContext &Ctx, const RecordDecl *RD, const FieldDecl *FD) {
944 int64_t Offset = 0;
945
946 if (GetFieldOffset(Ctx, RD, FD, Offset))
947 return std::optional<int64_t>(Offset);
948
949 return std::nullopt;
950}
951
952llvm::Value *CodeGenFunction::emitCountedBySize(const Expr *E,
953 llvm::Value *EmittedE,
954 unsigned Type,
955 llvm::IntegerType *ResType) {
956 // Note: If the whole struct is specificed in the __bdos (i.e. Visitor
957 // returns a DeclRefExpr). The calculation of the whole size of the structure
958 // with a flexible array member can be done in two ways:
959 //
960 // 1) sizeof(struct S) + count * sizeof(typeof(fam))
961 // 2) offsetof(struct S, fam) + count * sizeof(typeof(fam))
962 //
963 // The first will add additional padding after the end of the array
964 // allocation while the second method is more precise, but not quite expected
965 // from programmers. See
966 // https://lore.kernel.org/lkml/ZvV6X5FPBBW7CO1f@archlinux/ for a discussion
967 // of the topic.
968 //
969 // GCC isn't (currently) able to calculate __bdos on a pointer to the whole
970 // structure. Therefore, because of the above issue, we choose to match what
971 // GCC does for consistency's sake.
972
973 const Expr *Idx = nullptr;
974 // FIXME: `ArrayElementTy` is misleadingly named. `findStructFieldAccess()`
975 // sets it to the type of the array-subscript base, i.e. the (possibly cast)
976 // *pointer* being indexed (not an element type) or a null QualType when there
977 // is no subscript.
978 QualType ArrayElementTy;
979 E = findStructFieldAccess(E, OutArrayIndex: &Idx, OutArrayElementTy: &ArrayElementTy);
980 if (!E)
981 return nullptr;
982
983 if (Idx) {
984 if (Idx->HasSideEffects(Ctx: getContext()))
985 // We can't have side-effects.
986 return getDefaultBuiltinObjectSizeResult(Type, ResType);
987
988 if (const auto *IL = dyn_cast<IntegerLiteral>(Val: Idx)) {
989 int64_t Val = IL->getValue().getSExtValue();
990 if (Val < 0)
991 return getDefaultBuiltinObjectSizeResult(Type, ResType);
992
993 // The index is 0, so we don't need to take it into account.
994 if (Val == 0)
995 Idx = nullptr;
996 }
997 }
998
999 // __counted_by on either a flexible array member or a pointer into a struct
1000 // with a flexible array member.
1001 if (const auto *ME = dyn_cast<MemberExpr>(Val: E))
1002 return emitCountedByMemberSize(E: ME, Idx, EmittedE, CastedArrayElementTy: ArrayElementTy, Type,
1003 ResType);
1004
1005 // __counted_by on a pointer in a struct.
1006 if (const auto *ICE = dyn_cast<ImplicitCastExpr>(Val: E);
1007 ICE && ICE->getCastKind() == CK_LValueToRValue)
1008 return emitCountedByPointerSize(E: ICE, Idx, EmittedE, CastedArrayElementTy: ArrayElementTy, Type,
1009 ResType);
1010
1011 return nullptr;
1012}
1013
1014static llvm::Value *EmitPositiveResultOrZero(CodeGenFunction &CGF,
1015 llvm::Value *Res,
1016 llvm::Value *Index,
1017 llvm::IntegerType *ResType,
1018 bool IsSigned) {
1019 // cmp = (array_size >= 0)
1020 Value *Cmp = CGF.Builder.CreateIsNotNeg(Arg: Res);
1021 if (Index)
1022 // cmp = (cmp && index >= 0)
1023 Cmp = CGF.Builder.CreateAnd(LHS: CGF.Builder.CreateIsNotNeg(Arg: Index), RHS: Cmp);
1024
1025 // return cmp ? result : 0
1026 return CGF.Builder.CreateSelect(C: Cmp, True: Res,
1027 False: ConstantInt::get(Ty: ResType, V: 0, IsSigned));
1028}
1029
1030static std::pair<llvm::Value *, llvm::Value *>
1031GetCountFieldAndIndex(CodeGenFunction &CGF, const MemberExpr *ME,
1032 const FieldDecl *ArrayFD, const FieldDecl *CountFD,
1033 const Expr *Idx, llvm::IntegerType *ResType,
1034 bool IsSigned) {
1035 // count = ptr->count;
1036 Value *Count = CGF.EmitLoadOfCountedByField(Base: ME, FD: ArrayFD, CountDecl: CountFD);
1037 if (!Count)
1038 return std::make_pair<Value *>(x: nullptr, y: nullptr);
1039 Count = CGF.Builder.CreateIntCast(V: Count, DestTy: ResType, isSigned: IsSigned, Name: "count");
1040
1041 // index = idx;
1042 Value *Index = nullptr;
1043 if (Idx) {
1044 bool IdxSigned = Idx->getType()->isSignedIntegerType();
1045 Index = CGF.EmitScalarExpr(E: Idx);
1046 Index = CGF.Builder.CreateIntCast(V: Index, DestTy: ResType, isSigned: IdxSigned, Name: "index");
1047 }
1048
1049 return std::make_pair(x&: Count, y&: Index);
1050}
1051
1052llvm::Value *CodeGenFunction::emitCountedByPointerSize(
1053 const ImplicitCastExpr *E, const Expr *Idx, llvm::Value *EmittedE,
1054 QualType CastedArrayElementTy, unsigned Type, llvm::IntegerType *ResType) {
1055 assert(E->getCastKind() == CK_LValueToRValue &&
1056 "must be an LValue to RValue cast");
1057 assert(EmittedE && "emitted must not be null");
1058
1059 const MemberExpr *ME =
1060 dyn_cast<MemberExpr>(Val: E->getSubExpr()->IgnoreParenNoopCasts(Ctx: getContext()));
1061 if (!ME)
1062 return nullptr;
1063
1064 const auto *ArrayBaseFD = dyn_cast<FieldDecl>(Val: ME->getMemberDecl());
1065 if (!ArrayBaseFD || !ArrayBaseFD->getType()->isPointerType() ||
1066 !ArrayBaseFD->getType()->isCountAttributedType())
1067 return nullptr;
1068
1069 // Get the 'count' FieldDecl.
1070 const FieldDecl *CountFD = ArrayBaseFD->findCountedByField();
1071 if (!CountFD)
1072 // Can't find the field referenced by the "counted_by" attribute.
1073 return nullptr;
1074
1075 // Calculate the array's object size using these formulae. (Note: if the
1076 // calculation is negative, we return 0.):
1077 //
1078 // struct p;
1079 // struct s {
1080 // /* ... */
1081 // struct p **array __attribute__((ATTR(count)));
1082 // int count;
1083 // };
1084 //
1085 // 1) 'ptr->array':
1086 //
1087 // #if ATTR is counted_by_or_null || ATTR is sized_by_or_null
1088 // count = ptr->array ? ptr->count : 0;
1089 // #else
1090 // count = ptr->count;
1091 // #endif
1092 //
1093 // #if ATTR is counted_by || ATTR is counted_by_or_null
1094 // array_element_size = sizeof (*ptr->array);
1095 // array_size = count * array_element_size;
1096 // #else
1097 // array_size = count;
1098 // #endif
1099 //
1100 // result = array_size;
1101 //
1102 // cmp = (result >= 0)
1103 // return cmp ? result : 0;
1104 //
1105 // 2) '&((cast) ptr->array)[idx]':
1106 //
1107 // #if ATTR is counted_by_or_null || ATTR is sized_by_or_null
1108 // count = ptr->array ? ptr->count : 0;
1109 // #else
1110 // count = ptr->count;
1111 // #endif
1112 // index = idx;
1113 //
1114 // #if ATTR is counted_by || ATTR is counted_by_or_null
1115 // array_element_size = sizeof (*ptr->array);
1116 // array_size = count * array_element_size;
1117 // #else
1118 // array_size = count;
1119 // #endif
1120 //
1121 // casted_array_element_size = sizeof (*((cast) ptr->array));
1122 //
1123 // index_size = index * casted_array_element_size;
1124 // result = array_size - index_size;
1125 //
1126 // cmp = (result >= 0)
1127 // if (index)
1128 // cmp = (cmp && index > 0)
1129 // return cmp ? result : 0;
1130
1131 auto GetPointeeSize = [&](QualType PtrTy) -> CharUnits {
1132 assert(!PtrTy.isNull());
1133 QualType PointeeTy = PtrTy->getPointeeType();
1134 assert(!PointeeTy.isNull() &&
1135 (PointeeTy->isVoidType() || !PointeeTy->isIncompleteType()) &&
1136 "pointee type must have a computable size");
1137
1138 CharUnits PointeeSize = getContext().getTypeSizeInChars(T: PointeeTy);
1139 if (PointeeSize.isZero()) {
1140 // Support GNU extension of treating `void` having size 1.
1141 PointeeSize = CharUnits::One();
1142 }
1143
1144 return PointeeSize;
1145 };
1146
1147 bool IsSigned = CountFD->getType()->isSignedIntegerType();
1148 const auto *CountAttributedTy =
1149 ArrayBaseFD->getType()->getAs<CountAttributedType>();
1150 assert(CountAttributedTy && "the field's type is not a CountAttributedType");
1151
1152 // count = ptr->count;
1153 // index = idx;
1154 Value *Count, *Index;
1155 std::tie(args&: Count, args&: Index) = GetCountFieldAndIndex(
1156 CGF&: *this, ME, ArrayFD: ArrayBaseFD, CountFD, Idx, ResType, IsSigned);
1157 if (!Count)
1158 return nullptr;
1159
1160 // For the _or_null variants, a null pointer describes no accessible memory:
1161 // count = ptr->array ? count : 0;
1162 if (CountAttributedTy->isOrNull()) {
1163 Value *Ptr = nullptr;
1164 if (!Idx) {
1165 // 1) 'ptr->array'
1166 // Reuse the already-emitted pointer value rather than re-loading `ME`.
1167 // Re-loading would produce a second, observable access for a volatile
1168 // pointer field
1169 Ptr = EmittedE;
1170 } else {
1171 // 2) '&((cast) ptr->array)[idx]'
1172 // FIXME: `EmittedE` is the element address, not `ptr->array`, so we fall
1173 // back to re-emitting `ME` and the pointer field is loaded twice. This is
1174 // normally harmless except when the pointer is `volatile`. Avoiding that
1175 // would require restructuring how the base pointer is emitted (it is
1176 // handled elsewhere in the callstack), so it is left as-is for now.
1177 Ptr = EmitScalarExpr(E: ME);
1178 }
1179 Value *IsNull = Builder.CreateIsNull(Arg: Ptr);
1180 Count = Builder.CreateSelect(C: IsNull, True: ConstantInt::get(Ty: ResType, V: 0, IsSigned),
1181 False: Count, Name: "count.or.null");
1182 }
1183
1184 // #if ATTR is counted_by || ATTR is counted_by_or_null
1185 // array_element_size = sizeof (*ptr->array);
1186 // array_size = count * array_element_size;
1187 // #else
1188 // array_size = count;
1189 // #endif
1190 Value *ArraySize;
1191 if (!CountAttributedTy->isCountInBytes()) {
1192 // `__counted_by`/`__counted_by_or_null` require a complete pointee at use
1193 // sites (enforced by Sema) so the element size is computable.
1194 CharUnits ArrayElementBaseSize = GetPointeeSize(ArrayBaseFD->getType());
1195
1196 // array_element_size = sizeof (*ptr->array)
1197 auto *ArrayElementSize = llvm::ConstantInt::get(
1198 Ty: ResType, V: ArrayElementBaseSize.getQuantity(), IsSigned);
1199
1200 // array_size = count * array_element_size;
1201 ArraySize = Builder.CreateMul(LHS: Count, RHS: ArrayElementSize, Name: "array_size",
1202 HasNUW: !IsSigned, HasNSW: IsSigned);
1203 } else {
1204 // array_size = count;
1205 ArraySize = Count;
1206 }
1207
1208 // Option (1) 'ptr->array'
1209 // result = array_size
1210 Value *Result = ArraySize;
1211
1212 if (Idx) { // Option (2) '&((cast) ptr->array)[idx]'
1213 // FIXME: CastedArrayElementTy is confusingly named. It's actually the base
1214 // expression of the ArraySubscriptExpr, not the element (pointee) type.
1215 CharUnits CastedArrayElementSizeInChars =
1216 GetPointeeSize(CastedArrayElementTy);
1217
1218 // casted_array_element_size = sizeof (*((cast) ptr->array));
1219 auto *CastedArrayElementSize = llvm::ConstantInt::get(
1220 Ty: ResType, V: CastedArrayElementSizeInChars.getQuantity(), IsSigned);
1221
1222 // index_size = index * casted_array_element_size;
1223 Value *IndexSize = Builder.CreateMul(LHS: Index, RHS: CastedArrayElementSize,
1224 Name: "index_size", HasNUW: !IsSigned, HasNSW: IsSigned);
1225
1226 // result = result - index_size;
1227 Result =
1228 Builder.CreateSub(LHS: Result, RHS: IndexSize, Name: "result", HasNUW: !IsSigned, HasNSW: IsSigned);
1229 }
1230
1231 return EmitPositiveResultOrZero(CGF&: *this, Res: Result, Index, ResType, IsSigned);
1232}
1233
1234llvm::Value *CodeGenFunction::emitCountedByMemberSize(
1235 const MemberExpr *ME, const Expr *Idx, llvm::Value *EmittedE,
1236 QualType CastedArrayElementTy, unsigned Type, llvm::IntegerType *ResType) {
1237 const auto *FD = dyn_cast<FieldDecl>(Val: ME->getMemberDecl());
1238 if (!FD)
1239 return nullptr;
1240
1241 // Find the flexible array member and check that it has the __counted_by
1242 // attribute.
1243 ASTContext &Ctx = getContext();
1244 const RecordDecl *RD = FD->getDeclContext()->getOuterLexicalRecordContext();
1245 const FieldDecl *FlexibleArrayMemberFD = nullptr;
1246
1247 if (Decl::isFlexibleArrayMemberLike(
1248 Context: Ctx, D: FD, Ty: FD->getType(), StrictFlexArraysLevel: getLangOpts().getStrictFlexArraysLevel(),
1249 /*IgnoreTemplateOrMacroSubstitution=*/true))
1250 FlexibleArrayMemberFD = FD;
1251 else
1252 FlexibleArrayMemberFD = FindFlexibleArrayMemberField(CGF&: *this, Ctx, RD);
1253
1254 if (!FlexibleArrayMemberFD ||
1255 !FlexibleArrayMemberFD->getType()->isCountAttributedType())
1256 return nullptr;
1257
1258 // Get the 'count' FieldDecl.
1259 const FieldDecl *CountFD = FlexibleArrayMemberFD->findCountedByField();
1260 if (!CountFD)
1261 // Can't find the field referenced by the "counted_by" attribute.
1262 return nullptr;
1263
1264 // Calculate the flexible array member's object size using these formulae.
1265 // (Note: if the calculation is negative, we return 0.):
1266 //
1267 // struct p;
1268 // struct s {
1269 // /* ... */
1270 // int count;
1271 // struct p *array[] __attribute__((counted_by(count)));
1272 // };
1273 //
1274 // 1) 'ptr->array':
1275 //
1276 // count = ptr->count;
1277 //
1278 // flexible_array_member_element_size = sizeof (*ptr->array);
1279 // flexible_array_member_size =
1280 // count * flexible_array_member_element_size;
1281 //
1282 // result = flexible_array_member_size;
1283 //
1284 // cmp = (result >= 0)
1285 // return cmp ? result : 0;
1286 //
1287 // 2) '&((cast) ptr->array)[idx]':
1288 //
1289 // count = ptr->count;
1290 // index = idx;
1291 //
1292 // flexible_array_member_element_size = sizeof (*ptr->array);
1293 // flexible_array_member_size =
1294 // count * flexible_array_member_element_size;
1295 //
1296 // casted_flexible_array_member_element_size =
1297 // sizeof (*((cast) ptr->array));
1298 // index_size = index * casted_flexible_array_member_element_size;
1299 //
1300 // result = flexible_array_member_size - index_size;
1301 //
1302 // cmp = (result >= 0)
1303 // if (index != 0)
1304 // cmp = (cmp && index >= 0)
1305 // return cmp ? result : 0;
1306 //
1307 // 3) '&ptr->field':
1308 //
1309 // count = ptr->count;
1310 // sizeof_struct = sizeof (struct s);
1311 //
1312 // flexible_array_member_element_size = sizeof (*ptr->array);
1313 // flexible_array_member_size =
1314 // count * flexible_array_member_element_size;
1315 //
1316 // field_offset = offsetof (struct s, field);
1317 // offset_diff = sizeof_struct - field_offset;
1318 //
1319 // result = offset_diff + flexible_array_member_size;
1320 //
1321 // cmp = (result >= 0)
1322 // return cmp ? result : 0;
1323 //
1324 // 4) '&((cast) ptr->field_array)[idx]':
1325 //
1326 // count = ptr->count;
1327 // index = idx;
1328 // sizeof_struct = sizeof (struct s);
1329 //
1330 // flexible_array_member_element_size = sizeof (*ptr->array);
1331 // flexible_array_member_size =
1332 // count * flexible_array_member_element_size;
1333 //
1334 // casted_field_element_size = sizeof (*((cast) ptr->field_array));
1335 // field_offset = offsetof (struct s, field)
1336 // field_offset += index * casted_field_element_size;
1337 //
1338 // offset_diff = sizeof_struct - field_offset;
1339 //
1340 // result = offset_diff + flexible_array_member_size;
1341 //
1342 // cmp = (result >= 0)
1343 // if (index != 0)
1344 // cmp = (cmp && index >= 0)
1345 // return cmp ? result : 0;
1346
1347 bool IsSigned = CountFD->getType()->isSignedIntegerType();
1348
1349 QualType FlexibleArrayMemberTy = FlexibleArrayMemberFD->getType();
1350
1351 // Explicit cast because otherwise the CharWidth will promote an i32's into
1352 // u64's leading to overflows.
1353 int64_t CharWidth = static_cast<int64_t>(CGM.getContext().getCharWidth());
1354
1355 // field_offset = offsetof (struct s, field);
1356 Value *FieldOffset = nullptr;
1357 if (FlexibleArrayMemberFD != FD) {
1358 std::optional<int64_t> Offset = GetFieldOffset(Ctx, RD, FD);
1359 if (!Offset)
1360 return nullptr;
1361 FieldOffset =
1362 llvm::ConstantInt::get(Ty: ResType, V: *Offset / CharWidth, IsSigned);
1363 }
1364
1365 // count = ptr->count;
1366 // index = ptr->index;
1367 Value *Count, *Index;
1368 std::tie(args&: Count, args&: Index) = GetCountFieldAndIndex(
1369 CGF&: *this, ME, ArrayFD: FlexibleArrayMemberFD, CountFD, Idx, ResType, IsSigned);
1370 if (!Count)
1371 return nullptr;
1372
1373 // flexible_array_member_element_size = sizeof (*ptr->array);
1374 const ArrayType *ArrayTy = Ctx.getAsArrayType(T: FlexibleArrayMemberTy);
1375 CharUnits BaseSize = Ctx.getTypeSizeInChars(T: ArrayTy->getElementType());
1376 auto *FlexibleArrayMemberElementSize =
1377 llvm::ConstantInt::get(Ty: ResType, V: BaseSize.getQuantity(), IsSigned);
1378
1379 // flexible_array_member_size = count * flexible_array_member_element_size;
1380 Value *FlexibleArrayMemberSize =
1381 Builder.CreateMul(LHS: Count, RHS: FlexibleArrayMemberElementSize,
1382 Name: "flexible_array_member_size", HasNUW: !IsSigned, HasNSW: IsSigned);
1383
1384 Value *Result = nullptr;
1385 if (FlexibleArrayMemberFD == FD) {
1386 if (Idx) { // Option (2) '&((cast) ptr->array)[idx]'
1387 // casted_flexible_array_member_element_size =
1388 // sizeof (*((cast) ptr->array));
1389 llvm::ConstantInt *CastedFlexibleArrayMemberElementSize =
1390 FlexibleArrayMemberElementSize;
1391 if (!CastedArrayElementTy.isNull() &&
1392 CastedArrayElementTy->isPointerType()) {
1393 CharUnits BaseSize =
1394 Ctx.getTypeSizeInChars(T: CastedArrayElementTy->getPointeeType());
1395 CastedFlexibleArrayMemberElementSize =
1396 llvm::ConstantInt::get(Ty: ResType, V: BaseSize.getQuantity(), IsSigned);
1397 }
1398
1399 // index_size = index * casted_flexible_array_member_element_size;
1400 Value *IndexSize =
1401 Builder.CreateMul(LHS: Index, RHS: CastedFlexibleArrayMemberElementSize,
1402 Name: "index_size", HasNUW: !IsSigned, HasNSW: IsSigned);
1403
1404 // result = flexible_array_member_size - index_size;
1405 Result = Builder.CreateSub(LHS: FlexibleArrayMemberSize, RHS: IndexSize, Name: "result",
1406 HasNUW: !IsSigned, HasNSW: IsSigned);
1407 } else { // Option (1) 'ptr->array'
1408 // result = flexible_array_member_size;
1409 Result = FlexibleArrayMemberSize;
1410 }
1411 } else {
1412 // sizeof_struct = sizeof (struct s);
1413 llvm::StructType *StructTy = getTypes().getCGRecordLayout(RD).getLLVMType();
1414 const llvm::DataLayout &Layout = CGM.getDataLayout();
1415 TypeSize Size = Layout.getTypeSizeInBits(Ty: StructTy);
1416 Value *SizeofStruct =
1417 llvm::ConstantInt::get(Ty: ResType, V: Size.getKnownMinValue() / CharWidth);
1418
1419 if (Idx) { // Option (4) '&((cast) ptr->field_array)[idx]'
1420 // casted_field_element_size = sizeof (*((cast) ptr->field_array));
1421 CharUnits BaseSize;
1422 if (!CastedArrayElementTy.isNull() &&
1423 CastedArrayElementTy->isPointerType()) {
1424 BaseSize =
1425 Ctx.getTypeSizeInChars(T: CastedArrayElementTy->getPointeeType());
1426 } else {
1427 const ArrayType *ArrayTy = Ctx.getAsArrayType(T: FD->getType());
1428 BaseSize = Ctx.getTypeSizeInChars(T: ArrayTy->getElementType());
1429 }
1430
1431 llvm::ConstantInt *CastedFieldElementSize =
1432 llvm::ConstantInt::get(Ty: ResType, V: BaseSize.getQuantity(), IsSigned);
1433
1434 // field_offset += index * casted_field_element_size;
1435 Value *Mul = Builder.CreateMul(LHS: Index, RHS: CastedFieldElementSize,
1436 Name: "field_offset", HasNUW: !IsSigned, HasNSW: IsSigned);
1437 FieldOffset = Builder.CreateAdd(LHS: FieldOffset, RHS: Mul);
1438 }
1439 // Option (3) '&ptr->field', and Option (4) continuation.
1440 // offset_diff = flexible_array_member_offset - field_offset;
1441 Value *OffsetDiff = Builder.CreateSub(LHS: SizeofStruct, RHS: FieldOffset,
1442 Name: "offset_diff", HasNUW: !IsSigned, HasNSW: IsSigned);
1443
1444 // result = offset_diff + flexible_array_member_size;
1445 Result = Builder.CreateAdd(LHS: FlexibleArrayMemberSize, RHS: OffsetDiff, Name: "result");
1446 }
1447
1448 return EmitPositiveResultOrZero(CGF&: *this, Res: Result, Index, ResType, IsSigned);
1449}
1450
1451/// Returns a Value corresponding to the size of the given expression.
1452/// This Value may be either of the following:
1453/// - A llvm::Argument (if E is a param with the pass_object_size attribute on
1454/// it)
1455/// - A call to the @llvm.objectsize intrinsic
1456///
1457/// EmittedE is the result of emitting `E` as a scalar expr. If it's non-null
1458/// and we wouldn't otherwise try to reference a pass_object_size parameter,
1459/// we'll call @llvm.objectsize on EmittedE, rather than emitting E.
1460llvm::Value *
1461CodeGenFunction::emitBuiltinObjectSize(const Expr *E, unsigned Type,
1462 llvm::IntegerType *ResType,
1463 llvm::Value *EmittedE, bool IsDynamic) {
1464 // We need to reference an argument if the pointer is a parameter with the
1465 // pass_object_size attribute.
1466 if (auto *D = dyn_cast<DeclRefExpr>(Val: E->IgnoreParenImpCasts())) {
1467 auto *Param = dyn_cast<ParmVarDecl>(Val: D->getDecl());
1468 auto *PS = D->getDecl()->getAttr<PassObjectSizeAttr>();
1469 if (Param != nullptr && PS != nullptr &&
1470 areBOSTypesCompatible(From: PS->getType(), To: Type)) {
1471 auto Iter = SizeArguments.find(Val: Param);
1472 assert(Iter != SizeArguments.end());
1473
1474 const ImplicitParamDecl *D = Iter->second;
1475 auto DIter = LocalDeclMap.find(Val: D);
1476 assert(DIter != LocalDeclMap.end());
1477
1478 return EmitLoadOfScalar(Addr: DIter->second, /*Volatile=*/false,
1479 Ty: getContext().getSizeType(), Loc: E->getBeginLoc());
1480 }
1481 }
1482
1483 // LLVM can't handle Type=3 appropriately, and __builtin_object_size shouldn't
1484 // evaluate E for side-effects. In either case, we shouldn't lower to
1485 // @llvm.objectsize.
1486 if (Type == 3 || (!EmittedE && E->HasSideEffects(Ctx: getContext())))
1487 return getDefaultBuiltinObjectSizeResult(Type, ResType);
1488
1489 Value *Ptr = EmittedE ? EmittedE : EmitScalarExpr(E);
1490 assert(Ptr->getType()->isPointerTy() &&
1491 "Non-pointer passed to __builtin_object_size?");
1492
1493 if (IsDynamic)
1494 // Emit special code for a flexible array member with the "counted_by"
1495 // attribute.
1496 if (Value *V = emitCountedBySize(E, EmittedE: Ptr, Type, ResType))
1497 return V;
1498
1499 Function *F =
1500 CGM.getIntrinsic(IID: Intrinsic::objectsize, Tys: {ResType, Ptr->getType()});
1501
1502 // LLVM only supports 0 and 2, make sure that we pass along that as a boolean.
1503 Value *Min = Builder.getInt1(V: (Type & 2) != 0);
1504 // For GCC compatibility, __builtin_object_size treat NULL as unknown size.
1505 Value *NullIsUnknown = Builder.getTrue();
1506 Value *Dynamic = Builder.getInt1(V: IsDynamic);
1507 return Builder.CreateCall(Callee: F, Args: {Ptr, Min, NullIsUnknown, Dynamic});
1508}
1509
1510namespace {
1511/// A struct to generically describe a bit test intrinsic.
1512struct BitTest {
1513 enum ActionKind : uint8_t { TestOnly, Complement, Reset, Set };
1514 enum InterlockingKind : uint8_t {
1515 Unlocked,
1516 Sequential,
1517 Acquire,
1518 Release,
1519 NoFence
1520 };
1521
1522 ActionKind Action;
1523 InterlockingKind Interlocking;
1524 bool Is64Bit;
1525
1526 static BitTest decodeBitTestBuiltin(unsigned BuiltinID);
1527};
1528
1529} // namespace
1530
1531BitTest BitTest::decodeBitTestBuiltin(unsigned BuiltinID) {
1532 switch (BuiltinID) {
1533 // Main portable variants.
1534 case Builtin::BI_bittest:
1535 return {.Action: TestOnly, .Interlocking: Unlocked, .Is64Bit: false};
1536 case Builtin::BI_bittestandcomplement:
1537 return {.Action: Complement, .Interlocking: Unlocked, .Is64Bit: false};
1538 case Builtin::BI_bittestandreset:
1539 return {.Action: Reset, .Interlocking: Unlocked, .Is64Bit: false};
1540 case Builtin::BI_bittestandset:
1541 return {.Action: Set, .Interlocking: Unlocked, .Is64Bit: false};
1542 case Builtin::BI_interlockedbittestandreset:
1543 return {.Action: Reset, .Interlocking: Sequential, .Is64Bit: false};
1544 case Builtin::BI_interlockedbittestandset:
1545 return {.Action: Set, .Interlocking: Sequential, .Is64Bit: false};
1546
1547 // 64-bit variants.
1548 case Builtin::BI_bittest64:
1549 return {.Action: TestOnly, .Interlocking: Unlocked, .Is64Bit: true};
1550 case Builtin::BI_bittestandcomplement64:
1551 return {.Action: Complement, .Interlocking: Unlocked, .Is64Bit: true};
1552 case Builtin::BI_bittestandreset64:
1553 return {.Action: Reset, .Interlocking: Unlocked, .Is64Bit: true};
1554 case Builtin::BI_bittestandset64:
1555 return {.Action: Set, .Interlocking: Unlocked, .Is64Bit: true};
1556 case Builtin::BI_interlockedbittestandreset64:
1557 return {.Action: Reset, .Interlocking: Sequential, .Is64Bit: true};
1558 case Builtin::BI_interlockedbittestandset64:
1559 return {.Action: Set, .Interlocking: Sequential, .Is64Bit: true};
1560
1561 // ARM/AArch64-specific ordering variants.
1562 case Builtin::BI_interlockedbittestandset_acq:
1563 return {.Action: Set, .Interlocking: Acquire, .Is64Bit: false};
1564 case Builtin::BI_interlockedbittestandset_rel:
1565 return {.Action: Set, .Interlocking: Release, .Is64Bit: false};
1566 case Builtin::BI_interlockedbittestandset_nf:
1567 return {.Action: Set, .Interlocking: NoFence, .Is64Bit: false};
1568 case Builtin::BI_interlockedbittestandreset_acq:
1569 return {.Action: Reset, .Interlocking: Acquire, .Is64Bit: false};
1570 case Builtin::BI_interlockedbittestandreset_rel:
1571 return {.Action: Reset, .Interlocking: Release, .Is64Bit: false};
1572 case Builtin::BI_interlockedbittestandreset_nf:
1573 return {.Action: Reset, .Interlocking: NoFence, .Is64Bit: false};
1574 case Builtin::BI_interlockedbittestandreset64_acq:
1575 return {.Action: Reset, .Interlocking: Acquire, .Is64Bit: false};
1576 case Builtin::BI_interlockedbittestandreset64_rel:
1577 return {.Action: Reset, .Interlocking: Release, .Is64Bit: false};
1578 case Builtin::BI_interlockedbittestandreset64_nf:
1579 return {.Action: Reset, .Interlocking: NoFence, .Is64Bit: false};
1580 case Builtin::BI_interlockedbittestandset64_acq:
1581 return {.Action: Set, .Interlocking: Acquire, .Is64Bit: false};
1582 case Builtin::BI_interlockedbittestandset64_rel:
1583 return {.Action: Set, .Interlocking: Release, .Is64Bit: false};
1584 case Builtin::BI_interlockedbittestandset64_nf:
1585 return {.Action: Set, .Interlocking: NoFence, .Is64Bit: false};
1586 }
1587 llvm_unreachable("expected only bittest intrinsics");
1588}
1589
1590static char bitActionToX86BTCode(BitTest::ActionKind A) {
1591 switch (A) {
1592 case BitTest::TestOnly: return '\0';
1593 case BitTest::Complement: return 'c';
1594 case BitTest::Reset: return 'r';
1595 case BitTest::Set: return 's';
1596 }
1597 llvm_unreachable("invalid action");
1598}
1599
1600static llvm::Value *EmitX86BitTestIntrinsic(CodeGenFunction &CGF,
1601 BitTest BT,
1602 const CallExpr *E, Value *BitBase,
1603 Value *BitPos) {
1604 char Action = bitActionToX86BTCode(A: BT.Action);
1605 char SizeSuffix = BT.Is64Bit ? 'q' : 'l';
1606
1607 // Build the assembly.
1608 SmallString<64> Asm;
1609 raw_svector_ostream AsmOS(Asm);
1610 if (BT.Interlocking != BitTest::Unlocked)
1611 AsmOS << "lock ";
1612 AsmOS << "bt";
1613 if (Action)
1614 AsmOS << Action;
1615 AsmOS << SizeSuffix << " $2, ($1)";
1616
1617 // Build the constraints. FIXME: We should support immediates when possible.
1618 std::string Constraints = "={@ccc},r,r,~{cc},~{memory}";
1619 std::string_view MachineClobbers = CGF.getTarget().getClobbers();
1620 if (!MachineClobbers.empty()) {
1621 Constraints += ',';
1622 Constraints += MachineClobbers;
1623 }
1624 llvm::IntegerType *IntType = llvm::IntegerType::get(
1625 C&: CGF.getLLVMContext(),
1626 NumBits: CGF.getContext().getTypeSize(T: E->getArg(Arg: 1)->getType()));
1627 llvm::FunctionType *FTy =
1628 llvm::FunctionType::get(Result: CGF.Int8Ty, Params: {CGF.DefaultPtrTy, IntType}, isVarArg: false);
1629
1630 llvm::InlineAsm *IA =
1631 llvm::InlineAsm::get(Ty: FTy, AsmString: Asm, Constraints, /*hasSideEffects=*/true);
1632 return CGF.Builder.CreateCall(Callee: IA, Args: {BitBase, BitPos});
1633}
1634
1635static llvm::AtomicOrdering
1636getBitTestAtomicOrdering(BitTest::InterlockingKind I) {
1637 switch (I) {
1638 case BitTest::Unlocked: return llvm::AtomicOrdering::NotAtomic;
1639 case BitTest::Sequential: return llvm::AtomicOrdering::SequentiallyConsistent;
1640 case BitTest::Acquire: return llvm::AtomicOrdering::Acquire;
1641 case BitTest::Release: return llvm::AtomicOrdering::Release;
1642 case BitTest::NoFence: return llvm::AtomicOrdering::Monotonic;
1643 }
1644 llvm_unreachable("invalid interlocking");
1645}
1646
1647static llvm::Value *EmitBitCountExpr(CodeGenFunction &CGF, const Expr *E) {
1648 llvm::Value *ArgValue = CGF.EmitScalarExpr(E);
1649 llvm::Type *ArgType = ArgValue->getType();
1650
1651 // Boolean vectors can be casted directly to its bitfield representation. We
1652 // intentionally do not round up to the next power of two size and let LLVM
1653 // handle the trailing bits.
1654 //
1655 // In big endian mode, the bitfield representation has a reversed bit order,
1656 // hence the need to add an operation to reverse it back to the expected
1657 // order.
1658 if (auto *VT = dyn_cast<llvm::FixedVectorType>(Val: ArgType);
1659 VT && VT->getElementType()->isIntegerTy(BitWidth: 1)) {
1660 llvm::Type *StorageType =
1661 llvm::Type::getIntNTy(C&: CGF.getLLVMContext(), N: VT->getNumElements());
1662 ArgValue = CGF.Builder.CreateBitCast(V: ArgValue, DestTy: StorageType);
1663
1664 if (CGF.getTarget().isBigEndian())
1665 ArgValue = CGF.Builder.CreateIntrinsic(ID: Intrinsic::bitreverse,
1666 OverloadTypes: {StorageType}, Args: ArgValue);
1667 }
1668
1669 return ArgValue;
1670}
1671
1672/// Emit a _bittest* intrinsic. These intrinsics take a pointer to an array of
1673/// bits and a bit position and read and optionally modify the bit at that
1674/// position. The position index can be arbitrarily large, i.e. it can be larger
1675/// than 31 or 63, so we need an indexed load in the general case.
1676static llvm::Value *EmitBitTestIntrinsic(CodeGenFunction &CGF,
1677 unsigned BuiltinID,
1678 const CallExpr *E) {
1679 Value *BitBase = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
1680 Value *BitPos = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
1681
1682 BitTest BT = BitTest::decodeBitTestBuiltin(BuiltinID);
1683
1684 // X86 has special BT, BTC, BTR, and BTS instructions that handle the array
1685 // indexing operation internally. Use them if possible.
1686 if (CGF.getTarget().getTriple().isX86())
1687 return EmitX86BitTestIntrinsic(CGF, BT, E, BitBase, BitPos);
1688
1689 // Otherwise, use generic code to load one byte and test the bit. Use all but
1690 // the bottom three bits as the array index, and the bottom three bits to form
1691 // a mask.
1692 // Bit = BitBaseI8[BitPos >> 3] & (1 << (BitPos & 0x7)) != 0;
1693 Value *ByteIndex = CGF.Builder.CreateAShr(
1694 LHS: BitPos, RHS: llvm::ConstantInt::get(Ty: BitPos->getType(), V: 3), Name: "bittest.byteidx");
1695 Address ByteAddr(CGF.Builder.CreateInBoundsGEP(Ty: CGF.Int8Ty, Ptr: BitBase, IdxList: ByteIndex,
1696 Name: "bittest.byteaddr"),
1697 CGF.Int8Ty, CharUnits::One());
1698 Value *PosLow =
1699 CGF.Builder.CreateAnd(LHS: CGF.Builder.CreateTrunc(V: BitPos, DestTy: CGF.Int8Ty),
1700 RHS: llvm::ConstantInt::get(Ty: CGF.Int8Ty, V: 0x7));
1701
1702 // The updating instructions will need a mask.
1703 Value *Mask = nullptr;
1704 if (BT.Action != BitTest::TestOnly) {
1705 Mask = CGF.Builder.CreateShl(LHS: llvm::ConstantInt::get(Ty: CGF.Int8Ty, V: 1), RHS: PosLow,
1706 Name: "bittest.mask");
1707 }
1708
1709 // Check the action and ordering of the interlocked intrinsics.
1710 llvm::AtomicOrdering Ordering = getBitTestAtomicOrdering(I: BT.Interlocking);
1711
1712 Value *OldByte = nullptr;
1713 if (Ordering != llvm::AtomicOrdering::NotAtomic) {
1714 // Emit a combined atomicrmw load/store operation for the interlocked
1715 // intrinsics.
1716 llvm::AtomicRMWInst::BinOp RMWOp = llvm::AtomicRMWInst::Or;
1717 if (BT.Action == BitTest::Reset) {
1718 Mask = CGF.Builder.CreateNot(V: Mask);
1719 RMWOp = llvm::AtomicRMWInst::And;
1720 }
1721 OldByte = CGF.Builder.CreateAtomicRMW(Op: RMWOp, Addr: ByteAddr, Val: Mask, Ordering);
1722 } else {
1723 // Emit a plain load for the non-interlocked intrinsics.
1724 OldByte = CGF.Builder.CreateLoad(Addr: ByteAddr, Name: "bittest.byte");
1725 Value *NewByte = nullptr;
1726 switch (BT.Action) {
1727 case BitTest::TestOnly:
1728 // Don't store anything.
1729 break;
1730 case BitTest::Complement:
1731 NewByte = CGF.Builder.CreateXor(LHS: OldByte, RHS: Mask);
1732 break;
1733 case BitTest::Reset:
1734 NewByte = CGF.Builder.CreateAnd(LHS: OldByte, RHS: CGF.Builder.CreateNot(V: Mask));
1735 break;
1736 case BitTest::Set:
1737 NewByte = CGF.Builder.CreateOr(LHS: OldByte, RHS: Mask);
1738 break;
1739 }
1740 if (NewByte)
1741 CGF.Builder.CreateStore(Val: NewByte, Addr: ByteAddr);
1742 }
1743
1744 // However we loaded the old byte, either by plain load or atomicrmw, shift
1745 // the bit into the low position and mask it to 0 or 1.
1746 Value *ShiftedByte = CGF.Builder.CreateLShr(LHS: OldByte, RHS: PosLow, Name: "bittest.shr");
1747 return CGF.Builder.CreateAnd(
1748 LHS: ShiftedByte, RHS: llvm::ConstantInt::get(Ty: CGF.Int8Ty, V: 1), Name: "bittest.res");
1749}
1750
1751namespace {
1752enum class MSVCSetJmpKind {
1753 _setjmpex,
1754 _setjmp3,
1755 _setjmp
1756};
1757}
1758
1759/// MSVC handles setjmp a bit differently on different platforms. On 32-bit x86
1760/// extra parameters can be passed as variadic arguments, but we always pass
1761/// none. Everywhere else a frame value is passed: the stack pointer as it was
1762/// on entry to the function for AArch64 and 32-bit Arm, and the frame address
1763/// for the rest.
1764static RValue EmitMSVCRTSetJmp(CodeGenFunction &CGF, MSVCSetJmpKind SJKind,
1765 const CallExpr *E) {
1766 llvm::Value *Arg1 = nullptr;
1767 llvm::Type *Arg1Ty = nullptr;
1768 StringRef Name;
1769 bool IsVarArg = false;
1770 if (SJKind == MSVCSetJmpKind::_setjmp3) {
1771 Name = "_setjmp3";
1772 Arg1Ty = CGF.Int32Ty;
1773 Arg1 = llvm::ConstantInt::get(Ty: CGF.IntTy, V: 0);
1774 IsVarArg = true;
1775 } else {
1776 Name = SJKind == MSVCSetJmpKind::_setjmp ? "_setjmp" : "_setjmpex";
1777 Arg1Ty = CGF.Int8PtrTy;
1778 const llvm::Triple &T = CGF.getTarget().getTriple();
1779 if (T.getArch() == llvm::Triple::aarch64 || T.isARM() || T.isThumb()) {
1780 Arg1 = CGF.Builder.CreateCall(
1781 Callee: CGF.CGM.getIntrinsic(IID: Intrinsic::sponentry, Tys: CGF.AllocaInt8PtrTy));
1782 } else
1783 Arg1 = CGF.Builder.CreateCall(
1784 Callee: CGF.CGM.getIntrinsic(IID: Intrinsic::frameaddress, Tys: CGF.AllocaInt8PtrTy),
1785 Args: llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: 0));
1786 }
1787
1788 // Mark the call site and declaration with ReturnsTwice.
1789 llvm::Type *ArgTypes[2] = {CGF.Int8PtrTy, Arg1Ty};
1790 llvm::AttributeList ReturnsTwiceAttr = llvm::AttributeList::get(
1791 C&: CGF.getLLVMContext(), Index: llvm::AttributeList::FunctionIndex,
1792 Kinds: llvm::Attribute::ReturnsTwice);
1793 llvm::FunctionCallee SetJmpFn = CGF.CGM.CreateRuntimeFunction(
1794 Ty: llvm::FunctionType::get(Result: CGF.IntTy, Params: ArgTypes, isVarArg: IsVarArg), Name,
1795 ExtraAttrs: ReturnsTwiceAttr, /*Local=*/true);
1796
1797 llvm::Value *Buf = CGF.Builder.CreateBitOrPointerCast(
1798 V: CGF.EmitScalarExpr(E: E->getArg(Arg: 0)), DestTy: CGF.Int8PtrTy);
1799 llvm::Value *Args[] = {Buf, Arg1};
1800 llvm::CallBase *CB = CGF.EmitRuntimeCallOrInvoke(callee: SetJmpFn, args: Args);
1801 CB->setAttributes(ReturnsTwiceAttr);
1802 return RValue::get(V: CB);
1803}
1804
1805// Emit an MSVC intrinsic. Assumes that arguments have *not* been evaluated.
1806Value *CodeGenFunction::EmitMSVCBuiltinExpr(MSVCIntrin BuiltinID,
1807 const CallExpr *E) {
1808 switch (BuiltinID) {
1809 case MSVCIntrin::_BitScanForward:
1810 case MSVCIntrin::_BitScanReverse: {
1811 Address IndexAddress(EmitPointerWithAlignment(Addr: E->getArg(Arg: 0)));
1812 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 1));
1813
1814 llvm::Type *ArgType = ArgValue->getType();
1815 llvm::Type *IndexType = IndexAddress.getElementType();
1816 llvm::Type *ResultType = ConvertType(T: E->getType());
1817
1818 Value *ArgZero = llvm::Constant::getNullValue(Ty: ArgType);
1819 Value *ResZero = llvm::Constant::getNullValue(Ty: ResultType);
1820 Value *ResOne = llvm::ConstantInt::get(Ty: ResultType, V: 1);
1821
1822 BasicBlock *Begin = Builder.GetInsertBlock();
1823 BasicBlock *End = createBasicBlock(name: "bitscan_end", parent: this->CurFn);
1824 Builder.SetInsertPoint(End);
1825 PHINode *Result = Builder.CreatePHI(Ty: ResultType, NumReservedValues: 2, Name: "bitscan_result");
1826
1827 Builder.SetInsertPoint(Begin);
1828 Value *IsZero = Builder.CreateICmpEQ(LHS: ArgValue, RHS: ArgZero);
1829 BasicBlock *NotZero = createBasicBlock(name: "bitscan_not_zero", parent: this->CurFn);
1830 Builder.CreateCondBr(Cond: IsZero, True: End, False: NotZero);
1831 Result->addIncoming(V: ResZero, BB: Begin);
1832
1833 Builder.SetInsertPoint(NotZero);
1834
1835 if (BuiltinID == MSVCIntrin::_BitScanForward) {
1836 Function *F = CGM.getIntrinsic(IID: Intrinsic::cttz, Tys: ArgType);
1837 Value *ZeroCount = Builder.CreateCall(Callee: F, Args: {ArgValue, Builder.getTrue()});
1838 ZeroCount = Builder.CreateIntCast(V: ZeroCount, DestTy: IndexType, isSigned: false);
1839 Builder.CreateStore(Val: ZeroCount, Addr: IndexAddress, IsVolatile: false);
1840 } else {
1841 unsigned ArgWidth = cast<llvm::IntegerType>(Val: ArgType)->getBitWidth();
1842 Value *ArgTypeLastIndex = llvm::ConstantInt::get(Ty: IndexType, V: ArgWidth - 1);
1843
1844 Function *F = CGM.getIntrinsic(IID: Intrinsic::ctlz, Tys: ArgType);
1845 Value *ZeroCount = Builder.CreateCall(Callee: F, Args: {ArgValue, Builder.getTrue()});
1846 ZeroCount = Builder.CreateIntCast(V: ZeroCount, DestTy: IndexType, isSigned: false);
1847 Value *Index = Builder.CreateNSWSub(LHS: ArgTypeLastIndex, RHS: ZeroCount);
1848 Builder.CreateStore(Val: Index, Addr: IndexAddress, IsVolatile: false);
1849 }
1850 Builder.CreateBr(Dest: End);
1851 Result->addIncoming(V: ResOne, BB: NotZero);
1852
1853 Builder.SetInsertPoint(End);
1854 return Result;
1855 }
1856 case MSVCIntrin::_InterlockedAnd:
1857 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::And, E);
1858 case MSVCIntrin::_InterlockedExchange:
1859 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Xchg, E);
1860 case MSVCIntrin::_InterlockedExchangeAdd:
1861 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Add, E);
1862 case MSVCIntrin::_InterlockedExchangeSub:
1863 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Sub, E);
1864 case MSVCIntrin::_InterlockedOr:
1865 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Or, E);
1866 case MSVCIntrin::_InterlockedXor:
1867 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Xor, E);
1868 case MSVCIntrin::_InterlockedExchangeAdd_acq:
1869 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Add, E,
1870 Ordering: AtomicOrdering::Acquire);
1871 case MSVCIntrin::_InterlockedExchangeAdd_rel:
1872 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Add, E,
1873 Ordering: AtomicOrdering::Release);
1874 case MSVCIntrin::_InterlockedExchangeAdd_nf:
1875 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Add, E,
1876 Ordering: AtomicOrdering::Monotonic);
1877 case MSVCIntrin::_InterlockedExchange_acq:
1878 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Xchg, E,
1879 Ordering: AtomicOrdering::Acquire);
1880 case MSVCIntrin::_InterlockedExchange_rel:
1881 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Xchg, E,
1882 Ordering: AtomicOrdering::Release);
1883 case MSVCIntrin::_InterlockedExchange_nf:
1884 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Xchg, E,
1885 Ordering: AtomicOrdering::Monotonic);
1886 case MSVCIntrin::_InterlockedCompareExchange:
1887 return EmitAtomicCmpXchgForMSIntrin(CGF&: *this, E);
1888 case MSVCIntrin::_InterlockedCompareExchange_acq:
1889 return EmitAtomicCmpXchgForMSIntrin(CGF&: *this, E, SuccessOrdering: AtomicOrdering::Acquire);
1890 case MSVCIntrin::_InterlockedCompareExchange_rel:
1891 return EmitAtomicCmpXchgForMSIntrin(CGF&: *this, E, SuccessOrdering: AtomicOrdering::Release);
1892 case MSVCIntrin::_InterlockedCompareExchange_nf:
1893 return EmitAtomicCmpXchgForMSIntrin(CGF&: *this, E, SuccessOrdering: AtomicOrdering::Monotonic);
1894 case MSVCIntrin::_InterlockedCompareExchange128:
1895 return EmitAtomicCmpXchg128ForMSIntrin(
1896 CGF&: *this, E, SuccessOrdering: AtomicOrdering::SequentiallyConsistent);
1897 case MSVCIntrin::_InterlockedCompareExchange128_acq:
1898 return EmitAtomicCmpXchg128ForMSIntrin(CGF&: *this, E, SuccessOrdering: AtomicOrdering::Acquire);
1899 case MSVCIntrin::_InterlockedCompareExchange128_rel:
1900 return EmitAtomicCmpXchg128ForMSIntrin(CGF&: *this, E, SuccessOrdering: AtomicOrdering::Release);
1901 case MSVCIntrin::_InterlockedCompareExchange128_nf:
1902 return EmitAtomicCmpXchg128ForMSIntrin(CGF&: *this, E, SuccessOrdering: AtomicOrdering::Monotonic);
1903 case MSVCIntrin::_InterlockedOr_acq:
1904 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Or, E,
1905 Ordering: AtomicOrdering::Acquire);
1906 case MSVCIntrin::_InterlockedOr_rel:
1907 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Or, E,
1908 Ordering: AtomicOrdering::Release);
1909 case MSVCIntrin::_InterlockedOr_nf:
1910 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Or, E,
1911 Ordering: AtomicOrdering::Monotonic);
1912 case MSVCIntrin::_InterlockedXor_acq:
1913 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Xor, E,
1914 Ordering: AtomicOrdering::Acquire);
1915 case MSVCIntrin::_InterlockedXor_rel:
1916 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Xor, E,
1917 Ordering: AtomicOrdering::Release);
1918 case MSVCIntrin::_InterlockedXor_nf:
1919 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::Xor, E,
1920 Ordering: AtomicOrdering::Monotonic);
1921 case MSVCIntrin::_InterlockedAnd_acq:
1922 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::And, E,
1923 Ordering: AtomicOrdering::Acquire);
1924 case MSVCIntrin::_InterlockedAnd_rel:
1925 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::And, E,
1926 Ordering: AtomicOrdering::Release);
1927 case MSVCIntrin::_InterlockedAnd_nf:
1928 return MakeBinaryAtomicValue(CGF&: *this, Kind: AtomicRMWInst::And, E,
1929 Ordering: AtomicOrdering::Monotonic);
1930 case MSVCIntrin::_InterlockedIncrement_acq:
1931 return EmitAtomicIncrementValue(CGF&: *this, E, Ordering: AtomicOrdering::Acquire);
1932 case MSVCIntrin::_InterlockedIncrement_rel:
1933 return EmitAtomicIncrementValue(CGF&: *this, E, Ordering: AtomicOrdering::Release);
1934 case MSVCIntrin::_InterlockedIncrement_nf:
1935 return EmitAtomicIncrementValue(CGF&: *this, E, Ordering: AtomicOrdering::Monotonic);
1936 case MSVCIntrin::_InterlockedDecrement_acq:
1937 return EmitAtomicDecrementValue(CGF&: *this, E, Ordering: AtomicOrdering::Acquire);
1938 case MSVCIntrin::_InterlockedDecrement_rel:
1939 return EmitAtomicDecrementValue(CGF&: *this, E, Ordering: AtomicOrdering::Release);
1940 case MSVCIntrin::_InterlockedDecrement_nf:
1941 return EmitAtomicDecrementValue(CGF&: *this, E, Ordering: AtomicOrdering::Monotonic);
1942
1943 case MSVCIntrin::_InterlockedDecrement:
1944 return EmitAtomicDecrementValue(CGF&: *this, E);
1945 case MSVCIntrin::_InterlockedIncrement:
1946 return EmitAtomicIncrementValue(CGF&: *this, E);
1947
1948 case MSVCIntrin::_ReadWriteBarrier:
1949 return Builder.CreateFence(Ordering: llvm::AtomicOrdering::SequentiallyConsistent,
1950 SSID: llvm::SyncScope::SingleThread);
1951
1952 case MSVCIntrin::__fastfail: {
1953 // Request immediate process termination from the kernel. The instruction
1954 // sequences to do this are documented on MSDN:
1955 // https://msdn.microsoft.com/en-us/library/dn774154.aspx
1956 llvm::Triple::ArchType ISA = getTarget().getTriple().getArch();
1957 StringRef Asm, Constraints;
1958 switch (ISA) {
1959 default:
1960 ErrorUnsupported(S: E, Type: "__fastfail call for this architecture");
1961 break;
1962 case llvm::Triple::x86:
1963 case llvm::Triple::x86_64:
1964 Asm = "int $$0x29";
1965 Constraints = "{cx}";
1966 break;
1967 case llvm::Triple::thumb:
1968 Asm = "udf #251";
1969 Constraints = "{r0}";
1970 break;
1971 case llvm::Triple::aarch64:
1972 Asm = "brk #0xF003";
1973 Constraints = "{w0}";
1974 }
1975 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: VoidTy, Params: {Int32Ty}, isVarArg: false);
1976 llvm::InlineAsm *IA =
1977 llvm::InlineAsm::get(Ty: FTy, AsmString: Asm, Constraints, /*hasSideEffects=*/true);
1978 llvm::AttributeList NoReturnAttr = llvm::AttributeList::get(
1979 C&: getLLVMContext(), Index: llvm::AttributeList::FunctionIndex,
1980 Kinds: llvm::Attribute::NoReturn);
1981 llvm::CallInst *CI = Builder.CreateCall(Callee: IA, Args: EmitScalarExpr(E: E->getArg(Arg: 0)));
1982 CI->setAttributes(NoReturnAttr);
1983 return CI;
1984 }
1985 }
1986 llvm_unreachable("Incorrect MSVC intrinsic!");
1987}
1988
1989namespace {
1990// ARC cleanup for __builtin_os_log_format
1991struct CallObjCArcUse final : EHScopeStack::Cleanup {
1992 CallObjCArcUse(llvm::Value *object) : object(object) {}
1993 llvm::Value *object;
1994
1995 void Emit(CodeGenFunction &CGF, Flags flags) override {
1996 CGF.EmitARCIntrinsicUse(values: object);
1997 }
1998};
1999}
2000
2001Value *CodeGenFunction::EmitCheckedArgForBuiltin(const Expr *E,
2002 BuiltinCheckKind Kind) {
2003 assert((Kind == BCK_CLZPassedZero || Kind == BCK_CTZPassedZero) &&
2004 "Unsupported builtin check kind");
2005
2006 Value *ArgValue = EmitBitCountExpr(CGF&: *this, E);
2007 if (!SanOpts.has(K: SanitizerKind::Builtin))
2008 return ArgValue;
2009
2010 auto CheckOrdinal = SanitizerKind::SO_Builtin;
2011 auto CheckHandler = SanitizerHandler::InvalidBuiltin;
2012 SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);
2013 Value *Cond = Builder.CreateICmpNE(
2014 LHS: ArgValue, RHS: llvm::Constant::getNullValue(Ty: ArgValue->getType()));
2015 EmitCheck(Checked: std::make_pair(x&: Cond, y&: CheckOrdinal), Check: CheckHandler,
2016 StaticArgs: {EmitCheckSourceLocation(Loc: E->getExprLoc()),
2017 llvm::ConstantInt::get(Ty: Builder.getInt8Ty(), V: Kind)},
2018 DynamicArgs: {});
2019 return ArgValue;
2020}
2021
2022Value *CodeGenFunction::EmitCheckedArgForAssume(const Expr *E) {
2023 Value *ArgValue = EvaluateExprAsBool(E);
2024 if (!SanOpts.has(K: SanitizerKind::Builtin))
2025 return ArgValue;
2026
2027 auto CheckOrdinal = SanitizerKind::SO_Builtin;
2028 auto CheckHandler = SanitizerHandler::InvalidBuiltin;
2029 SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);
2030 EmitCheck(
2031 Checked: std::make_pair(x&: ArgValue, y&: CheckOrdinal), Check: CheckHandler,
2032 StaticArgs: {EmitCheckSourceLocation(Loc: E->getExprLoc()),
2033 llvm::ConstantInt::get(Ty: Builder.getInt8Ty(), V: BCK_AssumePassedFalse)},
2034 DynamicArgs: {});
2035 return ArgValue;
2036}
2037
2038static Value *EmitAbs(CodeGenFunction &CGF, Value *ArgValue, bool HasNSW) {
2039 return CGF.Builder.CreateBinaryIntrinsic(
2040 ID: Intrinsic::abs, LHS: ArgValue,
2041 RHS: ConstantInt::get(Ty: CGF.Builder.getInt1Ty(), V: HasNSW));
2042}
2043
2044static Value *EmitOverflowCheckedAbs(CodeGenFunction &CGF, const CallExpr *E,
2045 bool SanitizeOverflow) {
2046 Value *ArgValue = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
2047
2048 // Try to eliminate overflow check.
2049 if (const auto *VCI = dyn_cast<llvm::ConstantInt>(Val: ArgValue)) {
2050 if (!VCI->isMinSignedValue())
2051 return EmitAbs(CGF, ArgValue, HasNSW: true);
2052 }
2053
2054 SmallVector<SanitizerKind::SanitizerOrdinal, 1> Ordinals;
2055 SanitizerHandler CheckHandler;
2056 if (SanitizeOverflow) {
2057 Ordinals.push_back(Elt: SanitizerKind::SO_SignedIntegerOverflow);
2058 CheckHandler = SanitizerHandler::NegateOverflow;
2059 } else
2060 CheckHandler = SanitizerHandler::SubOverflow;
2061
2062 SanitizerDebugLocation SanScope(&CGF, Ordinals, CheckHandler);
2063
2064 Constant *Zero = Constant::getNullValue(Ty: ArgValue->getType());
2065 Value *ResultAndOverflow = CGF.Builder.CreateBinaryIntrinsic(
2066 ID: Intrinsic::ssub_with_overflow, LHS: Zero, RHS: ArgValue);
2067 Value *Result = CGF.Builder.CreateExtractValue(Agg: ResultAndOverflow, Idxs: 0);
2068 Value *NotOverflow = CGF.Builder.CreateNot(
2069 V: CGF.Builder.CreateExtractValue(Agg: ResultAndOverflow, Idxs: 1));
2070
2071 // TODO: support -ftrapv-handler.
2072 if (SanitizeOverflow) {
2073 CGF.EmitCheck(Checked: {{NotOverflow, SanitizerKind::SO_SignedIntegerOverflow}},
2074 Check: CheckHandler,
2075 StaticArgs: {CGF.EmitCheckSourceLocation(Loc: E->getArg(Arg: 0)->getExprLoc()),
2076 CGF.EmitCheckTypeDescriptor(T: E->getType())},
2077 DynamicArgs: {ArgValue});
2078 } else
2079 CGF.EmitTrapCheck(Checked: NotOverflow, CheckHandlerID: CheckHandler);
2080
2081 Value *CmpResult = CGF.Builder.CreateICmpSLT(LHS: ArgValue, RHS: Zero, Name: "abscond");
2082 return CGF.Builder.CreateSelect(C: CmpResult, True: Result, False: ArgValue, Name: "abs");
2083}
2084
2085/// Get the argument type for arguments to os_log_helper.
2086static CanQualType getOSLogArgType(ASTContext &C, int Size) {
2087 QualType UnsignedTy = C.getIntTypeForBitwidth(DestWidth: Size * 8, /*Signed=*/false);
2088 return C.getCanonicalType(T: UnsignedTy);
2089}
2090
2091llvm::Function *CodeGenFunction::generateBuiltinOSLogHelperFunction(
2092 const analyze_os_log::OSLogBufferLayout &Layout,
2093 CharUnits BufferAlignment) {
2094 ASTContext &Ctx = getContext();
2095
2096 llvm::SmallString<64> Name;
2097 {
2098 raw_svector_ostream OS(Name);
2099 OS << "__os_log_helper";
2100 OS << "_" << BufferAlignment.getQuantity();
2101 OS << "_" << int(Layout.getSummaryByte());
2102 OS << "_" << int(Layout.getNumArgsByte());
2103 for (const auto &Item : Layout.Items)
2104 OS << "_" << int(Item.getSizeByte()) << "_"
2105 << int(Item.getDescriptorByte());
2106 }
2107
2108 if (llvm::Function *F = CGM.getModule().getFunction(Name))
2109 return F;
2110
2111 llvm::SmallVector<QualType, 4> ArgTys;
2112 FunctionArgList Args;
2113 Args.push_back(Elt: ImplicitParamDecl::Create(
2114 C&: Ctx, DC: nullptr, IdLoc: SourceLocation(), Id: &Ctx.Idents.get(Name: "buffer"), T: Ctx.VoidPtrTy,
2115 ParamKind: ImplicitParamKind::Other));
2116 ArgTys.emplace_back(Args&: Ctx.VoidPtrTy);
2117
2118 for (unsigned int I = 0, E = Layout.Items.size(); I < E; ++I) {
2119 char Size = Layout.Items[I].getSizeByte();
2120 if (!Size)
2121 continue;
2122
2123 QualType ArgTy = getOSLogArgType(C&: Ctx, Size);
2124 Args.push_back(Elt: ImplicitParamDecl::Create(
2125 C&: Ctx, DC: nullptr, IdLoc: SourceLocation(),
2126 Id: &Ctx.Idents.get(Name: std::string("arg") + llvm::to_string(Value: I)), T: ArgTy,
2127 ParamKind: ImplicitParamKind::Other));
2128 ArgTys.emplace_back(Args&: ArgTy);
2129 }
2130
2131 QualType ReturnTy = Ctx.VoidTy;
2132
2133 // The helper function has linkonce_odr linkage to enable the linker to merge
2134 // identical functions. To ensure the merging always happens, 'noinline' is
2135 // attached to the function when compiling with -Oz.
2136 const CGFunctionInfo &FI =
2137 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: ReturnTy, args: Args);
2138 llvm::FunctionType *FuncTy = CGM.getTypes().GetFunctionType(Info: FI);
2139 llvm::Function *Fn = llvm::Function::Create(
2140 Ty: FuncTy, Linkage: llvm::GlobalValue::LinkOnceODRLinkage, N: Name, M: &CGM.getModule());
2141 Fn->setVisibility(llvm::GlobalValue::HiddenVisibility);
2142 CGM.SetLLVMFunctionAttributes(GD: GlobalDecl(), Info: FI, F: Fn, /*IsThunk=*/false);
2143 CGM.SetLLVMFunctionAttributesForDefinition(D: nullptr, F: Fn);
2144 Fn->setDoesNotThrow();
2145
2146 // Attach 'noinline' at -Oz.
2147 if (CGM.getCodeGenOpts().OptimizeSize == 2)
2148 Fn->addFnAttr(Kind: llvm::Attribute::NoInline);
2149
2150 auto NL = ApplyDebugLocation::CreateEmpty(CGF&: *this);
2151 StartFunction(GD: GlobalDecl(), RetTy: ReturnTy, Fn, FnInfo: FI, Args);
2152
2153 // Create a scope with an artificial location for the body of this function.
2154 auto AL = ApplyDebugLocation::CreateArtificial(CGF&: *this);
2155
2156 CharUnits Offset;
2157 Address BufAddr = makeNaturalAddressForPointer(
2158 Ptr: Builder.CreateLoad(Addr: GetAddrOfLocalVar(VD: Args[0]), Name: "buf"), T: Ctx.VoidTy,
2159 Alignment: BufferAlignment);
2160 Builder.CreateStore(Val: Builder.getInt8(C: Layout.getSummaryByte()),
2161 Addr: Builder.CreateConstByteGEP(Addr: BufAddr, Offset: Offset++, Name: "summary"));
2162 Builder.CreateStore(Val: Builder.getInt8(C: Layout.getNumArgsByte()),
2163 Addr: Builder.CreateConstByteGEP(Addr: BufAddr, Offset: Offset++, Name: "numArgs"));
2164
2165 unsigned I = 1;
2166 for (const auto &Item : Layout.Items) {
2167 Builder.CreateStore(
2168 Val: Builder.getInt8(C: Item.getDescriptorByte()),
2169 Addr: Builder.CreateConstByteGEP(Addr: BufAddr, Offset: Offset++, Name: "argDescriptor"));
2170 Builder.CreateStore(
2171 Val: Builder.getInt8(C: Item.getSizeByte()),
2172 Addr: Builder.CreateConstByteGEP(Addr: BufAddr, Offset: Offset++, Name: "argSize"));
2173
2174 CharUnits Size = Item.size();
2175 if (!Size.getQuantity())
2176 continue;
2177
2178 Address Arg = GetAddrOfLocalVar(VD: Args[I]);
2179 Address Addr = Builder.CreateConstByteGEP(Addr: BufAddr, Offset, Name: "argData");
2180 Addr = Addr.withElementType(ElemTy: Arg.getElementType());
2181 Builder.CreateStore(Val: Builder.CreateLoad(Addr: Arg), Addr);
2182 Offset += Size;
2183 ++I;
2184 }
2185
2186 FinishFunction();
2187
2188 return Fn;
2189}
2190
2191RValue CodeGenFunction::emitBuiltinOSLogFormat(const CallExpr &E) {
2192 assert(E.getNumArgs() >= 2 &&
2193 "__builtin_os_log_format takes at least 2 arguments");
2194 ASTContext &Ctx = getContext();
2195 analyze_os_log::OSLogBufferLayout Layout;
2196 analyze_os_log::computeOSLogBufferLayout(Ctx, E: &E, layout&: Layout);
2197 Address BufAddr = EmitPointerWithAlignment(Addr: E.getArg(Arg: 0));
2198
2199 // Ignore argument 1, the format string. It is not currently used.
2200 CallArgList Args;
2201 Args.add(rvalue: RValue::get(V: BufAddr.emitRawPointer(CGF&: *this)), type: Ctx.VoidPtrTy);
2202
2203 for (const auto &Item : Layout.Items) {
2204 int Size = Item.getSizeByte();
2205 if (!Size)
2206 continue;
2207
2208 llvm::Value *ArgVal;
2209
2210 if (Item.getKind() == analyze_os_log::OSLogBufferItem::MaskKind) {
2211 uint64_t Val = 0;
2212 for (unsigned I = 0, E = Item.getMaskType().size(); I < E; ++I)
2213 Val |= ((uint64_t)Item.getMaskType()[I]) << I * 8;
2214 ArgVal = llvm::Constant::getIntegerValue(Ty: Int64Ty, V: llvm::APInt(64, Val));
2215 } else if (const Expr *TheExpr = Item.getExpr()) {
2216 ArgVal = EmitScalarExpr(E: TheExpr, /*Ignore*/ IgnoreResultAssign: false);
2217
2218 // If a temporary object that requires destruction after the full
2219 // expression is passed, push a lifetime-extended cleanup to extend its
2220 // lifetime to the end of the enclosing block scope.
2221 auto LifetimeExtendObject = [&](const Expr *E) {
2222 E = E->IgnoreParenCasts();
2223 // Extend lifetimes of objects returned by function calls and message
2224 // sends.
2225
2226 // FIXME: We should do this in other cases in which temporaries are
2227 // created including arguments of non-ARC types (e.g., C++
2228 // temporaries).
2229 if (isa<CallExpr>(Val: E) || isa<ObjCMessageExpr>(Val: E))
2230 return true;
2231 return false;
2232 };
2233
2234 if (TheExpr->getType()->isObjCRetainableType() &&
2235 getLangOpts().ObjCAutoRefCount && LifetimeExtendObject(TheExpr)) {
2236 assert(getEvaluationKind(TheExpr->getType()) == TEK_Scalar &&
2237 "Only scalar can be a ObjC retainable type");
2238 if (!isa<Constant>(Val: ArgVal)) {
2239 CleanupKind Cleanup = getARCCleanupKind();
2240 QualType Ty = TheExpr->getType();
2241 RawAddress Alloca = CreateMemTempWithoutCast(T: Ty, Name: "os.log.arg");
2242 ArgVal = EmitARCRetain(type: Ty, value: ArgVal);
2243 Builder.CreateStore(Val: ArgVal, Addr: Alloca);
2244 pushLifetimeExtendedDestroy(kind: Cleanup, addr: Alloca, type: Ty,
2245 destroyer: CodeGenFunction::destroyARCStrongPrecise,
2246 useEHCleanupForArray: Cleanup & EHCleanup);
2247
2248 // Push a clang.arc.use call to ensure ARC optimizer knows that the
2249 // argument has to be alive.
2250 if (CGM.getCodeGenOpts().OptimizationLevel != 0)
2251 pushCleanupAfterFullExpr<CallObjCArcUse>(Kind: Cleanup, A: ArgVal);
2252 }
2253 }
2254 } else {
2255 ArgVal = Builder.getInt32(C: Item.getConstValue().getQuantity());
2256 }
2257
2258 unsigned ArgValSize =
2259 CGM.getDataLayout().getTypeSizeInBits(Ty: ArgVal->getType());
2260 llvm::IntegerType *IntTy = llvm::Type::getIntNTy(C&: getLLVMContext(),
2261 N: ArgValSize);
2262 ArgVal = Builder.CreateBitOrPointerCast(V: ArgVal, DestTy: IntTy);
2263 CanQualType ArgTy = getOSLogArgType(C&: Ctx, Size);
2264 // If ArgVal has type x86_fp80, zero-extend ArgVal.
2265 ArgVal = Builder.CreateZExtOrBitCast(V: ArgVal, DestTy: ConvertType(T: ArgTy));
2266 Args.add(rvalue: RValue::get(V: ArgVal), type: ArgTy);
2267 }
2268
2269 const CGFunctionInfo &FI =
2270 CGM.getTypes().arrangeBuiltinFunctionCall(resultType: Ctx.VoidTy, args: Args);
2271 llvm::Function *F = CodeGenFunction(CGM).generateBuiltinOSLogHelperFunction(
2272 Layout, BufferAlignment: BufAddr.getAlignment());
2273 EmitCall(CallInfo: FI, Callee: CGCallee::forDirect(functionPtr: F), ReturnValue: ReturnValueSlot(), Args);
2274 return RValue::get(Addr: BufAddr, CGF&: *this);
2275}
2276
2277static bool isSpecialUnsignedMultiplySignedResult(
2278 unsigned BuiltinID, WidthAndSignedness Op1Info, WidthAndSignedness Op2Info,
2279 WidthAndSignedness ResultInfo) {
2280 return BuiltinID == Builtin::BI__builtin_mul_overflow &&
2281 Op1Info.Width == Op2Info.Width && Op2Info.Width == ResultInfo.Width &&
2282 !Op1Info.Signed && !Op2Info.Signed && ResultInfo.Signed;
2283}
2284
2285static RValue EmitCheckedUnsignedMultiplySignedResult(
2286 CodeGenFunction &CGF, const clang::Expr *Op1, WidthAndSignedness Op1Info,
2287 const clang::Expr *Op2, WidthAndSignedness Op2Info,
2288 const clang::Expr *ResultArg, QualType ResultQTy,
2289 WidthAndSignedness ResultInfo) {
2290 assert(isSpecialUnsignedMultiplySignedResult(
2291 Builtin::BI__builtin_mul_overflow, Op1Info, Op2Info, ResultInfo) &&
2292 "Cannot specialize this multiply");
2293
2294 llvm::Value *V1 = CGF.EmitScalarExpr(E: Op1);
2295 llvm::Value *V2 = CGF.EmitScalarExpr(E: Op2);
2296
2297 llvm::Value *HasOverflow;
2298 llvm::Value *Result = EmitOverflowIntrinsic(
2299 CGF, IntrinsicID: Intrinsic::umul_with_overflow, X: V1, Y: V2, Carry&: HasOverflow);
2300
2301 // The intrinsic call will detect overflow when the value is > UINT_MAX,
2302 // however, since the original builtin had a signed result, we need to report
2303 // an overflow when the result is greater than INT_MAX.
2304 auto IntMax = llvm::APInt::getSignedMaxValue(numBits: ResultInfo.Width);
2305 llvm::Value *IntMaxValue = llvm::ConstantInt::get(Ty: Result->getType(), V: IntMax);
2306
2307 llvm::Value *IntMaxOverflow = CGF.Builder.CreateICmpUGT(LHS: Result, RHS: IntMaxValue);
2308 HasOverflow = CGF.Builder.CreateOr(LHS: HasOverflow, RHS: IntMaxOverflow);
2309
2310 bool isVolatile =
2311 ResultArg->getType()->getPointeeType().isVolatileQualified();
2312 Address ResultPtr = CGF.EmitPointerWithAlignment(Addr: ResultArg);
2313 CGF.Builder.CreateStore(Val: CGF.EmitToMemory(Value: Result, Ty: ResultQTy), Addr: ResultPtr,
2314 IsVolatile: isVolatile);
2315 return RValue::get(V: HasOverflow);
2316}
2317
2318/// Determine if a binop is a checked mixed-sign multiply we can specialize.
2319static bool isSpecialMixedSignMultiply(unsigned BuiltinID,
2320 WidthAndSignedness Op1Info,
2321 WidthAndSignedness Op2Info,
2322 WidthAndSignedness ResultInfo) {
2323 return BuiltinID == Builtin::BI__builtin_mul_overflow &&
2324 std::max(a: Op1Info.Width, b: Op2Info.Width) >= ResultInfo.Width &&
2325 Op1Info.Signed != Op2Info.Signed;
2326}
2327
2328/// Emit a checked mixed-sign multiply. This is a cheaper specialization of
2329/// the generic checked-binop irgen.
2330static RValue
2331EmitCheckedMixedSignMultiply(CodeGenFunction &CGF, const clang::Expr *Op1,
2332 WidthAndSignedness Op1Info, const clang::Expr *Op2,
2333 WidthAndSignedness Op2Info,
2334 const clang::Expr *ResultArg, QualType ResultQTy,
2335 WidthAndSignedness ResultInfo) {
2336 assert(isSpecialMixedSignMultiply(Builtin::BI__builtin_mul_overflow, Op1Info,
2337 Op2Info, ResultInfo) &&
2338 "Not a mixed-sign multipliction we can specialize");
2339
2340 // Emit the signed and unsigned operands.
2341 const clang::Expr *SignedOp = Op1Info.Signed ? Op1 : Op2;
2342 const clang::Expr *UnsignedOp = Op1Info.Signed ? Op2 : Op1;
2343 llvm::Value *Signed = CGF.EmitScalarExpr(E: SignedOp);
2344 llvm::Value *Unsigned = CGF.EmitScalarExpr(E: UnsignedOp);
2345 unsigned SignedOpWidth = Op1Info.Signed ? Op1Info.Width : Op2Info.Width;
2346 unsigned UnsignedOpWidth = Op1Info.Signed ? Op2Info.Width : Op1Info.Width;
2347
2348 // One of the operands may be smaller than the other. If so, [s|z]ext it.
2349 if (SignedOpWidth < UnsignedOpWidth)
2350 Signed = CGF.Builder.CreateSExt(V: Signed, DestTy: Unsigned->getType(), Name: "op.sext");
2351 if (UnsignedOpWidth < SignedOpWidth)
2352 Unsigned = CGF.Builder.CreateZExt(V: Unsigned, DestTy: Signed->getType(), Name: "op.zext");
2353
2354 llvm::Type *OpTy = Signed->getType();
2355 llvm::Value *Zero = llvm::Constant::getNullValue(Ty: OpTy);
2356 Address ResultPtr = CGF.EmitPointerWithAlignment(Addr: ResultArg);
2357 llvm::Type *ResTy = CGF.getTypes().ConvertType(T: ResultQTy);
2358 unsigned OpWidth = std::max(a: Op1Info.Width, b: Op2Info.Width);
2359
2360 // Take the absolute value of the signed operand.
2361 llvm::Value *IsNegative = CGF.Builder.CreateICmpSLT(LHS: Signed, RHS: Zero);
2362 llvm::Value *AbsOfNegative = CGF.Builder.CreateSub(LHS: Zero, RHS: Signed);
2363 llvm::Value *AbsSigned =
2364 CGF.Builder.CreateSelect(C: IsNegative, True: AbsOfNegative, False: Signed);
2365
2366 // Perform a checked unsigned multiplication.
2367 llvm::Value *UnsignedOverflow;
2368 llvm::Value *UnsignedResult =
2369 EmitOverflowIntrinsic(CGF, IntrinsicID: Intrinsic::umul_with_overflow, X: AbsSigned,
2370 Y: Unsigned, Carry&: UnsignedOverflow);
2371
2372 llvm::Value *Overflow, *Result;
2373 if (ResultInfo.Signed) {
2374 // Signed overflow occurs if the result is greater than INT_MAX or lesser
2375 // than INT_MIN, i.e when |Result| > (INT_MAX + IsNegative).
2376 auto IntMax =
2377 llvm::APInt::getSignedMaxValue(numBits: ResultInfo.Width).zext(width: OpWidth);
2378 llvm::Value *MaxResult =
2379 CGF.Builder.CreateAdd(LHS: llvm::ConstantInt::get(Ty: OpTy, V: IntMax),
2380 RHS: CGF.Builder.CreateZExt(V: IsNegative, DestTy: OpTy));
2381 llvm::Value *SignedOverflow =
2382 CGF.Builder.CreateICmpUGT(LHS: UnsignedResult, RHS: MaxResult);
2383 Overflow = CGF.Builder.CreateOr(LHS: UnsignedOverflow, RHS: SignedOverflow);
2384
2385 // Prepare the signed result (possibly by negating it).
2386 llvm::Value *NegativeResult = CGF.Builder.CreateNeg(V: UnsignedResult);
2387 llvm::Value *SignedResult =
2388 CGF.Builder.CreateSelect(C: IsNegative, True: NegativeResult, False: UnsignedResult);
2389 Result = CGF.Builder.CreateTrunc(V: SignedResult, DestTy: ResTy);
2390 } else {
2391 // Unsigned overflow occurs if the result is < 0 or greater than UINT_MAX.
2392 llvm::Value *Underflow = CGF.Builder.CreateAnd(
2393 LHS: IsNegative, RHS: CGF.Builder.CreateIsNotNull(Arg: UnsignedResult));
2394 Overflow = CGF.Builder.CreateOr(LHS: UnsignedOverflow, RHS: Underflow);
2395 if (ResultInfo.Width < OpWidth) {
2396 auto IntMax =
2397 llvm::APInt::getMaxValue(numBits: ResultInfo.Width).zext(width: OpWidth);
2398 llvm::Value *TruncOverflow = CGF.Builder.CreateICmpUGT(
2399 LHS: UnsignedResult, RHS: llvm::ConstantInt::get(Ty: OpTy, V: IntMax));
2400 Overflow = CGF.Builder.CreateOr(LHS: Overflow, RHS: TruncOverflow);
2401 }
2402
2403 // Negate the product if it would be negative in infinite precision.
2404 Result = CGF.Builder.CreateSelect(
2405 C: IsNegative, True: CGF.Builder.CreateNeg(V: UnsignedResult), False: UnsignedResult);
2406
2407 Result = CGF.Builder.CreateTrunc(V: Result, DestTy: ResTy);
2408 }
2409 assert(Overflow && Result && "Missing overflow or result");
2410
2411 bool isVolatile =
2412 ResultArg->getType()->getPointeeType().isVolatileQualified();
2413 CGF.Builder.CreateStore(Val: CGF.EmitToMemory(Value: Result, Ty: ResultQTy), Addr: ResultPtr,
2414 IsVolatile: isVolatile);
2415 return RValue::get(V: Overflow);
2416}
2417
2418/// Determine if the specified type requires laundering by checking if it is a
2419/// dynamic class type or contains a subobject which is a dynamic class type.
2420static bool TypeRequiresBuiltinLaunder(CodeGenModule &CGM, QualType Ty) {
2421 if (!CGM.getCodeGenOpts().StrictVTablePointers)
2422 return false;
2423 return Ty.requiresBuiltinLaunder(Context: CGM.getContext());
2424}
2425
2426RValue CodeGenFunction::emitRotate(const CallExpr *E, bool IsRotateRight) {
2427 llvm::Value *Src = EmitScalarExpr(E: E->getArg(Arg: 0));
2428 llvm::Value *ShiftAmt = EmitScalarExpr(E: E->getArg(Arg: 1));
2429
2430 // The builtin's shift arg may have a different type than the source arg and
2431 // result, but the LLVM intrinsic uses the same type for all values.
2432 llvm::Type *Ty = Src->getType();
2433 llvm::Type *ShiftTy = ShiftAmt->getType();
2434
2435 unsigned BitWidth = Ty->getIntegerBitWidth();
2436
2437 // Normalize shift amount to [0, BitWidth) range to match runtime behavior.
2438 // This matches the algorithm in ExprConstant.cpp for constant evaluation.
2439 if (BitWidth == 1) {
2440 // Rotating a 1-bit value is always a no-op
2441 ShiftAmt = ConstantInt::get(Ty: ShiftTy, V: 0);
2442 } else if (BitWidth == 2) {
2443 // For 2-bit values: rotation amount is 0 or 1 based on
2444 // whether the amount is even or odd. We can't use srem here because
2445 // the divisor (2) would be misinterpreted as -2 in 2-bit signed arithmetic.
2446 llvm::Value *One = ConstantInt::get(Ty: ShiftTy, V: 1);
2447 ShiftAmt = Builder.CreateAnd(LHS: ShiftAmt, RHS: One);
2448 } else {
2449 unsigned ShiftAmtBitWidth = ShiftTy->getIntegerBitWidth();
2450 bool ShiftAmtIsSigned = E->getArg(Arg: 1)->getType()->isSignedIntegerType();
2451
2452 // Choose the wider type for the divisor to avoid truncation
2453 llvm::Type *DivisorTy = ShiftAmtBitWidth > BitWidth ? ShiftTy : Ty;
2454 llvm::Value *Divisor = ConstantInt::get(Ty: DivisorTy, V: BitWidth);
2455
2456 // Extend ShiftAmt to match Divisor width if needed
2457 if (ShiftAmtBitWidth < DivisorTy->getIntegerBitWidth()) {
2458 ShiftAmt = Builder.CreateIntCast(V: ShiftAmt, DestTy: DivisorTy, isSigned: ShiftAmtIsSigned);
2459 }
2460
2461 // Normalize to [0, BitWidth)
2462 llvm::Value *RemResult;
2463 if (ShiftAmtIsSigned) {
2464 RemResult = Builder.CreateSRem(LHS: ShiftAmt, RHS: Divisor);
2465 // Signed remainder can be negative, convert to positive equivalent
2466 llvm::Value *Zero = ConstantInt::get(Ty: DivisorTy, V: 0);
2467 llvm::Value *IsNegative = Builder.CreateICmpSLT(LHS: RemResult, RHS: Zero);
2468 llvm::Value *PositiveShift = Builder.CreateAdd(LHS: RemResult, RHS: Divisor);
2469 ShiftAmt = Builder.CreateSelect(C: IsNegative, True: PositiveShift, False: RemResult);
2470 } else {
2471 ShiftAmt = Builder.CreateURem(LHS: ShiftAmt, RHS: Divisor);
2472 }
2473 }
2474
2475 // Convert to the source type if needed
2476 if (ShiftAmt->getType() != Ty) {
2477 ShiftAmt = Builder.CreateIntCast(V: ShiftAmt, DestTy: Ty, isSigned: false);
2478 }
2479
2480 // Rotate is a special case of LLVM funnel shift - 1st 2 args are the same.
2481 unsigned IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;
2482 Function *F = CGM.getIntrinsic(IID, Tys: Ty);
2483 return RValue::get(V: Builder.CreateCall(Callee: F, Args: {Src, Src, ShiftAmt}));
2484}
2485
2486// Map math builtins for long-double to f128 version.
2487static unsigned mutateLongDoubleBuiltin(unsigned BuiltinID) {
2488 switch (BuiltinID) {
2489#define MUTATE_LDBL(func) \
2490 case Builtin::BI__builtin_##func##l: \
2491 return Builtin::BI__builtin_##func##f128;
2492 MUTATE_LDBL(sqrt)
2493 MUTATE_LDBL(cbrt)
2494 MUTATE_LDBL(fabs)
2495 MUTATE_LDBL(log)
2496 MUTATE_LDBL(log2)
2497 MUTATE_LDBL(log10)
2498 MUTATE_LDBL(log1p)
2499 MUTATE_LDBL(logb)
2500 MUTATE_LDBL(exp)
2501 MUTATE_LDBL(exp2)
2502 MUTATE_LDBL(expm1)
2503 MUTATE_LDBL(fdim)
2504 MUTATE_LDBL(hypot)
2505 MUTATE_LDBL(ilogb)
2506 MUTATE_LDBL(pow)
2507 MUTATE_LDBL(fmin)
2508 MUTATE_LDBL(fmax)
2509 MUTATE_LDBL(ceil)
2510 MUTATE_LDBL(trunc)
2511 MUTATE_LDBL(rint)
2512 MUTATE_LDBL(nearbyint)
2513 MUTATE_LDBL(round)
2514 MUTATE_LDBL(floor)
2515 MUTATE_LDBL(lround)
2516 MUTATE_LDBL(llround)
2517 MUTATE_LDBL(lrint)
2518 MUTATE_LDBL(llrint)
2519 MUTATE_LDBL(fmod)
2520 MUTATE_LDBL(modf)
2521 MUTATE_LDBL(nan)
2522 MUTATE_LDBL(nans)
2523 MUTATE_LDBL(inf)
2524 MUTATE_LDBL(fma)
2525 MUTATE_LDBL(sin)
2526 MUTATE_LDBL(cos)
2527 MUTATE_LDBL(tan)
2528 MUTATE_LDBL(sinh)
2529 MUTATE_LDBL(cosh)
2530 MUTATE_LDBL(tanh)
2531 MUTATE_LDBL(asin)
2532 MUTATE_LDBL(acos)
2533 MUTATE_LDBL(atan)
2534 MUTATE_LDBL(asinh)
2535 MUTATE_LDBL(acosh)
2536 MUTATE_LDBL(atanh)
2537 MUTATE_LDBL(atan2)
2538 MUTATE_LDBL(erf)
2539 MUTATE_LDBL(erfc)
2540 MUTATE_LDBL(ldexp)
2541 MUTATE_LDBL(frexp)
2542 MUTATE_LDBL(huge_val)
2543 MUTATE_LDBL(copysign)
2544 MUTATE_LDBL(nextafter)
2545 MUTATE_LDBL(nexttoward)
2546 MUTATE_LDBL(remainder)
2547 MUTATE_LDBL(remquo)
2548 MUTATE_LDBL(scalbln)
2549 MUTATE_LDBL(scalbn)
2550 MUTATE_LDBL(tgamma)
2551 MUTATE_LDBL(lgamma)
2552#undef MUTATE_LDBL
2553 default:
2554 return BuiltinID;
2555 }
2556}
2557
2558static Value *tryUseTestFPKind(CodeGenFunction &CGF, unsigned BuiltinID,
2559 Value *V) {
2560 if (CGF.Builder.getIsFPConstrained() &&
2561 CGF.Builder.getDefaultConstrainedExcept() != fp::ebIgnore) {
2562 if (Value *Result =
2563 CGF.getTargetHooks().testFPKind(V, BuiltinID, Builder&: CGF.Builder, CGM&: CGF.CGM))
2564 return Result;
2565 }
2566 return nullptr;
2567}
2568
2569static RValue EmitHipStdParUnsupportedBuiltin(CodeGenFunction *CGF,
2570 const FunctionDecl *FD) {
2571 auto Name = FD->getNameAsString() + "__hipstdpar_unsupported";
2572 auto FnTy = CGF->CGM.getTypes().GetFunctionType(GD: FD);
2573 auto UBF = CGF->CGM.getModule().getOrInsertFunction(Name, T: FnTy);
2574
2575 SmallVector<Value *, 16> Args;
2576 for (auto &&FormalTy : FnTy->params())
2577 Args.push_back(Elt: llvm::PoisonValue::get(T: FormalTy));
2578
2579 return RValue::get(V: CGF->Builder.CreateCall(Callee: UBF, Args));
2580}
2581
2582// stdc_{leading,trailing}_{zeros,ones} and stdc_count_ones: counts bits using
2583// ctlz, cttz, or ctpop (IsPop). InvertArg flips the input to count the
2584// opposite bit value.
2585RValue CodeGenFunction::emitStdcCountIntrinsic(const CallExpr *E,
2586 Intrinsic::ID IntID,
2587 bool InvertArg, bool IsPop) {
2588 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
2589 llvm::Type *ArgType = ArgValue->getType();
2590 llvm::Type *ResultType = ConvertType(T: E->getType());
2591 Value *ActualArg = InvertArg ? Builder.CreateNot(V: ArgValue) : ArgValue;
2592 Function *F = CGM.getIntrinsic(IID: IntID, Tys: ArgType);
2593 Value *Result = IsPop
2594 ? Builder.CreateCall(Callee: F, Args: ActualArg)
2595 : Builder.CreateCall(Callee: F, Args: {ActualArg, Builder.getFalse()});
2596 if (Result->getType() != ResultType)
2597 Result = Builder.CreateIntCast(V: Result, DestTy: ResultType, isSigned: false);
2598 return RValue::get(V: Result);
2599}
2600
2601// stdc_count_zeros (BitWidth - ctpop) and stdc_bit_width (BitWidth - ctlz).
2602// IsPop selects ctpop; otherwise ctlz is used.
2603RValue CodeGenFunction::emitStdcBitWidthMinus(const CallExpr *E,
2604 Intrinsic::ID IntID, bool IsPop) {
2605 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
2606 llvm::Type *ArgType = ArgValue->getType();
2607 llvm::Type *ResultType = ConvertType(T: E->getType());
2608 unsigned BitWidth = ArgType->getIntegerBitWidth();
2609 Function *F = CGM.getIntrinsic(IID: IntID, Tys: ArgType);
2610 Value *Cnt = IsPop ? Builder.CreateCall(Callee: F, Args: ArgValue)
2611 : Builder.CreateCall(Callee: F, Args: {ArgValue, Builder.getFalse()});
2612 Value *Result = Builder.CreateSub(LHS: ConstantInt::get(Ty: ArgType, V: BitWidth), RHS: Cnt);
2613 if (Result->getType() != ResultType)
2614 Result = Builder.CreateIntCast(V: Result, DestTy: ResultType, isSigned: false);
2615 return RValue::get(V: Result);
2616}
2617
2618// stdc_first_{leading,trailing}_{zero,one}: returns the 1-based position of
2619// the first matching bit, or 0 if no such bit exists. InvertArg flips the
2620// input to search for zeros instead of ones.
2621RValue CodeGenFunction::emitStdcFirstBit(const CallExpr *E, Intrinsic::ID IntID,
2622 bool InvertArg) {
2623 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
2624 llvm::Type *ArgType = ArgValue->getType();
2625 llvm::Type *ResultType = ConvertType(T: E->getType());
2626 Value *Zero = ConstantInt::get(Ty: ArgType, V: 0);
2627 Value *One = ConstantInt::get(Ty: ArgType, V: 1);
2628 Value *ActualArg = InvertArg ? Builder.CreateNot(V: ArgValue) : ArgValue;
2629 Function *F = CGM.getIntrinsic(IID: IntID, Tys: ArgType);
2630 Value *Cnt = Builder.CreateCall(Callee: F, Args: {ActualArg, Builder.getFalse()});
2631 Value *Tmp = Builder.CreateAdd(LHS: Cnt, RHS: One);
2632 Value *IsZero = Builder.CreateICmpEQ(LHS: ActualArg, RHS: Zero);
2633 Value *Result = Builder.CreateSelect(C: IsZero, True: Zero, False: Tmp);
2634 if (Result->getType() != ResultType)
2635 Result = Builder.CreateIntCast(V: Result, DestTy: ResultType, isSigned: false);
2636 return RValue::get(V: Result);
2637}
2638
2639static void ClearPadding(CodeGenFunction &CGF, Address Src,
2640 const ASTContext::BitInterval &PaddingInterval) {
2641 uint64_t CharWidth = CGF.getContext().getCharWidth();
2642
2643 auto *I8Ptr = CGF.Builder.CreateBitCast(V: Src.getBasePointer(), DestTy: CGF.Int8PtrTy);
2644 auto *Zero = ConstantInt::get(Ty: CGF.Int8Ty, V: 0);
2645
2646 // Calculate byte indices and bit positions
2647 auto StartByte = PaddingInterval.First / CharWidth;
2648 auto StartBit = PaddingInterval.First % CharWidth;
2649 auto EndByte = PaddingInterval.Last / CharWidth;
2650 auto EndBit = PaddingInterval.Last % CharWidth;
2651
2652 if (StartByte == EndByte) {
2653 // Interval is within a single byte
2654 auto *Index = ConstantInt::get(Ty: CGF.IntTy, V: StartByte);
2655 auto *Element = CGF.Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: I8Ptr, IdxList: Index);
2656 Address ElementAddr(Element, CGF.Int8Ty,
2657 Src.getAlignment().alignmentAtOffset(
2658 offset: CharUnits::fromQuantity(Quantity: StartByte)));
2659
2660 auto *Value = CGF.Builder.CreateLoad(Addr: ElementAddr);
2661
2662 // Create mask to clear bits within the byte
2663 // We want to clear bits from StartBit to EndBit-1
2664 uint8_t bitsToClear = ((1 << EndBit) - 1) & ~((1 << StartBit) - 1);
2665 uint8_t bitsToKeep = ~bitsToClear;
2666 auto *MaskValue = ConstantInt::get(Ty: CGF.Int8Ty, V: bitsToKeep);
2667 auto *NewValue = CGF.Builder.CreateAnd(LHS: Value, RHS: MaskValue);
2668
2669 CGF.Builder.CreateStore(Val: NewValue, Addr: ElementAddr);
2670 } else {
2671 // Handle the start byte
2672 if (StartBit != 0) {
2673 auto *Index = ConstantInt::get(Ty: CGF.IntTy, V: StartByte);
2674 auto *Element = CGF.Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: I8Ptr, IdxList: Index);
2675 Address ElementAddr(Element, CGF.Int8Ty,
2676 Src.getAlignment().alignmentAtOffset(
2677 offset: CharUnits::fromQuantity(Quantity: StartByte)));
2678
2679 auto *Value = CGF.Builder.CreateLoad(Addr: ElementAddr);
2680
2681 uint8_t bitsToClear = ((1 << (CharWidth - StartBit)) - 1) << StartBit;
2682 uint8_t bitsToKeep = ~bitsToClear;
2683 auto *MaskValue = ConstantInt::get(Ty: CGF.Int8Ty, V: bitsToKeep);
2684 auto *NewValue = CGF.Builder.CreateAnd(LHS: Value, RHS: MaskValue);
2685
2686 CGF.Builder.CreateStore(Val: NewValue, Addr: ElementAddr);
2687 ++StartByte;
2688 }
2689
2690 // Handle full bytes in the middle
2691 for (auto Offset = StartByte; Offset < EndByte; ++Offset) {
2692 auto *Index = ConstantInt::get(Ty: CGF.IntTy, V: Offset);
2693 auto *Element = CGF.Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: I8Ptr, IdxList: Index);
2694 Address ElementAddr(Element, CGF.Int8Ty,
2695 Src.getAlignment().alignmentAtOffset(
2696 offset: CharUnits::fromQuantity(Quantity: Offset)));
2697
2698 CGF.Builder.CreateStore(Val: Zero, Addr: ElementAddr);
2699 }
2700
2701 // Handle the end byte
2702 if (EndBit != 0) {
2703 auto *Index = ConstantInt::get(Ty: CGF.IntTy, V: EndByte);
2704 auto *Element = CGF.Builder.CreateGEP(Ty: CGF.Int8Ty, Ptr: I8Ptr, IdxList: Index);
2705 Address ElementAddr(Element, CGF.Int8Ty,
2706 Src.getAlignment().alignmentAtOffset(
2707 offset: CharUnits::fromQuantity(Quantity: EndByte)));
2708
2709 auto *Value = CGF.Builder.CreateLoad(Addr: ElementAddr);
2710
2711 uint8_t bitsToClear = (1 << EndBit) - 1;
2712 uint8_t bitsToKeep = ~bitsToClear;
2713 auto *MaskValue = ConstantInt::get(Ty: CGF.Int8Ty, V: bitsToKeep);
2714 auto *NewValue = CGF.Builder.CreateAnd(LHS: Value, RHS: MaskValue);
2715
2716 CGF.Builder.CreateStore(Val: NewValue, Addr: ElementAddr);
2717 }
2718 }
2719}
2720
2721RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
2722 const CallExpr *E,
2723 ReturnValueSlot ReturnValue) {
2724 assert(!getContext().BuiltinInfo.isImmediate(BuiltinID) &&
2725 "Should not codegen for consteval builtins");
2726
2727 const FunctionDecl *FD = GD.getDecl()->getAsFunction();
2728 // See if we can constant fold this builtin. If so, don't emit it at all.
2729 // TODO: Extend this handling to all builtin calls that we can constant-fold.
2730 Expr::EvalResult Result;
2731 if (E->isPRValue() && E->EvaluateAsRValue(Result, Ctx: CGM.getContext()) &&
2732 !Result.hasSideEffects()) {
2733 if (Result.Val.isInt())
2734 return RValue::get(V: llvm::ConstantInt::get(Context&: getLLVMContext(),
2735 V: Result.Val.getInt()));
2736 if (Result.Val.isFloat())
2737 return RValue::get(V: llvm::ConstantFP::get(Context&: getLLVMContext(),
2738 V: Result.Val.getFloat()));
2739 }
2740
2741 // If current long-double semantics is IEEE 128-bit, replace math builtins
2742 // of long-double with f128 equivalent.
2743 // TODO: This mutation should also be applied to other targets other than PPC,
2744 // after backend supports IEEE 128-bit style libcalls.
2745 if (getTarget().getTriple().isPPC64() &&
2746 &getTarget().getLongDoubleFormat() == &llvm::APFloat::IEEEquad())
2747 BuiltinID = mutateLongDoubleBuiltin(BuiltinID);
2748
2749 // If the builtin has been declared explicitly with an assembler label,
2750 // disable the specialized emitting below. Ideally we should communicate the
2751 // rename in IR, or at least avoid generating the intrinsic calls that are
2752 // likely to get lowered to the renamed library functions.
2753 const unsigned BuiltinIDIfNoAsmLabel =
2754 FD->hasAttr<AsmLabelAttr>() ? 0 : BuiltinID;
2755
2756 std::optional<bool> ErrnoOverriden;
2757 // ErrnoOverriden is true if math-errno is overriden via the
2758 // '#pragma float_control(precise, on)'. This pragma disables fast-math,
2759 // which implies math-errno.
2760 if (E->hasStoredFPFeatures()) {
2761 FPOptionsOverride OP = E->getFPFeatures();
2762 if (OP.hasMathErrnoOverride())
2763 ErrnoOverriden = OP.getMathErrnoOverride();
2764 }
2765 // True if 'attribute__((optnone))' is used. This attribute overrides
2766 // fast-math which implies math-errno.
2767 bool OptNone = CurFuncDecl && CurFuncDecl->hasAttr<OptimizeNoneAttr>();
2768
2769 bool IsOptimizationEnabled = CGM.getCodeGenOpts().OptimizationLevel != 0;
2770
2771 bool GenerateFPMathIntrinsics =
2772 getContext().BuiltinInfo.shouldGenerateFPMathIntrinsic(
2773 BuiltinID, Trip: CGM.getTriple(), ErrnoOverwritten: ErrnoOverriden, MathErrnoEnabled: getLangOpts().MathErrno,
2774 HasOptNoneAttr: OptNone, IsOptimizationEnabled);
2775
2776 if (GenerateFPMathIntrinsics) {
2777 switch (BuiltinIDIfNoAsmLabel) {
2778 case Builtin::BIacos:
2779 case Builtin::BIacosf:
2780 case Builtin::BIacosl:
2781 case Builtin::BI__builtin_acos:
2782 case Builtin::BI__builtin_acosf:
2783 case Builtin::BI__builtin_acosf16:
2784 case Builtin::BI__builtin_acosl:
2785 case Builtin::BI__builtin_acosf128:
2786 case Builtin::BI__builtin_elementwise_acos:
2787 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(
2788 CGF&: *this, E, IntrinsicID: Intrinsic::acos, ConstrainedIntrinsicID: Intrinsic::experimental_constrained_acos));
2789
2790 case Builtin::BIasin:
2791 case Builtin::BIasinf:
2792 case Builtin::BIasinl:
2793 case Builtin::BI__builtin_asin:
2794 case Builtin::BI__builtin_asinf:
2795 case Builtin::BI__builtin_asinf16:
2796 case Builtin::BI__builtin_asinl:
2797 case Builtin::BI__builtin_asinf128:
2798 case Builtin::BI__builtin_elementwise_asin:
2799 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(
2800 CGF&: *this, E, IntrinsicID: Intrinsic::asin, ConstrainedIntrinsicID: Intrinsic::experimental_constrained_asin));
2801
2802 case Builtin::BIatan:
2803 case Builtin::BIatanf:
2804 case Builtin::BIatanl:
2805 case Builtin::BI__builtin_atan:
2806 case Builtin::BI__builtin_atanf:
2807 case Builtin::BI__builtin_atanf16:
2808 case Builtin::BI__builtin_atanl:
2809 case Builtin::BI__builtin_atanf128:
2810 case Builtin::BI__builtin_elementwise_atan:
2811 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(
2812 CGF&: *this, E, IntrinsicID: Intrinsic::atan, ConstrainedIntrinsicID: Intrinsic::experimental_constrained_atan));
2813
2814 case Builtin::BIatan2:
2815 case Builtin::BIatan2f:
2816 case Builtin::BIatan2l:
2817 case Builtin::BI__builtin_atan2:
2818 case Builtin::BI__builtin_atan2f:
2819 case Builtin::BI__builtin_atan2f16:
2820 case Builtin::BI__builtin_atan2l:
2821 case Builtin::BI__builtin_atan2f128:
2822 case Builtin::BI__builtin_elementwise_atan2:
2823 return RValue::get(V: emitBinaryMaybeConstrainedFPBuiltin(
2824 CGF&: *this, E, IntrinsicID: Intrinsic::atan2,
2825 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_atan2));
2826
2827 case Builtin::BIceil:
2828 case Builtin::BIceilf:
2829 case Builtin::BIceill:
2830 case Builtin::BI__builtin_ceil:
2831 case Builtin::BI__builtin_ceilf:
2832 case Builtin::BI__builtin_ceilf16:
2833 case Builtin::BI__builtin_ceill:
2834 case Builtin::BI__builtin_ceilf128:
2835 case Builtin::BI__builtin_elementwise_ceil:
2836 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
2837 IntrinsicID: Intrinsic::ceil,
2838 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_ceil));
2839
2840 case Builtin::BIcopysign:
2841 case Builtin::BIcopysignf:
2842 case Builtin::BIcopysignl:
2843 case Builtin::BI__builtin_copysign:
2844 case Builtin::BI__builtin_copysignf:
2845 case Builtin::BI__builtin_copysignf16:
2846 case Builtin::BI__builtin_copysignl:
2847 case Builtin::BI__builtin_copysignf128:
2848 return RValue::get(
2849 V: emitBuiltinWithOneOverloadedType<2>(CGF&: *this, E, IntrinsicID: Intrinsic::copysign));
2850
2851 case Builtin::BIcos:
2852 case Builtin::BIcosf:
2853 case Builtin::BIcosl:
2854 case Builtin::BI__builtin_cos:
2855 case Builtin::BI__builtin_cosf:
2856 case Builtin::BI__builtin_cosf16:
2857 case Builtin::BI__builtin_cosl:
2858 case Builtin::BI__builtin_cosf128:
2859 case Builtin::BI__builtin_elementwise_cos:
2860 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
2861 IntrinsicID: Intrinsic::cos,
2862 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_cos));
2863
2864 case Builtin::BIcosh:
2865 case Builtin::BIcoshf:
2866 case Builtin::BIcoshl:
2867 case Builtin::BI__builtin_cosh:
2868 case Builtin::BI__builtin_coshf:
2869 case Builtin::BI__builtin_coshf16:
2870 case Builtin::BI__builtin_coshl:
2871 case Builtin::BI__builtin_coshf128:
2872 case Builtin::BI__builtin_elementwise_cosh:
2873 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(
2874 CGF&: *this, E, IntrinsicID: Intrinsic::cosh, ConstrainedIntrinsicID: Intrinsic::experimental_constrained_cosh));
2875
2876 case Builtin::BIexp:
2877 case Builtin::BIexpf:
2878 case Builtin::BIexpl:
2879 case Builtin::BI__builtin_exp:
2880 case Builtin::BI__builtin_expf:
2881 case Builtin::BI__builtin_expf16:
2882 case Builtin::BI__builtin_expl:
2883 case Builtin::BI__builtin_expf128:
2884 case Builtin::BI__builtin_elementwise_exp:
2885 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
2886 IntrinsicID: Intrinsic::exp,
2887 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_exp));
2888
2889 case Builtin::BIexp2:
2890 case Builtin::BIexp2f:
2891 case Builtin::BIexp2l:
2892 case Builtin::BI__builtin_exp2:
2893 case Builtin::BI__builtin_exp2f:
2894 case Builtin::BI__builtin_exp2f16:
2895 case Builtin::BI__builtin_exp2l:
2896 case Builtin::BI__builtin_exp2f128:
2897 case Builtin::BI__builtin_elementwise_exp2:
2898 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
2899 IntrinsicID: Intrinsic::exp2,
2900 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_exp2));
2901 case Builtin::BI__builtin_exp10:
2902 case Builtin::BI__builtin_exp10f:
2903 case Builtin::BI__builtin_exp10f16:
2904 case Builtin::BI__builtin_exp10l:
2905 case Builtin::BI__builtin_exp10f128:
2906 case Builtin::BI__builtin_elementwise_exp10: {
2907 // TODO: strictfp support
2908 if (Builder.getIsFPConstrained())
2909 break;
2910 return RValue::get(
2911 V: emitBuiltinWithOneOverloadedType<1>(CGF&: *this, E, IntrinsicID: Intrinsic::exp10));
2912 }
2913 case Builtin::BIfabs:
2914 case Builtin::BIfabsf:
2915 case Builtin::BIfabsl:
2916 case Builtin::BI__builtin_fabs:
2917 case Builtin::BI__builtin_fabsf:
2918 case Builtin::BI__builtin_fabsf16:
2919 case Builtin::BI__builtin_fabsl:
2920 case Builtin::BI__builtin_fabsf128:
2921 return RValue::get(
2922 V: emitBuiltinWithOneOverloadedType<1>(CGF&: *this, E, IntrinsicID: Intrinsic::fabs));
2923
2924 case Builtin::BIfloor:
2925 case Builtin::BIfloorf:
2926 case Builtin::BIfloorl:
2927 case Builtin::BI__builtin_floor:
2928 case Builtin::BI__builtin_floorf:
2929 case Builtin::BI__builtin_floorf16:
2930 case Builtin::BI__builtin_floorl:
2931 case Builtin::BI__builtin_floorf128:
2932 case Builtin::BI__builtin_elementwise_floor:
2933 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
2934 IntrinsicID: Intrinsic::floor,
2935 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_floor));
2936
2937 case Builtin::BIfma:
2938 case Builtin::BIfmaf:
2939 case Builtin::BIfmal:
2940 case Builtin::BI__builtin_fma:
2941 case Builtin::BI__builtin_fmaf:
2942 case Builtin::BI__builtin_fmaf16:
2943 case Builtin::BI__builtin_fmal:
2944 case Builtin::BI__builtin_fmaf128:
2945 case Builtin::BI__builtin_elementwise_fma:
2946 return RValue::get(V: emitTernaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
2947 IntrinsicID: Intrinsic::fma,
2948 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_fma));
2949
2950 case Builtin::BIfmax:
2951 case Builtin::BIfmaxf:
2952 case Builtin::BIfmaxl:
2953 case Builtin::BI__builtin_fmax:
2954 case Builtin::BI__builtin_fmaxf:
2955 case Builtin::BI__builtin_fmaxf16:
2956 case Builtin::BI__builtin_fmaxl:
2957 case Builtin::BI__builtin_fmaxf128: {
2958 IRBuilder<>::FastMathFlagGuard FMFGuard(Builder);
2959 Builder.getFastMathFlags().setNoSignedZeros();
2960 return RValue::get(V: emitBinaryMaybeConstrainedFPBuiltin(
2961 CGF&: *this, E, IntrinsicID: Intrinsic::maxnum,
2962 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_maxnum));
2963 }
2964
2965 case Builtin::BIfmin:
2966 case Builtin::BIfminf:
2967 case Builtin::BIfminl:
2968 case Builtin::BI__builtin_fmin:
2969 case Builtin::BI__builtin_fminf:
2970 case Builtin::BI__builtin_fminf16:
2971 case Builtin::BI__builtin_fminl:
2972 case Builtin::BI__builtin_fminf128: {
2973 IRBuilder<>::FastMathFlagGuard FMFGuard(Builder);
2974 Builder.getFastMathFlags().setNoSignedZeros();
2975 return RValue::get(V: emitBinaryMaybeConstrainedFPBuiltin(
2976 CGF&: *this, E, IntrinsicID: Intrinsic::minnum,
2977 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_minnum));
2978 }
2979
2980 case Builtin::BIfmaximum_num:
2981 case Builtin::BIfmaximum_numf:
2982 case Builtin::BIfmaximum_numl:
2983 case Builtin::BI__builtin_fmaximum_num:
2984 case Builtin::BI__builtin_fmaximum_numf:
2985 case Builtin::BI__builtin_fmaximum_numf16:
2986 case Builtin::BI__builtin_fmaximum_numl:
2987 case Builtin::BI__builtin_fmaximum_numf128:
2988 return RValue::get(
2989 V: emitBuiltinWithOneOverloadedType<2>(CGF&: *this, E, IntrinsicID: Intrinsic::maximumnum));
2990
2991 case Builtin::BIfminimum_num:
2992 case Builtin::BIfminimum_numf:
2993 case Builtin::BIfminimum_numl:
2994 case Builtin::BI__builtin_fminimum_num:
2995 case Builtin::BI__builtin_fminimum_numf:
2996 case Builtin::BI__builtin_fminimum_numf16:
2997 case Builtin::BI__builtin_fminimum_numl:
2998 case Builtin::BI__builtin_fminimum_numf128:
2999 return RValue::get(
3000 V: emitBuiltinWithOneOverloadedType<2>(CGF&: *this, E, IntrinsicID: Intrinsic::minimumnum));
3001
3002 // fmod() is a special-case. It maps to the frem instruction rather than an
3003 // LLVM intrinsic.
3004 case Builtin::BIfmod:
3005 case Builtin::BIfmodf:
3006 case Builtin::BIfmodl:
3007 case Builtin::BI__builtin_fmod:
3008 case Builtin::BI__builtin_fmodf:
3009 case Builtin::BI__builtin_fmodf16:
3010 case Builtin::BI__builtin_fmodl:
3011 case Builtin::BI__builtin_fmodf128:
3012 case Builtin::BI__builtin_elementwise_fmod: {
3013 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
3014 Value *Arg1 = EmitScalarExpr(E: E->getArg(Arg: 0));
3015 Value *Arg2 = EmitScalarExpr(E: E->getArg(Arg: 1));
3016 if (Builder.getIsFPConstrained()) {
3017 Function *F = CGM.getIntrinsic(IID: Intrinsic::experimental_constrained_frem,
3018 Tys: Arg1->getType());
3019 return RValue::get(V: Builder.CreateConstrainedFPCall(Callee: F, Args: {Arg1, Arg2}));
3020 } else {
3021 return RValue::get(V: Builder.CreateFRem(L: Arg1, R: Arg2, Name: "fmod"));
3022 }
3023 }
3024
3025 case Builtin::BIlog:
3026 case Builtin::BIlogf:
3027 case Builtin::BIlogl:
3028 case Builtin::BI__builtin_log:
3029 case Builtin::BI__builtin_logf:
3030 case Builtin::BI__builtin_logf16:
3031 case Builtin::BI__builtin_logl:
3032 case Builtin::BI__builtin_logf128:
3033 case Builtin::BI__builtin_elementwise_log:
3034 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
3035 IntrinsicID: Intrinsic::log,
3036 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_log));
3037
3038 case Builtin::BIlog10:
3039 case Builtin::BIlog10f:
3040 case Builtin::BIlog10l:
3041 case Builtin::BI__builtin_log10:
3042 case Builtin::BI__builtin_log10f:
3043 case Builtin::BI__builtin_log10f16:
3044 case Builtin::BI__builtin_log10l:
3045 case Builtin::BI__builtin_log10f128:
3046 case Builtin::BI__builtin_elementwise_log10:
3047 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
3048 IntrinsicID: Intrinsic::log10,
3049 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_log10));
3050
3051 case Builtin::BIlog2:
3052 case Builtin::BIlog2f:
3053 case Builtin::BIlog2l:
3054 case Builtin::BI__builtin_log2:
3055 case Builtin::BI__builtin_log2f:
3056 case Builtin::BI__builtin_log2f16:
3057 case Builtin::BI__builtin_log2l:
3058 case Builtin::BI__builtin_log2f128:
3059 case Builtin::BI__builtin_elementwise_log2:
3060 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
3061 IntrinsicID: Intrinsic::log2,
3062 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_log2));
3063
3064 case Builtin::BInearbyint:
3065 case Builtin::BInearbyintf:
3066 case Builtin::BInearbyintl:
3067 case Builtin::BI__builtin_nearbyint:
3068 case Builtin::BI__builtin_nearbyintf:
3069 case Builtin::BI__builtin_nearbyintl:
3070 case Builtin::BI__builtin_nearbyintf128:
3071 case Builtin::BI__builtin_elementwise_nearbyint:
3072 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
3073 IntrinsicID: Intrinsic::nearbyint,
3074 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_nearbyint));
3075
3076 case Builtin::BIpow:
3077 case Builtin::BIpowf:
3078 case Builtin::BIpowl:
3079 case Builtin::BI__builtin_pow:
3080 case Builtin::BI__builtin_powf:
3081 case Builtin::BI__builtin_powf16:
3082 case Builtin::BI__builtin_powl:
3083 case Builtin::BI__builtin_powf128:
3084 case Builtin::BI__builtin_elementwise_pow:
3085 return RValue::get(V: emitBinaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
3086 IntrinsicID: Intrinsic::pow,
3087 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_pow));
3088
3089 case Builtin::BIrint:
3090 case Builtin::BIrintf:
3091 case Builtin::BIrintl:
3092 case Builtin::BI__builtin_rint:
3093 case Builtin::BI__builtin_rintf:
3094 case Builtin::BI__builtin_rintf16:
3095 case Builtin::BI__builtin_rintl:
3096 case Builtin::BI__builtin_rintf128:
3097 case Builtin::BI__builtin_elementwise_rint:
3098 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
3099 IntrinsicID: Intrinsic::rint,
3100 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_rint));
3101
3102 case Builtin::BIround:
3103 case Builtin::BIroundf:
3104 case Builtin::BIroundl:
3105 case Builtin::BI__builtin_round:
3106 case Builtin::BI__builtin_roundf:
3107 case Builtin::BI__builtin_roundf16:
3108 case Builtin::BI__builtin_roundl:
3109 case Builtin::BI__builtin_roundf128:
3110 case Builtin::BI__builtin_elementwise_round:
3111 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
3112 IntrinsicID: Intrinsic::round,
3113 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_round));
3114
3115 case Builtin::BIroundeven:
3116 case Builtin::BIroundevenf:
3117 case Builtin::BIroundevenl:
3118 case Builtin::BI__builtin_roundeven:
3119 case Builtin::BI__builtin_roundevenf:
3120 case Builtin::BI__builtin_roundevenf16:
3121 case Builtin::BI__builtin_roundevenl:
3122 case Builtin::BI__builtin_roundevenf128:
3123 case Builtin::BI__builtin_elementwise_roundeven:
3124 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
3125 IntrinsicID: Intrinsic::roundeven,
3126 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_roundeven));
3127
3128 case Builtin::BIsin:
3129 case Builtin::BIsinf:
3130 case Builtin::BIsinl:
3131 case Builtin::BI__builtin_sin:
3132 case Builtin::BI__builtin_sinf:
3133 case Builtin::BI__builtin_sinf16:
3134 case Builtin::BI__builtin_sinl:
3135 case Builtin::BI__builtin_sinf128:
3136 case Builtin::BI__builtin_elementwise_sin:
3137 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
3138 IntrinsicID: Intrinsic::sin,
3139 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_sin));
3140
3141 case Builtin::BIsinh:
3142 case Builtin::BIsinhf:
3143 case Builtin::BIsinhl:
3144 case Builtin::BI__builtin_sinh:
3145 case Builtin::BI__builtin_sinhf:
3146 case Builtin::BI__builtin_sinhf16:
3147 case Builtin::BI__builtin_sinhl:
3148 case Builtin::BI__builtin_sinhf128:
3149 case Builtin::BI__builtin_elementwise_sinh:
3150 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(
3151 CGF&: *this, E, IntrinsicID: Intrinsic::sinh, ConstrainedIntrinsicID: Intrinsic::experimental_constrained_sinh));
3152
3153 case Builtin::BI__builtin_sincospi:
3154 case Builtin::BI__builtin_sincospif:
3155 case Builtin::BI__builtin_sincospil:
3156 if (Builder.getIsFPConstrained())
3157 break; // TODO: Emit constrained sincospi intrinsic once one exists.
3158 emitSincosBuiltin(CGF&: *this, E, IntrinsicID: Intrinsic::sincospi);
3159 return RValue::get(V: nullptr);
3160
3161 case Builtin::BIsincos:
3162 case Builtin::BIsincosf:
3163 case Builtin::BIsincosl:
3164 case Builtin::BI__builtin_sincos:
3165 case Builtin::BI__builtin_sincosf:
3166 case Builtin::BI__builtin_sincosf16:
3167 case Builtin::BI__builtin_sincosl:
3168 case Builtin::BI__builtin_sincosf128:
3169 if (Builder.getIsFPConstrained())
3170 break; // TODO: Emit constrained sincos intrinsic once one exists.
3171 emitSincosBuiltin(CGF&: *this, E, IntrinsicID: Intrinsic::sincos);
3172 return RValue::get(V: nullptr);
3173
3174 case Builtin::BIsqrt:
3175 case Builtin::BIsqrtf:
3176 case Builtin::BIsqrtl:
3177 case Builtin::BI__builtin_sqrt:
3178 case Builtin::BI__builtin_sqrtf:
3179 case Builtin::BI__builtin_sqrtf16:
3180 case Builtin::BI__builtin_sqrtl:
3181 case Builtin::BI__builtin_sqrtf128:
3182 case Builtin::BI__builtin_elementwise_sqrt: {
3183 llvm::Value *Call = emitUnaryMaybeConstrainedFPBuiltin(
3184 CGF&: *this, E, IntrinsicID: Intrinsic::sqrt, ConstrainedIntrinsicID: Intrinsic::experimental_constrained_sqrt);
3185 SetSqrtFPAccuracy(Call);
3186 return RValue::get(V: Call);
3187 }
3188
3189 case Builtin::BItan:
3190 case Builtin::BItanf:
3191 case Builtin::BItanl:
3192 case Builtin::BI__builtin_tan:
3193 case Builtin::BI__builtin_tanf:
3194 case Builtin::BI__builtin_tanf16:
3195 case Builtin::BI__builtin_tanl:
3196 case Builtin::BI__builtin_tanf128:
3197 case Builtin::BI__builtin_elementwise_tan:
3198 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(
3199 CGF&: *this, E, IntrinsicID: Intrinsic::tan, ConstrainedIntrinsicID: Intrinsic::experimental_constrained_tan));
3200
3201 case Builtin::BItanh:
3202 case Builtin::BItanhf:
3203 case Builtin::BItanhl:
3204 case Builtin::BI__builtin_tanh:
3205 case Builtin::BI__builtin_tanhf:
3206 case Builtin::BI__builtin_tanhf16:
3207 case Builtin::BI__builtin_tanhl:
3208 case Builtin::BI__builtin_tanhf128:
3209 case Builtin::BI__builtin_elementwise_tanh:
3210 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(
3211 CGF&: *this, E, IntrinsicID: Intrinsic::tanh, ConstrainedIntrinsicID: Intrinsic::experimental_constrained_tanh));
3212
3213 case Builtin::BItrunc:
3214 case Builtin::BItruncf:
3215 case Builtin::BItruncl:
3216 case Builtin::BI__builtin_trunc:
3217 case Builtin::BI__builtin_truncf:
3218 case Builtin::BI__builtin_truncf16:
3219 case Builtin::BI__builtin_truncl:
3220 case Builtin::BI__builtin_truncf128:
3221 case Builtin::BI__builtin_elementwise_trunc:
3222 return RValue::get(V: emitUnaryMaybeConstrainedFPBuiltin(CGF&: *this, E,
3223 IntrinsicID: Intrinsic::trunc,
3224 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_trunc));
3225
3226 case Builtin::BIlround:
3227 case Builtin::BIlroundf:
3228 case Builtin::BIlroundl:
3229 case Builtin::BI__builtin_lround:
3230 case Builtin::BI__builtin_lroundf:
3231 case Builtin::BI__builtin_lroundl:
3232 case Builtin::BI__builtin_lroundf128:
3233 return RValue::get(V: emitMaybeConstrainedFPToIntRoundBuiltin(
3234 CGF&: *this, E, IntrinsicID: Intrinsic::lround,
3235 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_lround));
3236
3237 case Builtin::BIllround:
3238 case Builtin::BIllroundf:
3239 case Builtin::BIllroundl:
3240 case Builtin::BI__builtin_llround:
3241 case Builtin::BI__builtin_llroundf:
3242 case Builtin::BI__builtin_llroundl:
3243 case Builtin::BI__builtin_llroundf128:
3244 return RValue::get(V: emitMaybeConstrainedFPToIntRoundBuiltin(
3245 CGF&: *this, E, IntrinsicID: Intrinsic::llround,
3246 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_llround));
3247
3248 case Builtin::BIlrint:
3249 case Builtin::BIlrintf:
3250 case Builtin::BIlrintl:
3251 case Builtin::BI__builtin_lrint:
3252 case Builtin::BI__builtin_lrintf:
3253 case Builtin::BI__builtin_lrintl:
3254 case Builtin::BI__builtin_lrintf128:
3255 return RValue::get(V: emitMaybeConstrainedFPToIntRoundBuiltin(
3256 CGF&: *this, E, IntrinsicID: Intrinsic::lrint,
3257 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_lrint));
3258
3259 case Builtin::BIllrint:
3260 case Builtin::BIllrintf:
3261 case Builtin::BIllrintl:
3262 case Builtin::BI__builtin_llrint:
3263 case Builtin::BI__builtin_llrintf:
3264 case Builtin::BI__builtin_llrintl:
3265 case Builtin::BI__builtin_llrintf128:
3266 return RValue::get(V: emitMaybeConstrainedFPToIntRoundBuiltin(
3267 CGF&: *this, E, IntrinsicID: Intrinsic::llrint,
3268 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_llrint));
3269 case Builtin::BI__builtin_ldexp:
3270 case Builtin::BI__builtin_ldexpf:
3271 case Builtin::BI__builtin_ldexpl:
3272 case Builtin::BI__builtin_ldexpf16:
3273 case Builtin::BI__builtin_ldexpf128:
3274 case Builtin::BI__builtin_elementwise_ldexp:
3275 return RValue::get(V: emitBinaryExpMaybeConstrainedFPBuiltin(
3276 CGF&: *this, E, IntrinsicID: Intrinsic::ldexp,
3277 ConstrainedIntrinsicID: Intrinsic::experimental_constrained_ldexp));
3278 default:
3279 break;
3280 }
3281 }
3282
3283 // Check NonnullAttribute/NullabilityArg and Alignment.
3284 auto EmitArgCheck = [&](TypeCheckKind Kind, Address A, const Expr *Arg,
3285 unsigned ParmNum) {
3286 Value *Val = A.emitRawPointer(CGF&: *this);
3287 EmitNonNullArgCheck(RV: RValue::get(V: Val), ArgType: Arg->getType(), ArgLoc: Arg->getExprLoc(), AC: FD,
3288 ParmNum);
3289
3290 if (SanOpts.has(K: SanitizerKind::Alignment)) {
3291 SanitizerSet SkippedChecks;
3292 SkippedChecks.set(SanitizerKind::All);
3293 SkippedChecks.clear(K: SanitizerKind::Alignment);
3294 SourceLocation Loc = Arg->getExprLoc();
3295 // Strip an implicit cast.
3296 if (auto *CE = dyn_cast<ImplicitCastExpr>(Val: Arg))
3297 if (CE->getCastKind() == CK_BitCast)
3298 Arg = CE->getSubExpr();
3299 EmitTypeCheck(TCK: Kind, Loc, V: Val, Type: Arg->getType(), Alignment: A.getAlignment(),
3300 SkippedChecks);
3301 }
3302 };
3303
3304 switch (BuiltinIDIfNoAsmLabel) {
3305 default: break;
3306 case Builtin::BI__builtin___CFStringMakeConstantString:
3307 case Builtin::BI__builtin___NSStringMakeConstantString:
3308 return RValue::get(V: ConstantEmitter(*this).emitAbstract(E, T: E->getType()));
3309 case Builtin::BI__builtin_stdarg_start:
3310 case Builtin::BI__builtin_va_start:
3311 case Builtin::BI__va_start:
3312 case Builtin::BI__builtin_c23_va_start:
3313 case Builtin::BI__builtin_va_end:
3314 EmitVAStartEnd(ArgValue: BuiltinID == Builtin::BI__va_start
3315 ? EmitScalarExpr(E: E->getArg(Arg: 0))
3316 : EmitVAListRef(E: E->getArg(Arg: 0)).emitRawPointer(CGF&: *this),
3317 IsStart: BuiltinID != Builtin::BI__builtin_va_end);
3318 return RValue::get(V: nullptr);
3319 case Builtin::BI__builtin_va_copy: {
3320 Value *DstPtr = EmitVAListRef(E: E->getArg(Arg: 0)).emitRawPointer(CGF&: *this);
3321 Value *SrcPtr = EmitVAListRef(E: E->getArg(Arg: 1)).emitRawPointer(CGF&: *this);
3322 Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Intrinsic::vacopy, Tys: {DstPtr->getType()}),
3323 Args: {DstPtr, SrcPtr});
3324 return RValue::get(V: nullptr);
3325 }
3326 case Builtin::BIabs:
3327 case Builtin::BIlabs:
3328 case Builtin::BIllabs:
3329 case Builtin::BI__builtin_abs:
3330 case Builtin::BI__builtin_labs:
3331 case Builtin::BI__builtin_llabs: {
3332 bool SanitizeOverflow = SanOpts.has(K: SanitizerKind::SignedIntegerOverflow);
3333
3334 Value *Result;
3335 switch (getLangOpts().getSignedOverflowBehavior()) {
3336 case LangOptions::SOB_Defined:
3337 Result = EmitAbs(CGF&: *this, ArgValue: EmitScalarExpr(E: E->getArg(Arg: 0)), HasNSW: false);
3338 break;
3339 case LangOptions::SOB_Undefined:
3340 if (!SanitizeOverflow) {
3341 Result = EmitAbs(CGF&: *this, ArgValue: EmitScalarExpr(E: E->getArg(Arg: 0)), HasNSW: true);
3342 break;
3343 }
3344 [[fallthrough]];
3345 case LangOptions::SOB_Trapping:
3346 // TODO: Somehow handle the corner case when the address of abs is taken.
3347 Result = EmitOverflowCheckedAbs(CGF&: *this, E, SanitizeOverflow);
3348 break;
3349 }
3350 return RValue::get(V: Result);
3351 }
3352 case Builtin::BI__builtin_complex: {
3353 Value *Real = EmitScalarExpr(E: E->getArg(Arg: 0));
3354 Value *Imag = EmitScalarExpr(E: E->getArg(Arg: 1));
3355 return RValue::getComplex(C: {Real, Imag});
3356 }
3357 case Builtin::BI__builtin_conj:
3358 case Builtin::BI__builtin_conjf:
3359 case Builtin::BI__builtin_conjl:
3360 case Builtin::BIconj:
3361 case Builtin::BIconjf:
3362 case Builtin::BIconjl: {
3363 ComplexPairTy ComplexVal = EmitComplexExpr(E: E->getArg(Arg: 0));
3364 Value *Real = ComplexVal.first;
3365 Value *Imag = ComplexVal.second;
3366 Imag = Builder.CreateFNeg(V: Imag, Name: "neg");
3367 return RValue::getComplex(C: std::make_pair(x&: Real, y&: Imag));
3368 }
3369 case Builtin::BI__builtin_creal:
3370 case Builtin::BI__builtin_crealf:
3371 case Builtin::BI__builtin_creall:
3372 case Builtin::BIcreal:
3373 case Builtin::BIcrealf:
3374 case Builtin::BIcreall: {
3375 ComplexPairTy ComplexVal = EmitComplexExpr(E: E->getArg(Arg: 0));
3376 return RValue::get(V: ComplexVal.first);
3377 }
3378
3379 case Builtin::BI__builtin_preserve_access_index: {
3380 // Only enabled preserved access index region when debuginfo
3381 // is available as debuginfo is needed to preserve user-level
3382 // access pattern.
3383 if (!getDebugInfo()) {
3384 CGM.Error(loc: E->getExprLoc(), error: "using builtin_preserve_access_index() without -g");
3385 return RValue::get(V: EmitScalarExpr(E: E->getArg(Arg: 0)));
3386 }
3387
3388 // Nested builtin_preserve_access_index() not supported
3389 if (IsInPreservedAIRegion) {
3390 CGM.Error(loc: E->getExprLoc(), error: "nested builtin_preserve_access_index() not supported");
3391 return RValue::get(V: EmitScalarExpr(E: E->getArg(Arg: 0)));
3392 }
3393
3394 IsInPreservedAIRegion = true;
3395 Value *Res = EmitScalarExpr(E: E->getArg(Arg: 0));
3396 IsInPreservedAIRegion = false;
3397 return RValue::get(V: Res);
3398 }
3399
3400 case Builtin::BI__builtin_cimag:
3401 case Builtin::BI__builtin_cimagf:
3402 case Builtin::BI__builtin_cimagl:
3403 case Builtin::BIcimag:
3404 case Builtin::BIcimagf:
3405 case Builtin::BIcimagl: {
3406 ComplexPairTy ComplexVal = EmitComplexExpr(E: E->getArg(Arg: 0));
3407 return RValue::get(V: ComplexVal.second);
3408 }
3409
3410 case Builtin::BI__builtin_clrsb:
3411 case Builtin::BI__builtin_clrsbl:
3412 case Builtin::BI__builtin_clrsbll: {
3413 // clrsb(x) -> clz(x < 0 ? ~x : x) - 1 or
3414 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
3415
3416 llvm::Type *ArgType = ArgValue->getType();
3417 Function *F = CGM.getIntrinsic(IID: Intrinsic::ctlz, Tys: ArgType);
3418
3419 llvm::Type *ResultType = ConvertType(T: E->getType());
3420 Value *Zero = llvm::Constant::getNullValue(Ty: ArgType);
3421 Value *IsNeg = Builder.CreateICmpSLT(LHS: ArgValue, RHS: Zero, Name: "isneg");
3422 Value *Inverse = Builder.CreateNot(V: ArgValue, Name: "not");
3423 Value *Tmp = Builder.CreateSelect(C: IsNeg, True: Inverse, False: ArgValue);
3424 Value *Ctlz = Builder.CreateCall(Callee: F, Args: {Tmp, Builder.getFalse()});
3425 Value *Result =
3426 Builder.CreateNUWSub(LHS: Ctlz, RHS: llvm::ConstantInt::get(Ty: ArgType, V: 1));
3427 Result = Builder.CreateIntCast(V: Result, DestTy: ResultType, /*isSigned*/true,
3428 Name: "cast");
3429 return RValue::get(V: Result);
3430 }
3431 case Builtin::BI__builtin_ctzs:
3432 case Builtin::BI__builtin_ctz:
3433 case Builtin::BI__builtin_ctzl:
3434 case Builtin::BI__builtin_ctzll:
3435 case Builtin::BI__builtin_ctzg:
3436 case Builtin::BI__builtin_elementwise_ctzg: {
3437 bool HasFallback =
3438 (BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_ctzg ||
3439 BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_elementwise_ctzg) &&
3440 E->getNumArgs() > 1;
3441
3442 Value *ArgValue =
3443 HasFallback ? EmitBitCountExpr(CGF&: *this, E: E->getArg(Arg: 0))
3444 : EmitCheckedArgForBuiltin(E: E->getArg(Arg: 0), Kind: BCK_CTZPassedZero);
3445
3446 llvm::Type *ArgType = ArgValue->getType();
3447 Function *F = CGM.getIntrinsic(IID: Intrinsic::cttz, Tys: ArgType);
3448
3449 llvm::Type *ResultType = ConvertType(T: E->getType());
3450 // The elementwise builtins always exhibit zero-is-undef behaviour
3451 Value *ZeroUndef = Builder.getInt1(
3452 V: HasFallback || getTarget().isCLZForZeroUndef() ||
3453 BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_elementwise_ctzg);
3454 Value *Result = Builder.CreateCall(Callee: F, Args: {ArgValue, ZeroUndef});
3455 if (Result->getType() != ResultType)
3456 Result =
3457 Builder.CreateIntCast(V: Result, DestTy: ResultType, /*isSigned*/ false, Name: "cast");
3458 if (!HasFallback)
3459 return RValue::get(V: Result);
3460
3461 Value *Zero = Constant::getNullValue(Ty: ArgType);
3462 Value *IsZero = Builder.CreateICmpEQ(LHS: ArgValue, RHS: Zero, Name: "iszero");
3463 Value *FallbackValue = EmitScalarExpr(E: E->getArg(Arg: 1));
3464 Value *ResultOrFallback =
3465 Builder.CreateSelect(C: IsZero, True: FallbackValue, False: Result, Name: "ctzg");
3466 return RValue::get(V: ResultOrFallback);
3467 }
3468 case Builtin::BI__builtin_clzs:
3469 case Builtin::BI__builtin_clz:
3470 case Builtin::BI__builtin_clzl:
3471 case Builtin::BI__builtin_clzll:
3472 case Builtin::BI__builtin_clzg:
3473 case Builtin::BI__builtin_elementwise_clzg: {
3474 bool HasFallback =
3475 (BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_clzg ||
3476 BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_elementwise_clzg) &&
3477 E->getNumArgs() > 1;
3478
3479 Value *ArgValue =
3480 HasFallback ? EmitBitCountExpr(CGF&: *this, E: E->getArg(Arg: 0))
3481 : EmitCheckedArgForBuiltin(E: E->getArg(Arg: 0), Kind: BCK_CLZPassedZero);
3482
3483 llvm::Type *ArgType = ArgValue->getType();
3484 Function *F = CGM.getIntrinsic(IID: Intrinsic::ctlz, Tys: ArgType);
3485
3486 llvm::Type *ResultType = ConvertType(T: E->getType());
3487 // The elementwise builtins always exhibit zero-is-undef behaviour
3488 Value *ZeroUndef = Builder.getInt1(
3489 V: HasFallback || getTarget().isCLZForZeroUndef() ||
3490 BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_elementwise_clzg);
3491 Value *Result = Builder.CreateCall(Callee: F, Args: {ArgValue, ZeroUndef});
3492 if (Result->getType() != ResultType)
3493 Result =
3494 Builder.CreateIntCast(V: Result, DestTy: ResultType, /*isSigned*/ false, Name: "cast");
3495 if (!HasFallback)
3496 return RValue::get(V: Result);
3497
3498 Value *Zero = Constant::getNullValue(Ty: ArgType);
3499 Value *IsZero = Builder.CreateICmpEQ(LHS: ArgValue, RHS: Zero, Name: "iszero");
3500 Value *FallbackValue = EmitScalarExpr(E: E->getArg(Arg: 1));
3501 Value *ResultOrFallback =
3502 Builder.CreateSelect(C: IsZero, True: FallbackValue, False: Result, Name: "clzg");
3503 return RValue::get(V: ResultOrFallback);
3504 }
3505 case Builtin::BI__builtin_ffs:
3506 case Builtin::BI__builtin_ffsl:
3507 case Builtin::BI__builtin_ffsll: {
3508 // ffs(x) -> x ? cttz(x) + 1 : 0
3509 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
3510
3511 llvm::Type *ArgType = ArgValue->getType();
3512 Function *F = CGM.getIntrinsic(IID: Intrinsic::cttz, Tys: ArgType);
3513
3514 llvm::Type *ResultType = ConvertType(T: E->getType());
3515 Value *Tmp =
3516 Builder.CreateAdd(LHS: Builder.CreateCall(Callee: F, Args: {ArgValue, Builder.getTrue()}),
3517 RHS: llvm::ConstantInt::get(Ty: ArgType, V: 1));
3518 Value *Zero = llvm::Constant::getNullValue(Ty: ArgType);
3519 Value *IsZero = Builder.CreateICmpEQ(LHS: ArgValue, RHS: Zero, Name: "iszero");
3520 Value *Result = Builder.CreateSelect(C: IsZero, True: Zero, False: Tmp, Name: "ffs");
3521 if (Result->getType() != ResultType)
3522 Result = Builder.CreateIntCast(V: Result, DestTy: ResultType, /*isSigned*/true,
3523 Name: "cast");
3524 return RValue::get(V: Result);
3525 }
3526 case Builtin::BI__builtin_parity:
3527 case Builtin::BI__builtin_parityl:
3528 case Builtin::BI__builtin_parityll: {
3529 // parity(x) -> ctpop(x) & 1
3530 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
3531
3532 llvm::Type *ArgType = ArgValue->getType();
3533 Function *F = CGM.getIntrinsic(IID: Intrinsic::ctpop, Tys: ArgType);
3534
3535 llvm::Type *ResultType = ConvertType(T: E->getType());
3536 Value *Tmp = Builder.CreateCall(Callee: F, Args: ArgValue);
3537 Value *Result = Builder.CreateAnd(LHS: Tmp, RHS: llvm::ConstantInt::get(Ty: ArgType, V: 1));
3538 if (Result->getType() != ResultType)
3539 Result = Builder.CreateIntCast(V: Result, DestTy: ResultType, /*isSigned*/true,
3540 Name: "cast");
3541 return RValue::get(V: Result);
3542 }
3543 case Builtin::BI__lzcnt16:
3544 case Builtin::BI__lzcnt:
3545 case Builtin::BI__lzcnt64: {
3546 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
3547
3548 llvm::Type *ArgType = ArgValue->getType();
3549 Function *F = CGM.getIntrinsic(IID: Intrinsic::ctlz, Tys: ArgType);
3550
3551 llvm::Type *ResultType = ConvertType(T: E->getType());
3552 Value *Result = Builder.CreateCall(Callee: F, Args: {ArgValue, Builder.getFalse()});
3553 if (Result->getType() != ResultType)
3554 Result = Builder.CreateIntCast(V: Result, DestTy: ResultType, /*isSigned*/true,
3555 Name: "cast");
3556 return RValue::get(V: Result);
3557 }
3558 case Builtin::BI__popcnt16:
3559 case Builtin::BI__popcnt:
3560 case Builtin::BI__popcnt64:
3561 case Builtin::BI__builtin_popcount:
3562 case Builtin::BI__builtin_popcountl:
3563 case Builtin::BI__builtin_popcountll:
3564 case Builtin::BI__builtin_popcountg: {
3565 Value *ArgValue = EmitBitCountExpr(CGF&: *this, E: E->getArg(Arg: 0));
3566
3567 llvm::Type *ArgType = ArgValue->getType();
3568 Function *F = CGM.getIntrinsic(IID: Intrinsic::ctpop, Tys: ArgType);
3569
3570 llvm::Type *ResultType = ConvertType(T: E->getType());
3571 Value *Result = Builder.CreateCall(Callee: F, Args: ArgValue);
3572 if (Result->getType() != ResultType)
3573 Result =
3574 Builder.CreateIntCast(V: Result, DestTy: ResultType, /*isSigned*/ false, Name: "cast");
3575 return RValue::get(V: Result);
3576 }
3577 case Builtin::BI__builtin_unpredictable: {
3578 // Always return the argument of __builtin_unpredictable. LLVM does not
3579 // handle this builtin. Metadata for this builtin should be added directly
3580 // to instructions such as branches or switches that use it.
3581 return RValue::get(V: EmitScalarExpr(E: E->getArg(Arg: 0)));
3582 }
3583 case Builtin::BI__builtin_expect: {
3584 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
3585 llvm::Type *ArgType = ArgValue->getType();
3586
3587 Value *ExpectedValue = EmitScalarExpr(E: E->getArg(Arg: 1));
3588 // Don't generate llvm.expect on -O0 as the backend won't use it for
3589 // anything.
3590 // Note, we still IRGen ExpectedValue because it could have side-effects.
3591 if (CGM.getCodeGenOpts().OptimizationLevel == 0)
3592 return RValue::get(V: ArgValue);
3593
3594 Function *FnExpect = CGM.getIntrinsic(IID: Intrinsic::expect, Tys: ArgType);
3595 Value *Result =
3596 Builder.CreateCall(Callee: FnExpect, Args: {ArgValue, ExpectedValue}, Name: "expval");
3597 return RValue::get(V: Result);
3598 }
3599 case Builtin::BI__builtin_expect_with_probability: {
3600 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
3601 llvm::Type *ArgType = ArgValue->getType();
3602
3603 Value *ExpectedValue = EmitScalarExpr(E: E->getArg(Arg: 1));
3604 llvm::APFloat Probability(0.0);
3605 const Expr *ProbArg = E->getArg(Arg: 2);
3606 bool EvalSucceed = ProbArg->EvaluateAsFloat(Result&: Probability, Ctx: CGM.getContext());
3607 assert(EvalSucceed && "probability should be able to evaluate as float");
3608 (void)EvalSucceed;
3609 bool LoseInfo = false;
3610 Probability.convert(ToSemantics: llvm::APFloat::IEEEdouble(),
3611 RM: llvm::RoundingMode::Dynamic, losesInfo: &LoseInfo);
3612 llvm::Type *Ty = ConvertType(T: ProbArg->getType());
3613 Constant *Confidence = ConstantFP::get(Ty, V: Probability);
3614 // Don't generate llvm.expect.with.probability on -O0 as the backend
3615 // won't use it for anything.
3616 // Note, we still IRGen ExpectedValue because it could have side-effects.
3617 if (CGM.getCodeGenOpts().OptimizationLevel == 0)
3618 return RValue::get(V: ArgValue);
3619
3620 Function *FnExpect =
3621 CGM.getIntrinsic(IID: Intrinsic::expect_with_probability, Tys: ArgType);
3622 Value *Result = Builder.CreateCall(
3623 Callee: FnExpect, Args: {ArgValue, ExpectedValue, Confidence}, Name: "expval");
3624 return RValue::get(V: Result);
3625 }
3626 case Builtin::BI__builtin_assume_aligned: {
3627 const Expr *Ptr = E->getArg(Arg: 0);
3628 Value *PtrValue = EmitScalarExpr(E: Ptr);
3629 Value *OffsetValue =
3630 (E->getNumArgs() > 2) ? EmitScalarExpr(E: E->getArg(Arg: 2)) : nullptr;
3631
3632 Value *AlignmentValue = EmitScalarExpr(E: E->getArg(Arg: 1));
3633 ConstantInt *AlignmentCI = cast<ConstantInt>(Val: AlignmentValue);
3634 if (AlignmentCI->getValue().ugt(RHS: llvm::Value::MaximumAlignment))
3635 AlignmentCI = ConstantInt::get(Ty: AlignmentCI->getIntegerType(),
3636 V: llvm::Value::MaximumAlignment);
3637
3638 emitAlignmentAssumption(PtrValue, E: Ptr,
3639 /*The expr loc is sufficient.*/ AssumptionLoc: SourceLocation(),
3640 Alignment: AlignmentCI, OffsetValue);
3641 return RValue::get(V: PtrValue);
3642 }
3643 case Builtin::BI__builtin_assume_dereferenceable: {
3644 const Expr *Ptr = E->getArg(Arg: 0);
3645 const Expr *Size = E->getArg(Arg: 1);
3646 Value *PtrValue = EmitScalarExpr(E: Ptr);
3647 Value *SizeValue = EmitScalarExpr(E: Size);
3648 if (SizeValue->getType() != IntPtrTy)
3649 SizeValue =
3650 Builder.CreateIntCast(V: SizeValue, DestTy: IntPtrTy, isSigned: false, Name: "casted.size");
3651 Builder.CreateDereferenceableAssumption(PtrValue, SizeValue);
3652 return RValue::get(V: nullptr);
3653 }
3654 case Builtin::BI__assume:
3655 case Builtin::BI__builtin_assume: {
3656 if (E->getArg(Arg: 0)->HasSideEffects(Ctx: getContext()))
3657 return RValue::get(V: nullptr);
3658
3659 Value *ArgValue = EmitCheckedArgForAssume(E: E->getArg(Arg: 0));
3660 Function *FnAssume = CGM.getIntrinsic(IID: Intrinsic::assume);
3661 Builder.CreateCall(Callee: FnAssume, Args: ArgValue);
3662 return RValue::get(V: nullptr);
3663 }
3664 case Builtin::BI__builtin_assume_separate_storage: {
3665 const Expr *Arg0 = E->getArg(Arg: 0);
3666 const Expr *Arg1 = E->getArg(Arg: 1);
3667
3668 Value *Value0 = EmitScalarExpr(E: Arg0);
3669 Value *Value1 = EmitScalarExpr(E: Arg1);
3670
3671 Value *Values[] = {Value0, Value1};
3672 OperandBundleDefT<Value *> OBD("separate_storage", Values);
3673 Builder.CreateAssumption(OpBundles: {OBD});
3674 return RValue::get(V: nullptr);
3675 }
3676 case Builtin::BI__builtin_allow_runtime_check: {
3677 StringRef Kind =
3678 cast<StringLiteral>(Val: E->getArg(Arg: 0)->IgnoreParenCasts())->getString();
3679 LLVMContext &Ctx = CGM.getLLVMContext();
3680 llvm::Value *Allow = Builder.CreateCall(
3681 Callee: CGM.getIntrinsic(IID: Intrinsic::allow_runtime_check),
3682 Args: llvm::MetadataAsValue::get(Context&: Ctx, MD: llvm::MDString::get(Context&: Ctx, Str: Kind)));
3683 return RValue::get(V: Allow);
3684 }
3685 case Builtin::BI__builtin_allow_sanitize_check: {
3686 Intrinsic::ID IntrID = Intrinsic::not_intrinsic;
3687 StringRef Name =
3688 cast<StringLiteral>(Val: E->getArg(Arg: 0)->IgnoreParenCasts())->getString();
3689
3690 // We deliberately allow the use of kernel- and non-kernel names
3691 // interchangably, even when one or the other is enabled. This is consistent
3692 // with the no_sanitize-attribute, which allows either kernel- or non-kernel
3693 // name to disable instrumentation (see CodeGenFunction::StartFunction).
3694 if (getLangOpts().Sanitize.hasOneOf(K: SanitizerKind::Address |
3695 SanitizerKind::KernelAddress) &&
3696 (Name == "address" || Name == "kernel-address")) {
3697 IntrID = Intrinsic::allow_sanitize_address;
3698 } else if (getLangOpts().Sanitize.has(K: SanitizerKind::Thread) &&
3699 Name == "thread") {
3700 IntrID = Intrinsic::allow_sanitize_thread;
3701 } else if (getLangOpts().Sanitize.hasOneOf(K: SanitizerKind::Memory |
3702 SanitizerKind::KernelMemory) &&
3703 (Name == "memory" || Name == "kernel-memory")) {
3704 IntrID = Intrinsic::allow_sanitize_memory;
3705 } else if (getLangOpts().Sanitize.hasOneOf(
3706 K: SanitizerKind::HWAddress | SanitizerKind::KernelHWAddress) &&
3707 (Name == "hwaddress" || Name == "kernel-hwaddress")) {
3708 IntrID = Intrinsic::allow_sanitize_hwaddress;
3709 }
3710
3711 if (IntrID != Intrinsic::not_intrinsic) {
3712 llvm::Value *Allow = Builder.CreateCall(Callee: CGM.getIntrinsic(IID: IntrID));
3713 return RValue::get(V: Allow);
3714 }
3715 // If the checked sanitizer is not enabled, we can safely lower to false
3716 // right away. This is also more efficient, since the LowerAllowCheckPass
3717 // must not always be enabled if none of the above sanitizers are enabled.
3718 return RValue::get(V: Builder.getFalse());
3719 }
3720 case Builtin::BI__arithmetic_fence: {
3721 // Create the builtin call if FastMath is selected, and the target
3722 // supports the builtin, otherwise just return the argument.
3723 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
3724 llvm::FastMathFlags FMF = Builder.getFastMathFlags();
3725 bool isArithmeticFenceEnabled =
3726 FMF.allowReassoc() &&
3727 getContext().getTargetInfo().checkArithmeticFenceSupported();
3728 QualType ArgType = E->getArg(Arg: 0)->getType();
3729 if (ArgType->isComplexType()) {
3730 if (isArithmeticFenceEnabled) {
3731 QualType ElementType = ArgType->castAs<ComplexType>()->getElementType();
3732 ComplexPairTy ComplexVal = EmitComplexExpr(E: E->getArg(Arg: 0));
3733 Value *Real = Builder.CreateArithmeticFence(Val: ComplexVal.first,
3734 DstType: ConvertType(T: ElementType));
3735 Value *Imag = Builder.CreateArithmeticFence(Val: ComplexVal.second,
3736 DstType: ConvertType(T: ElementType));
3737 return RValue::getComplex(C: std::make_pair(x&: Real, y&: Imag));
3738 }
3739 ComplexPairTy ComplexVal = EmitComplexExpr(E: E->getArg(Arg: 0));
3740 Value *Real = ComplexVal.first;
3741 Value *Imag = ComplexVal.second;
3742 return RValue::getComplex(C: std::make_pair(x&: Real, y&: Imag));
3743 }
3744 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
3745 if (isArithmeticFenceEnabled)
3746 return RValue::get(
3747 V: Builder.CreateArithmeticFence(Val: ArgValue, DstType: ConvertType(T: ArgType)));
3748 return RValue::get(V: ArgValue);
3749 }
3750 case Builtin::BI__builtin_bswapg: {
3751 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
3752 llvm::IntegerType *IntTy = cast<llvm::IntegerType>(Val: ArgValue->getType());
3753 assert(IntTy && "LLVM's __builtin_bswapg only supports integer variants");
3754 if (IntTy->getBitWidth() == 1 || IntTy->getBitWidth() == 8)
3755 return RValue::get(V: ArgValue);
3756 assert(((IntTy->getBitWidth() % 16 == 0 && IntTy->getBitWidth() != 0)) &&
3757 "LLVM's __builtin_bswapg only supports integer variants that has a "
3758 "multiple of 16 bits as well as a single byte");
3759 return RValue::get(
3760 V: emitBuiltinWithOneOverloadedType<1>(CGF&: *this, E, IntrinsicID: Intrinsic::bswap));
3761 }
3762 case Builtin::BI__builtin_bswap16:
3763 case Builtin::BI__builtin_bswap32:
3764 case Builtin::BI__builtin_bswap64:
3765 case Builtin::BI_byteswap_ushort:
3766 case Builtin::BI_byteswap_ulong:
3767 case Builtin::BI_byteswap_uint64: {
3768 return RValue::get(
3769 V: emitBuiltinWithOneOverloadedType<1>(CGF&: *this, E, IntrinsicID: Intrinsic::bswap));
3770 }
3771 case Builtin::BI__builtin_bitreverseg: {
3772 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
3773 llvm::IntegerType *IntTy = cast<llvm::IntegerType>(Val: ArgValue->getType());
3774 assert(IntTy &&
3775 "LLVM's __builtin_bitreverseg only support integer variants");
3776 if (IntTy->getBitWidth() == 1)
3777 return RValue::get(V: ArgValue);
3778 return RValue::get(
3779 V: emitBuiltinWithOneOverloadedType<1>(CGF&: *this, E, IntrinsicID: Intrinsic::bitreverse));
3780 }
3781 case Builtin::BI__builtin_bitreverse8:
3782 case Builtin::BI__builtin_bitreverse16:
3783 case Builtin::BI__builtin_bitreverse32:
3784 case Builtin::BI__builtin_bitreverse64: {
3785 return RValue::get(
3786 V: emitBuiltinWithOneOverloadedType<1>(CGF&: *this, E, IntrinsicID: Intrinsic::bitreverse));
3787 }
3788 case Builtin::BI__builtin_rotateleft8:
3789 case Builtin::BI__builtin_rotateleft16:
3790 case Builtin::BI__builtin_rotateleft32:
3791 case Builtin::BI__builtin_rotateleft64:
3792 case Builtin::BI__builtin_stdc_rotate_left:
3793 case Builtin::BIstdc_rotate_left_uc:
3794 case Builtin::BIstdc_rotate_left_us:
3795 case Builtin::BIstdc_rotate_left_ui:
3796 case Builtin::BIstdc_rotate_left_ul:
3797 case Builtin::BIstdc_rotate_left_ull:
3798 case Builtin::BI_rotl8: // Microsoft variants of rotate left
3799 case Builtin::BI_rotl16:
3800 case Builtin::BI_rotl:
3801 case Builtin::BI_lrotl:
3802 case Builtin::BI_rotl64:
3803 return emitRotate(E, IsRotateRight: false);
3804
3805 case Builtin::BI__builtin_rotateright8:
3806 case Builtin::BI__builtin_rotateright16:
3807 case Builtin::BI__builtin_rotateright32:
3808 case Builtin::BI__builtin_rotateright64:
3809 case Builtin::BI__builtin_stdc_rotate_right:
3810 case Builtin::BIstdc_rotate_right_uc:
3811 case Builtin::BIstdc_rotate_right_us:
3812 case Builtin::BIstdc_rotate_right_ui:
3813 case Builtin::BIstdc_rotate_right_ul:
3814 case Builtin::BIstdc_rotate_right_ull:
3815 case Builtin::BI_rotr8: // Microsoft variants of rotate right
3816 case Builtin::BI_rotr16:
3817 case Builtin::BI_rotr:
3818 case Builtin::BI_lrotr:
3819 case Builtin::BI_rotr64:
3820 return emitRotate(E, IsRotateRight: true);
3821
3822 case Builtin::BIstdc_leading_zeros_uc:
3823 case Builtin::BIstdc_leading_zeros_us:
3824 case Builtin::BIstdc_leading_zeros_ui:
3825 case Builtin::BIstdc_leading_zeros_ul:
3826 case Builtin::BIstdc_leading_zeros_ull:
3827 case Builtin::BI__builtin_stdc_leading_zeros:
3828 return emitStdcCountIntrinsic(E, IntID: Intrinsic::ctlz, /*InvertArg=*/false);
3829 case Builtin::BIstdc_leading_ones_uc:
3830 case Builtin::BIstdc_leading_ones_us:
3831 case Builtin::BIstdc_leading_ones_ui:
3832 case Builtin::BIstdc_leading_ones_ul:
3833 case Builtin::BIstdc_leading_ones_ull:
3834 case Builtin::BI__builtin_stdc_leading_ones:
3835 return emitStdcCountIntrinsic(E, IntID: Intrinsic::ctlz, /*InvertArg=*/true);
3836 case Builtin::BIstdc_trailing_zeros_uc:
3837 case Builtin::BIstdc_trailing_zeros_us:
3838 case Builtin::BIstdc_trailing_zeros_ui:
3839 case Builtin::BIstdc_trailing_zeros_ul:
3840 case Builtin::BIstdc_trailing_zeros_ull:
3841 case Builtin::BI__builtin_stdc_trailing_zeros:
3842 return emitStdcCountIntrinsic(E, IntID: Intrinsic::cttz, /*InvertArg=*/false);
3843 case Builtin::BIstdc_trailing_ones_uc:
3844 case Builtin::BIstdc_trailing_ones_us:
3845 case Builtin::BIstdc_trailing_ones_ui:
3846 case Builtin::BIstdc_trailing_ones_ul:
3847 case Builtin::BIstdc_trailing_ones_ull:
3848 case Builtin::BI__builtin_stdc_trailing_ones:
3849 return emitStdcCountIntrinsic(E, IntID: Intrinsic::cttz, /*InvertArg=*/true);
3850 case Builtin::BIstdc_first_leading_zero_uc:
3851 case Builtin::BIstdc_first_leading_zero_us:
3852 case Builtin::BIstdc_first_leading_zero_ui:
3853 case Builtin::BIstdc_first_leading_zero_ul:
3854 case Builtin::BIstdc_first_leading_zero_ull:
3855 case Builtin::BI__builtin_stdc_first_leading_zero:
3856 return emitStdcFirstBit(E, IntID: Intrinsic::ctlz, /*InvertArg=*/true);
3857 case Builtin::BIstdc_first_leading_one_uc:
3858 case Builtin::BIstdc_first_leading_one_us:
3859 case Builtin::BIstdc_first_leading_one_ui:
3860 case Builtin::BIstdc_first_leading_one_ul:
3861 case Builtin::BIstdc_first_leading_one_ull:
3862 case Builtin::BI__builtin_stdc_first_leading_one:
3863 return emitStdcFirstBit(E, IntID: Intrinsic::ctlz, /*InvertArg=*/false);
3864 case Builtin::BIstdc_first_trailing_zero_uc:
3865 case Builtin::BIstdc_first_trailing_zero_us:
3866 case Builtin::BIstdc_first_trailing_zero_ui:
3867 case Builtin::BIstdc_first_trailing_zero_ul:
3868 case Builtin::BIstdc_first_trailing_zero_ull:
3869 case Builtin::BI__builtin_stdc_first_trailing_zero:
3870 return emitStdcFirstBit(E, IntID: Intrinsic::cttz, /*InvertArg=*/true);
3871 case Builtin::BIstdc_first_trailing_one_uc:
3872 case Builtin::BIstdc_first_trailing_one_us:
3873 case Builtin::BIstdc_first_trailing_one_ui:
3874 case Builtin::BIstdc_first_trailing_one_ul:
3875 case Builtin::BIstdc_first_trailing_one_ull:
3876 case Builtin::BI__builtin_stdc_first_trailing_one:
3877 return emitStdcFirstBit(E, IntID: Intrinsic::cttz, /*InvertArg=*/false);
3878 case Builtin::BIstdc_count_zeros_uc:
3879 case Builtin::BIstdc_count_zeros_us:
3880 case Builtin::BIstdc_count_zeros_ui:
3881 case Builtin::BIstdc_count_zeros_ul:
3882 case Builtin::BIstdc_count_zeros_ull:
3883 case Builtin::BI__builtin_stdc_count_zeros:
3884 return emitStdcBitWidthMinus(E, IntID: Intrinsic::ctpop, /*IsPop=*/true);
3885 case Builtin::BIstdc_count_ones_uc:
3886 case Builtin::BIstdc_count_ones_us:
3887 case Builtin::BIstdc_count_ones_ui:
3888 case Builtin::BIstdc_count_ones_ul:
3889 case Builtin::BIstdc_count_ones_ull:
3890 case Builtin::BI__builtin_stdc_count_ones:
3891 return emitStdcCountIntrinsic(E, IntID: Intrinsic::ctpop, /*InvertArg=*/false,
3892 /*IsPop=*/true);
3893 case Builtin::BIstdc_has_single_bit_uc:
3894 case Builtin::BIstdc_has_single_bit_us:
3895 case Builtin::BIstdc_has_single_bit_ui:
3896 case Builtin::BIstdc_has_single_bit_ul:
3897 case Builtin::BIstdc_has_single_bit_ull:
3898 case Builtin::BI__builtin_stdc_has_single_bit: {
3899 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
3900 llvm::Type *ArgType = ArgValue->getType();
3901 Value *One = ConstantInt::get(Ty: ArgType, V: 1);
3902 Function *F = CGM.getIntrinsic(IID: Intrinsic::ctpop, Tys: ArgType);
3903 Value *PopCnt = Builder.CreateCall(Callee: F, Args: ArgValue);
3904 return RValue::get(V: Builder.CreateICmpEQ(LHS: PopCnt, RHS: One));
3905 }
3906 case Builtin::BIstdc_bit_width_uc:
3907 case Builtin::BIstdc_bit_width_us:
3908 case Builtin::BIstdc_bit_width_ui:
3909 case Builtin::BIstdc_bit_width_ul:
3910 case Builtin::BIstdc_bit_width_ull:
3911 case Builtin::BI__builtin_stdc_bit_width:
3912 return emitStdcBitWidthMinus(E, IntID: Intrinsic::ctlz, /*IsPop=*/false);
3913 case Builtin::BIstdc_bit_floor_uc:
3914 case Builtin::BIstdc_bit_floor_us:
3915 case Builtin::BIstdc_bit_floor_ui:
3916 case Builtin::BIstdc_bit_floor_ul:
3917 case Builtin::BIstdc_bit_floor_ull:
3918 case Builtin::BI__builtin_stdc_bit_floor: {
3919 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
3920 llvm::Type *ArgType = ArgValue->getType();
3921 unsigned BitWidth = ArgType->getIntegerBitWidth();
3922 Value *Zero = ConstantInt::get(Ty: ArgType, V: 0);
3923 Value *One = ConstantInt::get(Ty: ArgType, V: 1);
3924 Function *F = CGM.getIntrinsic(IID: Intrinsic::ctlz, Tys: ArgType);
3925 Value *LZ = Builder.CreateCall(Callee: F, Args: {ArgValue, Builder.getTrue()});
3926 Value *ShiftAmt =
3927 Builder.CreateSub(LHS: ConstantInt::get(Ty: ArgType, V: BitWidth - 1), RHS: LZ);
3928 Value *Shifted = Builder.CreateShl(LHS: One, RHS: ShiftAmt);
3929 Value *IsZero = Builder.CreateICmpEQ(LHS: ArgValue, RHS: Zero);
3930 Value *Result = Builder.CreateSelect(C: IsZero, True: Zero, False: Shifted);
3931 return RValue::get(V: Result);
3932 }
3933 case Builtin::BIstdc_bit_ceil_uc:
3934 case Builtin::BIstdc_bit_ceil_us:
3935 case Builtin::BIstdc_bit_ceil_ui:
3936 case Builtin::BIstdc_bit_ceil_ul:
3937 case Builtin::BIstdc_bit_ceil_ull:
3938 case Builtin::BI__builtin_stdc_bit_ceil: {
3939 Value *ArgValue = EmitScalarExpr(E: E->getArg(Arg: 0));
3940 llvm::Type *ArgType = ArgValue->getType();
3941 Value *One = ConstantInt::get(Ty: ArgType, V: 1);
3942 Value *IsLEOne = Builder.CreateICmpULE(LHS: ArgValue, RHS: One, Name: "isleone");
3943
3944 BasicBlock *EntryBB = Builder.GetInsertBlock();
3945 BasicBlock *CalcBB = createBasicBlock(name: "bitceil.calc", parent: CurFn);
3946 BasicBlock *MergeBB = createBasicBlock(name: "bitceil.merge", parent: CurFn);
3947
3948 Builder.CreateCondBr(Cond: IsLEOne, True: MergeBB, False: CalcBB);
3949
3950 Builder.SetInsertPoint(CalcBB);
3951 Function *F = CGM.getIntrinsic(IID: Intrinsic::ctlz, Tys: ArgType);
3952 Value *ArgMinusOne = Builder.CreateSub(LHS: ArgValue, RHS: One);
3953 Value *LZ = Builder.CreateCall(Callee: F, Args: {ArgMinusOne, Builder.getFalse()});
3954 // 2<<(BitWidth-1-LZ) to get the next power of two. The shift
3955 // amount is always in [0, BitWidth-1], so when LZ==0 (argument has its MSB
3956 // set), the result wraps to 0
3957 unsigned BitWidth = ArgType->getIntegerBitWidth();
3958 Value *ShiftAmt =
3959 Builder.CreateSub(LHS: ConstantInt::get(Ty: ArgType, V: BitWidth - 1), RHS: LZ);
3960 Value *Two = Builder.CreateShl(LHS: One, RHS: One);
3961 Value *Tmp = Builder.CreateShl(LHS: Two, RHS: ShiftAmt);
3962 Builder.CreateBr(Dest: MergeBB);
3963
3964 Builder.SetInsertPoint(MergeBB);
3965 PHINode *Phi = Builder.CreatePHI(Ty: ArgType, NumReservedValues: 2);
3966 Phi->addIncoming(V: One, BB: EntryBB);
3967 Phi->addIncoming(V: Tmp, BB: CalcBB);
3968 return RValue::get(V: Phi);
3969 }
3970
3971 // stdc_memreverse8u8 is a no-op (single byte, nothing to swap).
3972 case Builtin::BIstdc_memreverse8u8:
3973 return RValue::get(V: EmitScalarExpr(E: E->getArg(Arg: 0)));
3974
3975 case Builtin::BIstdc_memreverse8u16:
3976 case Builtin::BIstdc_memreverse8u32:
3977 case Builtin::BIstdc_memreverse8u64:
3978 return RValue::get(
3979 V: emitBuiltinWithOneOverloadedType<1>(CGF&: *this, E, IntrinsicID: Intrinsic::bswap));
3980
3981 case Builtin::BIstdc_memreverse8:
3982 case Builtin::BI__builtin_stdc_memreverse8: {
3983 Expr::EvalResult R;
3984 if (E->getArg(Arg: 0)->EvaluateAsInt(Result&: R, Ctx: getContext())) {
3985 uint64_t Size = R.Val.getInt().getZExtValue();
3986 if (Size <= 1) {
3987 EmitIgnoredExpr(E: E->getArg(Arg: 1));
3988 return RValue::get(V: nullptr);
3989 }
3990 if (Size == 2 || Size == 4 || Size == 8) {
3991 llvm::Type *IntTy = Builder.getIntNTy(N: Size * 8);
3992 Address PtrAddr = EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
3993 Address Addr = PtrAddr.withElementType(ElemTy: IntTy);
3994 Value *Val = Builder.CreateLoad(Addr);
3995 Function *F = CGM.getIntrinsic(IID: Intrinsic::bswap, Tys: IntTy);
3996 Value *Swapped = Builder.CreateCall(Callee: F, Args: Val);
3997 Builder.CreateStore(Val: Swapped, Addr);
3998 return RValue::get(V: nullptr);
3999 }
4000 }
4001
4002 // General case: fall back to the library function stdc_memreverse8.
4003 break;
4004 }
4005
4006 // stdc_load8_*: load N bytes from a const unsigned char* and assemble
4007 // into an integer using little-endian or big-endian byte order.
4008 case Builtin::BIstdc_load8_leu8:
4009 case Builtin::BIstdc_load8_les8:
4010 case Builtin::BIstdc_load8_beu8:
4011 case Builtin::BIstdc_load8_bes8:
4012 case Builtin::BIstdc_load8_aligned_leu8:
4013 case Builtin::BIstdc_load8_aligned_les8:
4014 case Builtin::BIstdc_load8_aligned_beu8:
4015 case Builtin::BIstdc_load8_aligned_bes8:
4016 case Builtin::BIstdc_load8_leu16:
4017 case Builtin::BIstdc_load8_les16:
4018 case Builtin::BIstdc_load8_beu16:
4019 case Builtin::BIstdc_load8_bes16:
4020 case Builtin::BIstdc_load8_aligned_leu16:
4021 case Builtin::BIstdc_load8_aligned_les16:
4022 case Builtin::BIstdc_load8_aligned_beu16:
4023 case Builtin::BIstdc_load8_aligned_bes16:
4024 case Builtin::BIstdc_load8_leu32:
4025 case Builtin::BIstdc_load8_les32:
4026 case Builtin::BIstdc_load8_beu32:
4027 case Builtin::BIstdc_load8_bes32:
4028 case Builtin::BIstdc_load8_aligned_leu32:
4029 case Builtin::BIstdc_load8_aligned_les32:
4030 case Builtin::BIstdc_load8_aligned_beu32:
4031 case Builtin::BIstdc_load8_aligned_bes32:
4032 case Builtin::BIstdc_load8_leu64:
4033 case Builtin::BIstdc_load8_les64:
4034 case Builtin::BIstdc_load8_beu64:
4035 case Builtin::BIstdc_load8_bes64:
4036 case Builtin::BIstdc_load8_aligned_leu64:
4037 case Builtin::BIstdc_load8_aligned_les64:
4038 case Builtin::BIstdc_load8_aligned_beu64:
4039 case Builtin::BIstdc_load8_aligned_bes64: {
4040 bool IsBE = BuiltinID == Builtin::BIstdc_load8_beu8 ||
4041 BuiltinID == Builtin::BIstdc_load8_bes8 ||
4042 BuiltinID == Builtin::BIstdc_load8_beu16 ||
4043 BuiltinID == Builtin::BIstdc_load8_bes16 ||
4044 BuiltinID == Builtin::BIstdc_load8_beu32 ||
4045 BuiltinID == Builtin::BIstdc_load8_bes32 ||
4046 BuiltinID == Builtin::BIstdc_load8_beu64 ||
4047 BuiltinID == Builtin::BIstdc_load8_bes64 ||
4048 BuiltinID == Builtin::BIstdc_load8_aligned_beu8 ||
4049 BuiltinID == Builtin::BIstdc_load8_aligned_bes8 ||
4050 BuiltinID == Builtin::BIstdc_load8_aligned_beu16 ||
4051 BuiltinID == Builtin::BIstdc_load8_aligned_bes16 ||
4052 BuiltinID == Builtin::BIstdc_load8_aligned_beu32 ||
4053 BuiltinID == Builtin::BIstdc_load8_aligned_bes32 ||
4054 BuiltinID == Builtin::BIstdc_load8_aligned_beu64 ||
4055 BuiltinID == Builtin::BIstdc_load8_aligned_bes64;
4056 bool IsAligned = BuiltinID == Builtin::BIstdc_load8_aligned_leu8 ||
4057 BuiltinID == Builtin::BIstdc_load8_aligned_les8 ||
4058 BuiltinID == Builtin::BIstdc_load8_aligned_beu8 ||
4059 BuiltinID == Builtin::BIstdc_load8_aligned_bes8 ||
4060 BuiltinID == Builtin::BIstdc_load8_aligned_leu16 ||
4061 BuiltinID == Builtin::BIstdc_load8_aligned_les16 ||
4062 BuiltinID == Builtin::BIstdc_load8_aligned_beu16 ||
4063 BuiltinID == Builtin::BIstdc_load8_aligned_bes16 ||
4064 BuiltinID == Builtin::BIstdc_load8_aligned_leu32 ||
4065 BuiltinID == Builtin::BIstdc_load8_aligned_les32 ||
4066 BuiltinID == Builtin::BIstdc_load8_aligned_beu32 ||
4067 BuiltinID == Builtin::BIstdc_load8_aligned_bes32 ||
4068 BuiltinID == Builtin::BIstdc_load8_aligned_leu64 ||
4069 BuiltinID == Builtin::BIstdc_load8_aligned_les64 ||
4070 BuiltinID == Builtin::BIstdc_load8_aligned_beu64 ||
4071 BuiltinID == Builtin::BIstdc_load8_aligned_bes64;
4072
4073 llvm::Type *IntTy = ConvertType(T: E->getType());
4074 unsigned BitWidth = IntTy->getIntegerBitWidth();
4075 assert(
4076 (BitWidth == 8 || BitWidth == 16 || BitWidth == 32 || BitWidth == 64) &&
4077 "unexpected bit width for stdc_load8_*");
4078 Address Addr = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4079 if (IsAligned) {
4080 CharUnits TypeAlign = getContext().getTypeAlignInChars(T: E->getType());
4081 Addr = Addr.withAlignment(NewAlignment: std::max(a: Addr.getAlignment(), b: TypeAlign));
4082 }
4083 Addr = Addr.withElementType(ElemTy: IntTy);
4084 Value *Val = Builder.CreateLoad(Addr);
4085 if ((BitWidth == 16 || BitWidth == 32 || BitWidth == 64) &&
4086 IsBE != getTarget().isBigEndian())
4087 Val = Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Intrinsic::bswap, Tys: IntTy), Args: Val);
4088
4089 return RValue::get(V: Val);
4090 }
4091
4092 case Builtin::BI__builtin_constant_p: {
4093 llvm::Type *ResultType = ConvertType(T: E->getType());
4094
4095 const Expr *Arg = E->getArg(Arg: 0);
4096 QualType ArgType = Arg->getType();
4097 // FIXME: The allowance for Obj-C pointers and block pointers is historical
4098 // and likely a mistake.
4099 if (!ArgType->isIntegralOrEnumerationType() && !ArgType->isFloatingType() &&
4100 !ArgType->isObjCObjectPointerType() && !ArgType->isBlockPointerType())
4101 // Per the GCC documentation, only numeric constants are recognized after
4102 // inlining.
4103 return RValue::get(V: ConstantInt::get(Ty: ResultType, V: 0));
4104
4105 if (Arg->HasSideEffects(Ctx: getContext()))
4106 // The argument is unevaluated, so be conservative if it might have
4107 // side-effects.
4108 return RValue::get(V: ConstantInt::get(Ty: ResultType, V: 0));
4109
4110 Value *ArgValue = EmitScalarExpr(E: Arg);
4111 if (ArgType->isObjCObjectPointerType()) {
4112 // Convert Objective-C objects to id because we cannot distinguish between
4113 // LLVM types for Obj-C classes as they are opaque.
4114 ArgType = CGM.getContext().getObjCIdType();
4115 ArgValue = Builder.CreateBitCast(V: ArgValue, DestTy: ConvertType(T: ArgType));
4116 }
4117 Function *F =
4118 CGM.getIntrinsic(IID: Intrinsic::is_constant, Tys: ConvertType(T: ArgType));
4119 Value *Result = Builder.CreateCall(Callee: F, Args: ArgValue);
4120 if (Result->getType() != ResultType)
4121 Result = Builder.CreateIntCast(V: Result, DestTy: ResultType, /*isSigned*/false);
4122 return RValue::get(V: Result);
4123 }
4124 case Builtin::BI__builtin_dynamic_object_size:
4125 case Builtin::BI__builtin_object_size: {
4126 unsigned Type =
4127 E->getArg(Arg: 1)->EvaluateKnownConstInt(Ctx: getContext()).getZExtValue();
4128 auto *ResType = cast<llvm::IntegerType>(Val: ConvertType(T: E->getType()));
4129
4130 // We pass this builtin onto the optimizer so that it can figure out the
4131 // object size in more complex cases.
4132 bool IsDynamic = BuiltinID == Builtin::BI__builtin_dynamic_object_size;
4133 return RValue::get(V: emitBuiltinObjectSize(E: E->getArg(Arg: 0), Type, ResType,
4134 /*EmittedE=*/nullptr, IsDynamic));
4135 }
4136 case Builtin::BI__builtin_counted_by_ref: {
4137 // Default to returning '(void *) 0'.
4138 llvm::Value *Result = llvm::ConstantPointerNull::get(
4139 T: llvm::PointerType::getUnqual(C&: getLLVMContext()));
4140
4141 const Expr *Arg = E->getArg(Arg: 0)->IgnoreParenImpCasts();
4142
4143 if (auto *UO = dyn_cast<UnaryOperator>(Val: Arg);
4144 UO && UO->getOpcode() == UO_AddrOf) {
4145 Arg = UO->getSubExpr()->IgnoreParenImpCasts();
4146
4147 if (auto *ASE = dyn_cast<ArraySubscriptExpr>(Val: Arg))
4148 Arg = ASE->getBase()->IgnoreParenImpCasts();
4149 }
4150
4151 if (const MemberExpr *ME = dyn_cast_if_present<MemberExpr>(Val: Arg)) {
4152 if (auto *CATy =
4153 ME->getMemberDecl()->getType()->getAs<CountAttributedType>();
4154 CATy && CATy->getKind() == CountAttributedType::CountedBy) {
4155 const auto *MemberDecl = cast<FieldDecl>(Val: ME->getMemberDecl());
4156 if (const FieldDecl *CountFD = MemberDecl->findCountedByField())
4157 Result = GetCountedByFieldExprGEP(Base: Arg, FD: MemberDecl, CountDecl: CountFD);
4158 else
4159 llvm::report_fatal_error(reason: "Cannot find the counted_by 'count' field");
4160 }
4161 }
4162
4163 return RValue::get(V: Result);
4164 }
4165 case Builtin::BI__builtin_prefetch: {
4166 Value *Locality, *RW, *Address = EmitScalarExpr(E: E->getArg(Arg: 0));
4167 unsigned ICEArguments = (1 << 1) | (1 << 2);
4168 RW = (E->getNumArgs() > 1) ? EmitScalarOrConstFoldImmArg(ICEArguments, Idx: 1, E)
4169 : llvm::ConstantInt::get(Ty: Int32Ty, V: 0);
4170 RW = Builder.CreateZExtOrTrunc(V: RW, DestTy: Int32Ty);
4171 Locality = (E->getNumArgs() > 2)
4172 ? EmitScalarOrConstFoldImmArg(ICEArguments, Idx: 2, E)
4173 : llvm::ConstantInt::get(Ty: Int32Ty, V: 3);
4174 Locality = Builder.CreateZExtOrTrunc(V: Locality, DestTy: Int32Ty);
4175 Value *Data = llvm::ConstantInt::get(Ty: Int32Ty, V: 1);
4176 Function *F = CGM.getIntrinsic(IID: Intrinsic::prefetch, Tys: Address->getType());
4177 Builder.CreateCall(Callee: F, Args: {Address, RW, Locality, Data});
4178 return RValue::get(V: nullptr);
4179 }
4180 case Builtin::BI__builtin_readcyclecounter: {
4181 Function *F = CGM.getIntrinsic(IID: Intrinsic::readcyclecounter);
4182 return RValue::get(V: Builder.CreateCall(Callee: F));
4183 }
4184 case Builtin::BI__builtin_readsteadycounter: {
4185 Function *F = CGM.getIntrinsic(IID: Intrinsic::readsteadycounter);
4186 return RValue::get(V: Builder.CreateCall(Callee: F));
4187 }
4188 case Builtin::BI__builtin___clear_cache: {
4189 Value *Begin = EmitScalarExpr(E: E->getArg(Arg: 0));
4190 Value *End = EmitScalarExpr(E: E->getArg(Arg: 1));
4191 Function *F = CGM.getIntrinsic(IID: Intrinsic::clear_cache, Tys: {CGM.DefaultPtrTy});
4192 return RValue::get(V: Builder.CreateCall(Callee: F, Args: {Begin, End}));
4193 }
4194 case Builtin::BI__builtin_trap:
4195 EmitTrapCall(IntrID: Intrinsic::trap);
4196 return RValue::get(V: nullptr);
4197 case Builtin::BI__builtin_verbose_trap: {
4198 llvm::DebugLoc CallLocation = Builder.getCurrentDebugLocation();
4199 llvm::DILocation *TrapLocation = CallLocation;
4200 if (getDebugInfo()) {
4201 TrapLocation = getDebugInfo()->CreateTrapFailureMessageFor(
4202 TrapLocation, Category: *E->getArg(Arg: 0)->tryEvaluateString(Ctx&: getContext()),
4203 FailureMsg: *E->getArg(Arg: 1)->tryEvaluateString(Ctx&: getContext()));
4204 // Keep the trap on the builtin's source line. A line-zero location would
4205 // leave the trap attributed to the preceding line in the line table.
4206 TrapLocation = llvm::DILocation::get(
4207 Context&: getLLVMContext(), Line: CallLocation.getLine(), Column: CallLocation.getCol(),
4208 Scope: TrapLocation->getScope(), InlinedAt: TrapLocation->getInlinedAt());
4209 }
4210 ApplyDebugLocation ApplyTrapDI(*this, TrapLocation);
4211 // Currently no attempt is made to prevent traps from being merged.
4212 EmitTrapCall(IntrID: Intrinsic::trap);
4213 return RValue::get(V: nullptr);
4214 }
4215 case Builtin::BI__debugbreak:
4216 EmitTrapCall(IntrID: Intrinsic::debugtrap);
4217 return RValue::get(V: nullptr);
4218 case Builtin::BI__builtin_unreachable: {
4219 EmitUnreachable(Loc: E->getExprLoc());
4220
4221 // We do need to preserve an insertion point.
4222 EmitBlock(BB: createBasicBlock(name: "unreachable.cont"));
4223
4224 return RValue::get(V: nullptr);
4225 }
4226
4227 case Builtin::BI__builtin_powi:
4228 case Builtin::BI__builtin_powif:
4229 case Builtin::BI__builtin_powil: {
4230 llvm::Value *Src0 = EmitScalarExpr(E: E->getArg(Arg: 0));
4231 llvm::Value *Src1 = EmitScalarExpr(E: E->getArg(Arg: 1));
4232
4233 if (Builder.getIsFPConstrained()) {
4234 // FIXME: llvm.powi has 2 mangling types,
4235 // llvm.experimental.constrained.powi has one.
4236 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
4237 Function *F = CGM.getIntrinsic(IID: Intrinsic::experimental_constrained_powi,
4238 Tys: Src0->getType());
4239 return RValue::get(V: Builder.CreateConstrainedFPCall(Callee: F, Args: { Src0, Src1 }));
4240 }
4241
4242 Function *F = CGM.getIntrinsic(IID: Intrinsic::powi,
4243 Tys: { Src0->getType(), Src1->getType() });
4244 return RValue::get(V: Builder.CreateCall(Callee: F, Args: { Src0, Src1 }));
4245 }
4246 case Builtin::BI__builtin_frexpl: {
4247 // Linux PPC will not be adding additional PPCDoubleDouble support.
4248 // WIP to switch default to IEEE long double. Will emit libcall for
4249 // frexpl instead of legalizing this type in the BE.
4250 if (&getTarget().getLongDoubleFormat() == &llvm::APFloat::PPCDoubleDouble())
4251 break;
4252 [[fallthrough]];
4253 }
4254 case Builtin::BI__builtin_frexp:
4255 case Builtin::BI__builtin_frexpf:
4256 case Builtin::BI__builtin_frexpf128:
4257 case Builtin::BI__builtin_frexpf16:
4258 return RValue::get(V: emitFrexpBuiltin(CGF&: *this, E, IntrinsicID: Intrinsic::frexp));
4259 case Builtin::BImodf:
4260 case Builtin::BImodff:
4261 case Builtin::BImodfl:
4262 case Builtin::BI__builtin_modf:
4263 case Builtin::BI__builtin_modff:
4264 case Builtin::BI__builtin_modfl:
4265 if (Builder.getIsFPConstrained())
4266 break; // TODO: Emit constrained modf intrinsic once one exists.
4267 return RValue::get(V: emitModfBuiltin(CGF&: *this, E, IntrinsicID: Intrinsic::modf));
4268 case Builtin::BI__builtin_isgreater:
4269 case Builtin::BI__builtin_isgreaterequal:
4270 case Builtin::BI__builtin_isless:
4271 case Builtin::BI__builtin_islessequal:
4272 case Builtin::BI__builtin_islessgreater:
4273 case Builtin::BI__builtin_isunordered: {
4274 // Ordered comparisons: we know the arguments to these are matching scalar
4275 // floating point values.
4276 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
4277 Value *LHS = EmitScalarExpr(E: E->getArg(Arg: 0));
4278 Value *RHS = EmitScalarExpr(E: E->getArg(Arg: 1));
4279
4280 switch (BuiltinID) {
4281 default: llvm_unreachable("Unknown ordered comparison");
4282 case Builtin::BI__builtin_isgreater:
4283 LHS = Builder.CreateFCmpOGT(LHS, RHS, Name: "cmp");
4284 break;
4285 case Builtin::BI__builtin_isgreaterequal:
4286 LHS = Builder.CreateFCmpOGE(LHS, RHS, Name: "cmp");
4287 break;
4288 case Builtin::BI__builtin_isless:
4289 LHS = Builder.CreateFCmpOLT(LHS, RHS, Name: "cmp");
4290 break;
4291 case Builtin::BI__builtin_islessequal:
4292 LHS = Builder.CreateFCmpOLE(LHS, RHS, Name: "cmp");
4293 break;
4294 case Builtin::BI__builtin_islessgreater:
4295 LHS = Builder.CreateFCmpONE(LHS, RHS, Name: "cmp");
4296 break;
4297 case Builtin::BI__builtin_isunordered:
4298 LHS = Builder.CreateFCmpUNO(LHS, RHS, Name: "cmp");
4299 break;
4300 }
4301 // ZExt bool to int type.
4302 return RValue::get(V: Builder.CreateZExt(V: LHS, DestTy: ConvertType(T: E->getType())));
4303 }
4304
4305 case Builtin::BI__builtin_isnan: {
4306 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
4307 Value *V = EmitScalarExpr(E: E->getArg(Arg: 0));
4308 if (Value *Result = tryUseTestFPKind(CGF&: *this, BuiltinID, V))
4309 return RValue::get(V: Result);
4310 return RValue::get(
4311 V: Builder.CreateZExt(V: Builder.createIsFPClass(FPNum: V, Test: FPClassTest::fcNan),
4312 DestTy: ConvertType(T: E->getType())));
4313 }
4314
4315 case Builtin::BI__builtin_issignaling: {
4316 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
4317 Value *V = EmitScalarExpr(E: E->getArg(Arg: 0));
4318 return RValue::get(
4319 V: Builder.CreateZExt(V: Builder.createIsFPClass(FPNum: V, Test: FPClassTest::fcSNan),
4320 DestTy: ConvertType(T: E->getType())));
4321 }
4322
4323 case Builtin::BI__builtin_isinf: {
4324 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
4325 Value *V = EmitScalarExpr(E: E->getArg(Arg: 0));
4326 if (Value *Result = tryUseTestFPKind(CGF&: *this, BuiltinID, V))
4327 return RValue::get(V: Result);
4328 return RValue::get(
4329 V: Builder.CreateZExt(V: Builder.createIsFPClass(FPNum: V, Test: FPClassTest::fcInf),
4330 DestTy: ConvertType(T: E->getType())));
4331 }
4332
4333 case Builtin::BIfinite:
4334 case Builtin::BI__finite:
4335 case Builtin::BIfinitef:
4336 case Builtin::BI__finitef:
4337 case Builtin::BIfinitel:
4338 case Builtin::BI__finitel:
4339 case Builtin::BI__builtin_isfinite: {
4340 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
4341 Value *V = EmitScalarExpr(E: E->getArg(Arg: 0));
4342 if (Value *Result = tryUseTestFPKind(CGF&: *this, BuiltinID, V))
4343 return RValue::get(V: Result);
4344 return RValue::get(
4345 V: Builder.CreateZExt(V: Builder.createIsFPClass(FPNum: V, Test: FPClassTest::fcFinite),
4346 DestTy: ConvertType(T: E->getType())));
4347 }
4348
4349 case Builtin::BI__builtin_isnormal: {
4350 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
4351 Value *V = EmitScalarExpr(E: E->getArg(Arg: 0));
4352 return RValue::get(
4353 V: Builder.CreateZExt(V: Builder.createIsFPClass(FPNum: V, Test: FPClassTest::fcNormal),
4354 DestTy: ConvertType(T: E->getType())));
4355 }
4356
4357 case Builtin::BI__builtin_issubnormal: {
4358 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
4359 Value *V = EmitScalarExpr(E: E->getArg(Arg: 0));
4360 return RValue::get(
4361 V: Builder.CreateZExt(V: Builder.createIsFPClass(FPNum: V, Test: FPClassTest::fcSubnormal),
4362 DestTy: ConvertType(T: E->getType())));
4363 }
4364
4365 case Builtin::BI__builtin_iszero: {
4366 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
4367 Value *V = EmitScalarExpr(E: E->getArg(Arg: 0));
4368 return RValue::get(
4369 V: Builder.CreateZExt(V: Builder.createIsFPClass(FPNum: V, Test: FPClassTest::fcZero),
4370 DestTy: ConvertType(T: E->getType())));
4371 }
4372
4373 case Builtin::BI__builtin_isfpclass: {
4374 Expr::EvalResult Result;
4375 if (!E->getArg(Arg: 1)->EvaluateAsInt(Result, Ctx: CGM.getContext()))
4376 break;
4377 uint64_t Test = Result.Val.getInt().getLimitedValue();
4378 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
4379 Value *V = EmitScalarExpr(E: E->getArg(Arg: 0));
4380 return RValue::get(V: Builder.CreateZExt(V: Builder.createIsFPClass(FPNum: V, Test),
4381 DestTy: ConvertType(T: E->getType())));
4382 }
4383
4384 case Builtin::BI__builtin_nondeterministic_value: {
4385 llvm::Type *Ty = ConvertType(T: E->getArg(Arg: 0)->getType());
4386
4387 Value *Result = PoisonValue::get(T: Ty);
4388 Result = Builder.CreateFreeze(V: Result);
4389
4390 return RValue::get(V: Result);
4391 }
4392
4393 case Builtin::BI__builtin_elementwise_abs: {
4394 Value *Result;
4395 QualType QT = E->getArg(Arg: 0)->getType();
4396
4397 if (auto *VecTy = QT->getAs<VectorType>())
4398 QT = VecTy->getElementType();
4399 else if (auto *MatTy = QT->getAs<ConstantMatrixType>())
4400 QT = MatTy->getElementType();
4401 if (QT->isIntegerType())
4402 Result = Builder.CreateBinaryIntrinsic(
4403 ID: Intrinsic::abs, LHS: EmitScalarExpr(E: E->getArg(Arg: 0)), RHS: Builder.getFalse(),
4404 FMFSource: nullptr, Name: "elt.abs");
4405 else
4406 Result = emitBuiltinWithOneOverloadedType<1>(CGF&: *this, E, IntrinsicID: Intrinsic::fabs,
4407 Name: "elt.abs");
4408
4409 return RValue::get(V: Result);
4410 }
4411 case Builtin::BI__builtin_elementwise_bitreverse:
4412 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4413 CGF&: *this, E, IntrinsicID: Intrinsic::bitreverse, Name: "elt.bitreverse"));
4414 case Builtin::BI__builtin_elementwise_popcount:
4415 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4416 CGF&: *this, E, IntrinsicID: Intrinsic::ctpop, Name: "elt.ctpop"));
4417 case Builtin::BI__builtin_elementwise_canonicalize:
4418 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4419 CGF&: *this, E, IntrinsicID: Intrinsic::canonicalize, Name: "elt.canonicalize"));
4420 case Builtin::BI__builtin_elementwise_copysign:
4421 return RValue::get(
4422 V: emitBuiltinWithOneOverloadedType<2>(CGF&: *this, E, IntrinsicID: Intrinsic::copysign));
4423 case Builtin::BI__builtin_elementwise_fshl:
4424 return RValue::get(
4425 V: emitBuiltinWithOneOverloadedType<3>(CGF&: *this, E, IntrinsicID: Intrinsic::fshl));
4426 case Builtin::BI__builtin_elementwise_fshr:
4427 return RValue::get(
4428 V: emitBuiltinWithOneOverloadedType<3>(CGF&: *this, E, IntrinsicID: Intrinsic::fshr));
4429 case Builtin::BI__builtin_elementwise_clmul:
4430 return RValue::get(
4431 V: emitBuiltinWithOneOverloadedType<2>(CGF&: *this, E, IntrinsicID: Intrinsic::clmul));
4432 case Builtin::BI__builtin_elementwise_pext:
4433 return RValue::get(
4434 V: emitBuiltinWithOneOverloadedType<2>(CGF&: *this, E, IntrinsicID: Intrinsic::pext));
4435 case Builtin::BI__builtin_elementwise_pdep:
4436 return RValue::get(
4437 V: emitBuiltinWithOneOverloadedType<2>(CGF&: *this, E, IntrinsicID: Intrinsic::pdep));
4438
4439 case Builtin::BI__builtin_elementwise_add_sat:
4440 case Builtin::BI__builtin_elementwise_sub_sat: {
4441 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
4442 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
4443 Value *Result;
4444 assert(Op0->getType()->isIntOrIntVectorTy() && "integer type expected");
4445 QualType Ty = E->getArg(Arg: 0)->getType();
4446 if (auto *VecTy = Ty->getAs<VectorType>())
4447 Ty = VecTy->getElementType();
4448 bool IsSigned = Ty->isSignedIntegerType();
4449 unsigned Opc;
4450 if (BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_elementwise_add_sat)
4451 Opc = IsSigned ? Intrinsic::sadd_sat : Intrinsic::uadd_sat;
4452 else
4453 Opc = IsSigned ? Intrinsic::ssub_sat : Intrinsic::usub_sat;
4454 Result = Builder.CreateBinaryIntrinsic(ID: Opc, LHS: Op0, RHS: Op1, FMFSource: nullptr, Name: "elt.sat");
4455 return RValue::get(V: Result);
4456 }
4457
4458 case Builtin::BI__builtin_elementwise_max: {
4459 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
4460 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
4461 Value *Result;
4462 if (Op0->getType()->isIntOrIntVectorTy()) {
4463 QualType Ty = E->getArg(Arg: 0)->getType();
4464 Result = Builder.CreateBinaryIntrinsic(
4465 ID: Ty->hasSignedIntegerRepresentation() ? Intrinsic::smax
4466 : Intrinsic::umax,
4467 LHS: Op0, RHS: Op1, FMFSource: nullptr, Name: "elt.max");
4468 } else
4469 Result = Builder.CreateMaxNum(LHS: Op0, RHS: Op1, /*FMFSource=*/nullptr, Name: "elt.max");
4470 return RValue::get(V: Result);
4471 }
4472 case Builtin::BI__builtin_elementwise_min: {
4473 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
4474 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
4475 Value *Result;
4476 if (Op0->getType()->isIntOrIntVectorTy()) {
4477 QualType Ty = E->getArg(Arg: 0)->getType();
4478 Result = Builder.CreateBinaryIntrinsic(
4479 ID: Ty->hasSignedIntegerRepresentation() ? Intrinsic::smin
4480 : Intrinsic::umin,
4481 LHS: Op0, RHS: Op1, FMFSource: nullptr, Name: "elt.min");
4482 } else
4483 Result = Builder.CreateMinNum(LHS: Op0, RHS: Op1, /*FMFSource=*/nullptr, Name: "elt.min");
4484 return RValue::get(V: Result);
4485 }
4486
4487 case Builtin::BI__builtin_elementwise_maxnum: {
4488 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
4489 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
4490 Value *Result = Builder.CreateBinaryIntrinsic(ID: llvm::Intrinsic::maxnum, LHS: Op0,
4491 RHS: Op1, FMFSource: nullptr, Name: "elt.maxnum");
4492 return RValue::get(V: Result);
4493 }
4494
4495 case Builtin::BI__builtin_elementwise_minnum: {
4496 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
4497 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
4498 Value *Result = Builder.CreateBinaryIntrinsic(ID: llvm::Intrinsic::minnum, LHS: Op0,
4499 RHS: Op1, FMFSource: nullptr, Name: "elt.minnum");
4500 return RValue::get(V: Result);
4501 }
4502
4503 case Builtin::BI__builtin_elementwise_maximum: {
4504 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
4505 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
4506 Value *Result = Builder.CreateBinaryIntrinsic(ID: Intrinsic::maximum, LHS: Op0, RHS: Op1,
4507 FMFSource: nullptr, Name: "elt.maximum");
4508 return RValue::get(V: Result);
4509 }
4510
4511 case Builtin::BI__builtin_elementwise_minimum: {
4512 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
4513 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
4514 Value *Result = Builder.CreateBinaryIntrinsic(ID: Intrinsic::minimum, LHS: Op0, RHS: Op1,
4515 FMFSource: nullptr, Name: "elt.minimum");
4516 return RValue::get(V: Result);
4517 }
4518
4519 case Builtin::BI__builtin_elementwise_maximumnum: {
4520 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
4521 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
4522 Value *Result = Builder.CreateBinaryIntrinsic(
4523 ID: Intrinsic::maximumnum, LHS: Op0, RHS: Op1, FMFSource: nullptr, Name: "elt.maximumnum");
4524 return RValue::get(V: Result);
4525 }
4526
4527 case Builtin::BI__builtin_elementwise_minimumnum: {
4528 Value *Op0 = EmitScalarExpr(E: E->getArg(Arg: 0));
4529 Value *Op1 = EmitScalarExpr(E: E->getArg(Arg: 1));
4530 Value *Result = Builder.CreateBinaryIntrinsic(
4531 ID: Intrinsic::minimumnum, LHS: Op0, RHS: Op1, FMFSource: nullptr, Name: "elt.minimumnum");
4532 return RValue::get(V: Result);
4533 }
4534
4535 case Builtin::BI__builtin_reduce_max: {
4536 auto GetIntrinsicID = [this](QualType QT) {
4537 if (auto *VecTy = QT->getAs<VectorType>())
4538 QT = VecTy->getElementType();
4539 else if (QT->isSizelessVectorType())
4540 QT = QT->getSizelessVectorEltType(Ctx: CGM.getContext());
4541
4542 if (QT->isSignedIntegerType())
4543 return Intrinsic::vector_reduce_smax;
4544 if (QT->isUnsignedIntegerType())
4545 return Intrinsic::vector_reduce_umax;
4546 assert(QT->isFloatingType() && "must have a float here");
4547 return Intrinsic::vector_reduce_fmax;
4548 };
4549 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4550 CGF&: *this, E, IntrinsicID: GetIntrinsicID(E->getArg(Arg: 0)->getType()), Name: "rdx.min"));
4551 }
4552
4553 case Builtin::BI__builtin_reduce_min: {
4554 auto GetIntrinsicID = [this](QualType QT) {
4555 if (auto *VecTy = QT->getAs<VectorType>())
4556 QT = VecTy->getElementType();
4557 else if (QT->isSizelessVectorType())
4558 QT = QT->getSizelessVectorEltType(Ctx: CGM.getContext());
4559
4560 if (QT->isSignedIntegerType())
4561 return Intrinsic::vector_reduce_smin;
4562 if (QT->isUnsignedIntegerType())
4563 return Intrinsic::vector_reduce_umin;
4564 assert(QT->isFloatingType() && "must have a float here");
4565 return Intrinsic::vector_reduce_fmin;
4566 };
4567
4568 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4569 CGF&: *this, E, IntrinsicID: GetIntrinsicID(E->getArg(Arg: 0)->getType()), Name: "rdx.min"));
4570 }
4571
4572 case Builtin::BI__builtin_reduce_add:
4573 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4574 CGF&: *this, E, IntrinsicID: Intrinsic::vector_reduce_add, Name: "rdx.add"));
4575 case Builtin::BI__builtin_reduce_mul:
4576 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4577 CGF&: *this, E, IntrinsicID: Intrinsic::vector_reduce_mul, Name: "rdx.mul"));
4578 case Builtin::BI__builtin_reduce_xor:
4579 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4580 CGF&: *this, E, IntrinsicID: Intrinsic::vector_reduce_xor, Name: "rdx.xor"));
4581 case Builtin::BI__builtin_reduce_or:
4582 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4583 CGF&: *this, E, IntrinsicID: Intrinsic::vector_reduce_or, Name: "rdx.or"));
4584 case Builtin::BI__builtin_reduce_and:
4585 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4586 CGF&: *this, E, IntrinsicID: Intrinsic::vector_reduce_and, Name: "rdx.and"));
4587 case Builtin::BI__builtin_reduce_maximum:
4588 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4589 CGF&: *this, E, IntrinsicID: Intrinsic::vector_reduce_fmaximum, Name: "rdx.maximum"));
4590 case Builtin::BI__builtin_reduce_minimum:
4591 return RValue::get(V: emitBuiltinWithOneOverloadedType<1>(
4592 CGF&: *this, E, IntrinsicID: Intrinsic::vector_reduce_fminimum, Name: "rdx.minimum"));
4593 case Builtin::BI__builtin_reduce_assoc_fadd:
4594 case Builtin::BI__builtin_reduce_in_order_fadd: {
4595 llvm::Value *Vector = EmitScalarExpr(E: E->getArg(Arg: 0));
4596 llvm::Type *ScalarTy = Vector->getType()->getScalarType();
4597 llvm::Value *StartValue = nullptr;
4598 if (E->getNumArgs() == 2)
4599 StartValue = Builder.CreateFPCast(V: EmitScalarExpr(E: E->getArg(Arg: 1)), DestTy: ScalarTy);
4600 llvm::Value *Args[] = {/*start_value=*/StartValue
4601 ? StartValue
4602 : llvm::ConstantFP::get(Ty: ScalarTy, V: -0.0F),
4603 /*vector=*/Vector};
4604 llvm::Function *F =
4605 CGM.getIntrinsic(IID: Intrinsic::vector_reduce_fadd, Tys: Vector->getType());
4606 llvm::CallBase *Reduce = Builder.CreateCall(Callee: F, Args, Name: "rdx.addf");
4607 if (BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_reduce_assoc_fadd) {
4608 // `__builtin_reduce_assoc_fadd` is an associative reduction which
4609 // requires the reassoc FMF flag.
4610 llvm::FastMathFlags FMF;
4611 FMF.setAllowReassoc();
4612 cast<llvm::CallBase>(Val: Reduce)->setFastMathFlags(FMF);
4613 }
4614 return RValue::get(V: Reduce);
4615 }
4616
4617 case Builtin::BI__builtin_matrix_transpose: {
4618 auto *MatrixTy = E->getArg(Arg: 0)->getType()->castAs<ConstantMatrixType>();
4619 Value *MatValue = EmitScalarExpr(E: E->getArg(Arg: 0));
4620 MatrixBuilder MB(Builder);
4621 Value *Result = MB.CreateMatrixTranspose(Matrix: MatValue, Rows: MatrixTy->getNumRows(),
4622 Columns: MatrixTy->getNumColumns());
4623 return RValue::get(V: Result);
4624 }
4625
4626 case Builtin::BI__builtin_matrix_column_major_load: {
4627 MatrixBuilder MB(Builder);
4628 // Emit everything that isn't dependent on the first parameter type
4629 Value *Stride = EmitScalarExpr(E: E->getArg(Arg: 3));
4630 const auto *ResultTy = E->getType()->getAs<ConstantMatrixType>();
4631 auto *PtrTy = E->getArg(Arg: 0)->getType()->getAs<PointerType>();
4632 assert(PtrTy && "arg0 must be of pointer type");
4633 bool IsVolatile = PtrTy->getPointeeType().isVolatileQualified();
4634
4635 Address Src = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4636 EmitNonNullArgCheck(RV: RValue::get(V: Src.emitRawPointer(CGF&: *this)),
4637 ArgType: E->getArg(Arg: 0)->getType(), ArgLoc: E->getArg(Arg: 0)->getExprLoc(), AC: FD,
4638 ParmNum: 0);
4639 Value *Result = MB.CreateColumnMajorLoad(
4640 EltTy: Src.getElementType(), DataPtr: Src.emitRawPointer(CGF&: *this),
4641 Alignment: Align(Src.getAlignment().getQuantity()), Stride, IsVolatile,
4642 Rows: ResultTy->getNumRows(), Columns: ResultTy->getNumColumns(), Name: "matrix");
4643 return RValue::get(V: Result);
4644 }
4645
4646 case Builtin::BI__builtin_matrix_column_major_store: {
4647 MatrixBuilder MB(Builder);
4648 Value *Matrix = EmitScalarExpr(E: E->getArg(Arg: 0));
4649 Address Dst = EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
4650 Value *Stride = EmitScalarExpr(E: E->getArg(Arg: 2));
4651
4652 const auto *MatrixTy = E->getArg(Arg: 0)->getType()->getAs<ConstantMatrixType>();
4653 auto *PtrTy = E->getArg(Arg: 1)->getType()->getAs<PointerType>();
4654 assert(PtrTy && "arg1 must be of pointer type");
4655 bool IsVolatile = PtrTy->getPointeeType().isVolatileQualified();
4656
4657 EmitNonNullArgCheck(RV: RValue::get(V: Dst.emitRawPointer(CGF&: *this)),
4658 ArgType: E->getArg(Arg: 1)->getType(), ArgLoc: E->getArg(Arg: 1)->getExprLoc(), AC: FD,
4659 ParmNum: 0);
4660 Value *Result = MB.CreateColumnMajorStore(
4661 Matrix, Ptr: Dst.emitRawPointer(CGF&: *this),
4662 Alignment: Align(Dst.getAlignment().getQuantity()), Stride, IsVolatile,
4663 Rows: MatrixTy->getNumRows(), Columns: MatrixTy->getNumColumns());
4664 addInstToNewSourceAtom(KeyInstruction: cast<Instruction>(Val: Result), Backup: Matrix);
4665 return RValue::get(V: Result);
4666 }
4667
4668 case Builtin::BI__builtin_masked_load:
4669 case Builtin::BI__builtin_masked_expand_load: {
4670 llvm::Value *Mask = EmitScalarExpr(E: E->getArg(Arg: 0));
4671 llvm::Value *Ptr = EmitScalarExpr(E: E->getArg(Arg: 1));
4672
4673 llvm::Type *RetTy = CGM.getTypes().ConvertType(T: E->getType());
4674 llvm::Value *PassThru = llvm::PoisonValue::get(T: RetTy);
4675 if (E->getNumArgs() > 2)
4676 PassThru = EmitScalarExpr(E: E->getArg(Arg: 2));
4677
4678 CharUnits Align = CGM.getNaturalTypeAlignment(
4679 T: E->getType()->getAs<VectorType>()->getElementType(), BaseInfo: nullptr);
4680
4681 llvm::Value *Result;
4682 if (BuiltinID == Builtin::BI__builtin_masked_load)
4683 Result = Builder.CreateMaskedLoad(Ty: RetTy, Ptr, Alignment: Align.getAsAlign(), Mask,
4684 PassThru, Name: "masked_load");
4685 else
4686 Result = Builder.CreateMaskedExpandLoad(Ty: RetTy, Ptr, Align: MaybeAlign(), Mask,
4687 PassThru, Name: "masked_expand_load");
4688
4689 return RValue::get(V: Result);
4690 };
4691 case Builtin::BI__builtin_masked_gather: {
4692 llvm::Value *Mask = EmitScalarExpr(E: E->getArg(Arg: 0));
4693 llvm::Value *Idx = EmitScalarExpr(E: E->getArg(Arg: 1));
4694 llvm::Value *Ptr = EmitScalarExpr(E: E->getArg(Arg: 2));
4695
4696 llvm::Type *RetTy = CGM.getTypes().ConvertType(T: E->getType());
4697 CharUnits Align = CGM.getNaturalTypeAlignment(
4698 T: E->getType()->getAs<VectorType>()->getElementType(), BaseInfo: nullptr);
4699
4700 llvm::Value *PassThru = llvm::PoisonValue::get(T: RetTy);
4701 if (E->getNumArgs() > 3)
4702 PassThru = EmitScalarExpr(E: E->getArg(Arg: 3));
4703
4704 llvm::Type *ElemTy = CGM.getTypes().ConvertType(
4705 T: E->getType()->getAs<VectorType>()->getElementType());
4706 llvm::Value *PtrVec = Builder.CreateGEP(Ty: ElemTy, Ptr, IdxList: Idx);
4707
4708 llvm::Value *Result = Builder.CreateMaskedGather(
4709 Ty: RetTy, Ptrs: PtrVec, Alignment: Align.getAsAlign(), Mask, PassThru, Name: "masked_gather");
4710 return RValue::get(V: Result);
4711 }
4712 case Builtin::BI__builtin_masked_store:
4713 case Builtin::BI__builtin_masked_compress_store: {
4714 llvm::Value *Mask = EmitScalarExpr(E: E->getArg(Arg: 0));
4715 llvm::Value *Val = EmitScalarExpr(E: E->getArg(Arg: 1));
4716 llvm::Value *Ptr = EmitScalarExpr(E: E->getArg(Arg: 2));
4717
4718 CharUnits Align = CGM.getNaturalTypeAlignment(
4719 T: E->getArg(Arg: 1)->getType()->getAs<VectorType>()->getElementType(),
4720 BaseInfo: nullptr);
4721
4722 if (BuiltinID == Builtin::BI__builtin_masked_store)
4723 Builder.CreateMaskedStore(Val, Ptr, Alignment: Align.getAsAlign(), Mask);
4724 else
4725 Builder.CreateMaskedCompressStore(Val, Ptr, Align: MaybeAlign(), Mask);
4726
4727 return RValue::get(V: nullptr);
4728 }
4729 case Builtin::BI__builtin_masked_scatter: {
4730 llvm::Value *Mask = EmitScalarExpr(E: E->getArg(Arg: 0));
4731 llvm::Value *Idx = EmitScalarExpr(E: E->getArg(Arg: 1));
4732 llvm::Value *Val = EmitScalarExpr(E: E->getArg(Arg: 2));
4733 llvm::Value *Ptr = EmitScalarExpr(E: E->getArg(Arg: 3));
4734
4735 CharUnits Align = CGM.getNaturalTypeAlignment(
4736 T: E->getArg(Arg: 2)->getType()->getAs<VectorType>()->getElementType(),
4737 BaseInfo: nullptr);
4738
4739 llvm::Type *ElemTy = CGM.getTypes().ConvertType(
4740 T: E->getArg(Arg: 1)->getType()->getAs<VectorType>()->getElementType());
4741 llvm::Value *PtrVec = Builder.CreateGEP(Ty: ElemTy, Ptr, IdxList: Idx);
4742
4743 Builder.CreateMaskedScatter(Val, Ptrs: PtrVec, Alignment: Align.getAsAlign(), Mask);
4744 return RValue();
4745 }
4746 case Builtin::BI__builtin_isinf_sign: {
4747 // isinf_sign(x) -> fabs(x) == infinity ? (signbit(x) ? -1 : 1) : 0
4748 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
4749 // FIXME: for strictfp/IEEE-754 we need to not trap on SNaN here.
4750 Value *Arg = EmitScalarExpr(E: E->getArg(Arg: 0));
4751 Value *AbsArg = EmitFAbs(CGF&: *this, V: Arg);
4752 Value *IsInf = Builder.CreateFCmpOEQ(
4753 LHS: AbsArg, RHS: ConstantFP::getInfinity(Ty: Arg->getType()), Name: "isinf");
4754 Value *IsNeg = EmitSignBit(CGF&: *this, V: Arg);
4755
4756 llvm::Type *IntTy = ConvertType(T: E->getType());
4757 Value *Zero = Constant::getNullValue(Ty: IntTy);
4758 Value *One = ConstantInt::get(Ty: IntTy, V: 1);
4759 Value *NegativeOne = ConstantInt::getAllOnesValue(Ty: IntTy);
4760 Value *SignResult = Builder.CreateSelect(C: IsNeg, True: NegativeOne, False: One);
4761 Value *Result = Builder.CreateSelect(C: IsInf, True: SignResult, False: Zero);
4762 return RValue::get(V: Result);
4763 }
4764
4765 case Builtin::BI__builtin_flt_rounds: {
4766 Function *F = CGM.getIntrinsic(IID: Intrinsic::get_rounding);
4767
4768 llvm::Type *ResultType = ConvertType(T: E->getType());
4769 Value *Result = Builder.CreateCall(Callee: F);
4770 if (Result->getType() != ResultType)
4771 Result = Builder.CreateIntCast(V: Result, DestTy: ResultType, /*isSigned*/true,
4772 Name: "cast");
4773 return RValue::get(V: Result);
4774 }
4775
4776 case Builtin::BI__builtin_set_flt_rounds: {
4777 Function *F = CGM.getIntrinsic(IID: Intrinsic::set_rounding);
4778
4779 Value *V = EmitScalarExpr(E: E->getArg(Arg: 0));
4780 Builder.CreateCall(Callee: F, Args: V);
4781 return RValue::get(V: nullptr);
4782 }
4783
4784 case Builtin::BI__builtin_fpclassify: {
4785 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
4786 // FIXME: for strictfp/IEEE-754 we need to not trap on SNaN here.
4787 Value *V = EmitScalarExpr(E: E->getArg(Arg: 5));
4788 llvm::Type *Ty = ConvertType(T: E->getArg(Arg: 5)->getType());
4789
4790 // Create Result
4791 BasicBlock *Begin = Builder.GetInsertBlock();
4792 BasicBlock *End = createBasicBlock(name: "fpclassify_end", parent: this->CurFn);
4793 Builder.SetInsertPoint(End);
4794 PHINode *Result =
4795 Builder.CreatePHI(Ty: ConvertType(T: E->getArg(Arg: 0)->getType()), NumReservedValues: 4,
4796 Name: "fpclassify_result");
4797
4798 // if (V==0) return FP_ZERO
4799 Builder.SetInsertPoint(Begin);
4800 Value *IsZero = Builder.CreateFCmpOEQ(LHS: V, RHS: Constant::getNullValue(Ty),
4801 Name: "iszero");
4802 Value *ZeroLiteral = EmitScalarExpr(E: E->getArg(Arg: 4));
4803 BasicBlock *NotZero = createBasicBlock(name: "fpclassify_not_zero", parent: this->CurFn);
4804 Builder.CreateCondBr(Cond: IsZero, True: End, False: NotZero);
4805 Result->addIncoming(V: ZeroLiteral, BB: Begin);
4806
4807 // if (V != V) return FP_NAN
4808 Builder.SetInsertPoint(NotZero);
4809 Value *IsNan = Builder.CreateFCmpUNO(LHS: V, RHS: V, Name: "cmp");
4810 Value *NanLiteral = EmitScalarExpr(E: E->getArg(Arg: 0));
4811 BasicBlock *NotNan = createBasicBlock(name: "fpclassify_not_nan", parent: this->CurFn);
4812 Builder.CreateCondBr(Cond: IsNan, True: End, False: NotNan);
4813 Result->addIncoming(V: NanLiteral, BB: NotZero);
4814
4815 // if (fabs(V) == infinity) return FP_INFINITY
4816 Builder.SetInsertPoint(NotNan);
4817 Value *VAbs = EmitFAbs(CGF&: *this, V);
4818 Value *IsInf =
4819 Builder.CreateFCmpOEQ(LHS: VAbs, RHS: ConstantFP::getInfinity(Ty: V->getType()),
4820 Name: "isinf");
4821 Value *InfLiteral = EmitScalarExpr(E: E->getArg(Arg: 1));
4822 BasicBlock *NotInf = createBasicBlock(name: "fpclassify_not_inf", parent: this->CurFn);
4823 Builder.CreateCondBr(Cond: IsInf, True: End, False: NotInf);
4824 Result->addIncoming(V: InfLiteral, BB: NotNan);
4825
4826 // if (fabs(V) >= MIN_NORMAL) return FP_NORMAL else FP_SUBNORMAL
4827 Builder.SetInsertPoint(NotInf);
4828 APFloat Smallest = APFloat::getSmallestNormalized(
4829 Sem: getContext().getFloatTypeSemantics(T: E->getArg(Arg: 5)->getType()));
4830 Value *IsNormal =
4831 Builder.CreateFCmpUGE(LHS: VAbs, RHS: ConstantFP::get(Context&: V->getContext(), V: Smallest),
4832 Name: "isnormal");
4833 Value *NormalResult =
4834 Builder.CreateSelect(C: IsNormal, True: EmitScalarExpr(E: E->getArg(Arg: 2)),
4835 False: EmitScalarExpr(E: E->getArg(Arg: 3)));
4836 Builder.CreateBr(Dest: End);
4837 Result->addIncoming(V: NormalResult, BB: NotInf);
4838
4839 // return Result
4840 Builder.SetInsertPoint(End);
4841 return RValue::get(V: Result);
4842 }
4843
4844 // An alloca will always return a pointer to the alloca (stack) address
4845 // space. This address space need not be the same as the AST / Language
4846 // default (e.g. in C / C++ auto vars are in the generic address space). At
4847 // the AST level this is handled within CreateTempAlloca et al., but for the
4848 // builtin / dynamic alloca we have to handle it here. We use an explicit cast
4849 // instead of passing an AS to CreateAlloca so as to not inhibit optimisation.
4850 case Builtin::BIalloca:
4851 case Builtin::BI_alloca:
4852 case Builtin::BI__builtin_alloca_uninitialized:
4853 case Builtin::BI__builtin_alloca: {
4854 Value *Size = EmitScalarExpr(E: E->getArg(Arg: 0));
4855 const TargetInfo &TI = getContext().getTargetInfo();
4856 // The alignment of the alloca should correspond to __BIGGEST_ALIGNMENT__.
4857 const Align SuitableAlignmentInBytes =
4858 CGM.getContext()
4859 .toCharUnitsFromBits(BitSize: TI.getSuitableAlign())
4860 .getAsAlign();
4861 AllocaInst *AI = Builder.CreateAlloca(Ty: Builder.getInt8Ty(), ArraySize: Size);
4862 AI->setAlignment(SuitableAlignmentInBytes);
4863 if (BuiltinID != Builtin::BI__builtin_alloca_uninitialized)
4864 initializeAlloca(CGF&: *this, AI, Size, AlignmentInBytes: SuitableAlignmentInBytes);
4865 if (AI->getAddressSpace() !=
4866 CGM.getContext().getTargetAddressSpace(
4867 AS: E->getType()->getPointeeType().getAddressSpace())) {
4868 llvm::Type *Ty = CGM.getTypes().ConvertType(T: E->getType());
4869 return RValue::get(V: performAddrSpaceCast(Src: AI, DestTy: Ty));
4870 }
4871 return RValue::get(V: AI);
4872 }
4873
4874 case Builtin::BI__builtin_alloca_with_align_uninitialized:
4875 case Builtin::BI__builtin_alloca_with_align: {
4876 Value *Size = EmitScalarExpr(E: E->getArg(Arg: 0));
4877 Value *AlignmentInBitsValue = EmitScalarExpr(E: E->getArg(Arg: 1));
4878 auto *AlignmentInBitsCI = cast<ConstantInt>(Val: AlignmentInBitsValue);
4879 unsigned AlignmentInBits = AlignmentInBitsCI->getZExtValue();
4880 const Align AlignmentInBytes =
4881 CGM.getContext().toCharUnitsFromBits(BitSize: AlignmentInBits).getAsAlign();
4882 AllocaInst *AI = Builder.CreateAlloca(Ty: Builder.getInt8Ty(), ArraySize: Size);
4883 AI->setAlignment(AlignmentInBytes);
4884 if (BuiltinID != Builtin::BI__builtin_alloca_with_align_uninitialized)
4885 initializeAlloca(CGF&: *this, AI, Size, AlignmentInBytes);
4886 if (AI->getAddressSpace() !=
4887 CGM.getContext().getTargetAddressSpace(
4888 AS: E->getType()->getPointeeType().getAddressSpace())) {
4889 llvm::Type *Ty = CGM.getTypes().ConvertType(T: E->getType());
4890 return RValue::get(V: performAddrSpaceCast(Src: AI, DestTy: Ty));
4891 }
4892 return RValue::get(V: AI);
4893 }
4894
4895 case Builtin::BI__builtin_infer_alloc_token: {
4896 // The builtin has no enclosing allocation whose placement-new uses could
4897 // be inspected, so only the call-based inference applies.
4898 llvm::MDNode *MDN = buildAllocToken(
4899 AllocType: infer_alloc::inferPossibleType(E, Ctx: getContext(), CastE: CurCast));
4900 llvm::Value *MDV = MetadataAsValue::get(Context&: getLLVMContext(), MD: MDN);
4901 llvm::Function *F =
4902 CGM.getIntrinsic(IID: llvm::Intrinsic::alloc_token_id, Tys: {IntPtrTy});
4903 llvm::CallBase *TokenID = Builder.CreateCall(Callee: F, Args: MDV);
4904 return RValue::get(V: TokenID);
4905 }
4906
4907 case Builtin::BIbzero:
4908 case Builtin::BI__builtin_bzero: {
4909 Address Dest = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4910 Value *SizeVal = EmitScalarExpr(E: E->getArg(Arg: 1));
4911 EmitNonNullArgCheck(Addr: Dest, ArgType: E->getArg(Arg: 0)->getType(),
4912 ArgLoc: E->getArg(Arg: 0)->getExprLoc(), AC: FD, ParmNum: 0);
4913 auto *I = Builder.CreateMemSet(Dest, Value: Builder.getInt8(C: 0), Size: SizeVal, IsVolatile: false);
4914 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
4915 return RValue::get(V: nullptr);
4916 }
4917
4918 case Builtin::BIbcopy:
4919 case Builtin::BI__builtin_bcopy: {
4920 Address Src = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4921 Address Dest = EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
4922 Value *SizeVal = EmitScalarExpr(E: E->getArg(Arg: 2));
4923 EmitNonNullArgCheck(RV: RValue::get(V: Src.emitRawPointer(CGF&: *this)),
4924 ArgType: E->getArg(Arg: 0)->getType(), ArgLoc: E->getArg(Arg: 0)->getExprLoc(), AC: FD,
4925 ParmNum: 0);
4926 EmitNonNullArgCheck(RV: RValue::get(V: Dest.emitRawPointer(CGF&: *this)),
4927 ArgType: E->getArg(Arg: 1)->getType(), ArgLoc: E->getArg(Arg: 1)->getExprLoc(), AC: FD,
4928 ParmNum: 0);
4929 auto *I = Builder.CreateMemMove(Dest, Src, Size: SizeVal, IsVolatile: false);
4930 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
4931 return RValue::get(V: nullptr);
4932 }
4933
4934 case Builtin::BImemcpy:
4935 case Builtin::BI__builtin_memcpy:
4936 case Builtin::BImempcpy:
4937 case Builtin::BI__builtin_mempcpy: {
4938 Address Dest = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4939 Address Src = EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
4940 Value *SizeVal = EmitScalarExpr(E: E->getArg(Arg: 2));
4941 EmitArgCheck(TCK_Store, Dest, E->getArg(Arg: 0), 0);
4942 EmitArgCheck(TCK_Load, Src, E->getArg(Arg: 1), 1);
4943 auto *I = Builder.CreateMemCpy(Dest, Src, Size: SizeVal, IsVolatile: false);
4944 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
4945 if (BuiltinID == Builtin::BImempcpy ||
4946 BuiltinID == Builtin::BI__builtin_mempcpy)
4947 return RValue::get(V: Builder.CreateInBoundsGEP(
4948 Ty: Dest.getElementType(), Ptr: Dest.emitRawPointer(CGF&: *this), IdxList: SizeVal));
4949 else
4950 return RValue::get(Addr: Dest, CGF&: *this);
4951 }
4952
4953 case Builtin::BI__builtin_memcpy_inline: {
4954 Address Dest = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4955 Address Src = EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
4956 uint64_t Size =
4957 E->getArg(Arg: 2)->EvaluateKnownConstInt(Ctx: getContext()).getZExtValue();
4958 EmitArgCheck(TCK_Store, Dest, E->getArg(Arg: 0), 0);
4959 EmitArgCheck(TCK_Load, Src, E->getArg(Arg: 1), 1);
4960 auto *I = Builder.CreateMemCpyInline(Dest, Src, Size);
4961 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
4962 return RValue::get(V: nullptr);
4963 }
4964
4965 case Builtin::BI__builtin_char_memchr:
4966 BuiltinID = Builtin::BI__builtin_memchr;
4967 break;
4968
4969 case Builtin::BI__builtin___memcpy_chk: {
4970 // fold __builtin_memcpy_chk(x, y, cst1, cst2) to memcpy iff cst1<=cst2.
4971 Expr::EvalResult SizeResult, DstSizeResult;
4972 if (!E->getArg(Arg: 2)->EvaluateAsInt(Result&: SizeResult, Ctx: CGM.getContext()) ||
4973 !E->getArg(Arg: 3)->EvaluateAsInt(Result&: DstSizeResult, Ctx: CGM.getContext()))
4974 break;
4975 llvm::APSInt Size = SizeResult.Val.getInt();
4976 llvm::APSInt DstSize = DstSizeResult.Val.getInt();
4977 if (Size.ugt(RHS: DstSize))
4978 break;
4979 Address Dest = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4980 Address Src = EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
4981 Value *SizeVal = llvm::ConstantInt::get(Context&: Builder.getContext(), V: Size);
4982 auto *I = Builder.CreateMemCpy(Dest, Src, Size: SizeVal, IsVolatile: false);
4983 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
4984 return RValue::get(Addr: Dest, CGF&: *this);
4985 }
4986
4987 case Builtin::BI__builtin_objc_memmove_collectable: {
4988 Address DestAddr = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
4989 Address SrcAddr = EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
4990 Value *SizeVal = EmitScalarExpr(E: E->getArg(Arg: 2));
4991 CGM.getObjCRuntime().EmitGCMemmoveCollectable(CGF&: *this,
4992 DestPtr: DestAddr, SrcPtr: SrcAddr, Size: SizeVal);
4993 return RValue::get(Addr: DestAddr, CGF&: *this);
4994 }
4995
4996 case Builtin::BI__builtin___memmove_chk: {
4997 // fold __builtin_memmove_chk(x, y, cst1, cst2) to memmove iff cst1<=cst2.
4998 Expr::EvalResult SizeResult, DstSizeResult;
4999 if (!E->getArg(Arg: 2)->EvaluateAsInt(Result&: SizeResult, Ctx: CGM.getContext()) ||
5000 !E->getArg(Arg: 3)->EvaluateAsInt(Result&: DstSizeResult, Ctx: CGM.getContext()))
5001 break;
5002 llvm::APSInt Size = SizeResult.Val.getInt();
5003 llvm::APSInt DstSize = DstSizeResult.Val.getInt();
5004 if (Size.ugt(RHS: DstSize))
5005 break;
5006 Address Dest = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
5007 Address Src = EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
5008 Value *SizeVal = llvm::ConstantInt::get(Context&: Builder.getContext(), V: Size);
5009 auto *I = Builder.CreateMemMove(Dest, Src, Size: SizeVal, IsVolatile: false);
5010 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
5011 return RValue::get(Addr: Dest, CGF&: *this);
5012 }
5013
5014 case Builtin::BI__builtin_trivially_relocate:
5015 case Builtin::BImemmove:
5016 case Builtin::BI__builtin_memmove: {
5017 Address Dest = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
5018 Address Src = EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
5019 Value *SizeVal = EmitScalarExpr(E: E->getArg(Arg: 2));
5020 if (BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_trivially_relocate)
5021 SizeVal = Builder.CreateMul(
5022 LHS: SizeVal,
5023 RHS: ConstantInt::get(
5024 Ty: SizeVal->getType(),
5025 V: getContext()
5026 .getTypeSizeInChars(T: E->getArg(Arg: 0)->getType()->getPointeeType())
5027 .getQuantity()));
5028 EmitArgCheck(TCK_Store, Dest, E->getArg(Arg: 0), 0);
5029 EmitArgCheck(TCK_Load, Src, E->getArg(Arg: 1), 1);
5030 auto *I = Builder.CreateMemMove(Dest, Src, Size: SizeVal, IsVolatile: false);
5031 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
5032 return RValue::get(Addr: Dest, CGF&: *this);
5033 }
5034 case Builtin::BImemset:
5035 case Builtin::BI__builtin_memset: {
5036 Address Dest = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
5037 Value *ByteVal = Builder.CreateTrunc(V: EmitScalarExpr(E: E->getArg(Arg: 1)),
5038 DestTy: Builder.getInt8Ty());
5039 Value *SizeVal = EmitScalarExpr(E: E->getArg(Arg: 2));
5040 EmitNonNullArgCheck(Addr: Dest, ArgType: E->getArg(Arg: 0)->getType(),
5041 ArgLoc: E->getArg(Arg: 0)->getExprLoc(), AC: FD, ParmNum: 0);
5042 auto *I = Builder.CreateMemSet(Dest, Value: ByteVal, Size: SizeVal, IsVolatile: false);
5043 addInstToNewSourceAtom(KeyInstruction: I, Backup: ByteVal);
5044 return RValue::get(Addr: Dest, CGF&: *this);
5045 }
5046 case Builtin::BI__builtin_memset_inline: {
5047 Address Dest = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
5048 Value *ByteVal =
5049 Builder.CreateTrunc(V: EmitScalarExpr(E: E->getArg(Arg: 1)), DestTy: Builder.getInt8Ty());
5050 uint64_t Size =
5051 E->getArg(Arg: 2)->EvaluateKnownConstInt(Ctx: getContext()).getZExtValue();
5052 EmitNonNullArgCheck(RV: RValue::get(V: Dest.emitRawPointer(CGF&: *this)),
5053 ArgType: E->getArg(Arg: 0)->getType(), ArgLoc: E->getArg(Arg: 0)->getExprLoc(), AC: FD,
5054 ParmNum: 0);
5055 auto *I = Builder.CreateMemSetInline(Dest, Value: ByteVal, Size);
5056 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
5057 return RValue::get(V: nullptr);
5058 }
5059 case Builtin::BI__builtin___memset_chk: {
5060 // fold __builtin_memset_chk(x, y, cst1, cst2) to memset iff cst1<=cst2.
5061 Expr::EvalResult SizeResult, DstSizeResult;
5062 if (!E->getArg(Arg: 2)->EvaluateAsInt(Result&: SizeResult, Ctx: CGM.getContext()) ||
5063 !E->getArg(Arg: 3)->EvaluateAsInt(Result&: DstSizeResult, Ctx: CGM.getContext()))
5064 break;
5065 llvm::APSInt Size = SizeResult.Val.getInt();
5066 llvm::APSInt DstSize = DstSizeResult.Val.getInt();
5067 if (Size.ugt(RHS: DstSize))
5068 break;
5069 Address Dest = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
5070 Value *ByteVal = Builder.CreateTrunc(V: EmitScalarExpr(E: E->getArg(Arg: 1)),
5071 DestTy: Builder.getInt8Ty());
5072 Value *SizeVal = llvm::ConstantInt::get(Context&: Builder.getContext(), V: Size);
5073 auto *I = Builder.CreateMemSet(Dest, Value: ByteVal, Size: SizeVal, IsVolatile: false);
5074 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
5075 return RValue::get(Addr: Dest, CGF&: *this);
5076 }
5077 case Builtin::BI__builtin_wmemchr: {
5078 // The MSVC runtime library does not provide a definition of wmemchr, so we
5079 // need an inline implementation.
5080 if (!getTarget().getTriple().isOSMSVCRT())
5081 break;
5082
5083 llvm::Type *WCharTy = ConvertType(T: getContext().WCharTy);
5084 Value *Str = EmitScalarExpr(E: E->getArg(Arg: 0));
5085 Value *Chr = EmitScalarExpr(E: E->getArg(Arg: 1));
5086 Value *Size = EmitScalarExpr(E: E->getArg(Arg: 2));
5087
5088 BasicBlock *Entry = Builder.GetInsertBlock();
5089 BasicBlock *CmpEq = createBasicBlock(name: "wmemchr.eq");
5090 BasicBlock *Next = createBasicBlock(name: "wmemchr.next");
5091 BasicBlock *Exit = createBasicBlock(name: "wmemchr.exit");
5092 Value *SizeEq0 = Builder.CreateICmpEQ(LHS: Size, RHS: ConstantInt::get(Ty: SizeTy, V: 0));
5093 Builder.CreateCondBr(Cond: SizeEq0, True: Exit, False: CmpEq);
5094
5095 EmitBlock(BB: CmpEq);
5096 PHINode *StrPhi = Builder.CreatePHI(Ty: Str->getType(), NumReservedValues: 2);
5097 StrPhi->addIncoming(V: Str, BB: Entry);
5098 PHINode *SizePhi = Builder.CreatePHI(Ty: SizeTy, NumReservedValues: 2);
5099 SizePhi->addIncoming(V: Size, BB: Entry);
5100 CharUnits WCharAlign =
5101 getContext().getTypeAlignInChars(T: getContext().WCharTy);
5102 Value *StrCh = Builder.CreateAlignedLoad(Ty: WCharTy, Addr: StrPhi, Align: WCharAlign);
5103 Value *FoundChr = Builder.CreateConstInBoundsGEP1_32(Ty: WCharTy, Ptr: StrPhi, Idx0: 0);
5104 Value *StrEqChr = Builder.CreateICmpEQ(LHS: StrCh, RHS: Chr);
5105 Builder.CreateCondBr(Cond: StrEqChr, True: Exit, False: Next);
5106
5107 EmitBlock(BB: Next);
5108 Value *NextStr = Builder.CreateConstInBoundsGEP1_32(Ty: WCharTy, Ptr: StrPhi, Idx0: 1);
5109 Value *NextSize = Builder.CreateSub(LHS: SizePhi, RHS: ConstantInt::get(Ty: SizeTy, V: 1));
5110 Value *NextSizeEq0 =
5111 Builder.CreateICmpEQ(LHS: NextSize, RHS: ConstantInt::get(Ty: SizeTy, V: 0));
5112 Builder.CreateCondBr(Cond: NextSizeEq0, True: Exit, False: CmpEq);
5113 StrPhi->addIncoming(V: NextStr, BB: Next);
5114 SizePhi->addIncoming(V: NextSize, BB: Next);
5115
5116 EmitBlock(BB: Exit);
5117 PHINode *Ret = Builder.CreatePHI(Ty: Str->getType(), NumReservedValues: 3);
5118 Ret->addIncoming(V: llvm::Constant::getNullValue(Ty: Str->getType()), BB: Entry);
5119 Ret->addIncoming(V: llvm::Constant::getNullValue(Ty: Str->getType()), BB: Next);
5120 Ret->addIncoming(V: FoundChr, BB: CmpEq);
5121 return RValue::get(V: Ret);
5122 }
5123 case Builtin::BI__builtin_wmemcmp: {
5124 // The MSVC runtime library does not provide a definition of wmemcmp, so we
5125 // need an inline implementation.
5126 if (!getTarget().getTriple().isOSMSVCRT())
5127 break;
5128
5129 llvm::Type *WCharTy = ConvertType(T: getContext().WCharTy);
5130
5131 Value *Dst = EmitScalarExpr(E: E->getArg(Arg: 0));
5132 Value *Src = EmitScalarExpr(E: E->getArg(Arg: 1));
5133 Value *Size = EmitScalarExpr(E: E->getArg(Arg: 2));
5134
5135 BasicBlock *Entry = Builder.GetInsertBlock();
5136 BasicBlock *CmpGT = createBasicBlock(name: "wmemcmp.gt");
5137 BasicBlock *CmpLT = createBasicBlock(name: "wmemcmp.lt");
5138 BasicBlock *Next = createBasicBlock(name: "wmemcmp.next");
5139 BasicBlock *Exit = createBasicBlock(name: "wmemcmp.exit");
5140 Value *SizeEq0 = Builder.CreateICmpEQ(LHS: Size, RHS: ConstantInt::get(Ty: SizeTy, V: 0));
5141 Builder.CreateCondBr(Cond: SizeEq0, True: Exit, False: CmpGT);
5142
5143 EmitBlock(BB: CmpGT);
5144 PHINode *DstPhi = Builder.CreatePHI(Ty: Dst->getType(), NumReservedValues: 2);
5145 DstPhi->addIncoming(V: Dst, BB: Entry);
5146 PHINode *SrcPhi = Builder.CreatePHI(Ty: Src->getType(), NumReservedValues: 2);
5147 SrcPhi->addIncoming(V: Src, BB: Entry);
5148 PHINode *SizePhi = Builder.CreatePHI(Ty: SizeTy, NumReservedValues: 2);
5149 SizePhi->addIncoming(V: Size, BB: Entry);
5150 CharUnits WCharAlign =
5151 getContext().getTypeAlignInChars(T: getContext().WCharTy);
5152 Value *DstCh = Builder.CreateAlignedLoad(Ty: WCharTy, Addr: DstPhi, Align: WCharAlign);
5153 Value *SrcCh = Builder.CreateAlignedLoad(Ty: WCharTy, Addr: SrcPhi, Align: WCharAlign);
5154 Value *DstGtSrc = Builder.CreateICmpUGT(LHS: DstCh, RHS: SrcCh);
5155 Builder.CreateCondBr(Cond: DstGtSrc, True: Exit, False: CmpLT);
5156
5157 EmitBlock(BB: CmpLT);
5158 Value *DstLtSrc = Builder.CreateICmpULT(LHS: DstCh, RHS: SrcCh);
5159 Builder.CreateCondBr(Cond: DstLtSrc, True: Exit, False: Next);
5160
5161 EmitBlock(BB: Next);
5162 Value *NextDst = Builder.CreateConstInBoundsGEP1_32(Ty: WCharTy, Ptr: DstPhi, Idx0: 1);
5163 Value *NextSrc = Builder.CreateConstInBoundsGEP1_32(Ty: WCharTy, Ptr: SrcPhi, Idx0: 1);
5164 Value *NextSize = Builder.CreateSub(LHS: SizePhi, RHS: ConstantInt::get(Ty: SizeTy, V: 1));
5165 Value *NextSizeEq0 =
5166 Builder.CreateICmpEQ(LHS: NextSize, RHS: ConstantInt::get(Ty: SizeTy, V: 0));
5167 Builder.CreateCondBr(Cond: NextSizeEq0, True: Exit, False: CmpGT);
5168 DstPhi->addIncoming(V: NextDst, BB: Next);
5169 SrcPhi->addIncoming(V: NextSrc, BB: Next);
5170 SizePhi->addIncoming(V: NextSize, BB: Next);
5171
5172 EmitBlock(BB: Exit);
5173 PHINode *Ret = Builder.CreatePHI(Ty: IntTy, NumReservedValues: 4);
5174 Ret->addIncoming(V: ConstantInt::get(Ty: IntTy, V: 0), BB: Entry);
5175 Ret->addIncoming(V: ConstantInt::get(Ty: IntTy, V: 1), BB: CmpGT);
5176 Ret->addIncoming(V: ConstantInt::getAllOnesValue(Ty: IntTy), BB: CmpLT);
5177 Ret->addIncoming(V: ConstantInt::get(Ty: IntTy, V: 0), BB: Next);
5178 return RValue::get(V: Ret);
5179 }
5180 case Builtin::BI__builtin_dwarf_cfa: {
5181 // The offset in bytes from the first argument to the CFA.
5182 //
5183 // Why on earth is this in the frontend? Is there any reason at
5184 // all that the backend can't reasonably determine this while
5185 // lowering llvm.eh.dwarf.cfa()?
5186 //
5187 // TODO: If there's a satisfactory reason, add a target hook for
5188 // this instead of hard-coding 0, which is correct for most targets.
5189 int32_t Offset = 0;
5190
5191 Function *F = CGM.getIntrinsic(IID: Intrinsic::eh_dwarf_cfa);
5192 return RValue::get(V: Builder.CreateCall(Callee: F,
5193 Args: llvm::ConstantInt::get(Ty: Int32Ty, V: Offset)));
5194 }
5195 case Builtin::BI__builtin_return_address: {
5196 Value *Depth = ConstantEmitter(*this).emitAbstract(E: E->getArg(Arg: 0),
5197 T: getContext().UnsignedIntTy);
5198 Depth = Builder.CreateZExtOrTrunc(V: Depth, DestTy: Int32Ty);
5199 Function *F =
5200 CGM.getIntrinsic(IID: Intrinsic::returnaddress, Tys: {CGM.ProgramPtrTy});
5201 return RValue::get(V: Builder.CreateCall(Callee: F, Args: Depth));
5202 }
5203 case Builtin::BI_ReturnAddress: {
5204 Function *F =
5205 CGM.getIntrinsic(IID: Intrinsic::returnaddress, Tys: {CGM.ProgramPtrTy});
5206 return RValue::get(V: Builder.CreateCall(Callee: F, Args: Builder.getInt32(C: 0)));
5207 }
5208 case Builtin::BI__builtin_frame_address: {
5209 Value *Depth = ConstantEmitter(*this).emitAbstract(E: E->getArg(Arg: 0),
5210 T: getContext().UnsignedIntTy);
5211 Depth = Builder.CreateZExtOrTrunc(V: Depth, DestTy: Int32Ty);
5212 Function *F = CGM.getIntrinsic(IID: Intrinsic::frameaddress, Tys: AllocaInt8PtrTy);
5213 return RValue::get(V: Builder.CreateCall(Callee: F, Args: Depth));
5214 }
5215 case Builtin::BI__builtin_stack_address: {
5216 return RValue::get(V: Builder.CreateCall(
5217 Callee: CGM.getIntrinsic(IID: Intrinsic::stackaddress, Tys: AllocaInt8PtrTy)));
5218 }
5219 case Builtin::BI__builtin_extract_return_addr: {
5220 Value *Address = EmitScalarExpr(E: E->getArg(Arg: 0));
5221 Value *Result = getTargetHooks().decodeReturnAddress(CGF&: *this, Address);
5222 return RValue::get(V: Result);
5223 }
5224 case Builtin::BI__builtin_frob_return_addr: {
5225 Value *Address = EmitScalarExpr(E: E->getArg(Arg: 0));
5226 Value *Result = getTargetHooks().encodeReturnAddress(CGF&: *this, Address);
5227 return RValue::get(V: Result);
5228 }
5229 case Builtin::BI__builtin_dwarf_sp_column: {
5230 llvm::IntegerType *Ty
5231 = cast<llvm::IntegerType>(Val: ConvertType(T: E->getType()));
5232 int Column = getTargetHooks().getDwarfEHStackPointer(M&: CGM);
5233 if (Column == -1) {
5234 CGM.ErrorUnsupported(S: E, Type: "__builtin_dwarf_sp_column");
5235 return RValue::get(V: llvm::UndefValue::get(T: Ty));
5236 }
5237 return RValue::get(V: llvm::ConstantInt::get(Ty, V: Column, IsSigned: true));
5238 }
5239 case Builtin::BI__builtin_init_dwarf_reg_size_table: {
5240 Value *Address = EmitScalarExpr(E: E->getArg(Arg: 0));
5241 if (getTargetHooks().initDwarfEHRegSizeTable(CGF&: *this, Address))
5242 CGM.ErrorUnsupported(S: E, Type: "__builtin_init_dwarf_reg_size_table");
5243 return RValue::get(V: llvm::UndefValue::get(T: ConvertType(T: E->getType())));
5244 }
5245 case Builtin::BI__builtin_eh_return: {
5246 Value *Int = EmitScalarExpr(E: E->getArg(Arg: 0));
5247 Value *Ptr = EmitScalarExpr(E: E->getArg(Arg: 1));
5248
5249 llvm::IntegerType *IntTy = cast<llvm::IntegerType>(Val: Int->getType());
5250 assert((IntTy->getBitWidth() == 32 || IntTy->getBitWidth() == 64) &&
5251 "LLVM's __builtin_eh_return only supports 32- and 64-bit variants");
5252 Function *F =
5253 CGM.getIntrinsic(IID: IntTy->getBitWidth() == 32 ? Intrinsic::eh_return_i32
5254 : Intrinsic::eh_return_i64);
5255 Builder.CreateCall(Callee: F, Args: {Int, Ptr});
5256 Builder.CreateUnreachable();
5257
5258 // We do need to preserve an insertion point.
5259 EmitBlock(BB: createBasicBlock(name: "builtin_eh_return.cont"));
5260
5261 return RValue::get(V: nullptr);
5262 }
5263 case Builtin::BI__builtin_unwind_init: {
5264 Function *F = CGM.getIntrinsic(IID: Intrinsic::eh_unwind_init);
5265 Builder.CreateCall(Callee: F);
5266 return RValue::get(V: nullptr);
5267 }
5268 case Builtin::BI__builtin_extend_pointer: {
5269 // Extends a pointer to the size of an _Unwind_Word, which is
5270 // uint64_t on all platforms. Generally this gets poked into a
5271 // register and eventually used as an address, so if the
5272 // addressing registers are wider than pointers and the platform
5273 // doesn't implicitly ignore high-order bits when doing
5274 // addressing, we need to make sure we zext / sext based on
5275 // the platform's expectations.
5276 //
5277 // See: http://gcc.gnu.org/ml/gcc-bugs/2002-02/msg00237.html
5278
5279 // Cast the pointer to intptr_t.
5280 Value *Ptr = EmitScalarExpr(E: E->getArg(Arg: 0));
5281 Value *Result = Builder.CreatePtrToInt(V: Ptr, DestTy: IntPtrTy, Name: "extend.cast");
5282
5283 // If that's 64 bits, we're done.
5284 if (IntPtrTy->getBitWidth() == 64)
5285 return RValue::get(V: Result);
5286
5287 // Otherwise, ask the codegen data what to do.
5288 if (getTargetHooks().extendPointerWithSExt())
5289 return RValue::get(V: Builder.CreateSExt(V: Result, DestTy: Int64Ty, Name: "extend.sext"));
5290 else
5291 return RValue::get(V: Builder.CreateZExt(V: Result, DestTy: Int64Ty, Name: "extend.zext"));
5292 }
5293 case Builtin::BI__builtin_setjmp: {
5294 // Buffer is a void**.
5295 Address Buf = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
5296
5297 if (getTarget().getTriple().getArch() == llvm::Triple::systemz) {
5298 // On this target, the back end fills in the context buffer completely.
5299 // It doesn't really matter if the frontend stores to the buffer before
5300 // calling setjmp, the back-end is going to overwrite them anyway.
5301 Function *F = CGM.getIntrinsic(IID: Intrinsic::eh_sjlj_setjmp);
5302 return RValue::get(V: Builder.CreateCall(Callee: F, Args: Buf.emitRawPointer(CGF&: *this)));
5303 }
5304
5305 // Store the frame pointer to the setjmp buffer.
5306 Value *FrameAddr = Builder.CreateCall(
5307 Callee: CGM.getIntrinsic(IID: Intrinsic::frameaddress, Tys: AllocaInt8PtrTy),
5308 Args: ConstantInt::get(Ty: Int32Ty, V: 0));
5309 Builder.CreateStore(Val: FrameAddr, Addr: Buf);
5310
5311 // Store the stack pointer to the setjmp buffer.
5312 Value *StackAddr = Builder.CreateStackSave();
5313 assert(Buf.emitRawPointer(*this)->getType() == StackAddr->getType());
5314
5315 Address StackSaveSlot = Builder.CreateConstInBoundsGEP(Addr: Buf, Index: 2);
5316 Builder.CreateStore(Val: StackAddr, Addr: StackSaveSlot);
5317
5318 // Call LLVM's EH setjmp, which is lightweight.
5319 Function *F = CGM.getIntrinsic(IID: Intrinsic::eh_sjlj_setjmp);
5320 return RValue::get(V: Builder.CreateCall(Callee: F, Args: Buf.emitRawPointer(CGF&: *this)));
5321 }
5322 case Builtin::BI__builtin_longjmp: {
5323 Value *Buf = EmitScalarExpr(E: E->getArg(Arg: 0));
5324
5325 // Call LLVM's EH longjmp, which is lightweight.
5326 Builder.CreateCall(Callee: CGM.getIntrinsic(IID: Intrinsic::eh_sjlj_longjmp), Args: Buf);
5327
5328 // longjmp doesn't return; mark this as unreachable.
5329 Builder.CreateUnreachable();
5330
5331 // We do need to preserve an insertion point.
5332 EmitBlock(BB: createBasicBlock(name: "longjmp.cont"));
5333
5334 return RValue::get(V: nullptr);
5335 }
5336 case Builtin::BI__builtin_launder: {
5337 const Expr *Arg = E->getArg(Arg: 0);
5338 QualType ArgTy = Arg->getType()->getPointeeType();
5339 Value *Ptr = EmitScalarExpr(E: Arg);
5340 if (TypeRequiresBuiltinLaunder(CGM, Ty: ArgTy))
5341 Ptr = Builder.CreateLaunderInvariantGroup(Ptr);
5342
5343 return RValue::get(V: Ptr);
5344 }
5345 case Builtin::BI__builtin_clear_padding: {
5346 Address Src = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
5347 auto PointeeTy = E->getArg(Arg: 0)->getType()->getPointeeType();
5348
5349 llvm::ArrayRef<ASTContext::BitInterval> Padding =
5350 getContext().getPaddingIntervals(Ty: PointeeTy);
5351 for (const auto &Interval : Padding)
5352 ClearPadding(CGF&: *this, Src, PaddingInterval: Interval);
5353
5354 return RValue::get(V: nullptr);
5355 }
5356 case Builtin::BI__sync_fetch_and_add:
5357 case Builtin::BI__sync_fetch_and_sub:
5358 case Builtin::BI__sync_fetch_and_or:
5359 case Builtin::BI__sync_fetch_and_and:
5360 case Builtin::BI__sync_fetch_and_xor:
5361 case Builtin::BI__sync_fetch_and_nand:
5362 case Builtin::BI__sync_add_and_fetch:
5363 case Builtin::BI__sync_sub_and_fetch:
5364 case Builtin::BI__sync_and_and_fetch:
5365 case Builtin::BI__sync_or_and_fetch:
5366 case Builtin::BI__sync_xor_and_fetch:
5367 case Builtin::BI__sync_nand_and_fetch:
5368 case Builtin::BI__sync_val_compare_and_swap:
5369 case Builtin::BI__sync_bool_compare_and_swap:
5370 case Builtin::BI__sync_lock_test_and_set:
5371 case Builtin::BI__sync_lock_release:
5372 case Builtin::BI__sync_swap:
5373 llvm_unreachable("Shouldn't make it through sema");
5374 case Builtin::BI__sync_fetch_and_add_1:
5375 case Builtin::BI__sync_fetch_and_add_2:
5376 case Builtin::BI__sync_fetch_and_add_4:
5377 case Builtin::BI__sync_fetch_and_add_8:
5378 case Builtin::BI__sync_fetch_and_add_16:
5379 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::Add, E);
5380 case Builtin::BI__sync_fetch_and_sub_1:
5381 case Builtin::BI__sync_fetch_and_sub_2:
5382 case Builtin::BI__sync_fetch_and_sub_4:
5383 case Builtin::BI__sync_fetch_and_sub_8:
5384 case Builtin::BI__sync_fetch_and_sub_16:
5385 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::Sub, E);
5386 case Builtin::BI__sync_fetch_and_or_1:
5387 case Builtin::BI__sync_fetch_and_or_2:
5388 case Builtin::BI__sync_fetch_and_or_4:
5389 case Builtin::BI__sync_fetch_and_or_8:
5390 case Builtin::BI__sync_fetch_and_or_16:
5391 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::Or, E);
5392 case Builtin::BI__sync_fetch_and_and_1:
5393 case Builtin::BI__sync_fetch_and_and_2:
5394 case Builtin::BI__sync_fetch_and_and_4:
5395 case Builtin::BI__sync_fetch_and_and_8:
5396 case Builtin::BI__sync_fetch_and_and_16:
5397 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::And, E);
5398 case Builtin::BI__sync_fetch_and_xor_1:
5399 case Builtin::BI__sync_fetch_and_xor_2:
5400 case Builtin::BI__sync_fetch_and_xor_4:
5401 case Builtin::BI__sync_fetch_and_xor_8:
5402 case Builtin::BI__sync_fetch_and_xor_16:
5403 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::Xor, E);
5404 case Builtin::BI__sync_fetch_and_nand_1:
5405 case Builtin::BI__sync_fetch_and_nand_2:
5406 case Builtin::BI__sync_fetch_and_nand_4:
5407 case Builtin::BI__sync_fetch_and_nand_8:
5408 case Builtin::BI__sync_fetch_and_nand_16:
5409 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::Nand, E);
5410
5411 // Clang extensions: not overloaded yet.
5412 case Builtin::BI__sync_fetch_and_min:
5413 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::Min, E);
5414 case Builtin::BI__sync_fetch_and_max:
5415 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::Max, E);
5416 case Builtin::BI__sync_fetch_and_umin:
5417 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::UMin, E);
5418 case Builtin::BI__sync_fetch_and_umax:
5419 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::UMax, E);
5420
5421 case Builtin::BI__sync_add_and_fetch_1:
5422 case Builtin::BI__sync_add_and_fetch_2:
5423 case Builtin::BI__sync_add_and_fetch_4:
5424 case Builtin::BI__sync_add_and_fetch_8:
5425 case Builtin::BI__sync_add_and_fetch_16:
5426 return EmitBinaryAtomicPost(CGF&: *this, Kind: llvm::AtomicRMWInst::Add, E,
5427 Op: llvm::Instruction::Add);
5428 case Builtin::BI__sync_sub_and_fetch_1:
5429 case Builtin::BI__sync_sub_and_fetch_2:
5430 case Builtin::BI__sync_sub_and_fetch_4:
5431 case Builtin::BI__sync_sub_and_fetch_8:
5432 case Builtin::BI__sync_sub_and_fetch_16:
5433 return EmitBinaryAtomicPost(CGF&: *this, Kind: llvm::AtomicRMWInst::Sub, E,
5434 Op: llvm::Instruction::Sub);
5435 case Builtin::BI__sync_and_and_fetch_1:
5436 case Builtin::BI__sync_and_and_fetch_2:
5437 case Builtin::BI__sync_and_and_fetch_4:
5438 case Builtin::BI__sync_and_and_fetch_8:
5439 case Builtin::BI__sync_and_and_fetch_16:
5440 return EmitBinaryAtomicPost(CGF&: *this, Kind: llvm::AtomicRMWInst::And, E,
5441 Op: llvm::Instruction::And);
5442 case Builtin::BI__sync_or_and_fetch_1:
5443 case Builtin::BI__sync_or_and_fetch_2:
5444 case Builtin::BI__sync_or_and_fetch_4:
5445 case Builtin::BI__sync_or_and_fetch_8:
5446 case Builtin::BI__sync_or_and_fetch_16:
5447 return EmitBinaryAtomicPost(CGF&: *this, Kind: llvm::AtomicRMWInst::Or, E,
5448 Op: llvm::Instruction::Or);
5449 case Builtin::BI__sync_xor_and_fetch_1:
5450 case Builtin::BI__sync_xor_and_fetch_2:
5451 case Builtin::BI__sync_xor_and_fetch_4:
5452 case Builtin::BI__sync_xor_and_fetch_8:
5453 case Builtin::BI__sync_xor_and_fetch_16:
5454 return EmitBinaryAtomicPost(CGF&: *this, Kind: llvm::AtomicRMWInst::Xor, E,
5455 Op: llvm::Instruction::Xor);
5456 case Builtin::BI__sync_nand_and_fetch_1:
5457 case Builtin::BI__sync_nand_and_fetch_2:
5458 case Builtin::BI__sync_nand_and_fetch_4:
5459 case Builtin::BI__sync_nand_and_fetch_8:
5460 case Builtin::BI__sync_nand_and_fetch_16:
5461 return EmitBinaryAtomicPost(CGF&: *this, Kind: llvm::AtomicRMWInst::Nand, E,
5462 Op: llvm::Instruction::And, Invert: true);
5463
5464 case Builtin::BI__sync_val_compare_and_swap_1:
5465 case Builtin::BI__sync_val_compare_and_swap_2:
5466 case Builtin::BI__sync_val_compare_and_swap_4:
5467 case Builtin::BI__sync_val_compare_and_swap_8:
5468 case Builtin::BI__sync_val_compare_and_swap_16:
5469 return RValue::get(V: MakeAtomicCmpXchgValue(
5470 CGF&: *this, E, ReturnBool: false, SuccessOrdering: AtomicOrdering::SequentiallyConsistent,
5471 FailureOrdering: AtomicOrdering::SequentiallyConsistent));
5472
5473 case Builtin::BI__sync_bool_compare_and_swap_1:
5474 case Builtin::BI__sync_bool_compare_and_swap_2:
5475 case Builtin::BI__sync_bool_compare_and_swap_4:
5476 case Builtin::BI__sync_bool_compare_and_swap_8:
5477 case Builtin::BI__sync_bool_compare_and_swap_16:
5478 return RValue::get(V: MakeAtomicCmpXchgValue(
5479 CGF&: *this, E, ReturnBool: true, SuccessOrdering: AtomicOrdering::SequentiallyConsistent,
5480 FailureOrdering: AtomicOrdering::SequentiallyConsistent));
5481
5482 case Builtin::BI__sync_swap_1:
5483 case Builtin::BI__sync_swap_2:
5484 case Builtin::BI__sync_swap_4:
5485 case Builtin::BI__sync_swap_8:
5486 case Builtin::BI__sync_swap_16:
5487 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::Xchg, E);
5488
5489 case Builtin::BI__sync_lock_test_and_set_1:
5490 case Builtin::BI__sync_lock_test_and_set_2:
5491 case Builtin::BI__sync_lock_test_and_set_4:
5492 case Builtin::BI__sync_lock_test_and_set_8:
5493 case Builtin::BI__sync_lock_test_and_set_16:
5494 return EmitBinaryAtomic(CGF&: *this, Kind: llvm::AtomicRMWInst::Xchg, E);
5495
5496 case Builtin::BI__sync_lock_release_1:
5497 case Builtin::BI__sync_lock_release_2:
5498 case Builtin::BI__sync_lock_release_4:
5499 case Builtin::BI__sync_lock_release_8:
5500 case Builtin::BI__sync_lock_release_16: {
5501 Address Ptr = CheckAtomicAlignment(CGF&: *this, E);
5502 QualType ElTy = E->getArg(Arg: 0)->getType()->getPointeeType();
5503
5504 llvm::Type *ITy = llvm::IntegerType::get(C&: getLLVMContext(),
5505 NumBits: getContext().getTypeSize(T: ElTy));
5506 llvm::StoreInst *Store =
5507 Builder.CreateStore(Val: llvm::Constant::getNullValue(Ty: ITy), Addr: Ptr);
5508 Store->setAtomic(Ordering: llvm::AtomicOrdering::Release);
5509 return RValue::get(V: nullptr);
5510 }
5511
5512 case Builtin::BI__sync_synchronize: {
5513 // We assume this is supposed to correspond to a C++0x-style
5514 // sequentially-consistent fence (i.e. this is only usable for
5515 // synchronization, not device I/O or anything like that). This intrinsic
5516 // is really badly designed in the sense that in theory, there isn't
5517 // any way to safely use it... but in practice, it mostly works
5518 // to use it with non-atomic loads and stores to get acquire/release
5519 // semantics.
5520 Builder.CreateFence(Ordering: llvm::AtomicOrdering::SequentiallyConsistent);
5521 return RValue::get(V: nullptr);
5522 }
5523
5524 case Builtin::BI__builtin_nontemporal_load:
5525 return RValue::get(V: EmitNontemporalLoad(CGF&: *this, E));
5526 case Builtin::BI__builtin_nontemporal_store:
5527 return RValue::get(V: EmitNontemporalStore(CGF&: *this, E));
5528 case Builtin::BI__c11_atomic_is_lock_free:
5529 case Builtin::BI__atomic_is_lock_free: {
5530 // Call "bool __atomic_is_lock_free(size_t size, void *ptr)". For the
5531 // __c11 builtin, ptr is 0 (indicating a properly-aligned object), since
5532 // _Atomic(T) is always properly-aligned.
5533 const char *LibCallName = "__atomic_is_lock_free";
5534 CallArgList Args;
5535 Args.add(rvalue: RValue::get(V: EmitScalarExpr(E: E->getArg(Arg: 0))),
5536 type: getContext().getSizeType());
5537 if (BuiltinID == Builtin::BI__atomic_is_lock_free)
5538 Args.add(rvalue: RValue::get(V: EmitScalarExpr(E: E->getArg(Arg: 1))),
5539 type: getContext().VoidPtrTy);
5540 else
5541 Args.add(rvalue: RValue::get(V: llvm::Constant::getNullValue(Ty: VoidPtrTy)),
5542 type: getContext().VoidPtrTy);
5543 const CGFunctionInfo &FuncInfo =
5544 CGM.getTypes().arrangeBuiltinFunctionCall(resultType: E->getType(), args: Args);
5545 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(Info: FuncInfo);
5546 llvm::FunctionCallee Func = CGM.CreateRuntimeFunction(Ty: FTy, Name: LibCallName);
5547 return EmitCall(CallInfo: FuncInfo, Callee: CGCallee::forDirect(functionPtr: Func),
5548 ReturnValue: ReturnValueSlot(), Args);
5549 }
5550
5551 case Builtin::BI__atomic_thread_fence:
5552 case Builtin::BI__atomic_signal_fence:
5553 case Builtin::BI__c11_atomic_thread_fence:
5554 case Builtin::BI__c11_atomic_signal_fence: {
5555 llvm::SyncScope::ID SSID;
5556 if (BuiltinID == Builtin::BI__atomic_signal_fence ||
5557 BuiltinID == Builtin::BI__c11_atomic_signal_fence)
5558 SSID = llvm::SyncScope::SingleThread;
5559 else
5560 SSID = llvm::SyncScope::System;
5561 Value *Order = EmitScalarExpr(E: E->getArg(Arg: 0));
5562 if (isa<llvm::ConstantInt>(Val: Order)) {
5563 int ord = cast<llvm::ConstantInt>(Val: Order)->getZExtValue();
5564 switch (ord) {
5565 case 0: // memory_order_relaxed
5566 default: // invalid order
5567 break;
5568 case 1: // memory_order_consume
5569 case 2: // memory_order_acquire
5570 emitAtomicFence(Order: llvm::AtomicOrdering::Acquire, SSID);
5571 break;
5572 case 3: // memory_order_release
5573 emitAtomicFence(Order: llvm::AtomicOrdering::Release, SSID);
5574 break;
5575 case 4: // memory_order_acq_rel
5576 emitAtomicFence(Order: llvm::AtomicOrdering::AcquireRelease, SSID);
5577 break;
5578 case 5: // memory_order_seq_cst
5579 emitAtomicFence(Order: llvm::AtomicOrdering::SequentiallyConsistent, SSID);
5580 break;
5581 }
5582 return RValue::get(V: nullptr);
5583 }
5584
5585 llvm::BasicBlock *AcquireBB, *ReleaseBB, *AcqRelBB, *SeqCstBB;
5586 AcquireBB = createBasicBlock(name: "acquire", parent: CurFn);
5587 ReleaseBB = createBasicBlock(name: "release", parent: CurFn);
5588 AcqRelBB = createBasicBlock(name: "acqrel", parent: CurFn);
5589 SeqCstBB = createBasicBlock(name: "seqcst", parent: CurFn);
5590 llvm::BasicBlock *ContBB = createBasicBlock(name: "atomic.continue", parent: CurFn);
5591
5592 Order = Builder.CreateIntCast(V: Order, DestTy: Builder.getInt32Ty(), isSigned: false);
5593 llvm::SwitchInst *SI = Builder.CreateSwitch(V: Order, Dest: ContBB);
5594
5595 Builder.SetInsertPoint(AcquireBB);
5596 emitAtomicFence(Order: llvm::AtomicOrdering::Acquire, SSID);
5597 Builder.CreateBr(Dest: ContBB);
5598 SI->addCase(OnVal: Builder.getInt32(C: 1), Dest: AcquireBB);
5599 SI->addCase(OnVal: Builder.getInt32(C: 2), Dest: AcquireBB);
5600
5601 Builder.SetInsertPoint(ReleaseBB);
5602 emitAtomicFence(Order: llvm::AtomicOrdering::Release, SSID);
5603 Builder.CreateBr(Dest: ContBB);
5604 SI->addCase(OnVal: Builder.getInt32(C: 3), Dest: ReleaseBB);
5605
5606 Builder.SetInsertPoint(AcqRelBB);
5607 emitAtomicFence(Order: llvm::AtomicOrdering::AcquireRelease, SSID);
5608 Builder.CreateBr(Dest: ContBB);
5609 SI->addCase(OnVal: Builder.getInt32(C: 4), Dest: AcqRelBB);
5610
5611 Builder.SetInsertPoint(SeqCstBB);
5612 emitAtomicFence(Order: llvm::AtomicOrdering::SequentiallyConsistent, SSID);
5613 Builder.CreateBr(Dest: ContBB);
5614 SI->addCase(OnVal: Builder.getInt32(C: 5), Dest: SeqCstBB);
5615
5616 Builder.SetInsertPoint(ContBB);
5617 return RValue::get(V: nullptr);
5618 }
5619 case Builtin::BI__scoped_atomic_thread_fence: {
5620 auto ScopeModel = AtomicScopeModel::create(K: AtomicScopeModelKind::Generic);
5621
5622 Value *Order = EmitScalarExpr(E: E->getArg(Arg: 0));
5623 Value *Scope = EmitScalarExpr(E: E->getArg(Arg: 1));
5624 auto Ord = dyn_cast<llvm::ConstantInt>(Val: Order);
5625 auto Scp = dyn_cast<llvm::ConstantInt>(Val: Scope);
5626 if (Ord && Scp) {
5627 SyncScope SS = ScopeModel->isValid(S: Scp->getZExtValue())
5628 ? ScopeModel->map(S: Scp->getZExtValue())
5629 : ScopeModel->map(S: ScopeModel->getFallBackValue());
5630 switch (Ord->getZExtValue()) {
5631 case 0: // memory_order_relaxed
5632 default: // invalid order
5633 break;
5634 case 1: // memory_order_consume
5635 case 2: // memory_order_acquire
5636 emitAtomicFence(Order: llvm::AtomicOrdering::Acquire,
5637 SSID: getTargetHooks().getLLVMSyncScopeID(
5638 LangOpts: getLangOpts(), Scope: SS, Ordering: llvm::AtomicOrdering::Acquire,
5639 Ctx&: getLLVMContext()));
5640 break;
5641 case 3: // memory_order_release
5642 emitAtomicFence(Order: llvm::AtomicOrdering::Release,
5643 SSID: getTargetHooks().getLLVMSyncScopeID(
5644 LangOpts: getLangOpts(), Scope: SS, Ordering: llvm::AtomicOrdering::Release,
5645 Ctx&: getLLVMContext()));
5646 break;
5647 case 4: // memory_order_acq_rel
5648 emitAtomicFence(Order: llvm::AtomicOrdering::AcquireRelease,
5649 SSID: getTargetHooks().getLLVMSyncScopeID(
5650 LangOpts: getLangOpts(), Scope: SS,
5651 Ordering: llvm::AtomicOrdering::AcquireRelease,
5652 Ctx&: getLLVMContext()));
5653 break;
5654 case 5: // memory_order_seq_cst
5655 emitAtomicFence(Order: llvm::AtomicOrdering::SequentiallyConsistent,
5656 SSID: getTargetHooks().getLLVMSyncScopeID(
5657 LangOpts: getLangOpts(), Scope: SS,
5658 Ordering: llvm::AtomicOrdering::SequentiallyConsistent,
5659 Ctx&: getLLVMContext()));
5660 break;
5661 }
5662 return RValue::get(V: nullptr);
5663 }
5664
5665 llvm::BasicBlock *ContBB = createBasicBlock(name: "atomic.scope.continue", parent: CurFn);
5666
5667 llvm::SmallVector<std::pair<llvm::BasicBlock *, llvm::AtomicOrdering>>
5668 OrderBBs;
5669 if (Ord) {
5670 switch (Ord->getZExtValue()) {
5671 case 0: // memory_order_relaxed
5672 default: // invalid order
5673 ContBB->eraseFromParent();
5674 return RValue::get(V: nullptr);
5675 case 1: // memory_order_consume
5676 case 2: // memory_order_acquire
5677 OrderBBs.emplace_back(Args: Builder.GetInsertBlock(),
5678 Args: llvm::AtomicOrdering::Acquire);
5679 break;
5680 case 3: // memory_order_release
5681 OrderBBs.emplace_back(Args: Builder.GetInsertBlock(),
5682 Args: llvm::AtomicOrdering::Release);
5683 break;
5684 case 4: // memory_order_acq_rel
5685 OrderBBs.emplace_back(Args: Builder.GetInsertBlock(),
5686 Args: llvm::AtomicOrdering::AcquireRelease);
5687 break;
5688 case 5: // memory_order_seq_cst
5689 OrderBBs.emplace_back(Args: Builder.GetInsertBlock(),
5690 Args: llvm::AtomicOrdering::SequentiallyConsistent);
5691 break;
5692 }
5693 } else {
5694 llvm::BasicBlock *AcquireBB = createBasicBlock(name: "acquire", parent: CurFn);
5695 llvm::BasicBlock *ReleaseBB = createBasicBlock(name: "release", parent: CurFn);
5696 llvm::BasicBlock *AcqRelBB = createBasicBlock(name: "acqrel", parent: CurFn);
5697 llvm::BasicBlock *SeqCstBB = createBasicBlock(name: "seqcst", parent: CurFn);
5698
5699 Order = Builder.CreateIntCast(V: Order, DestTy: Builder.getInt32Ty(), isSigned: false);
5700 llvm::SwitchInst *SI = Builder.CreateSwitch(V: Order, Dest: ContBB);
5701 SI->addCase(OnVal: Builder.getInt32(C: 1), Dest: AcquireBB);
5702 SI->addCase(OnVal: Builder.getInt32(C: 2), Dest: AcquireBB);
5703 SI->addCase(OnVal: Builder.getInt32(C: 3), Dest: ReleaseBB);
5704 SI->addCase(OnVal: Builder.getInt32(C: 4), Dest: AcqRelBB);
5705 SI->addCase(OnVal: Builder.getInt32(C: 5), Dest: SeqCstBB);
5706
5707 OrderBBs.emplace_back(Args&: AcquireBB, Args: llvm::AtomicOrdering::Acquire);
5708 OrderBBs.emplace_back(Args&: ReleaseBB, Args: llvm::AtomicOrdering::Release);
5709 OrderBBs.emplace_back(Args&: AcqRelBB, Args: llvm::AtomicOrdering::AcquireRelease);
5710 OrderBBs.emplace_back(Args&: SeqCstBB,
5711 Args: llvm::AtomicOrdering::SequentiallyConsistent);
5712 }
5713
5714 for (auto &[OrderBB, Ordering] : OrderBBs) {
5715 Builder.SetInsertPoint(OrderBB);
5716 if (Scp) {
5717 SyncScope SS = ScopeModel->isValid(S: Scp->getZExtValue())
5718 ? ScopeModel->map(S: Scp->getZExtValue())
5719 : ScopeModel->map(S: ScopeModel->getFallBackValue());
5720 emitAtomicFence(Order: Ordering,
5721 SSID: getTargetHooks().getLLVMSyncScopeID(
5722 LangOpts: getLangOpts(), Scope: SS, Ordering, Ctx&: getLLVMContext()));
5723 Builder.CreateBr(Dest: ContBB);
5724 } else {
5725 llvm::DenseMap<unsigned, llvm::BasicBlock *> BBs;
5726 for (unsigned Scp : ScopeModel->getRuntimeValues())
5727 BBs[Scp] = createBasicBlock(name: getAsString(S: ScopeModel->map(S: Scp)), parent: CurFn);
5728
5729 auto *SC = Builder.CreateIntCast(V: Scope, DestTy: Builder.getInt32Ty(), isSigned: false);
5730 llvm::SwitchInst *SI = Builder.CreateSwitch(V: SC, Dest: ContBB);
5731 for (unsigned Scp : ScopeModel->getRuntimeValues()) {
5732 auto *B = BBs[Scp];
5733 SI->addCase(OnVal: Builder.getInt32(C: Scp), Dest: B);
5734
5735 Builder.SetInsertPoint(B);
5736 emitAtomicFence(Order: Ordering, SSID: getTargetHooks().getLLVMSyncScopeID(
5737 LangOpts: getLangOpts(), Scope: ScopeModel->map(S: Scp),
5738 Ordering, Ctx&: getLLVMContext()));
5739 Builder.CreateBr(Dest: ContBB);
5740 }
5741 }
5742 }
5743
5744 Builder.SetInsertPoint(ContBB);
5745 return RValue::get(V: nullptr);
5746 }
5747
5748 case Builtin::BI__builtin_signbit:
5749 case Builtin::BI__builtin_signbitf:
5750 case Builtin::BI__builtin_signbitl: {
5751 return RValue::get(
5752 V: Builder.CreateZExt(V: EmitSignBit(CGF&: *this, V: EmitScalarExpr(E: E->getArg(Arg: 0))),
5753 DestTy: ConvertType(T: E->getType())));
5754 }
5755 case Builtin::BI__warn_memset_zero_len:
5756 return RValue::getIgnored();
5757 case Builtin::BI__annotation: {
5758 // Re-encode each wide string to UTF8 and make an MDString.
5759 SmallVector<Metadata *, 1> Strings;
5760 for (const Expr *Arg : E->arguments()) {
5761 const auto *Str = cast<StringLiteral>(Val: Arg->IgnoreParenCasts());
5762 assert(Str->getCharByteWidth() == 2 || Str->getCharByteWidth() == 4);
5763 StringRef WideBytes = Str->getBytes();
5764 std::string StrUtf8;
5765 bool Converted =
5766 (Str->getCharByteWidth() == 2)
5767 ? convertUTF16ToUTF8String(
5768 SrcBytes: ArrayRef(WideBytes.data(), WideBytes.size()), Out&: StrUtf8)
5769 : convertUTF32ToUTF8String(
5770 SrcBytes: ArrayRef(WideBytes.data(), WideBytes.size()), Out&: StrUtf8);
5771 if (!Converted) {
5772 CGM.ErrorUnsupported(S: E, Type: "non-Unicode __annotation argument");
5773 continue;
5774 }
5775 Strings.push_back(Elt: llvm::MDString::get(Context&: getLLVMContext(), Str: StrUtf8));
5776 }
5777
5778 // Build and MDTuple of MDStrings and emit the intrinsic call.
5779 llvm::Function *F = CGM.getIntrinsic(IID: Intrinsic::codeview_annotation, Tys: {});
5780 MDTuple *StrTuple = MDTuple::get(Context&: getLLVMContext(), MDs: Strings);
5781 Builder.CreateCall(Callee: F, Args: MetadataAsValue::get(Context&: getLLVMContext(), MD: StrTuple));
5782 return RValue::getIgnored();
5783 }
5784 case Builtin::BI__builtin_annotation: {
5785 llvm::Value *AnnVal = EmitScalarExpr(E: E->getArg(Arg: 0));
5786 llvm::Function *F = CGM.getIntrinsic(
5787 IID: Intrinsic::annotation, Tys: {AnnVal->getType(), CGM.ConstGlobalsPtrTy});
5788
5789 // Get the annotation string, go through casts. Sema requires this to be a
5790 // non-wide string literal, potentially casted, so the cast<> is safe.
5791 const Expr *AnnotationStrExpr = E->getArg(Arg: 1)->IgnoreParenCasts();
5792 StringRef Str = cast<StringLiteral>(Val: AnnotationStrExpr)->getString();
5793 return RValue::get(
5794 V: EmitAnnotationCall(AnnotationFn: F, AnnotatedVal: AnnVal, AnnotationStr: Str, Location: E->getExprLoc(), Attr: nullptr));
5795 }
5796 case Builtin::BI__builtin_addcb:
5797 case Builtin::BI__builtin_addcs:
5798 case Builtin::BI__builtin_addc:
5799 case Builtin::BI__builtin_addcl:
5800 case Builtin::BI__builtin_addcll:
5801 case Builtin::BI__builtin_subcb:
5802 case Builtin::BI__builtin_subcs:
5803 case Builtin::BI__builtin_subc:
5804 case Builtin::BI__builtin_subcl:
5805 case Builtin::BI__builtin_subcll: {
5806
5807 // We translate all of these builtins from expressions of the form:
5808 // int x = ..., y = ..., carryin = ..., carryout, result;
5809 // result = __builtin_addc(x, y, carryin, &carryout);
5810 //
5811 // to LLVM IR of the form:
5812 //
5813 // %tmp1 = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %x, i32 %y)
5814 // %tmpsum1 = extractvalue {i32, i1} %tmp1, 0
5815 // %carry1 = extractvalue {i32, i1} %tmp1, 1
5816 // %tmp2 = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %tmpsum1,
5817 // i32 %carryin)
5818 // %result = extractvalue {i32, i1} %tmp2, 0
5819 // %carry2 = extractvalue {i32, i1} %tmp2, 1
5820 // %tmp3 = or i1 %carry1, %carry2
5821 // %tmp4 = zext i1 %tmp3 to i32
5822 // store i32 %tmp4, i32* %carryout
5823
5824 // Scalarize our inputs.
5825 llvm::Value *X = EmitScalarExpr(E: E->getArg(Arg: 0));
5826 llvm::Value *Y = EmitScalarExpr(E: E->getArg(Arg: 1));
5827 llvm::Value *Carryin = EmitScalarExpr(E: E->getArg(Arg: 2));
5828 Address CarryOutPtr = EmitPointerWithAlignment(Addr: E->getArg(Arg: 3));
5829
5830 // Decide if we are lowering to a uadd.with.overflow or usub.with.overflow.
5831 Intrinsic::ID IntrinsicId;
5832 switch (BuiltinID) {
5833 default: llvm_unreachable("Unknown multiprecision builtin id.");
5834 case Builtin::BI__builtin_addcb:
5835 case Builtin::BI__builtin_addcs:
5836 case Builtin::BI__builtin_addc:
5837 case Builtin::BI__builtin_addcl:
5838 case Builtin::BI__builtin_addcll:
5839 IntrinsicId = Intrinsic::uadd_with_overflow;
5840 break;
5841 case Builtin::BI__builtin_subcb:
5842 case Builtin::BI__builtin_subcs:
5843 case Builtin::BI__builtin_subc:
5844 case Builtin::BI__builtin_subcl:
5845 case Builtin::BI__builtin_subcll:
5846 IntrinsicId = Intrinsic::usub_with_overflow;
5847 break;
5848 }
5849
5850 // Construct our resulting LLVM IR expression.
5851 llvm::Value *Carry1;
5852 llvm::Value *Sum1 = EmitOverflowIntrinsic(CGF&: *this, IntrinsicID: IntrinsicId,
5853 X, Y, Carry&: Carry1);
5854 llvm::Value *Carry2;
5855 llvm::Value *Sum2 = EmitOverflowIntrinsic(CGF&: *this, IntrinsicID: IntrinsicId,
5856 X: Sum1, Y: Carryin, Carry&: Carry2);
5857 llvm::Value *CarryOut = Builder.CreateZExt(V: Builder.CreateOr(LHS: Carry1, RHS: Carry2),
5858 DestTy: X->getType());
5859 Builder.CreateStore(Val: CarryOut, Addr: CarryOutPtr);
5860 return RValue::get(V: Sum2);
5861 }
5862
5863 case Builtin::BI__builtin_add_overflow:
5864 case Builtin::BI__builtin_sub_overflow:
5865 case Builtin::BI__builtin_mul_overflow: {
5866 const clang::Expr *LeftArg = E->getArg(Arg: 0);
5867 const clang::Expr *RightArg = E->getArg(Arg: 1);
5868 const clang::Expr *ResultArg = E->getArg(Arg: 2);
5869
5870 clang::QualType ResultQTy =
5871 ResultArg->getType()->castAs<PointerType>()->getPointeeType();
5872
5873 WidthAndSignedness LeftInfo =
5874 getIntegerWidthAndSignedness(context: CGM.getContext(), Type: LeftArg->getType());
5875 WidthAndSignedness RightInfo =
5876 getIntegerWidthAndSignedness(context: CGM.getContext(), Type: RightArg->getType());
5877 WidthAndSignedness ResultInfo =
5878 getIntegerWidthAndSignedness(context: CGM.getContext(), Type: ResultQTy);
5879
5880 // Handle mixed-sign multiplication as a special case, because adding
5881 // runtime or backend support for our generic irgen would be too expensive.
5882 if (isSpecialMixedSignMultiply(BuiltinID, Op1Info: LeftInfo, Op2Info: RightInfo, ResultInfo))
5883 return EmitCheckedMixedSignMultiply(CGF&: *this, Op1: LeftArg, Op1Info: LeftInfo, Op2: RightArg,
5884 Op2Info: RightInfo, ResultArg, ResultQTy,
5885 ResultInfo);
5886
5887 if (isSpecialUnsignedMultiplySignedResult(BuiltinID, Op1Info: LeftInfo, Op2Info: RightInfo,
5888 ResultInfo))
5889 return EmitCheckedUnsignedMultiplySignedResult(
5890 CGF&: *this, Op1: LeftArg, Op1Info: LeftInfo, Op2: RightArg, Op2Info: RightInfo, ResultArg, ResultQTy,
5891 ResultInfo);
5892
5893 WidthAndSignedness EncompassingInfo =
5894 EncompassingIntegerType(Types: {LeftInfo, RightInfo, ResultInfo});
5895
5896 llvm::Type *EncompassingLLVMTy =
5897 llvm::IntegerType::get(C&: CGM.getLLVMContext(), NumBits: EncompassingInfo.Width);
5898
5899 llvm::Type *ResultLLVMTy = CGM.getTypes().ConvertType(T: ResultQTy);
5900
5901 Intrinsic::ID IntrinsicId;
5902 switch (BuiltinID) {
5903 default:
5904 llvm_unreachable("Unknown overflow builtin id.");
5905 case Builtin::BI__builtin_add_overflow:
5906 IntrinsicId = EncompassingInfo.Signed ? Intrinsic::sadd_with_overflow
5907 : Intrinsic::uadd_with_overflow;
5908 break;
5909 case Builtin::BI__builtin_sub_overflow:
5910 IntrinsicId = EncompassingInfo.Signed ? Intrinsic::ssub_with_overflow
5911 : Intrinsic::usub_with_overflow;
5912 break;
5913 case Builtin::BI__builtin_mul_overflow:
5914 IntrinsicId = EncompassingInfo.Signed ? Intrinsic::smul_with_overflow
5915 : Intrinsic::umul_with_overflow;
5916 break;
5917 }
5918
5919 llvm::Value *Left = EmitScalarExpr(E: LeftArg);
5920 llvm::Value *Right = EmitScalarExpr(E: RightArg);
5921 Address ResultPtr = EmitPointerWithAlignment(Addr: ResultArg);
5922
5923 // Extend each operand to the encompassing type.
5924 Left = Builder.CreateIntCast(V: Left, DestTy: EncompassingLLVMTy, isSigned: LeftInfo.Signed);
5925 Right = Builder.CreateIntCast(V: Right, DestTy: EncompassingLLVMTy, isSigned: RightInfo.Signed);
5926
5927 // Perform the operation on the extended values.
5928 llvm::Value *Overflow, *Result;
5929 Result = EmitOverflowIntrinsic(CGF&: *this, IntrinsicID: IntrinsicId, X: Left, Y: Right, Carry&: Overflow);
5930
5931 if (EncompassingInfo.Width > ResultInfo.Width) {
5932 // The encompassing type is wider than the result type, so we need to
5933 // truncate it.
5934 llvm::Value *ResultTrunc = Builder.CreateTrunc(V: Result, DestTy: ResultLLVMTy);
5935
5936 // To see if the truncation caused an overflow, we will extend
5937 // the result and then compare it to the original result.
5938 llvm::Value *ResultTruncExt = Builder.CreateIntCast(
5939 V: ResultTrunc, DestTy: EncompassingLLVMTy, isSigned: ResultInfo.Signed);
5940 llvm::Value *TruncationOverflow =
5941 Builder.CreateICmpNE(LHS: Result, RHS: ResultTruncExt);
5942
5943 Overflow = Builder.CreateOr(LHS: Overflow, RHS: TruncationOverflow);
5944 Result = ResultTrunc;
5945 }
5946
5947 // Finally, store the result using the pointer.
5948 bool isVolatile =
5949 ResultArg->getType()->getPointeeType().isVolatileQualified();
5950 Builder.CreateStore(Val: EmitToMemory(Value: Result, Ty: ResultQTy), Addr: ResultPtr, IsVolatile: isVolatile);
5951
5952 return RValue::get(V: Overflow);
5953 }
5954
5955 case Builtin::BI__builtin_uadd_overflow:
5956 case Builtin::BI__builtin_uaddl_overflow:
5957 case Builtin::BI__builtin_uaddll_overflow:
5958 case Builtin::BI__builtin_usub_overflow:
5959 case Builtin::BI__builtin_usubl_overflow:
5960 case Builtin::BI__builtin_usubll_overflow:
5961 case Builtin::BI__builtin_umul_overflow:
5962 case Builtin::BI__builtin_umull_overflow:
5963 case Builtin::BI__builtin_umulll_overflow:
5964 case Builtin::BI__builtin_sadd_overflow:
5965 case Builtin::BI__builtin_saddl_overflow:
5966 case Builtin::BI__builtin_saddll_overflow:
5967 case Builtin::BI__builtin_ssub_overflow:
5968 case Builtin::BI__builtin_ssubl_overflow:
5969 case Builtin::BI__builtin_ssubll_overflow:
5970 case Builtin::BI__builtin_smul_overflow:
5971 case Builtin::BI__builtin_smull_overflow:
5972 case Builtin::BI__builtin_smulll_overflow: {
5973
5974 // We translate all of these builtins directly to the relevant llvm IR node.
5975
5976 // Scalarize our inputs.
5977 llvm::Value *X = EmitScalarExpr(E: E->getArg(Arg: 0));
5978 llvm::Value *Y = EmitScalarExpr(E: E->getArg(Arg: 1));
5979 Address SumOutPtr = EmitPointerWithAlignment(Addr: E->getArg(Arg: 2));
5980
5981 // Decide which of the overflow intrinsics we are lowering to:
5982 Intrinsic::ID IntrinsicId;
5983 switch (BuiltinID) {
5984 default: llvm_unreachable("Unknown overflow builtin id.");
5985 case Builtin::BI__builtin_uadd_overflow:
5986 case Builtin::BI__builtin_uaddl_overflow:
5987 case Builtin::BI__builtin_uaddll_overflow:
5988 IntrinsicId = Intrinsic::uadd_with_overflow;
5989 break;
5990 case Builtin::BI__builtin_usub_overflow:
5991 case Builtin::BI__builtin_usubl_overflow:
5992 case Builtin::BI__builtin_usubll_overflow:
5993 IntrinsicId = Intrinsic::usub_with_overflow;
5994 break;
5995 case Builtin::BI__builtin_umul_overflow:
5996 case Builtin::BI__builtin_umull_overflow:
5997 case Builtin::BI__builtin_umulll_overflow:
5998 IntrinsicId = Intrinsic::umul_with_overflow;
5999 break;
6000 case Builtin::BI__builtin_sadd_overflow:
6001 case Builtin::BI__builtin_saddl_overflow:
6002 case Builtin::BI__builtin_saddll_overflow:
6003 IntrinsicId = Intrinsic::sadd_with_overflow;
6004 break;
6005 case Builtin::BI__builtin_ssub_overflow:
6006 case Builtin::BI__builtin_ssubl_overflow:
6007 case Builtin::BI__builtin_ssubll_overflow:
6008 IntrinsicId = Intrinsic::ssub_with_overflow;
6009 break;
6010 case Builtin::BI__builtin_smul_overflow:
6011 case Builtin::BI__builtin_smull_overflow:
6012 case Builtin::BI__builtin_smulll_overflow:
6013 IntrinsicId = Intrinsic::smul_with_overflow;
6014 break;
6015 }
6016
6017
6018 llvm::Value *Carry;
6019 llvm::Value *Sum = EmitOverflowIntrinsic(CGF&: *this, IntrinsicID: IntrinsicId, X, Y, Carry);
6020 Builder.CreateStore(Val: Sum, Addr: SumOutPtr);
6021
6022 return RValue::get(V: Carry);
6023 }
6024 case Builtin::BIaddressof:
6025 case Builtin::BI__addressof:
6026 case Builtin::BI__builtin_addressof:
6027 return RValue::get(V: EmitLValue(E: E->getArg(Arg: 0)).getPointer(CGF&: *this));
6028 case Builtin::BI__builtin_function_start:
6029 return RValue::get(V: CGM.GetFunctionStart(
6030 Decl: E->getArg(Arg: 0)->getAsBuiltinConstantDeclRef(Context: CGM.getContext())));
6031 case Builtin::BI__builtin_operator_new:
6032 return EmitBuiltinNewDeleteCall(
6033 Type: E->getCallee()->getType()->castAs<FunctionProtoType>(), TheCallExpr: E, IsDelete: false);
6034 case Builtin::BI__builtin_operator_delete:
6035 EmitBuiltinNewDeleteCall(
6036 Type: E->getCallee()->getType()->castAs<FunctionProtoType>(), TheCallExpr: E, IsDelete: true);
6037 return RValue::get(V: nullptr);
6038
6039 case Builtin::BI__builtin_is_aligned:
6040 return EmitBuiltinIsAligned(E);
6041 case Builtin::BI__builtin_align_up:
6042 return EmitBuiltinAlignTo(E, AlignUp: true);
6043 case Builtin::BI__builtin_align_down:
6044 return EmitBuiltinAlignTo(E, AlignUp: false);
6045
6046 case Builtin::BI__noop:
6047 // __noop always evaluates to an integer literal zero.
6048 return RValue::get(V: ConstantInt::get(Ty: IntTy, V: 0));
6049 case Builtin::BI__builtin_call_with_static_chain: {
6050 const CallExpr *Call = cast<CallExpr>(Val: E->getArg(Arg: 0));
6051 const Expr *Chain = E->getArg(Arg: 1);
6052 return EmitCall(FnType: Call->getCallee()->getType(),
6053 Callee: EmitCallee(E: Call->getCallee()), E: Call, ReturnValue,
6054 Chain: EmitScalarExpr(E: Chain));
6055 }
6056 case Builtin::BI_InterlockedExchange8:
6057 case Builtin::BI_InterlockedExchange16:
6058 case Builtin::BI_InterlockedExchange:
6059 case Builtin::BI_InterlockedExchangePointer:
6060 return RValue::get(
6061 V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::_InterlockedExchange, E));
6062 case Builtin::BI_InterlockedCompareExchangePointer:
6063 return RValue::get(
6064 V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::_InterlockedCompareExchange, E));
6065 case Builtin::BI_InterlockedCompareExchangePointer_nf:
6066 return RValue::get(
6067 V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::_InterlockedCompareExchange_nf, E));
6068 case Builtin::BI_InterlockedCompareExchange8:
6069 case Builtin::BI_InterlockedCompareExchange16:
6070 case Builtin::BI_InterlockedCompareExchange:
6071 case Builtin::BI_InterlockedCompareExchange64:
6072 return RValue::get(V: EmitAtomicCmpXchgForMSIntrin(CGF&: *this, E));
6073 case Builtin::BI_InterlockedIncrement16:
6074 case Builtin::BI_InterlockedIncrement:
6075 return RValue::get(
6076 V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::_InterlockedIncrement, E));
6077 case Builtin::BI_InterlockedDecrement16:
6078 case Builtin::BI_InterlockedDecrement:
6079 return RValue::get(
6080 V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::_InterlockedDecrement, E));
6081 case Builtin::BI_InterlockedAnd8:
6082 case Builtin::BI_InterlockedAnd16:
6083 case Builtin::BI_InterlockedAnd:
6084 return RValue::get(V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::_InterlockedAnd, E));
6085 case Builtin::BI_InterlockedExchangeAdd8:
6086 case Builtin::BI_InterlockedExchangeAdd16:
6087 case Builtin::BI_InterlockedExchangeAdd:
6088 return RValue::get(
6089 V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::_InterlockedExchangeAdd, E));
6090 case Builtin::BI_InterlockedExchangeSub8:
6091 case Builtin::BI_InterlockedExchangeSub16:
6092 case Builtin::BI_InterlockedExchangeSub:
6093 return RValue::get(
6094 V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::_InterlockedExchangeSub, E));
6095 case Builtin::BI_InterlockedOr8:
6096 case Builtin::BI_InterlockedOr16:
6097 case Builtin::BI_InterlockedOr:
6098 return RValue::get(V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::_InterlockedOr, E));
6099 case Builtin::BI_InterlockedXor8:
6100 case Builtin::BI_InterlockedXor16:
6101 case Builtin::BI_InterlockedXor:
6102 return RValue::get(V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::_InterlockedXor, E));
6103
6104 case Builtin::BI_bittest64:
6105 case Builtin::BI_bittest:
6106 case Builtin::BI_bittestandcomplement64:
6107 case Builtin::BI_bittestandcomplement:
6108 case Builtin::BI_bittestandreset64:
6109 case Builtin::BI_bittestandreset:
6110 case Builtin::BI_bittestandset64:
6111 case Builtin::BI_bittestandset:
6112 case Builtin::BI_interlockedbittestandreset:
6113 case Builtin::BI_interlockedbittestandreset64:
6114 case Builtin::BI_interlockedbittestandreset64_acq:
6115 case Builtin::BI_interlockedbittestandreset64_rel:
6116 case Builtin::BI_interlockedbittestandreset64_nf:
6117 case Builtin::BI_interlockedbittestandset64:
6118 case Builtin::BI_interlockedbittestandset64_acq:
6119 case Builtin::BI_interlockedbittestandset64_rel:
6120 case Builtin::BI_interlockedbittestandset64_nf:
6121 case Builtin::BI_interlockedbittestandset:
6122 case Builtin::BI_interlockedbittestandset_acq:
6123 case Builtin::BI_interlockedbittestandset_rel:
6124 case Builtin::BI_interlockedbittestandset_nf:
6125 case Builtin::BI_interlockedbittestandreset_acq:
6126 case Builtin::BI_interlockedbittestandreset_rel:
6127 case Builtin::BI_interlockedbittestandreset_nf:
6128 return RValue::get(V: EmitBitTestIntrinsic(CGF&: *this, BuiltinID, E));
6129
6130 // These builtins exist to emit regular volatile loads and stores not
6131 // affected by the -fms-volatile setting.
6132 case Builtin::BI__iso_volatile_load8:
6133 case Builtin::BI__iso_volatile_load16:
6134 case Builtin::BI__iso_volatile_load32:
6135 case Builtin::BI__iso_volatile_load64:
6136 return RValue::get(V: EmitISOVolatileLoad(CGF&: *this, E));
6137 case Builtin::BI__iso_volatile_store8:
6138 case Builtin::BI__iso_volatile_store16:
6139 case Builtin::BI__iso_volatile_store32:
6140 case Builtin::BI__iso_volatile_store64:
6141 return RValue::get(V: EmitISOVolatileStore(CGF&: *this, E));
6142
6143 case Builtin::BI__builtin_ptrauth_sign_constant:
6144 return RValue::get(V: ConstantEmitter(*this).emitAbstract(E, T: E->getType()));
6145
6146 case Builtin::BI__builtin_ptrauth_auth:
6147 case Builtin::BI__builtin_ptrauth_auth_and_resign:
6148 case Builtin::BI__builtin_ptrauth_auth_with_pc_and_resign:
6149 case Builtin::BI__builtin_ptrauth_auth_load_relative_and_sign:
6150 case Builtin::BI__builtin_ptrauth_blend_discriminator:
6151 case Builtin::BI__builtin_ptrauth_sign_generic_data:
6152 case Builtin::BI__builtin_ptrauth_sign_unauthenticated:
6153 case Builtin::BI__builtin_ptrauth_strip: {
6154 // Emit the arguments.
6155 SmallVector<llvm::Value *, 6> Args;
6156 for (auto argExpr : E->arguments())
6157 Args.push_back(Elt: EmitScalarExpr(E: argExpr));
6158
6159 // Cast the value to intptr_t, saving its original type.
6160 llvm::Type *OrigValueType = Args[0]->getType();
6161 if (OrigValueType->isPointerTy())
6162 Args[0] = Builder.CreatePtrToInt(V: Args[0], DestTy: IntPtrTy);
6163
6164 switch (BuiltinID) {
6165 case Builtin::BI__builtin_ptrauth_auth_with_pc_and_resign:
6166 // Convert oldDiscriminator (arg 2), oldPC (arg 3) and newDiscriminator
6167 // (arg 5) to intptr_t
6168 if (Args[2]->getType()->isPointerTy())
6169 Args[2] = Builder.CreatePtrToInt(V: Args[2], DestTy: IntPtrTy);
6170 if (Args[3]->getType()->isPointerTy())
6171 Args[3] = Builder.CreatePtrToInt(V: Args[3], DestTy: IntPtrTy);
6172 if (Args[5]->getType()->isPointerTy())
6173 Args[5] = Builder.CreatePtrToInt(V: Args[5], DestTy: IntPtrTy);
6174 break;
6175
6176 case Builtin::BI__builtin_ptrauth_auth_and_resign:
6177 case Builtin::BI__builtin_ptrauth_auth_load_relative_and_sign:
6178 if (Args[4]->getType()->isPointerTy())
6179 Args[4] = Builder.CreatePtrToInt(V: Args[4], DestTy: IntPtrTy);
6180 [[fallthrough]];
6181
6182 case Builtin::BI__builtin_ptrauth_auth:
6183 case Builtin::BI__builtin_ptrauth_sign_unauthenticated:
6184 if (Args[2]->getType()->isPointerTy())
6185 Args[2] = Builder.CreatePtrToInt(V: Args[2], DestTy: IntPtrTy);
6186 break;
6187
6188 case Builtin::BI__builtin_ptrauth_sign_generic_data:
6189 if (Args[1]->getType()->isPointerTy())
6190 Args[1] = Builder.CreatePtrToInt(V: Args[1], DestTy: IntPtrTy);
6191 break;
6192
6193 case Builtin::BI__builtin_ptrauth_blend_discriminator:
6194 case Builtin::BI__builtin_ptrauth_strip:
6195 break;
6196 }
6197
6198 // Call the intrinsic.
6199 auto IntrinsicID = [&]() -> unsigned {
6200 switch (BuiltinID) {
6201 case Builtin::BI__builtin_ptrauth_auth:
6202 return Intrinsic::ptrauth_auth;
6203 case Builtin::BI__builtin_ptrauth_auth_and_resign:
6204 return Intrinsic::ptrauth_resign;
6205 case Builtin::BI__builtin_ptrauth_auth_with_pc_and_resign:
6206 return Intrinsic::ptrauth_auth_with_pc_and_resign;
6207 case Builtin::BI__builtin_ptrauth_auth_load_relative_and_sign:
6208 return Intrinsic::ptrauth_resign_load_relative;
6209 case Builtin::BI__builtin_ptrauth_blend_discriminator:
6210 return Intrinsic::ptrauth_blend;
6211 case Builtin::BI__builtin_ptrauth_sign_generic_data:
6212 return Intrinsic::ptrauth_sign_generic;
6213 case Builtin::BI__builtin_ptrauth_sign_unauthenticated:
6214 return Intrinsic::ptrauth_sign;
6215 case Builtin::BI__builtin_ptrauth_strip:
6216 return Intrinsic::ptrauth_strip;
6217 }
6218 llvm_unreachable("bad ptrauth intrinsic");
6219 }();
6220 auto Intrinsic = CGM.getIntrinsic(IID: IntrinsicID);
6221 llvm::Value *Result = EmitRuntimeCall(callee: Intrinsic, args: Args);
6222
6223 if (BuiltinID != Builtin::BI__builtin_ptrauth_sign_generic_data &&
6224 BuiltinID != Builtin::BI__builtin_ptrauth_blend_discriminator &&
6225 OrigValueType->isPointerTy()) {
6226 Result = Builder.CreateIntToPtr(V: Result, DestTy: OrigValueType);
6227 }
6228 return RValue::get(V: Result);
6229 }
6230
6231 case Builtin::BI__builtin_get_vtable_pointer: {
6232 const Expr *Target = E->getArg(Arg: 0);
6233 QualType TargetType = Target->getType();
6234 const CXXRecordDecl *Decl = TargetType->getPointeeCXXRecordDecl();
6235 assert(Decl);
6236 auto ThisAddress = EmitPointerWithAlignment(Addr: Target);
6237 assert(ThisAddress.isValid());
6238 llvm::Value *VTablePointer =
6239 GetVTablePtr(This: ThisAddress, VTableTy: Int8PtrTy, VTableClass: Decl, AuthMode: VTableAuthMode::MustTrap);
6240 return RValue::get(V: VTablePointer);
6241 }
6242
6243 case Builtin::BI__exception_code:
6244 case Builtin::BI_exception_code:
6245 return RValue::get(V: EmitSEHExceptionCode());
6246 case Builtin::BI__exception_info:
6247 case Builtin::BI_exception_info:
6248 return RValue::get(V: EmitSEHExceptionInfo());
6249 case Builtin::BI__abnormal_termination:
6250 case Builtin::BI_abnormal_termination:
6251 return RValue::get(V: EmitSEHAbnormalTermination());
6252 case Builtin::BI_setjmpex:
6253 if (getTarget().getTriple().isOSMSVCRT() && E->getNumArgs() == 1 &&
6254 E->getArg(Arg: 0)->getType()->isPointerType())
6255 return EmitMSVCRTSetJmp(CGF&: *this, SJKind: MSVCSetJmpKind::_setjmpex, E);
6256 break;
6257 case Builtin::BI_setjmp:
6258 if (getTarget().getTriple().isOSMSVCRT() && E->getNumArgs() == 1 &&
6259 E->getArg(Arg: 0)->getType()->isPointerType()) {
6260 if (getTarget().getTriple().getArch() == llvm::Triple::x86)
6261 return EmitMSVCRTSetJmp(CGF&: *this, SJKind: MSVCSetJmpKind::_setjmp3, E);
6262 else if (getTarget().getTriple().getArch() == llvm::Triple::aarch64)
6263 return EmitMSVCRTSetJmp(CGF&: *this, SJKind: MSVCSetJmpKind::_setjmpex, E);
6264 return EmitMSVCRTSetJmp(CGF&: *this, SJKind: MSVCSetJmpKind::_setjmp, E);
6265 }
6266 break;
6267
6268 // C++ std:: builtins.
6269 case Builtin::BImove:
6270 case Builtin::BImove_if_noexcept:
6271 case Builtin::BIforward:
6272 case Builtin::BIforward_like:
6273 case Builtin::BIas_const:
6274 return RValue::get(V: EmitLValue(E: E->getArg(Arg: 0)).getPointer(CGF&: *this));
6275 case Builtin::BI__GetExceptionInfo: {
6276 if (llvm::GlobalVariable *GV =
6277 CGM.getCXXABI().getThrowInfo(T: FD->getParamDecl(i: 0)->getType()))
6278 return RValue::get(V: GV);
6279 break;
6280 }
6281
6282 case Builtin::BI__fastfail:
6283 return RValue::get(V: EmitMSVCBuiltinExpr(BuiltinID: MSVCIntrin::__fastfail, E));
6284
6285 case Builtin::BI__builtin_coro_id:
6286 return EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_id);
6287 case Builtin::BI__builtin_coro_promise:
6288 return EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_promise);
6289 case Builtin::BI__builtin_coro_resume:
6290 EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_resume);
6291 return RValue::get(V: nullptr);
6292 case Builtin::BI__builtin_coro_frame:
6293 return EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_frame);
6294 case Builtin::BI__builtin_coro_noop:
6295 return EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_noop);
6296 case Builtin::BI__builtin_coro_free:
6297 return EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_free);
6298 case Builtin::BI__builtin_coro_destroy:
6299 EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_destroy);
6300 return RValue::get(V: nullptr);
6301 case Builtin::BI__builtin_coro_done:
6302 return EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_done);
6303 case Builtin::BI__builtin_coro_alloc:
6304 return EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_alloc);
6305 case Builtin::BI__builtin_coro_begin:
6306 return EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_begin);
6307 case Builtin::BI__builtin_coro_end:
6308 return EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_end);
6309 case Builtin::BI__builtin_coro_suspend:
6310 return EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_suspend);
6311 case Builtin::BI__builtin_coro_size:
6312 return EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_size);
6313 case Builtin::BI__builtin_coro_align:
6314 return EmitCoroutineIntrinsic(E, IID: Intrinsic::coro_align);
6315
6316 // OpenCL v2.0 s6.13.16.2, Built-in pipe read and write functions
6317 case Builtin::BIread_pipe:
6318 case Builtin::BIwrite_pipe: {
6319 Value *Arg0 = EmitScalarExpr(E: E->getArg(Arg: 0)),
6320 *Arg1 = EmitScalarExpr(E: E->getArg(Arg: 1));
6321 CGOpenCLRuntime OpenCLRT(CGM);
6322 Value *PacketSize = OpenCLRT.getPipeElemSize(PipeArg: E->getArg(Arg: 0));
6323 Value *PacketAlign = OpenCLRT.getPipeElemAlign(PipeArg: E->getArg(Arg: 0));
6324
6325 // Type of the generic packet parameter.
6326 unsigned GenericAS =
6327 getContext().getTargetAddressSpace(AS: LangAS::opencl_generic);
6328 llvm::Type *I8PTy = llvm::PointerType::get(C&: getLLVMContext(), AddressSpace: GenericAS);
6329
6330 // Testing which overloaded version we should generate the call for.
6331 if (2U == E->getNumArgs()) {
6332 const char *Name = (BuiltinID == Builtin::BIread_pipe) ? "__read_pipe_2"
6333 : "__write_pipe_2";
6334 // Creating a generic function type to be able to call with any builtin or
6335 // user defined type.
6336 llvm::Type *ArgTys[] = {Arg0->getType(), I8PTy, Int32Ty, Int32Ty};
6337 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: Int32Ty, Params: ArgTys, isVarArg: false);
6338 Value *ACast = Builder.CreateAddrSpaceCast(V: Arg1, DestTy: I8PTy);
6339 return RValue::get(
6340 V: EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(Ty: FTy, Name),
6341 args: {Arg0, ACast, PacketSize, PacketAlign}));
6342 } else {
6343 assert(4 == E->getNumArgs() &&
6344 "Illegal number of parameters to pipe function");
6345 const char *Name = (BuiltinID == Builtin::BIread_pipe) ? "__read_pipe_4"
6346 : "__write_pipe_4";
6347
6348 llvm::Type *ArgTys[] = {Arg0->getType(), Arg1->getType(), Int32Ty, I8PTy,
6349 Int32Ty, Int32Ty};
6350 Value *Arg2 = EmitScalarExpr(E: E->getArg(Arg: 2)),
6351 *Arg3 = EmitScalarExpr(E: E->getArg(Arg: 3));
6352 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: Int32Ty, Params: ArgTys, isVarArg: false);
6353 Value *ACast = Builder.CreateAddrSpaceCast(V: Arg3, DestTy: I8PTy);
6354 // We know the third argument is an integer type, but we may need to cast
6355 // it to i32.
6356 if (Arg2->getType() != Int32Ty)
6357 Arg2 = Builder.CreateZExtOrTrunc(V: Arg2, DestTy: Int32Ty);
6358 return RValue::get(
6359 V: EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(Ty: FTy, Name),
6360 args: {Arg0, Arg1, Arg2, ACast, PacketSize, PacketAlign}));
6361 }
6362 }
6363 // OpenCL v2.0 s6.13.16 ,s9.17.3.5 - Built-in pipe reserve read and write
6364 // functions
6365 case Builtin::BIreserve_read_pipe:
6366 case Builtin::BIreserve_write_pipe:
6367 case Builtin::BIwork_group_reserve_read_pipe:
6368 case Builtin::BIwork_group_reserve_write_pipe:
6369 case Builtin::BIsub_group_reserve_read_pipe:
6370 case Builtin::BIsub_group_reserve_write_pipe: {
6371 // Composing the mangled name for the function.
6372 const char *Name;
6373 if (BuiltinID == Builtin::BIreserve_read_pipe)
6374 Name = "__reserve_read_pipe";
6375 else if (BuiltinID == Builtin::BIreserve_write_pipe)
6376 Name = "__reserve_write_pipe";
6377 else if (BuiltinID == Builtin::BIwork_group_reserve_read_pipe)
6378 Name = "__work_group_reserve_read_pipe";
6379 else if (BuiltinID == Builtin::BIwork_group_reserve_write_pipe)
6380 Name = "__work_group_reserve_write_pipe";
6381 else if (BuiltinID == Builtin::BIsub_group_reserve_read_pipe)
6382 Name = "__sub_group_reserve_read_pipe";
6383 else
6384 Name = "__sub_group_reserve_write_pipe";
6385
6386 Value *Arg0 = EmitScalarExpr(E: E->getArg(Arg: 0)),
6387 *Arg1 = EmitScalarExpr(E: E->getArg(Arg: 1));
6388 llvm::Type *ReservedIDTy = ConvertType(T: getContext().OCLReserveIDTy);
6389 CGOpenCLRuntime OpenCLRT(CGM);
6390 Value *PacketSize = OpenCLRT.getPipeElemSize(PipeArg: E->getArg(Arg: 0));
6391 Value *PacketAlign = OpenCLRT.getPipeElemAlign(PipeArg: E->getArg(Arg: 0));
6392
6393 // Building the generic function prototype.
6394 llvm::Type *ArgTys[] = {Arg0->getType(), Int32Ty, Int32Ty, Int32Ty};
6395 llvm::FunctionType *FTy =
6396 llvm::FunctionType::get(Result: ReservedIDTy, Params: ArgTys, isVarArg: false);
6397 // We know the second argument is an integer type, but we may need to cast
6398 // it to i32.
6399 if (Arg1->getType() != Int32Ty)
6400 Arg1 = Builder.CreateZExtOrTrunc(V: Arg1, DestTy: Int32Ty);
6401 return RValue::get(V: EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(Ty: FTy, Name),
6402 args: {Arg0, Arg1, PacketSize, PacketAlign}));
6403 }
6404 // OpenCL v2.0 s6.13.16, s9.17.3.5 - Built-in pipe commit read and write
6405 // functions
6406 case Builtin::BIcommit_read_pipe:
6407 case Builtin::BIcommit_write_pipe:
6408 case Builtin::BIwork_group_commit_read_pipe:
6409 case Builtin::BIwork_group_commit_write_pipe:
6410 case Builtin::BIsub_group_commit_read_pipe:
6411 case Builtin::BIsub_group_commit_write_pipe: {
6412 const char *Name;
6413 if (BuiltinID == Builtin::BIcommit_read_pipe)
6414 Name = "__commit_read_pipe";
6415 else if (BuiltinID == Builtin::BIcommit_write_pipe)
6416 Name = "__commit_write_pipe";
6417 else if (BuiltinID == Builtin::BIwork_group_commit_read_pipe)
6418 Name = "__work_group_commit_read_pipe";
6419 else if (BuiltinID == Builtin::BIwork_group_commit_write_pipe)
6420 Name = "__work_group_commit_write_pipe";
6421 else if (BuiltinID == Builtin::BIsub_group_commit_read_pipe)
6422 Name = "__sub_group_commit_read_pipe";
6423 else
6424 Name = "__sub_group_commit_write_pipe";
6425
6426 Value *Arg0 = EmitScalarExpr(E: E->getArg(Arg: 0)),
6427 *Arg1 = EmitScalarExpr(E: E->getArg(Arg: 1));
6428 CGOpenCLRuntime OpenCLRT(CGM);
6429 Value *PacketSize = OpenCLRT.getPipeElemSize(PipeArg: E->getArg(Arg: 0));
6430 Value *PacketAlign = OpenCLRT.getPipeElemAlign(PipeArg: E->getArg(Arg: 0));
6431
6432 // Building the generic function prototype.
6433 llvm::Type *ArgTys[] = {Arg0->getType(), Arg1->getType(), Int32Ty, Int32Ty};
6434 llvm::FunctionType *FTy = llvm::FunctionType::get(
6435 Result: llvm::Type::getVoidTy(C&: getLLVMContext()), Params: ArgTys, isVarArg: false);
6436
6437 return RValue::get(V: EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(Ty: FTy, Name),
6438 args: {Arg0, Arg1, PacketSize, PacketAlign}));
6439 }
6440 // OpenCL v2.0 s6.13.16.4 Built-in pipe query functions
6441 case Builtin::BIget_pipe_num_packets:
6442 case Builtin::BIget_pipe_max_packets: {
6443 const char *BaseName;
6444 const auto *PipeTy = E->getArg(Arg: 0)->getType()->castAs<PipeType>();
6445 if (BuiltinID == Builtin::BIget_pipe_num_packets)
6446 BaseName = "__get_pipe_num_packets";
6447 else
6448 BaseName = "__get_pipe_max_packets";
6449 std::string Name = std::string(BaseName) +
6450 std::string(PipeTy->isReadOnly() ? "_ro" : "_wo");
6451
6452 // Building the generic function prototype.
6453 Value *Arg0 = EmitScalarExpr(E: E->getArg(Arg: 0));
6454 CGOpenCLRuntime OpenCLRT(CGM);
6455 Value *PacketSize = OpenCLRT.getPipeElemSize(PipeArg: E->getArg(Arg: 0));
6456 Value *PacketAlign = OpenCLRT.getPipeElemAlign(PipeArg: E->getArg(Arg: 0));
6457 llvm::Type *ArgTys[] = {Arg0->getType(), Int32Ty, Int32Ty};
6458 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: Int32Ty, Params: ArgTys, isVarArg: false);
6459
6460 return RValue::get(V: EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(Ty: FTy, Name),
6461 args: {Arg0, PacketSize, PacketAlign}));
6462 }
6463
6464 // OpenCL v2.0 s6.13.9 - Address space qualifier functions.
6465 case Builtin::BIto_global:
6466 case Builtin::BIto_local:
6467 case Builtin::BIto_private: {
6468 auto Arg0 = EmitScalarExpr(E: E->getArg(Arg: 0));
6469 auto NewArgT = llvm::PointerType::get(
6470 C&: getLLVMContext(),
6471 AddressSpace: CGM.getContext().getTargetAddressSpace(AS: LangAS::opencl_generic));
6472 auto NewRetT = llvm::PointerType::get(
6473 C&: getLLVMContext(),
6474 AddressSpace: CGM.getContext().getTargetAddressSpace(
6475 AS: E->getType()->getPointeeType().getAddressSpace()));
6476 auto FTy = llvm::FunctionType::get(Result: NewRetT, Params: {NewArgT}, isVarArg: false);
6477 llvm::Value *NewArg;
6478 if (Arg0->getType()->getPointerAddressSpace() !=
6479 NewArgT->getPointerAddressSpace())
6480 NewArg = Builder.CreateAddrSpaceCast(V: Arg0, DestTy: NewArgT);
6481 else
6482 NewArg = Builder.CreateBitOrPointerCast(V: Arg0, DestTy: NewArgT);
6483 auto NewName = std::string("__") + E->getDirectCallee()->getName().str();
6484 auto NewCall =
6485 EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(Ty: FTy, Name: NewName), args: {NewArg});
6486 return RValue::get(V: Builder.CreateBitOrPointerCast(V: NewCall,
6487 DestTy: ConvertType(T: E->getType())));
6488 }
6489
6490 // OpenCL v2.0, s6.13.17 - Enqueue kernel function.
6491 // Table 6.13.17.1 specifies four overload forms of enqueue_kernel.
6492 // The code below expands the builtin call to a call to one of the following
6493 // functions that an OpenCL runtime library will have to provide:
6494 // __enqueue_kernel_basic
6495 // __enqueue_kernel_varargs
6496 // __enqueue_kernel_basic_events
6497 // __enqueue_kernel_events_varargs
6498 case Builtin::BIenqueue_kernel: {
6499 StringRef Name; // Generated function call name
6500 unsigned NumArgs = E->getNumArgs();
6501
6502 llvm::Type *QueueTy = ConvertType(T: getContext().OCLQueueTy);
6503 llvm::Type *GenericVoidPtrTy = Builder.getPtrTy(
6504 AddrSpace: getContext().getTargetAddressSpace(AS: LangAS::opencl_generic));
6505
6506 llvm::Value *Queue = EmitScalarExpr(E: E->getArg(Arg: 0));
6507 llvm::Value *Flags = EmitScalarExpr(E: E->getArg(Arg: 1));
6508 LValue NDRangeL = EmitAggExprToLValue(E: E->getArg(Arg: 2));
6509 llvm::Value *Range = NDRangeL.getAddress().emitRawPointer(CGF&: *this);
6510
6511 // FIXME: Look through the addrspacecast which may exist to the stack
6512 // temporary as a hack.
6513 //
6514 // This is hardcoding the assumed ABI of the target function. This assumes
6515 // direct passing for every argument except NDRange, which is assumed to be
6516 // byval or byref indirect passed.
6517 //
6518 // This should be fixed to query a signature from CGOpenCLRuntime, and go
6519 // through EmitCallArgs to get the correct target ABI.
6520 Range = Range->stripPointerCasts();
6521
6522 llvm::Type *RangePtrTy = Range->getType();
6523
6524 if (NumArgs == 4) {
6525 // The most basic form of the call with parameters:
6526 // queue_t, kernel_enqueue_flags_t, ndrange_t, block(void)
6527 Name = "__enqueue_kernel_basic";
6528 llvm::Type *ArgTys[] = {QueueTy, Int32Ty, RangePtrTy, GenericVoidPtrTy,
6529 GenericVoidPtrTy};
6530 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: Int32Ty, Params: ArgTys, isVarArg: false);
6531
6532 auto Info =
6533 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(CGF&: *this, E: E->getArg(Arg: 3));
6534 llvm::Value *Kernel =
6535 Builder.CreatePointerCast(V: Info.KernelHandle, DestTy: GenericVoidPtrTy);
6536 llvm::Value *Block =
6537 Builder.CreatePointerCast(V: Info.BlockArg, DestTy: GenericVoidPtrTy);
6538
6539 auto RTCall = EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(Ty: FTy, Name),
6540 args: {Queue, Flags, Range, Kernel, Block});
6541 return RValue::get(V: RTCall);
6542 }
6543 assert(NumArgs >= 5 && "Invalid enqueue_kernel signature");
6544
6545 // Create a temporary array to hold the sizes of local pointer arguments
6546 // for the block. \p First is the position of the first size argument.
6547 auto CreateArrayForSizeVar =
6548 [=](unsigned First) -> std::pair<llvm::Value *, llvm::Value *> {
6549 llvm::APInt ArraySize(32, NumArgs - First);
6550 QualType SizeArrayTy = getContext().getConstantArrayType(
6551 EltTy: getContext().getSizeType(), ArySize: ArraySize, SizeExpr: nullptr,
6552 ASM: ArraySizeModifier::Normal,
6553 /*IndexTypeQuals=*/0);
6554 auto Tmp = CreateMemTempWithoutCast(T: SizeArrayTy, Name: "block_sizes");
6555 llvm::Value *Alloca = Tmp.getPointer();
6556 llvm::Value *ElemPtr;
6557 EmitLifetimeStart(Addr: Alloca);
6558 // Each of the following arguments specifies the size of the corresponding
6559 // argument passed to the enqueued block.
6560 auto *Zero = llvm::ConstantInt::get(Ty: IntTy, V: 0);
6561 for (unsigned I = First; I < NumArgs; ++I) {
6562 auto *Index = llvm::ConstantInt::get(Ty: IntTy, V: I - First);
6563 auto *GEP =
6564 Builder.CreateGEP(Ty: Tmp.getElementType(), Ptr: Alloca, IdxList: {Zero, Index});
6565 if (I == First)
6566 ElemPtr = GEP;
6567 auto *V =
6568 Builder.CreateZExtOrTrunc(V: EmitScalarExpr(E: E->getArg(Arg: I)), DestTy: SizeTy);
6569 Builder.CreateAlignedStore(
6570 Val: V, Ptr: GEP, Align: CGM.getDataLayout().getPrefTypeAlign(Ty: SizeTy));
6571 }
6572 return {ElemPtr, Alloca};
6573 };
6574
6575 // Could have events and/or varargs.
6576 if (E->getArg(Arg: 3)->getType()->isBlockPointerType()) {
6577 // No events passed, but has variadic arguments.
6578 Name = "__enqueue_kernel_varargs";
6579 auto Info =
6580 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(CGF&: *this, E: E->getArg(Arg: 3));
6581 llvm::Value *Kernel =
6582 Builder.CreatePointerCast(V: Info.KernelHandle, DestTy: GenericVoidPtrTy);
6583 auto *Block = Builder.CreatePointerCast(V: Info.BlockArg, DestTy: GenericVoidPtrTy);
6584 auto [ElemPtr, TmpPtr] = CreateArrayForSizeVar(4);
6585
6586 // Create a vector of the arguments, as well as a constant value to
6587 // express to the runtime the number of variadic arguments.
6588 llvm::Value *const Args[] = {Queue, Flags,
6589 Range, Kernel,
6590 Block, ConstantInt::get(Ty: IntTy, V: NumArgs - 4),
6591 ElemPtr};
6592 llvm::Type *const ArgTys[] = {
6593 QueueTy, IntTy, RangePtrTy, GenericVoidPtrTy,
6594 GenericVoidPtrTy, IntTy, ElemPtr->getType()};
6595
6596 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: Int32Ty, Params: ArgTys, isVarArg: false);
6597 auto Call = RValue::get(
6598 V: EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(Ty: FTy, Name), args: Args));
6599 EmitLifetimeEnd(Addr: TmpPtr);
6600 return Call;
6601 }
6602 // Any calls now have event arguments passed.
6603 if (NumArgs >= 7) {
6604 llvm::PointerType *PtrTy = llvm::PointerType::get(
6605 C&: CGM.getLLVMContext(),
6606 AddressSpace: CGM.getContext().getTargetAddressSpace(AS: LangAS::opencl_generic));
6607
6608 llvm::Value *NumEvents =
6609 Builder.CreateZExtOrTrunc(V: EmitScalarExpr(E: E->getArg(Arg: 3)), DestTy: Int32Ty);
6610
6611 // Since SemaOpenCLBuiltinEnqueueKernel allows fifth and sixth arguments
6612 // to be a null pointer constant (including `0` literal), we can take it
6613 // into account and emit null pointer directly.
6614 llvm::Value *EventWaitList = nullptr;
6615 if (E->getArg(Arg: 4)->isNullPointerConstant(
6616 Ctx&: getContext(), NPC: Expr::NPC_ValueDependentIsNotNull)) {
6617 EventWaitList = llvm::ConstantPointerNull::get(T: PtrTy);
6618 } else {
6619 EventWaitList =
6620 E->getArg(Arg: 4)->getType()->isArrayType()
6621 ? EmitArrayToPointerDecay(Array: E->getArg(Arg: 4)).emitRawPointer(CGF&: *this)
6622 : EmitScalarExpr(E: E->getArg(Arg: 4));
6623 // Convert to generic address space.
6624 EventWaitList = Builder.CreatePointerCast(V: EventWaitList, DestTy: PtrTy);
6625 }
6626 llvm::Value *EventRet = nullptr;
6627 if (E->getArg(Arg: 5)->isNullPointerConstant(
6628 Ctx&: getContext(), NPC: Expr::NPC_ValueDependentIsNotNull)) {
6629 EventRet = llvm::ConstantPointerNull::get(T: PtrTy);
6630 } else {
6631 EventRet =
6632 Builder.CreatePointerCast(V: EmitScalarExpr(E: E->getArg(Arg: 5)), DestTy: PtrTy);
6633 }
6634
6635 auto Info =
6636 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(CGF&: *this, E: E->getArg(Arg: 6));
6637 llvm::Value *Kernel =
6638 Builder.CreatePointerCast(V: Info.KernelHandle, DestTy: GenericVoidPtrTy);
6639 llvm::Value *Block =
6640 Builder.CreatePointerCast(V: Info.BlockArg, DestTy: GenericVoidPtrTy);
6641
6642 std::vector<llvm::Type *> ArgTys = {
6643 QueueTy, Int32Ty, RangePtrTy, Int32Ty,
6644 PtrTy, PtrTy, GenericVoidPtrTy, GenericVoidPtrTy};
6645
6646 std::vector<llvm::Value *> Args = {Queue, Flags, Range,
6647 NumEvents, EventWaitList, EventRet,
6648 Kernel, Block};
6649
6650 if (NumArgs == 7) {
6651 // Has events but no variadics.
6652 Name = "__enqueue_kernel_basic_events";
6653 llvm::FunctionType *FTy =
6654 llvm::FunctionType::get(Result: Int32Ty, Params: ArgTys, isVarArg: false);
6655 return RValue::get(
6656 V: EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(Ty: FTy, Name), args: Args));
6657 }
6658 // Has event info and variadics
6659 // Pass the number of variadics to the runtime function too.
6660 Args.push_back(x: ConstantInt::get(Ty: Int32Ty, V: NumArgs - 7));
6661 ArgTys.push_back(x: Int32Ty);
6662 Name = "__enqueue_kernel_events_varargs";
6663
6664 auto [ElemPtr, TmpPtr] = CreateArrayForSizeVar(7);
6665 Args.push_back(x: ElemPtr);
6666 ArgTys.push_back(x: ElemPtr->getType());
6667
6668 llvm::FunctionType *FTy = llvm::FunctionType::get(Result: Int32Ty, Params: ArgTys, isVarArg: false);
6669 auto Call = RValue::get(
6670 V: EmitRuntimeCall(callee: CGM.CreateRuntimeFunction(Ty: FTy, Name), args: Args));
6671 EmitLifetimeEnd(Addr: TmpPtr);
6672 return Call;
6673 }
6674 llvm_unreachable("Unexpected enqueue_kernel signature");
6675 }
6676 // OpenCL v2.0 s6.13.17.6 - Kernel query functions need bitcast of block
6677 // parameter.
6678 case Builtin::BIget_kernel_work_group_size: {
6679 llvm::Type *GenericVoidPtrTy = Builder.getPtrTy(
6680 AddrSpace: getContext().getTargetAddressSpace(AS: LangAS::opencl_generic));
6681 auto Info =
6682 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(CGF&: *this, E: E->getArg(Arg: 0));
6683 Value *Kernel =
6684 Builder.CreatePointerCast(V: Info.KernelHandle, DestTy: GenericVoidPtrTy);
6685 Value *Arg = Builder.CreatePointerCast(V: Info.BlockArg, DestTy: GenericVoidPtrTy);
6686 return RValue::get(V: EmitRuntimeCall(
6687 callee: CGM.CreateRuntimeFunction(
6688 Ty: llvm::FunctionType::get(Result: IntTy, Params: {GenericVoidPtrTy, GenericVoidPtrTy},
6689 isVarArg: false),
6690 Name: "__get_kernel_work_group_size_impl"),
6691 args: {Kernel, Arg}));
6692 }
6693 case Builtin::BIget_kernel_preferred_work_group_size_multiple: {
6694 llvm::Type *GenericVoidPtrTy = Builder.getPtrTy(
6695 AddrSpace: getContext().getTargetAddressSpace(AS: LangAS::opencl_generic));
6696 auto Info =
6697 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(CGF&: *this, E: E->getArg(Arg: 0));
6698 Value *Kernel =
6699 Builder.CreatePointerCast(V: Info.KernelHandle, DestTy: GenericVoidPtrTy);
6700 Value *Arg = Builder.CreatePointerCast(V: Info.BlockArg, DestTy: GenericVoidPtrTy);
6701 return RValue::get(V: EmitRuntimeCall(
6702 callee: CGM.CreateRuntimeFunction(
6703 Ty: llvm::FunctionType::get(Result: IntTy, Params: {GenericVoidPtrTy, GenericVoidPtrTy},
6704 isVarArg: false),
6705 Name: "__get_kernel_preferred_work_group_size_multiple_impl"),
6706 args: {Kernel, Arg}));
6707 }
6708 case Builtin::BIget_kernel_max_sub_group_size_for_ndrange:
6709 case Builtin::BIget_kernel_sub_group_count_for_ndrange: {
6710 llvm::Type *GenericVoidPtrTy = Builder.getPtrTy(
6711 AddrSpace: getContext().getTargetAddressSpace(AS: LangAS::opencl_generic));
6712 LValue NDRangeL = EmitAggExprToLValue(E: E->getArg(Arg: 0));
6713 llvm::Value *NDRange = NDRangeL.getAddress().emitRawPointer(CGF&: *this);
6714 auto Info =
6715 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(CGF&: *this, E: E->getArg(Arg: 1));
6716 Value *Kernel =
6717 Builder.CreatePointerCast(V: Info.KernelHandle, DestTy: GenericVoidPtrTy);
6718 Value *Block = Builder.CreatePointerCast(V: Info.BlockArg, DestTy: GenericVoidPtrTy);
6719 const char *Name =
6720 BuiltinID == Builtin::BIget_kernel_max_sub_group_size_for_ndrange
6721 ? "__get_kernel_max_sub_group_size_for_ndrange_impl"
6722 : "__get_kernel_sub_group_count_for_ndrange_impl";
6723 return RValue::get(V: EmitRuntimeCall(
6724 callee: CGM.CreateRuntimeFunction(
6725 Ty: llvm::FunctionType::get(
6726 Result: IntTy, Params: {NDRange->getType(), GenericVoidPtrTy, GenericVoidPtrTy},
6727 isVarArg: false),
6728 Name),
6729 args: {NDRange, Kernel, Block}));
6730 }
6731 case Builtin::BI__builtin_store_half:
6732 case Builtin::BI__builtin_store_halff: {
6733 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
6734 Value *Val = EmitScalarExpr(E: E->getArg(Arg: 0));
6735 Address Address = EmitPointerWithAlignment(Addr: E->getArg(Arg: 1));
6736 Value *HalfVal = Builder.CreateFPTrunc(V: Val, DestTy: Builder.getHalfTy());
6737 Builder.CreateStore(Val: HalfVal, Addr: Address);
6738 return RValue::get(V: nullptr);
6739 }
6740 case Builtin::BI__builtin_load_half: {
6741 Address Address = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
6742 Value *HalfVal = Builder.CreateLoad(Addr: Address);
6743 return RValue::get(V: Builder.CreateFPExt(V: HalfVal, DestTy: Builder.getDoubleTy()));
6744 }
6745 case Builtin::BI__builtin_load_halff: {
6746 Address Address = EmitPointerWithAlignment(Addr: E->getArg(Arg: 0));
6747 Value *HalfVal = Builder.CreateLoad(Addr: Address);
6748 return RValue::get(V: Builder.CreateFPExt(V: HalfVal, DestTy: Builder.getFloatTy()));
6749 }
6750 case Builtin::BI__builtin_printf:
6751 case Builtin::BIprintf:
6752 if (getTarget().getTriple().isNVPTX() ||
6753 getTarget().getTriple().isAMDGCN() ||
6754 (getTarget().getTriple().isSPIRV() &&
6755 getTarget().getTriple().getVendor() == Triple::VendorType::AMD)) {
6756 if (getTarget().getTriple().isNVPTX())
6757 return EmitNVPTXDevicePrintfCallExpr(E);
6758 if ((getTarget().getTriple().isAMDGCN() ||
6759 getTarget().getTriple().isSPIRV()) &&
6760 getLangOpts().HIP)
6761 return EmitAMDGPUDevicePrintfCallExpr(E);
6762 }
6763
6764 break;
6765 case Builtin::BI__builtin_canonicalize:
6766 case Builtin::BI__builtin_canonicalizef:
6767 case Builtin::BI__builtin_canonicalizef16:
6768 case Builtin::BI__builtin_canonicalizel:
6769 return RValue::get(
6770 V: emitBuiltinWithOneOverloadedType<1>(CGF&: *this, E, IntrinsicID: Intrinsic::canonicalize));
6771
6772 case Builtin::BI__builtin_thread_pointer: {
6773 if (!getContext().getTargetInfo().isTLSSupported())
6774 CGM.ErrorUnsupported(S: E, Type: "__builtin_thread_pointer");
6775
6776 return RValue::get(V: Builder.CreateIntrinsic(ID: llvm::Intrinsic::thread_pointer,
6777 OverloadTypes: {GlobalsInt8PtrTy}, Args: {}));
6778 }
6779 case Builtin::BI__builtin_os_log_format:
6780 return emitBuiltinOSLogFormat(E: *E);
6781
6782 case Builtin::BI__xray_customevent: {
6783 if (!ShouldXRayInstrumentFunction())
6784 return RValue::getIgnored();
6785
6786 if (!CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
6787 K: XRayInstrKind::Custom))
6788 return RValue::getIgnored();
6789
6790 if (const auto *XRayAttr = CurFuncDecl->getAttr<XRayInstrumentAttr>())
6791 if (XRayAttr->neverXRayInstrument() && !AlwaysEmitXRayCustomEvents())
6792 return RValue::getIgnored();
6793
6794 Function *F = CGM.getIntrinsic(IID: Intrinsic::xray_customevent);
6795 auto FTy = F->getFunctionType();
6796 auto Arg0 = E->getArg(Arg: 0);
6797 auto Arg0Val = EmitScalarExpr(E: Arg0);
6798 auto Arg0Ty = Arg0->getType();
6799 auto PTy0 = FTy->getParamType(i: 0);
6800 if (PTy0 != Arg0Val->getType()) {
6801 if (Arg0Ty->isArrayType())
6802 Arg0Val = EmitArrayToPointerDecay(Array: Arg0).emitRawPointer(CGF&: *this);
6803 else
6804 Arg0Val = Builder.CreatePointerCast(V: Arg0Val, DestTy: PTy0);
6805 }
6806 auto Arg1 = EmitScalarExpr(E: E->getArg(Arg: 1));
6807 auto PTy1 = FTy->getParamType(i: 1);
6808 if (PTy1 != Arg1->getType())
6809 Arg1 = Builder.CreateTruncOrBitCast(V: Arg1, DestTy: PTy1);
6810 return RValue::get(V: Builder.CreateCall(Callee: F, Args: {Arg0Val, Arg1}));
6811 }
6812
6813 case Builtin::BI__xray_typedevent: {
6814 // TODO: There should be a way to always emit events even if the current
6815 // function is not instrumented. Losing events in a stream can cripple
6816 // a trace.
6817 if (!ShouldXRayInstrumentFunction())
6818 return RValue::getIgnored();
6819
6820 if (!CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
6821 K: XRayInstrKind::Typed))
6822 return RValue::getIgnored();
6823
6824 if (const auto *XRayAttr = CurFuncDecl->getAttr<XRayInstrumentAttr>())
6825 if (XRayAttr->neverXRayInstrument() && !AlwaysEmitXRayTypedEvents())
6826 return RValue::getIgnored();
6827
6828 Function *F = CGM.getIntrinsic(IID: Intrinsic::xray_typedevent);
6829 auto FTy = F->getFunctionType();
6830 auto Arg0 = EmitScalarExpr(E: E->getArg(Arg: 0));
6831 auto PTy0 = FTy->getParamType(i: 0);
6832 if (PTy0 != Arg0->getType())
6833 Arg0 = Builder.CreateTruncOrBitCast(V: Arg0, DestTy: PTy0);
6834 auto Arg1 = E->getArg(Arg: 1);
6835 auto Arg1Val = EmitScalarExpr(E: Arg1);
6836 auto Arg1Ty = Arg1->getType();
6837 auto PTy1 = FTy->getParamType(i: 1);
6838 if (PTy1 != Arg1Val->getType()) {
6839 if (Arg1Ty->isArrayType())
6840 Arg1Val = EmitArrayToPointerDecay(Array: Arg1).emitRawPointer(CGF&: *this);
6841 else
6842 Arg1Val = Builder.CreatePointerCast(V: Arg1Val, DestTy: PTy1);
6843 }
6844 auto Arg2 = EmitScalarExpr(E: E->getArg(Arg: 2));
6845 auto PTy2 = FTy->getParamType(i: 2);
6846 if (PTy2 != Arg2->getType())
6847 Arg2 = Builder.CreateTruncOrBitCast(V: Arg2, DestTy: PTy2);
6848 return RValue::get(V: Builder.CreateCall(Callee: F, Args: {Arg0, Arg1Val, Arg2}));
6849 }
6850
6851 case Builtin::BI__builtin_ms_va_start:
6852 case Builtin::BI__builtin_ms_va_end:
6853 return RValue::get(
6854 V: EmitVAStartEnd(ArgValue: EmitMSVAListRef(E: E->getArg(Arg: 0)).emitRawPointer(CGF&: *this),
6855 IsStart: BuiltinID == Builtin::BI__builtin_ms_va_start));
6856
6857 case Builtin::BI__builtin_ms_va_copy: {
6858 // Lower this manually. We can't reliably determine whether or not any
6859 // given va_copy() is for a Win64 va_list from the calling convention
6860 // alone, because it's legal to do this from a System V ABI function.
6861 // With opaque pointer types, we won't have enough information in LLVM
6862 // IR to determine this from the argument types, either. Best to do it
6863 // now, while we have enough information.
6864 Address DestAddr = EmitMSVAListRef(E: E->getArg(Arg: 0));
6865 Address SrcAddr = EmitMSVAListRef(E: E->getArg(Arg: 1));
6866
6867 DestAddr = DestAddr.withElementType(ElemTy: Int8PtrTy);
6868 SrcAddr = SrcAddr.withElementType(ElemTy: Int8PtrTy);
6869
6870 Value *ArgPtr = Builder.CreateLoad(Addr: SrcAddr, Name: "ap.val");
6871 return RValue::get(V: Builder.CreateStore(Val: ArgPtr, Addr: DestAddr));
6872 }
6873
6874 case Builtin::BI__builtin_zos_va_start:
6875 case Builtin::BI__builtin_zos_va_end: {
6876 // The va_list is an array with 2 elements, called curr and next.
6877 // Element curr is set to 0. For builtin_zos_va_start, next is initialized
6878 // with a call to @llvm.va_start. Otherwise, next is passed to @llvm.va_end.
6879 Address VAList = EmitZOSVAListRef(E: E->getArg(Arg: 0));
6880 llvm::Type *VAListTy = ConvertType(T: getContext().getBuiltinZOSVaListType());
6881 VAList = VAList.withElementType(ElemTy: VAListTy);
6882 Address Curr = Builder.CreateConstArrayGEP(Addr: VAList, Index: 0, Name: "curr");
6883 Value *Zero = llvm::Constant::getNullValue(Ty: VoidPtrTy);
6884 Builder.CreateStore(Val: Zero, Addr: Curr);
6885 Address Next = Builder.CreateConstArrayGEP(Addr: VAList, Index: 1, Name: "next");
6886 return RValue::get(
6887 V: EmitVAStartEnd(ArgValue: Next.emitRawPointer(CGF&: *this),
6888 IsStart: BuiltinID == Builtin::BI__builtin_zos_va_start));
6889 }
6890 case Builtin::BI__builtin_zos_va_copy: {
6891 // Lower this manually because later can't reliably determine the type.
6892 Address Dest = EmitZOSVAListRef(E: E->getArg(Arg: 0));
6893 Address Src = EmitZOSVAListRef(E: E->getArg(Arg: 1));
6894 llvm::Type *VAListTy = ConvertType(T: getContext().getBuiltinZOSVaListType());
6895 uint64_t SizeBytes =
6896 CGM.getDataLayout().getTypeAllocSize(Ty: VAListTy).getFixedValue();
6897 Value *SizeVal = llvm::ConstantInt::get(Ty: Int64Ty, V: SizeBytes);
6898 Builder.CreateMemCpy(Dest, Src, Size: SizeVal, IsVolatile: false);
6899 return RValue::get(V: Dest.emitRawPointer(CGF&: *this));
6900 }
6901
6902 case Builtin::BI__builtin_get_device_side_mangled_name: {
6903 auto Name = CGM.getCUDARuntime().getDeviceSideName(
6904 ND: cast<DeclRefExpr>(Val: E->getArg(Arg: 0)->IgnoreImpCasts())->getDecl());
6905 auto Str = CGM.GetAddrOfConstantCString(Str: Name, GlobalName: "");
6906 return RValue::get(V: Str.getPointer());
6907 }
6908 }
6909
6910 // If this is an alias for a lib function (e.g. __builtin_sin), emit
6911 // the call using the normal call path, but using the unmangled
6912 // version of the function name.
6913 const auto &BI = getContext().BuiltinInfo;
6914 if (!shouldEmitBuiltinAsIR(BuiltinID, BI, CGF: *this) &&
6915 BI.isLibFunction(ID: BuiltinID))
6916 return emitLibraryCall(CGF&: *this, FD, E,
6917 calleeValue: CGM.getBuiltinLibFunction(FD, BuiltinID));
6918
6919 // If this is a predefined lib function (e.g. malloc), emit the call
6920 // using exactly the normal call path.
6921 if (BI.isPredefinedLibFunction(ID: BuiltinID))
6922 return emitLibraryCall(CGF&: *this, FD, E, calleeValue: CGM.getRawFunctionPointer(GD: FD));
6923
6924 // Check that a call to a target specific builtin has the correct target
6925 // features.
6926 // This is down here to avoid non-target specific builtins, however, if
6927 // generic builtins start to require generic target features then we
6928 // can move this up to the beginning of the function.
6929 checkTargetFeatures(E, TargetDecl: FD);
6930
6931 if (unsigned VectorWidth = getContext().BuiltinInfo.getRequiredVectorWidth(ID: BuiltinID))
6932 LargestVectorWidth = std::max(a: LargestVectorWidth, b: VectorWidth);
6933
6934 // See if we have a target specific intrinsic.
6935 std::string Name = getContext().BuiltinInfo.getName(ID: BuiltinID);
6936 Intrinsic::ID IntrinsicID = Intrinsic::not_intrinsic;
6937 StringRef Prefix =
6938 llvm::Triple::getArchTypePrefix(Kind: getTarget().getTriple().getArch());
6939 if (!Prefix.empty()) {
6940 IntrinsicID = Intrinsic::getIntrinsicForClangBuiltin(TargetPrefix: Prefix.data(), BuiltinName: Name);
6941 if (IntrinsicID == Intrinsic::not_intrinsic && Prefix == "spv" &&
6942 getTarget().getTriple().getOS() == llvm::Triple::OSType::AMDHSA)
6943 IntrinsicID = Intrinsic::getIntrinsicForClangBuiltin(TargetPrefix: "amdgcn", BuiltinName: Name);
6944 // NOTE we don't need to perform a compatibility flag check here since the
6945 // intrinsics are declared in Builtins*.def via LANGBUILTIN which filter the
6946 // MS builtins via ALL_MS_LANGUAGES and are filtered earlier.
6947 if (IntrinsicID == Intrinsic::not_intrinsic)
6948 IntrinsicID = Intrinsic::getIntrinsicForMSBuiltin(TargetPrefix: Prefix.data(), BuiltinName: Name);
6949 }
6950
6951 if (IntrinsicID != Intrinsic::not_intrinsic) {
6952 SmallVector<Value*, 16> Args;
6953
6954 // Find out if any arguments are required to be integer constant
6955 // expressions.
6956 unsigned ICEArguments = 0;
6957 ASTContext::GetBuiltinTypeError Error;
6958 getContext().GetBuiltinType(ID: BuiltinID, Error, IntegerConstantArgs: &ICEArguments);
6959 assert(Error == ASTContext::GE_None && "Should not codegen an error");
6960
6961 Function *F = CGM.getIntrinsic(IID: IntrinsicID);
6962 llvm::FunctionType *FTy = F->getFunctionType();
6963
6964 for (unsigned i = 0, e = E->getNumArgs(); i != e; ++i) {
6965 Value *ArgValue = EmitScalarOrConstFoldImmArg(ICEArguments, Idx: i, E);
6966 // If the intrinsic arg type is different from the builtin arg type
6967 // we need to do a bit cast.
6968 llvm::Type *PTy = FTy->getParamType(i);
6969 if (PTy != ArgValue->getType()) {
6970 // XXX - vector of pointers?
6971 if (auto *PtrTy = dyn_cast<llvm::PointerType>(Val: PTy)) {
6972 if (PtrTy->getAddressSpace() !=
6973 ArgValue->getType()->getPointerAddressSpace()) {
6974 ArgValue = Builder.CreateAddrSpaceCast(
6975 V: ArgValue, DestTy: llvm::PointerType::get(C&: getLLVMContext(),
6976 AddressSpace: PtrTy->getAddressSpace()));
6977 }
6978 }
6979
6980 // Cast vector type (e.g., v256i32) to x86_amx, this only happen
6981 // in amx intrinsics.
6982 if (PTy->isX86_AMXTy())
6983 ArgValue = Builder.CreateIntrinsic(ID: Intrinsic::x86_cast_vector_to_tile,
6984 OverloadTypes: {ArgValue->getType()}, Args: {ArgValue});
6985 else
6986 ArgValue = Builder.CreateBitCast(V: ArgValue, DestTy: PTy);
6987 }
6988
6989 Args.push_back(Elt: ArgValue);
6990 }
6991
6992 appendDefaultIntrinsicArgs(Args, F);
6993
6994 Value *V = Builder.CreateCall(Callee: F, Args);
6995 QualType BuiltinRetType = E->getType();
6996
6997 llvm::Type *RetTy = VoidTy;
6998 if (!BuiltinRetType->isVoidType())
6999 RetTy = ConvertType(T: BuiltinRetType);
7000
7001 if (RetTy != V->getType()) {
7002 // XXX - vector of pointers?
7003 if (auto *PtrTy = dyn_cast<llvm::PointerType>(Val: RetTy)) {
7004 if (PtrTy->getAddressSpace() != V->getType()->getPointerAddressSpace()) {
7005 V = Builder.CreateAddrSpaceCast(
7006 V, DestTy: llvm::PointerType::get(C&: getLLVMContext(),
7007 AddressSpace: PtrTy->getAddressSpace()));
7008 }
7009 }
7010
7011 // Cast x86_amx to vector type (e.g., v256i32), this only happen
7012 // in amx intrinsics.
7013 if (V->getType()->isX86_AMXTy())
7014 V = Builder.CreateIntrinsic(ID: Intrinsic::x86_cast_tile_to_vector, OverloadTypes: {RetTy},
7015 Args: {V});
7016 else
7017 V = Builder.CreateBitCast(V, DestTy: RetTy);
7018 }
7019
7020 if (RetTy->isVoidTy())
7021 return RValue::get(V: nullptr);
7022
7023 return RValue::get(V);
7024 }
7025
7026 // Some target-specific builtins can have aggregate return values, e.g.
7027 // __builtin_arm_mve_vld2q_u32. So if the result is an aggregate, force
7028 // ReturnValue to be non-null, so that the target-specific emission code can
7029 // always just emit into it.
7030 TypeEvaluationKind EvalKind = getEvaluationKind(T: E->getType());
7031 if (EvalKind == TEK_Aggregate && ReturnValue.isNull()) {
7032 Address DestPtr = CreateMemTemp(T: E->getType(), Name: "agg.tmp");
7033 ReturnValue = ReturnValueSlot(DestPtr, false);
7034 }
7035
7036 // Now see if we can emit a target-specific builtin.
7037 if (Value *V = EmitTargetBuiltinExpr(BuiltinID, E, ReturnValue)) {
7038 switch (EvalKind) {
7039 case TEK_Scalar:
7040 if (V->getType()->isVoidTy())
7041 return RValue::get(V: nullptr);
7042 return RValue::get(V);
7043 case TEK_Aggregate:
7044 return RValue::getAggregate(addr: ReturnValue.getAddress(),
7045 isVolatile: ReturnValue.isVolatile());
7046 case TEK_Complex:
7047 llvm_unreachable("No current target builtin returns complex");
7048 }
7049 llvm_unreachable("Bad evaluation kind in EmitBuiltinExpr");
7050 }
7051
7052 // EmitHLSLBuiltinExpr will check getLangOpts().HLSL
7053 if (Value *V = EmitHLSLBuiltinExpr(BuiltinID, E, ReturnValue)) {
7054 switch (EvalKind) {
7055 case TEK_Scalar:
7056 if (V->getType()->isVoidTy())
7057 return RValue::get(V: nullptr);
7058 return RValue::get(V);
7059 case TEK_Aggregate:
7060 return RValue::getAggregate(addr: ReturnValue.getAddress(),
7061 isVolatile: ReturnValue.isVolatile());
7062 case TEK_Complex:
7063 llvm_unreachable("No current hlsl builtin returns complex");
7064 }
7065 llvm_unreachable("Bad evaluation kind in EmitBuiltinExpr");
7066 }
7067
7068 if (getLangOpts().HIPStdPar && getLangOpts().CUDAIsDevice)
7069 return EmitHipStdParUnsupportedBuiltin(CGF: this, FD);
7070
7071 ErrorUnsupported(S: E, Type: "builtin function");
7072
7073 // Unknown builtin, for now just dump it out and return undef.
7074 return GetUndefRValue(Ty: E->getType());
7075}
7076
7077namespace {
7078struct BuiltinAlignArgs {
7079 llvm::Value *Src = nullptr;
7080 llvm::Type *SrcType = nullptr;
7081 llvm::Value *Alignment = nullptr;
7082 llvm::Value *Mask = nullptr;
7083 llvm::IntegerType *IntType = nullptr;
7084
7085 BuiltinAlignArgs(const CallExpr *E, CodeGenFunction &CGF) {
7086 QualType AstType = E->getArg(Arg: 0)->getType();
7087 if (AstType->isArrayType())
7088 Src = CGF.EmitArrayToPointerDecay(Array: E->getArg(Arg: 0)).emitRawPointer(CGF);
7089 else
7090 Src = CGF.EmitScalarExpr(E: E->getArg(Arg: 0));
7091 SrcType = Src->getType();
7092 if (SrcType->isPointerTy()) {
7093 IntType = IntegerType::get(
7094 C&: CGF.getLLVMContext(),
7095 NumBits: CGF.CGM.getDataLayout().getIndexTypeSizeInBits(Ty: SrcType));
7096 } else {
7097 assert(SrcType->isIntegerTy());
7098 IntType = cast<llvm::IntegerType>(Val: SrcType);
7099 }
7100 Alignment = CGF.EmitScalarExpr(E: E->getArg(Arg: 1));
7101 Alignment = CGF.Builder.CreateZExtOrTrunc(V: Alignment, DestTy: IntType, Name: "alignment");
7102 auto *One = llvm::ConstantInt::get(Ty: IntType, V: 1);
7103 Mask = CGF.Builder.CreateSub(LHS: Alignment, RHS: One, Name: "mask");
7104 }
7105};
7106} // namespace
7107
7108/// Generate (x & (y-1)) == 0.
7109RValue CodeGenFunction::EmitBuiltinIsAligned(const CallExpr *E) {
7110 BuiltinAlignArgs Args(E, *this);
7111 llvm::Value *SrcAddress = Args.Src;
7112 if (Args.SrcType->isPointerTy())
7113 SrcAddress =
7114 Builder.CreateBitOrPointerCast(V: Args.Src, DestTy: Args.IntType, Name: "src_addr");
7115 return RValue::get(V: Builder.CreateICmpEQ(
7116 LHS: Builder.CreateAnd(LHS: SrcAddress, RHS: Args.Mask, Name: "set_bits"),
7117 RHS: llvm::Constant::getNullValue(Ty: Args.IntType), Name: "is_aligned"));
7118}
7119
7120/// Generate (x & ~(y-1)) to align down or ((x+(y-1)) & ~(y-1)) to align up.
7121/// Note: For pointer types we can avoid ptrtoint/inttoptr pairs by using the
7122/// llvm.ptrmask intrinsic (with a GEP before in the align_up case).
7123RValue CodeGenFunction::EmitBuiltinAlignTo(const CallExpr *E, bool AlignUp) {
7124 BuiltinAlignArgs Args(E, *this);
7125 llvm::Value *SrcForMask = Args.Src;
7126 if (AlignUp) {
7127 // When aligning up we have to first add the mask to ensure we go over the
7128 // next alignment value and then align down to the next valid multiple.
7129 // By adding the mask, we ensure that align_up on an already aligned
7130 // value will not change the value.
7131 if (Args.Src->getType()->isPointerTy()) {
7132 if (getLangOpts().PointerOverflowDefined)
7133 SrcForMask =
7134 Builder.CreateGEP(Ty: Int8Ty, Ptr: SrcForMask, IdxList: Args.Mask, Name: "over_boundary");
7135 else
7136 SrcForMask = EmitCheckedInBoundsGEP(ElemTy: Int8Ty, Ptr: SrcForMask, IdxList: Args.Mask,
7137 /*SignedIndices=*/true,
7138 /*isSubtraction=*/IsSubtraction: false,
7139 Loc: E->getExprLoc(), Name: "over_boundary");
7140 } else {
7141 SrcForMask = Builder.CreateAdd(LHS: SrcForMask, RHS: Args.Mask, Name: "over_boundary");
7142 }
7143 }
7144 // Invert the mask to only clear the lower bits.
7145 llvm::Value *InvertedMask = Builder.CreateNot(V: Args.Mask, Name: "inverted_mask");
7146 llvm::Value *Result = nullptr;
7147 if (Args.Src->getType()->isPointerTy()) {
7148 Result = Builder.CreateIntrinsic(
7149 ID: Intrinsic::ptrmask, OverloadTypes: {Args.SrcType, Args.IntType},
7150 Args: {SrcForMask, InvertedMask}, FMFSource: nullptr, Name: "aligned_result");
7151 } else {
7152 Result = Builder.CreateAnd(LHS: SrcForMask, RHS: InvertedMask, Name: "aligned_result");
7153 }
7154 assert(Result->getType() == Args.SrcType);
7155 return RValue::get(V: Result);
7156}
7157