1//===--- InterpBuiltin.cpp - Interpreter for the constexpr VM ---*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8#include "../ExprConstShared.h"
9#include "Boolean.h"
10#include "Char.h"
11#include "EvalEmitter.h"
12#include "Interp.h"
13#include "InterpBuiltinBitCast.h"
14#include "InterpHelpers.h"
15#include "PrimType.h"
16#include "Program.h"
17#include "clang/AST/ASTContext.h"
18#include "clang/AST/ExprCXX.h"
19#include "clang/AST/InferAlloc.h"
20#include "clang/AST/OSLog.h"
21#include "clang/AST/RecordLayout.h"
22#include "clang/Basic/Builtins.h"
23#include "clang/Basic/TargetBuiltins.h"
24#include "clang/Basic/TargetInfo.h"
25#include "llvm/ADT/StringExtras.h"
26#include "llvm/Support/AllocToken.h"
27#include "llvm/Support/CRC.h"
28#include "llvm/Support/ErrorHandling.h"
29#include "llvm/Support/MathExtras.h"
30#include "llvm/Support/SipHash.h"
31
32namespace clang {
33namespace interp {
34
35[[maybe_unused]] static bool isNoopBuiltin(unsigned ID) {
36 switch (ID) {
37 case Builtin::BIas_const:
38 case Builtin::BIforward:
39 case Builtin::BIforward_like:
40 case Builtin::BImove:
41 case Builtin::BImove_if_noexcept:
42 case Builtin::BIaddressof:
43 case Builtin::BI__addressof:
44 case Builtin::BI__builtin_addressof:
45 case Builtin::BI__builtin_launder:
46 return true;
47 default:
48 return false;
49 }
50 return false;
51}
52
53static void discard(InterpStack &Stk, PrimType T) {
54 TYPE_SWITCH(T, { Stk.discard<T>(); });
55}
56
57static bool popToUInt64(const InterpState &S, const Expr *E, uint64_t &Out) {
58 INT_TYPE_SWITCH(*S.getContext().classify(E->getType()), {
59 const auto &Val = S.Stk.pop<T>();
60 if (!Val.isNumber())
61 return false;
62 Out = static_cast<uint64_t>(Val);
63 return true;
64 });
65}
66
67static bool popToAPSInt(InterpStack &Stk, PrimType T, APSInt &Out) {
68 INT_TYPE_SWITCH(T, {
69 const auto &Val = Stk.pop<T>();
70 if (!Val.isNumber())
71 return false;
72 Out = Val.toAPSInt();
73 return true;
74 });
75}
76
77static bool popToAPSInt(InterpState &S, const Expr *E, APSInt &Out) {
78 return popToAPSInt(Stk&: S.Stk, T: *S.getContext().classify(T: E->getType()), Out);
79}
80static bool popToAPSInt(InterpState &S, QualType T, APSInt &Out) {
81 return popToAPSInt(Stk&: S.Stk, T: *S.getContext().classify(T), Out);
82}
83
84/// Check for common reasons a pointer can't be read from, which
85/// are usually not diagnosed in a builtin function.
86static bool isReadable(const Pointer &P) {
87 if (P.isDummy())
88 return false;
89 if (!P.isReadablePointerType())
90 return false;
91 if (!P.isLive())
92 return false;
93 if (P.isOnePastEnd())
94 return false;
95 return true;
96}
97
98/// Pushes \p Val on the stack as the type given by \p QT.
99static void pushInteger(InterpState &S, const APSInt &Val, QualType QT) {
100 assert(QT->isSignedIntegerOrEnumerationType() ||
101 QT->isUnsignedIntegerOrEnumerationType());
102 OptPrimType T = *S.getContext().classify(T: QT);
103 assert(T);
104
105 if (T == PT_IntAPS) {
106 unsigned BitWidth = S.getASTContext().getIntWidth(T: QT);
107 auto Result = S.allocAP<IntegralAP<true>>(BitWidth);
108 Result.copy(V: Val.extOrTrunc(width: BitWidth));
109 S.Stk.push<IntegralAP<true>>(Args&: Result);
110 return;
111 }
112
113 if (T == PT_IntAP) {
114 unsigned BitWidth = S.getASTContext().getIntWidth(T: QT);
115 auto Result = S.allocAP<IntegralAP<false>>(BitWidth);
116 Result.copy(V: Val.extOrTrunc(width: BitWidth));
117 S.Stk.push<IntegralAP<false>>(Args&: Result);
118 return;
119 }
120
121 if (isSignedType(T: *T)) {
122 int64_t V = Val.getSExtValue();
123 INT_TYPE_SWITCH(*T, { S.Stk.push<T>(T::from(V)); });
124 } else {
125 assert(QT->isUnsignedIntegerOrEnumerationType());
126 uint64_t V = Val.getZExtValue();
127 INT_TYPE_SWITCH(*T, { S.Stk.push<T>(T::from(V)); });
128 }
129}
130
131template <typename T>
132static void pushInteger(InterpState &S, T Val, QualType QT) {
133 if constexpr (std::is_same_v<T, APInt>)
134 pushInteger(S, Val: APSInt(Val, !std::is_signed_v<T>), QT);
135 else if constexpr (std::is_same_v<T, APSInt>)
136 pushInteger(S, Val, QT);
137 else
138 pushInteger(S,
139 Val: APSInt(APInt(sizeof(T) * 8, static_cast<uint64_t>(Val),
140 std::is_signed_v<T>),
141 !std::is_signed_v<T>),
142 QT);
143}
144
145static void assignIntegral(InterpState &S, const Pointer &Dest, PrimType ValueT,
146 const APSInt &Value) {
147
148 if (ValueT == PT_IntAPS) {
149 Dest.deref<IntegralAP<true>>() =
150 S.allocAP<IntegralAP<true>>(BitWidth: Value.getBitWidth());
151 Dest.deref<IntegralAP<true>>().copy(V: Value);
152 } else if (ValueT == PT_IntAP) {
153 Dest.deref<IntegralAP<false>>() =
154 S.allocAP<IntegralAP<false>>(BitWidth: Value.getBitWidth());
155 Dest.deref<IntegralAP<false>>().copy(V: Value);
156 } else if (ValueT == PT_Bool) {
157 Dest.deref<Boolean>() = Boolean::from(Value: !Value.isZero());
158 } else {
159 INT_TYPE_SWITCH_NO_BOOL(
160 ValueT, { Dest.deref<T>() = T::from(static_cast<T>(Value)); });
161 }
162}
163
164static QualType getElemType(const Pointer &P) {
165 if (P.isStringPointer()) {
166 return P.asStringPointer()
167 .getLiteral()
168 ->getType()
169 ->getAsArrayTypeUnsafe()
170 ->getElementType();
171 }
172
173 if (P.isOpaquePointer() || P.isIntegralPointer())
174 return P.getType();
175
176 const Descriptor *Desc = P.getFieldDesc();
177 QualType T = Desc->getType();
178 if (Desc->isPrimitive())
179 return T;
180 if (T->isPointerType())
181 return T->castAs<PointerType>()->getPointeeType();
182 if (Desc->isArray())
183 return Desc->getElemQualType();
184 if (const auto *AT = T->getAsArrayTypeUnsafe())
185 return AT->getElementType();
186 return T;
187}
188
189static void diagnoseNonConstexprBuiltin(InterpState &S, CodePtr OpPC,
190 unsigned ID) {
191 if (!S.diagnosing())
192 return;
193
194 auto Loc = S.Current->getSource(PC: OpPC);
195 if (S.getLangOpts().CPlusPlus11)
196 S.CCEDiag(SI: Loc, DiagId: diag::note_constexpr_invalid_function)
197 << /*isConstexpr=*/0 << /*isConstructor=*/0
198 << S.getASTContext().BuiltinInfo.getQuotedName(ID);
199 else
200 S.CCEDiag(SI: Loc, DiagId: diag::note_invalid_subexpr_in_const_expr);
201}
202
203static bool diagnoseUnreadableObject(InterpState &S, CodePtr OpPC,
204 const Pointer &Ptr, AccessKinds AK) {
205 S.FFDiag(SI: S.Current->getSource(PC: OpPC),
206 DiagId: diag::note_constexpr_access_unreadable_object)
207 << AK << Ptr.toDiagnosticString(Ctx: S.getASTContext());
208 noteValueLocation(S, Ptr);
209 return false;
210}
211
212static llvm::APSInt convertBoolVectorToInt(const Pointer &Val) {
213 assert(Val.getFieldDesc()->isPrimitiveArray() &&
214 Val.getFieldDesc()->getElemQualType()->isBooleanType() &&
215 "Not a boolean vector");
216 unsigned NumElems = Val.getNumElems();
217
218 // Each element is one bit, so create an integer with NumElts bits.
219 llvm::APSInt Result(NumElems, 0);
220 for (unsigned I = 0; I != NumElems; ++I) {
221 if (Val.elem<bool>(I))
222 Result.setBit(I);
223 }
224
225 return Result;
226}
227
228// Strict double -> float conversion used for X86 PD2PS/cvtsd2ss intrinsics.
229// Reject NaN/Inf/Subnormal inputs and any lossy/inexact conversions.
230static bool convertDoubleToFloatStrict(const APFloat &Src, Floating &Dst,
231 InterpState &S, const Expr *DiagExpr) {
232 if (Src.isInfinity()) {
233 if (S.diagnosing())
234 S.CCEDiag(E: DiagExpr, DiagId: diag::note_constexpr_float_arithmetic) << 0;
235 return false;
236 }
237 if (Src.isNaN()) {
238 if (S.diagnosing())
239 S.CCEDiag(E: DiagExpr, DiagId: diag::note_constexpr_float_arithmetic) << 1;
240 return false;
241 }
242 APFloat Val = Src;
243 bool LosesInfo = false;
244 APFloat::opStatus Status = Val.convert(
245 ToSemantics: APFloat::IEEEsingle(), RM: APFloat::rmNearestTiesToEven, losesInfo: &LosesInfo);
246 if (LosesInfo || Val.isDenormal()) {
247 if (S.diagnosing())
248 S.CCEDiag(E: DiagExpr, DiagId: diag::note_constexpr_float_arithmetic_strict);
249 return false;
250 }
251 if (Status != APFloat::opOK) {
252 if (S.diagnosing())
253 S.CCEDiag(E: DiagExpr, DiagId: diag::note_invalid_subexpr_in_const_expr);
254 return false;
255 }
256 Dst.copy(F: Val);
257 return true;
258}
259
260static bool interp__builtin_is_constant_evaluated(InterpState &S, CodePtr OpPC,
261 const InterpFrame *Frame,
262 const CallExpr *Call) {
263 unsigned Depth = S.FrameDepth;
264 auto isStdCall = [](const FunctionDecl *F) -> bool {
265 return F && F->isInStdNamespace() && F->getIdentifier() &&
266 F->getIdentifier()->isStr(Str: "is_constant_evaluated");
267 };
268 const InterpFrame *Caller = Frame->Caller;
269 // The current frame is the one for __builtin_is_constant_evaluated.
270 // The one above that, potentially the one for std::is_constant_evaluated().
271 if (S.inConstantContext() && !S.checkingPotentialConstantExpression() &&
272 S.getEvalStatus().Diag &&
273 (Depth == 0 || (Depth == 1 && isStdCall(Frame->getCallee())))) {
274 if (Caller && isStdCall(Frame->getCallee())) {
275 const Expr *E = Caller->getExpr(PC: Caller->getRetPC());
276 S.report(Loc: E->getExprLoc(),
277 DiagId: diag::warn_is_constant_evaluated_always_true_constexpr)
278 << "std::is_constant_evaluated" << E->getSourceRange();
279 } else {
280 S.report(Loc: Call->getExprLoc(),
281 DiagId: diag::warn_is_constant_evaluated_always_true_constexpr)
282 << "__builtin_is_constant_evaluated" << Call->getSourceRange();
283 }
284 }
285
286 S.Stk.push<Boolean>(Args: Boolean::from(Value: S.inConstantContext()));
287 return true;
288}
289
290// __builtin_assume
291// __assume (MS extension)
292static bool interp__builtin_assume(InterpState &S, CodePtr OpPC,
293 const InterpFrame *Frame,
294 const CallExpr *Call) {
295 // Nothing to be done here since the argument is NOT evaluated.
296 assert(Call->getNumArgs() == 1);
297 return true;
298}
299
300static bool interp__builtin_strcmp(InterpState &S, CodePtr OpPC,
301 const InterpFrame *Frame,
302 const CallExpr *Call, unsigned ID) {
303 uint64_t Limit = ~static_cast<uint64_t>(0);
304 if (ID == Builtin::BIstrncmp || ID == Builtin::BI__builtin_strncmp ||
305 ID == Builtin::BIwcsncmp || ID == Builtin::BI__builtin_wcsncmp) {
306 if (!popToUInt64(S, E: Call->getArg(Arg: 2), Out&: Limit))
307 return false;
308 }
309
310 const Pointer &B = S.Stk.pop<Pointer>();
311 const Pointer &A = S.Stk.pop<Pointer>();
312 if (ID == Builtin::BIstrcmp || ID == Builtin::BIstrncmp ||
313 ID == Builtin::BIwcscmp || ID == Builtin::BIwcsncmp)
314 diagnoseNonConstexprBuiltin(S, OpPC, ID);
315
316 if (Limit == 0) {
317 pushInteger(S, Val: 0, QT: Call->getType());
318 return true;
319 }
320
321 if (!CheckLive(S, OpPC, Ptr: A, AK: AK_Read) || !CheckLive(S, OpPC, Ptr: B, AK: AK_Read))
322 return false;
323
324 if (!A.isReadablePointerType() || !B.isReadablePointerType())
325 return false;
326
327 if (A.isDummy() || B.isDummy() || A.isUnknownSizeArray() ||
328 B.isUnknownSizeArray())
329 return false;
330
331 bool IsWide = ID == Builtin::BIwcscmp || ID == Builtin::BIwcsncmp ||
332 ID == Builtin::BI__builtin_wcscmp ||
333 ID == Builtin::BI__builtin_wcsncmp;
334
335 QualType ElemTy = getElemType(P: A);
336 // Different element types shouldn't happen, but with casts they can.
337 if (!S.getASTContext().hasSameUnqualifiedType(T1: ElemTy, T2: getElemType(P: B)) ||
338 !S.getContext().canClassify(T: ElemTy))
339 return false;
340
341 PrimType ElemT = *S.getContext().classify(T: ElemTy);
342
343 auto returnResult = [&](int V) -> bool {
344 pushInteger(S, Val: V, QT: Call->getType());
345 return true;
346 };
347
348 unsigned IndexA = A.getIndex();
349 unsigned IndexB = B.getIndex();
350 unsigned NumElemsA = A.getNumElems();
351 unsigned NumElemsB = B.getNumElems();
352 uint64_t Steps = 0;
353 for (;; ++IndexA, ++IndexB, ++Steps) {
354
355 if (Steps >= Limit)
356 break;
357
358 // Diagnose this as a read of one-past-the-end.
359 if (IndexA >= NumElemsA || IndexB >= NumElemsB) {
360 S.FFDiag(SI: S.Current->getSource(PC: OpPC), DiagId: diag::note_constexpr_access_past_end)
361 << AK_Read << S.Current->getRange(PC: OpPC);
362 return false;
363 }
364
365 if (IsWide) {
366 INT_TYPE_SWITCH(ElemT, {
367 T CA = A.loadElem<T>(IndexA);
368 T CB = B.loadElem<T>(IndexB);
369 if (CA > CB)
370 return returnResult(1);
371 if (CA < CB)
372 return returnResult(-1);
373 if (CA.isZero() || CB.isZero())
374 return returnResult(0);
375 });
376 continue;
377 }
378
379 uint8_t CA = A.loadElem<uint8_t>(I: IndexA);
380 uint8_t CB = B.loadElem<uint8_t>(I: IndexB);
381
382 if (CA > CB)
383 return returnResult(1);
384 if (CA < CB)
385 return returnResult(-1);
386 if (CA == 0 || CB == 0)
387 return returnResult(0);
388 }
389
390 return returnResult(0);
391}
392
393static bool interp__builtin_strlen(InterpState &S, CodePtr OpPC,
394 const InterpFrame *Frame,
395 const CallExpr *Call, unsigned ID) {
396 const Pointer &StrPtr = S.Stk.pop<Pointer>().expand();
397
398 if (ID == Builtin::BIstrlen || ID == Builtin::BIwcslen)
399 diagnoseNonConstexprBuiltin(S, OpPC, ID);
400
401 if (StrPtr.isConstexprUnknown())
402 return false;
403
404 if (!CheckArray(S, OpPC, Ptr: StrPtr))
405 return false;
406
407 if (!CheckLive(S, OpPC, Ptr: StrPtr, AK: AK_Read))
408 return false;
409
410 // For string literal pointers, this is pretty simple.
411 if (StrPtr.isStringPointer()) {
412 if (StrPtr.isOnePastEnd())
413 return CheckRange(S, OpPC, Ptr: StrPtr, AK: AK_Read);
414
415 const auto *Lit = StrPtr.asStringPointer().getLiteral();
416 int64_t Off = StrPtr.getByteOffset();
417 if (Off < 0)
418 return false;
419
420 UnsignedOrNone ZeroIndex = Lit->findZeroCodeUnit(StartIndex: Off);
421 if (!ZeroIndex)
422 return false;
423 pushInteger(S, Val: *ZeroIndex, QT: Call->getType());
424 return true;
425 }
426
427 if (!StrPtr.isBlockPointer())
428 return false;
429
430 if (!CheckDummy(S, OpPC, Ptr: StrPtr, AK: AK_Read))
431 return false;
432
433 if (!StrPtr.getFieldDesc()->isPrimitiveArray())
434 return false;
435
436 assert(StrPtr.getFieldDesc()->isPrimitiveArray());
437 PrimType ElemT = StrPtr.getFieldDesc()->getPrimType();
438 unsigned ElemSize = StrPtr.getFieldDesc()->getElemDataSize();
439 if (ElemSize != 1 && ElemSize != 2 && ElemSize != 4)
440 return Invalid(S, OpPC);
441
442 if (ID == Builtin::BI__builtin_wcslen || ID == Builtin::BIwcslen) {
443 const ASTContext &AC = S.getASTContext();
444 unsigned WCharSize = AC.getTypeSizeInChars(T: AC.getWCharType()).getQuantity();
445 if (StrPtr.getFieldDesc()->getElemDataSize() != WCharSize)
446 return false;
447 }
448
449 size_t Len = 0;
450 for (size_t I = StrPtr.getIndex();; ++I, ++Len) {
451 PtrView ElemPtr = StrPtr.view().atIndex(Idx: I);
452
453 if (!CheckRange(S, OpPC, Ptr: ElemPtr, AK: AK_Read))
454 return false;
455
456 uint32_t Val;
457 FIXED_SIZE_INT_TYPE_SWITCH(
458 ElemT, { Val = static_cast<uint32_t>(ElemPtr.deref<T>()); });
459 if (Val == 0)
460 break;
461 }
462
463 pushInteger(S, Val: Len, QT: Call->getType());
464
465 return true;
466}
467
468static bool interp__builtin_nan(InterpState &S, CodePtr OpPC,
469 const InterpFrame *Frame, const CallExpr *Call,
470 bool Signaling) {
471 const Pointer &Arg = S.Stk.pop<Pointer>();
472
473 if (!CheckLoad(S, OpPC, Ptr: Arg))
474 return false;
475
476 // Convert the given string to an integer using StringRef's API.
477 llvm::APInt Fill;
478 if (Arg.isBlockPointer()) {
479 if (!Arg.getFieldDesc()->isPrimitiveArray())
480 return Invalid(S, OpPC);
481
482 std::string Str;
483 unsigned ArgLength = Arg.getNumElems();
484 bool FoundZero = false;
485 for (unsigned I = 0; I != ArgLength; ++I) {
486 if (!Arg.isElementInitialized(Index: I))
487 return false;
488
489 if (Arg.loadElem<int8_t>(I) == 0) {
490 FoundZero = true;
491 break;
492 }
493 Str += Arg.elem<char>(I);
494 }
495
496 // If we didn't find a NUL byte, diagnose as a one-past-the-end read.
497 if (!FoundZero)
498 return CheckRange(S, OpPC, Ptr: Arg.atIndex(Idx: ArgLength), AK: AK_Read);
499
500 // Treat empty strings as if they were zero.
501 if (Str.empty())
502 Fill = llvm::APInt(32, 0);
503 else if (StringRef(Str).getAsInteger(Radix: 0, Result&: Fill))
504 return false;
505 } else if (Arg.isStringPointer()) {
506 if (!Arg.asStringPointer().getLiteral()->isOrdinary())
507 return false;
508 StringRef Str = Arg.asStringPointer().getLiteral()->getString();
509 // Treat empty strings as if they were zero.
510 if (Str.empty())
511 Fill = llvm::APInt(32, 0);
512 else if (StringRef(Str).getAsInteger(Radix: 0, Result&: Fill))
513 return false;
514 } else {
515 return false;
516 }
517
518 const llvm::fltSemantics &TargetSemantics =
519 S.getASTContext().getFloatTypeSemantics(
520 T: Call->getDirectCallee()->getReturnType());
521
522 Floating Result = S.allocFloat(Sem: TargetSemantics);
523 if (S.getASTContext().getTargetInfo().isNan2008()) {
524 if (Signaling)
525 Result.copy(
526 F: llvm::APFloat::getSNaN(Sem: TargetSemantics, /*Negative=*/false, payload: &Fill));
527 else
528 Result.copy(
529 F: llvm::APFloat::getQNaN(Sem: TargetSemantics, /*Negative=*/false, payload: &Fill));
530 } else {
531 // Prior to IEEE 754-2008, architectures were allowed to choose whether
532 // the first bit of their significand was set for qNaN or sNaN. MIPS chose
533 // a different encoding to what became a standard in 2008, and for pre-
534 // 2008 revisions, MIPS interpreted sNaN-2008 as qNan and qNaN-2008 as
535 // sNaN. This is now known as "legacy NaN" encoding.
536 if (Signaling)
537 Result.copy(
538 F: llvm::APFloat::getQNaN(Sem: TargetSemantics, /*Negative=*/false, payload: &Fill));
539 else
540 Result.copy(
541 F: llvm::APFloat::getSNaN(Sem: TargetSemantics, /*Negative=*/false, payload: &Fill));
542 }
543
544 S.Stk.push<Floating>(Args&: Result);
545 return true;
546}
547
548static bool interp__builtin_inf(InterpState &S, CodePtr OpPC,
549 const InterpFrame *Frame,
550 const CallExpr *Call) {
551 const llvm::fltSemantics &TargetSemantics =
552 S.getASTContext().getFloatTypeSemantics(
553 T: Call->getDirectCallee()->getReturnType());
554
555 Floating Result = S.allocFloat(Sem: TargetSemantics);
556 Result.copy(F: APFloat::getInf(Sem: TargetSemantics));
557 S.Stk.push<Floating>(Args&: Result);
558 return true;
559}
560
561static bool interp__builtin_copysign(InterpState &S, CodePtr OpPC,
562 const InterpFrame *Frame) {
563 const Floating &Arg2 = S.Stk.pop<Floating>();
564 const Floating &Arg1 = S.Stk.pop<Floating>();
565 Floating Result = S.allocFloat(Sem: Arg1.getSemantics());
566
567 APFloat Copy = Arg1.getAPFloat();
568 Copy.copySign(RHS: Arg2.getAPFloat());
569 Result.copy(F: Copy);
570 S.Stk.push<Floating>(Args&: Result);
571
572 return true;
573}
574
575static bool interp__builtin_fmin(InterpState &S, CodePtr OpPC,
576 const InterpFrame *Frame, bool IsNumBuiltin) {
577 const Floating &RHS = S.Stk.pop<Floating>();
578 const Floating &LHS = S.Stk.pop<Floating>();
579 Floating Result = S.allocFloat(Sem: LHS.getSemantics());
580
581 if (IsNumBuiltin)
582 Result.copy(F: llvm::minimumnum(A: LHS.getAPFloat(), B: RHS.getAPFloat()));
583 else
584 Result.copy(F: minnum(A: LHS.getAPFloat(), B: RHS.getAPFloat()));
585 S.Stk.push<Floating>(Args&: Result);
586 return true;
587}
588
589static bool interp__builtin_fmax(InterpState &S, CodePtr OpPC,
590 const InterpFrame *Frame, bool IsNumBuiltin) {
591 const Floating &RHS = S.Stk.pop<Floating>();
592 const Floating &LHS = S.Stk.pop<Floating>();
593 Floating Result = S.allocFloat(Sem: LHS.getSemantics());
594
595 if (IsNumBuiltin)
596 Result.copy(F: llvm::maximumnum(A: LHS.getAPFloat(), B: RHS.getAPFloat()));
597 else
598 Result.copy(F: maxnum(A: LHS.getAPFloat(), B: RHS.getAPFloat()));
599 S.Stk.push<Floating>(Args&: Result);
600 return true;
601}
602
603/// Defined as __builtin_isnan(...), to accommodate the fact that it can
604/// take a float, double, long double, etc.
605/// But for us, that's all a Floating anyway.
606static bool interp__builtin_isnan(InterpState &S, CodePtr OpPC,
607 const InterpFrame *Frame,
608 const CallExpr *Call) {
609 const Floating &Arg = S.Stk.pop<Floating>();
610
611 pushInteger(S, Val: Arg.isNan(), QT: Call->getType());
612 return true;
613}
614
615static bool interp__builtin_issignaling(InterpState &S, CodePtr OpPC,
616 const InterpFrame *Frame,
617 const CallExpr *Call) {
618 const Floating &Arg = S.Stk.pop<Floating>();
619
620 pushInteger(S, Val: Arg.isSignaling(), QT: Call->getType());
621 return true;
622}
623
624static bool interp__builtin_isinf(InterpState &S, CodePtr OpPC,
625 const InterpFrame *Frame, bool CheckSign,
626 const CallExpr *Call) {
627 const Floating &Arg = S.Stk.pop<Floating>();
628 APFloat F = Arg.getAPFloat();
629 bool IsInf = F.isInfinity();
630
631 if (CheckSign)
632 pushInteger(S, Val: IsInf ? (F.isNegative() ? -1 : 1) : 0, QT: Call->getType());
633 else
634 pushInteger(S, Val: IsInf, QT: Call->getType());
635 return true;
636}
637
638static bool interp__builtin_isfinite(InterpState &S, CodePtr OpPC,
639 const InterpFrame *Frame,
640 const CallExpr *Call) {
641 const Floating &Arg = S.Stk.pop<Floating>();
642
643 pushInteger(S, Val: Arg.isFinite(), QT: Call->getType());
644 return true;
645}
646
647static bool interp__builtin_isnormal(InterpState &S, CodePtr OpPC,
648 const InterpFrame *Frame,
649 const CallExpr *Call) {
650 const Floating &Arg = S.Stk.pop<Floating>();
651
652 pushInteger(S, Val: Arg.isNormal(), QT: Call->getType());
653 return true;
654}
655
656static bool interp__builtin_issubnormal(InterpState &S, CodePtr OpPC,
657 const InterpFrame *Frame,
658 const CallExpr *Call) {
659 const Floating &Arg = S.Stk.pop<Floating>();
660
661 pushInteger(S, Val: Arg.isDenormal(), QT: Call->getType());
662 return true;
663}
664
665static bool interp__builtin_iszero(InterpState &S, CodePtr OpPC,
666 const InterpFrame *Frame,
667 const CallExpr *Call) {
668 const Floating &Arg = S.Stk.pop<Floating>();
669
670 pushInteger(S, Val: Arg.isZero(), QT: Call->getType());
671 return true;
672}
673
674static bool interp__builtin_signbit(InterpState &S, CodePtr OpPC,
675 const InterpFrame *Frame,
676 const CallExpr *Call) {
677 const Floating &Arg = S.Stk.pop<Floating>();
678
679 pushInteger(S, Val: Arg.isNegative(), QT: Call->getType());
680 return true;
681}
682
683static bool interp_floating_comparison(InterpState &S, CodePtr OpPC,
684 const CallExpr *Call, unsigned ID) {
685 const Floating &RHS = S.Stk.pop<Floating>();
686 const Floating &LHS = S.Stk.pop<Floating>();
687
688 pushInteger(
689 S,
690 Val: [&] {
691 switch (ID) {
692 case Builtin::BI__builtin_isgreater:
693 return LHS > RHS;
694 case Builtin::BI__builtin_isgreaterequal:
695 return LHS >= RHS;
696 case Builtin::BI__builtin_isless:
697 return LHS < RHS;
698 case Builtin::BI__builtin_islessequal:
699 return LHS <= RHS;
700 case Builtin::BI__builtin_islessgreater: {
701 ComparisonCategoryResult Cmp = LHS.compare(RHS);
702 return Cmp == ComparisonCategoryResult::Less ||
703 Cmp == ComparisonCategoryResult::Greater;
704 }
705 case Builtin::BI__builtin_isunordered:
706 return LHS.compare(RHS) == ComparisonCategoryResult::Unordered;
707 default:
708 llvm_unreachable("Unexpected builtin ID: Should be a floating point "
709 "comparison function");
710 }
711 }(),
712 QT: Call->getType());
713 return true;
714}
715
716/// First parameter to __builtin_isfpclass is the floating value, the
717/// second one is an integral value.
718static bool interp__builtin_isfpclass(InterpState &S, CodePtr OpPC,
719 const InterpFrame *Frame,
720 const CallExpr *Call) {
721 APSInt FPClassArg;
722 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: FPClassArg))
723 return false;
724 const Floating &F = S.Stk.pop<Floating>();
725
726 int32_t Result = static_cast<int32_t>(
727 (F.classify() & std::move(FPClassArg)).getZExtValue());
728 pushInteger(S, Val: Result, QT: Call->getType());
729
730 return true;
731}
732
733/// Five int values followed by one floating value.
734/// __builtin_fpclassify(int, int, int, int, int, float)
735static bool interp__builtin_fpclassify(InterpState &S, CodePtr OpPC,
736 const InterpFrame *Frame,
737 const CallExpr *Call) {
738 const Floating &Val = S.Stk.pop<Floating>();
739
740 PrimType IntT = *S.getContext().classify(E: Call->getArg(Arg: 0));
741 APSInt Values[5];
742 for (unsigned I = 0; I != 5; ++I) {
743 if (!popToAPSInt(Stk&: S.Stk, T: IntT, Out&: Values[4 - I]))
744 return false;
745 }
746
747 unsigned Index;
748 switch (Val.getCategory()) {
749 case APFloat::fcNaN:
750 Index = 0;
751 break;
752 case APFloat::fcInfinity:
753 Index = 1;
754 break;
755 case APFloat::fcNormal:
756 Index = Val.isDenormal() ? 3 : 2;
757 break;
758 case APFloat::fcZero:
759 Index = 4;
760 break;
761 }
762
763 // The last argument is first on the stack.
764 assert(Index <= 4);
765
766 pushInteger(S, Val: Values[Index], QT: Call->getType());
767 return true;
768}
769
770static inline Floating abs(InterpState &S, const Floating &In) {
771 if (!In.isNegative())
772 return In;
773
774 Floating Output = S.allocFloat(Sem: In.getSemantics());
775 APFloat New = In.getAPFloat();
776 New.changeSign();
777 Output.copy(F: New);
778 return Output;
779}
780
781// The C standard says "fabs raises no floating-point exceptions,
782// even if x is a signaling NaN. The returned value is independent of
783// the current rounding direction mode." Therefore constant folding can
784// proceed without regard to the floating point settings.
785// Reference, WG14 N2478 F.10.4.3
786static bool interp__builtin_fabs(InterpState &S, CodePtr OpPC,
787 const InterpFrame *Frame) {
788 const Floating &Val = S.Stk.pop<Floating>();
789 S.Stk.push<Floating>(Args: abs(S, In: Val));
790 return true;
791}
792
793static bool interp__builtin_abs(InterpState &S, CodePtr OpPC,
794 const InterpFrame *Frame,
795 const CallExpr *Call) {
796 APSInt Val;
797 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: Val))
798 return false;
799 if (Val ==
800 APSInt(APInt::getSignedMinValue(numBits: Val.getBitWidth()), /*IsUnsigned=*/false))
801 return false;
802 if (Val.isNegative())
803 Val.negate();
804 pushInteger(S, Val, QT: Call->getType());
805 return true;
806}
807
808static bool interp__builtin_popcount(InterpState &S, CodePtr OpPC,
809 const InterpFrame *Frame,
810 const CallExpr *Call) {
811 APSInt Val;
812 if (Call->getArg(Arg: 0)->getType()->isExtVectorBoolType()) {
813 const Pointer &Arg = S.Stk.pop<Pointer>();
814 Val = convertBoolVectorToInt(Val: Arg);
815 } else {
816 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: Val))
817 return false;
818 }
819 pushInteger(S, Val: Val.popcount(), QT: Call->getType());
820 return true;
821}
822
823static bool interp__builtin_ia32_crc32(InterpState &S, CodePtr OpPC,
824 const InterpFrame *Frame,
825 const CallExpr *Call,
826 unsigned DataBytes) {
827 uint64_t DataVal;
828 if (!popToUInt64(S, E: Call->getArg(Arg: 1), Out&: DataVal))
829 return false;
830 uint64_t CRCVal;
831 if (!popToUInt64(S, E: Call->getArg(Arg: 0), Out&: CRCVal))
832 return false;
833
834 // CRC32C polynomial (iSCSI polynomial, bit-reversed)
835 static const uint32_t CRC32C_POLY = 0x82F63B78;
836
837 uint32_t Result = llvm::calculateReflectedCRC32(
838 Crc: static_cast<uint32_t>(CRCVal), Data: DataVal, DataBytes, Poly: CRC32C_POLY);
839
840 pushInteger(S, Val: Result, QT: Call->getType());
841 return true;
842}
843
844static bool interp__builtin_classify_type(InterpState &S, CodePtr OpPC,
845 const InterpFrame *Frame,
846 const CallExpr *Call) {
847 // This is an unevaluated call, so there are no arguments on the stack.
848 assert(Call->getNumArgs() == 1);
849 const Expr *Arg = Call->getArg(Arg: 0);
850
851 GCCTypeClass ResultClass =
852 EvaluateBuiltinClassifyType(T: Arg->getType(), LangOpts: S.getLangOpts());
853 int32_t ReturnVal = static_cast<int32_t>(ResultClass);
854 pushInteger(S, Val: ReturnVal, QT: Call->getType());
855 return true;
856}
857
858// __builtin_expect(long, long)
859// __builtin_expect_with_probability(long, long, double)
860static bool interp__builtin_expect(InterpState &S, CodePtr OpPC,
861 const InterpFrame *Frame,
862 const CallExpr *Call) {
863 // The return value is simply the value of the first parameter.
864 // We ignore the probability.
865 unsigned NumArgs = Call->getNumArgs();
866 assert(NumArgs == 2 || NumArgs == 3);
867
868 PrimType ArgT = *S.getContext().classify(T: Call->getArg(Arg: 0)->getType());
869 if (NumArgs == 3)
870 S.Stk.discard<Floating>();
871 discard(Stk&: S.Stk, T: ArgT);
872 // Top of the stack is now the first paramter. Leave it there as the return
873 // value.
874
875 return true;
876}
877
878static bool interp__builtin_addressof(InterpState &S, CodePtr OpPC,
879 const InterpFrame *Frame,
880 const CallExpr *Call) {
881#ifndef NDEBUG
882 assert(Call->getArg(0)->isLValue());
883 PrimType PtrT = S.getContext().classify(Call->getArg(0)).value_or(PT_Ptr);
884 assert(PtrT == PT_Ptr &&
885 "Unsupported pointer type passed to __builtin_addressof()");
886#endif
887 return true;
888}
889
890static bool interp__builtin_move(InterpState &S, CodePtr OpPC,
891 const InterpFrame *Frame,
892 const CallExpr *Call) {
893 return Call->getDirectCallee()->isConstexpr();
894}
895
896static bool interp__builtin_eh_return_data_regno(InterpState &S, CodePtr OpPC,
897 const InterpFrame *Frame,
898 const CallExpr *Call) {
899 APSInt Arg;
900 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: Arg))
901 return false;
902
903 int Result = S.getASTContext().getTargetInfo().getEHDataRegisterNumber(
904 RegNo: Arg.getZExtValue());
905 pushInteger(S, Val: Result, QT: Call->getType());
906 return true;
907}
908
909// Two integral values followed by a pointer (lhs, rhs, resultOut)
910static bool interp__builtin_overflowop(InterpState &S, CodePtr OpPC,
911 const CallExpr *Call,
912 unsigned BuiltinOp) {
913 const Pointer &ResultPtr = S.Stk.pop<Pointer>();
914 if (ResultPtr.isDummy() || !ResultPtr.isBlockPointer())
915 return false;
916
917 PrimType RHST = *S.getContext().classify(T: Call->getArg(Arg: 1)->getType());
918 PrimType LHST = *S.getContext().classify(T: Call->getArg(Arg: 0)->getType());
919 APSInt RHS;
920 if (!popToAPSInt(Stk&: S.Stk, T: RHST, Out&: RHS))
921 return false;
922 APSInt LHS;
923 if (!popToAPSInt(Stk&: S.Stk, T: LHST, Out&: LHS))
924 return false;
925 QualType ResultType = Call->getArg(Arg: 2)->getType()->getPointeeType();
926 PrimType ResultT = *S.getContext().classify(T: ResultType);
927 bool Overflow;
928
929 APSInt Result;
930 if (BuiltinOp == Builtin::BI__builtin_add_overflow ||
931 BuiltinOp == Builtin::BI__builtin_sub_overflow ||
932 BuiltinOp == Builtin::BI__builtin_mul_overflow) {
933 bool IsSigned = LHS.isSigned() || RHS.isSigned() ||
934 ResultType->isSignedIntegerOrEnumerationType();
935 bool AllSigned = LHS.isSigned() && RHS.isSigned() &&
936 ResultType->isSignedIntegerOrEnumerationType();
937 uint64_t LHSSize = LHS.getBitWidth();
938 uint64_t RHSSize = RHS.getBitWidth();
939 uint64_t ResultSize = S.getASTContext().getIntWidth(T: ResultType);
940 uint64_t MaxBits = std::max(a: std::max(a: LHSSize, b: RHSSize), b: ResultSize);
941
942 // Add an additional bit if the signedness isn't uniformly agreed to. We
943 // could do this ONLY if there is a signed and an unsigned that both have
944 // MaxBits, but the code to check that is pretty nasty. The issue will be
945 // caught in the shrink-to-result later anyway.
946 if (IsSigned && !AllSigned)
947 ++MaxBits;
948
949 LHS = APSInt(LHS.extOrTrunc(width: MaxBits), !IsSigned);
950 RHS = APSInt(RHS.extOrTrunc(width: MaxBits), !IsSigned);
951 Result = APSInt(MaxBits, !IsSigned);
952 }
953
954 // Find largest int.
955 switch (BuiltinOp) {
956 default:
957 llvm_unreachable("Invalid value for BuiltinOp");
958 case Builtin::BI__builtin_add_overflow:
959 case Builtin::BI__builtin_sadd_overflow:
960 case Builtin::BI__builtin_saddl_overflow:
961 case Builtin::BI__builtin_saddll_overflow:
962 case Builtin::BI__builtin_uadd_overflow:
963 case Builtin::BI__builtin_uaddl_overflow:
964 case Builtin::BI__builtin_uaddll_overflow:
965 Result = LHS.isSigned() ? LHS.sadd_ov(RHS, Overflow)
966 : LHS.uadd_ov(RHS, Overflow);
967 break;
968 case Builtin::BI__builtin_sub_overflow:
969 case Builtin::BI__builtin_ssub_overflow:
970 case Builtin::BI__builtin_ssubl_overflow:
971 case Builtin::BI__builtin_ssubll_overflow:
972 case Builtin::BI__builtin_usub_overflow:
973 case Builtin::BI__builtin_usubl_overflow:
974 case Builtin::BI__builtin_usubll_overflow:
975 Result = LHS.isSigned() ? LHS.ssub_ov(RHS, Overflow)
976 : LHS.usub_ov(RHS, Overflow);
977 break;
978 case Builtin::BI__builtin_mul_overflow:
979 case Builtin::BI__builtin_smul_overflow:
980 case Builtin::BI__builtin_smull_overflow:
981 case Builtin::BI__builtin_smulll_overflow:
982 case Builtin::BI__builtin_umul_overflow:
983 case Builtin::BI__builtin_umull_overflow:
984 case Builtin::BI__builtin_umulll_overflow:
985 Result = LHS.isSigned() ? LHS.smul_ov(RHS, Overflow)
986 : LHS.umul_ov(RHS, Overflow);
987 break;
988 }
989
990 // In the case where multiple sizes are allowed, truncate and see if
991 // the values are the same.
992 if (BuiltinOp == Builtin::BI__builtin_add_overflow ||
993 BuiltinOp == Builtin::BI__builtin_sub_overflow ||
994 BuiltinOp == Builtin::BI__builtin_mul_overflow) {
995 // APSInt doesn't have a TruncOrSelf, so we use extOrTrunc instead,
996 // since it will give us the behavior of a TruncOrSelf in the case where
997 // its parameter <= its size. We previously set Result to be at least the
998 // integer width of the result, so getIntWidth(ResultType) <=
999 // Result.BitWidth
1000 APSInt Temp = Result.extOrTrunc(width: S.getASTContext().getIntWidth(T: ResultType));
1001 Temp.setIsSigned(ResultType->isSignedIntegerOrEnumerationType());
1002
1003 if (!APSInt::isSameValue(I1: Temp, I2: Result))
1004 Overflow = true;
1005 Result = std::move(Temp);
1006 }
1007
1008 // Write Result to ResultPtr and put Overflow on the stack.
1009 assignIntegral(S, Dest: ResultPtr, ValueT: ResultT, Value: Result);
1010 if (ResultPtr.canBeInitialized())
1011 ResultPtr.initialize();
1012
1013 assert(Call->getDirectCallee()->getReturnType()->isBooleanType());
1014 S.Stk.push<Boolean>(Args&: Overflow);
1015 return true;
1016}
1017
1018/// Three integral values followed by a pointer (lhs, rhs, carry, carryOut).
1019static bool interp__builtin_carryop(InterpState &S, CodePtr OpPC,
1020 const InterpFrame *Frame,
1021 const CallExpr *Call, unsigned BuiltinOp) {
1022 const Pointer &CarryOutPtr = S.Stk.pop<Pointer>();
1023 PrimType LHST = *S.getContext().classify(T: Call->getArg(Arg: 0)->getType());
1024 PrimType RHST = *S.getContext().classify(T: Call->getArg(Arg: 1)->getType());
1025 APSInt CarryIn;
1026 if (!popToAPSInt(Stk&: S.Stk, T: LHST, Out&: CarryIn))
1027 return false;
1028 APSInt RHS;
1029 if (!popToAPSInt(Stk&: S.Stk, T: RHST, Out&: RHS))
1030 return false;
1031 APSInt LHS;
1032 if (!popToAPSInt(Stk&: S.Stk, T: LHST, Out&: LHS))
1033 return false;
1034
1035 if (!isReadable(P: CarryOutPtr))
1036 return false;
1037
1038 APSInt CarryOut;
1039
1040 APSInt Result;
1041 // Copy the number of bits and sign.
1042 Result = LHS;
1043 CarryOut = LHS;
1044
1045 bool FirstOverflowed = false;
1046 bool SecondOverflowed = false;
1047 switch (BuiltinOp) {
1048 default:
1049 llvm_unreachable("Invalid value for BuiltinOp");
1050 case Builtin::BI__builtin_addcb:
1051 case Builtin::BI__builtin_addcs:
1052 case Builtin::BI__builtin_addc:
1053 case Builtin::BI__builtin_addcl:
1054 case Builtin::BI__builtin_addcll:
1055 Result =
1056 LHS.uadd_ov(RHS, Overflow&: FirstOverflowed).uadd_ov(RHS: CarryIn, Overflow&: SecondOverflowed);
1057 break;
1058 case Builtin::BI__builtin_subcb:
1059 case Builtin::BI__builtin_subcs:
1060 case Builtin::BI__builtin_subc:
1061 case Builtin::BI__builtin_subcl:
1062 case Builtin::BI__builtin_subcll:
1063 Result =
1064 LHS.usub_ov(RHS, Overflow&: FirstOverflowed).usub_ov(RHS: CarryIn, Overflow&: SecondOverflowed);
1065 break;
1066 }
1067 // It is possible for both overflows to happen but CGBuiltin uses an OR so
1068 // this is consistent.
1069 CarryOut = (uint64_t)(FirstOverflowed | SecondOverflowed);
1070
1071 QualType CarryOutType = Call->getArg(Arg: 3)->getType()->getPointeeType();
1072 PrimType CarryOutT = *S.getContext().classify(T: CarryOutType);
1073 assignIntegral(S, Dest: CarryOutPtr, ValueT: CarryOutT, Value: CarryOut);
1074 if (CarryOutPtr.canBeInitialized())
1075 CarryOutPtr.initialize();
1076
1077 assert(S.getASTContext().hasSimilarType(Call->getType(),
1078 Call->getArg(0)->getType()));
1079 pushInteger(S, Val: Result, QT: Call->getType());
1080 return true;
1081}
1082
1083static bool interp__builtin_clz(InterpState &S, CodePtr OpPC,
1084 const InterpFrame *Frame, const CallExpr *Call,
1085 unsigned BuiltinOp) {
1086
1087 std::optional<APSInt> Fallback;
1088 if (BuiltinOp == Builtin::BI__builtin_clzg && Call->getNumArgs() == 2) {
1089 APSInt FallbackVal;
1090 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: FallbackVal))
1091 return false;
1092 Fallback = FallbackVal;
1093 }
1094
1095 APSInt Val;
1096 if (Call->getArg(Arg: 0)->getType()->isExtVectorBoolType()) {
1097 const Pointer &Arg = S.Stk.pop<Pointer>();
1098 Val = convertBoolVectorToInt(Val: Arg);
1099 } else {
1100 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: Val))
1101 return false;
1102 }
1103
1104 // When the argument is 0, the result of GCC builtins is undefined, whereas
1105 // for Microsoft intrinsics, the result is the bit-width of the argument.
1106 bool ZeroIsUndefined = BuiltinOp != Builtin::BI__lzcnt16 &&
1107 BuiltinOp != Builtin::BI__lzcnt &&
1108 BuiltinOp != Builtin::BI__lzcnt64;
1109
1110 if (Val == 0) {
1111 if (Fallback) {
1112 pushInteger(S, Val: *Fallback, QT: Call->getType());
1113 return true;
1114 }
1115
1116 if (ZeroIsUndefined)
1117 return false;
1118 }
1119
1120 pushInteger(S, Val: Val.countl_zero(), QT: Call->getType());
1121 return true;
1122}
1123
1124static bool interp__builtin_ctz(InterpState &S, CodePtr OpPC,
1125 const InterpFrame *Frame, const CallExpr *Call,
1126 unsigned BuiltinID) {
1127 std::optional<APSInt> Fallback;
1128 if (BuiltinID == Builtin::BI__builtin_ctzg && Call->getNumArgs() == 2) {
1129 APSInt FallbackVal;
1130 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: FallbackVal))
1131 return false;
1132 Fallback = FallbackVal;
1133 }
1134
1135 APSInt Val;
1136 if (Call->getArg(Arg: 0)->getType()->isExtVectorBoolType()) {
1137 const Pointer &Arg = S.Stk.pop<Pointer>();
1138 Val = convertBoolVectorToInt(Val: Arg);
1139 } else {
1140 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: Val))
1141 return false;
1142 }
1143
1144 if (Val == 0) {
1145 if (Fallback) {
1146 pushInteger(S, Val: *Fallback, QT: Call->getType());
1147 return true;
1148 }
1149 return false;
1150 }
1151
1152 pushInteger(S, Val: Val.countr_zero(), QT: Call->getType());
1153 return true;
1154}
1155
1156static bool interp__builtin_bswap(InterpState &S, CodePtr OpPC,
1157 const InterpFrame *Frame,
1158 const CallExpr *Call) {
1159 APSInt Val;
1160 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: Val))
1161 return false;
1162 if (Val.getBitWidth() == 8 || Val.getBitWidth() == 1)
1163 pushInteger(S, Val, QT: Call->getType());
1164 else
1165 pushInteger(S, Val: Val.byteSwap(), QT: Call->getType());
1166 return true;
1167}
1168
1169/// bool __atomic_always_lock_free(size_t, void const volatile*)
1170/// bool __atomic_is_lock_free(size_t, void const volatile*)
1171static bool interp__builtin_atomic_lock_free(InterpState &S, CodePtr OpPC,
1172 const InterpFrame *Frame,
1173 const CallExpr *Call,
1174 unsigned BuiltinOp) {
1175 auto returnBool = [&S](bool Value) -> bool {
1176 S.Stk.push<Boolean>(Args&: Value);
1177 return true;
1178 };
1179
1180 const Pointer &Ptr = S.Stk.pop<Pointer>();
1181 uint64_t SizeVal;
1182 if (!popToUInt64(S, E: Call->getArg(Arg: 0), Out&: SizeVal))
1183 return false;
1184
1185 // For __atomic_is_lock_free(sizeof(_Atomic(T))), if the size is a power
1186 // of two less than or equal to the maximum inline atomic width, we know it
1187 // is lock-free. If the size isn't a power of two, or greater than the
1188 // maximum alignment where we promote atomics, we know it is not lock-free
1189 // (at least not in the sense of atomic_is_lock_free). Otherwise,
1190 // the answer can only be determined at runtime; for example, 16-byte
1191 // atomics have lock-free implementations on some, but not all,
1192 // x86-64 processors.
1193
1194 // Check power-of-two.
1195 CharUnits Size = CharUnits::fromQuantity(Quantity: SizeVal);
1196 if (Size.isPowerOfTwo()) {
1197 // Check against inlining width.
1198 unsigned InlineWidthBits =
1199 S.getASTContext().getTargetInfo().getMaxAtomicInlineWidth();
1200 if (Size <= S.getASTContext().toCharUnitsFromBits(BitSize: InlineWidthBits)) {
1201
1202 // OK, we will inline appropriately-aligned operations of this size,
1203 // and _Atomic(T) is appropriately-aligned.
1204 if (Size == CharUnits::One())
1205 return returnBool(true);
1206
1207 // Same for null pointers.
1208 assert(BuiltinOp != Builtin::BI__c11_atomic_is_lock_free);
1209 if (Ptr.isZero())
1210 return returnBool(true);
1211
1212 if (Ptr.isIntegralPointer()) {
1213 uint64_t IntVal = Ptr.getIntegerRepresentation();
1214 if (APSInt(APInt(64, IntVal, false), true).isAligned(A: Size.getAsAlign()))
1215 return returnBool(true);
1216 }
1217
1218 const Expr *PtrArg = Call->getArg(Arg: 1);
1219 // Otherwise, check if the type's alignment against Size.
1220 if (const auto *ICE = dyn_cast<ImplicitCastExpr>(Val: PtrArg)) {
1221 // Drop the potential implicit-cast to 'const volatile void*', getting
1222 // the underlying type.
1223 if (ICE->getCastKind() == CK_BitCast)
1224 PtrArg = ICE->getSubExpr();
1225 }
1226
1227 if (const auto *PtrTy = PtrArg->getType()->getAs<PointerType>()) {
1228 QualType PointeeType = PtrTy->getPointeeType();
1229 if (!PointeeType->isIncompleteType() &&
1230 S.getASTContext().getTypeAlignInChars(T: PointeeType) >= Size) {
1231 // OK, we will inline operations on this object.
1232 return returnBool(true);
1233 }
1234 }
1235 }
1236 }
1237
1238 if (BuiltinOp == Builtin::BI__atomic_always_lock_free)
1239 return returnBool(false);
1240
1241 return Invalid(S, OpPC);
1242}
1243
1244/// bool __c11_atomic_is_lock_free(size_t)
1245static bool interp__builtin_c11_atomic_is_lock_free(InterpState &S,
1246 CodePtr OpPC,
1247 const InterpFrame *Frame,
1248 const CallExpr *Call) {
1249 uint64_t SizeVal;
1250 if (!popToUInt64(S, E: Call->getArg(Arg: 0), Out&: SizeVal))
1251 return false;
1252
1253 CharUnits Size = CharUnits::fromQuantity(Quantity: SizeVal);
1254 if (Size.isPowerOfTwo()) {
1255 // Check against inlining width.
1256 unsigned InlineWidthBits =
1257 S.getASTContext().getTargetInfo().getMaxAtomicInlineWidth();
1258 if (Size <= S.getASTContext().toCharUnitsFromBits(BitSize: InlineWidthBits)) {
1259 S.Stk.push<Boolean>(Args: true);
1260 return true;
1261 }
1262 }
1263
1264 return false; // returnBool(false);
1265}
1266
1267/// __builtin_complex(Float A, float B);
1268static bool interp__builtin_complex(InterpState &S, CodePtr OpPC,
1269 const InterpFrame *Frame,
1270 const CallExpr *Call) {
1271 const Floating &Arg2 = S.Stk.pop<Floating>();
1272 const Floating &Arg1 = S.Stk.pop<Floating>();
1273 Pointer &Result = S.Stk.peek<Pointer>();
1274
1275 Result.elem<Floating>(I: 0) = Arg1;
1276 Result.elem<Floating>(I: 1) = Arg2;
1277 Result.initializeAllElements();
1278
1279 return true;
1280}
1281
1282/// __builtin_is_aligned()
1283/// __builtin_align_up()
1284/// __builtin_align_down()
1285/// The first parameter is either an integer or a pointer.
1286/// The second parameter is the requested alignment as an integer.
1287static bool interp__builtin_is_aligned_up_down(InterpState &S, CodePtr OpPC,
1288 const InterpFrame *Frame,
1289 const CallExpr *Call,
1290 unsigned BuiltinOp) {
1291 APSInt Alignment;
1292 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: Alignment))
1293 return false;
1294
1295 if (Alignment < 0 || !Alignment.isPowerOf2()) {
1296 S.FFDiag(E: Call, DiagId: diag::note_constexpr_invalid_alignment) << Alignment;
1297 return false;
1298 }
1299 unsigned SrcWidth = S.getASTContext().getIntWidth(T: Call->getArg(Arg: 0)->getType());
1300 APSInt MaxValue(APInt::getOneBitSet(numBits: SrcWidth, BitNo: SrcWidth - 1));
1301 if (APSInt::compareValues(I1: Alignment, I2: MaxValue) > 0) {
1302 S.FFDiag(E: Call, DiagId: diag::note_constexpr_alignment_too_big)
1303 << MaxValue << Call->getArg(Arg: 0)->getType() << Alignment;
1304 return false;
1305 }
1306
1307 // The first parameter is either an integer or a pointer.
1308 PrimType FirstArgT = *S.Ctx.classify(E: Call->getArg(Arg: 0));
1309
1310 if (isIntegerType(T: FirstArgT)) {
1311 APSInt Src;
1312 if (!popToAPSInt(Stk&: S.Stk, T: FirstArgT, Out&: Src))
1313 return false;
1314 APInt AlignMinusOne = Alignment.extOrTrunc(width: Src.getBitWidth()) - 1;
1315 if (BuiltinOp == Builtin::BI__builtin_align_up) {
1316 APSInt AlignedVal =
1317 APSInt((Src + AlignMinusOne) & ~AlignMinusOne, Src.isUnsigned());
1318 pushInteger(S, Val: AlignedVal, QT: Call->getType());
1319 } else if (BuiltinOp == Builtin::BI__builtin_align_down) {
1320 APSInt AlignedVal = APSInt(Src & ~AlignMinusOne, Src.isUnsigned());
1321 pushInteger(S, Val: AlignedVal, QT: Call->getType());
1322 } else {
1323 assert(*S.Ctx.classify(Call->getType()) == PT_Bool);
1324 S.Stk.push<Boolean>(Args: (Src & AlignMinusOne) == 0);
1325 }
1326 return true;
1327 }
1328 assert(FirstArgT == PT_Ptr);
1329 const Pointer &Ptr = S.Stk.pop<Pointer>();
1330
1331 // Null pointers are always aligned. Preserve null pointers for
1332 // align_up/align_down and return true for is_aligned.
1333 if (Ptr.isZero()) {
1334 if (BuiltinOp == Builtin::BI__builtin_is_aligned) {
1335 S.Stk.push<Boolean>(Args: true);
1336 return true;
1337 }
1338
1339 assert(BuiltinOp == Builtin::BI__builtin_align_up ||
1340 BuiltinOp == Builtin::BI__builtin_align_down);
1341
1342 S.Stk.push<Pointer>(Args: Ptr);
1343 return true;
1344 }
1345
1346 if (!Ptr.isBlockPointer() && !Ptr.isOpaquePointer()) {
1347 S.FFDiag(E: Call->getArg(Arg: 0), DiagId: diag::note_constexpr_alignment_compute)
1348 << Alignment;
1349 return false;
1350 }
1351
1352 const VarDecl *PtrDecl = Ptr.getRootVarDecl();
1353 // We need a pointer for a declaration here.
1354 if (!PtrDecl) {
1355 if (BuiltinOp == Builtin::BI__builtin_is_aligned)
1356 S.FFDiag(E: Call->getArg(Arg: 0), DiagId: diag::note_constexpr_alignment_compute)
1357 << Alignment;
1358 else
1359 S.FFDiag(E: Call->getArg(Arg: 0), DiagId: diag::note_constexpr_alignment_adjust)
1360 << Alignment;
1361 return false;
1362 }
1363
1364 unsigned PtrOffset;
1365 if (Ptr.isBlockPointer()) {
1366 // For one-past-end pointers, we can't call getIndex() since it asserts.
1367 // Use getNumElems() instead which gives the correct index for past-end.
1368 PtrOffset = Ptr.isElementPastEnd() ? Ptr.getNumElems() : Ptr.getIndex();
1369 } else {
1370 if (std::optional<size_t> PtrOff =
1371 Ptr.computeLayoutOffset(ASTCtx: S.getASTContext()))
1372 PtrOffset = *PtrOff;
1373 else
1374 return false;
1375 }
1376
1377 CharUnits BaseAlignment = S.getASTContext().getDeclAlign(D: PtrDecl);
1378 CharUnits PtrAlign =
1379 BaseAlignment.alignmentAtOffset(offset: CharUnits::fromQuantity(Quantity: PtrOffset));
1380
1381 if (BuiltinOp == Builtin::BI__builtin_is_aligned) {
1382 if (PtrAlign.getQuantity() >= Alignment) {
1383 S.Stk.push<Boolean>(Args: true);
1384 return true;
1385 }
1386 // If the alignment is not known to be sufficient, some cases could still
1387 // be aligned at run time. However, if the requested alignment is less or
1388 // equal to the base alignment and the offset is not aligned, we know that
1389 // the run-time value can never be aligned.
1390 if (BaseAlignment.getQuantity() >= Alignment &&
1391 PtrAlign.getQuantity() < Alignment) {
1392 S.Stk.push<Boolean>(Args: false);
1393 return true;
1394 }
1395
1396 S.FFDiag(E: Call->getArg(Arg: 0), DiagId: diag::note_constexpr_alignment_compute)
1397 << Alignment;
1398 return false;
1399 }
1400
1401 assert(BuiltinOp == Builtin::BI__builtin_align_down ||
1402 BuiltinOp == Builtin::BI__builtin_align_up);
1403
1404 // For align_up/align_down, we can return the same value if the alignment
1405 // is known to be greater or equal to the requested value.
1406 if (PtrAlign.getQuantity() >= Alignment) {
1407 S.Stk.push<Pointer>(Args: Ptr);
1408 return true;
1409 }
1410
1411 // The alignment could be greater than the minimum at run-time, so we cannot
1412 // infer much about the resulting pointer value. One case is possible:
1413 // For `_Alignas(32) char buf[N]; __builtin_align_down(&buf[idx], 32)` we
1414 // can infer the correct index if the requested alignment is smaller than
1415 // the base alignment so we can perform the computation on the offset.
1416 if (BaseAlignment.getQuantity() >= Alignment) {
1417 assert(Alignment.getBitWidth() <= 64 &&
1418 "Cannot handle > 64-bit address-space");
1419 uint64_t Alignment64 = Alignment.getZExtValue();
1420 CharUnits NewOffset =
1421 CharUnits::fromQuantity(Quantity: BuiltinOp == Builtin::BI__builtin_align_down
1422 ? llvm::alignDown(Value: PtrOffset, Align: Alignment64)
1423 : llvm::alignTo(Value: PtrOffset, Align: Alignment64));
1424
1425 if (Ptr.isBlockPointer()) {
1426 S.Stk.push<Pointer>(Args: Ptr.atIndex(Idx: NewOffset.getQuantity()));
1427 return true;
1428 }
1429
1430 assert(Ptr.isOpaquePointer());
1431
1432 APSInt APOffset =
1433 APSInt(APInt(64, NewOffset.getQuantity(), /*IsSigned=*/true),
1434 /*IsUnsigned=*/false);
1435 return arrayElemPtrOpaque(S, OpPC, Ptr, Index: std::move(APOffset),
1436 /*AllocReplace=*/AllowReplace: true);
1437 }
1438
1439 // Otherwise, we cannot constant-evaluate the result.
1440 S.FFDiag(E: Call->getArg(Arg: 0), DiagId: diag::note_constexpr_alignment_adjust) << Alignment;
1441 return false;
1442}
1443
1444/// Returns the alignment of the object Ptr points into.
1445static CharUnits getBaseAlignment(const ASTContext &ASTCtx,
1446 const Pointer &Ptr) {
1447 CharUnits BaseAlignment;
1448 if (Ptr.isBlockPointer() && Ptr.block()->isDynamic())
1449 BaseAlignment = Ptr.getDeclDesc()->computeAlignForDynamicAlloc(Ctx: ASTCtx);
1450 else if (const auto *VD = Ptr.getRootVarDecl())
1451 BaseAlignment = ASTCtx.getDeclAlign(D: VD);
1452 else if (const auto *E = Ptr.getRootExpr())
1453 BaseAlignment = GetAlignOfExpr(Ctx: ASTCtx, E, ExprKind: UETT_AlignOf);
1454 return BaseAlignment;
1455}
1456
1457/// __builtin_assume_aligned(Ptr, Alignment[, ExtraOffset])
1458static bool interp__builtin_assume_aligned(InterpState &S, CodePtr OpPC,
1459 const InterpFrame *Frame,
1460 const CallExpr *Call) {
1461 assert(Call->getNumArgs() == 2 || Call->getNumArgs() == 3);
1462
1463 std::optional<APSInt> ExtraOffset;
1464 if (Call->getNumArgs() == 3) {
1465 APSInt ExtraOffsetVal;
1466 if (!popToAPSInt(Stk&: S.Stk, T: *S.Ctx.classify(E: Call->getArg(Arg: 2)), Out&: ExtraOffsetVal))
1467 return false;
1468 ExtraOffset = ExtraOffsetVal;
1469 }
1470
1471 APSInt Alignment;
1472 if (!popToAPSInt(Stk&: S.Stk, T: *S.Ctx.classify(E: Call->getArg(Arg: 1)), Out&: Alignment))
1473 return false;
1474 const Pointer &Ptr = S.Stk.pop<Pointer>();
1475
1476 const ASTContext &ASTCtx = S.getASTContext();
1477 CharUnits Align = CharUnits::fromQuantity(Quantity: Alignment.getZExtValue());
1478
1479 // If there is a base object, then it must have the correct alignment.
1480 if (Ptr.isBlockPointer() || Ptr.isOpaquePointer()) {
1481 CharUnits BaseAlignment = getBaseAlignment(ASTCtx, Ptr);
1482 if (BaseAlignment < Align) {
1483 S.CCEDiag(E: Call->getArg(Arg: 0),
1484 DiagId: diag::note_constexpr_baa_insufficient_alignment)
1485 << 0 << BaseAlignment.getQuantity() << Align.getQuantity();
1486 return false;
1487 }
1488 }
1489
1490 std::optional<size_t> LayoutOffset = Ptr.computeLayoutOffset(ASTCtx);
1491 if (!LayoutOffset)
1492 return false;
1493
1494 CharUnits AVOffset = CharUnits::fromQuantity(Quantity: *LayoutOffset);
1495 if (ExtraOffset)
1496 AVOffset -= CharUnits::fromQuantity(Quantity: ExtraOffset->getZExtValue());
1497 if (AVOffset.alignTo(Align) != AVOffset) {
1498 if (Ptr.isBlockPointer() || Ptr.isOpaquePointer())
1499 S.CCEDiag(E: Call->getArg(Arg: 0),
1500 DiagId: diag::note_constexpr_baa_insufficient_alignment)
1501 << 1 << AVOffset.getQuantity() << Align.getQuantity();
1502 else
1503 S.CCEDiag(E: Call->getArg(Arg: 0),
1504 DiagId: diag::note_constexpr_baa_value_insufficient_alignment)
1505 << AVOffset.getQuantity() << Align.getQuantity();
1506 return false;
1507 }
1508
1509 S.Stk.push<Pointer>(Args: Ptr);
1510 return true;
1511}
1512
1513/// (CarryIn, LHS, RHS, Result)
1514static bool interp__builtin_ia32_addcarry_subborrow(InterpState &S,
1515 CodePtr OpPC,
1516 const InterpFrame *Frame,
1517 const CallExpr *Call,
1518 bool IsAdd) {
1519 if (Call->getNumArgs() != 4 || !Call->getArg(Arg: 0)->getType()->isIntegerType() ||
1520 !Call->getArg(Arg: 1)->getType()->isIntegerType() ||
1521 !Call->getArg(Arg: 2)->getType()->isIntegerType())
1522 return false;
1523
1524 const Pointer &CarryOutPtr = S.Stk.pop<Pointer>();
1525
1526 APSInt RHS;
1527 if (!popToAPSInt(S, E: Call->getArg(Arg: 2), Out&: RHS))
1528 return false;
1529 APSInt LHS;
1530 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: LHS))
1531 return false;
1532 APSInt CarryIn;
1533 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: CarryIn))
1534 return false;
1535
1536 unsigned BitWidth = LHS.getBitWidth();
1537 unsigned CarryInBit = CarryIn.ugt(RHS: 0) ? 1 : 0;
1538 APInt ExResult =
1539 IsAdd ? (LHS.zext(width: BitWidth + 1) + (RHS.zext(width: BitWidth + 1) + CarryInBit))
1540 : (LHS.zext(width: BitWidth + 1) - (RHS.zext(width: BitWidth + 1) + CarryInBit));
1541
1542 APInt Result = ExResult.extractBits(numBits: BitWidth, bitPosition: 0);
1543 APSInt CarryOut =
1544 APSInt(ExResult.extractBits(numBits: 1, bitPosition: BitWidth), /*IsUnsigned=*/true);
1545
1546 QualType CarryOutType = Call->getArg(Arg: 3)->getType()->getPointeeType();
1547 PrimType CarryOutT = *S.getContext().classify(T: CarryOutType);
1548 assignIntegral(S, Dest: CarryOutPtr, ValueT: CarryOutT, Value: APSInt(std::move(Result), true));
1549
1550 pushInteger(S, Val: CarryOut, QT: Call->getType());
1551
1552 return true;
1553}
1554
1555static bool interp__builtin_os_log_format_buffer_size(InterpState &S,
1556 CodePtr OpPC,
1557 const InterpFrame *Frame,
1558 const CallExpr *Call) {
1559 analyze_os_log::OSLogBufferLayout Layout;
1560 analyze_os_log::computeOSLogBufferLayout(Ctx&: S.getASTContext(), E: Call, layout&: Layout);
1561 pushInteger(S, Val: Layout.size().getQuantity(), QT: Call->getType());
1562 return true;
1563}
1564
1565static bool
1566interp__builtin_ptrauth_string_discriminator(InterpState &S, CodePtr OpPC,
1567 const InterpFrame *Frame,
1568 const CallExpr *Call) {
1569 const auto &Ptr = S.Stk.pop<Pointer>();
1570 if (!Ptr.isStringPointer())
1571 return false;
1572
1573 uint64_t Result = getPointerAuthStableSipHash(
1574 S: cast<StringLiteral>(Val: Ptr.getRootExpr())->getString());
1575 pushInteger(S, Val: Result, QT: Call->getType());
1576 return true;
1577}
1578
1579static bool interp__builtin_infer_alloc_token(InterpState &S, CodePtr OpPC,
1580 const InterpFrame *Frame,
1581 const CallExpr *Call) {
1582 const ASTContext &ASTCtx = S.getASTContext();
1583 uint64_t BitWidth = ASTCtx.getTypeSize(T: ASTCtx.getSizeType());
1584 auto Mode =
1585 ASTCtx.getLangOpts().AllocTokenMode.value_or(u: llvm::DefaultAllocTokenMode);
1586 auto MaxTokensOpt = ASTCtx.getLangOpts().AllocTokenMax;
1587 uint64_t MaxTokens =
1588 MaxTokensOpt.value_or(u: 0) ? *MaxTokensOpt : (~0ULL >> (64 - BitWidth));
1589
1590 // We do not read any of the arguments; discard them.
1591 for (int I = Call->getNumArgs() - 1; I >= 0; --I)
1592 discard(Stk&: S.Stk, T: S.getContext().classify(E: Call->getArg(Arg: I)).value_or(PT: PT_Ptr));
1593
1594 // Note: Type inference from a surrounding cast is not supported in
1595 // constexpr evaluation.
1596 QualType AllocType = infer_alloc::inferPossibleType(E: Call, Ctx: ASTCtx, CastE: nullptr);
1597 if (AllocType.isNull()) {
1598 S.CCEDiag(E: Call,
1599 DiagId: diag::note_constexpr_infer_alloc_token_type_inference_failed);
1600 return false;
1601 }
1602
1603 auto ATMD = infer_alloc::getAllocTokenMetadata(T: AllocType, Ctx: ASTCtx);
1604 if (!ATMD) {
1605 S.CCEDiag(E: Call, DiagId: diag::note_constexpr_infer_alloc_token_no_metadata);
1606 return false;
1607 }
1608
1609 auto MaybeToken = llvm::getAllocToken(Mode, Metadata: *ATMD, MaxTokens);
1610 if (!MaybeToken) {
1611 S.CCEDiag(E: Call, DiagId: diag::note_constexpr_infer_alloc_token_unsupported_mode);
1612 return false;
1613 }
1614
1615 pushInteger(S, Val: llvm::APInt(BitWidth, *MaybeToken), QT: ASTCtx.getSizeType());
1616 return true;
1617}
1618
1619static bool interp__builtin_operator_new(InterpState &S, CodePtr OpPC,
1620 const InterpFrame *Frame,
1621 const CallExpr *Call) {
1622 // A call to __operator_new is only valid within std::allocate<>::allocate.
1623 // Walk up the call stack to find the appropriate caller and get the
1624 // element type from it.
1625 auto [NewCall, ElemType] = S.getStdAllocatorCaller(Name: "allocate");
1626
1627 if (ElemType.isNull()) {
1628 S.FFDiag(E: Call, DiagId: S.getLangOpts().CPlusPlus20
1629 ? diag::note_constexpr_new_untyped
1630 : diag::note_constexpr_new);
1631 return false;
1632 }
1633 assert(NewCall);
1634
1635 if (ElemType->isIncompleteType() || ElemType->isFunctionType()) {
1636 S.FFDiag(E: Call, DiagId: diag::note_constexpr_new_not_complete_object_type)
1637 << (ElemType->isIncompleteType() ? 0 : 1) << ElemType;
1638 return false;
1639 }
1640
1641 // We only care about the first parameter (the size), so discard all the
1642 // others.
1643 {
1644 unsigned NumArgs = Call->getNumArgs();
1645 assert(NumArgs >= 1);
1646
1647 // The std::nothrow_t arg never gets put on the stack.
1648 if (Call->getArg(Arg: NumArgs - 1)->getType()->isNothrowT())
1649 --NumArgs;
1650 auto Args = ArrayRef(Call->getArgs(), Call->getNumArgs());
1651 // First arg is needed.
1652 Args = Args.drop_front();
1653
1654 // Discard the rest.
1655 for (const Expr *Arg : Args)
1656 discard(Stk&: S.Stk, T: *S.getContext().classify(E: Arg));
1657 }
1658
1659 APSInt Bytes;
1660 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: Bytes))
1661 return false;
1662 CharUnits ElemSize = S.getASTContext().getTypeSizeInChars(T: ElemType);
1663 assert(!ElemSize.isZero());
1664 // Divide the number of bytes by sizeof(ElemType), so we get the number of
1665 // elements we should allocate.
1666 APInt NumElems, Remainder;
1667 APInt ElemSizeAP(Bytes.getBitWidth(), ElemSize.getQuantity());
1668 APInt::udivrem(LHS: Bytes, RHS: ElemSizeAP, Quotient&: NumElems, Remainder);
1669 if (Remainder != 0) {
1670 // This likely indicates a bug in the implementation of 'std::allocator'.
1671 S.FFDiag(E: Call, DiagId: diag::note_constexpr_operator_new_bad_size)
1672 << Bytes << APSInt(ElemSizeAP, true) << ElemType;
1673 return false;
1674 }
1675
1676 // NB: The same check we're using in CheckArraySize()
1677 if (NumElems.getActiveBits() >
1678 ConstantArrayType::getMaxSizeBits(Context: S.getASTContext()) ||
1679 NumElems.ugt(RHS: Descriptor::MaxArrayElemBytes / ElemSize.getQuantity())) {
1680 // FIXME: NoThrow check?
1681 const SourceInfo &Loc = S.Current->getSource(PC: OpPC);
1682 S.FFDiag(SI: Loc, DiagId: diag::note_constexpr_new_too_large)
1683 << NumElems.getZExtValue();
1684 return false;
1685 }
1686
1687 if (!CheckArraySize(S, OpPC, NumElems: NumElems.getZExtValue()))
1688 return false;
1689
1690 bool IsArray = NumElems.ugt(RHS: 1);
1691 OptPrimType ElemT = S.getContext().classify(T: ElemType);
1692 DynamicAllocator &Allocator = S.getAllocator();
1693 if (ElemT) {
1694 Block *B =
1695 Allocator.allocate(Source: NewCall, T: *ElemT, NumElements: NumElems.getZExtValue(),
1696 EvalID: S.Ctx.getEvalID(), AllocForm: DynamicAllocator::Form::Operator);
1697 assert(B);
1698 S.Stk.push<Pointer>(Args: Pointer(B).atIndex(Idx: 0));
1699 return true;
1700 }
1701
1702 assert(!ElemT);
1703
1704 // Composite arrays
1705 if (IsArray) {
1706 const Descriptor *Desc =
1707 S.P.createDescriptor(D: NewCall, Ty: ElemType.getTypePtr());
1708 Block *B =
1709 Allocator.allocate(D: Desc, NumElements: NumElems.getZExtValue(), EvalID: S.Ctx.getEvalID(),
1710 AllocForm: DynamicAllocator::Form::Operator);
1711 assert(B);
1712 S.Stk.push<Pointer>(Args: Pointer(B).atIndex(Idx: 0).narrow());
1713 return true;
1714 }
1715
1716 // Records. Still allocate them as single-element arrays.
1717 QualType AllocType = S.getASTContext().getConstantArrayType(
1718 EltTy: ElemType, ArySize: NumElems, SizeExpr: nullptr, ASM: ArraySizeModifier::Normal, IndexTypeQuals: 0);
1719
1720 const Descriptor *Desc =
1721 S.P.createDescriptor(D: NewCall, Ty: AllocType.getTypePtr());
1722 Block *B = Allocator.allocate(D: Desc, EvalID: S.getContext().getEvalID(),
1723 AllocForm: DynamicAllocator::Form::Operator);
1724 assert(B);
1725 S.Stk.push<Pointer>(Args: Pointer(B).atIndex(Idx: 0).narrow());
1726 return true;
1727}
1728
1729static bool interp__builtin_operator_delete(InterpState &S, CodePtr OpPC,
1730 const InterpFrame *Frame,
1731 const CallExpr *Call) {
1732 const Expr *Source = nullptr;
1733 const Block *BlockToDelete = nullptr;
1734
1735 unsigned NumArgs = Call->getNumArgs();
1736 assert(NumArgs >= 1);
1737
1738 // Args are pushed in source order. The trailing sized/aligned delete
1739 // operands are above the pointer on the stack.
1740 for (unsigned I = NumArgs - 1; I != 0; --I)
1741 discard(Stk&: S.Stk, T: *S.getContext().classify(E: Call->getArg(Arg: I)));
1742
1743 if (S.checkingPotentialConstantExpression()) {
1744 S.Stk.discard<Pointer>();
1745 return false;
1746 }
1747
1748 // This is permitted only within a call to std::allocator<T>::deallocate.
1749 if (!S.getStdAllocatorCaller(Name: "deallocate")) {
1750 S.FFDiag(E: Call);
1751 S.Stk.discard<Pointer>();
1752 return true;
1753 }
1754
1755 {
1756 const Pointer &Ptr = S.Stk.pop<Pointer>();
1757
1758 if (Ptr.isZero()) {
1759 S.CCEDiag(E: Call, DiagId: diag::note_constexpr_deallocate_null);
1760 return true;
1761 }
1762
1763 Source = Ptr.getRootExpr();
1764 BlockToDelete = Ptr.block();
1765
1766 if (!BlockToDelete->isDynamic()) {
1767 S.FFDiag(E: Call, DiagId: diag::note_constexpr_delete_not_heap_alloc)
1768 << Ptr.toDiagnosticString(Ctx: S.getASTContext());
1769 if (const auto *D = Ptr.getFieldDesc()->asDecl())
1770 S.Note(Loc: D->getLocation(), DiagId: diag::note_declared_at);
1771 }
1772 }
1773 assert(BlockToDelete);
1774
1775 DynamicAllocator &Allocator = S.getAllocator();
1776 const Descriptor *BlockDesc = BlockToDelete->getDescriptor();
1777 std::optional<DynamicAllocator::Form> AllocForm =
1778 Allocator.getAllocationForm(Source);
1779
1780 if (!Allocator.deallocate(Source, BlockToDelete)) {
1781 // Nothing has been deallocated, this must be a double-delete.
1782 const SourceInfo &Loc = S.Current->getSource(PC: OpPC);
1783 S.FFDiag(SI: Loc, DiagId: diag::note_constexpr_double_delete);
1784 return false;
1785 }
1786 assert(AllocForm);
1787
1788 return CheckNewDeleteForms(
1789 S, OpPC, AllocForm: *AllocForm, DeleteForm: DynamicAllocator::Form::Operator, D: BlockDesc, NewExpr: Source);
1790}
1791
1792static bool interp__builtin_arithmetic_fence(InterpState &S, CodePtr OpPC,
1793 const InterpFrame *Frame,
1794 const CallExpr *Call) {
1795 const Floating &Arg0 = S.Stk.pop<Floating>();
1796 S.Stk.push<Floating>(Args: Arg0);
1797 return true;
1798}
1799
1800static bool interp__builtin_vector_reduce(InterpState &S, CodePtr OpPC,
1801 const CallExpr *Call, unsigned ID) {
1802 const Pointer &Arg = S.Stk.pop<Pointer>();
1803 assert(Arg.getFieldDesc()->isPrimitiveArray());
1804
1805 QualType ElemType = Arg.getFieldDesc()->getElemQualType();
1806 assert(Call->getType() == ElemType);
1807 PrimType ElemT = *S.getContext().classify(T: ElemType);
1808 unsigned NumElems = Arg.getNumElems();
1809
1810 if (!isIntegerType(T: ElemT))
1811 return false;
1812
1813 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
1814 T Result = Arg.elem<T>(0);
1815 unsigned BitWidth = Result.bitWidth();
1816 for (unsigned I = 1; I != NumElems; ++I) {
1817 T Elem = Arg.elem<T>(I);
1818 T PrevResult = Result;
1819
1820 if (ID == Builtin::BI__builtin_reduce_add) {
1821 if (T::add(Result, Elem, BitWidth, &Result)) {
1822 unsigned OverflowBits = BitWidth + 1;
1823 (void)handleOverflow(S, OpPC,
1824 (PrevResult.toAPSInt(OverflowBits) +
1825 Elem.toAPSInt(OverflowBits)));
1826 return false;
1827 }
1828 } else if (ID == Builtin::BI__builtin_reduce_mul) {
1829 if (T::mul(Result, Elem, BitWidth, &Result)) {
1830 unsigned OverflowBits = BitWidth * 2;
1831 (void)handleOverflow(S, OpPC,
1832 (PrevResult.toAPSInt(OverflowBits) *
1833 Elem.toAPSInt(OverflowBits)));
1834 return false;
1835 }
1836
1837 } else if (ID == Builtin::BI__builtin_reduce_and) {
1838 (void)T::bitAnd(Result, Elem, BitWidth, &Result);
1839 } else if (ID == Builtin::BI__builtin_reduce_or) {
1840 (void)T::bitOr(Result, Elem, BitWidth, &Result);
1841 } else if (ID == Builtin::BI__builtin_reduce_xor) {
1842 (void)T::bitXor(Result, Elem, BitWidth, &Result);
1843 } else if (ID == Builtin::BI__builtin_reduce_min) {
1844 if (Elem < Result)
1845 Result = Elem;
1846 } else if (ID == Builtin::BI__builtin_reduce_max) {
1847 if (Elem > Result)
1848 Result = Elem;
1849 } else {
1850 llvm_unreachable("Unhandled vector reduce builtin");
1851 }
1852 }
1853 pushInteger(S, Result.toAPSInt(), Call->getType());
1854 });
1855
1856 return true;
1857}
1858
1859static bool interp__builtin_elementwise_abs(InterpState &S, CodePtr OpPC,
1860 const InterpFrame *Frame,
1861 const CallExpr *Call,
1862 unsigned BuiltinID) {
1863 assert(Call->getNumArgs() == 1);
1864 QualType Ty = Call->getArg(Arg: 0)->getType();
1865 if (Ty->isIntegerType()) {
1866 APSInt Val;
1867 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: Val))
1868 return false;
1869 pushInteger(S, Val: Val.abs(), QT: Call->getType());
1870 return true;
1871 }
1872
1873 if (Ty->isFloatingType()) {
1874 Floating Val = S.Stk.pop<Floating>();
1875 Floating Result = abs(S, In: Val);
1876 S.Stk.push<Floating>(Args&: Result);
1877 return true;
1878 }
1879
1880 // Otherwise, the argument must be a vector.
1881 assert(Call->getArg(0)->getType()->isVectorType());
1882 const Pointer &Arg = S.Stk.pop<Pointer>();
1883 assert(Arg.getFieldDesc()->isPrimitiveArray());
1884 const Pointer &Dst = S.Stk.peek<Pointer>();
1885 assert(Dst.getFieldDesc()->isPrimitiveArray());
1886 assert(Arg.getFieldDesc()->getNumElems() ==
1887 Dst.getFieldDesc()->getNumElems());
1888
1889 QualType ElemType = Arg.getFieldDesc()->getElemQualType();
1890 PrimType ElemT = *S.getContext().classify(T: ElemType);
1891 unsigned NumElems = Arg.getNumElems();
1892 // we can either have a vector of integer or a vector of floating point
1893 for (unsigned I = 0; I != NumElems; ++I) {
1894 if (ElemType->isIntegerType()) {
1895 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
1896 Dst.elem<T>(I) = T::from(static_cast<T>(
1897 APSInt(Arg.elem<T>(I).toAPSInt().abs(),
1898 ElemType->isUnsignedIntegerOrEnumerationType())));
1899 });
1900 } else {
1901 Floating Val = Arg.elem<Floating>(I);
1902 Dst.elem<Floating>(I) = abs(S, In: Val);
1903 }
1904 }
1905 Dst.initializeAllElements();
1906
1907 return true;
1908}
1909
1910/// Can be called with an integer or vector as the first and only parameter.
1911static bool interp__builtin_elementwise_countzeroes(InterpState &S,
1912 CodePtr OpPC,
1913 const InterpFrame *Frame,
1914 const CallExpr *Call,
1915 unsigned BuiltinID) {
1916 bool HasZeroArg = Call->getNumArgs() == 2;
1917 bool IsCTTZ = BuiltinID == Builtin::BI__builtin_elementwise_ctzg;
1918 assert(Call->getNumArgs() == 1 || HasZeroArg);
1919 if (Call->getArg(Arg: 0)->getType()->isIntegerType()) {
1920 PrimType ArgT = *S.getContext().classify(T: Call->getArg(Arg: 0)->getType());
1921 APSInt Val;
1922 if (!popToAPSInt(Stk&: S.Stk, T: ArgT, Out&: Val))
1923 return false;
1924 std::optional<APSInt> ZeroVal;
1925 if (HasZeroArg) {
1926 ZeroVal = Val;
1927 if (!popToAPSInt(Stk&: S.Stk, T: ArgT, Out&: Val))
1928 return false;
1929 }
1930
1931 if (Val.isZero()) {
1932 if (ZeroVal) {
1933 pushInteger(S, Val: *ZeroVal, QT: Call->getType());
1934 return true;
1935 }
1936 // If we haven't been provided the second argument, the result is
1937 // undefined
1938 S.FFDiag(SI: S.Current->getSource(PC: OpPC),
1939 DiagId: diag::note_constexpr_countzeroes_zero)
1940 << /*IsTrailing=*/IsCTTZ;
1941 return false;
1942 }
1943
1944 if (BuiltinID == Builtin::BI__builtin_elementwise_clzg) {
1945 pushInteger(S, Val: Val.countLeadingZeros(), QT: Call->getType());
1946 } else {
1947 pushInteger(S, Val: Val.countTrailingZeros(), QT: Call->getType());
1948 }
1949 return true;
1950 }
1951 // Otherwise, the argument must be a vector.
1952 const ASTContext &ASTCtx = S.getASTContext();
1953 Pointer ZeroArg;
1954 if (HasZeroArg) {
1955 assert(Call->getArg(1)->getType()->isVectorType() &&
1956 ASTCtx.hasSameUnqualifiedType(Call->getArg(0)->getType(),
1957 Call->getArg(1)->getType()));
1958 (void)ASTCtx;
1959 ZeroArg = S.Stk.pop<Pointer>();
1960 assert(ZeroArg.getFieldDesc()->isPrimitiveArray());
1961 }
1962 assert(Call->getArg(0)->getType()->isVectorType());
1963 const Pointer &Arg = S.Stk.pop<Pointer>();
1964 assert(Arg.getFieldDesc()->isPrimitiveArray());
1965 const Pointer &Dst = S.Stk.peek<Pointer>();
1966 assert(Dst.getFieldDesc()->isPrimitiveArray());
1967 assert(Arg.getFieldDesc()->getNumElems() ==
1968 Dst.getFieldDesc()->getNumElems());
1969
1970 QualType ElemType = Arg.getFieldDesc()->getElemQualType();
1971 PrimType ElemT = *S.getContext().classify(T: ElemType);
1972 unsigned NumElems = Arg.getNumElems();
1973
1974 // FIXME: Reading from uninitialized vector elements?
1975 for (unsigned I = 0; I != NumElems; ++I) {
1976 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
1977 APInt EltVal = Arg.atIndex(I).deref<T>().toAPSInt();
1978 if (EltVal.isZero()) {
1979 if (HasZeroArg) {
1980 Dst.atIndex(I).deref<T>() = ZeroArg.atIndex(I).deref<T>();
1981 } else {
1982 // If we haven't been provided the second argument, the result is
1983 // undefined
1984 S.FFDiag(S.Current->getSource(OpPC),
1985 diag::note_constexpr_countzeroes_zero)
1986 << /*IsTrailing=*/IsCTTZ;
1987 return false;
1988 }
1989 } else if (IsCTTZ) {
1990 Dst.atIndex(I).deref<T>() = T::from(EltVal.countTrailingZeros());
1991 } else {
1992 Dst.atIndex(I).deref<T>() = T::from(EltVal.countLeadingZeros());
1993 }
1994 Dst.atIndex(I).initialize();
1995 });
1996 }
1997
1998 return true;
1999}
2000
2001static bool interp__builtin_memcpy(InterpState &S, CodePtr OpPC,
2002 const InterpFrame *Frame,
2003 const CallExpr *Call, unsigned ID) {
2004 assert(Call->getNumArgs() == 3);
2005 const ASTContext &ASTCtx = S.getASTContext();
2006 uint64_t Size;
2007 if (!popToUInt64(S, E: Call->getArg(Arg: 2), Out&: Size))
2008 return false;
2009 Pointer SrcPtr = S.Stk.pop<Pointer>().expand();
2010 Pointer DestPtr = S.Stk.pop<Pointer>().expand();
2011
2012 if (ID == Builtin::BImemcpy || ID == Builtin::BImemmove)
2013 diagnoseNonConstexprBuiltin(S, OpPC, ID);
2014
2015 bool Move =
2016 (ID == Builtin::BI__builtin_memmove || ID == Builtin::BImemmove ||
2017 ID == Builtin::BI__builtin_wmemmove || ID == Builtin::BIwmemmove);
2018 bool WChar = ID == Builtin::BIwmemcpy || ID == Builtin::BIwmemmove ||
2019 ID == Builtin::BI__builtin_wmemcpy ||
2020 ID == Builtin::BI__builtin_wmemmove;
2021
2022 // If the size is zero, we treat this as always being a valid no-op.
2023 if (Size == 0) {
2024 S.Stk.push<Pointer>(Args&: DestPtr);
2025 return true;
2026 }
2027
2028 if (SrcPtr.isZero() || DestPtr.isZero()) {
2029 Pointer DiagPtr = (SrcPtr.isZero() ? SrcPtr : DestPtr);
2030 S.FFDiag(SI: S.Current->getSource(PC: OpPC), DiagId: diag::note_constexpr_memcpy_null)
2031 << /*IsMove=*/Move << /*IsWchar=*/WChar << !SrcPtr.isZero()
2032 << DiagPtr.toDiagnosticString(Ctx: ASTCtx);
2033 return false;
2034 }
2035
2036 // Diagnose integral src/dest pointers specially.
2037 if (SrcPtr.isIntegralPointer() || DestPtr.isIntegralPointer()) {
2038 std::string DiagVal = "(void *)";
2039 DiagVal += SrcPtr.isIntegralPointer()
2040 ? std::to_string(val: SrcPtr.getIntegerRepresentation())
2041 : std::to_string(val: DestPtr.getIntegerRepresentation());
2042 S.FFDiag(SI: S.Current->getSource(PC: OpPC), DiagId: diag::note_constexpr_memcpy_null)
2043 << Move << WChar << DestPtr.isIntegralPointer() << DiagVal;
2044 return false;
2045 }
2046
2047 if (!DestPtr.isBlockPointer())
2048 return diagnoseUnreadableObject(S, OpPC, Ptr: DestPtr, AK: AK_Assign);
2049
2050 if (!isReadable(P: SrcPtr))
2051 return false;
2052
2053 if (DestPtr.getType()->isIncompleteType()) {
2054 S.FFDiag(SI: S.Current->getSource(PC: OpPC),
2055 DiagId: diag::note_constexpr_memcpy_incomplete_type)
2056 << Move << DestPtr.getType();
2057 return false;
2058 }
2059 if (SrcPtr.getType()->isIncompleteType()) {
2060 S.FFDiag(SI: S.Current->getSource(PC: OpPC),
2061 DiagId: diag::note_constexpr_memcpy_incomplete_type)
2062 << Move << SrcPtr.getType();
2063 return false;
2064 }
2065
2066 QualType DestElemType = getElemType(P: DestPtr);
2067 if (DestElemType->isIncompleteType()) {
2068 S.FFDiag(SI: S.Current->getSource(PC: OpPC),
2069 DiagId: diag::note_constexpr_memcpy_incomplete_type)
2070 << Move << DestElemType;
2071 return false;
2072 }
2073
2074 size_t RemainingDestElems;
2075 if (DestPtr.inArray()) {
2076 RemainingDestElems = DestPtr.isUnknownSizeArray()
2077 ? 0
2078 : (DestPtr.getNumElems() - DestPtr.getIndex());
2079 } else {
2080 RemainingDestElems = 1;
2081 }
2082 unsigned DestElemSize = ASTCtx.getTypeSizeInChars(T: DestElemType).getQuantity();
2083
2084 if (WChar) {
2085 uint64_t WCharSize =
2086 ASTCtx.getTypeSizeInChars(T: ASTCtx.getWCharType()).getQuantity();
2087 Size *= WCharSize;
2088 }
2089
2090 if (Size % DestElemSize != 0) {
2091 S.FFDiag(SI: S.Current->getSource(PC: OpPC),
2092 DiagId: diag::note_constexpr_memcpy_unsupported)
2093 << Move << WChar << 0 << DestElemType << Size << DestElemSize;
2094 return false;
2095 }
2096
2097 QualType SrcElemType = getElemType(P: SrcPtr);
2098 size_t RemainingSrcElems;
2099 if (SrcPtr.inArray()) {
2100 RemainingSrcElems = SrcPtr.isUnknownSizeArray()
2101 ? 0
2102 : (SrcPtr.getNumElems() - SrcPtr.getIndex());
2103 } else {
2104 RemainingSrcElems = 1;
2105 }
2106 unsigned SrcElemSize = ASTCtx.getTypeSizeInChars(T: SrcElemType).getQuantity();
2107
2108 if (!ASTCtx.hasSameUnqualifiedType(T1: DestElemType, T2: SrcElemType)) {
2109 S.FFDiag(SI: S.Current->getSource(PC: OpPC), DiagId: diag::note_constexpr_memcpy_type_pun)
2110 << Move << SrcElemType << DestElemType;
2111 return false;
2112 }
2113
2114 if (!DestElemType.isTriviallyCopyableType(Context: ASTCtx)) {
2115 S.FFDiag(SI: S.Current->getSource(PC: OpPC), DiagId: diag::note_constexpr_memcpy_nontrivial)
2116 << Move << DestElemType;
2117 return false;
2118 }
2119
2120 // Check if we have enough elements to read from and write to.
2121 size_t RemainingDestBytes = RemainingDestElems * DestElemSize;
2122 size_t RemainingSrcBytes = RemainingSrcElems * SrcElemSize;
2123 if (Size > RemainingDestBytes || Size > RemainingSrcBytes) {
2124 APInt N = APInt(64, Size / DestElemSize);
2125 S.FFDiag(SI: S.Current->getSource(PC: OpPC),
2126 DiagId: diag::note_constexpr_memcpy_unsupported)
2127 << Move << WChar << (Size > RemainingSrcBytes ? 1 : 2) << DestElemType
2128 << toString(I: N, Radix: 10, /*Signed=*/false);
2129 return false;
2130 }
2131
2132 // Check for overlapping memory regions.
2133 if (!Move && Pointer::pointToSameBlock(A: SrcPtr, B: DestPtr)) {
2134 // Remove base casts.
2135 Pointer SrcP = SrcPtr.stripBaseCasts();
2136 Pointer DestP = DestPtr.stripBaseCasts();
2137
2138 unsigned SrcIndex = SrcP.expand().getIndex() * SrcElemSize;
2139 unsigned DstIndex = DestP.expand().getIndex() * DestElemSize;
2140
2141 if ((SrcIndex <= DstIndex && (SrcIndex + Size) > DstIndex) ||
2142 (DstIndex <= SrcIndex && (DstIndex + Size) > SrcIndex)) {
2143 S.FFDiag(SI: S.Current->getSource(PC: OpPC), DiagId: diag::note_constexpr_memcpy_overlap)
2144 << /*IsWChar=*/false;
2145 return false;
2146 }
2147 }
2148
2149 assert(Size % DestElemSize == 0);
2150 if (!DoMemcpy(S, OpPC, SrcPtr, DestPtr, Size: Bytes(Size).toBits()))
2151 return false;
2152
2153 S.Stk.push<Pointer>(Args&: DestPtr);
2154 return true;
2155}
2156
2157/// Determine if T is a character type for which we guarantee that
2158/// sizeof(T) == 1.
2159static bool isOneByteCharacterType(QualType T) {
2160 return T->isCharType() || T->isChar8Type();
2161}
2162
2163// stdc_memreverse8(size_t N, unsigned char *P)
2164static bool interp__builtin_stdc_memreverse8(InterpState &S, CodePtr OpPC,
2165 const InterpFrame *Frame,
2166 const CallExpr *Call) {
2167 Pointer Ptr = S.Stk.pop<Pointer>();
2168
2169 uint64_t NElems;
2170 if (!popToUInt64(S, E: Call->getArg(Arg: 0), Out&: NElems))
2171 return false;
2172
2173 if (Ptr.isZero()) {
2174 S.FFDiag(SI: S.Current->getSource(PC: OpPC), DiagId: diag::note_constexpr_access_null)
2175 << AK_Assign;
2176 return false;
2177 }
2178
2179 if (!isReadable(P: Ptr) && !Ptr.isOnePastEnd())
2180 return false;
2181
2182 const Descriptor *Desc = Ptr.getFieldDesc();
2183 bool IsArray = Desc->isArray();
2184 QualType ElemTy = IsArray ? Desc->getElemQualType() : Desc->getType();
2185
2186 if (IsArray)
2187 Ptr = Ptr.expand();
2188
2189 uint64_t BaseIdx = Ptr.getIndex();
2190 uint64_t ArraySize = Ptr.getNumElems();
2191 uint64_t RemainingElems = ArraySize - BaseIdx;
2192 if (NElems > RemainingElems) {
2193 uint64_t LastIndex = llvm::SaturatingAdd(X: BaseIdx, Y: NElems - 1);
2194 if (IsArray)
2195 S.FFDiag(SI: S.Current->getSource(PC: OpPC), DiagId: diag::note_constexpr_array_index)
2196 << LastIndex << /*array*/ 0 << ArraySize;
2197 else
2198 S.FFDiag(SI: S.Current->getSource(PC: OpPC), DiagId: diag::note_constexpr_array_index)
2199 << LastIndex << /*non-array*/ 1;
2200 return false;
2201 }
2202
2203 if (NElems <= 1)
2204 return true;
2205
2206 PrimType ElemT = *S.getContext().classify(T: ElemTy);
2207
2208 for (uint64_t I = 0, Half = NElems / 2; I < Half; ++I) {
2209 Pointer LoPtr = Ptr.atIndex(Idx: BaseIdx + I);
2210 Pointer HiPtr = Ptr.atIndex(Idx: BaseIdx + NElems - 1 - I);
2211
2212 if (!CheckLoad(S, OpPC, Ptr: LoPtr, AK: AK_Read) ||
2213 !CheckLoad(S, OpPC, Ptr: HiPtr, AK: AK_Read) || !CheckStore(S, OpPC, Ptr: LoPtr) ||
2214 !CheckStore(S, OpPC, Ptr: HiPtr))
2215 return false;
2216
2217 INT_TYPE_SWITCH_NO_BOOL(ElemT,
2218 { std::swap(LoPtr.deref<T>(), HiPtr.deref<T>()); });
2219 LoPtr.initialize();
2220 HiPtr.initialize();
2221 }
2222 return true;
2223}
2224
2225static bool interp__builtin_load8(InterpState &S, CodePtr OpPC,
2226 const InterpFrame *Frame,
2227 const CallExpr *Call, bool IsBigEndian,
2228 bool IsAligned) {
2229 Pointer Ptr = S.Stk.pop<Pointer>();
2230
2231 if (Ptr.isZero()) {
2232 S.FFDiag(SI: S.Current->getSource(PC: OpPC), DiagId: diag::note_constexpr_access_null)
2233 << AK_Read;
2234 return false;
2235 }
2236
2237 if (!isReadable(P: Ptr) && !Ptr.isOnePastEnd())
2238 return false;
2239
2240 if (IsAligned) {
2241 CharUnits RequiredAlign =
2242 S.getASTContext().getTypeAlignInChars(T: Call->getType());
2243 CharUnits BaseAlignment = getBaseAlignment(ASTCtx: S.getASTContext(), Ptr);
2244 std::optional<size_t> LayoutOffset =
2245 Ptr.computeLayoutOffset(ASTCtx: S.getASTContext());
2246 if (!LayoutOffset)
2247 return false;
2248 CharUnits PtrOffset = CharUnits::fromQuantity(Quantity: *LayoutOffset);
2249 CharUnits PtrAlign = BaseAlignment.alignmentAtOffset(offset: PtrOffset);
2250 if (PtrAlign < RequiredAlign) {
2251 S.FFDiag(SI: S.Current->getSource(PC: OpPC), DiagId: diag::note_constexpr_load8_unaligned)
2252 << S.getASTContext().BuiltinInfo.getQuotedName(
2253 ID: Call->getBuiltinCallee())
2254 << RequiredAlign.getQuantity() << PtrAlign.getQuantity();
2255 return false;
2256 }
2257 }
2258
2259 // A string pointer has no Descriptor; treat it as an array of its
2260 // character type.
2261 bool IsArray = Ptr.isStringPointer() || Ptr.getFieldDesc()->isArray();
2262 QualType ElemTy = getElemType(P: Ptr);
2263
2264 if (IsArray)
2265 Ptr = Ptr.expand();
2266
2267 uint64_t BaseIdx = Ptr.getIndex();
2268 uint64_t ArraySize = Ptr.getNumElems();
2269 uint64_t RemainingElems = ArraySize - BaseIdx;
2270
2271 unsigned ByteWidth = S.getASTContext().getTypeSize(T: Call->getType()) / 8;
2272 if (ByteWidth > RemainingElems) {
2273 uint64_t LastIndex = llvm::SaturatingAdd(X: BaseIdx, Y: uint64_t(ByteWidth - 1));
2274 S.FFDiag(SI: S.Current->getSource(PC: OpPC), DiagId: diag::note_constexpr_array_index)
2275 << LastIndex << /*array*/ !IsArray << ArraySize;
2276 return false;
2277 }
2278
2279 OptPrimType ElemT = S.getContext().classify(T: ElemTy);
2280 if (!ElemT || !isIntegerOrBoolType(T: *ElemT))
2281 return false;
2282 uint64_t Result = 0;
2283
2284 if (IsArray) {
2285 // C2y §7.18.21: result = sum(b_index * 2^(8*index)) for index in [0, N/8)
2286 // where b_index = ptr[index] (LE) or ptr[N/8 - index - 1] (BE).
2287 for (unsigned I = 0; I != ByteWidth; ++I) {
2288 size_t SrcIdx = IsBigEndian ? (ByteWidth - I - 1) : I;
2289 Pointer BytePtr = Ptr.atIndex(Idx: BaseIdx + SrcIdx);
2290 if (!CheckLoad(S, OpPC, Ptr: BytePtr, AK: AK_Read))
2291 return false;
2292 INT_TYPE_SWITCH(*ElemT, {
2293 Result |= BytePtr.load<T>().toAPSInt().getZExtValue() << (8 * I);
2294 });
2295 }
2296 } else {
2297 assert(ByteWidth == 1 && "non-array pointer, expected a 1-byte load");
2298 if (!CheckLoad(S, OpPC, Ptr, AK: AK_Read))
2299 return false;
2300 INT_TYPE_SWITCH(*ElemT,
2301 { Result = Ptr.load<T>().toAPSInt().getZExtValue(); });
2302 }
2303
2304 bool IsSigned = Call->getType()->isSignedIntegerType();
2305 pushInteger(S, Val: APSInt(APInt(ByteWidth * 8, Result), !IsSigned),
2306 QT: Call->getType());
2307 return true;
2308}
2309
2310static bool interp__builtin_memcmp(InterpState &S, CodePtr OpPC,
2311 const InterpFrame *Frame,
2312 const CallExpr *Call, unsigned ID) {
2313 assert(Call->getNumArgs() == 3);
2314 uint64_t Size;
2315 if (!popToUInt64(S, E: Call->getArg(Arg: 2), Out&: Size))
2316 return false;
2317 const Pointer &PtrB = S.Stk.pop<Pointer>();
2318 const Pointer &PtrA = S.Stk.pop<Pointer>();
2319
2320 if (ID == Builtin::BImemcmp || ID == Builtin::BIbcmp ||
2321 ID == Builtin::BIwmemcmp)
2322 diagnoseNonConstexprBuiltin(S, OpPC, ID);
2323
2324 if (Size == 0) {
2325 pushInteger(S, Val: 0, QT: Call->getType());
2326 return true;
2327 }
2328 bool IsWide =
2329 (ID == Builtin::BIwmemcmp || ID == Builtin::BI__builtin_wmemcmp);
2330
2331 const ASTContext &ASTCtx = S.getASTContext();
2332 QualType ElemTypeA = getElemType(P: PtrA);
2333 QualType ElemTypeB = getElemType(P: PtrB);
2334 // FIXME: This is an arbitrary limitation the current constant interpreter
2335 // had. We could remove this.
2336 if (!IsWide && (!isOneByteCharacterType(T: ElemTypeA) ||
2337 !isOneByteCharacterType(T: ElemTypeB))) {
2338 S.FFDiag(SI: S.Current->getSource(PC: OpPC),
2339 DiagId: diag::note_constexpr_memcmp_unsupported)
2340 << ASTCtx.BuiltinInfo.getQuotedName(ID) << PtrA.getType()
2341 << PtrB.getType();
2342 return false;
2343 }
2344
2345 if (!PtrA.isReadablePointerType() || !PtrB.isReadablePointerType())
2346 return false;
2347
2348 if (!CheckLoad(S, OpPC, Ptr: PtrA, AK: AK_Read) || !CheckLoad(S, OpPC, Ptr: PtrB, AK: AK_Read))
2349 return false;
2350
2351 // Now, read both pointers to a buffer and compare those.
2352 BitcastBuffer BufferA(
2353 Bits(ASTCtx.getTypeSize(T: ElemTypeA) * PtrA.getNumElems()));
2354 readPointerToBuffer(Ctx: S.getContext(), FromPtr: PtrA, Buffer&: BufferA, /*ReturnOnUninit=*/false);
2355
2356 // FIXME: The swapping here is UNDOING something we do when reading the
2357 // data into the buffer.
2358 if (ASTCtx.getTargetInfo().isBigEndian())
2359 swapBytes(M: BufferA.Data.get(), N: BufferA.byteSize().getQuantity());
2360
2361 BitcastBuffer BufferB(
2362 Bits(ASTCtx.getTypeSize(T: ElemTypeB) * PtrB.getNumElems()));
2363 readPointerToBuffer(Ctx: S.getContext(), FromPtr: PtrB, Buffer&: BufferB, /*ReturnOnUninit=*/false);
2364 // FIXME: The swapping here is UNDOING something we do when reading the
2365 // data into the buffer.
2366 if (ASTCtx.getTargetInfo().isBigEndian())
2367 swapBytes(M: BufferB.Data.get(), N: BufferB.byteSize().getQuantity());
2368
2369 size_t MinBufferSize = std::min(a: BufferA.byteSize().getQuantity(),
2370 b: BufferB.byteSize().getQuantity());
2371
2372 unsigned ElemSize = 1;
2373 if (IsWide)
2374 ElemSize = ASTCtx.getTypeSizeInChars(T: ASTCtx.getWCharType()).getQuantity();
2375 // The Size given for the wide variants is in wide-char units. Convert it
2376 // to bytes.
2377 size_t ByteSize = Size * ElemSize;
2378 size_t CmpSize = std::min(a: MinBufferSize, b: ByteSize);
2379
2380 for (size_t I = 0; I != CmpSize; I += ElemSize) {
2381 if (IsWide) {
2382 FIXED_SIZE_INT_TYPE_SWITCH(
2383 *S.getContext().classify(ASTCtx.getWCharType()), {
2384 T A = T::bitcastFromMemory(BufferA.atByte(I), T::bitWidth());
2385 T B = T::bitcastFromMemory(BufferB.atByte(I), T::bitWidth());
2386 if (A < B) {
2387 pushInteger(S, -1, Call->getType());
2388 return true;
2389 }
2390 if (A > B) {
2391 pushInteger(S, 1, Call->getType());
2392 return true;
2393 }
2394 });
2395 } else {
2396 auto A = BufferA.deref<std::byte>(Offset: Bytes(I));
2397 auto B = BufferB.deref<std::byte>(Offset: Bytes(I));
2398
2399 if (A < B) {
2400 pushInteger(S, Val: -1, QT: Call->getType());
2401 return true;
2402 }
2403 if (A > B) {
2404 pushInteger(S, Val: 1, QT: Call->getType());
2405 return true;
2406 }
2407 }
2408 }
2409
2410 // We compared CmpSize bytes above. If the limiting factor was the Size
2411 // passed, we're done and the result is equality (0).
2412 if (ByteSize <= CmpSize) {
2413 pushInteger(S, Val: 0, QT: Call->getType());
2414 return true;
2415 }
2416
2417 // However, if we read all the available bytes but were instructed to read
2418 // even more, diagnose this as a "read of dereferenced one-past-the-end
2419 // pointer". This is what would happen if we called CheckLoad() on every array
2420 // element.
2421 S.FFDiag(SI: S.Current->getSource(PC: OpPC), DiagId: diag::note_constexpr_access_past_end)
2422 << AK_Read << S.Current->getRange(PC: OpPC);
2423 return false;
2424}
2425
2426// __builtin_memchr(ptr, int, int)
2427// __builtin_strchr(ptr, int)
2428static bool interp__builtin_memchr(InterpState &S, CodePtr OpPC,
2429 const CallExpr *Call, unsigned ID) {
2430 if (ID == Builtin::BImemchr || ID == Builtin::BIwcschr ||
2431 ID == Builtin::BIstrchr || ID == Builtin::BIwmemchr)
2432 diagnoseNonConstexprBuiltin(S, OpPC, ID);
2433
2434 std::optional<APSInt> MaxLength;
2435 if (Call->getNumArgs() == 3) {
2436 APSInt MaxLengthVal;
2437 if (!popToAPSInt(S, E: Call->getArg(Arg: 2), Out&: MaxLengthVal))
2438 return false;
2439 MaxLength = MaxLengthVal;
2440 }
2441
2442 APSInt Desired;
2443 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: Desired))
2444 return false;
2445 const Pointer &Ptr = S.Stk.pop<Pointer>();
2446
2447 if (MaxLength && MaxLength->isZero()) {
2448 S.Stk.push<Pointer>();
2449 return true;
2450 }
2451
2452 if (Ptr.isDummy()) {
2453 if (Ptr.getType()->isIncompleteType())
2454 S.FFDiag(SI: S.Current->getSource(PC: OpPC),
2455 DiagId: diag::note_constexpr_ltor_incomplete_type)
2456 << Ptr.getType();
2457 return false;
2458 }
2459
2460 // Null is only okay if the given size is 0.
2461 if (Ptr.isZero()) {
2462 S.FFDiag(SI: S.Current->getSource(PC: OpPC), DiagId: diag::note_constexpr_access_null)
2463 << AK_Read;
2464 return false;
2465 }
2466
2467 if (!Ptr.isReadablePointerType())
2468 return false;
2469
2470 QualType ElemTy = getElemType(P: Ptr);
2471 bool IsRawByte = ID == Builtin::BImemchr || ID == Builtin::BI__builtin_memchr;
2472
2473 // Give up on byte-oriented matching against multibyte elements.
2474 if (IsRawByte && !isOneByteCharacterType(T: ElemTy)) {
2475 S.FFDiag(SI: S.Current->getSource(PC: OpPC),
2476 DiagId: diag::note_constexpr_memchr_unsupported)
2477 << S.getASTContext().BuiltinInfo.getQuotedName(ID) << ElemTy;
2478 return false;
2479 }
2480
2481 if (!isReadable(P: Ptr))
2482 return false;
2483
2484 if (ID == Builtin::BIstrchr || ID == Builtin::BI__builtin_strchr) {
2485 int64_t DesiredTrunc;
2486 if (S.getASTContext().CharTy->isSignedIntegerType())
2487 DesiredTrunc =
2488 Desired.trunc(width: S.getASTContext().getCharWidth()).getSExtValue();
2489 else
2490 DesiredTrunc =
2491 Desired.trunc(width: S.getASTContext().getCharWidth()).getZExtValue();
2492 // strchr compares directly to the passed integer, and therefore
2493 // always fails if given an int that is not a char.
2494 if (Desired != DesiredTrunc) {
2495 S.Stk.push<Pointer>();
2496 return true;
2497 }
2498 }
2499
2500 uint64_t DesiredVal;
2501 if (ID == Builtin::BIwmemchr || ID == Builtin::BI__builtin_wmemchr ||
2502 ID == Builtin::BIwcschr || ID == Builtin::BI__builtin_wcschr) {
2503 // wcschr and wmemchr are given a wchar_t to look for. Just use it.
2504 DesiredVal = Desired.getZExtValue();
2505 } else {
2506 DesiredVal = Desired.trunc(width: S.getASTContext().getCharWidth()).getZExtValue();
2507 }
2508
2509 bool StopAtZero =
2510 (ID == Builtin::BIstrchr || ID == Builtin::BI__builtin_strchr ||
2511 ID == Builtin::BIwcschr || ID == Builtin::BI__builtin_wcschr);
2512
2513 PrimType ElemT =
2514 IsRawByte ? PT_Sint8 : *S.getContext().classify(T: getElemType(P: Ptr));
2515
2516 size_t Index = Ptr.getIndex();
2517 size_t Step = 0;
2518 for (;;) {
2519 const Pointer &ElemPtr =
2520 (Index + Step) > 0 ? Ptr.atIndex(Idx: Index + Step) : Ptr;
2521
2522 if (!CheckLoad(S, OpPC, Ptr: ElemPtr))
2523 return false;
2524
2525 uint64_t V;
2526 INT_TYPE_SWITCH_NO_BOOL(
2527 ElemT, { V = static_cast<uint64_t>(ElemPtr.load<T>().toUnsigned()); });
2528
2529 if (V == DesiredVal) {
2530 S.Stk.push<Pointer>(Args: ElemPtr);
2531 return true;
2532 }
2533
2534 if (StopAtZero && V == 0)
2535 break;
2536
2537 ++Step;
2538 if (MaxLength && Step == MaxLength->getZExtValue())
2539 break;
2540 }
2541
2542 S.Stk.push<Pointer>();
2543 return true;
2544}
2545
2546static bool interp__builtin_object_size(InterpState &S, CodePtr OpPC,
2547 const InterpFrame *Frame,
2548 const CallExpr *Call, bool IsDynamic) {
2549 const ASTContext &ASTCtx = S.getASTContext();
2550 // From the GCC docs:
2551 // Kind is an integer constant from 0 to 3. If the least significant bit is
2552 // clear, objects are whole variables. If it is set, a closest surrounding
2553 // subobject is considered the object a pointer points to. The second bit
2554 // determines if maximum or minimum of remaining bytes is computed.
2555 APSInt Kind;
2556 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: Kind))
2557 return false;
2558 if (Kind < 0 || Kind > 3) {
2559 S.FFDiag(E: Call->getArg(Arg: 1), DiagId: diag::note_constexpr_builtin_arg_out_of_range)
2560 << Kind << 0 << 3;
2561 return false;
2562 }
2563 unsigned KindValue = Kind.getZExtValue();
2564 Pointer Ptr = S.Stk.pop<Pointer>();
2565
2566 if (auto Result = evaluateBuiltinObjectSize(ASTCtx, Kind: KindValue, Ptr,
2567 E: Call->getArg(Arg: 0), IsDynamic)) {
2568 pushInteger(S, Val: *Result, QT: Call->getType());
2569 return true;
2570 }
2571
2572 if (Call->getArg(Arg: 0)->HasSideEffects(Ctx: ASTCtx)) {
2573 // "If there are any side effects in them, it returns (size_t) -1
2574 // for type 0 or 1 and (size_t) 0 for type 2 or 3."
2575 pushInteger(S, Val: KindValue <= 1 ? (size_t)-1 : (size_t)0, QT: Call->getType());
2576 return true;
2577 }
2578
2579 switch (S.EvalMode) {
2580 case EvaluationMode::ConstantExpression:
2581 case EvaluationMode::ConstantFold:
2582 case EvaluationMode::IgnoreSideEffects:
2583 // Leave it to IR generation.
2584 return Invalid(S, OpPC);
2585 case EvaluationMode::ConstantExpressionUnevaluated:
2586 // Reduce it to a constant now.
2587 pushInteger(S, Val: ((KindValue & 2u) ? (size_t)0 : (size_t)-1),
2588 QT: Call->getType());
2589 return true;
2590 }
2591
2592 return false;
2593}
2594
2595static bool interp__builtin_is_within_lifetime(InterpState &S, CodePtr OpPC,
2596 const CallExpr *Call) {
2597
2598 if (!S.inConstantContext())
2599 return false;
2600
2601 const Pointer &Ptr = S.Stk.pop<Pointer>();
2602
2603 auto Error = [&](int Diag) {
2604 bool CalledFromStd = false;
2605 const auto *Callee = S.Current->getCallee();
2606 if (Callee && Callee->isInStdNamespace()) {
2607 const IdentifierInfo *Identifier = Callee->getIdentifier();
2608 CalledFromStd = Identifier && Identifier->isStr(Str: "is_within_lifetime");
2609 }
2610 S.CCEDiag(SI: CalledFromStd
2611 ? S.Current->Caller->getSource(PC: S.Current->getRetPC())
2612 : S.Current->getSource(PC: OpPC),
2613 DiagId: diag::err_invalid_is_within_lifetime)
2614 << (CalledFromStd ? "std::is_within_lifetime"
2615 : "__builtin_is_within_lifetime")
2616 << Diag;
2617 return false;
2618 };
2619
2620 if (Ptr.isZero())
2621 return Error(0);
2622 if (Ptr.isOnePastEnd())
2623 return Error(1);
2624
2625 bool Result = Ptr.getLifetime() != Lifetime::Ended;
2626 if (!Ptr.isActive()) {
2627 Result = false;
2628 } else {
2629 if (!CheckLive(S, OpPC, Ptr, AK: AK_Read))
2630 return false;
2631 if (!CheckMutable(S, OpPC, Ptr))
2632 return false;
2633 if (!CheckDummy(S, OpPC, Ptr, AK: AK_Read))
2634 return false;
2635 }
2636
2637 if (!Ptr.isBlockPointer())
2638 return diagnoseUnreadableObject(S, OpPC, Ptr, AK: AK_Read);
2639
2640 // Check if we're currently running an initializer.
2641 if (S.initializingBlock(B: Ptr.block()))
2642 return Error(2);
2643 if (S.EvaluatingDecl && Ptr.getRootVarDecl() == S.EvaluatingDecl)
2644 return Error(2);
2645
2646 pushInteger(S, Val: Result, QT: Call->getType());
2647 return true;
2648}
2649
2650static bool interp__builtin_elementwise_int_unaryop(
2651 InterpState &S, CodePtr OpPC, const CallExpr *Call,
2652 llvm::function_ref<APInt(const APSInt &)> Fn) {
2653 assert(Call->getNumArgs() == 1);
2654
2655 // Single integer case.
2656 if (!Call->getArg(Arg: 0)->getType()->isVectorType()) {
2657 assert(Call->getType()->isIntegerType());
2658 APSInt Src;
2659 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: Src))
2660 return false;
2661 APInt Result = Fn(Src);
2662 pushInteger(S, Val: APSInt(std::move(Result), !Src.isSigned()), QT: Call->getType());
2663 return true;
2664 }
2665
2666 // Vector case.
2667 const Pointer &Arg = S.Stk.pop<Pointer>();
2668 assert(Arg.getFieldDesc()->isPrimitiveArray());
2669 const Pointer &Dst = S.Stk.peek<Pointer>();
2670 assert(Dst.getFieldDesc()->isPrimitiveArray());
2671 assert(Arg.getFieldDesc()->getNumElems() ==
2672 Dst.getFieldDesc()->getNumElems());
2673
2674 QualType ElemType = Arg.getFieldDesc()->getElemQualType();
2675 PrimType ElemT = *S.getContext().classify(T: ElemType);
2676 unsigned NumElems = Arg.getNumElems();
2677 bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
2678
2679 for (unsigned I = 0; I != NumElems; ++I) {
2680 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
2681 APSInt Src = Arg.elem<T>(I).toAPSInt();
2682 APInt Result = Fn(Src);
2683 Dst.elem<T>(I) = static_cast<T>(APSInt(std::move(Result), DestUnsigned));
2684 });
2685 }
2686 Dst.initializeAllElements();
2687
2688 return true;
2689}
2690
2691static bool interp__builtin_elementwise_fp_binop(
2692 InterpState &S, CodePtr OpPC, const CallExpr *Call,
2693 llvm::function_ref<std::optional<APFloat>(
2694 const APFloat &, const APFloat &, std::optional<APSInt> RoundingMode)>
2695 Fn,
2696 bool IsScalar = false) {
2697 assert((Call->getNumArgs() == 2) || (Call->getNumArgs() == 3));
2698 const auto *VT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
2699 assert(VT->getElementType()->isFloatingType());
2700 unsigned NumElems = VT->getNumElements();
2701
2702 // Vector case.
2703 assert(Call->getArg(0)->getType()->isVectorType() &&
2704 Call->getArg(1)->getType()->isVectorType());
2705 assert(VT->getElementType() ==
2706 Call->getArg(1)->getType()->castAs<VectorType>()->getElementType());
2707 assert(VT->getNumElements() ==
2708 Call->getArg(1)->getType()->castAs<VectorType>()->getNumElements());
2709
2710 std::optional<APSInt> RoundingMode = std::nullopt;
2711 if (Call->getNumArgs() == 3) {
2712 APSInt RoundingModeVal;
2713 if (!popToAPSInt(S, E: Call->getArg(Arg: 2), Out&: RoundingModeVal))
2714 return false;
2715 RoundingMode = RoundingModeVal;
2716 }
2717
2718 const Pointer &BPtr = S.Stk.pop<Pointer>();
2719 const Pointer &APtr = S.Stk.pop<Pointer>();
2720 const Pointer &Dst = S.Stk.peek<Pointer>();
2721 for (unsigned ElemIdx = 0; ElemIdx != NumElems; ++ElemIdx) {
2722 using T = PrimConv<PT_Float>::T;
2723 if (IsScalar && ElemIdx > 0) {
2724 Dst.elem<T>(I: ElemIdx) = APtr.elem<T>(I: ElemIdx);
2725 continue;
2726 }
2727 APFloat ElemA = APtr.elem<T>(I: ElemIdx).getAPFloat();
2728 APFloat ElemB = BPtr.elem<T>(I: ElemIdx).getAPFloat();
2729 std::optional<APFloat> Result = Fn(ElemA, ElemB, RoundingMode);
2730 if (!Result)
2731 return false;
2732 Dst.elem<T>(I: ElemIdx) = static_cast<T>(*Result);
2733 }
2734
2735 Dst.initializeAllElements();
2736
2737 return true;
2738}
2739
2740static bool interp__builtin_scalar_fp_round_mask_binop(
2741 InterpState &S, CodePtr OpPC, const CallExpr *Call,
2742 llvm::function_ref<std::optional<APFloat>(const APFloat &, const APFloat &,
2743 std::optional<APSInt>)>
2744 Fn) {
2745 assert(Call->getNumArgs() == 5);
2746 const auto *VT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
2747 unsigned NumElems = VT->getNumElements();
2748
2749 APSInt RoundingMode;
2750 if (!popToAPSInt(S, E: Call->getArg(Arg: 4), Out&: RoundingMode))
2751 return false;
2752 uint64_t MaskVal;
2753 if (!popToUInt64(S, E: Call->getArg(Arg: 3), Out&: MaskVal))
2754 return false;
2755 const Pointer &SrcPtr = S.Stk.pop<Pointer>();
2756 const Pointer &BPtr = S.Stk.pop<Pointer>();
2757 const Pointer &APtr = S.Stk.pop<Pointer>();
2758 const Pointer &Dst = S.Stk.peek<Pointer>();
2759
2760 using T = PrimConv<PT_Float>::T;
2761
2762 if (MaskVal & 1) {
2763 APFloat ElemA = APtr.elem<T>(I: 0).getAPFloat();
2764 APFloat ElemB = BPtr.elem<T>(I: 0).getAPFloat();
2765 std::optional<APFloat> Result = Fn(ElemA, ElemB, RoundingMode);
2766 if (!Result)
2767 return false;
2768 Dst.elem<T>(I: 0) = static_cast<T>(*Result);
2769 } else {
2770 Dst.elem<T>(I: 0) = SrcPtr.elem<T>(I: 0);
2771 }
2772
2773 for (unsigned I = 1; I < NumElems; ++I)
2774 Dst.elem<T>(I) = APtr.elem<T>(I);
2775
2776 Dst.initializeAllElements();
2777
2778 return true;
2779}
2780
2781static bool interp__builtin_elementwise_int_binop(
2782 InterpState &S, CodePtr OpPC, const CallExpr *Call,
2783 llvm::function_ref<APInt(const APSInt &, const APSInt &)> Fn) {
2784 assert(Call->getNumArgs() == 2);
2785
2786 // Single integer case.
2787 if (!Call->getArg(Arg: 0)->getType()->isVectorType()) {
2788 assert(!Call->getArg(1)->getType()->isVectorType());
2789 APSInt RHS;
2790 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: RHS))
2791 return false;
2792 APSInt LHS;
2793 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: LHS))
2794 return false;
2795 APInt Result = Fn(LHS, RHS);
2796 pushInteger(S, Val: APSInt(std::move(Result), !LHS.isSigned()), QT: Call->getType());
2797 return true;
2798 }
2799
2800 const auto *VT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
2801 assert(VT->getElementType()->isIntegralOrEnumerationType());
2802 PrimType ElemT = *S.getContext().classify(T: VT->getElementType());
2803 unsigned NumElems = VT->getNumElements();
2804 bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
2805
2806 // Vector + Scalar case.
2807 if (!Call->getArg(Arg: 1)->getType()->isVectorType()) {
2808 assert(Call->getArg(1)->getType()->isIntegralOrEnumerationType());
2809
2810 APSInt RHS;
2811 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: RHS))
2812 return false;
2813 const Pointer &LHS = S.Stk.pop<Pointer>();
2814 const Pointer &Dst = S.Stk.peek<Pointer>();
2815
2816 for (unsigned I = 0; I != NumElems; ++I) {
2817 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
2818 Dst.elem<T>(I) = static_cast<T>(
2819 APSInt(Fn(LHS.elem<T>(I).toAPSInt(), RHS), DestUnsigned));
2820 });
2821 }
2822 Dst.initializeAllElements();
2823 return true;
2824 }
2825
2826 // Vector case.
2827 assert(Call->getArg(0)->getType()->isVectorType() &&
2828 Call->getArg(1)->getType()->isVectorType());
2829 assert(VT->getElementType() ==
2830 Call->getArg(1)->getType()->castAs<VectorType>()->getElementType());
2831 assert(VT->getNumElements() ==
2832 Call->getArg(1)->getType()->castAs<VectorType>()->getNumElements());
2833 assert(VT->getElementType()->isIntegralOrEnumerationType());
2834
2835 const Pointer &RHS = S.Stk.pop<Pointer>();
2836 const Pointer &LHS = S.Stk.pop<Pointer>();
2837 const Pointer &Dst = S.Stk.peek<Pointer>();
2838 for (unsigned I = 0; I != NumElems; ++I) {
2839 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
2840 APSInt Elem1 = LHS.elem<T>(I).toAPSInt();
2841 APSInt Elem2 = RHS.elem<T>(I).toAPSInt();
2842 Dst.elem<T>(I) = static_cast<T>(APSInt(Fn(Elem1, Elem2), DestUnsigned));
2843 });
2844 }
2845 Dst.initializeAllElements();
2846
2847 return true;
2848}
2849
2850static bool
2851interp__builtin_ia32_pack(InterpState &S, CodePtr, const CallExpr *E,
2852 llvm::function_ref<APInt(const APSInt &)> PackFn) {
2853 const auto *VT0 = E->getArg(Arg: 0)->getType()->castAs<VectorType>();
2854 [[maybe_unused]] const auto *VT1 =
2855 E->getArg(Arg: 1)->getType()->castAs<VectorType>();
2856 assert(VT0 && VT1 && "pack builtin VT0 and VT1 must be VectorType");
2857 assert(VT0->getElementType() == VT1->getElementType() &&
2858 VT0->getNumElements() == VT1->getNumElements() &&
2859 "pack builtin VT0 and VT1 ElementType must be same");
2860
2861 const Pointer &RHS = S.Stk.pop<Pointer>();
2862 const Pointer &LHS = S.Stk.pop<Pointer>();
2863 const Pointer &Dst = S.Stk.peek<Pointer>();
2864
2865 const ASTContext &ASTCtx = S.getASTContext();
2866 unsigned SrcBits = ASTCtx.getIntWidth(T: VT0->getElementType());
2867 unsigned LHSVecLen = VT0->getNumElements();
2868 unsigned SrcPerLane = 128 / SrcBits;
2869 unsigned Lanes = LHSVecLen * SrcBits / 128;
2870
2871 PrimType SrcT = *S.getContext().classify(T: VT0->getElementType());
2872 PrimType DstT = *S.getContext().classify(T: getElemType(P: Dst));
2873 bool IsUnsigend = getElemType(P: Dst)->isUnsignedIntegerType();
2874
2875 for (unsigned Lane = 0; Lane != Lanes; ++Lane) {
2876 unsigned BaseSrc = Lane * SrcPerLane;
2877 unsigned BaseDst = Lane * (2 * SrcPerLane);
2878
2879 for (unsigned I = 0; I != SrcPerLane; ++I) {
2880 INT_TYPE_SWITCH_NO_BOOL(SrcT, {
2881 APSInt A = LHS.elem<T>(BaseSrc + I).toAPSInt();
2882 APSInt B = RHS.elem<T>(BaseSrc + I).toAPSInt();
2883
2884 assignIntegral(S, Dst.atIndex(BaseDst + I), DstT,
2885 APSInt(PackFn(A), IsUnsigend));
2886 assignIntegral(S, Dst.atIndex(BaseDst + SrcPerLane + I), DstT,
2887 APSInt(PackFn(B), IsUnsigend));
2888 });
2889 }
2890 }
2891
2892 Dst.initializeAllElements();
2893 return true;
2894}
2895
2896static bool interp__builtin_elementwise_maxmin(InterpState &S, CodePtr OpPC,
2897 const CallExpr *Call,
2898 unsigned BuiltinID) {
2899 assert(Call->getNumArgs() == 2);
2900
2901 QualType Arg0Type = Call->getArg(Arg: 0)->getType();
2902
2903 // TODO: Support floating-point types.
2904 if (!(Arg0Type->isIntegerType() ||
2905 (Arg0Type->isVectorType() &&
2906 Arg0Type->castAs<VectorType>()->getElementType()->isIntegerType())))
2907 return false;
2908
2909 if (!Arg0Type->isVectorType()) {
2910 assert(!Call->getArg(1)->getType()->isVectorType());
2911 APSInt RHS;
2912 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: RHS))
2913 return false;
2914 APSInt LHS;
2915 if (!popToAPSInt(S, T: Arg0Type, Out&: LHS))
2916 return false;
2917 APInt Result;
2918 if (BuiltinID == Builtin::BI__builtin_elementwise_max) {
2919 Result = std::max(a: LHS, b: RHS);
2920 } else if (BuiltinID == Builtin::BI__builtin_elementwise_min) {
2921 Result = std::min(a: LHS, b: RHS);
2922 } else {
2923 llvm_unreachable("Wrong builtin ID");
2924 }
2925
2926 pushInteger(S, Val: APSInt(Result, !LHS.isSigned()), QT: Call->getType());
2927 return true;
2928 }
2929
2930 // Vector case.
2931 assert(Call->getArg(0)->getType()->isVectorType() &&
2932 Call->getArg(1)->getType()->isVectorType());
2933 const auto *VT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
2934 assert(VT->getElementType() ==
2935 Call->getArg(1)->getType()->castAs<VectorType>()->getElementType());
2936 assert(VT->getNumElements() ==
2937 Call->getArg(1)->getType()->castAs<VectorType>()->getNumElements());
2938 assert(VT->getElementType()->isIntegralOrEnumerationType());
2939
2940 const Pointer &RHS = S.Stk.pop<Pointer>();
2941 const Pointer &LHS = S.Stk.pop<Pointer>();
2942 const Pointer &Dst = S.Stk.peek<Pointer>();
2943 PrimType ElemT = *S.getContext().classify(T: VT->getElementType());
2944 unsigned NumElems = VT->getNumElements();
2945 for (unsigned I = 0; I != NumElems; ++I) {
2946 APSInt Elem1;
2947 APSInt Elem2;
2948 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
2949 Elem1 = LHS.elem<T>(I).toAPSInt();
2950 Elem2 = RHS.elem<T>(I).toAPSInt();
2951 });
2952
2953 APSInt Result;
2954 if (BuiltinID == Builtin::BI__builtin_elementwise_max) {
2955 Result = APSInt(std::max(a: Elem1, b: Elem2),
2956 Call->getType()->isUnsignedIntegerOrEnumerationType());
2957 } else if (BuiltinID == Builtin::BI__builtin_elementwise_min) {
2958 Result = APSInt(std::min(a: Elem1, b: Elem2),
2959 Call->getType()->isUnsignedIntegerOrEnumerationType());
2960 } else {
2961 llvm_unreachable("Wrong builtin ID");
2962 }
2963
2964 INT_TYPE_SWITCH_NO_BOOL(ElemT,
2965 { Dst.elem<T>(I) = static_cast<T>(Result); });
2966 }
2967 Dst.initializeAllElements();
2968
2969 return true;
2970}
2971
2972static bool interp__builtin_ia32_pmul(
2973 InterpState &S, CodePtr OpPC, const CallExpr *Call,
2974 llvm::function_ref<APInt(const APSInt &, const APSInt &, const APSInt &,
2975 const APSInt &)>
2976 Fn) {
2977 assert(Call->getArg(0)->getType()->isVectorType() &&
2978 Call->getArg(1)->getType()->isVectorType());
2979 const Pointer &RHS = S.Stk.pop<Pointer>();
2980 const Pointer &LHS = S.Stk.pop<Pointer>();
2981 const Pointer &Dst = S.Stk.peek<Pointer>();
2982
2983 const auto *VT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
2984 PrimType ElemT = *S.getContext().classify(T: VT->getElementType());
2985 unsigned NumElems = VT->getNumElements();
2986 const auto *DestVT = Call->getType()->castAs<VectorType>();
2987 PrimType DestElemT = *S.getContext().classify(T: DestVT->getElementType());
2988 bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
2989
2990 unsigned DstElem = 0;
2991 for (unsigned I = 0; I != NumElems; I += 2) {
2992 APSInt Result;
2993 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
2994 APSInt LoLHS = LHS.elem<T>(I).toAPSInt();
2995 APSInt HiLHS = LHS.elem<T>(I + 1).toAPSInt();
2996 APSInt LoRHS = RHS.elem<T>(I).toAPSInt();
2997 APSInt HiRHS = RHS.elem<T>(I + 1).toAPSInt();
2998 Result = APSInt(Fn(LoLHS, HiLHS, LoRHS, HiRHS), DestUnsigned);
2999 });
3000
3001 INT_TYPE_SWITCH_NO_BOOL(DestElemT,
3002 { Dst.elem<T>(DstElem) = static_cast<T>(Result); });
3003 ++DstElem;
3004 }
3005
3006 Dst.initializeAllElements();
3007 return true;
3008}
3009
3010static bool interp__builtin_ia32_psadbw(InterpState &S, CodePtr OpPC,
3011 const CallExpr *Call) {
3012 assert(Call->getNumArgs() == 2);
3013
3014 const Pointer &RHS = S.Stk.pop<Pointer>();
3015 const Pointer &LHS = S.Stk.pop<Pointer>();
3016 const Pointer &Dst = S.Stk.peek<Pointer>();
3017
3018 const auto *SrcVT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
3019 PrimType SrcElemT = *S.getContext().classify(T: SrcVT->getElementType());
3020 unsigned SourceLen = SrcVT->getNumElements();
3021 assert((SourceLen % 8) == 0);
3022
3023 const auto *DestVT = Call->getType()->castAs<VectorType>();
3024 PrimType DestElemT = *S.getContext().classify(T: DestVT->getElementType());
3025 bool DestUnsigned =
3026 DestVT->getElementType()->isUnsignedIntegerOrEnumerationType();
3027
3028 unsigned DstElem = 0;
3029 for (unsigned Lane = 0; Lane != SourceLen; Lane += 8) {
3030 APInt Sum(64, 0);
3031 for (unsigned I = 0; I != 8; ++I) {
3032 INT_TYPE_SWITCH_NO_BOOL(SrcElemT, {
3033 APSInt L = LHS.elem<T>(Lane + I).toAPSInt();
3034 APSInt R = RHS.elem<T>(Lane + I).toAPSInt();
3035 Sum += llvm::APIntOps::abdu(L.extOrTrunc(8), R.extOrTrunc(8)).zext(64);
3036 });
3037 }
3038
3039 INT_TYPE_SWITCH_NO_BOOL(DestElemT, {
3040 Dst.elem<T>(DstElem) = static_cast<T>(APSInt(Sum, DestUnsigned));
3041 });
3042 ++DstElem;
3043 }
3044
3045 Dst.initializeAllElements();
3046 return true;
3047}
3048
3049static bool interp__builtin_ia32_dbpsadbw(InterpState &S, CodePtr OpPC,
3050 const CallExpr *Call) {
3051 assert(Call->getNumArgs() == 3);
3052 uint64_t Imm;
3053 if (!popToUInt64(S, E: Call->getArg(Arg: 2), Out&: Imm))
3054 return false;
3055
3056 const Pointer &Src2 = S.Stk.pop<Pointer>();
3057 const Pointer &Src1 = S.Stk.pop<Pointer>();
3058 const Pointer &Dst = S.Stk.peek<Pointer>();
3059
3060 const auto *SrcVT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
3061 PrimType SrcElemT = *S.getContext().classify(T: SrcVT->getElementType());
3062 unsigned SourceLen = SrcVT->getNumElements();
3063
3064 const auto *DestVT = Call->getType()->castAs<VectorType>();
3065 PrimType DestElemT = *S.getContext().classify(T: DestVT->getElementType());
3066 bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
3067
3068 constexpr unsigned LaneSize = 16; // 128-bit lane = 16 bytes
3069
3070 // Phase 1: Shuffle Src2 using all four 2-bit fields of imm8.
3071 // Within each 128-bit lane, for group j (0..3), select a 4-byte block
3072 // from Src2 based on bits [2*j+1:2*j] of imm8.
3073 SmallVector<uint8_t, 64> Shuffled(SourceLen);
3074 for (unsigned I = 0; I < SourceLen; I += LaneSize) {
3075 for (unsigned J = 0; J < 4; ++J) {
3076 unsigned Part = (Imm >> (2 * J)) & 3;
3077 for (unsigned K = 0; K < 4; ++K) {
3078 INT_TYPE_SWITCH_NO_BOOL(SrcElemT, {
3079 Shuffled[I + 4 * J + K] =
3080 static_cast<uint8_t>(Src2.elem<T>(I + 4 * Part + K));
3081 });
3082 }
3083 }
3084 }
3085
3086 // Phase 2: Sliding SAD computation.
3087 // For every group of 4 output u16 values, compute absolute differences
3088 // using overlapping windows into Src1 and the shuffled array.
3089 unsigned Size = SourceLen / 2; // number of output u16 elements
3090 for (unsigned I = 0; I < Size; I += 4) {
3091 unsigned Sad[4] = {0, 0, 0, 0};
3092 for (unsigned J = 0; J < 4; ++J) {
3093 uint8_t A1, A2;
3094 INT_TYPE_SWITCH_NO_BOOL(SrcElemT, {
3095 A1 = static_cast<uint8_t>(Src1.elem<T>(2 * I + J));
3096 A2 = static_cast<uint8_t>(Src1.elem<T>(2 * I + J + 4));
3097 });
3098 uint8_t B0 = Shuffled[2 * I + J];
3099 uint8_t B1 = Shuffled[2 * I + J + 1];
3100 uint8_t B2 = Shuffled[2 * I + J + 2];
3101 uint8_t B3 = Shuffled[2 * I + J + 3];
3102 Sad[0] += (A1 > B0) ? (A1 - B0) : (B0 - A1);
3103 Sad[1] += (A1 > B1) ? (A1 - B1) : (B1 - A1);
3104 Sad[2] += (A2 > B2) ? (A2 - B2) : (B2 - A2);
3105 Sad[3] += (A2 > B3) ? (A2 - B3) : (B3 - A2);
3106 }
3107 for (unsigned R = 0; R < 4; ++R) {
3108 INT_TYPE_SWITCH_NO_BOOL(DestElemT, {
3109 Dst.elem<T>(I + R) =
3110 static_cast<T>(APSInt(APInt(16, Sad[R]), DestUnsigned));
3111 });
3112 }
3113 }
3114
3115 Dst.initializeAllElements();
3116 return true;
3117}
3118
3119static bool interp__builtin_ia32_mpsadbw(InterpState &S, CodePtr OpPC,
3120 const CallExpr *Call) {
3121 assert(Call->getNumArgs() == 3);
3122 uint64_t Imm;
3123 if (!popToUInt64(S, E: Call->getArg(Arg: 2), Out&: Imm))
3124 return false;
3125
3126 const Pointer &Src2 = S.Stk.pop<Pointer>();
3127 const Pointer &Src1 = S.Stk.pop<Pointer>();
3128 const Pointer &Dst = S.Stk.peek<Pointer>();
3129
3130 const auto *SrcVT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
3131 PrimType SrcElemT = *S.getContext().classify(T: SrcVT->getElementType());
3132 unsigned SourceLen = SrcVT->getNumElements();
3133 assert((SourceLen == 16 || SourceLen == 32) &&
3134 "MPSADBW operates on 128-bit or 256-bit vectors");
3135
3136 const auto *DestVT = Call->getType()->castAs<VectorType>();
3137 PrimType DestElemT = *S.getContext().classify(T: DestVT->getElementType());
3138 bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
3139
3140 constexpr unsigned LaneSize = 16; // 128-bit lane = 16 bytes
3141 unsigned NumLanes = SourceLen / LaneSize;
3142
3143 for (unsigned Lane = 0; Lane != NumLanes; ++Lane) {
3144 unsigned Ctrl = (Imm >> (3 * Lane)) & 0x7;
3145 unsigned AOff = ((Ctrl >> 2) & 1) * 4;
3146 unsigned BOff = (Ctrl & 3) * 4;
3147 for (unsigned J = 0; J != 8; ++J) {
3148 uint16_t Sad = 0;
3149 for (unsigned K = 0; K != 4; ++K) {
3150 uint8_t A, B;
3151 INT_TYPE_SWITCH_NO_BOOL(SrcElemT, {
3152 A = static_cast<uint8_t>(
3153 Src1.elem<T>(Lane * LaneSize + AOff + J + K));
3154 B = static_cast<uint8_t>(Src2.elem<T>(Lane * LaneSize + BOff + K));
3155 });
3156 Sad += (A > B) ? (A - B) : (B - A);
3157 }
3158 INT_TYPE_SWITCH_NO_BOOL(DestElemT, {
3159 Dst.elem<T>(Lane * 8 + J) =
3160 static_cast<T>(APSInt(APInt(16, Sad), DestUnsigned));
3161 });
3162 }
3163 }
3164
3165 Dst.initializeAllElements();
3166 return true;
3167}
3168
3169static bool interp_builtin_horizontal_int_binop(
3170 InterpState &S, CodePtr OpPC, const CallExpr *Call,
3171 llvm::function_ref<APInt(const APSInt &, const APSInt &)> Fn) {
3172 const auto *VT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
3173 PrimType ElemT = *S.getContext().classify(T: VT->getElementType());
3174 bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
3175
3176 const Pointer &RHS = S.Stk.pop<Pointer>();
3177 const Pointer &LHS = S.Stk.pop<Pointer>();
3178 const Pointer &Dst = S.Stk.peek<Pointer>();
3179 unsigned NumElts = VT->getNumElements();
3180 unsigned EltBits = S.getASTContext().getIntWidth(T: VT->getElementType());
3181 unsigned EltsPerLane = 128 / EltBits;
3182 unsigned Lanes = NumElts * EltBits / 128;
3183 unsigned DestIndex = 0;
3184
3185 for (unsigned Lane = 0; Lane < Lanes; ++Lane) {
3186 unsigned LaneStart = Lane * EltsPerLane;
3187 for (unsigned I = 0; I < EltsPerLane; I += 2) {
3188 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
3189 APSInt Elem1 = LHS.elem<T>(LaneStart + I).toAPSInt();
3190 APSInt Elem2 = LHS.elem<T>(LaneStart + I + 1).toAPSInt();
3191 APSInt ResL = APSInt(Fn(Elem1, Elem2), DestUnsigned);
3192 Dst.elem<T>(DestIndex++) = static_cast<T>(ResL);
3193 });
3194 }
3195
3196 for (unsigned I = 0; I < EltsPerLane; I += 2) {
3197 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
3198 APSInt Elem1 = RHS.elem<T>(LaneStart + I).toAPSInt();
3199 APSInt Elem2 = RHS.elem<T>(LaneStart + I + 1).toAPSInt();
3200 APSInt ResR = APSInt(Fn(Elem1, Elem2), DestUnsigned);
3201 Dst.elem<T>(DestIndex++) = static_cast<T>(ResR);
3202 });
3203 }
3204 }
3205 Dst.initializeAllElements();
3206 return true;
3207}
3208
3209static bool interp_builtin_horizontal_fp_binop(
3210 InterpState &S, CodePtr OpPC, const CallExpr *Call,
3211 llvm::function_ref<APFloat(const APFloat &, const APFloat &,
3212 llvm::RoundingMode)>
3213 Fn) {
3214 const Pointer &RHS = S.Stk.pop<Pointer>();
3215 const Pointer &LHS = S.Stk.pop<Pointer>();
3216 const Pointer &Dst = S.Stk.peek<Pointer>();
3217 FPOptions FPO = Call->getFPFeaturesInEffect(LO: S.Ctx.getLangOpts());
3218 llvm::RoundingMode RM = getRoundingMode(FPO);
3219 const auto *VT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
3220
3221 unsigned NumElts = VT->getNumElements();
3222 unsigned EltBits = S.getASTContext().getTypeSize(T: VT->getElementType());
3223 unsigned NumLanes = NumElts * EltBits / 128;
3224 unsigned NumElemsPerLane = NumElts / NumLanes;
3225 unsigned HalfElemsPerLane = NumElemsPerLane / 2;
3226
3227 for (unsigned L = 0; L != NumElts; L += NumElemsPerLane) {
3228 using T = PrimConv<PT_Float>::T;
3229 for (unsigned E = 0; E != HalfElemsPerLane; ++E) {
3230 APFloat Elem1 = LHS.elem<T>(I: L + (2 * E) + 0).getAPFloat();
3231 APFloat Elem2 = LHS.elem<T>(I: L + (2 * E) + 1).getAPFloat();
3232 Dst.elem<T>(I: L + E) = static_cast<T>(Fn(Elem1, Elem2, RM));
3233 }
3234 for (unsigned E = 0; E != HalfElemsPerLane; ++E) {
3235 APFloat Elem1 = RHS.elem<T>(I: L + (2 * E) + 0).getAPFloat();
3236 APFloat Elem2 = RHS.elem<T>(I: L + (2 * E) + 1).getAPFloat();
3237 Dst.elem<T>(I: L + E + HalfElemsPerLane) =
3238 static_cast<T>(Fn(Elem1, Elem2, RM));
3239 }
3240 }
3241 Dst.initializeAllElements();
3242 return true;
3243}
3244
3245static bool interp__builtin_ia32_addsub(InterpState &S, CodePtr OpPC,
3246 const CallExpr *Call) {
3247 // Addsub: alternates between subtraction and addition
3248 // Result[i] = (i % 2 == 0) ? (a[i] - b[i]) : (a[i] + b[i])
3249 const Pointer &RHS = S.Stk.pop<Pointer>();
3250 const Pointer &LHS = S.Stk.pop<Pointer>();
3251 const Pointer &Dst = S.Stk.peek<Pointer>();
3252 FPOptions FPO = Call->getFPFeaturesInEffect(LO: S.Ctx.getLangOpts());
3253 llvm::RoundingMode RM = getRoundingMode(FPO);
3254 const auto *VT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
3255 unsigned NumElems = VT->getNumElements();
3256
3257 using T = PrimConv<PT_Float>::T;
3258 for (unsigned I = 0; I != NumElems; ++I) {
3259 APFloat LElem = LHS.elem<T>(I).getAPFloat();
3260 APFloat RElem = RHS.elem<T>(I).getAPFloat();
3261 if (I % 2 == 0) {
3262 // Even indices: subtract
3263 LElem.subtract(RHS: RElem, RM);
3264 } else {
3265 // Odd indices: add
3266 LElem.add(RHS: RElem, RM);
3267 }
3268 Dst.elem<T>(I) = static_cast<T>(LElem);
3269 }
3270 Dst.initializeAllElements();
3271 return true;
3272}
3273
3274static bool interp__builtin_ia32_pclmulqdq(InterpState &S, CodePtr OpPC,
3275 const CallExpr *Call) {
3276 // PCLMULQDQ: carry-less multiplication of selected 64-bit halves
3277 // imm8 bit 0: selects lower (0) or upper (1) 64 bits of first operand
3278 // imm8 bit 4: selects lower (0) or upper (1) 64 bits of second operand
3279 assert(Call->getArg(0)->getType()->isVectorType() &&
3280 Call->getArg(1)->getType()->isVectorType());
3281
3282 // Extract imm8 argument
3283 APSInt Imm8;
3284 if (!popToAPSInt(S, E: Call->getArg(Arg: 2), Out&: Imm8))
3285 return false;
3286 bool SelectUpperA = (Imm8 & 0x01) != 0;
3287 bool SelectUpperB = (Imm8 & 0x10) != 0;
3288
3289 const Pointer &RHS = S.Stk.pop<Pointer>();
3290 const Pointer &LHS = S.Stk.pop<Pointer>();
3291 const Pointer &Dst = S.Stk.peek<Pointer>();
3292
3293 const auto *VT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
3294 PrimType ElemT = *S.getContext().classify(T: VT->getElementType());
3295 unsigned NumElems = VT->getNumElements();
3296 const auto *DestVT = Call->getType()->castAs<VectorType>();
3297 PrimType DestElemT = *S.getContext().classify(T: DestVT->getElementType());
3298 bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
3299
3300 // Process each 128-bit lane (2 elements at a time)
3301 for (unsigned Lane = 0; Lane < NumElems; Lane += 2) {
3302 APSInt A0, A1, B0, B1;
3303 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
3304 A0 = LHS.elem<T>(Lane + 0).toAPSInt();
3305 A1 = LHS.elem<T>(Lane + 1).toAPSInt();
3306 B0 = RHS.elem<T>(Lane + 0).toAPSInt();
3307 B1 = RHS.elem<T>(Lane + 1).toAPSInt();
3308 });
3309
3310 // Select the appropriate 64-bit values based on imm8
3311 APInt A = SelectUpperA ? A1 : A0;
3312 APInt B = SelectUpperB ? B1 : B0;
3313
3314 // Extend both operands to 128 bits for carry-less multiplication
3315 APInt A128 = A.zext(width: 128);
3316 APInt B128 = B.zext(width: 128);
3317
3318 // Use APIntOps::clmul for carry-less multiplication
3319 APInt Result = llvm::APIntOps::clmul(LHS: A128, RHS: B128);
3320
3321 // Split the 128-bit result into two 64-bit halves
3322 APSInt ResultLow(Result.extractBits(numBits: 64, bitPosition: 0), DestUnsigned);
3323 APSInt ResultHigh(Result.extractBits(numBits: 64, bitPosition: 64), DestUnsigned);
3324
3325 INT_TYPE_SWITCH_NO_BOOL(DestElemT, {
3326 Dst.elem<T>(Lane + 0) = static_cast<T>(ResultLow);
3327 Dst.elem<T>(Lane + 1) = static_cast<T>(ResultHigh);
3328 });
3329 }
3330
3331 Dst.initializeAllElements();
3332 return true;
3333}
3334
3335static bool interp__builtin_elementwise_triop_fp(
3336 InterpState &S, CodePtr OpPC, const CallExpr *Call,
3337 llvm::function_ref<APFloat(const APFloat &, const APFloat &,
3338 const APFloat &, llvm::RoundingMode)>
3339 Fn) {
3340 assert(Call->getNumArgs() == 3);
3341
3342 FPOptions FPO = Call->getFPFeaturesInEffect(LO: S.Ctx.getLangOpts());
3343 llvm::RoundingMode RM = getRoundingMode(FPO);
3344 QualType Arg1Type = Call->getArg(Arg: 0)->getType();
3345 QualType Arg2Type = Call->getArg(Arg: 1)->getType();
3346 QualType Arg3Type = Call->getArg(Arg: 2)->getType();
3347
3348 // Non-vector floating point types.
3349 if (!Arg1Type->isVectorType()) {
3350 assert(!Arg2Type->isVectorType());
3351 assert(!Arg3Type->isVectorType());
3352 (void)Arg2Type;
3353 (void)Arg3Type;
3354
3355 const Floating &Z = S.Stk.pop<Floating>();
3356 const Floating &Y = S.Stk.pop<Floating>();
3357 const Floating &X = S.Stk.pop<Floating>();
3358 APFloat F = Fn(X.getAPFloat(), Y.getAPFloat(), Z.getAPFloat(), RM);
3359 Floating Result = S.allocFloat(Sem: X.getSemantics());
3360 Result.copy(F);
3361 S.Stk.push<Floating>(Args&: Result);
3362 return true;
3363 }
3364
3365 // Vector type.
3366 assert(Arg1Type->isVectorType() && Arg2Type->isVectorType() &&
3367 Arg3Type->isVectorType());
3368
3369 const VectorType *VecTy = Arg1Type->castAs<VectorType>();
3370 QualType ElemQT = VecTy->getElementType();
3371 unsigned NumElems = VecTy->getNumElements();
3372
3373 assert(ElemQT == Arg2Type->castAs<VectorType>()->getElementType() &&
3374 ElemQT == Arg3Type->castAs<VectorType>()->getElementType());
3375 assert(NumElems == Arg2Type->castAs<VectorType>()->getNumElements() &&
3376 NumElems == Arg3Type->castAs<VectorType>()->getNumElements());
3377 assert(ElemQT->isRealFloatingType());
3378 (void)ElemQT;
3379
3380 const Pointer &VZ = S.Stk.pop<Pointer>();
3381 const Pointer &VY = S.Stk.pop<Pointer>();
3382 const Pointer &VX = S.Stk.pop<Pointer>();
3383 const Pointer &Dst = S.Stk.peek<Pointer>();
3384 for (unsigned I = 0; I != NumElems; ++I) {
3385 using T = PrimConv<PT_Float>::T;
3386 APFloat X = VX.elem<T>(I).getAPFloat();
3387 APFloat Y = VY.elem<T>(I).getAPFloat();
3388 APFloat Z = VZ.elem<T>(I).getAPFloat();
3389 APFloat F = Fn(X, Y, Z, RM);
3390 Dst.elem<Floating>(I) = Floating(F);
3391 }
3392 Dst.initializeAllElements();
3393 return true;
3394}
3395
3396/// AVX512 predicated move: "Result = Mask[] ? LHS[] : RHS[]".
3397static bool interp__builtin_ia32_select(InterpState &S, CodePtr OpPC,
3398 const CallExpr *Call) {
3399 const Pointer &RHS = S.Stk.pop<Pointer>();
3400 const Pointer &LHS = S.Stk.pop<Pointer>();
3401 APSInt Mask;
3402 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: Mask))
3403 return false;
3404 const Pointer &Dst = S.Stk.peek<Pointer>();
3405
3406 assert(LHS.getNumElems() == RHS.getNumElems());
3407 assert(LHS.getNumElems() == Dst.getNumElems());
3408 unsigned NumElems = LHS.getNumElems();
3409 PrimType ElemT = LHS.getFieldDesc()->getPrimType();
3410 PrimType DstElemT = Dst.getFieldDesc()->getPrimType();
3411
3412 for (unsigned I = 0; I != NumElems; ++I) {
3413 if (ElemT == PT_Float) {
3414 assert(DstElemT == PT_Float);
3415 Dst.elem<Floating>(I) =
3416 Mask[I] ? LHS.elem<Floating>(I) : RHS.elem<Floating>(I);
3417 } else {
3418 APSInt Elem;
3419 INT_TYPE_SWITCH(ElemT, {
3420 Elem = Mask[I] ? LHS.elem<T>(I).toAPSInt() : RHS.elem<T>(I).toAPSInt();
3421 });
3422 INT_TYPE_SWITCH_NO_BOOL(DstElemT,
3423 { Dst.elem<T>(I) = static_cast<T>(Elem); });
3424 }
3425 }
3426 Dst.initializeAllElements();
3427
3428 return true;
3429}
3430
3431/// Scalar variant of AVX512 predicated select:
3432/// Result[i] = (Mask bit 0) ? LHS[i] : RHS[i], but only element 0 may change.
3433/// All other elements are taken from RHS.
3434static bool interp__builtin_ia32_select_scalar(InterpState &S,
3435 const CallExpr *Call) {
3436 unsigned N =
3437 Call->getArg(Arg: 1)->getType()->castAs<VectorType>()->getNumElements();
3438
3439 const Pointer &W = S.Stk.pop<Pointer>();
3440 const Pointer &A = S.Stk.pop<Pointer>();
3441 APSInt U;
3442 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: U))
3443 return false;
3444 const Pointer &Dst = S.Stk.peek<Pointer>();
3445
3446 bool TakeA0 = U.getZExtValue() & 1ULL;
3447
3448 for (unsigned I = TakeA0; I != N; ++I)
3449 Dst.elem<Floating>(I) = W.elem<Floating>(I);
3450 if (TakeA0)
3451 Dst.elem<Floating>(I: 0) = A.elem<Floating>(I: 0);
3452
3453 Dst.initializeAllElements();
3454 return true;
3455}
3456
3457static bool interp__builtin_ia32_test_op(
3458 InterpState &S, CodePtr OpPC, const CallExpr *Call,
3459 llvm::function_ref<bool(const APInt &A, const APInt &B)> Fn) {
3460 const Pointer &RHS = S.Stk.pop<Pointer>();
3461 const Pointer &LHS = S.Stk.pop<Pointer>();
3462
3463 assert(LHS.getNumElems() == RHS.getNumElems());
3464
3465 unsigned SourceLen = LHS.getNumElems();
3466 QualType ElemQT = getElemType(P: LHS);
3467 OptPrimType ElemPT = S.getContext().classify(T: ElemQT);
3468 unsigned LaneWidth = S.getASTContext().getTypeSize(T: ElemQT);
3469
3470 APInt AWide(LaneWidth * SourceLen, 0);
3471 APInt BWide(LaneWidth * SourceLen, 0);
3472
3473 for (unsigned I = 0; I != SourceLen; ++I) {
3474 APInt ALane;
3475 APInt BLane;
3476
3477 if (ElemQT->isIntegerType()) { // Get value.
3478 INT_TYPE_SWITCH_NO_BOOL(*ElemPT, {
3479 ALane = LHS.elem<T>(I).toAPSInt();
3480 BLane = RHS.elem<T>(I).toAPSInt();
3481 });
3482 } else if (ElemQT->isFloatingType()) { // Get only sign bit.
3483 using T = PrimConv<PT_Float>::T;
3484 ALane = LHS.elem<T>(I).getAPFloat().bitcastToAPInt().isNegative();
3485 BLane = RHS.elem<T>(I).getAPFloat().bitcastToAPInt().isNegative();
3486 } else { // Must be integer or floating type.
3487 return false;
3488 }
3489 AWide.insertBits(SubBits: ALane, bitPosition: I * LaneWidth);
3490 BWide.insertBits(SubBits: BLane, bitPosition: I * LaneWidth);
3491 }
3492 pushInteger(S, Val: Fn(AWide, BWide), QT: Call->getType());
3493 return true;
3494}
3495
3496static bool interp__builtin_ia32_movmsk_op(InterpState &S, CodePtr OpPC,
3497 const CallExpr *Call) {
3498 assert(Call->getNumArgs() == 1);
3499
3500 const Pointer &Source = S.Stk.pop<Pointer>();
3501
3502 unsigned SourceLen = Source.getNumElems();
3503 QualType ElemQT = getElemType(P: Source);
3504 OptPrimType ElemT = S.getContext().classify(T: ElemQT);
3505 unsigned ResultLen =
3506 S.getASTContext().getTypeSize(T: Call->getType()); // Always 32-bit integer.
3507 APInt Result(ResultLen, 0);
3508
3509 for (unsigned I = 0; I != SourceLen; ++I) {
3510 APInt Elem;
3511 if (ElemQT->isIntegerType()) {
3512 INT_TYPE_SWITCH_NO_BOOL(*ElemT, { Elem = Source.elem<T>(I).toAPSInt(); });
3513 } else if (ElemQT->isRealFloatingType()) {
3514 using T = PrimConv<PT_Float>::T;
3515 Elem = Source.elem<T>(I).getAPFloat().bitcastToAPInt();
3516 } else {
3517 return false;
3518 }
3519 Result.setBitVal(BitPosition: I, BitValue: Elem.isNegative());
3520 }
3521 pushInteger(S, Val: Result, QT: Call->getType());
3522 return true;
3523}
3524
3525static bool interp__builtin_elementwise_triop(
3526 InterpState &S, CodePtr OpPC, const CallExpr *Call,
3527 llvm::function_ref<APInt(const APSInt &, const APSInt &, const APSInt &)>
3528 Fn) {
3529 assert(Call->getNumArgs() == 3);
3530
3531 QualType Arg0Type = Call->getArg(Arg: 0)->getType();
3532 QualType Arg2Type = Call->getArg(Arg: 2)->getType();
3533 // Non-vector integer types.
3534 if (!Arg0Type->isVectorType()) {
3535 APSInt Op2;
3536 if (!popToAPSInt(S, T: Arg2Type, Out&: Op2))
3537 return false;
3538 APSInt Op1;
3539 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: Op1))
3540 return false;
3541 APSInt Op0;
3542 if (!popToAPSInt(S, T: Arg0Type, Out&: Op0))
3543 return false;
3544 APSInt Result = APSInt(Fn(Op0, Op1, Op2), Op0.isUnsigned());
3545 pushInteger(S, Val: Result, QT: Call->getType());
3546 return true;
3547 }
3548
3549 const auto *VecT = Arg0Type->castAs<VectorType>();
3550 PrimType ElemT = *S.getContext().classify(T: VecT->getElementType());
3551 unsigned NumElems = VecT->getNumElements();
3552 bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
3553
3554 // Vector + Vector + Scalar case.
3555 if (!Arg2Type->isVectorType()) {
3556 APSInt Op2;
3557 if (!popToAPSInt(S, T: Arg2Type, Out&: Op2))
3558 return false;
3559
3560 const Pointer &Op1 = S.Stk.pop<Pointer>();
3561 const Pointer &Op0 = S.Stk.pop<Pointer>();
3562 const Pointer &Dst = S.Stk.peek<Pointer>();
3563 for (unsigned I = 0; I != NumElems; ++I) {
3564 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
3565 Dst.elem<T>(I) = static_cast<T>(APSInt(
3566 Fn(Op0.elem<T>(I).toAPSInt(), Op1.elem<T>(I).toAPSInt(), Op2),
3567 DestUnsigned));
3568 });
3569 }
3570 Dst.initializeAllElements();
3571
3572 return true;
3573 }
3574
3575 // Vector type.
3576 const Pointer &Op2 = S.Stk.pop<Pointer>();
3577 const Pointer &Op1 = S.Stk.pop<Pointer>();
3578 const Pointer &Op0 = S.Stk.pop<Pointer>();
3579 const Pointer &Dst = S.Stk.peek<Pointer>();
3580 for (unsigned I = 0; I != NumElems; ++I) {
3581 APSInt Val0, Val1, Val2;
3582 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
3583 Val0 = Op0.elem<T>(I).toAPSInt();
3584 Val1 = Op1.elem<T>(I).toAPSInt();
3585 Val2 = Op2.elem<T>(I).toAPSInt();
3586 });
3587 APSInt Result = APSInt(Fn(Val0, Val1, Val2), Val0.isUnsigned());
3588 INT_TYPE_SWITCH_NO_BOOL(ElemT,
3589 { Dst.elem<T>(I) = static_cast<T>(Result); });
3590 }
3591 Dst.initializeAllElements();
3592
3593 return true;
3594}
3595
3596static bool interp__builtin_ia32_extract_vector(InterpState &S, CodePtr OpPC,
3597 const CallExpr *Call,
3598 unsigned ID) {
3599 assert(Call->getNumArgs() == 2);
3600
3601 APSInt ImmAPS;
3602 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: ImmAPS))
3603 return false;
3604 uint64_t Index = ImmAPS.getZExtValue();
3605
3606 const Pointer &Src = S.Stk.pop<Pointer>();
3607 if (!Src.getFieldDesc()->isPrimitiveArray())
3608 return false;
3609
3610 const Pointer &Dst = S.Stk.peek<Pointer>();
3611 if (!Dst.getFieldDesc()->isPrimitiveArray())
3612 return false;
3613
3614 unsigned SrcElems = Src.getNumElems();
3615 unsigned DstElems = Dst.getNumElems();
3616
3617 unsigned NumLanes = SrcElems / DstElems;
3618 unsigned Lane = static_cast<unsigned>(Index % NumLanes);
3619 unsigned ExtractPos = Lane * DstElems;
3620
3621 PrimType ElemT = Src.getFieldDesc()->getPrimType();
3622
3623 TYPE_SWITCH(ElemT, {
3624 for (unsigned I = 0; I != DstElems; ++I) {
3625 Dst.elem<T>(I) = Src.elem<T>(ExtractPos + I);
3626 }
3627 });
3628
3629 Dst.initializeAllElements();
3630 return true;
3631}
3632
3633static bool interp__builtin_ia32_extract_vector_masked(InterpState &S,
3634 CodePtr OpPC,
3635 const CallExpr *Call,
3636 unsigned ID) {
3637 assert(Call->getNumArgs() == 4);
3638
3639 APSInt MaskAPS;
3640 if (!popToAPSInt(S, E: Call->getArg(Arg: 3), Out&: MaskAPS))
3641 return false;
3642 const Pointer &Merge = S.Stk.pop<Pointer>();
3643 APSInt ImmAPS;
3644 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: ImmAPS))
3645 return false;
3646 const Pointer &Src = S.Stk.pop<Pointer>();
3647
3648 if (!Src.getFieldDesc()->isPrimitiveArray() ||
3649 !Merge.getFieldDesc()->isPrimitiveArray())
3650 return false;
3651
3652 const Pointer &Dst = S.Stk.peek<Pointer>();
3653 if (!Dst.getFieldDesc()->isPrimitiveArray())
3654 return false;
3655
3656 unsigned SrcElems = Src.getNumElems();
3657 unsigned DstElems = Dst.getNumElems();
3658
3659 unsigned NumLanes = SrcElems / DstElems;
3660 unsigned Lane = static_cast<unsigned>(ImmAPS.getZExtValue() % NumLanes);
3661 unsigned Base = Lane * DstElems;
3662
3663 PrimType ElemT = Src.getFieldDesc()->getPrimType();
3664
3665 TYPE_SWITCH(ElemT, {
3666 for (unsigned I = 0; I != DstElems; ++I) {
3667 if (MaskAPS[I])
3668 Dst.elem<T>(I) = Src.elem<T>(Base + I);
3669 else
3670 Dst.elem<T>(I) = Merge.elem<T>(I);
3671 }
3672 });
3673
3674 Dst.initializeAllElements();
3675 return true;
3676}
3677
3678static bool interp__builtin_ia32_insert_subvector(InterpState &S, CodePtr OpPC,
3679 const CallExpr *Call,
3680 unsigned ID) {
3681 assert(Call->getNumArgs() == 3);
3682
3683 APSInt ImmAPS;
3684 if (!popToAPSInt(S, E: Call->getArg(Arg: 2), Out&: ImmAPS))
3685 return false;
3686 uint64_t Index = ImmAPS.getZExtValue();
3687
3688 const Pointer &SubVec = S.Stk.pop<Pointer>();
3689 if (!SubVec.getFieldDesc()->isPrimitiveArray())
3690 return false;
3691
3692 const Pointer &BaseVec = S.Stk.pop<Pointer>();
3693 if (!BaseVec.getFieldDesc()->isPrimitiveArray())
3694 return false;
3695
3696 const Pointer &Dst = S.Stk.peek<Pointer>();
3697
3698 unsigned BaseElements = BaseVec.getNumElems();
3699 unsigned SubElements = SubVec.getNumElems();
3700
3701 assert(SubElements != 0 && BaseElements != 0 &&
3702 (BaseElements % SubElements) == 0);
3703
3704 unsigned NumLanes = BaseElements / SubElements;
3705 unsigned Lane = static_cast<unsigned>(Index % NumLanes);
3706 unsigned InsertPos = Lane * SubElements;
3707
3708 PrimType ElemT = BaseVec.getFieldDesc()->getPrimType();
3709
3710 TYPE_SWITCH(ElemT, {
3711 for (unsigned I = 0; I != BaseElements; ++I)
3712 Dst.elem<T>(I) = BaseVec.elem<T>(I);
3713 for (unsigned I = 0; I != SubElements; ++I)
3714 Dst.elem<T>(InsertPos + I) = SubVec.elem<T>(I);
3715 });
3716
3717 Dst.initializeAllElements();
3718 return true;
3719}
3720
3721static bool interp__builtin_ia32_phminposuw(InterpState &S, CodePtr OpPC,
3722 const CallExpr *Call) {
3723 assert(Call->getNumArgs() == 1);
3724
3725 const Pointer &Source = S.Stk.pop<Pointer>();
3726 const Pointer &Dest = S.Stk.peek<Pointer>();
3727
3728 unsigned SourceLen = Source.getNumElems();
3729 QualType ElemQT = getElemType(P: Source);
3730 OptPrimType ElemT = S.getContext().classify(T: ElemQT);
3731 unsigned ElemBitWidth = S.getASTContext().getTypeSize(T: ElemQT);
3732
3733 bool DestUnsigned = Call->getCallReturnType(Ctx: S.getASTContext())
3734 ->castAs<VectorType>()
3735 ->getElementType()
3736 ->isUnsignedIntegerOrEnumerationType();
3737
3738 INT_TYPE_SWITCH_NO_BOOL(*ElemT, {
3739 APSInt MinIndex(ElemBitWidth, DestUnsigned);
3740 APSInt MinVal = Source.elem<T>(0).toAPSInt();
3741
3742 for (unsigned I = 1; I != SourceLen; ++I) {
3743 APSInt Val = Source.elem<T>(I).toAPSInt();
3744 if (MinVal.ugt(Val)) {
3745 MinVal = Val;
3746 MinIndex = I;
3747 }
3748 }
3749
3750 Dest.elem<T>(0) = static_cast<T>(MinVal);
3751 Dest.elem<T>(1) = static_cast<T>(MinIndex);
3752 for (unsigned I = 2; I != SourceLen; ++I) {
3753 Dest.elem<T>(I) = static_cast<T>(APSInt(ElemBitWidth, DestUnsigned));
3754 }
3755 });
3756 Dest.initializeAllElements();
3757 return true;
3758}
3759
3760static bool interp__builtin_ia32_pternlog(InterpState &S, CodePtr OpPC,
3761 const CallExpr *Call, bool MaskZ) {
3762 assert(Call->getNumArgs() == 5);
3763
3764 APSInt UVal;
3765 if (!popToAPSInt(S, E: Call->getArg(Arg: 4), Out&: UVal))
3766 return false;
3767 APInt U = UVal; // Lane mask
3768 APSInt ImmVal;
3769 if (!popToAPSInt(S, E: Call->getArg(Arg: 3), Out&: ImmVal))
3770 return false;
3771 APInt Imm = ImmVal; // Ternary truth table
3772 const Pointer &C = S.Stk.pop<Pointer>();
3773 const Pointer &B = S.Stk.pop<Pointer>();
3774 const Pointer &A = S.Stk.pop<Pointer>();
3775 const Pointer &Dst = S.Stk.peek<Pointer>();
3776
3777 unsigned DstLen = A.getNumElems();
3778 QualType ElemQT = getElemType(P: A);
3779 OptPrimType ElemT = S.getContext().classify(T: ElemQT);
3780 unsigned LaneWidth = S.getASTContext().getTypeSize(T: ElemQT);
3781 bool DstUnsigned = ElemQT->isUnsignedIntegerOrEnumerationType();
3782
3783 INT_TYPE_SWITCH_NO_BOOL(*ElemT, {
3784 for (unsigned I = 0; I != DstLen; ++I) {
3785 APInt ALane = A.elem<T>(I).toAPSInt();
3786 APInt BLane = B.elem<T>(I).toAPSInt();
3787 APInt CLane = C.elem<T>(I).toAPSInt();
3788 APInt RLane(LaneWidth, 0);
3789 if (U[I]) { // If lane not masked, compute ternary logic.
3790 for (unsigned Bit = 0; Bit != LaneWidth; ++Bit) {
3791 unsigned ABit = ALane[Bit];
3792 unsigned BBit = BLane[Bit];
3793 unsigned CBit = CLane[Bit];
3794 unsigned Idx = (ABit << 2) | (BBit << 1) | (CBit);
3795 RLane.setBitVal(Bit, Imm[Idx]);
3796 }
3797 Dst.elem<T>(I) = static_cast<T>(APSInt(RLane, DstUnsigned));
3798 } else if (MaskZ) { // If zero masked, zero the lane.
3799 Dst.elem<T>(I) = static_cast<T>(APSInt(RLane, DstUnsigned));
3800 } else { // Just masked, put in A lane.
3801 Dst.elem<T>(I) = static_cast<T>(APSInt(ALane, DstUnsigned));
3802 }
3803 }
3804 });
3805 Dst.initializeAllElements();
3806 return true;
3807}
3808
3809static bool interp__builtin_ia32_vec_ext(InterpState &S, CodePtr OpPC,
3810 const CallExpr *Call, unsigned ID) {
3811 assert(Call->getNumArgs() == 2);
3812
3813 APSInt ImmAPS;
3814 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: ImmAPS))
3815 return false;
3816 const Pointer &Vec = S.Stk.pop<Pointer>();
3817 if (!Vec.getFieldDesc()->isPrimitiveArray())
3818 return false;
3819
3820 unsigned NumElems = Vec.getNumElems();
3821 unsigned Index =
3822 static_cast<unsigned>(ImmAPS.getZExtValue() & (NumElems - 1));
3823
3824 PrimType ElemT = Vec.getFieldDesc()->getPrimType();
3825 // FIXME(#161685): Replace float+int split with a numeric-only type switch
3826 if (ElemT == PT_Float) {
3827 S.Stk.push<Floating>(Args&: Vec.elem<Floating>(I: Index));
3828 return true;
3829 }
3830 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
3831 APSInt V = Vec.elem<T>(Index).toAPSInt();
3832 pushInteger(S, V, Call->getType());
3833 });
3834
3835 return true;
3836}
3837
3838static bool interp__builtin_ia32_vec_set(InterpState &S, CodePtr OpPC,
3839 const CallExpr *Call, unsigned ID) {
3840 assert(Call->getNumArgs() == 3);
3841
3842 APSInt ImmAPS;
3843 if (!popToAPSInt(S, E: Call->getArg(Arg: 2), Out&: ImmAPS))
3844 return false;
3845 APSInt ValAPS;
3846 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: ValAPS))
3847 return false;
3848
3849 const Pointer &Base = S.Stk.pop<Pointer>();
3850 if (!Base.getFieldDesc()->isPrimitiveArray())
3851 return false;
3852
3853 const Pointer &Dst = S.Stk.peek<Pointer>();
3854
3855 unsigned NumElems = Base.getNumElems();
3856 unsigned Index =
3857 static_cast<unsigned>(ImmAPS.getZExtValue() & (NumElems - 1));
3858
3859 PrimType ElemT = Base.getFieldDesc()->getPrimType();
3860 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
3861 for (unsigned I = 0; I != NumElems; ++I)
3862 Dst.elem<T>(I) = Base.elem<T>(I);
3863 Dst.elem<T>(Index) = static_cast<T>(ValAPS);
3864 });
3865
3866 Dst.initializeAllElements();
3867 return true;
3868}
3869
3870static bool evalICmpImm(uint8_t Imm, const APSInt &A, const APSInt &B,
3871 bool IsUnsigned) {
3872 switch (Imm & 0x7) {
3873 case 0x00: // _MM_CMPINT_EQ
3874 return (A == B);
3875 case 0x01: // _MM_CMPINT_LT
3876 return IsUnsigned ? A.ult(RHS: B) : A.slt(RHS: B);
3877 case 0x02: // _MM_CMPINT_LE
3878 return IsUnsigned ? A.ule(RHS: B) : A.sle(RHS: B);
3879 case 0x03: // _MM_CMPINT_FALSE
3880 return false;
3881 case 0x04: // _MM_CMPINT_NE
3882 return (A != B);
3883 case 0x05: // _MM_CMPINT_NLT
3884 return IsUnsigned ? A.uge(RHS: B) : A.sge(RHS: B);
3885 case 0x06: // _MM_CMPINT_NLE
3886 return IsUnsigned ? A.ugt(RHS: B) : A.sgt(RHS: B);
3887 case 0x07: // _MM_CMPINT_TRUE
3888 return true;
3889 default:
3890 llvm_unreachable("Invalid Op");
3891 }
3892}
3893
3894static bool interp__builtin_ia32_cmp_mask(InterpState &S, CodePtr OpPC,
3895 const CallExpr *Call, unsigned ID,
3896 bool IsUnsigned) {
3897 assert(Call->getNumArgs() == 4);
3898
3899 APSInt Mask;
3900 if (!popToAPSInt(S, E: Call->getArg(Arg: 3), Out&: Mask))
3901 return false;
3902 APSInt Opcode;
3903 if (!popToAPSInt(S, E: Call->getArg(Arg: 2), Out&: Opcode))
3904 return false;
3905 unsigned CmpOp = static_cast<unsigned>(Opcode.getZExtValue());
3906 const Pointer &RHS = S.Stk.pop<Pointer>();
3907 const Pointer &LHS = S.Stk.pop<Pointer>();
3908
3909 assert(LHS.getNumElems() == RHS.getNumElems());
3910
3911 APInt RetMask = APInt::getZero(numBits: LHS.getNumElems());
3912 unsigned VectorLen = LHS.getNumElems();
3913 PrimType ElemT = LHS.getFieldDesc()->getPrimType();
3914
3915 for (unsigned ElemNum = 0; ElemNum < VectorLen; ++ElemNum) {
3916 APSInt A, B;
3917 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
3918 A = LHS.elem<T>(ElemNum).toAPSInt();
3919 B = RHS.elem<T>(ElemNum).toAPSInt();
3920 });
3921 RetMask.setBitVal(BitPosition: ElemNum,
3922 BitValue: Mask[ElemNum] && evalICmpImm(Imm: CmpOp, A, B, IsUnsigned));
3923 }
3924 pushInteger(S, Val: RetMask, QT: Call->getType());
3925 return true;
3926}
3927
3928static bool interp__builtin_ia32_vpconflict(InterpState &S, CodePtr OpPC,
3929 const CallExpr *Call) {
3930 assert(Call->getNumArgs() == 1);
3931
3932 QualType Arg0Type = Call->getArg(Arg: 0)->getType();
3933 const auto *VecT = Arg0Type->castAs<VectorType>();
3934 PrimType ElemT = *S.getContext().classify(T: VecT->getElementType());
3935 unsigned NumElems = VecT->getNumElements();
3936 bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
3937 const Pointer &Src = S.Stk.pop<Pointer>();
3938 const Pointer &Dst = S.Stk.peek<Pointer>();
3939
3940 for (unsigned I = 0; I != NumElems; ++I) {
3941 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
3942 APSInt ElemI = Src.elem<T>(I).toAPSInt();
3943 APInt ConflictMask(ElemI.getBitWidth(), 0);
3944 for (unsigned J = 0; J != I; ++J) {
3945 APSInt ElemJ = Src.elem<T>(J).toAPSInt();
3946 ConflictMask.setBitVal(J, ElemI == ElemJ);
3947 }
3948 Dst.elem<T>(I) = static_cast<T>(APSInt(ConflictMask, DestUnsigned));
3949 });
3950 }
3951 Dst.initializeAllElements();
3952 return true;
3953}
3954
3955static bool interp__builtin_ia32_cvt_vec2mask(InterpState &S, CodePtr OpPC,
3956 const CallExpr *Call,
3957 unsigned ID) {
3958 assert(Call->getNumArgs() == 1);
3959
3960 const Pointer &Vec = S.Stk.pop<Pointer>();
3961 unsigned RetWidth = S.getASTContext().getIntWidth(T: Call->getType());
3962 APInt RetMask(RetWidth, 0);
3963
3964 unsigned VectorLen = Vec.getNumElems();
3965 PrimType ElemT = Vec.getFieldDesc()->getPrimType();
3966
3967 for (unsigned ElemNum = 0; ElemNum != VectorLen; ++ElemNum) {
3968 APSInt A;
3969 INT_TYPE_SWITCH_NO_BOOL(ElemT, { A = Vec.elem<T>(ElemNum).toAPSInt(); });
3970 unsigned MSB = A[A.getBitWidth() - 1];
3971 RetMask.setBitVal(BitPosition: ElemNum, BitValue: MSB);
3972 }
3973 pushInteger(S, Val: RetMask, QT: Call->getType());
3974 return true;
3975}
3976
3977static bool interp__builtin_ia32_cvt_mask2vec(InterpState &S, CodePtr OpPC,
3978 const CallExpr *Call,
3979 unsigned ID) {
3980 assert(Call->getNumArgs() == 1);
3981
3982 APSInt Mask;
3983 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: Mask))
3984 return false;
3985
3986 const Pointer &Vec = S.Stk.peek<Pointer>();
3987 unsigned NumElems = Vec.getNumElems();
3988 PrimType ElemT = Vec.getFieldDesc()->getPrimType();
3989
3990 for (unsigned I = 0; I != NumElems; ++I) {
3991 bool BitSet = Mask[I];
3992
3993 INT_TYPE_SWITCH_NO_BOOL(
3994 ElemT, { Vec.elem<T>(I) = BitSet ? T::from(-1) : T::from(0); });
3995 }
3996
3997 Vec.initializeAllElements();
3998
3999 return true;
4000}
4001
4002static bool interp__builtin_ia32_cvtsd2ss(InterpState &S, CodePtr OpPC,
4003 const CallExpr *Call,
4004 bool HasRoundingMask) {
4005 APSInt Rounding, MaskInt;
4006 Pointer Src, B, A;
4007
4008 if (HasRoundingMask) {
4009 assert(Call->getNumArgs() == 5);
4010 if (!popToAPSInt(S, E: Call->getArg(Arg: 4), Out&: Rounding))
4011 return false;
4012 if (!popToAPSInt(S, E: Call->getArg(Arg: 3), Out&: MaskInt))
4013 return false;
4014 Src = S.Stk.pop<Pointer>();
4015 B = S.Stk.pop<Pointer>();
4016 A = S.Stk.pop<Pointer>();
4017 if (!CheckLoad(S, OpPC, Ptr: A) || !CheckLoad(S, OpPC, Ptr: B) ||
4018 !CheckLoad(S, OpPC, Ptr: Src))
4019 return false;
4020 } else {
4021 assert(Call->getNumArgs() == 2);
4022 B = S.Stk.pop<Pointer>();
4023 A = S.Stk.pop<Pointer>();
4024 if (!CheckLoad(S, OpPC, Ptr: A) || !CheckLoad(S, OpPC, Ptr: B))
4025 return false;
4026 }
4027
4028 const auto *DstVTy = Call->getType()->castAs<VectorType>();
4029 unsigned NumElems = DstVTy->getNumElements();
4030 const Pointer &Dst = S.Stk.peek<Pointer>();
4031
4032 // Copy all elements except lane 0 (overwritten below) from A to Dst.
4033 for (unsigned I = 1; I != NumElems; ++I)
4034 Dst.elem<Floating>(I) = A.elem<Floating>(I);
4035
4036 // Convert element 0 from double to float, or use Src if masked off.
4037 if (!HasRoundingMask || (MaskInt.getZExtValue() & 0x1)) {
4038 assert(S.getASTContext().FloatTy == DstVTy->getElementType() &&
4039 "cvtsd2ss requires float element type in destination vector");
4040
4041 Floating Conv = S.allocFloat(
4042 Sem: S.getASTContext().getFloatTypeSemantics(T: DstVTy->getElementType()));
4043 APFloat SrcVal = B.elem<Floating>(I: 0).getAPFloat();
4044 if (!convertDoubleToFloatStrict(Src: SrcVal, Dst&: Conv, S, DiagExpr: Call))
4045 return false;
4046 Dst.elem<Floating>(I: 0) = Conv;
4047 } else {
4048 Dst.elem<Floating>(I: 0) = Src.elem<Floating>(I: 0);
4049 }
4050
4051 Dst.initializeAllElements();
4052 return true;
4053}
4054
4055static bool interp__builtin_ia32_cvtpd2ps(InterpState &S, CodePtr OpPC,
4056 const CallExpr *Call, bool IsMasked,
4057 bool HasRounding) {
4058 APSInt MaskVal;
4059 Pointer PassThrough;
4060 Pointer Src;
4061 APSInt Rounding;
4062
4063 if (IsMasked) {
4064 // Pop in reverse order.
4065 if (HasRounding) {
4066 if (!popToAPSInt(S, E: Call->getArg(Arg: 3), Out&: Rounding))
4067 return false;
4068 if (!popToAPSInt(S, E: Call->getArg(Arg: 2), Out&: MaskVal))
4069 return false;
4070 PassThrough = S.Stk.pop<Pointer>();
4071 Src = S.Stk.pop<Pointer>();
4072 } else {
4073 if (!popToAPSInt(S, E: Call->getArg(Arg: 2), Out&: MaskVal))
4074 return false;
4075 PassThrough = S.Stk.pop<Pointer>();
4076 Src = S.Stk.pop<Pointer>();
4077 }
4078
4079 if (!CheckLoad(S, OpPC, Ptr: PassThrough))
4080 return false;
4081 } else {
4082 // Pop source only.
4083 Src = S.Stk.pop<Pointer>();
4084 }
4085
4086 if (!CheckLoad(S, OpPC, Ptr: Src))
4087 return false;
4088
4089 const auto *RetVTy = Call->getType()->castAs<VectorType>();
4090 unsigned RetElems = RetVTy->getNumElements();
4091 unsigned SrcElems = Src.getNumElems();
4092 const Pointer &Dst = S.Stk.peek<Pointer>();
4093
4094 // Initialize destination with passthrough or zeros.
4095 for (unsigned I = 0; I != RetElems; ++I)
4096 if (IsMasked)
4097 Dst.elem<Floating>(I) = PassThrough.elem<Floating>(I);
4098 else
4099 Dst.elem<Floating>(I) = Floating(APFloat(0.0f));
4100
4101 assert(S.getASTContext().FloatTy == RetVTy->getElementType() &&
4102 "cvtpd2ps requires float element type in return vector");
4103
4104 // Convert double to float for enabled elements (only process source elements
4105 // that exist).
4106 for (unsigned I = 0; I != SrcElems; ++I) {
4107 if (IsMasked && !MaskVal[I])
4108 continue;
4109
4110 APFloat SrcVal = Src.elem<Floating>(I).getAPFloat();
4111
4112 Floating Conv = S.allocFloat(
4113 Sem: S.getASTContext().getFloatTypeSemantics(T: RetVTy->getElementType()));
4114 if (!convertDoubleToFloatStrict(Src: SrcVal, Dst&: Conv, S, DiagExpr: Call))
4115 return false;
4116 Dst.elem<Floating>(I) = Conv;
4117 }
4118
4119 Dst.initializeAllElements();
4120 return true;
4121}
4122
4123static bool interp__builtin_ia32_shuffle_generic(
4124 InterpState &S, CodePtr OpPC, const CallExpr *Call,
4125 llvm::function_ref<std::pair<unsigned, int>(unsigned, const APInt &)>
4126 GetSourceIndex) {
4127
4128 assert(Call->getNumArgs() == 2 || Call->getNumArgs() == 3);
4129
4130 APInt ShuffleMask;
4131 Pointer A, MaskVector, B;
4132 bool IsVectorMask = false;
4133 bool IsSingleOperand = (Call->getNumArgs() == 2);
4134
4135 if (IsSingleOperand) {
4136 QualType MaskType = Call->getArg(Arg: 1)->getType();
4137 if (MaskType->isVectorType()) {
4138 IsVectorMask = true;
4139 MaskVector = S.Stk.pop<Pointer>();
4140 A = S.Stk.pop<Pointer>();
4141 B = A;
4142 } else if (MaskType->isIntegerType()) {
4143 APSInt MaskVal;
4144 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: MaskVal))
4145 return false;
4146 ShuffleMask = MaskVal;
4147 A = S.Stk.pop<Pointer>();
4148 B = A;
4149 } else {
4150 return false;
4151 }
4152 } else {
4153 QualType Arg2Type = Call->getArg(Arg: 2)->getType();
4154 if (Arg2Type->isVectorType()) {
4155 IsVectorMask = true;
4156 B = S.Stk.pop<Pointer>();
4157 MaskVector = S.Stk.pop<Pointer>();
4158 A = S.Stk.pop<Pointer>();
4159 } else if (Arg2Type->isIntegerType()) {
4160 APSInt MaskVal;
4161 if (!popToAPSInt(S, E: Call->getArg(Arg: 2), Out&: MaskVal))
4162 return false;
4163 ShuffleMask = MaskVal;
4164 B = S.Stk.pop<Pointer>();
4165 A = S.Stk.pop<Pointer>();
4166 } else {
4167 return false;
4168 }
4169 }
4170
4171 QualType Arg0Type = Call->getArg(Arg: 0)->getType();
4172 const auto *VecT = Arg0Type->castAs<VectorType>();
4173 PrimType ElemT = *S.getContext().classify(T: VecT->getElementType());
4174 unsigned NumElems = VecT->getNumElements();
4175
4176 const Pointer &Dst = S.Stk.peek<Pointer>();
4177
4178 PrimType MaskElemT = PT_Uint32;
4179 if (IsVectorMask) {
4180 QualType Arg1Type = Call->getArg(Arg: 1)->getType();
4181 const auto *MaskVecT = Arg1Type->castAs<VectorType>();
4182 QualType MaskElemType = MaskVecT->getElementType();
4183 MaskElemT = *S.getContext().classify(T: MaskElemType);
4184 }
4185
4186 for (unsigned DstIdx = 0; DstIdx != NumElems; ++DstIdx) {
4187 if (IsVectorMask) {
4188 INT_TYPE_SWITCH(MaskElemT,
4189 { ShuffleMask = MaskVector.elem<T>(DstIdx).toAPSInt(); });
4190 }
4191
4192 auto [SrcVecIdx, SrcIdx] = GetSourceIndex(DstIdx, ShuffleMask);
4193
4194 if (SrcIdx < 0) {
4195 // Zero out this element
4196 if (ElemT == PT_Float) {
4197 Dst.elem<Floating>(I: DstIdx) = Floating(
4198 S.getASTContext().getFloatTypeSemantics(T: VecT->getElementType()));
4199 } else {
4200 INT_TYPE_SWITCH_NO_BOOL(ElemT, { Dst.elem<T>(DstIdx) = T::from(0); });
4201 }
4202 } else {
4203 const Pointer &Src = (SrcVecIdx == 0) ? A : B;
4204 TYPE_SWITCH(ElemT, { Dst.elem<T>(DstIdx) = Src.elem<T>(SrcIdx); });
4205 }
4206 }
4207 Dst.initializeAllElements();
4208
4209 return true;
4210}
4211
4212static bool interp__builtin_ia32_shuffle_generic(
4213 InterpState &S, CodePtr OpPC, const CallExpr *Call,
4214 llvm::function_ref<std::pair<unsigned, int>(unsigned, unsigned)>
4215 GetSourceIndex) {
4216 return interp__builtin_ia32_shuffle_generic(
4217 S, OpPC, Call,
4218 GetSourceIndex: [&GetSourceIndex](unsigned DstIdx,
4219 const APInt &Mask) -> std::pair<unsigned, int> {
4220 return GetSourceIndex(DstIdx, Mask.getZExtValue());
4221 });
4222}
4223
4224static bool interp__builtin_ia32_shift_with_count(
4225 InterpState &S, CodePtr OpPC, const CallExpr *Call,
4226 llvm::function_ref<APInt(const APInt &, uint64_t)> ShiftOp,
4227 llvm::function_ref<APInt(const APInt &, unsigned)> OverflowOp) {
4228
4229 assert(Call->getNumArgs() == 2);
4230
4231 const Pointer &Count = S.Stk.pop<Pointer>();
4232 const Pointer &Source = S.Stk.pop<Pointer>();
4233
4234 QualType SourceType = Call->getArg(Arg: 0)->getType();
4235 QualType CountType = Call->getArg(Arg: 1)->getType();
4236 assert(SourceType->isVectorType() && CountType->isVectorType());
4237
4238 const auto *SourceVecT = SourceType->castAs<VectorType>();
4239 const auto *CountVecT = CountType->castAs<VectorType>();
4240 PrimType SourceElemT = *S.getContext().classify(T: SourceVecT->getElementType());
4241 PrimType CountElemT = *S.getContext().classify(T: CountVecT->getElementType());
4242
4243 const Pointer &Dst = S.Stk.peek<Pointer>();
4244
4245 unsigned DestEltWidth =
4246 S.getASTContext().getTypeSize(T: SourceVecT->getElementType());
4247 bool IsDestUnsigned = SourceVecT->getElementType()->isUnsignedIntegerType();
4248 unsigned DestLen = SourceVecT->getNumElements();
4249 unsigned CountEltWidth =
4250 S.getASTContext().getTypeSize(T: CountVecT->getElementType());
4251 unsigned NumBitsInQWord = 64;
4252 unsigned NumCountElts = NumBitsInQWord / CountEltWidth;
4253
4254 uint64_t CountLQWord = 0;
4255 for (unsigned EltIdx = 0; EltIdx != NumCountElts; ++EltIdx) {
4256 uint64_t Elt = 0;
4257 INT_TYPE_SWITCH(CountElemT,
4258 { Elt = static_cast<uint64_t>(Count.elem<T>(EltIdx)); });
4259 CountLQWord |= (Elt << (EltIdx * CountEltWidth));
4260 }
4261
4262 for (unsigned EltIdx = 0; EltIdx != DestLen; ++EltIdx) {
4263 APSInt Elt;
4264 INT_TYPE_SWITCH(SourceElemT, { Elt = Source.elem<T>(EltIdx).toAPSInt(); });
4265
4266 APInt Result;
4267 if (CountLQWord < DestEltWidth) {
4268 Result = ShiftOp(Elt, CountLQWord);
4269 } else {
4270 Result = OverflowOp(Elt, DestEltWidth);
4271 }
4272 if (IsDestUnsigned) {
4273 INT_TYPE_SWITCH(SourceElemT, {
4274 Dst.elem<T>(EltIdx) = T::from(Result.getZExtValue());
4275 });
4276 } else {
4277 INT_TYPE_SWITCH(SourceElemT, {
4278 Dst.elem<T>(EltIdx) = T::from(Result.getSExtValue());
4279 });
4280 }
4281 }
4282
4283 Dst.initializeAllElements();
4284 return true;
4285}
4286
4287static bool interp__builtin_ia32_shufbitqmb_mask(InterpState &S, CodePtr OpPC,
4288 const CallExpr *Call) {
4289
4290 assert(Call->getNumArgs() == 3);
4291
4292 QualType SourceType = Call->getArg(Arg: 0)->getType();
4293 QualType ShuffleMaskType = Call->getArg(Arg: 1)->getType();
4294 QualType ZeroMaskType = Call->getArg(Arg: 2)->getType();
4295 if (!SourceType->isVectorType() || !ShuffleMaskType->isVectorType() ||
4296 !ZeroMaskType->isIntegerType()) {
4297 return false;
4298 }
4299
4300 Pointer Source, ShuffleMask;
4301 APSInt ZeroMask;
4302 if (!popToAPSInt(S, E: Call->getArg(Arg: 2), Out&: ZeroMask))
4303 return false;
4304 ShuffleMask = S.Stk.pop<Pointer>();
4305 Source = S.Stk.pop<Pointer>();
4306
4307 const auto *SourceVecT = SourceType->castAs<VectorType>();
4308 const auto *ShuffleMaskVecT = ShuffleMaskType->castAs<VectorType>();
4309 assert(SourceVecT->getNumElements() == ShuffleMaskVecT->getNumElements());
4310 assert(ZeroMask.getBitWidth() == SourceVecT->getNumElements());
4311
4312 PrimType SourceElemT = *S.getContext().classify(T: SourceVecT->getElementType());
4313 PrimType ShuffleMaskElemT =
4314 *S.getContext().classify(T: ShuffleMaskVecT->getElementType());
4315
4316 unsigned NumBytesInQWord = 8;
4317 unsigned NumBitsInByte = 8;
4318 unsigned NumBytes = SourceVecT->getNumElements();
4319 unsigned NumQWords = NumBytes / NumBytesInQWord;
4320 unsigned RetWidth = ZeroMask.getBitWidth();
4321 APSInt RetMask(llvm::APInt(RetWidth, 0), /*isUnsigned=*/true);
4322
4323 for (unsigned QWordId = 0; QWordId != NumQWords; ++QWordId) {
4324 APInt SourceQWord(64, 0);
4325 for (unsigned ByteIdx = 0; ByteIdx != NumBytesInQWord; ++ByteIdx) {
4326 uint64_t Byte = 0;
4327 INT_TYPE_SWITCH(SourceElemT, {
4328 Byte = static_cast<uint64_t>(
4329 Source.elem<T>(QWordId * NumBytesInQWord + ByteIdx));
4330 });
4331 SourceQWord.insertBits(SubBits: APInt(8, Byte & 0xFF), bitPosition: ByteIdx * NumBitsInByte);
4332 }
4333
4334 for (unsigned ByteIdx = 0; ByteIdx != NumBytesInQWord; ++ByteIdx) {
4335 unsigned SelIdx = QWordId * NumBytesInQWord + ByteIdx;
4336 unsigned M = 0;
4337 INT_TYPE_SWITCH(ShuffleMaskElemT, {
4338 M = static_cast<unsigned>(ShuffleMask.elem<T>(SelIdx)) & 0x3F;
4339 });
4340
4341 if (ZeroMask[SelIdx]) {
4342 RetMask.setBitVal(BitPosition: SelIdx, BitValue: SourceQWord[M]);
4343 }
4344 }
4345 }
4346
4347 pushInteger(S, Val: RetMask, QT: Call->getType());
4348 return true;
4349}
4350
4351static bool interp__builtin_ia32_vcvtps2ph(InterpState &S, CodePtr OpPC,
4352 const CallExpr *Call) {
4353 // Arguments are: vector of floats, rounding immediate
4354 assert(Call->getNumArgs() == 2);
4355
4356 APSInt Imm;
4357 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: Imm))
4358 return false;
4359 const Pointer &Src = S.Stk.pop<Pointer>();
4360 const Pointer &Dst = S.Stk.peek<Pointer>();
4361
4362 assert(Src.getFieldDesc()->isPrimitiveArray());
4363 assert(Dst.getFieldDesc()->isPrimitiveArray());
4364
4365 const auto *SrcVTy = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
4366 unsigned SrcNumElems = SrcVTy->getNumElements();
4367 const auto *DstVTy = Call->getType()->castAs<VectorType>();
4368 unsigned DstNumElems = DstVTy->getNumElements();
4369
4370 const llvm::fltSemantics &HalfSem =
4371 S.getASTContext().getFloatTypeSemantics(T: S.getASTContext().HalfTy);
4372
4373 // imm[2] == 1 means use MXCSR rounding mode.
4374 // In that case, we can only evaluate if the conversion is exact.
4375 int ImmVal = Imm.getZExtValue();
4376 bool UseMXCSR = (ImmVal & 4) != 0;
4377 bool IsFPConstrained =
4378 Call->getFPFeaturesInEffect(LO: S.getASTContext().getLangOpts())
4379 .isFPConstrained();
4380
4381 llvm::RoundingMode RM;
4382 if (!UseMXCSR) {
4383 switch (ImmVal & 3) {
4384 case 0:
4385 RM = llvm::RoundingMode::NearestTiesToEven;
4386 break;
4387 case 1:
4388 RM = llvm::RoundingMode::TowardNegative;
4389 break;
4390 case 2:
4391 RM = llvm::RoundingMode::TowardPositive;
4392 break;
4393 case 3:
4394 RM = llvm::RoundingMode::TowardZero;
4395 break;
4396 default:
4397 llvm_unreachable("Invalid immediate rounding mode");
4398 }
4399 } else {
4400 // For MXCSR, we must check for exactness. We can use any rounding mode
4401 // for the trial conversion since the result is the same if it's exact.
4402 RM = llvm::RoundingMode::NearestTiesToEven;
4403 }
4404
4405 QualType DstElemQT = Dst.getFieldDesc()->getElemQualType();
4406 PrimType DstElemT = *S.getContext().classify(T: DstElemQT);
4407
4408 for (unsigned I = 0; I != SrcNumElems; ++I) {
4409 Floating SrcVal = Src.elem<Floating>(I);
4410 APFloat DstVal = SrcVal.getAPFloat();
4411
4412 bool LostInfo;
4413 APFloat::opStatus St = DstVal.convert(ToSemantics: HalfSem, RM, losesInfo: &LostInfo);
4414
4415 if (UseMXCSR && IsFPConstrained && St != APFloat::opOK) {
4416 S.FFDiag(SI: S.Current->getSource(PC: OpPC),
4417 DiagId: diag::note_constexpr_dynamic_rounding);
4418 return false;
4419 }
4420
4421 INT_TYPE_SWITCH_NO_BOOL(DstElemT, {
4422 // Convert the destination value's bit pattern to an unsigned integer,
4423 // then reconstruct the element using the target type's 'from' method.
4424 uint64_t RawBits = DstVal.bitcastToAPInt().getZExtValue();
4425 Dst.elem<T>(I) = T::from(RawBits);
4426 });
4427 }
4428
4429 // Zero out remaining elements if the destination has more elements
4430 // (e.g., vcvtps2ph converting 4 floats to 8 shorts).
4431 if (DstNumElems > SrcNumElems) {
4432 for (unsigned I = SrcNumElems; I != DstNumElems; ++I) {
4433 INT_TYPE_SWITCH_NO_BOOL(DstElemT, { Dst.elem<T>(I) = T::from(0); });
4434 }
4435 }
4436
4437 Dst.initializeAllElements();
4438 return true;
4439}
4440
4441static bool interp__builtin_ia32_multishiftqb(InterpState &S, CodePtr OpPC,
4442 const CallExpr *Call) {
4443 assert(Call->getNumArgs() == 2);
4444
4445 QualType ATy = Call->getArg(Arg: 0)->getType();
4446 QualType BTy = Call->getArg(Arg: 1)->getType();
4447 if (!ATy->isVectorType() || !BTy->isVectorType()) {
4448 return false;
4449 }
4450
4451 const Pointer &BPtr = S.Stk.pop<Pointer>();
4452 const Pointer &APtr = S.Stk.pop<Pointer>();
4453 const auto *AVecT = ATy->castAs<VectorType>();
4454 assert(AVecT->getNumElements() ==
4455 BTy->castAs<VectorType>()->getNumElements());
4456
4457 PrimType ElemT = *S.getContext().classify(T: AVecT->getElementType());
4458
4459 unsigned NumBytesInQWord = 8;
4460 unsigned NumBitsInByte = 8;
4461 unsigned NumBytes = AVecT->getNumElements();
4462 unsigned NumQWords = NumBytes / NumBytesInQWord;
4463 const Pointer &Dst = S.Stk.peek<Pointer>();
4464
4465 for (unsigned QWordId = 0; QWordId != NumQWords; ++QWordId) {
4466 APInt BQWord(64, 0);
4467 for (unsigned ByteIdx = 0; ByteIdx != NumBytesInQWord; ++ByteIdx) {
4468 unsigned Idx = QWordId * NumBytesInQWord + ByteIdx;
4469 INT_TYPE_SWITCH(ElemT, {
4470 uint64_t Byte = static_cast<uint64_t>(BPtr.elem<T>(Idx));
4471 BQWord.insertBits(APInt(8, Byte & 0xFF), ByteIdx * NumBitsInByte);
4472 });
4473 }
4474
4475 for (unsigned ByteIdx = 0; ByteIdx != NumBytesInQWord; ++ByteIdx) {
4476 unsigned Idx = QWordId * NumBytesInQWord + ByteIdx;
4477 uint64_t Ctrl = 0;
4478 INT_TYPE_SWITCH(
4479 ElemT, { Ctrl = static_cast<uint64_t>(APtr.elem<T>(Idx)) & 0x3F; });
4480
4481 APInt Byte(8, 0);
4482 for (unsigned BitIdx = 0; BitIdx != NumBitsInByte; ++BitIdx) {
4483 Byte.setBitVal(BitPosition: BitIdx, BitValue: BQWord[(Ctrl + BitIdx) & 0x3F]);
4484 }
4485 INT_TYPE_SWITCH(ElemT,
4486 { Dst.elem<T>(Idx) = T::from(Byte.getZExtValue()); });
4487 }
4488 }
4489
4490 Dst.initializeAllElements();
4491
4492 return true;
4493}
4494
4495static bool interp__builtin_ia32_gfni_affine(InterpState &S, CodePtr OpPC,
4496 const CallExpr *Call,
4497 bool Inverse) {
4498 assert(Call->getNumArgs() == 3);
4499 QualType XType = Call->getArg(Arg: 0)->getType();
4500 QualType AType = Call->getArg(Arg: 1)->getType();
4501 QualType ImmType = Call->getArg(Arg: 2)->getType();
4502 if (!XType->isVectorType() || !AType->isVectorType() ||
4503 !ImmType->isIntegerType()) {
4504 return false;
4505 }
4506
4507 Pointer X, A;
4508 APSInt Imm;
4509 if (!popToAPSInt(S, E: Call->getArg(Arg: 2), Out&: Imm))
4510 return false;
4511 A = S.Stk.pop<Pointer>();
4512 X = S.Stk.pop<Pointer>();
4513
4514 const Pointer &Dst = S.Stk.peek<Pointer>();
4515 const auto *AVecT = AType->castAs<VectorType>();
4516 assert(XType->castAs<VectorType>()->getNumElements() ==
4517 AVecT->getNumElements());
4518 unsigned NumBytesInQWord = 8;
4519 unsigned NumBytes = AVecT->getNumElements();
4520 unsigned NumBitsInQWord = 64;
4521 unsigned NumQWords = NumBytes / NumBytesInQWord;
4522 unsigned NumBitsInByte = 8;
4523 PrimType AElemT = *S.getContext().classify(T: AVecT->getElementType());
4524
4525 // computing A*X + Imm
4526 for (unsigned QWordIdx = 0; QWordIdx != NumQWords; ++QWordIdx) {
4527 // Extract the QWords from X, A
4528 APInt XQWord(NumBitsInQWord, 0);
4529 APInt AQWord(NumBitsInQWord, 0);
4530 for (unsigned ByteIdx = 0; ByteIdx != NumBytesInQWord; ++ByteIdx) {
4531 unsigned Idx = QWordIdx * NumBytesInQWord + ByteIdx;
4532 uint8_t XByte;
4533 uint8_t AByte;
4534 INT_TYPE_SWITCH(AElemT, {
4535 XByte = static_cast<uint8_t>(X.elem<T>(Idx));
4536 AByte = static_cast<uint8_t>(A.elem<T>(Idx));
4537 });
4538
4539 XQWord.insertBits(SubBits: APInt(NumBitsInByte, XByte), bitPosition: ByteIdx * NumBitsInByte);
4540 AQWord.insertBits(SubBits: APInt(NumBitsInByte, AByte), bitPosition: ByteIdx * NumBitsInByte);
4541 }
4542
4543 for (unsigned ByteIdx = 0; ByteIdx != NumBytesInQWord; ++ByteIdx) {
4544 unsigned Idx = QWordIdx * NumBytesInQWord + ByteIdx;
4545 uint8_t XByte =
4546 XQWord.lshr(shiftAmt: ByteIdx * NumBitsInByte).getLoBits(numBits: 8).getZExtValue();
4547 INT_TYPE_SWITCH(AElemT, {
4548 Dst.elem<T>(Idx) = T::from(GFNIAffine(XByte, AQWord, Imm, Inverse));
4549 });
4550 }
4551 }
4552 Dst.initializeAllElements();
4553 return true;
4554}
4555
4556static bool interp__builtin_ia32_gfni_mul(InterpState &S, CodePtr OpPC,
4557 const CallExpr *Call) {
4558 assert(Call->getNumArgs() == 2);
4559
4560 QualType AType = Call->getArg(Arg: 0)->getType();
4561 QualType BType = Call->getArg(Arg: 1)->getType();
4562 if (!AType->isVectorType() || !BType->isVectorType()) {
4563 return false;
4564 }
4565
4566 Pointer A, B;
4567 B = S.Stk.pop<Pointer>();
4568 A = S.Stk.pop<Pointer>();
4569
4570 const Pointer &Dst = S.Stk.peek<Pointer>();
4571 const auto *AVecT = AType->castAs<VectorType>();
4572 assert(AVecT->getNumElements() ==
4573 BType->castAs<VectorType>()->getNumElements());
4574
4575 PrimType AElemT = *S.getContext().classify(T: AVecT->getElementType());
4576 unsigned NumBytes = A.getNumElems();
4577
4578 for (unsigned ByteIdx = 0; ByteIdx != NumBytes; ++ByteIdx) {
4579 uint8_t AByte, BByte;
4580 INT_TYPE_SWITCH(AElemT, {
4581 AByte = static_cast<uint8_t>(A.elem<T>(ByteIdx));
4582 BByte = static_cast<uint8_t>(B.elem<T>(ByteIdx));
4583 Dst.elem<T>(ByteIdx) = T::from(GFNIMul(AByte, BByte));
4584 });
4585 }
4586
4587 Dst.initializeAllElements();
4588 return true;
4589}
4590
4591static bool interp__builtin_ia32_vpdp(InterpState &S, CodePtr OpPC,
4592 const CallExpr *Call, bool IsSaturating) {
4593 assert(Call->getNumArgs() == 3);
4594
4595 QualType SrcT = Call->getArg(Arg: 0)->getType();
4596 QualType OpAT = Call->getArg(Arg: 1)->getType();
4597 QualType OpBT = Call->getArg(Arg: 2)->getType();
4598 QualType DstT = Call->getType();
4599 if (!SrcT->isVectorType() || !OpAT->isVectorType() || !OpBT->isVectorType() ||
4600 !DstT->isVectorType())
4601 return false;
4602
4603 const auto *SrcVecT = SrcT->castAs<VectorType>();
4604 const auto *OpAVecT = OpAT->castAs<VectorType>();
4605 const auto *OpBVecT = OpBT->castAs<VectorType>();
4606 const auto *DstVecT = DstT->castAs<VectorType>();
4607
4608 assert(OpAVecT->getNumElements() == OpBVecT->getNumElements());
4609
4610 unsigned NumSrcElems = SrcVecT->getNumElements();
4611 unsigned NumOperandElems = OpAVecT->getNumElements();
4612 unsigned ElemsPerLane = NumOperandElems / NumSrcElems;
4613
4614 PrimType SrcElemT = *S.getContext().classify(T: SrcVecT->getElementType());
4615 PrimType OpAElemT = *S.getContext().classify(T: OpAVecT->getElementType());
4616 PrimType OpBElemT = *S.getContext().classify(T: OpBVecT->getElementType());
4617 PrimType DstElemT = *S.getContext().classify(T: DstVecT->getElementType());
4618
4619 assert(SrcElemT == DstElemT);
4620
4621 const Pointer &OpBPtr = S.Stk.pop<Pointer>();
4622 const Pointer &OpAPtr = S.Stk.pop<Pointer>();
4623 const Pointer &SrcPtr = S.Stk.pop<Pointer>();
4624 const Pointer &Dst = S.Stk.peek<Pointer>();
4625
4626 for (unsigned I = 0; I != NumSrcElems; ++I) {
4627 APSInt Acc;
4628 INT_TYPE_SWITCH_NO_BOOL(SrcElemT, { Acc = SrcPtr.elem<T>(I).toAPSInt(); });
4629 Acc = Acc.sext(width: 64);
4630 for (unsigned J = 0; J != ElemsPerLane; ++J) {
4631 APSInt OpA, OpB;
4632 INT_TYPE_SWITCH_NO_BOOL(
4633 OpAElemT, { OpA = OpAPtr.elem<T>(ElemsPerLane * I + J).toAPSInt(); });
4634 INT_TYPE_SWITCH_NO_BOOL(
4635 OpBElemT, { OpB = OpBPtr.elem<T>(ElemsPerLane * I + J).toAPSInt(); });
4636 OpA = APSInt(OpA.extend(width: 64), false);
4637 OpB = APSInt(OpB.extend(width: 64), false);
4638 Acc += OpA * OpB;
4639 }
4640 if (IsSaturating)
4641 Acc = APSInt(Acc.truncSSat(width: 32), false);
4642 else
4643 Acc = APSInt(Acc.trunc(width: 32), false);
4644 INT_TYPE_SWITCH_NO_BOOL(DstElemT,
4645 { Dst.elem<T>(I) = static_cast<T>(Acc); });
4646 }
4647 Dst.initializeAllElements();
4648 return true;
4649}
4650
4651// Bit Matrix Multiply and Accumulate (AVX512BMM). Each 256-bit lane holds a
4652// 16x16 bit matrix as 16 x i16 elements; element i is row i and bit j of that
4653// element is entry [i][j]. The accumulator (third argument, src1 in the AMD
4654// ISA) provides the initial value of each result bit, into which the bit-matrix
4655// product of the first two arguments (src2 * src3) is reduced with OR (vbmacor)
4656// or XOR (vbmacxor):
4657// for i in 0..15, j in 0..15:
4658// bit = C[16*i+j]
4659// for k in 0..15: bit OP= A[16*i+k] & B[16*k+j]
4660// dest[16*i+j] = bit
4661static bool interp__builtin_ia32_bmac(InterpState &S, CodePtr OpPC,
4662 const CallExpr *Call, bool IsXor) {
4663 assert(Call->getNumArgs() == 3);
4664
4665 // AST-based type checks before popping the stack.
4666 QualType AType = Call->getArg(Arg: 0)->getType();
4667 QualType BType = Call->getArg(Arg: 1)->getType();
4668 QualType CType = Call->getArg(Arg: 2)->getType();
4669 if (!AType->isVectorType() || !BType->isVectorType() ||
4670 !CType->isVectorType())
4671 return false;
4672
4673 const Pointer &C = S.Stk.pop<Pointer>();
4674 const Pointer &B = S.Stk.pop<Pointer>();
4675 const Pointer &A = S.Stk.pop<Pointer>();
4676 const Pointer &Dst = S.Stk.peek<Pointer>();
4677
4678 // check if all three primitive arrays are with 16-bit elements.
4679 auto isValid16BitArray = [](const Pointer &P) {
4680 const Descriptor *D = P.getFieldDesc();
4681 if (!D->isPrimitiveArray())
4682 return false;
4683 PrimType PT = D->getPrimType();
4684 return ((PT == PT_Sint16) || (PT == PT_Uint16));
4685 };
4686
4687 if (!isValid16BitArray(A) || !isValid16BitArray(B) || !isValid16BitArray(C))
4688 return false;
4689
4690 PrimType ElemT = A.getFieldDesc()->getPrimType();
4691 unsigned NumElems = A.getNumElems();
4692 assert(NumElems % 16 == 0 && "BMM operates on 256-bit lanes of 16 x i16");
4693 bool DstUnsigned = ElemT == PT_Uint16;
4694
4695 // Lanes are always 16-bit; gather them so the reduction below is untyped.
4696 SmallVector<uint16_t> AVals(NumElems), BVals(NumElems), Acc(NumElems);
4697 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
4698 for (unsigned I = 0; I != NumElems; ++I) {
4699 AVals[I] = (uint16_t)A.elem<T>(I).toAPSInt().getZExtValue();
4700 BVals[I] = (uint16_t)B.elem<T>(I).toAPSInt().getZExtValue();
4701 Acc[I] = (uint16_t)C.elem<T>(I).toAPSInt().getZExtValue();
4702 }
4703 });
4704
4705 for (unsigned Lane = 0; Lane != NumElems; Lane += 16) {
4706 for (unsigned I = 0; I != 16; ++I) {
4707 uint16_t AVal = AVals[Lane + I], DVal = Acc[Lane + I];
4708 for (unsigned J = 0; J != 16; ++J) {
4709 // Seed the reduction with the accumulator bit, then fold in each
4710 // product term with the same operator (OR for vbmacor, XOR for
4711 // vbmacxor).
4712 unsigned Bit = (DVal >> J) & 1u;
4713 for (unsigned K = 0; K != 16; ++K) {
4714 unsigned Product = ((AVal >> K) & 1u) & ((BVals[Lane + K] >> J) & 1u);
4715 Bit = IsXor ? (Bit ^ Product) : (Bit | Product);
4716 }
4717 DVal = (DVal & ~(uint16_t(1) << J)) | (uint16_t(Bit) << J);
4718 }
4719 Acc[Lane + I] = DVal;
4720 }
4721 }
4722
4723 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
4724 for (unsigned I = 0; I != NumElems; ++I)
4725 Dst.elem<T>(I) = static_cast<T>(APSInt(APInt(16, Acc[I]), DstUnsigned));
4726 });
4727 Dst.initializeAllElements();
4728 return true;
4729}
4730
4731static bool interp_builtin_ia32_cvt_scalar_to_int(InterpState &S, CodePtr OpPC,
4732 const CallExpr *E) {
4733 Pointer SrcVecPtr = S.Stk.pop<Pointer>();
4734 const Floating &FloatElem = SrcVecPtr.elem<Floating>(I: 0);
4735
4736 unsigned BitWidth = S.getASTContext().getIntWidth(T: E->getType());
4737 bool IsUnsigned = E->getType()->isUnsignedIntegerType();
4738
4739 llvm::APSInt IntResult(BitWidth, IsUnsigned);
4740 bool IsExact = false;
4741 // We only allow exact conversions so rounding mode does not matter for cvt*
4742 // and cvtt* builtins
4743 FloatElem.getAPFloat().convertToInteger(
4744 Result&: IntResult, RM: llvm::APFloat::rmTowardZero, IsExact: &IsExact);
4745 if (!IsExact)
4746 return false;
4747
4748 pushInteger(S, Val: IntResult, QT: E->getType());
4749 return true;
4750}
4751
4752static bool interp_builtin_ia32_cvt_vector_to_int(InterpState &S, CodePtr OpPC,
4753 const CallExpr *E) {
4754 Pointer SrcVecPtr = S.Stk.pop<Pointer>();
4755 const Pointer &Dst = S.Stk.peek<Pointer>();
4756
4757 unsigned NumSrcElems = SrcVecPtr.getNumElems();
4758 unsigned NumDstElems = Dst.getNumElems();
4759
4760 if (NumSrcElems > NumDstElems)
4761 return false;
4762
4763 QualType ElemType = Dst.getFieldDesc()->getElemQualType();
4764 unsigned BitWidth = S.getASTContext().getIntWidth(T: ElemType);
4765 bool IsUnsigned = ElemType->isUnsignedIntegerType();
4766
4767 PrimType ElemT = *S.getContext().classify(T: ElemType);
4768 for (unsigned I = 0; I != NumSrcElems; ++I) {
4769 const Floating &FloatElem = SrcVecPtr.elem<Floating>(I);
4770 llvm::APSInt IntResult(BitWidth, IsUnsigned);
4771
4772 bool IsExact = false;
4773 // We only allow exact conversions so rounding mode does not matter for
4774 // cvt* and cvtt* builtins
4775 FloatElem.getAPFloat().convertToInteger(
4776 Result&: IntResult, RM: llvm::APFloat::rmTowardZero, IsExact: &IsExact);
4777 if (!IsExact)
4778 return false;
4779 INT_TYPE_SWITCH_NO_BOOL(
4780 ElemT, { Dst.elem<T>(I) = T::from(IntResult.getZExtValue()); });
4781 }
4782
4783 // Zero out remaining elements if the destination has more elements
4784 // (e.g., cvtpd2dq converting 2 doubles(_m128d) to 2 ints stored in _m128i).
4785 for (unsigned I = NumSrcElems; I != NumDstElems; ++I)
4786 INT_TYPE_SWITCH_NO_BOOL(ElemT, { Dst.elem<T>(I) = T::from(0); });
4787
4788 Dst.initializeAllElements();
4789 return true;
4790}
4791
4792bool InterpretBuiltin(InterpState &S, CodePtr OpPC, const CallExpr *Call,
4793 uint32_t BuiltinID) {
4794 const InterpFrame *Frame = S.Current;
4795 switch (BuiltinID) {
4796 case Builtin::BI__builtin_is_constant_evaluated:
4797 return interp__builtin_is_constant_evaluated(S, OpPC, Frame, Call);
4798
4799 case Builtin::BI__builtin_assume:
4800 case Builtin::BI__assume:
4801 return interp__builtin_assume(S, OpPC, Frame, Call);
4802
4803 case Builtin::BI__builtin_strcmp:
4804 case Builtin::BIstrcmp:
4805 case Builtin::BI__builtin_strncmp:
4806 case Builtin::BIstrncmp:
4807 case Builtin::BI__builtin_wcsncmp:
4808 case Builtin::BIwcsncmp:
4809 case Builtin::BI__builtin_wcscmp:
4810 case Builtin::BIwcscmp:
4811 return interp__builtin_strcmp(S, OpPC, Frame, Call, ID: BuiltinID);
4812
4813 case Builtin::BI__builtin_strlen:
4814 case Builtin::BIstrlen:
4815 case Builtin::BI__builtin_wcslen:
4816 case Builtin::BIwcslen:
4817 return interp__builtin_strlen(S, OpPC, Frame, Call, ID: BuiltinID);
4818
4819 case Builtin::BI__builtin_nan:
4820 case Builtin::BI__builtin_nanf:
4821 case Builtin::BI__builtin_nanl:
4822 case Builtin::BI__builtin_nanf16:
4823 case Builtin::BI__builtin_nanf128:
4824 return interp__builtin_nan(S, OpPC, Frame, Call, /*Signaling=*/false);
4825
4826 case Builtin::BI__builtin_nans:
4827 case Builtin::BI__builtin_nansf:
4828 case Builtin::BI__builtin_nansl:
4829 case Builtin::BI__builtin_nansf16:
4830 case Builtin::BI__builtin_nansf128:
4831 return interp__builtin_nan(S, OpPC, Frame, Call, /*Signaling=*/true);
4832
4833 case Builtin::BI__builtin_huge_val:
4834 case Builtin::BI__builtin_huge_valf:
4835 case Builtin::BI__builtin_huge_vall:
4836 case Builtin::BI__builtin_huge_valf16:
4837 case Builtin::BI__builtin_huge_valf128:
4838 case Builtin::BI__builtin_inf:
4839 case Builtin::BI__builtin_inff:
4840 case Builtin::BI__builtin_infl:
4841 case Builtin::BI__builtin_inff16:
4842 case Builtin::BI__builtin_inff128:
4843 return interp__builtin_inf(S, OpPC, Frame, Call);
4844
4845 case Builtin::BI__builtin_copysign:
4846 case Builtin::BI__builtin_copysignf:
4847 case Builtin::BI__builtin_copysignl:
4848 case Builtin::BI__builtin_copysignf128:
4849 return interp__builtin_copysign(S, OpPC, Frame);
4850
4851 case Builtin::BI__builtin_fmin:
4852 case Builtin::BI__builtin_fminf:
4853 case Builtin::BI__builtin_fminl:
4854 case Builtin::BI__builtin_fminf16:
4855 case Builtin::BI__builtin_fminf128:
4856 return interp__builtin_fmin(S, OpPC, Frame, /*IsNumBuiltin=*/false);
4857
4858 case Builtin::BI__builtin_fminimum_num:
4859 case Builtin::BI__builtin_fminimum_numf:
4860 case Builtin::BI__builtin_fminimum_numl:
4861 case Builtin::BI__builtin_fminimum_numf16:
4862 case Builtin::BI__builtin_fminimum_numf128:
4863 return interp__builtin_fmin(S, OpPC, Frame, /*IsNumBuiltin=*/true);
4864
4865 case Builtin::BI__builtin_fmax:
4866 case Builtin::BI__builtin_fmaxf:
4867 case Builtin::BI__builtin_fmaxl:
4868 case Builtin::BI__builtin_fmaxf16:
4869 case Builtin::BI__builtin_fmaxf128:
4870 return interp__builtin_fmax(S, OpPC, Frame, /*IsNumBuiltin=*/false);
4871
4872 case Builtin::BI__builtin_fmaximum_num:
4873 case Builtin::BI__builtin_fmaximum_numf:
4874 case Builtin::BI__builtin_fmaximum_numl:
4875 case Builtin::BI__builtin_fmaximum_numf16:
4876 case Builtin::BI__builtin_fmaximum_numf128:
4877 return interp__builtin_fmax(S, OpPC, Frame, /*IsNumBuiltin=*/true);
4878
4879 case Builtin::BI__builtin_isnan:
4880 return interp__builtin_isnan(S, OpPC, Frame, Call);
4881
4882 case Builtin::BI__builtin_issignaling:
4883 return interp__builtin_issignaling(S, OpPC, Frame, Call);
4884
4885 case Builtin::BI__builtin_isinf:
4886 return interp__builtin_isinf(S, OpPC, Frame, /*Sign=*/CheckSign: false, Call);
4887
4888 case Builtin::BI__builtin_isinf_sign:
4889 return interp__builtin_isinf(S, OpPC, Frame, /*Sign=*/CheckSign: true, Call);
4890
4891 case Builtin::BI__builtin_isfinite:
4892 return interp__builtin_isfinite(S, OpPC, Frame, Call);
4893
4894 case Builtin::BI__builtin_isnormal:
4895 return interp__builtin_isnormal(S, OpPC, Frame, Call);
4896
4897 case Builtin::BI__builtin_issubnormal:
4898 return interp__builtin_issubnormal(S, OpPC, Frame, Call);
4899
4900 case Builtin::BI__builtin_iszero:
4901 return interp__builtin_iszero(S, OpPC, Frame, Call);
4902
4903 case Builtin::BI__builtin_signbit:
4904 case Builtin::BI__builtin_signbitf:
4905 case Builtin::BI__builtin_signbitl:
4906 return interp__builtin_signbit(S, OpPC, Frame, Call);
4907
4908 case Builtin::BI__builtin_isgreater:
4909 case Builtin::BI__builtin_isgreaterequal:
4910 case Builtin::BI__builtin_isless:
4911 case Builtin::BI__builtin_islessequal:
4912 case Builtin::BI__builtin_islessgreater:
4913 case Builtin::BI__builtin_isunordered:
4914 return interp_floating_comparison(S, OpPC, Call, ID: BuiltinID);
4915
4916 case Builtin::BI__builtin_isfpclass:
4917 return interp__builtin_isfpclass(S, OpPC, Frame, Call);
4918
4919 case Builtin::BI__builtin_fpclassify:
4920 return interp__builtin_fpclassify(S, OpPC, Frame, Call);
4921
4922 case Builtin::BI__builtin_fabs:
4923 case Builtin::BI__builtin_fabsf:
4924 case Builtin::BI__builtin_fabsl:
4925 case Builtin::BI__builtin_fabsf128:
4926 return interp__builtin_fabs(S, OpPC, Frame);
4927
4928 case Builtin::BI__builtin_abs:
4929 case Builtin::BI__builtin_labs:
4930 case Builtin::BI__builtin_llabs:
4931 return interp__builtin_abs(S, OpPC, Frame, Call);
4932
4933 case Builtin::BI__builtin_popcount:
4934 case Builtin::BI__builtin_popcountl:
4935 case Builtin::BI__builtin_popcountll:
4936 case Builtin::BI__builtin_popcountg:
4937 case Builtin::BI__popcnt16: // Microsoft variants of popcount
4938 case Builtin::BI__popcnt:
4939 case Builtin::BI__popcnt64:
4940 return interp__builtin_popcount(S, OpPC, Frame, Call);
4941
4942 case Builtin::BI__builtin_parity:
4943 case Builtin::BI__builtin_parityl:
4944 case Builtin::BI__builtin_parityll:
4945 return interp__builtin_elementwise_int_unaryop(
4946 S, OpPC, Call, Fn: [](const APSInt &Val) {
4947 return APInt(Val.getBitWidth(), Val.popcount() % 2);
4948 });
4949 case Builtin::BI__builtin_clrsb:
4950 case Builtin::BI__builtin_clrsbl:
4951 case Builtin::BI__builtin_clrsbll:
4952 return interp__builtin_elementwise_int_unaryop(
4953 S, OpPC, Call, Fn: [](const APSInt &Val) {
4954 return APInt(Val.getBitWidth(),
4955 Val.getBitWidth() - Val.getSignificantBits());
4956 });
4957 case Builtin::BI__builtin_bitreverseg:
4958 case Builtin::BI__builtin_bitreverse8:
4959 case Builtin::BI__builtin_bitreverse16:
4960 case Builtin::BI__builtin_bitreverse32:
4961 case Builtin::BI__builtin_bitreverse64:
4962 return interp__builtin_elementwise_int_unaryop(
4963 S, OpPC, Call, Fn: [](const APSInt &Val) { return Val.reverseBits(); });
4964
4965 case Builtin::BI__builtin_classify_type:
4966 return interp__builtin_classify_type(S, OpPC, Frame, Call);
4967
4968 case Builtin::BI__builtin_expect:
4969 case Builtin::BI__builtin_expect_with_probability:
4970 return interp__builtin_expect(S, OpPC, Frame, Call);
4971
4972 case Builtin::BI__builtin_rotateleft8:
4973 case Builtin::BI__builtin_rotateleft16:
4974 case Builtin::BI__builtin_rotateleft32:
4975 case Builtin::BI__builtin_rotateleft64:
4976 case Builtin::BI__builtin_stdc_rotate_left:
4977 case Builtin::BIstdc_rotate_left_uc:
4978 case Builtin::BIstdc_rotate_left_us:
4979 case Builtin::BIstdc_rotate_left_ui:
4980 case Builtin::BIstdc_rotate_left_ul:
4981 case Builtin::BIstdc_rotate_left_ull:
4982 case Builtin::BI_rotl8: // Microsoft variants of rotate left
4983 case Builtin::BI_rotl16:
4984 case Builtin::BI_rotl:
4985 case Builtin::BI_lrotl:
4986 case Builtin::BI_rotl64:
4987 case Builtin::BI__builtin_rotateright8:
4988 case Builtin::BI__builtin_rotateright16:
4989 case Builtin::BI__builtin_rotateright32:
4990 case Builtin::BI__builtin_rotateright64:
4991 case Builtin::BI__builtin_stdc_rotate_right:
4992 case Builtin::BIstdc_rotate_right_uc:
4993 case Builtin::BIstdc_rotate_right_us:
4994 case Builtin::BIstdc_rotate_right_ui:
4995 case Builtin::BIstdc_rotate_right_ul:
4996 case Builtin::BIstdc_rotate_right_ull:
4997 case Builtin::BI_rotr8: // Microsoft variants of rotate right
4998 case Builtin::BI_rotr16:
4999 case Builtin::BI_rotr:
5000 case Builtin::BI_lrotr:
5001 case Builtin::BI_rotr64: {
5002 // Determine if this is a rotate right operation
5003 bool IsRotateRight;
5004 switch (BuiltinID) {
5005 case Builtin::BI__builtin_rotateright8:
5006 case Builtin::BI__builtin_rotateright16:
5007 case Builtin::BI__builtin_rotateright32:
5008 case Builtin::BI__builtin_rotateright64:
5009 case Builtin::BI__builtin_stdc_rotate_right:
5010 case Builtin::BIstdc_rotate_right_uc:
5011 case Builtin::BIstdc_rotate_right_us:
5012 case Builtin::BIstdc_rotate_right_ui:
5013 case Builtin::BIstdc_rotate_right_ul:
5014 case Builtin::BIstdc_rotate_right_ull:
5015 case Builtin::BI_rotr8:
5016 case Builtin::BI_rotr16:
5017 case Builtin::BI_rotr:
5018 case Builtin::BI_lrotr:
5019 case Builtin::BI_rotr64:
5020 IsRotateRight = true;
5021 break;
5022 default:
5023 IsRotateRight = false;
5024 break;
5025 }
5026
5027 return interp__builtin_elementwise_int_binop(
5028 S, OpPC, Call, Fn: [IsRotateRight](const APSInt &Value, APSInt Amount) {
5029 Amount = NormalizeRotateAmount(Value, Amount);
5030 return IsRotateRight ? Value.rotr(rotateAmt: Amount.getZExtValue())
5031 : Value.rotl(rotateAmt: Amount.getZExtValue());
5032 });
5033 }
5034
5035 case Builtin::BIstdc_leading_zeros_uc:
5036 case Builtin::BIstdc_leading_zeros_us:
5037 case Builtin::BIstdc_leading_zeros_ui:
5038 case Builtin::BIstdc_leading_zeros_ul:
5039 case Builtin::BIstdc_leading_zeros_ull:
5040 case Builtin::BI__builtin_stdc_leading_zeros: {
5041 unsigned ResWidth = S.getASTContext().getIntWidth(T: Call->getType());
5042 return interp__builtin_elementwise_int_unaryop(
5043 S, OpPC, Call, Fn: [ResWidth](const APSInt &Val) {
5044 return APInt(ResWidth, Val.countl_zero());
5045 });
5046 }
5047
5048 case Builtin::BIstdc_leading_ones_uc:
5049 case Builtin::BIstdc_leading_ones_us:
5050 case Builtin::BIstdc_leading_ones_ui:
5051 case Builtin::BIstdc_leading_ones_ul:
5052 case Builtin::BIstdc_leading_ones_ull:
5053 case Builtin::BI__builtin_stdc_leading_ones: {
5054 unsigned ResWidth = S.getASTContext().getIntWidth(T: Call->getType());
5055 return interp__builtin_elementwise_int_unaryop(
5056 S, OpPC, Call, Fn: [ResWidth](const APSInt &Val) {
5057 return APInt(ResWidth, Val.countl_one());
5058 });
5059 }
5060
5061 case Builtin::BIstdc_trailing_zeros_uc:
5062 case Builtin::BIstdc_trailing_zeros_us:
5063 case Builtin::BIstdc_trailing_zeros_ui:
5064 case Builtin::BIstdc_trailing_zeros_ul:
5065 case Builtin::BIstdc_trailing_zeros_ull:
5066 case Builtin::BI__builtin_stdc_trailing_zeros: {
5067 unsigned ResWidth = S.getASTContext().getIntWidth(T: Call->getType());
5068 return interp__builtin_elementwise_int_unaryop(
5069 S, OpPC, Call, Fn: [ResWidth](const APSInt &Val) {
5070 return APInt(ResWidth, Val.countr_zero());
5071 });
5072 }
5073
5074 case Builtin::BIstdc_trailing_ones_uc:
5075 case Builtin::BIstdc_trailing_ones_us:
5076 case Builtin::BIstdc_trailing_ones_ui:
5077 case Builtin::BIstdc_trailing_ones_ul:
5078 case Builtin::BIstdc_trailing_ones_ull:
5079 case Builtin::BI__builtin_stdc_trailing_ones: {
5080 unsigned ResWidth = S.getASTContext().getIntWidth(T: Call->getType());
5081 return interp__builtin_elementwise_int_unaryop(
5082 S, OpPC, Call, Fn: [ResWidth](const APSInt &Val) {
5083 return APInt(ResWidth, Val.countr_one());
5084 });
5085 }
5086
5087 case Builtin::BIstdc_first_leading_zero_uc:
5088 case Builtin::BIstdc_first_leading_zero_us:
5089 case Builtin::BIstdc_first_leading_zero_ui:
5090 case Builtin::BIstdc_first_leading_zero_ul:
5091 case Builtin::BIstdc_first_leading_zero_ull:
5092 case Builtin::BI__builtin_stdc_first_leading_zero: {
5093 unsigned ResWidth = S.getASTContext().getIntWidth(T: Call->getType());
5094 return interp__builtin_elementwise_int_unaryop(
5095 S, OpPC, Call, Fn: [ResWidth](const APSInt &Val) {
5096 return APInt(ResWidth, Val.isAllOnes() ? 0 : Val.countl_one() + 1);
5097 });
5098 }
5099
5100 case Builtin::BIstdc_first_leading_one_uc:
5101 case Builtin::BIstdc_first_leading_one_us:
5102 case Builtin::BIstdc_first_leading_one_ui:
5103 case Builtin::BIstdc_first_leading_one_ul:
5104 case Builtin::BIstdc_first_leading_one_ull:
5105 case Builtin::BI__builtin_stdc_first_leading_one: {
5106 unsigned ResWidth = S.getASTContext().getIntWidth(T: Call->getType());
5107 return interp__builtin_elementwise_int_unaryop(
5108 S, OpPC, Call, Fn: [ResWidth](const APSInt &Val) {
5109 return APInt(ResWidth, Val.isZero() ? 0 : Val.countl_zero() + 1);
5110 });
5111 }
5112
5113 case Builtin::BIstdc_first_trailing_zero_uc:
5114 case Builtin::BIstdc_first_trailing_zero_us:
5115 case Builtin::BIstdc_first_trailing_zero_ui:
5116 case Builtin::BIstdc_first_trailing_zero_ul:
5117 case Builtin::BIstdc_first_trailing_zero_ull:
5118 case Builtin::BI__builtin_stdc_first_trailing_zero: {
5119 unsigned ResWidth = S.getASTContext().getIntWidth(T: Call->getType());
5120 return interp__builtin_elementwise_int_unaryop(
5121 S, OpPC, Call, Fn: [ResWidth](const APSInt &Val) {
5122 return APInt(ResWidth, Val.isAllOnes() ? 0 : Val.countr_one() + 1);
5123 });
5124 }
5125
5126 case Builtin::BIstdc_first_trailing_one_uc:
5127 case Builtin::BIstdc_first_trailing_one_us:
5128 case Builtin::BIstdc_first_trailing_one_ui:
5129 case Builtin::BIstdc_first_trailing_one_ul:
5130 case Builtin::BIstdc_first_trailing_one_ull:
5131 case Builtin::BI__builtin_stdc_first_trailing_one: {
5132 unsigned ResWidth = S.getASTContext().getIntWidth(T: Call->getType());
5133 return interp__builtin_elementwise_int_unaryop(
5134 S, OpPC, Call, Fn: [ResWidth](const APSInt &Val) {
5135 return APInt(ResWidth, Val.isZero() ? 0 : Val.countr_zero() + 1);
5136 });
5137 }
5138
5139 case Builtin::BIstdc_count_zeros_uc:
5140 case Builtin::BIstdc_count_zeros_us:
5141 case Builtin::BIstdc_count_zeros_ui:
5142 case Builtin::BIstdc_count_zeros_ul:
5143 case Builtin::BIstdc_count_zeros_ull:
5144 case Builtin::BI__builtin_stdc_count_zeros: {
5145 unsigned ResWidth = S.getASTContext().getIntWidth(T: Call->getType());
5146 return interp__builtin_elementwise_int_unaryop(
5147 S, OpPC, Call, Fn: [ResWidth](const APSInt &Val) {
5148 unsigned BitWidth = Val.getBitWidth();
5149 return APInt(ResWidth, BitWidth - Val.popcount());
5150 });
5151 }
5152
5153 case Builtin::BIstdc_count_ones_uc:
5154 case Builtin::BIstdc_count_ones_us:
5155 case Builtin::BIstdc_count_ones_ui:
5156 case Builtin::BIstdc_count_ones_ul:
5157 case Builtin::BIstdc_count_ones_ull:
5158 case Builtin::BI__builtin_stdc_count_ones: {
5159 unsigned ResWidth = S.getASTContext().getIntWidth(T: Call->getType());
5160 return interp__builtin_elementwise_int_unaryop(
5161 S, OpPC, Call, Fn: [ResWidth](const APSInt &Val) {
5162 return APInt(ResWidth, Val.popcount());
5163 });
5164 }
5165
5166 case Builtin::BIstdc_has_single_bit_uc:
5167 case Builtin::BIstdc_has_single_bit_us:
5168 case Builtin::BIstdc_has_single_bit_ui:
5169 case Builtin::BIstdc_has_single_bit_ul:
5170 case Builtin::BIstdc_has_single_bit_ull:
5171 case Builtin::BI__builtin_stdc_has_single_bit: {
5172 unsigned ResWidth = S.getASTContext().getIntWidth(T: Call->getType());
5173 return interp__builtin_elementwise_int_unaryop(
5174 S, OpPC, Call, Fn: [ResWidth](const APSInt &Val) {
5175 return APInt(ResWidth, Val.popcount() == 1 ? 1 : 0);
5176 });
5177 }
5178
5179 case Builtin::BIstdc_bit_width_uc:
5180 case Builtin::BIstdc_bit_width_us:
5181 case Builtin::BIstdc_bit_width_ui:
5182 case Builtin::BIstdc_bit_width_ul:
5183 case Builtin::BIstdc_bit_width_ull:
5184 case Builtin::BI__builtin_stdc_bit_width: {
5185 unsigned ResWidth = S.getASTContext().getIntWidth(T: Call->getType());
5186 return interp__builtin_elementwise_int_unaryop(
5187 S, OpPC, Call, Fn: [ResWidth](const APSInt &Val) {
5188 unsigned BitWidth = Val.getBitWidth();
5189 return APInt(ResWidth, BitWidth - Val.countl_zero());
5190 });
5191 }
5192
5193 case Builtin::BIstdc_bit_floor_uc:
5194 case Builtin::BIstdc_bit_floor_us:
5195 case Builtin::BIstdc_bit_floor_ui:
5196 case Builtin::BIstdc_bit_floor_ul:
5197 case Builtin::BIstdc_bit_floor_ull:
5198 case Builtin::BI__builtin_stdc_bit_floor:
5199 return interp__builtin_elementwise_int_unaryop(
5200 S, OpPC, Call, Fn: [](const APSInt &Val) {
5201 unsigned BitWidth = Val.getBitWidth();
5202 if (Val.isZero())
5203 return APInt::getZero(numBits: BitWidth);
5204 return APInt::getOneBitSet(numBits: BitWidth,
5205 BitNo: BitWidth - Val.countl_zero() - 1);
5206 });
5207
5208 case Builtin::BIstdc_bit_ceil_uc:
5209 case Builtin::BIstdc_bit_ceil_us:
5210 case Builtin::BIstdc_bit_ceil_ui:
5211 case Builtin::BIstdc_bit_ceil_ul:
5212 case Builtin::BIstdc_bit_ceil_ull:
5213 case Builtin::BI__builtin_stdc_bit_ceil:
5214 return interp__builtin_elementwise_int_unaryop(
5215 S, OpPC, Call, Fn: [](const APSInt &Val) {
5216 unsigned BitWidth = Val.getBitWidth();
5217 if (Val.ule(RHS: 1))
5218 return APInt(BitWidth, 1);
5219 APInt V = Val;
5220 APInt ValMinusOne = V - 1;
5221 unsigned LeadingZeros = ValMinusOne.countl_zero();
5222 if (LeadingZeros == 0)
5223 return APInt(BitWidth, 0); // overflows; wrap to 0
5224 return APInt::getOneBitSet(numBits: BitWidth, BitNo: BitWidth - LeadingZeros);
5225 });
5226
5227 case Builtin::BI__builtin_ffs:
5228 case Builtin::BI__builtin_ffsl:
5229 case Builtin::BI__builtin_ffsll:
5230 return interp__builtin_elementwise_int_unaryop(
5231 S, OpPC, Call, Fn: [](const APSInt &Val) {
5232 return APInt(Val.getBitWidth(),
5233 Val.isZero() ? 0u : Val.countTrailingZeros() + 1u);
5234 });
5235
5236 case Builtin::BIaddressof:
5237 case Builtin::BI__addressof:
5238 case Builtin::BI__builtin_addressof:
5239 assert(isNoopBuiltin(BuiltinID));
5240 return interp__builtin_addressof(S, OpPC, Frame, Call);
5241
5242 case Builtin::BIas_const:
5243 case Builtin::BIforward:
5244 case Builtin::BIforward_like:
5245 case Builtin::BImove:
5246 case Builtin::BImove_if_noexcept:
5247 assert(isNoopBuiltin(BuiltinID));
5248 return interp__builtin_move(S, OpPC, Frame, Call);
5249
5250 case Builtin::BI__builtin_eh_return_data_regno:
5251 return interp__builtin_eh_return_data_regno(S, OpPC, Frame, Call);
5252
5253 case Builtin::BI__builtin_launder:
5254 assert(isNoopBuiltin(BuiltinID));
5255 return true;
5256
5257 case Builtin::BI__builtin_add_overflow:
5258 case Builtin::BI__builtin_sub_overflow:
5259 case Builtin::BI__builtin_mul_overflow:
5260 case Builtin::BI__builtin_sadd_overflow:
5261 case Builtin::BI__builtin_uadd_overflow:
5262 case Builtin::BI__builtin_uaddl_overflow:
5263 case Builtin::BI__builtin_uaddll_overflow:
5264 case Builtin::BI__builtin_usub_overflow:
5265 case Builtin::BI__builtin_usubl_overflow:
5266 case Builtin::BI__builtin_usubll_overflow:
5267 case Builtin::BI__builtin_umul_overflow:
5268 case Builtin::BI__builtin_umull_overflow:
5269 case Builtin::BI__builtin_umulll_overflow:
5270 case Builtin::BI__builtin_saddl_overflow:
5271 case Builtin::BI__builtin_saddll_overflow:
5272 case Builtin::BI__builtin_ssub_overflow:
5273 case Builtin::BI__builtin_ssubl_overflow:
5274 case Builtin::BI__builtin_ssubll_overflow:
5275 case Builtin::BI__builtin_smul_overflow:
5276 case Builtin::BI__builtin_smull_overflow:
5277 case Builtin::BI__builtin_smulll_overflow:
5278 return interp__builtin_overflowop(S, OpPC, Call, BuiltinOp: BuiltinID);
5279
5280 case Builtin::BI__builtin_addcb:
5281 case Builtin::BI__builtin_addcs:
5282 case Builtin::BI__builtin_addc:
5283 case Builtin::BI__builtin_addcl:
5284 case Builtin::BI__builtin_addcll:
5285 case Builtin::BI__builtin_subcb:
5286 case Builtin::BI__builtin_subcs:
5287 case Builtin::BI__builtin_subc:
5288 case Builtin::BI__builtin_subcl:
5289 case Builtin::BI__builtin_subcll:
5290 return interp__builtin_carryop(S, OpPC, Frame, Call, BuiltinOp: BuiltinID);
5291
5292 case Builtin::BI__builtin_clz:
5293 case Builtin::BI__builtin_clzl:
5294 case Builtin::BI__builtin_clzll:
5295 case Builtin::BI__builtin_clzs:
5296 case Builtin::BI__builtin_clzg:
5297 case Builtin::BI__lzcnt16: // Microsoft variants of count leading-zeroes
5298 case Builtin::BI__lzcnt:
5299 case Builtin::BI__lzcnt64:
5300 return interp__builtin_clz(S, OpPC, Frame, Call, BuiltinOp: BuiltinID);
5301
5302 case Builtin::BI__builtin_ctz:
5303 case Builtin::BI__builtin_ctzl:
5304 case Builtin::BI__builtin_ctzll:
5305 case Builtin::BI__builtin_ctzs:
5306 case Builtin::BI__builtin_ctzg:
5307 return interp__builtin_ctz(S, OpPC, Frame, Call, BuiltinID);
5308
5309 case Builtin::BI__builtin_elementwise_clzg:
5310 case Builtin::BI__builtin_elementwise_ctzg:
5311 return interp__builtin_elementwise_countzeroes(S, OpPC, Frame, Call,
5312 BuiltinID);
5313 case Builtin::BI__builtin_bswapg:
5314 case Builtin::BI__builtin_bswap16:
5315 case Builtin::BI__builtin_bswap32:
5316 case Builtin::BI__builtin_bswap64:
5317 case Builtin::BIstdc_memreverse8u8:
5318 case Builtin::BIstdc_memreverse8u16:
5319 case Builtin::BIstdc_memreverse8u32:
5320 case Builtin::BIstdc_memreverse8u64:
5321 return interp__builtin_bswap(S, OpPC, Frame, Call);
5322
5323 case Builtin::BIstdc_memreverse8:
5324 case Builtin::BI__builtin_stdc_memreverse8:
5325 return interp__builtin_stdc_memreverse8(S, OpPC, Frame, Call);
5326
5327 case Builtin::BIstdc_load8_leu8:
5328 case Builtin::BIstdc_load8_leu16:
5329 case Builtin::BIstdc_load8_leu32:
5330 case Builtin::BIstdc_load8_leu64:
5331 case Builtin::BIstdc_load8_les8:
5332 case Builtin::BIstdc_load8_les16:
5333 case Builtin::BIstdc_load8_les32:
5334 case Builtin::BIstdc_load8_les64:
5335 return interp__builtin_load8(S, OpPC, Frame, Call, /*IsBigEndian=*/false,
5336 /*IsAligned=*/false);
5337
5338 case Builtin::BIstdc_load8_aligned_leu8:
5339 case Builtin::BIstdc_load8_aligned_leu16:
5340 case Builtin::BIstdc_load8_aligned_leu32:
5341 case Builtin::BIstdc_load8_aligned_leu64:
5342 case Builtin::BIstdc_load8_aligned_les8:
5343 case Builtin::BIstdc_load8_aligned_les16:
5344 case Builtin::BIstdc_load8_aligned_les32:
5345 case Builtin::BIstdc_load8_aligned_les64:
5346 return interp__builtin_load8(S, OpPC, Frame, Call, /*IsBigEndian=*/false,
5347 /*IsAligned=*/true);
5348
5349 case Builtin::BIstdc_load8_beu8:
5350 case Builtin::BIstdc_load8_beu16:
5351 case Builtin::BIstdc_load8_beu32:
5352 case Builtin::BIstdc_load8_beu64:
5353 case Builtin::BIstdc_load8_bes8:
5354 case Builtin::BIstdc_load8_bes16:
5355 case Builtin::BIstdc_load8_bes32:
5356 case Builtin::BIstdc_load8_bes64:
5357 return interp__builtin_load8(S, OpPC, Frame, Call, /*IsBigEndian=*/true,
5358 /*IsAligned=*/false);
5359
5360 case Builtin::BIstdc_load8_aligned_beu8:
5361 case Builtin::BIstdc_load8_aligned_beu16:
5362 case Builtin::BIstdc_load8_aligned_beu32:
5363 case Builtin::BIstdc_load8_aligned_beu64:
5364 case Builtin::BIstdc_load8_aligned_bes8:
5365 case Builtin::BIstdc_load8_aligned_bes16:
5366 case Builtin::BIstdc_load8_aligned_bes32:
5367 case Builtin::BIstdc_load8_aligned_bes64:
5368 return interp__builtin_load8(S, OpPC, Frame, Call, /*IsBigEndian=*/true,
5369 /*IsAligned=*/true);
5370
5371 case Builtin::BI__atomic_always_lock_free:
5372 case Builtin::BI__atomic_is_lock_free:
5373 return interp__builtin_atomic_lock_free(S, OpPC, Frame, Call, BuiltinOp: BuiltinID);
5374
5375 case Builtin::BI__c11_atomic_is_lock_free:
5376 return interp__builtin_c11_atomic_is_lock_free(S, OpPC, Frame, Call);
5377
5378 case Builtin::BI__builtin_complex:
5379 return interp__builtin_complex(S, OpPC, Frame, Call);
5380
5381 case Builtin::BI__builtin_is_aligned:
5382 case Builtin::BI__builtin_align_up:
5383 case Builtin::BI__builtin_align_down:
5384 return interp__builtin_is_aligned_up_down(S, OpPC, Frame, Call, BuiltinOp: BuiltinID);
5385
5386 case Builtin::BI__builtin_assume_aligned:
5387 return interp__builtin_assume_aligned(S, OpPC, Frame, Call);
5388
5389 case clang::X86::BI__builtin_ia32_crc32qi:
5390 return interp__builtin_ia32_crc32(S, OpPC, Frame, Call, DataBytes: 1);
5391 case clang::X86::BI__builtin_ia32_crc32hi:
5392 return interp__builtin_ia32_crc32(S, OpPC, Frame, Call, DataBytes: 2);
5393 case clang::X86::BI__builtin_ia32_crc32si:
5394 return interp__builtin_ia32_crc32(S, OpPC, Frame, Call, DataBytes: 4);
5395 case clang::X86::BI__builtin_ia32_crc32di:
5396 return interp__builtin_ia32_crc32(S, OpPC, Frame, Call, DataBytes: 8);
5397
5398 case clang::X86::BI__builtin_ia32_bextr_u32:
5399 case clang::X86::BI__builtin_ia32_bextr_u64:
5400 case clang::X86::BI__builtin_ia32_bextri_u32:
5401 case clang::X86::BI__builtin_ia32_bextri_u64:
5402 return interp__builtin_elementwise_int_binop(
5403 S, OpPC, Call, Fn: [](const APSInt &Val, const APSInt &Idx) {
5404 unsigned BitWidth = Val.getBitWidth();
5405 uint64_t Shift = Idx.extractBitsAsZExtValue(numBits: 8, bitPosition: 0);
5406 uint64_t Length = Idx.extractBitsAsZExtValue(numBits: 8, bitPosition: 8);
5407 if (Length > BitWidth) {
5408 Length = BitWidth;
5409 }
5410
5411 // Handle out of bounds cases.
5412 if (Length == 0 || Shift >= BitWidth)
5413 return APInt(BitWidth, 0);
5414
5415 uint64_t Result = Val.getZExtValue() >> Shift;
5416 Result &= llvm::maskTrailingOnes<uint64_t>(N: Length);
5417 return APInt(BitWidth, Result);
5418 });
5419
5420 case clang::X86::BI__builtin_ia32_bzhi_si:
5421 case clang::X86::BI__builtin_ia32_bzhi_di:
5422 return interp__builtin_elementwise_int_binop(
5423 S, OpPC, Call, Fn: [](const APSInt &Val, const APSInt &Idx) {
5424 unsigned BitWidth = Val.getBitWidth();
5425 uint64_t Index = Idx.extractBitsAsZExtValue(numBits: 8, bitPosition: 0);
5426 APSInt Result = Val;
5427
5428 if (Index < BitWidth)
5429 Result.clearHighBits(hiBits: BitWidth - Index);
5430
5431 return Result;
5432 });
5433
5434 case clang::X86::BI__builtin_ia32_ktestcqi:
5435 case clang::X86::BI__builtin_ia32_ktestchi:
5436 case clang::X86::BI__builtin_ia32_ktestcsi:
5437 case clang::X86::BI__builtin_ia32_ktestcdi:
5438 return interp__builtin_elementwise_int_binop(
5439 S, OpPC, Call, Fn: [](const APSInt &A, const APSInt &B) {
5440 return APInt(sizeof(unsigned char) * 8, (~A & B) == 0);
5441 });
5442
5443 case clang::X86::BI__builtin_ia32_ktestzqi:
5444 case clang::X86::BI__builtin_ia32_ktestzhi:
5445 case clang::X86::BI__builtin_ia32_ktestzsi:
5446 case clang::X86::BI__builtin_ia32_ktestzdi:
5447 return interp__builtin_elementwise_int_binop(
5448 S, OpPC, Call, Fn: [](const APSInt &A, const APSInt &B) {
5449 return APInt(sizeof(unsigned char) * 8, (A & B) == 0);
5450 });
5451
5452 case clang::X86::BI__builtin_ia32_kortestcqi:
5453 case clang::X86::BI__builtin_ia32_kortestchi:
5454 case clang::X86::BI__builtin_ia32_kortestcsi:
5455 case clang::X86::BI__builtin_ia32_kortestcdi:
5456 return interp__builtin_elementwise_int_binop(
5457 S, OpPC, Call, Fn: [](const APSInt &A, const APSInt &B) {
5458 return APInt(sizeof(unsigned char) * 8, ~(A | B) == 0);
5459 });
5460
5461 case clang::X86::BI__builtin_ia32_kortestzqi:
5462 case clang::X86::BI__builtin_ia32_kortestzhi:
5463 case clang::X86::BI__builtin_ia32_kortestzsi:
5464 case clang::X86::BI__builtin_ia32_kortestzdi:
5465 return interp__builtin_elementwise_int_binop(
5466 S, OpPC, Call, Fn: [](const APSInt &A, const APSInt &B) {
5467 return APInt(sizeof(unsigned char) * 8, (A | B) == 0);
5468 });
5469
5470 case clang::X86::BI__builtin_ia32_kshiftliqi:
5471 case clang::X86::BI__builtin_ia32_kshiftlihi:
5472 case clang::X86::BI__builtin_ia32_kshiftlisi:
5473 case clang::X86::BI__builtin_ia32_kshiftlidi:
5474 return interp__builtin_elementwise_int_binop(
5475 S, OpPC, Call, Fn: [](const APSInt &LHS, const APSInt &RHS) {
5476 unsigned Amt = RHS.getZExtValue() & 0xFF;
5477 if (Amt >= LHS.getBitWidth())
5478 return APInt::getZero(numBits: LHS.getBitWidth());
5479 return LHS.shl(shiftAmt: Amt);
5480 });
5481
5482 case clang::X86::BI__builtin_ia32_kshiftriqi:
5483 case clang::X86::BI__builtin_ia32_kshiftrihi:
5484 case clang::X86::BI__builtin_ia32_kshiftrisi:
5485 case clang::X86::BI__builtin_ia32_kshiftridi:
5486 return interp__builtin_elementwise_int_binop(
5487 S, OpPC, Call, Fn: [](const APSInt &LHS, const APSInt &RHS) {
5488 unsigned Amt = RHS.getZExtValue() & 0xFF;
5489 if (Amt >= LHS.getBitWidth())
5490 return APInt::getZero(numBits: LHS.getBitWidth());
5491 return LHS.lshr(shiftAmt: Amt);
5492 });
5493
5494 case clang::X86::BI__builtin_ia32_lzcnt_u16:
5495 case clang::X86::BI__builtin_ia32_lzcnt_u32:
5496 case clang::X86::BI__builtin_ia32_lzcnt_u64:
5497 return interp__builtin_elementwise_int_unaryop(
5498 S, OpPC, Call, Fn: [](const APSInt &Src) {
5499 return APInt(Src.getBitWidth(), Src.countLeadingZeros());
5500 });
5501
5502 case clang::X86::BI__builtin_ia32_tzcnt_u16:
5503 case clang::X86::BI__builtin_ia32_tzcnt_u32:
5504 case clang::X86::BI__builtin_ia32_tzcnt_u64:
5505 return interp__builtin_elementwise_int_unaryop(
5506 S, OpPC, Call, Fn: [](const APSInt &Src) {
5507 return APInt(Src.getBitWidth(), Src.countTrailingZeros());
5508 });
5509
5510 case clang::X86::BI__builtin_ia32_addcarryx_u32:
5511 case clang::X86::BI__builtin_ia32_addcarryx_u64:
5512 return interp__builtin_ia32_addcarry_subborrow(S, OpPC, Frame, Call,
5513 /*IsAdd=*/true);
5514
5515 case clang::X86::BI__builtin_ia32_subborrow_u32:
5516 case clang::X86::BI__builtin_ia32_subborrow_u64:
5517 return interp__builtin_ia32_addcarry_subborrow(S, OpPC, Frame, Call,
5518 /*IsAdd=*/false);
5519
5520 case Builtin::BI__builtin_os_log_format_buffer_size:
5521 return interp__builtin_os_log_format_buffer_size(S, OpPC, Frame, Call);
5522
5523 case Builtin::BI__builtin_ptrauth_string_discriminator:
5524 return interp__builtin_ptrauth_string_discriminator(S, OpPC, Frame, Call);
5525
5526 case Builtin::BI__builtin_infer_alloc_token:
5527 return interp__builtin_infer_alloc_token(S, OpPC, Frame, Call);
5528
5529 case Builtin::BI__noop:
5530 pushInteger(S, Val: 0, QT: Call->getType());
5531 return true;
5532
5533 case Builtin::BI__builtin_operator_new:
5534 return interp__builtin_operator_new(S, OpPC, Frame, Call);
5535
5536 case Builtin::BI__builtin_operator_delete:
5537 return interp__builtin_operator_delete(S, OpPC, Frame, Call);
5538
5539 case Builtin::BI__arithmetic_fence:
5540 return interp__builtin_arithmetic_fence(S, OpPC, Frame, Call);
5541
5542 case Builtin::BI__builtin_reduce_add:
5543 case Builtin::BI__builtin_reduce_mul:
5544 case Builtin::BI__builtin_reduce_and:
5545 case Builtin::BI__builtin_reduce_or:
5546 case Builtin::BI__builtin_reduce_xor:
5547 case Builtin::BI__builtin_reduce_min:
5548 case Builtin::BI__builtin_reduce_max:
5549 return interp__builtin_vector_reduce(S, OpPC, Call, ID: BuiltinID);
5550
5551 case Builtin::BI__builtin_elementwise_popcount:
5552 return interp__builtin_elementwise_int_unaryop(
5553 S, OpPC, Call, Fn: [](const APSInt &Src) {
5554 return APInt(Src.getBitWidth(), Src.popcount());
5555 });
5556 case Builtin::BI__builtin_elementwise_bitreverse:
5557 return interp__builtin_elementwise_int_unaryop(
5558 S, OpPC, Call, Fn: [](const APSInt &Src) { return Src.reverseBits(); });
5559
5560 case Builtin::BI__builtin_elementwise_abs:
5561 return interp__builtin_elementwise_abs(S, OpPC, Frame, Call, BuiltinID);
5562
5563 case Builtin::BI__builtin_memcpy:
5564 case Builtin::BImemcpy:
5565 case Builtin::BI__builtin_wmemcpy:
5566 case Builtin::BIwmemcpy:
5567 case Builtin::BI__builtin_memmove:
5568 case Builtin::BImemmove:
5569 case Builtin::BI__builtin_wmemmove:
5570 case Builtin::BIwmemmove:
5571 return interp__builtin_memcpy(S, OpPC, Frame, Call, ID: BuiltinID);
5572
5573 case Builtin::BI__builtin_memcmp:
5574 case Builtin::BImemcmp:
5575 case Builtin::BI__builtin_bcmp:
5576 case Builtin::BIbcmp:
5577 case Builtin::BI__builtin_wmemcmp:
5578 case Builtin::BIwmemcmp:
5579 return interp__builtin_memcmp(S, OpPC, Frame, Call, ID: BuiltinID);
5580
5581 case Builtin::BImemchr:
5582 case Builtin::BI__builtin_memchr:
5583 case Builtin::BIstrchr:
5584 case Builtin::BI__builtin_strchr:
5585 case Builtin::BIwmemchr:
5586 case Builtin::BI__builtin_wmemchr:
5587 case Builtin::BIwcschr:
5588 case Builtin::BI__builtin_wcschr:
5589 case Builtin::BI__builtin_char_memchr:
5590 return interp__builtin_memchr(S, OpPC, Call, ID: BuiltinID);
5591
5592 case Builtin::BI__builtin_object_size:
5593 return interp__builtin_object_size(S, OpPC, Frame, Call,
5594 /*IsDynamic=*/false);
5595 case Builtin::BI__builtin_dynamic_object_size:
5596 return interp__builtin_object_size(S, OpPC, Frame, Call,
5597 /*IsDynamic=*/true);
5598
5599 case Builtin::BI__builtin_is_within_lifetime:
5600 return interp__builtin_is_within_lifetime(S, OpPC, Call);
5601
5602 case Builtin::BI__builtin_elementwise_add_sat:
5603 return interp__builtin_elementwise_int_binop(
5604 S, OpPC, Call, Fn: [](const APSInt &LHS, const APSInt &RHS) {
5605 return LHS.isSigned() ? LHS.sadd_sat(RHS) : LHS.uadd_sat(RHS);
5606 });
5607
5608 case Builtin::BI__builtin_elementwise_sub_sat:
5609 return interp__builtin_elementwise_int_binop(
5610 S, OpPC, Call, Fn: [](const APSInt &LHS, const APSInt &RHS) {
5611 return LHS.isSigned() ? LHS.ssub_sat(RHS) : LHS.usub_sat(RHS);
5612 });
5613
5614 case Builtin::BI__builtin_elementwise_pdep:
5615 return interp__builtin_elementwise_int_binop(S, OpPC, Call,
5616 Fn: llvm::APIntOps::pdep);
5617
5618 case Builtin::BI__builtin_elementwise_pext:
5619 return interp__builtin_elementwise_int_binop(S, OpPC, Call,
5620 Fn: llvm::APIntOps::pext);
5621
5622 case X86::BI__builtin_ia32_extract128i256:
5623 case X86::BI__builtin_ia32_vextractf128_pd256:
5624 case X86::BI__builtin_ia32_vextractf128_ps256:
5625 case X86::BI__builtin_ia32_vextractf128_si256:
5626 return interp__builtin_ia32_extract_vector(S, OpPC, Call, ID: BuiltinID);
5627
5628 case X86::BI__builtin_ia32_extractf32x4_256_mask:
5629 case X86::BI__builtin_ia32_extractf32x4_mask:
5630 case X86::BI__builtin_ia32_extractf32x8_mask:
5631 case X86::BI__builtin_ia32_extractf64x2_256_mask:
5632 case X86::BI__builtin_ia32_extractf64x2_512_mask:
5633 case X86::BI__builtin_ia32_extractf64x4_mask:
5634 case X86::BI__builtin_ia32_extracti32x4_256_mask:
5635 case X86::BI__builtin_ia32_extracti32x4_mask:
5636 case X86::BI__builtin_ia32_extracti32x8_mask:
5637 case X86::BI__builtin_ia32_extracti64x2_256_mask:
5638 case X86::BI__builtin_ia32_extracti64x2_512_mask:
5639 case X86::BI__builtin_ia32_extracti64x4_mask:
5640 return interp__builtin_ia32_extract_vector_masked(S, OpPC, Call, ID: BuiltinID);
5641
5642 case clang::X86::BI__builtin_ia32_pmulhrsw128:
5643 case clang::X86::BI__builtin_ia32_pmulhrsw256:
5644 case clang::X86::BI__builtin_ia32_pmulhrsw512:
5645 return interp__builtin_elementwise_int_binop(
5646 S, OpPC, Call, Fn: [](const APSInt &LHS, const APSInt &RHS) {
5647 return (llvm::APIntOps::mulsExtended(C1: LHS, C2: RHS).ashr(ShiftAmt: 14) + 1)
5648 .extractBits(numBits: 16, bitPosition: 1);
5649 });
5650
5651 case clang::X86::BI__builtin_ia32_movmskps:
5652 case clang::X86::BI__builtin_ia32_movmskpd:
5653 case clang::X86::BI__builtin_ia32_pmovmskb128:
5654 case clang::X86::BI__builtin_ia32_pmovmskb256:
5655 case clang::X86::BI__builtin_ia32_movmskps256:
5656 case clang::X86::BI__builtin_ia32_movmskpd256: {
5657 return interp__builtin_ia32_movmsk_op(S, OpPC, Call);
5658 }
5659
5660 case X86::BI__builtin_ia32_psignb128:
5661 case X86::BI__builtin_ia32_psignb256:
5662 case X86::BI__builtin_ia32_psignw128:
5663 case X86::BI__builtin_ia32_psignw256:
5664 case X86::BI__builtin_ia32_psignd128:
5665 case X86::BI__builtin_ia32_psignd256:
5666 return interp__builtin_elementwise_int_binop(
5667 S, OpPC, Call, Fn: [](const APInt &AElem, const APInt &BElem) {
5668 if (BElem.isZero())
5669 return APInt::getZero(numBits: AElem.getBitWidth());
5670 if (BElem.isNegative())
5671 return -AElem;
5672 return AElem;
5673 });
5674
5675 case clang::X86::BI__builtin_ia32_pavgb128:
5676 case clang::X86::BI__builtin_ia32_pavgw128:
5677 case clang::X86::BI__builtin_ia32_pavgb256:
5678 case clang::X86::BI__builtin_ia32_pavgw256:
5679 case clang::X86::BI__builtin_ia32_pavgb512:
5680 case clang::X86::BI__builtin_ia32_pavgw512:
5681 return interp__builtin_elementwise_int_binop(S, OpPC, Call,
5682 Fn: llvm::APIntOps::avgCeilU);
5683
5684 case clang::X86::BI__builtin_ia32_pmaddubsw128:
5685 case clang::X86::BI__builtin_ia32_pmaddubsw256:
5686 case clang::X86::BI__builtin_ia32_pmaddubsw512:
5687 return interp__builtin_ia32_pmul(
5688 S, OpPC, Call,
5689 Fn: [](const APSInt &LoLHS, const APSInt &HiLHS, const APSInt &LoRHS,
5690 const APSInt &HiRHS) {
5691 unsigned BitWidth = 2 * LoLHS.getBitWidth();
5692 return (LoLHS.zext(width: BitWidth) * LoRHS.sext(width: BitWidth))
5693 .sadd_sat(RHS: (HiLHS.zext(width: BitWidth) * HiRHS.sext(width: BitWidth)));
5694 });
5695
5696 case clang::X86::BI__builtin_ia32_pmaddwd128:
5697 case clang::X86::BI__builtin_ia32_pmaddwd256:
5698 case clang::X86::BI__builtin_ia32_pmaddwd512:
5699 return interp__builtin_ia32_pmul(
5700 S, OpPC, Call,
5701 Fn: [](const APSInt &LoLHS, const APSInt &HiLHS, const APSInt &LoRHS,
5702 const APSInt &HiRHS) {
5703 unsigned BitWidth = 2 * LoLHS.getBitWidth();
5704 return (LoLHS.sext(width: BitWidth) * LoRHS.sext(width: BitWidth)) +
5705 (HiLHS.sext(width: BitWidth) * HiRHS.sext(width: BitWidth));
5706 });
5707
5708 case clang::X86::BI__builtin_ia32_psadbw128:
5709 case clang::X86::BI__builtin_ia32_psadbw256:
5710 case clang::X86::BI__builtin_ia32_psadbw512:
5711 return interp__builtin_ia32_psadbw(S, OpPC, Call);
5712
5713 case clang::X86::BI__builtin_ia32_dbpsadbw128:
5714 case clang::X86::BI__builtin_ia32_dbpsadbw256:
5715 case clang::X86::BI__builtin_ia32_dbpsadbw512:
5716 return interp__builtin_ia32_dbpsadbw(S, OpPC, Call);
5717
5718 case clang::X86::BI__builtin_ia32_mpsadbw128:
5719 case clang::X86::BI__builtin_ia32_mpsadbw256:
5720 return interp__builtin_ia32_mpsadbw(S, OpPC, Call);
5721
5722 case clang::X86::BI__builtin_ia32_pmulhuw128:
5723 case clang::X86::BI__builtin_ia32_pmulhuw256:
5724 case clang::X86::BI__builtin_ia32_pmulhuw512:
5725 return interp__builtin_elementwise_int_binop(S, OpPC, Call,
5726 Fn: llvm::APIntOps::mulhu);
5727
5728 case clang::X86::BI__builtin_ia32_pmulhw128:
5729 case clang::X86::BI__builtin_ia32_pmulhw256:
5730 case clang::X86::BI__builtin_ia32_pmulhw512:
5731 return interp__builtin_elementwise_int_binop(S, OpPC, Call,
5732 Fn: llvm::APIntOps::mulhs);
5733
5734 case clang::X86::BI__builtin_ia32_psllv2di:
5735 case clang::X86::BI__builtin_ia32_psllv4di:
5736 case clang::X86::BI__builtin_ia32_psllv4si:
5737 case clang::X86::BI__builtin_ia32_psllv8di:
5738 case clang::X86::BI__builtin_ia32_psllv8hi:
5739 case clang::X86::BI__builtin_ia32_psllv8si:
5740 case clang::X86::BI__builtin_ia32_psllv16hi:
5741 case clang::X86::BI__builtin_ia32_psllv16si:
5742 case clang::X86::BI__builtin_ia32_psllv32hi:
5743 case clang::X86::BI__builtin_ia32_psllwi128:
5744 case clang::X86::BI__builtin_ia32_psllwi256:
5745 case clang::X86::BI__builtin_ia32_psllwi512:
5746 case clang::X86::BI__builtin_ia32_pslldi128:
5747 case clang::X86::BI__builtin_ia32_pslldi256:
5748 case clang::X86::BI__builtin_ia32_pslldi512:
5749 case clang::X86::BI__builtin_ia32_psllqi128:
5750 case clang::X86::BI__builtin_ia32_psllqi256:
5751 case clang::X86::BI__builtin_ia32_psllqi512:
5752 return interp__builtin_elementwise_int_binop(
5753 S, OpPC, Call, Fn: [](const APSInt &LHS, const APSInt &RHS) {
5754 if (RHS.uge(RHS: LHS.getBitWidth())) {
5755 return APInt::getZero(numBits: LHS.getBitWidth());
5756 }
5757 return LHS.shl(shiftAmt: RHS.getZExtValue());
5758 });
5759
5760 case clang::X86::BI__builtin_ia32_psrav4si:
5761 case clang::X86::BI__builtin_ia32_psrav8di:
5762 case clang::X86::BI__builtin_ia32_psrav8hi:
5763 case clang::X86::BI__builtin_ia32_psrav8si:
5764 case clang::X86::BI__builtin_ia32_psrav16hi:
5765 case clang::X86::BI__builtin_ia32_psrav16si:
5766 case clang::X86::BI__builtin_ia32_psrav32hi:
5767 case clang::X86::BI__builtin_ia32_psravq128:
5768 case clang::X86::BI__builtin_ia32_psravq256:
5769 case clang::X86::BI__builtin_ia32_psrawi128:
5770 case clang::X86::BI__builtin_ia32_psrawi256:
5771 case clang::X86::BI__builtin_ia32_psrawi512:
5772 case clang::X86::BI__builtin_ia32_psradi128:
5773 case clang::X86::BI__builtin_ia32_psradi256:
5774 case clang::X86::BI__builtin_ia32_psradi512:
5775 case clang::X86::BI__builtin_ia32_psraqi128:
5776 case clang::X86::BI__builtin_ia32_psraqi256:
5777 case clang::X86::BI__builtin_ia32_psraqi512:
5778 return interp__builtin_elementwise_int_binop(
5779 S, OpPC, Call, Fn: [](const APSInt &LHS, const APSInt &RHS) {
5780 if (RHS.uge(RHS: LHS.getBitWidth())) {
5781 return LHS.ashr(ShiftAmt: LHS.getBitWidth() - 1);
5782 }
5783 return LHS.ashr(ShiftAmt: RHS.getZExtValue());
5784 });
5785
5786 case clang::X86::BI__builtin_ia32_psrlv2di:
5787 case clang::X86::BI__builtin_ia32_psrlv4di:
5788 case clang::X86::BI__builtin_ia32_psrlv4si:
5789 case clang::X86::BI__builtin_ia32_psrlv8di:
5790 case clang::X86::BI__builtin_ia32_psrlv8hi:
5791 case clang::X86::BI__builtin_ia32_psrlv8si:
5792 case clang::X86::BI__builtin_ia32_psrlv16hi:
5793 case clang::X86::BI__builtin_ia32_psrlv16si:
5794 case clang::X86::BI__builtin_ia32_psrlv32hi:
5795 case clang::X86::BI__builtin_ia32_psrlwi128:
5796 case clang::X86::BI__builtin_ia32_psrlwi256:
5797 case clang::X86::BI__builtin_ia32_psrlwi512:
5798 case clang::X86::BI__builtin_ia32_psrldi128:
5799 case clang::X86::BI__builtin_ia32_psrldi256:
5800 case clang::X86::BI__builtin_ia32_psrldi512:
5801 case clang::X86::BI__builtin_ia32_psrlqi128:
5802 case clang::X86::BI__builtin_ia32_psrlqi256:
5803 case clang::X86::BI__builtin_ia32_psrlqi512:
5804 return interp__builtin_elementwise_int_binop(
5805 S, OpPC, Call, Fn: [](const APSInt &LHS, const APSInt &RHS) {
5806 if (RHS.uge(RHS: LHS.getBitWidth())) {
5807 return APInt::getZero(numBits: LHS.getBitWidth());
5808 }
5809 return LHS.lshr(shiftAmt: RHS.getZExtValue());
5810 });
5811 case clang::X86::BI__builtin_ia32_packsswb128:
5812 case clang::X86::BI__builtin_ia32_packsswb256:
5813 case clang::X86::BI__builtin_ia32_packsswb512:
5814 case clang::X86::BI__builtin_ia32_packssdw128:
5815 case clang::X86::BI__builtin_ia32_packssdw256:
5816 case clang::X86::BI__builtin_ia32_packssdw512:
5817 return interp__builtin_ia32_pack(S, OpPC, E: Call, PackFn: [](const APSInt &Src) {
5818 return APInt(Src).truncSSat(width: Src.getBitWidth() / 2);
5819 });
5820 case clang::X86::BI__builtin_ia32_packusdw128:
5821 case clang::X86::BI__builtin_ia32_packusdw256:
5822 case clang::X86::BI__builtin_ia32_packusdw512:
5823 case clang::X86::BI__builtin_ia32_packuswb128:
5824 case clang::X86::BI__builtin_ia32_packuswb256:
5825 case clang::X86::BI__builtin_ia32_packuswb512:
5826 return interp__builtin_ia32_pack(S, OpPC, E: Call, PackFn: [](const APSInt &Src) {
5827 return APInt(Src).truncSSatU(width: Src.getBitWidth() / 2);
5828 });
5829
5830 case clang::X86::BI__builtin_ia32_selectss_128:
5831 case clang::X86::BI__builtin_ia32_selectsd_128:
5832 case clang::X86::BI__builtin_ia32_selectsh_128:
5833 case clang::X86::BI__builtin_ia32_selectsbf_128:
5834 return interp__builtin_ia32_select_scalar(S, Call);
5835 case clang::X86::BI__builtin_ia32_vprotbi:
5836 case clang::X86::BI__builtin_ia32_vprotdi:
5837 case clang::X86::BI__builtin_ia32_vprotqi:
5838 case clang::X86::BI__builtin_ia32_vprotwi:
5839 case clang::X86::BI__builtin_ia32_prold128:
5840 case clang::X86::BI__builtin_ia32_prold256:
5841 case clang::X86::BI__builtin_ia32_prold512:
5842 case clang::X86::BI__builtin_ia32_prolq128:
5843 case clang::X86::BI__builtin_ia32_prolq256:
5844 case clang::X86::BI__builtin_ia32_prolq512:
5845 return interp__builtin_elementwise_int_binop(
5846 S, OpPC, Call,
5847 Fn: [](const APSInt &LHS, const APSInt &RHS) { return LHS.rotl(rotateAmt: RHS); });
5848
5849 case clang::X86::BI__builtin_ia32_prord128:
5850 case clang::X86::BI__builtin_ia32_prord256:
5851 case clang::X86::BI__builtin_ia32_prord512:
5852 case clang::X86::BI__builtin_ia32_prorq128:
5853 case clang::X86::BI__builtin_ia32_prorq256:
5854 case clang::X86::BI__builtin_ia32_prorq512:
5855 return interp__builtin_elementwise_int_binop(
5856 S, OpPC, Call,
5857 Fn: [](const APSInt &LHS, const APSInt &RHS) { return LHS.rotr(rotateAmt: RHS); });
5858
5859 case Builtin::BI__builtin_elementwise_max:
5860 case Builtin::BI__builtin_elementwise_min:
5861 return interp__builtin_elementwise_maxmin(S, OpPC, Call, BuiltinID);
5862
5863 case clang::X86::BI__builtin_ia32_phaddw128:
5864 case clang::X86::BI__builtin_ia32_phaddw256:
5865 case clang::X86::BI__builtin_ia32_phaddd128:
5866 case clang::X86::BI__builtin_ia32_phaddd256:
5867 return interp_builtin_horizontal_int_binop(
5868 S, OpPC, Call,
5869 Fn: [](const APSInt &LHS, const APSInt &RHS) { return LHS + RHS; });
5870 case clang::X86::BI__builtin_ia32_phaddsw128:
5871 case clang::X86::BI__builtin_ia32_phaddsw256:
5872 return interp_builtin_horizontal_int_binop(
5873 S, OpPC, Call,
5874 Fn: [](const APSInt &LHS, const APSInt &RHS) { return LHS.sadd_sat(RHS); });
5875 case clang::X86::BI__builtin_ia32_phsubw128:
5876 case clang::X86::BI__builtin_ia32_phsubw256:
5877 case clang::X86::BI__builtin_ia32_phsubd128:
5878 case clang::X86::BI__builtin_ia32_phsubd256:
5879 return interp_builtin_horizontal_int_binop(
5880 S, OpPC, Call,
5881 Fn: [](const APSInt &LHS, const APSInt &RHS) { return LHS - RHS; });
5882 case clang::X86::BI__builtin_ia32_phsubsw128:
5883 case clang::X86::BI__builtin_ia32_phsubsw256:
5884 return interp_builtin_horizontal_int_binop(
5885 S, OpPC, Call,
5886 Fn: [](const APSInt &LHS, const APSInt &RHS) { return LHS.ssub_sat(RHS); });
5887 case clang::X86::BI__builtin_ia32_haddpd:
5888 case clang::X86::BI__builtin_ia32_haddps:
5889 case clang::X86::BI__builtin_ia32_haddpd256:
5890 case clang::X86::BI__builtin_ia32_haddps256:
5891 return interp_builtin_horizontal_fp_binop(
5892 S, OpPC, Call,
5893 Fn: [](const APFloat &LHS, const APFloat &RHS, llvm::RoundingMode RM) {
5894 APFloat F = LHS;
5895 F.add(RHS, RM);
5896 return F;
5897 });
5898 case clang::X86::BI__builtin_ia32_hsubpd:
5899 case clang::X86::BI__builtin_ia32_hsubps:
5900 case clang::X86::BI__builtin_ia32_hsubpd256:
5901 case clang::X86::BI__builtin_ia32_hsubps256:
5902 return interp_builtin_horizontal_fp_binop(
5903 S, OpPC, Call,
5904 Fn: [](const APFloat &LHS, const APFloat &RHS, llvm::RoundingMode RM) {
5905 APFloat F = LHS;
5906 F.subtract(RHS, RM);
5907 return F;
5908 });
5909 case clang::X86::BI__builtin_ia32_addsubpd:
5910 case clang::X86::BI__builtin_ia32_addsubps:
5911 case clang::X86::BI__builtin_ia32_addsubpd256:
5912 case clang::X86::BI__builtin_ia32_addsubps256:
5913 return interp__builtin_ia32_addsub(S, OpPC, Call);
5914
5915 case clang::X86::BI__builtin_ia32_pmuldq128:
5916 case clang::X86::BI__builtin_ia32_pmuldq256:
5917 case clang::X86::BI__builtin_ia32_pmuldq512:
5918 return interp__builtin_ia32_pmul(
5919 S, OpPC, Call,
5920 Fn: [](const APSInt &LoLHS, const APSInt &HiLHS, const APSInt &LoRHS,
5921 const APSInt &HiRHS) {
5922 return llvm::APIntOps::mulsExtended(C1: LoLHS, C2: LoRHS);
5923 });
5924
5925 case clang::X86::BI__builtin_ia32_pmuludq128:
5926 case clang::X86::BI__builtin_ia32_pmuludq256:
5927 case clang::X86::BI__builtin_ia32_pmuludq512:
5928 return interp__builtin_ia32_pmul(
5929 S, OpPC, Call,
5930 Fn: [](const APSInt &LoLHS, const APSInt &HiLHS, const APSInt &LoRHS,
5931 const APSInt &HiRHS) {
5932 return llvm::APIntOps::muluExtended(C1: LoLHS, C2: LoRHS);
5933 });
5934
5935 case clang::X86::BI__builtin_ia32_pclmulqdq128:
5936 case clang::X86::BI__builtin_ia32_pclmulqdq256:
5937 case clang::X86::BI__builtin_ia32_pclmulqdq512:
5938 return interp__builtin_ia32_pclmulqdq(S, OpPC, Call);
5939 case Builtin::BI__builtin_elementwise_clmul:
5940 return interp__builtin_elementwise_int_binop(S, OpPC, Call,
5941 Fn: llvm::APIntOps::clmul);
5942
5943 case Builtin::BI__builtin_elementwise_fma:
5944 return interp__builtin_elementwise_triop_fp(
5945 S, OpPC, Call,
5946 Fn: [](const APFloat &X, const APFloat &Y, const APFloat &Z,
5947 llvm::RoundingMode RM) {
5948 APFloat F = X;
5949 F.fusedMultiplyAdd(Multiplicand: Y, Addend: Z, RM);
5950 return F;
5951 });
5952
5953 case X86::BI__builtin_ia32_vpmadd52luq128:
5954 case X86::BI__builtin_ia32_vpmadd52luq256:
5955 case X86::BI__builtin_ia32_vpmadd52luq512:
5956 return interp__builtin_elementwise_triop(
5957 S, OpPC, Call, Fn: [](const APSInt &A, const APSInt &B, const APSInt &C) {
5958 return A + (B.trunc(width: 52) * C.trunc(width: 52)).zext(width: 64);
5959 });
5960 case X86::BI__builtin_ia32_vpmadd52huq128:
5961 case X86::BI__builtin_ia32_vpmadd52huq256:
5962 case X86::BI__builtin_ia32_vpmadd52huq512:
5963 return interp__builtin_elementwise_triop(
5964 S, OpPC, Call, Fn: [](const APSInt &A, const APSInt &B, const APSInt &C) {
5965 return A + llvm::APIntOps::mulhu(C1: B.trunc(width: 52), C2: C.trunc(width: 52)).zext(width: 64);
5966 });
5967
5968 case X86::BI__builtin_ia32_vpshldd128:
5969 case X86::BI__builtin_ia32_vpshldd256:
5970 case X86::BI__builtin_ia32_vpshldd512:
5971 case X86::BI__builtin_ia32_vpshldq128:
5972 case X86::BI__builtin_ia32_vpshldq256:
5973 case X86::BI__builtin_ia32_vpshldq512:
5974 case X86::BI__builtin_ia32_vpshldw128:
5975 case X86::BI__builtin_ia32_vpshldw256:
5976 case X86::BI__builtin_ia32_vpshldw512:
5977 return interp__builtin_elementwise_triop(
5978 S, OpPC, Call,
5979 Fn: [](const APSInt &Hi, const APSInt &Lo, const APSInt &Amt) {
5980 return llvm::APIntOps::fshl(Hi, Lo, Shift: Amt);
5981 });
5982
5983 case X86::BI__builtin_ia32_vpshrdd128:
5984 case X86::BI__builtin_ia32_vpshrdd256:
5985 case X86::BI__builtin_ia32_vpshrdd512:
5986 case X86::BI__builtin_ia32_vpshrdq128:
5987 case X86::BI__builtin_ia32_vpshrdq256:
5988 case X86::BI__builtin_ia32_vpshrdq512:
5989 case X86::BI__builtin_ia32_vpshrdw128:
5990 case X86::BI__builtin_ia32_vpshrdw256:
5991 case X86::BI__builtin_ia32_vpshrdw512:
5992 // NOTE: Reversed Hi/Lo operands.
5993 return interp__builtin_elementwise_triop(
5994 S, OpPC, Call,
5995 Fn: [](const APSInt &Lo, const APSInt &Hi, const APSInt &Amt) {
5996 return llvm::APIntOps::fshr(Hi, Lo, Shift: Amt);
5997 });
5998 case X86::BI__builtin_ia32_vpconflictsi_128:
5999 case X86::BI__builtin_ia32_vpconflictsi_256:
6000 case X86::BI__builtin_ia32_vpconflictsi_512:
6001 case X86::BI__builtin_ia32_vpconflictdi_128:
6002 case X86::BI__builtin_ia32_vpconflictdi_256:
6003 case X86::BI__builtin_ia32_vpconflictdi_512:
6004 return interp__builtin_ia32_vpconflict(S, OpPC, Call);
6005 case X86::BI__builtin_ia32_compressdf128_mask:
6006 case X86::BI__builtin_ia32_compressdf256_mask:
6007 case X86::BI__builtin_ia32_compressdf512_mask:
6008 case X86::BI__builtin_ia32_compressdi128_mask:
6009 case X86::BI__builtin_ia32_compressdi256_mask:
6010 case X86::BI__builtin_ia32_compressdi512_mask:
6011 case X86::BI__builtin_ia32_compresshi128_mask:
6012 case X86::BI__builtin_ia32_compresshi256_mask:
6013 case X86::BI__builtin_ia32_compresshi512_mask:
6014 case X86::BI__builtin_ia32_compressqi128_mask:
6015 case X86::BI__builtin_ia32_compressqi256_mask:
6016 case X86::BI__builtin_ia32_compressqi512_mask:
6017 case X86::BI__builtin_ia32_compresssf128_mask:
6018 case X86::BI__builtin_ia32_compresssf256_mask:
6019 case X86::BI__builtin_ia32_compresssf512_mask:
6020 case X86::BI__builtin_ia32_compresssi128_mask:
6021 case X86::BI__builtin_ia32_compresssi256_mask:
6022 case X86::BI__builtin_ia32_compresssi512_mask: {
6023 unsigned NumElems =
6024 Call->getArg(Arg: 0)->getType()->castAs<VectorType>()->getNumElements();
6025 return interp__builtin_ia32_shuffle_generic(
6026 S, OpPC, Call, GetSourceIndex: [NumElems](unsigned DstIdx, const APInt &ShuffleMask) {
6027 APInt CompressMask = ShuffleMask.trunc(width: NumElems);
6028 if (DstIdx < CompressMask.popcount()) {
6029 while (DstIdx != 0) {
6030 CompressMask = CompressMask & (CompressMask - 1);
6031 DstIdx--;
6032 }
6033 return std::pair<unsigned, int>{
6034 0, static_cast<int>(CompressMask.countr_zero())};
6035 }
6036 return std::pair<unsigned, int>{1, static_cast<int>(DstIdx)};
6037 });
6038 }
6039 case X86::BI__builtin_ia32_expanddf128_mask:
6040 case X86::BI__builtin_ia32_expanddf256_mask:
6041 case X86::BI__builtin_ia32_expanddf512_mask:
6042 case X86::BI__builtin_ia32_expanddi128_mask:
6043 case X86::BI__builtin_ia32_expanddi256_mask:
6044 case X86::BI__builtin_ia32_expanddi512_mask:
6045 case X86::BI__builtin_ia32_expandhi128_mask:
6046 case X86::BI__builtin_ia32_expandhi256_mask:
6047 case X86::BI__builtin_ia32_expandhi512_mask:
6048 case X86::BI__builtin_ia32_expandqi128_mask:
6049 case X86::BI__builtin_ia32_expandqi256_mask:
6050 case X86::BI__builtin_ia32_expandqi512_mask:
6051 case X86::BI__builtin_ia32_expandsf128_mask:
6052 case X86::BI__builtin_ia32_expandsf256_mask:
6053 case X86::BI__builtin_ia32_expandsf512_mask:
6054 case X86::BI__builtin_ia32_expandsi128_mask:
6055 case X86::BI__builtin_ia32_expandsi256_mask:
6056 case X86::BI__builtin_ia32_expandsi512_mask: {
6057 return interp__builtin_ia32_shuffle_generic(
6058 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, const APInt &ShuffleMask) {
6059 // Trunc to the sub-mask for the dst index and count the number of
6060 // src elements used prior to that.
6061 APInt ExpandMask = ShuffleMask.trunc(width: DstIdx + 1);
6062 if (ExpandMask[DstIdx]) {
6063 int SrcIdx = ExpandMask.popcount() - 1;
6064 return std::pair<unsigned, int>{0, SrcIdx};
6065 }
6066 return std::pair<unsigned, int>{1, static_cast<int>(DstIdx)};
6067 });
6068 }
6069 case clang::X86::BI__builtin_ia32_blendpd:
6070 case clang::X86::BI__builtin_ia32_blendpd256:
6071 case clang::X86::BI__builtin_ia32_blendps:
6072 case clang::X86::BI__builtin_ia32_blendps256:
6073 case clang::X86::BI__builtin_ia32_pblendw128:
6074 case clang::X86::BI__builtin_ia32_pblendw256:
6075 case clang::X86::BI__builtin_ia32_pblendd128:
6076 case clang::X86::BI__builtin_ia32_pblendd256:
6077 return interp__builtin_ia32_shuffle_generic(
6078 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6079 // Bit index for mask.
6080 unsigned MaskBit = (ShuffleMask >> (DstIdx % 8)) & 0x1;
6081 unsigned SrcVecIdx = MaskBit ? 1 : 0; // 1 = TrueVec, 0 = FalseVec
6082 return std::pair<unsigned, int>{SrcVecIdx, static_cast<int>(DstIdx)};
6083 });
6084
6085
6086
6087 case clang::X86::BI__builtin_ia32_blendvpd:
6088 case clang::X86::BI__builtin_ia32_blendvpd256:
6089 case clang::X86::BI__builtin_ia32_blendvps:
6090 case clang::X86::BI__builtin_ia32_blendvps256:
6091 return interp__builtin_elementwise_triop_fp(
6092 S, OpPC, Call,
6093 Fn: [](const APFloat &F, const APFloat &T, const APFloat &C,
6094 llvm::RoundingMode) { return C.isNegative() ? T : F; });
6095
6096 case clang::X86::BI__builtin_ia32_pblendvb128:
6097 case clang::X86::BI__builtin_ia32_pblendvb256:
6098 return interp__builtin_elementwise_triop(
6099 S, OpPC, Call, Fn: [](const APSInt &F, const APSInt &T, const APSInt &C) {
6100 return ((APInt)C).isNegative() ? T : F;
6101 });
6102 case X86::BI__builtin_ia32_ptestz128:
6103 case X86::BI__builtin_ia32_ptestz256:
6104 case X86::BI__builtin_ia32_vtestzps:
6105 case X86::BI__builtin_ia32_vtestzps256:
6106 case X86::BI__builtin_ia32_vtestzpd:
6107 case X86::BI__builtin_ia32_vtestzpd256:
6108 return interp__builtin_ia32_test_op(
6109 S, OpPC, Call,
6110 Fn: [](const APInt &A, const APInt &B) { return (A & B) == 0; });
6111 case X86::BI__builtin_ia32_ptestc128:
6112 case X86::BI__builtin_ia32_ptestc256:
6113 case X86::BI__builtin_ia32_vtestcps:
6114 case X86::BI__builtin_ia32_vtestcps256:
6115 case X86::BI__builtin_ia32_vtestcpd:
6116 case X86::BI__builtin_ia32_vtestcpd256:
6117 return interp__builtin_ia32_test_op(
6118 S, OpPC, Call,
6119 Fn: [](const APInt &A, const APInt &B) { return (~A & B) == 0; });
6120 case X86::BI__builtin_ia32_ptestnzc128:
6121 case X86::BI__builtin_ia32_ptestnzc256:
6122 case X86::BI__builtin_ia32_vtestnzcps:
6123 case X86::BI__builtin_ia32_vtestnzcps256:
6124 case X86::BI__builtin_ia32_vtestnzcpd:
6125 case X86::BI__builtin_ia32_vtestnzcpd256:
6126 return interp__builtin_ia32_test_op(
6127 S, OpPC, Call, Fn: [](const APInt &A, const APInt &B) {
6128 return ((A & B) != 0) && ((~A & B) != 0);
6129 });
6130 case X86::BI__builtin_ia32_selectb_128:
6131 case X86::BI__builtin_ia32_selectb_256:
6132 case X86::BI__builtin_ia32_selectb_512:
6133 case X86::BI__builtin_ia32_selectw_128:
6134 case X86::BI__builtin_ia32_selectw_256:
6135 case X86::BI__builtin_ia32_selectw_512:
6136 case X86::BI__builtin_ia32_selectd_128:
6137 case X86::BI__builtin_ia32_selectd_256:
6138 case X86::BI__builtin_ia32_selectd_512:
6139 case X86::BI__builtin_ia32_selectq_128:
6140 case X86::BI__builtin_ia32_selectq_256:
6141 case X86::BI__builtin_ia32_selectq_512:
6142 case X86::BI__builtin_ia32_selectph_128:
6143 case X86::BI__builtin_ia32_selectph_256:
6144 case X86::BI__builtin_ia32_selectph_512:
6145 case X86::BI__builtin_ia32_selectpbf_128:
6146 case X86::BI__builtin_ia32_selectpbf_256:
6147 case X86::BI__builtin_ia32_selectpbf_512:
6148 case X86::BI__builtin_ia32_selectps_128:
6149 case X86::BI__builtin_ia32_selectps_256:
6150 case X86::BI__builtin_ia32_selectps_512:
6151 case X86::BI__builtin_ia32_selectpd_128:
6152 case X86::BI__builtin_ia32_selectpd_256:
6153 case X86::BI__builtin_ia32_selectpd_512:
6154 return interp__builtin_ia32_select(S, OpPC, Call);
6155
6156 case X86::BI__builtin_ia32_shufps:
6157 case X86::BI__builtin_ia32_shufps256:
6158 case X86::BI__builtin_ia32_shufps512:
6159 return interp__builtin_ia32_shuffle_generic(
6160 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6161 unsigned NumElemPerLane = 4;
6162 unsigned NumSelectableElems = NumElemPerLane / 2;
6163 unsigned BitsPerElem = 2;
6164 unsigned IndexMask = 0x3;
6165 unsigned MaskBits = 8;
6166 unsigned Lane = DstIdx / NumElemPerLane;
6167 unsigned ElemInLane = DstIdx % NumElemPerLane;
6168 unsigned LaneOffset = Lane * NumElemPerLane;
6169 unsigned SrcIdx = ElemInLane >= NumSelectableElems ? 1 : 0;
6170 unsigned BitIndex = (DstIdx * BitsPerElem) % MaskBits;
6171 unsigned Index = (ShuffleMask >> BitIndex) & IndexMask;
6172 return std::pair<unsigned, int>{SrcIdx,
6173 static_cast<int>(LaneOffset + Index)};
6174 });
6175 case X86::BI__builtin_ia32_shufpd:
6176 case X86::BI__builtin_ia32_shufpd256:
6177 case X86::BI__builtin_ia32_shufpd512:
6178 return interp__builtin_ia32_shuffle_generic(
6179 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6180 unsigned NumElemPerLane = 2;
6181 unsigned NumSelectableElems = NumElemPerLane / 2;
6182 unsigned BitsPerElem = 1;
6183 unsigned IndexMask = 0x1;
6184 unsigned MaskBits = 8;
6185 unsigned Lane = DstIdx / NumElemPerLane;
6186 unsigned ElemInLane = DstIdx % NumElemPerLane;
6187 unsigned LaneOffset = Lane * NumElemPerLane;
6188 unsigned SrcIdx = ElemInLane >= NumSelectableElems ? 1 : 0;
6189 unsigned BitIndex = (DstIdx * BitsPerElem) % MaskBits;
6190 unsigned Index = (ShuffleMask >> BitIndex) & IndexMask;
6191 return std::pair<unsigned, int>{SrcIdx,
6192 static_cast<int>(LaneOffset + Index)};
6193 });
6194
6195 case X86::BI__builtin_ia32_vgf2p8affineinvqb_v16qi:
6196 case X86::BI__builtin_ia32_vgf2p8affineinvqb_v32qi:
6197 case X86::BI__builtin_ia32_vgf2p8affineinvqb_v64qi:
6198 return interp__builtin_ia32_gfni_affine(S, OpPC, Call, Inverse: true);
6199 case X86::BI__builtin_ia32_vgf2p8affineqb_v16qi:
6200 case X86::BI__builtin_ia32_vgf2p8affineqb_v32qi:
6201 case X86::BI__builtin_ia32_vgf2p8affineqb_v64qi:
6202 return interp__builtin_ia32_gfni_affine(S, OpPC, Call, Inverse: false);
6203
6204 case X86::BI__builtin_ia32_vgf2p8mulb_v16qi:
6205 case X86::BI__builtin_ia32_vgf2p8mulb_v32qi:
6206 case X86::BI__builtin_ia32_vgf2p8mulb_v64qi:
6207 return interp__builtin_ia32_gfni_mul(S, OpPC, Call);
6208
6209 case X86::BI__builtin_ia32_bmacor16x16x16_v16hi:
6210 case X86::BI__builtin_ia32_bmacor16x16x16_v32hi:
6211 return interp__builtin_ia32_bmac(S, OpPC, Call, /*IsXor=*/false);
6212 case X86::BI__builtin_ia32_bmacxor16x16x16_v16hi:
6213 case X86::BI__builtin_ia32_bmacxor16x16x16_v32hi:
6214 return interp__builtin_ia32_bmac(S, OpPC, Call, /*IsXor=*/true);
6215
6216 case X86::BI__builtin_ia32_insertps128:
6217 return interp__builtin_ia32_shuffle_generic(
6218 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned Mask) {
6219 // Bits [3:0]: zero mask - if bit is set, zero this element
6220 if ((Mask & (1 << DstIdx)) != 0) {
6221 return std::pair<unsigned, int>{0, -1};
6222 }
6223 // Bits [7:6]: select element from source vector Y (0-3)
6224 // Bits [5:4]: select destination position (0-3)
6225 unsigned SrcElem = (Mask >> 6) & 0x3;
6226 unsigned DstElem = (Mask >> 4) & 0x3;
6227 if (DstIdx == DstElem) {
6228 // Insert element from source vector (B) at this position
6229 return std::pair<unsigned, int>{1, static_cast<int>(SrcElem)};
6230 } else {
6231 // Copy from destination vector (A)
6232 return std::pair<unsigned, int>{0, static_cast<int>(DstIdx)};
6233 }
6234 });
6235 case X86::BI__builtin_ia32_permvarsi256:
6236 case X86::BI__builtin_ia32_permvarsf256:
6237 case X86::BI__builtin_ia32_permvardf512:
6238 case X86::BI__builtin_ia32_permvardi512:
6239 case X86::BI__builtin_ia32_permvarhi128:
6240 return interp__builtin_ia32_shuffle_generic(
6241 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6242 int Offset = ShuffleMask & 0x7;
6243 return std::pair<unsigned, int>{0, Offset};
6244 });
6245 case X86::BI__builtin_ia32_permvarqi128:
6246 case X86::BI__builtin_ia32_permvarhi256:
6247 case X86::BI__builtin_ia32_permvarsi512:
6248 case X86::BI__builtin_ia32_permvarsf512:
6249 return interp__builtin_ia32_shuffle_generic(
6250 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6251 int Offset = ShuffleMask & 0xF;
6252 return std::pair<unsigned, int>{0, Offset};
6253 });
6254 case X86::BI__builtin_ia32_permvardi256:
6255 case X86::BI__builtin_ia32_permvardf256:
6256 return interp__builtin_ia32_shuffle_generic(
6257 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6258 int Offset = ShuffleMask & 0x3;
6259 return std::pair<unsigned, int>{0, Offset};
6260 });
6261 case X86::BI__builtin_ia32_permvarqi256:
6262 case X86::BI__builtin_ia32_permvarhi512:
6263 return interp__builtin_ia32_shuffle_generic(
6264 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6265 int Offset = ShuffleMask & 0x1F;
6266 return std::pair<unsigned, int>{0, Offset};
6267 });
6268 case X86::BI__builtin_ia32_permvarqi512:
6269 return interp__builtin_ia32_shuffle_generic(
6270 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6271 int Offset = ShuffleMask & 0x3F;
6272 return std::pair<unsigned, int>{0, Offset};
6273 });
6274 case X86::BI__builtin_ia32_vpermi2varq128:
6275 case X86::BI__builtin_ia32_vpermi2varpd128:
6276 return interp__builtin_ia32_shuffle_generic(
6277 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6278 int Offset = ShuffleMask & 0x1;
6279 unsigned SrcIdx = (ShuffleMask >> 1) & 0x1;
6280 return std::pair<unsigned, int>{SrcIdx, Offset};
6281 });
6282 case X86::BI__builtin_ia32_vpermi2vard128:
6283 case X86::BI__builtin_ia32_vpermi2varps128:
6284 case X86::BI__builtin_ia32_vpermi2varq256:
6285 case X86::BI__builtin_ia32_vpermi2varpd256:
6286 return interp__builtin_ia32_shuffle_generic(
6287 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6288 int Offset = ShuffleMask & 0x3;
6289 unsigned SrcIdx = (ShuffleMask >> 2) & 0x1;
6290 return std::pair<unsigned, int>{SrcIdx, Offset};
6291 });
6292 case X86::BI__builtin_ia32_vpermi2varhi128:
6293 case X86::BI__builtin_ia32_vpermi2vard256:
6294 case X86::BI__builtin_ia32_vpermi2varps256:
6295 case X86::BI__builtin_ia32_vpermi2varq512:
6296 case X86::BI__builtin_ia32_vpermi2varpd512:
6297 return interp__builtin_ia32_shuffle_generic(
6298 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6299 int Offset = ShuffleMask & 0x7;
6300 unsigned SrcIdx = (ShuffleMask >> 3) & 0x1;
6301 return std::pair<unsigned, int>{SrcIdx, Offset};
6302 });
6303 case X86::BI__builtin_ia32_vpermi2varqi128:
6304 case X86::BI__builtin_ia32_vpermi2varhi256:
6305 case X86::BI__builtin_ia32_vpermi2vard512:
6306 case X86::BI__builtin_ia32_vpermi2varps512:
6307 return interp__builtin_ia32_shuffle_generic(
6308 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6309 int Offset = ShuffleMask & 0xF;
6310 unsigned SrcIdx = (ShuffleMask >> 4) & 0x1;
6311 return std::pair<unsigned, int>{SrcIdx, Offset};
6312 });
6313 case X86::BI__builtin_ia32_vpermi2varqi256:
6314 case X86::BI__builtin_ia32_vpermi2varhi512:
6315 return interp__builtin_ia32_shuffle_generic(
6316 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6317 int Offset = ShuffleMask & 0x1F;
6318 unsigned SrcIdx = (ShuffleMask >> 5) & 0x1;
6319 return std::pair<unsigned, int>{SrcIdx, Offset};
6320 });
6321 case X86::BI__builtin_ia32_vpermi2varqi512:
6322 return interp__builtin_ia32_shuffle_generic(
6323 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6324 int Offset = ShuffleMask & 0x3F;
6325 unsigned SrcIdx = (ShuffleMask >> 6) & 0x1;
6326 return std::pair<unsigned, int>{SrcIdx, Offset};
6327 });
6328 case X86::BI__builtin_ia32_vperm2f128_pd256:
6329 case X86::BI__builtin_ia32_vperm2f128_ps256:
6330 case X86::BI__builtin_ia32_vperm2f128_si256:
6331 case X86::BI__builtin_ia32_permti256: {
6332 unsigned NumElements =
6333 Call->getArg(Arg: 0)->getType()->castAs<VectorType>()->getNumElements();
6334 unsigned PreservedBitsCnt = NumElements >> 2;
6335 return interp__builtin_ia32_shuffle_generic(
6336 S, OpPC, Call,
6337 GetSourceIndex: [PreservedBitsCnt](unsigned DstIdx, unsigned ShuffleMask) {
6338 unsigned ControlBitsCnt = DstIdx >> PreservedBitsCnt << 2;
6339 unsigned ControlBits = ShuffleMask >> ControlBitsCnt;
6340
6341 if (ControlBits & 0b1000)
6342 return std::make_pair(x: 0u, y: -1);
6343
6344 unsigned SrcVecIdx = (ControlBits & 0b10) >> 1;
6345 unsigned PreservedBitsMask = (1 << PreservedBitsCnt) - 1;
6346 int SrcIdx = ((ControlBits & 0b1) << PreservedBitsCnt) |
6347 (DstIdx & PreservedBitsMask);
6348 return std::make_pair(x&: SrcVecIdx, y&: SrcIdx);
6349 });
6350 }
6351 case X86::BI__builtin_ia32_pshufb128:
6352 case X86::BI__builtin_ia32_pshufb256:
6353 case X86::BI__builtin_ia32_pshufb512:
6354 return interp__builtin_ia32_shuffle_generic(
6355 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6356 uint8_t Ctlb = static_cast<uint8_t>(ShuffleMask);
6357 if (Ctlb & 0x80)
6358 return std::make_pair(x: 0, y: -1);
6359
6360 unsigned LaneBase = (DstIdx / 16) * 16;
6361 unsigned SrcOffset = Ctlb & 0x0F;
6362 unsigned SrcIdx = LaneBase + SrcOffset;
6363 return std::make_pair(x: 0, y: static_cast<int>(SrcIdx));
6364 });
6365
6366 case X86::BI__builtin_ia32_pshuflw:
6367 case X86::BI__builtin_ia32_pshuflw256:
6368 case X86::BI__builtin_ia32_pshuflw512:
6369 return interp__builtin_ia32_shuffle_generic(
6370 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6371 unsigned LaneBase = (DstIdx / 8) * 8;
6372 unsigned LaneIdx = DstIdx % 8;
6373 if (LaneIdx < 4) {
6374 unsigned Sel = (ShuffleMask >> (2 * LaneIdx)) & 0x3;
6375 return std::make_pair(x: 0, y: static_cast<int>(LaneBase + Sel));
6376 }
6377
6378 return std::make_pair(x: 0, y: static_cast<int>(DstIdx));
6379 });
6380
6381 case X86::BI__builtin_ia32_pshufhw:
6382 case X86::BI__builtin_ia32_pshufhw256:
6383 case X86::BI__builtin_ia32_pshufhw512:
6384 return interp__builtin_ia32_shuffle_generic(
6385 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6386 unsigned LaneBase = (DstIdx / 8) * 8;
6387 unsigned LaneIdx = DstIdx % 8;
6388 if (LaneIdx >= 4) {
6389 unsigned Sel = (ShuffleMask >> (2 * (LaneIdx - 4))) & 0x3;
6390 return std::make_pair(x: 0, y: static_cast<int>(LaneBase + 4 + Sel));
6391 }
6392
6393 return std::make_pair(x: 0, y: static_cast<int>(DstIdx));
6394 });
6395
6396 case X86::BI__builtin_ia32_pshufd:
6397 case X86::BI__builtin_ia32_pshufd256:
6398 case X86::BI__builtin_ia32_pshufd512:
6399 case X86::BI__builtin_ia32_vpermilps:
6400 case X86::BI__builtin_ia32_vpermilps256:
6401 case X86::BI__builtin_ia32_vpermilps512:
6402 return interp__builtin_ia32_shuffle_generic(
6403 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6404 unsigned LaneBase = (DstIdx / 4) * 4;
6405 unsigned LaneIdx = DstIdx % 4;
6406 unsigned Sel = (ShuffleMask >> (2 * LaneIdx)) & 0x3;
6407 return std::make_pair(x: 0, y: static_cast<int>(LaneBase + Sel));
6408 });
6409
6410 case X86::BI__builtin_ia32_vpermilvarpd:
6411 case X86::BI__builtin_ia32_vpermilvarpd256:
6412 case X86::BI__builtin_ia32_vpermilvarpd512:
6413 return interp__builtin_ia32_shuffle_generic(
6414 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6415 unsigned NumElemPerLane = 2;
6416 unsigned Lane = DstIdx / NumElemPerLane;
6417 unsigned Offset = ShuffleMask & 0b10 ? 1 : 0;
6418 return std::make_pair(
6419 x: 0, y: static_cast<int>(Lane * NumElemPerLane + Offset));
6420 });
6421
6422 case X86::BI__builtin_ia32_vpermilvarps:
6423 case X86::BI__builtin_ia32_vpermilvarps256:
6424 case X86::BI__builtin_ia32_vpermilvarps512:
6425 return interp__builtin_ia32_shuffle_generic(
6426 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6427 unsigned NumElemPerLane = 4;
6428 unsigned Lane = DstIdx / NumElemPerLane;
6429 unsigned Offset = ShuffleMask & 0b11;
6430 return std::make_pair(
6431 x: 0, y: static_cast<int>(Lane * NumElemPerLane + Offset));
6432 });
6433
6434 case X86::BI__builtin_ia32_vpermilpd:
6435 case X86::BI__builtin_ia32_vpermilpd256:
6436 case X86::BI__builtin_ia32_vpermilpd512:
6437 return interp__builtin_ia32_shuffle_generic(
6438 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned Control) {
6439 unsigned NumElemPerLane = 2;
6440 unsigned BitsPerElem = 1;
6441 unsigned MaskBits = 8;
6442 unsigned IndexMask = 0x1;
6443 unsigned Lane = DstIdx / NumElemPerLane;
6444 unsigned LaneOffset = Lane * NumElemPerLane;
6445 unsigned BitIndex = (DstIdx * BitsPerElem) % MaskBits;
6446 unsigned Index = (Control >> BitIndex) & IndexMask;
6447 return std::make_pair(x: 0, y: static_cast<int>(LaneOffset + Index));
6448 });
6449
6450 case X86::BI__builtin_ia32_permdf256:
6451 case X86::BI__builtin_ia32_permdi256:
6452 return interp__builtin_ia32_shuffle_generic(
6453 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned Control) {
6454 // permute4x64 operates on 4 64-bit elements
6455 // For element i (0-3), extract bits [2*i+1:2*i] from Control
6456 unsigned Index = (Control >> (2 * DstIdx)) & 0x3;
6457 return std::make_pair(x: 0, y: static_cast<int>(Index));
6458 });
6459
6460 case X86::BI__builtin_ia32_vpmultishiftqb128:
6461 case X86::BI__builtin_ia32_vpmultishiftqb256:
6462 case X86::BI__builtin_ia32_vpmultishiftqb512:
6463 return interp__builtin_ia32_multishiftqb(S, OpPC, Call);
6464 case X86::BI__builtin_ia32_kandqi:
6465 case X86::BI__builtin_ia32_kandhi:
6466 case X86::BI__builtin_ia32_kandsi:
6467 case X86::BI__builtin_ia32_kanddi:
6468 return interp__builtin_elementwise_int_binop(
6469 S, OpPC, Call,
6470 Fn: [](const APSInt &LHS, const APSInt &RHS) { return LHS & RHS; });
6471
6472 case X86::BI__builtin_ia32_kandnqi:
6473 case X86::BI__builtin_ia32_kandnhi:
6474 case X86::BI__builtin_ia32_kandnsi:
6475 case X86::BI__builtin_ia32_kandndi:
6476 return interp__builtin_elementwise_int_binop(
6477 S, OpPC, Call,
6478 Fn: [](const APSInt &LHS, const APSInt &RHS) { return ~LHS & RHS; });
6479
6480 case X86::BI__builtin_ia32_korqi:
6481 case X86::BI__builtin_ia32_korhi:
6482 case X86::BI__builtin_ia32_korsi:
6483 case X86::BI__builtin_ia32_kordi:
6484 return interp__builtin_elementwise_int_binop(
6485 S, OpPC, Call,
6486 Fn: [](const APSInt &LHS, const APSInt &RHS) { return LHS | RHS; });
6487
6488 case X86::BI__builtin_ia32_kxnorqi:
6489 case X86::BI__builtin_ia32_kxnorhi:
6490 case X86::BI__builtin_ia32_kxnorsi:
6491 case X86::BI__builtin_ia32_kxnordi:
6492 return interp__builtin_elementwise_int_binop(
6493 S, OpPC, Call,
6494 Fn: [](const APSInt &LHS, const APSInt &RHS) { return ~(LHS ^ RHS); });
6495
6496 case X86::BI__builtin_ia32_kxorqi:
6497 case X86::BI__builtin_ia32_kxorhi:
6498 case X86::BI__builtin_ia32_kxorsi:
6499 case X86::BI__builtin_ia32_kxordi:
6500 return interp__builtin_elementwise_int_binop(
6501 S, OpPC, Call,
6502 Fn: [](const APSInt &LHS, const APSInt &RHS) { return LHS ^ RHS; });
6503
6504 case X86::BI__builtin_ia32_knotqi:
6505 case X86::BI__builtin_ia32_knothi:
6506 case X86::BI__builtin_ia32_knotsi:
6507 case X86::BI__builtin_ia32_knotdi:
6508 return interp__builtin_elementwise_int_unaryop(
6509 S, OpPC, Call, Fn: [](const APSInt &Src) { return ~Src; });
6510
6511 case X86::BI__builtin_ia32_kaddqi:
6512 case X86::BI__builtin_ia32_kaddhi:
6513 case X86::BI__builtin_ia32_kaddsi:
6514 case X86::BI__builtin_ia32_kadddi:
6515 return interp__builtin_elementwise_int_binop(
6516 S, OpPC, Call,
6517 Fn: [](const APSInt &LHS, const APSInt &RHS) { return LHS + RHS; });
6518
6519 case X86::BI__builtin_ia32_kmovb:
6520 case X86::BI__builtin_ia32_kmovw:
6521 case X86::BI__builtin_ia32_kmovd:
6522 case X86::BI__builtin_ia32_kmovq:
6523 return interp__builtin_elementwise_int_unaryop(
6524 S, OpPC, Call, Fn: [](const APSInt &Src) { return Src; });
6525
6526 case X86::BI__builtin_ia32_kunpckhi:
6527 case X86::BI__builtin_ia32_kunpckdi:
6528 case X86::BI__builtin_ia32_kunpcksi:
6529 return interp__builtin_elementwise_int_binop(
6530 S, OpPC, Call, Fn: [](const APSInt &A, const APSInt &B) {
6531 // Generic kunpack: extract lower half of each operand and concatenate
6532 // Result = A[HalfWidth-1:0] concat B[HalfWidth-1:0]
6533 unsigned BW = A.getBitWidth();
6534 return APSInt(A.trunc(width: BW / 2).concat(NewLSB: B.trunc(width: BW / 2)),
6535 A.isUnsigned());
6536 });
6537
6538 case X86::BI__builtin_ia32_phminposuw128:
6539 return interp__builtin_ia32_phminposuw(S, OpPC, Call);
6540
6541 case X86::BI__builtin_ia32_psraq128:
6542 case X86::BI__builtin_ia32_psraq256:
6543 case X86::BI__builtin_ia32_psraq512:
6544 case X86::BI__builtin_ia32_psrad128:
6545 case X86::BI__builtin_ia32_psrad256:
6546 case X86::BI__builtin_ia32_psrad512:
6547 case X86::BI__builtin_ia32_psraw128:
6548 case X86::BI__builtin_ia32_psraw256:
6549 case X86::BI__builtin_ia32_psraw512:
6550 return interp__builtin_ia32_shift_with_count(
6551 S, OpPC, Call,
6552 ShiftOp: [](const APInt &Elt, uint64_t Count) { return Elt.ashr(ShiftAmt: Count); },
6553 OverflowOp: [](const APInt &Elt, unsigned Width) { return Elt.ashr(ShiftAmt: Width - 1); });
6554
6555 case X86::BI__builtin_ia32_psllq128:
6556 case X86::BI__builtin_ia32_psllq256:
6557 case X86::BI__builtin_ia32_psllq512:
6558 case X86::BI__builtin_ia32_pslld128:
6559 case X86::BI__builtin_ia32_pslld256:
6560 case X86::BI__builtin_ia32_pslld512:
6561 case X86::BI__builtin_ia32_psllw128:
6562 case X86::BI__builtin_ia32_psllw256:
6563 case X86::BI__builtin_ia32_psllw512:
6564 return interp__builtin_ia32_shift_with_count(
6565 S, OpPC, Call,
6566 ShiftOp: [](const APInt &Elt, uint64_t Count) { return Elt.shl(shiftAmt: Count); },
6567 OverflowOp: [](const APInt &Elt, unsigned Width) { return APInt::getZero(numBits: Width); });
6568
6569 case X86::BI__builtin_ia32_psrlq128:
6570 case X86::BI__builtin_ia32_psrlq256:
6571 case X86::BI__builtin_ia32_psrlq512:
6572 case X86::BI__builtin_ia32_psrld128:
6573 case X86::BI__builtin_ia32_psrld256:
6574 case X86::BI__builtin_ia32_psrld512:
6575 case X86::BI__builtin_ia32_psrlw128:
6576 case X86::BI__builtin_ia32_psrlw256:
6577 case X86::BI__builtin_ia32_psrlw512:
6578 return interp__builtin_ia32_shift_with_count(
6579 S, OpPC, Call,
6580 ShiftOp: [](const APInt &Elt, uint64_t Count) { return Elt.lshr(shiftAmt: Count); },
6581 OverflowOp: [](const APInt &Elt, unsigned Width) { return APInt::getZero(numBits: Width); });
6582
6583 case X86::BI__builtin_ia32_pternlogd128_mask:
6584 case X86::BI__builtin_ia32_pternlogd256_mask:
6585 case X86::BI__builtin_ia32_pternlogd512_mask:
6586 case X86::BI__builtin_ia32_pternlogq128_mask:
6587 case X86::BI__builtin_ia32_pternlogq256_mask:
6588 case X86::BI__builtin_ia32_pternlogq512_mask:
6589 return interp__builtin_ia32_pternlog(S, OpPC, Call, /*MaskZ=*/false);
6590 case X86::BI__builtin_ia32_pternlogd128_maskz:
6591 case X86::BI__builtin_ia32_pternlogd256_maskz:
6592 case X86::BI__builtin_ia32_pternlogd512_maskz:
6593 case X86::BI__builtin_ia32_pternlogq128_maskz:
6594 case X86::BI__builtin_ia32_pternlogq256_maskz:
6595 case X86::BI__builtin_ia32_pternlogq512_maskz:
6596 return interp__builtin_ia32_pternlog(S, OpPC, Call, /*MaskZ=*/true);
6597 case Builtin::BI__builtin_elementwise_fshl:
6598 return interp__builtin_elementwise_triop(S, OpPC, Call,
6599 Fn: llvm::APIntOps::fshl);
6600 case Builtin::BI__builtin_elementwise_fshr:
6601 return interp__builtin_elementwise_triop(S, OpPC, Call,
6602 Fn: llvm::APIntOps::fshr);
6603
6604 case X86::BI__builtin_ia32_shuf_f32x4_256:
6605 case X86::BI__builtin_ia32_shuf_i32x4_256:
6606 case X86::BI__builtin_ia32_shuf_f64x2_256:
6607 case X86::BI__builtin_ia32_shuf_i64x2_256:
6608 case X86::BI__builtin_ia32_shuf_f32x4:
6609 case X86::BI__builtin_ia32_shuf_i32x4:
6610 case X86::BI__builtin_ia32_shuf_f64x2:
6611 case X86::BI__builtin_ia32_shuf_i64x2: {
6612 // Destination and sources A, B all have the same type.
6613 QualType VecQT = Call->getArg(Arg: 0)->getType();
6614 const auto *VecT = VecQT->castAs<VectorType>();
6615 unsigned NumElems = VecT->getNumElements();
6616 unsigned ElemBits = S.getASTContext().getTypeSize(T: VecT->getElementType());
6617 unsigned LaneBits = 128u;
6618 unsigned NumLanes = (NumElems * ElemBits) / LaneBits;
6619 unsigned NumElemsPerLane = LaneBits / ElemBits;
6620
6621 return interp__builtin_ia32_shuffle_generic(
6622 S, OpPC, Call,
6623 GetSourceIndex: [NumLanes, NumElemsPerLane](unsigned DstIdx, unsigned ShuffleMask) {
6624 // DstIdx determines source. ShuffleMask selects lane in source.
6625 unsigned BitsPerElem = NumLanes / 2;
6626 unsigned IndexMask = (1u << BitsPerElem) - 1;
6627 unsigned Lane = DstIdx / NumElemsPerLane;
6628 unsigned SrcIdx = (Lane < NumLanes / 2) ? 0 : 1;
6629 unsigned BitIdx = BitsPerElem * Lane;
6630 unsigned SrcLaneIdx = (ShuffleMask >> BitIdx) & IndexMask;
6631 unsigned ElemInLane = DstIdx % NumElemsPerLane;
6632 unsigned IdxToPick = SrcLaneIdx * NumElemsPerLane + ElemInLane;
6633 return std::pair<unsigned, int>{SrcIdx, IdxToPick};
6634 });
6635 }
6636
6637 case X86::BI__builtin_ia32_insertf32x4_256:
6638 case X86::BI__builtin_ia32_inserti32x4_256:
6639 case X86::BI__builtin_ia32_insertf64x2_256:
6640 case X86::BI__builtin_ia32_inserti64x2_256:
6641 case X86::BI__builtin_ia32_insertf32x4:
6642 case X86::BI__builtin_ia32_inserti32x4:
6643 case X86::BI__builtin_ia32_insertf64x2_512:
6644 case X86::BI__builtin_ia32_inserti64x2_512:
6645 case X86::BI__builtin_ia32_insertf32x8:
6646 case X86::BI__builtin_ia32_inserti32x8:
6647 case X86::BI__builtin_ia32_insertf64x4:
6648 case X86::BI__builtin_ia32_inserti64x4:
6649 case X86::BI__builtin_ia32_vinsertf128_ps256:
6650 case X86::BI__builtin_ia32_vinsertf128_pd256:
6651 case X86::BI__builtin_ia32_vinsertf128_si256:
6652 case X86::BI__builtin_ia32_insert128i256:
6653 return interp__builtin_ia32_insert_subvector(S, OpPC, Call, ID: BuiltinID);
6654
6655 case clang::X86::BI__builtin_ia32_vcvtps2ph:
6656 case clang::X86::BI__builtin_ia32_vcvtps2ph256:
6657 return interp__builtin_ia32_vcvtps2ph(S, OpPC, Call);
6658
6659 case X86::BI__builtin_ia32_vec_ext_v4hi:
6660 case X86::BI__builtin_ia32_vec_ext_v16qi:
6661 case X86::BI__builtin_ia32_vec_ext_v8hi:
6662 case X86::BI__builtin_ia32_vec_ext_v4si:
6663 case X86::BI__builtin_ia32_vec_ext_v2di:
6664 case X86::BI__builtin_ia32_vec_ext_v32qi:
6665 case X86::BI__builtin_ia32_vec_ext_v16hi:
6666 case X86::BI__builtin_ia32_vec_ext_v8si:
6667 case X86::BI__builtin_ia32_vec_ext_v4di:
6668 case X86::BI__builtin_ia32_vec_ext_v4sf:
6669 return interp__builtin_ia32_vec_ext(S, OpPC, Call, ID: BuiltinID);
6670
6671 case X86::BI__builtin_ia32_vec_set_v4hi:
6672 case X86::BI__builtin_ia32_vec_set_v16qi:
6673 case X86::BI__builtin_ia32_vec_set_v8hi:
6674 case X86::BI__builtin_ia32_vec_set_v4si:
6675 case X86::BI__builtin_ia32_vec_set_v2di:
6676 case X86::BI__builtin_ia32_vec_set_v32qi:
6677 case X86::BI__builtin_ia32_vec_set_v16hi:
6678 case X86::BI__builtin_ia32_vec_set_v8si:
6679 case X86::BI__builtin_ia32_vec_set_v4di:
6680 return interp__builtin_ia32_vec_set(S, OpPC, Call, ID: BuiltinID);
6681
6682 case X86::BI__builtin_ia32_cvtb2mask128:
6683 case X86::BI__builtin_ia32_cvtb2mask256:
6684 case X86::BI__builtin_ia32_cvtb2mask512:
6685 case X86::BI__builtin_ia32_cvtw2mask128:
6686 case X86::BI__builtin_ia32_cvtw2mask256:
6687 case X86::BI__builtin_ia32_cvtw2mask512:
6688 case X86::BI__builtin_ia32_cvtd2mask128:
6689 case X86::BI__builtin_ia32_cvtd2mask256:
6690 case X86::BI__builtin_ia32_cvtd2mask512:
6691 case X86::BI__builtin_ia32_cvtq2mask128:
6692 case X86::BI__builtin_ia32_cvtq2mask256:
6693 case X86::BI__builtin_ia32_cvtq2mask512:
6694 return interp__builtin_ia32_cvt_vec2mask(S, OpPC, Call, ID: BuiltinID);
6695
6696 case X86::BI__builtin_ia32_cvtmask2b128:
6697 case X86::BI__builtin_ia32_cvtmask2b256:
6698 case X86::BI__builtin_ia32_cvtmask2b512:
6699 case X86::BI__builtin_ia32_cvtmask2w128:
6700 case X86::BI__builtin_ia32_cvtmask2w256:
6701 case X86::BI__builtin_ia32_cvtmask2w512:
6702 case X86::BI__builtin_ia32_cvtmask2d128:
6703 case X86::BI__builtin_ia32_cvtmask2d256:
6704 case X86::BI__builtin_ia32_cvtmask2d512:
6705 case X86::BI__builtin_ia32_cvtmask2q128:
6706 case X86::BI__builtin_ia32_cvtmask2q256:
6707 case X86::BI__builtin_ia32_cvtmask2q512:
6708 return interp__builtin_ia32_cvt_mask2vec(S, OpPC, Call, ID: BuiltinID);
6709
6710 case X86::BI__builtin_ia32_cvtsd2ss:
6711 return interp__builtin_ia32_cvtsd2ss(S, OpPC, Call, HasRoundingMask: false);
6712
6713 case X86::BI__builtin_ia32_cvtsd2ss_round_mask:
6714 return interp__builtin_ia32_cvtsd2ss(S, OpPC, Call, HasRoundingMask: true);
6715
6716 case X86::BI__builtin_ia32_cvtpd2ps:
6717 case X86::BI__builtin_ia32_cvtpd2ps256:
6718 return interp__builtin_ia32_cvtpd2ps(S, OpPC, Call, IsMasked: false, HasRounding: false);
6719 case X86::BI__builtin_ia32_cvtpd2ps_mask:
6720 return interp__builtin_ia32_cvtpd2ps(S, OpPC, Call, IsMasked: true, HasRounding: false);
6721 case X86::BI__builtin_ia32_cvtpd2ps512_mask:
6722 return interp__builtin_ia32_cvtpd2ps(S, OpPC, Call, IsMasked: true, HasRounding: true);
6723
6724 case X86::BI__builtin_ia32_cmpb128_mask:
6725 case X86::BI__builtin_ia32_cmpw128_mask:
6726 case X86::BI__builtin_ia32_cmpd128_mask:
6727 case X86::BI__builtin_ia32_cmpq128_mask:
6728 case X86::BI__builtin_ia32_cmpb256_mask:
6729 case X86::BI__builtin_ia32_cmpw256_mask:
6730 case X86::BI__builtin_ia32_cmpd256_mask:
6731 case X86::BI__builtin_ia32_cmpq256_mask:
6732 case X86::BI__builtin_ia32_cmpb512_mask:
6733 case X86::BI__builtin_ia32_cmpw512_mask:
6734 case X86::BI__builtin_ia32_cmpd512_mask:
6735 case X86::BI__builtin_ia32_cmpq512_mask:
6736 return interp__builtin_ia32_cmp_mask(S, OpPC, Call, ID: BuiltinID,
6737 /*IsUnsigned=*/false);
6738
6739 case X86::BI__builtin_ia32_ucmpb128_mask:
6740 case X86::BI__builtin_ia32_ucmpw128_mask:
6741 case X86::BI__builtin_ia32_ucmpd128_mask:
6742 case X86::BI__builtin_ia32_ucmpq128_mask:
6743 case X86::BI__builtin_ia32_ucmpb256_mask:
6744 case X86::BI__builtin_ia32_ucmpw256_mask:
6745 case X86::BI__builtin_ia32_ucmpd256_mask:
6746 case X86::BI__builtin_ia32_ucmpq256_mask:
6747 case X86::BI__builtin_ia32_ucmpb512_mask:
6748 case X86::BI__builtin_ia32_ucmpw512_mask:
6749 case X86::BI__builtin_ia32_ucmpd512_mask:
6750 case X86::BI__builtin_ia32_ucmpq512_mask:
6751 return interp__builtin_ia32_cmp_mask(S, OpPC, Call, ID: BuiltinID,
6752 /*IsUnsigned=*/true);
6753
6754 case X86::BI__builtin_ia32_vpshufbitqmb128_mask:
6755 case X86::BI__builtin_ia32_vpshufbitqmb256_mask:
6756 case X86::BI__builtin_ia32_vpshufbitqmb512_mask:
6757 return interp__builtin_ia32_shufbitqmb_mask(S, OpPC, Call);
6758
6759 case X86::BI__builtin_ia32_pslldqi128_byteshift:
6760 case X86::BI__builtin_ia32_pslldqi256_byteshift:
6761 case X86::BI__builtin_ia32_pslldqi512_byteshift:
6762 // These SLLDQ intrinsics always operate on byte elements (8 bits).
6763 // The lane width is hardcoded to 16 to match the SIMD register size,
6764 // but the algorithm processes one byte per iteration,
6765 // so APInt(8, ...) is correct and intentional.
6766 return interp__builtin_ia32_shuffle_generic(
6767 S, OpPC, Call,
6768 GetSourceIndex: [](unsigned DstIdx, unsigned Shift) -> std::pair<unsigned, int> {
6769 unsigned LaneBase = (DstIdx / 16) * 16;
6770 unsigned LaneIdx = DstIdx % 16;
6771 if (LaneIdx < Shift)
6772 return std::make_pair(x: 0, y: -1);
6773
6774 return std::make_pair(x: 0,
6775 y: static_cast<int>(LaneBase + LaneIdx - Shift));
6776 });
6777
6778 case X86::BI__builtin_ia32_psrldqi128_byteshift:
6779 case X86::BI__builtin_ia32_psrldqi256_byteshift:
6780 case X86::BI__builtin_ia32_psrldqi512_byteshift:
6781 // These SRLDQ intrinsics always operate on byte elements (8 bits).
6782 // The lane width is hardcoded to 16 to match the SIMD register size,
6783 // but the algorithm processes one byte per iteration,
6784 // so APInt(8, ...) is correct and intentional.
6785 return interp__builtin_ia32_shuffle_generic(
6786 S, OpPC, Call,
6787 GetSourceIndex: [](unsigned DstIdx, unsigned Shift) -> std::pair<unsigned, int> {
6788 unsigned LaneBase = (DstIdx / 16) * 16;
6789 unsigned LaneIdx = DstIdx % 16;
6790 if (LaneIdx + Shift < 16)
6791 return std::make_pair(x: 0,
6792 y: static_cast<int>(LaneBase + LaneIdx + Shift));
6793
6794 return std::make_pair(x: 0, y: -1);
6795 });
6796
6797 case X86::BI__builtin_ia32_palignr128:
6798 case X86::BI__builtin_ia32_palignr256:
6799 case X86::BI__builtin_ia32_palignr512:
6800 return interp__builtin_ia32_shuffle_generic(
6801 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned Shift) {
6802 // Default to -1 → zero-fill this destination element
6803 unsigned VecIdx = 1;
6804 int ElemIdx = -1;
6805
6806 int Lane = DstIdx / 16;
6807 int Offset = DstIdx % 16;
6808
6809 // Elements come from VecB first, then VecA after the shift boundary
6810 unsigned ShiftedIdx = Offset + (Shift & 0xFF);
6811 if (ShiftedIdx < 16) { // from VecB
6812 ElemIdx = ShiftedIdx + (Lane * 16);
6813 } else if (ShiftedIdx < 32) { // from VecA
6814 VecIdx = 0;
6815 ElemIdx = (ShiftedIdx - 16) + (Lane * 16);
6816 }
6817
6818 return std::pair<unsigned, int>{VecIdx, ElemIdx};
6819 });
6820
6821 case X86::BI__builtin_ia32_alignd128:
6822 case X86::BI__builtin_ia32_alignd256:
6823 case X86::BI__builtin_ia32_alignd512:
6824 case X86::BI__builtin_ia32_alignq128:
6825 case X86::BI__builtin_ia32_alignq256:
6826 case X86::BI__builtin_ia32_alignq512: {
6827 unsigned NumElems = Call->getType()->castAs<VectorType>()->getNumElements();
6828 return interp__builtin_ia32_shuffle_generic(
6829 S, OpPC, Call, GetSourceIndex: [NumElems](unsigned DstIdx, unsigned Shift) {
6830 unsigned Imm = Shift & 0xFF;
6831 unsigned EffectiveShift = Imm & (NumElems - 1);
6832 unsigned SourcePos = DstIdx + EffectiveShift;
6833 unsigned VecIdx = SourcePos < NumElems ? 1u : 0u;
6834 unsigned ElemIdx = SourcePos & (NumElems - 1);
6835 return std::pair<unsigned, int>{VecIdx, static_cast<int>(ElemIdx)};
6836 });
6837 }
6838
6839 case clang::X86::BI__builtin_ia32_minps:
6840 case clang::X86::BI__builtin_ia32_minpd:
6841 case clang::X86::BI__builtin_ia32_minph128:
6842 case clang::X86::BI__builtin_ia32_minph256:
6843 case clang::X86::BI__builtin_ia32_minps256:
6844 case clang::X86::BI__builtin_ia32_minpd256:
6845 case clang::X86::BI__builtin_ia32_minps512:
6846 case clang::X86::BI__builtin_ia32_minpd512:
6847 case clang::X86::BI__builtin_ia32_minph512:
6848 return interp__builtin_elementwise_fp_binop(
6849 S, OpPC, Call,
6850 Fn: [](const APFloat &A, const APFloat &B,
6851 std::optional<APSInt>) -> std::optional<APFloat> {
6852 if (A.isNaN() || A.isInfinity() || A.isDenormal() || B.isNaN() ||
6853 B.isInfinity() || B.isDenormal())
6854 return std::nullopt;
6855 if (A.isZero() && B.isZero())
6856 return B;
6857 return llvm::minimum(A, B);
6858 });
6859
6860 case clang::X86::BI__builtin_ia32_minss:
6861 case clang::X86::BI__builtin_ia32_minsd:
6862 return interp__builtin_elementwise_fp_binop(
6863 S, OpPC, Call,
6864 Fn: [](const APFloat &A, const APFloat &B,
6865 std::optional<APSInt> RoundingMode) -> std::optional<APFloat> {
6866 return EvalScalarMinMaxFp(A, B, RoundingMode, /*IsMin=*/true);
6867 },
6868 /*IsScalar=*/true);
6869
6870 case clang::X86::BI__builtin_ia32_minsd_round_mask:
6871 case clang::X86::BI__builtin_ia32_minss_round_mask:
6872 case clang::X86::BI__builtin_ia32_minsh_round_mask:
6873 case clang::X86::BI__builtin_ia32_maxsd_round_mask:
6874 case clang::X86::BI__builtin_ia32_maxss_round_mask:
6875 case clang::X86::BI__builtin_ia32_maxsh_round_mask: {
6876 bool IsMin = BuiltinID == clang::X86::BI__builtin_ia32_minsd_round_mask ||
6877 BuiltinID == clang::X86::BI__builtin_ia32_minss_round_mask ||
6878 BuiltinID == clang::X86::BI__builtin_ia32_minsh_round_mask;
6879 return interp__builtin_scalar_fp_round_mask_binop(
6880 S, OpPC, Call,
6881 Fn: [IsMin](const APFloat &A, const APFloat &B,
6882 std::optional<APSInt> RoundingMode) -> std::optional<APFloat> {
6883 return EvalScalarMinMaxFp(A, B, RoundingMode, IsMin);
6884 });
6885 }
6886
6887 case clang::X86::BI__builtin_ia32_maxps:
6888 case clang::X86::BI__builtin_ia32_maxpd:
6889 case clang::X86::BI__builtin_ia32_maxph128:
6890 case clang::X86::BI__builtin_ia32_maxph256:
6891 case clang::X86::BI__builtin_ia32_maxps256:
6892 case clang::X86::BI__builtin_ia32_maxpd256:
6893 case clang::X86::BI__builtin_ia32_maxps512:
6894 case clang::X86::BI__builtin_ia32_maxpd512:
6895 case clang::X86::BI__builtin_ia32_maxph512:
6896 return interp__builtin_elementwise_fp_binop(
6897 S, OpPC, Call,
6898 Fn: [](const APFloat &A, const APFloat &B,
6899 std::optional<APSInt>) -> std::optional<APFloat> {
6900 if (A.isNaN() || A.isInfinity() || A.isDenormal() || B.isNaN() ||
6901 B.isInfinity() || B.isDenormal())
6902 return std::nullopt;
6903 if (A.isZero() && B.isZero())
6904 return B;
6905 return llvm::maximum(A, B);
6906 });
6907
6908 case clang::X86::BI__builtin_ia32_maxss:
6909 case clang::X86::BI__builtin_ia32_maxsd:
6910 return interp__builtin_elementwise_fp_binop(
6911 S, OpPC, Call,
6912 Fn: [](const APFloat &A, const APFloat &B,
6913 std::optional<APSInt> RoundingMode) -> std::optional<APFloat> {
6914 return EvalScalarMinMaxFp(A, B, RoundingMode, /*IsMin=*/false);
6915 },
6916 /*IsScalar=*/true);
6917 case X86::BI__builtin_ia32_vpdpwssd128:
6918 case X86::BI__builtin_ia32_vpdpwssd256:
6919 case X86::BI__builtin_ia32_vpdpwssd512:
6920 case X86::BI__builtin_ia32_vpdpbusd128:
6921 case X86::BI__builtin_ia32_vpdpbusd256:
6922 case X86::BI__builtin_ia32_vpdpbusd512:
6923 return interp__builtin_ia32_vpdp(S, OpPC, Call, IsSaturating: false);
6924 case X86::BI__builtin_ia32_vpdpwssds128:
6925 case X86::BI__builtin_ia32_vpdpwssds256:
6926 case X86::BI__builtin_ia32_vpdpwssds512:
6927 case X86::BI__builtin_ia32_vpdpbusds128:
6928 case X86::BI__builtin_ia32_vpdpbusds256:
6929 case X86::BI__builtin_ia32_vpdpbusds512:
6930 return interp__builtin_ia32_vpdp(S, OpPC, Call, IsSaturating: true);
6931 case X86::BI__builtin_ia32_cvtss2si:
6932 case X86::BI__builtin_ia32_cvtsd2si:
6933 case X86::BI__builtin_ia32_cvttss2si:
6934 case X86::BI__builtin_ia32_cvttsd2si:
6935 case X86::BI__builtin_ia32_cvtss2si64:
6936 case X86::BI__builtin_ia32_cvtsd2si64:
6937 case X86::BI__builtin_ia32_cvttss2si64:
6938 case X86::BI__builtin_ia32_cvttsd2si64:
6939 return interp_builtin_ia32_cvt_scalar_to_int(S, OpPC, E: Call);
6940 case X86::BI__builtin_ia32_cvtpd2dq:
6941 case X86::BI__builtin_ia32_cvttpd2dq:
6942 case X86::BI__builtin_ia32_cvtps2dq:
6943 case X86::BI__builtin_ia32_cvtpd2dq256:
6944 case X86::BI__builtin_ia32_cvtps2dq256:
6945 case X86::BI__builtin_ia32_cvttps2dq:
6946 case X86::BI__builtin_ia32_cvttpd2dq256:
6947 case X86::BI__builtin_ia32_cvttps2dq256:
6948 return interp_builtin_ia32_cvt_vector_to_int(S, OpPC, E: Call);
6949 default:
6950 return Invalid(S, OpPC);
6951 }
6952
6953 llvm_unreachable("Unhandled builtin ID");
6954}
6955
6956bool InterpretOffsetOf(InterpState &S, CodePtr OpPC, const OffsetOfExpr *E,
6957 ArrayRef<int64_t> ArrayIndices, int64_t &IntResult) {
6958 S.getASTContext().recordOffsetOfEvaluation(E);
6959 CharUnits Result;
6960 unsigned N = E->getNumComponents();
6961 assert(N > 0);
6962
6963 unsigned ArrayIndex = 0;
6964 QualType CurrentType = E->getTypeSourceInfo()->getType();
6965 for (unsigned I = 0; I != N; ++I) {
6966 const OffsetOfNode &Node = E->getComponent(Idx: I);
6967 switch (Node.getKind()) {
6968 case OffsetOfNode::Field: {
6969 const FieldDecl *MemberDecl = Node.getField();
6970 const auto *RD = CurrentType->getAsRecordDecl();
6971 if (!RD || RD->isInvalidDecl())
6972 return false;
6973 const ASTRecordLayout &RL = S.getASTContext().getASTRecordLayout(D: RD);
6974 unsigned FieldIndex = MemberDecl->getFieldIndex();
6975 assert(FieldIndex < RL.getFieldCount() && "offsetof field in wrong type");
6976 Result +=
6977 S.getASTContext().toCharUnitsFromBits(BitSize: RL.getFieldOffset(FieldNo: FieldIndex));
6978 CurrentType = MemberDecl->getType().getNonReferenceType();
6979 break;
6980 }
6981 case OffsetOfNode::Array: {
6982 // When generating bytecode, we put all the index expressions as Sint64 on
6983 // the stack.
6984 int64_t Index = ArrayIndices[ArrayIndex];
6985 if (Index < 0)
6986 return Invalid(S, OpPC);
6987 const ArrayType *AT = S.getASTContext().getAsArrayType(T: CurrentType);
6988 if (!AT)
6989 return false;
6990 CurrentType = AT->getElementType();
6991 CharUnits ElementSize = S.getASTContext().getTypeSizeInChars(T: CurrentType);
6992 int64_t ElemSize = ElementSize.getQuantity();
6993 if (Index != 0 && ElemSize > (llvm::maxIntN(N: 64) / Index)) {
6994 S.FFDiag(Loc: S.Current->getLocation(PC: OpPC),
6995 DiagId: diag::note_constexpr_offsetof_overflow)
6996 << S.Current->getRange(PC: OpPC);
6997 return false;
6998 }
6999 int64_t Offset = Index * ElemSize;
7000 if (Result.getQuantity() > llvm::maxIntN(N: 64) - Offset) {
7001 S.FFDiag(Loc: S.Current->getLocation(PC: OpPC),
7002 DiagId: diag::note_constexpr_offsetof_overflow)
7003 << S.Current->getRange(PC: OpPC);
7004 return false;
7005 }
7006 Result += CharUnits::fromQuantity(Quantity: Offset);
7007 ++ArrayIndex;
7008 break;
7009 }
7010 case OffsetOfNode::Base: {
7011 const CXXBaseSpecifier *BaseSpec = Node.getBase();
7012 if (BaseSpec->isVirtual())
7013 return false;
7014
7015 // Find the layout of the class whose base we are looking into.
7016 const auto *RD = CurrentType->getAsCXXRecordDecl();
7017 if (!RD || RD->isInvalidDecl())
7018 return false;
7019 const ASTRecordLayout &RL = S.getASTContext().getASTRecordLayout(D: RD);
7020
7021 // Find the base class itself.
7022 CurrentType = BaseSpec->getType();
7023 const auto *BaseRD = CurrentType->getAsCXXRecordDecl();
7024 if (!BaseRD)
7025 return false;
7026
7027 // Add the offset to the base.
7028 Result += RL.getBaseClassOffset(Base: BaseRD);
7029 break;
7030 }
7031 case OffsetOfNode::Identifier:
7032 llvm_unreachable("Dependent OffsetOfExpr?");
7033 }
7034 }
7035
7036 IntResult = Result.getQuantity();
7037
7038 return true;
7039}
7040
7041bool SetThreeWayComparisonField(InterpState &S, CodePtr OpPC,
7042 const Pointer &Ptr, const APSInt &IntValue) {
7043
7044 const Record *R = Ptr.getRecord();
7045 assert(R);
7046 assert(R->getNumFields() == 1);
7047
7048 unsigned FieldOffset = R->getField(I: 0u)->Offset;
7049 PtrView FieldPtr = Ptr.view().atField(Offset: FieldOffset);
7050 PrimType FieldT = FieldPtr.getFieldDesc()->getPrimType();
7051
7052 INT_TYPE_SWITCH(FieldT,
7053 FieldPtr.deref<T>() = T::from(IntValue.getSExtValue()));
7054 FieldPtr.initialize();
7055 return true;
7056}
7057
7058static void zeroAll(PtrView Dest) {
7059 const Descriptor *Desc = Dest.getFieldDesc();
7060
7061 if (Desc->isPrimitive()) {
7062 TYPE_SWITCH(Desc->getPrimType(), {
7063 Dest.deref<T>().~T();
7064 new (&Dest.deref<T>()) T();
7065 });
7066 return;
7067 }
7068
7069 if (Desc->isRecord()) {
7070 const Record *R = Desc->ElemRecord;
7071 for (const Record::Field &F : R->fields()) {
7072 PtrView FieldPtr = Dest.atField(Offset: F.Offset);
7073 zeroAll(Dest: FieldPtr);
7074 }
7075 return;
7076 }
7077
7078 if (Desc->isPrimitiveArray()) {
7079 for (unsigned I = 0, N = Desc->getNumElems(); I != N; ++I) {
7080 TYPE_SWITCH(Desc->getPrimType(), {
7081 Dest.deref<T>().~T();
7082 new (&Dest.deref<T>()) T();
7083 });
7084 }
7085 return;
7086 }
7087
7088 if (Desc->isCompositeArray()) {
7089 for (unsigned I = 0, N = Desc->getNumElems(); I != N; ++I) {
7090 PtrView ElemPtr = Dest.atIndex(Idx: I).narrow();
7091 zeroAll(Dest: ElemPtr);
7092 }
7093 return;
7094 }
7095}
7096
7097static bool copyComposite(InterpState &S, CodePtr OpPC, PtrView Src,
7098 PtrView Dest, bool Activate, bool Diagnose);
7099static bool copyRecord(InterpState &S, CodePtr OpPC, PtrView Src, PtrView Dest,
7100 bool Activate = false, bool Diagnose = true) {
7101 [[maybe_unused]] const Descriptor *SrcDesc = Src.getFieldDesc();
7102 const Descriptor *DestDesc = Dest.getFieldDesc();
7103
7104 auto copyField = [&](const Record::Field &F, bool Activate) -> bool {
7105 PtrView DestField = Dest.atField(Offset: F.Offset);
7106 PtrView SrcField = Src.atField(Offset: F.Offset);
7107
7108 if (OptPrimType FT = F.T) {
7109 if (!SrcField.isInitialized()) {
7110 if (Diagnose)
7111 return diagnoseUninitialized(S, OpPC, Extern: false, B: SrcField.block(),
7112 LT: SrcField.getLifetime(), AK: AK_Read);
7113 // Just skip.
7114 return true;
7115 }
7116
7117 TYPE_SWITCH(*FT, DestField.deref<T>() = SrcField.deref<T>(););
7118 if (DestField.canBeInitialized())
7119 DestField.initialize();
7120 if (Activate)
7121 DestField.activate();
7122 return true;
7123 }
7124
7125 return copyComposite(S, OpPC, Src: SrcField, Dest: DestField, Activate, Diagnose);
7126 };
7127
7128 assert(SrcDesc->isRecord());
7129 assert(SrcDesc->ElemRecord == DestDesc->ElemRecord);
7130 const Record *R = DestDesc->ElemRecord;
7131 for (const Record::Field &F : R->fields()) {
7132 PtrView FP = Src.atField(Offset: F.Offset);
7133
7134 if (!CheckMutable(S, OpPC, Ptr: FP))
7135 return false;
7136
7137 if (R->isUnion()) {
7138 // For unions, only copy the active field. Zero all others.
7139 if (FP.isActive()) {
7140 if (!copyField(F, /*Activate=*/true))
7141 return false;
7142 } else {
7143 PtrView DestField = Dest.atField(Offset: F.Offset);
7144 zeroAll(Dest: DestField);
7145 }
7146 } else {
7147 if (!copyField(F, Activate))
7148 return false;
7149 }
7150 }
7151
7152 for (const Record::Base &B : R->bases()) {
7153 PtrView DestBase = Dest.atField(Offset: B.Offset);
7154 if (!copyRecord(S, OpPC, Src: Src.atField(Offset: B.Offset), Dest: DestBase, Activate,
7155 Diagnose))
7156 return false;
7157 }
7158
7159 Dest.initialize();
7160 if (Activate)
7161 Dest.activate();
7162 return true;
7163}
7164
7165static bool copyComposite(InterpState &S, CodePtr OpPC, PtrView Src,
7166 PtrView Dest, bool Activate = false,
7167 bool Diagnose = false) {
7168 assert(Src.isLive() && Dest.isLive());
7169
7170 [[maybe_unused]] const Descriptor *SrcDesc = Src.getFieldDesc();
7171 const Descriptor *DestDesc = Dest.getFieldDesc();
7172
7173 assert(!DestDesc->isPrimitive() && !SrcDesc->isPrimitive());
7174
7175 if (DestDesc->isPrimitiveArray()) {
7176 if (!SrcDesc->isPrimitiveArray())
7177 return false;
7178 // For floating types, check the actual QualType so we don't accidentally
7179 // mix up semantics.
7180 if (SrcDesc->getPrimType() == PT_Float) {
7181 if (!S.getASTContext().hasSimilarType(T1: SrcDesc->getElemQualType(),
7182 T2: DestDesc->getElemQualType()))
7183 return false;
7184 }
7185
7186 assert(SrcDesc->isPrimitiveArray());
7187 assert(SrcDesc->getNumElems() == DestDesc->getNumElems());
7188 assert(SrcDesc->getPrimType() == DestDesc->getPrimType());
7189 PrimType ET = DestDesc->getPrimType();
7190 for (unsigned I = 0, N = DestDesc->getNumElems(); I != N; ++I) {
7191 PtrView DestElem = Dest.atIndex(Idx: I);
7192 TYPE_SWITCH(ET, { DestElem.deref<T>() = Src.elem<T>(I); });
7193 DestElem.initializeElement(Index: I);
7194 }
7195 return true;
7196 }
7197
7198 if (DestDesc->isCompositeArray()) {
7199 if (!SrcDesc->isCompositeArray())
7200 return false;
7201 assert(SrcDesc->isCompositeArray());
7202 assert(SrcDesc->getNumElems() == DestDesc->getNumElems());
7203 for (unsigned I = 0, N = DestDesc->getNumElems(); I != N; ++I) {
7204 PtrView SrcElem = Src.atIndex(Idx: I).narrow();
7205 PtrView DestElem = Dest.atIndex(Idx: I).narrow();
7206 if (!copyComposite(S, OpPC, Src: SrcElem, Dest: DestElem, Activate))
7207 return false;
7208 }
7209 return true;
7210 }
7211
7212 if (DestDesc->isRecord()) {
7213 if (!SrcDesc->isRecord())
7214 return false;
7215 return copyRecord(S, OpPC, Src, Dest, Activate, Diagnose);
7216 }
7217 return Invalid(S, OpPC);
7218}
7219
7220bool DoMemcpy(InterpState &S, CodePtr OpPC, const Pointer &Src, Pointer &Dest,
7221 bool Activate, bool Diagnose) {
7222 if (!Src.isBlockPointer() || Src.getFieldDesc()->isPrimitive())
7223 return false;
7224 if (!Dest.isBlockPointer() || Dest.getFieldDesc()->isPrimitive())
7225 return false;
7226
7227 return copyComposite(S, OpPC, Src: Src.view(), Dest: Dest.view(), Activate, Diagnose);
7228}
7229
7230} // namespace interp
7231} // namespace clang
7232