1//===--- InterpBuiltin.cpp - Interpreter for the constexpr VM ---*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8#include "../ExprConstShared.h"
9#include "Boolean.h"
10#include "Char.h"
11#include "EvalEmitter.h"
12#include "InterpBuiltinBitCast.h"
13#include "InterpHelpers.h"
14#include "PrimType.h"
15#include "Program.h"
16#include "clang/AST/InferAlloc.h"
17#include "clang/AST/OSLog.h"
18#include "clang/AST/RecordLayout.h"
19#include "clang/Basic/Builtins.h"
20#include "clang/Basic/TargetBuiltins.h"
21#include "clang/Basic/TargetInfo.h"
22#include "llvm/ADT/StringExtras.h"
23#include "llvm/Support/AllocToken.h"
24#include "llvm/Support/ErrorHandling.h"
25#include "llvm/Support/SipHash.h"
26
27namespace clang {
28namespace interp {
29
30[[maybe_unused]] static bool isNoopBuiltin(unsigned ID) {
31 switch (ID) {
32 case Builtin::BIas_const:
33 case Builtin::BIforward:
34 case Builtin::BIforward_like:
35 case Builtin::BImove:
36 case Builtin::BImove_if_noexcept:
37 case Builtin::BIaddressof:
38 case Builtin::BI__addressof:
39 case Builtin::BI__builtin_addressof:
40 case Builtin::BI__builtin_launder:
41 return true;
42 default:
43 return false;
44 }
45 return false;
46}
47
48static void discard(InterpStack &Stk, PrimType T) {
49 TYPE_SWITCH(T, { Stk.discard<T>(); });
50}
51
52static bool popToUInt64(const InterpState &S, const Expr *E, uint64_t &Out) {
53 INT_TYPE_SWITCH(*S.getContext().classify(E->getType()), {
54 const auto &Val = S.Stk.pop<T>();
55 if (!Val.isNumber())
56 return false;
57 Out = static_cast<uint64_t>(Val);
58 return true;
59 });
60}
61
62static bool popToAPSInt(InterpStack &Stk, PrimType T, APSInt &Out) {
63 INT_TYPE_SWITCH(T, {
64 const auto &Val = Stk.pop<T>();
65 if (!Val.isNumber())
66 return false;
67 Out = Val.toAPSInt();
68 return true;
69 });
70}
71
72static bool popToAPSInt(InterpState &S, const Expr *E, APSInt &Out) {
73 return popToAPSInt(Stk&: S.Stk, T: *S.getContext().classify(T: E->getType()), Out);
74}
75static bool popToAPSInt(InterpState &S, QualType T, APSInt &Out) {
76 return popToAPSInt(Stk&: S.Stk, T: *S.getContext().classify(T), Out);
77}
78
79/// Check for common reasons a pointer can't be read from, which
80/// are usually not diagnosed in a builtin function.
81static bool isReadable(const Pointer &P) {
82 if (P.isDummy())
83 return false;
84 if (!P.isReadablePointerType())
85 return false;
86 if (!P.isLive())
87 return false;
88 if (P.isOnePastEnd())
89 return false;
90 return true;
91}
92
93/// Pushes \p Val on the stack as the type given by \p QT.
94static void pushInteger(InterpState &S, const APSInt &Val, QualType QT) {
95 assert(QT->isSignedIntegerOrEnumerationType() ||
96 QT->isUnsignedIntegerOrEnumerationType());
97 OptPrimType T = *S.getContext().classify(T: QT);
98 assert(T);
99
100 if (T == PT_IntAPS) {
101 unsigned BitWidth = S.getASTContext().getIntWidth(T: QT);
102 auto Result = S.allocAP<IntegralAP<true>>(BitWidth);
103 Result.copy(V: Val.extOrTrunc(width: BitWidth));
104 S.Stk.push<IntegralAP<true>>(Args&: Result);
105 return;
106 }
107
108 if (T == PT_IntAP) {
109 unsigned BitWidth = S.getASTContext().getIntWidth(T: QT);
110 auto Result = S.allocAP<IntegralAP<false>>(BitWidth);
111 Result.copy(V: Val.extOrTrunc(width: BitWidth));
112 S.Stk.push<IntegralAP<false>>(Args&: Result);
113 return;
114 }
115
116 if (isSignedType(T: *T)) {
117 int64_t V = Val.getSExtValue();
118 INT_TYPE_SWITCH(*T, { S.Stk.push<T>(T::from(V)); });
119 } else {
120 assert(QT->isUnsignedIntegerOrEnumerationType());
121 uint64_t V = Val.getZExtValue();
122 INT_TYPE_SWITCH(*T, { S.Stk.push<T>(T::from(V)); });
123 }
124}
125
126template <typename T>
127static void pushInteger(InterpState &S, T Val, QualType QT) {
128 if constexpr (std::is_same_v<T, APInt>)
129 pushInteger(S, Val: APSInt(Val, !std::is_signed_v<T>), QT);
130 else if constexpr (std::is_same_v<T, APSInt>)
131 pushInteger(S, Val, QT);
132 else
133 pushInteger(S,
134 Val: APSInt(APInt(sizeof(T) * 8, static_cast<uint64_t>(Val),
135 std::is_signed_v<T>),
136 !std::is_signed_v<T>),
137 QT);
138}
139
140static void assignIntegral(InterpState &S, const Pointer &Dest, PrimType ValueT,
141 const APSInt &Value) {
142
143 if (ValueT == PT_IntAPS) {
144 Dest.deref<IntegralAP<true>>() =
145 S.allocAP<IntegralAP<true>>(BitWidth: Value.getBitWidth());
146 Dest.deref<IntegralAP<true>>().copy(V: Value);
147 } else if (ValueT == PT_IntAP) {
148 Dest.deref<IntegralAP<false>>() =
149 S.allocAP<IntegralAP<false>>(BitWidth: Value.getBitWidth());
150 Dest.deref<IntegralAP<false>>().copy(V: Value);
151 } else if (ValueT == PT_Bool) {
152 Dest.deref<Boolean>() = Boolean::from(Value: !Value.isZero());
153 } else {
154 INT_TYPE_SWITCH_NO_BOOL(
155 ValueT, { Dest.deref<T>() = T::from(static_cast<T>(Value)); });
156 }
157}
158
159static QualType getElemType(const Pointer &P) {
160 if (P.isStringPointer()) {
161 return P.asStringPointer()
162 .getLiteral()
163 ->getType()
164 ->getAsArrayTypeUnsafe()
165 ->getElementType();
166 }
167
168 const Descriptor *Desc = P.getFieldDesc();
169 QualType T = Desc->getType();
170 if (Desc->isPrimitive())
171 return T;
172 if (T->isPointerType())
173 return T->castAs<PointerType>()->getPointeeType();
174 if (Desc->isArray())
175 return Desc->getElemQualType();
176 if (const auto *AT = T->getAsArrayTypeUnsafe())
177 return AT->getElementType();
178 return T;
179}
180
181static void diagnoseNonConstexprBuiltin(InterpState &S, CodePtr OpPC,
182 unsigned ID) {
183 if (!S.diagnosing())
184 return;
185
186 auto Loc = S.Current->getSource(PC: OpPC);
187 if (S.getLangOpts().CPlusPlus11)
188 S.CCEDiag(SI: Loc, DiagId: diag::note_constexpr_invalid_function)
189 << /*isConstexpr=*/0 << /*isConstructor=*/0
190 << S.getASTContext().BuiltinInfo.getQuotedName(ID);
191 else
192 S.CCEDiag(SI: Loc, DiagId: diag::note_invalid_subexpr_in_const_expr);
193}
194
195static llvm::APSInt convertBoolVectorToInt(const Pointer &Val) {
196 assert(Val.getFieldDesc()->isPrimitiveArray() &&
197 Val.getFieldDesc()->getElemQualType()->isBooleanType() &&
198 "Not a boolean vector");
199 unsigned NumElems = Val.getNumElems();
200
201 // Each element is one bit, so create an integer with NumElts bits.
202 llvm::APSInt Result(NumElems, 0);
203 for (unsigned I = 0; I != NumElems; ++I) {
204 if (Val.elem<bool>(I))
205 Result.setBit(I);
206 }
207
208 return Result;
209}
210
211// Strict double -> float conversion used for X86 PD2PS/cvtsd2ss intrinsics.
212// Reject NaN/Inf/Subnormal inputs and any lossy/inexact conversions.
213static bool convertDoubleToFloatStrict(const APFloat &Src, Floating &Dst,
214 InterpState &S, const Expr *DiagExpr) {
215 if (Src.isInfinity()) {
216 if (S.diagnosing())
217 S.CCEDiag(E: DiagExpr, DiagId: diag::note_constexpr_float_arithmetic) << 0;
218 return false;
219 }
220 if (Src.isNaN()) {
221 if (S.diagnosing())
222 S.CCEDiag(E: DiagExpr, DiagId: diag::note_constexpr_float_arithmetic) << 1;
223 return false;
224 }
225 APFloat Val = Src;
226 bool LosesInfo = false;
227 APFloat::opStatus Status = Val.convert(
228 ToSemantics: APFloat::IEEEsingle(), RM: APFloat::rmNearestTiesToEven, losesInfo: &LosesInfo);
229 if (LosesInfo || Val.isDenormal()) {
230 if (S.diagnosing())
231 S.CCEDiag(E: DiagExpr, DiagId: diag::note_constexpr_float_arithmetic_strict);
232 return false;
233 }
234 if (Status != APFloat::opOK) {
235 if (S.diagnosing())
236 S.CCEDiag(E: DiagExpr, DiagId: diag::note_invalid_subexpr_in_const_expr);
237 return false;
238 }
239 Dst.copy(F: Val);
240 return true;
241}
242
243static bool interp__builtin_is_constant_evaluated(InterpState &S, CodePtr OpPC,
244 const InterpFrame *Frame,
245 const CallExpr *Call) {
246 unsigned Depth = S.Current->getDepth();
247 auto isStdCall = [](const FunctionDecl *F) -> bool {
248 return F && F->isInStdNamespace() && F->getIdentifier() &&
249 F->getIdentifier()->isStr(Str: "is_constant_evaluated");
250 };
251 const InterpFrame *Caller = Frame->Caller;
252 // The current frame is the one for __builtin_is_constant_evaluated.
253 // The one above that, potentially the one for std::is_constant_evaluated().
254 if (S.inConstantContext() && !S.checkingPotentialConstantExpression() &&
255 S.getEvalStatus().Diag &&
256 (Depth == 0 || (Depth == 1 && isStdCall(Frame->getCallee())))) {
257 if (Caller && isStdCall(Frame->getCallee())) {
258 const Expr *E = Caller->getExpr(PC: Caller->getRetPC());
259 S.report(Loc: E->getExprLoc(),
260 DiagId: diag::warn_is_constant_evaluated_always_true_constexpr)
261 << "std::is_constant_evaluated" << E->getSourceRange();
262 } else {
263 S.report(Loc: Call->getExprLoc(),
264 DiagId: diag::warn_is_constant_evaluated_always_true_constexpr)
265 << "__builtin_is_constant_evaluated" << Call->getSourceRange();
266 }
267 }
268
269 S.Stk.push<Boolean>(Args: Boolean::from(Value: S.inConstantContext()));
270 return true;
271}
272
273// __builtin_assume
274// __assume (MS extension)
275static bool interp__builtin_assume(InterpState &S, CodePtr OpPC,
276 const InterpFrame *Frame,
277 const CallExpr *Call) {
278 // Nothing to be done here since the argument is NOT evaluated.
279 assert(Call->getNumArgs() == 1);
280 return true;
281}
282
283static bool interp__builtin_strcmp(InterpState &S, CodePtr OpPC,
284 const InterpFrame *Frame,
285 const CallExpr *Call, unsigned ID) {
286 uint64_t Limit = ~static_cast<uint64_t>(0);
287 if (ID == Builtin::BIstrncmp || ID == Builtin::BI__builtin_strncmp ||
288 ID == Builtin::BIwcsncmp || ID == Builtin::BI__builtin_wcsncmp) {
289 if (!popToUInt64(S, E: Call->getArg(Arg: 2), Out&: Limit))
290 return false;
291 }
292
293 const Pointer &B = S.Stk.pop<Pointer>();
294 const Pointer &A = S.Stk.pop<Pointer>();
295 if (ID == Builtin::BIstrcmp || ID == Builtin::BIstrncmp ||
296 ID == Builtin::BIwcscmp || ID == Builtin::BIwcsncmp)
297 diagnoseNonConstexprBuiltin(S, OpPC, ID);
298
299 if (Limit == 0) {
300 pushInteger(S, Val: 0, QT: Call->getType());
301 return true;
302 }
303
304 if (!CheckLive(S, OpPC, Ptr: A, AK: AK_Read) || !CheckLive(S, OpPC, Ptr: B, AK: AK_Read))
305 return false;
306
307 if (!A.isReadablePointerType() || !B.isReadablePointerType())
308 return false;
309 if (A.isDummy() || B.isDummy())
310 return false;
311
312 bool IsWide = ID == Builtin::BIwcscmp || ID == Builtin::BIwcsncmp ||
313 ID == Builtin::BI__builtin_wcscmp ||
314 ID == Builtin::BI__builtin_wcsncmp;
315
316 QualType ElemTy = getElemType(P: A);
317 // Different element types shouldn't happen, but with casts they can.
318 if (!S.getASTContext().hasSameUnqualifiedType(T1: ElemTy, T2: getElemType(P: B)))
319 return false;
320
321 PrimType ElemT = *S.getContext().classify(T: ElemTy);
322
323 auto returnResult = [&](int V) -> bool {
324 pushInteger(S, Val: V, QT: Call->getType());
325 return true;
326 };
327
328 unsigned IndexA = A.getIndex();
329 unsigned IndexB = B.getIndex();
330 unsigned NumElemsA = A.getNumElems();
331 unsigned NumElemsB = B.getNumElems();
332 uint64_t Steps = 0;
333 for (;; ++IndexA, ++IndexB, ++Steps) {
334
335 if (Steps >= Limit)
336 break;
337
338 // Diagnose this as a read of one-past-the-end.
339 if (IndexA >= NumElemsA || IndexB >= NumElemsB) {
340 S.FFDiag(SI: S.Current->getSource(PC: OpPC), DiagId: diag::note_constexpr_access_past_end)
341 << AK_Read << S.Current->getRange(PC: OpPC);
342 return false;
343 }
344
345 if (IsWide) {
346 INT_TYPE_SWITCH(ElemT, {
347 T CA = A.loadElem<T>(IndexA);
348 T CB = B.loadElem<T>(IndexB);
349 if (CA > CB)
350 return returnResult(1);
351 if (CA < CB)
352 return returnResult(-1);
353 if (CA.isZero() || CB.isZero())
354 return returnResult(0);
355 });
356 continue;
357 }
358
359 uint8_t CA = A.loadElem<uint8_t>(I: IndexA);
360 uint8_t CB = B.loadElem<uint8_t>(I: IndexB);
361
362 if (CA > CB)
363 return returnResult(1);
364 if (CA < CB)
365 return returnResult(-1);
366 if (CA == 0 || CB == 0)
367 return returnResult(0);
368 }
369
370 return returnResult(0);
371}
372
373static bool interp__builtin_strlen(InterpState &S, CodePtr OpPC,
374 const InterpFrame *Frame,
375 const CallExpr *Call, unsigned ID) {
376 const Pointer &StrPtr = S.Stk.pop<Pointer>().expand();
377
378 if (ID == Builtin::BIstrlen || ID == Builtin::BIwcslen)
379 diagnoseNonConstexprBuiltin(S, OpPC, ID);
380
381 if (StrPtr.isConstexprUnknown())
382 return false;
383
384 if (!CheckArray(S, OpPC, Ptr: StrPtr))
385 return false;
386
387 if (!CheckLive(S, OpPC, Ptr: StrPtr, AK: AK_Read))
388 return false;
389
390 // For string literal pointers, this is pretty simple.
391 if (StrPtr.isStringPointer()) {
392 if (StrPtr.isOnePastEnd())
393 return CheckRange(S, OpPC, Ptr: StrPtr, AK: AK_Read);
394
395 const auto *Lit = StrPtr.asStringPointer().getLiteral();
396 int64_t Off = StrPtr.getByteOffset();
397 if (Off < 0)
398 return false;
399
400 UnsignedOrNone ZeroIndex = Lit->findZeroCodeUnit(StartIndex: Off);
401 if (!ZeroIndex)
402 return false;
403 pushInteger(S, Val: *ZeroIndex, QT: Call->getType());
404 return true;
405 }
406
407 if (!StrPtr.isBlockPointer())
408 return false;
409
410 if (!CheckDummy(S, OpPC, B: StrPtr.block(), AK: AK_Read))
411 return false;
412
413 if (!StrPtr.getFieldDesc()->isPrimitiveArray())
414 return false;
415
416 assert(StrPtr.getFieldDesc()->isPrimitiveArray());
417 PrimType ElemT = StrPtr.getFieldDesc()->getPrimType();
418 unsigned ElemSize = StrPtr.getFieldDesc()->getElemDataSize();
419 if (ElemSize != 1 && ElemSize != 2 && ElemSize != 4)
420 return Invalid(S, OpPC);
421
422 if (ID == Builtin::BI__builtin_wcslen || ID == Builtin::BIwcslen) {
423 const ASTContext &AC = S.getASTContext();
424 unsigned WCharSize = AC.getTypeSizeInChars(T: AC.getWCharType()).getQuantity();
425 if (StrPtr.getFieldDesc()->getElemDataSize() != WCharSize)
426 return false;
427 }
428
429 size_t Len = 0;
430 for (size_t I = StrPtr.getIndex();; ++I, ++Len) {
431 PtrView ElemPtr = StrPtr.view().atIndex(Idx: I);
432
433 if (!CheckRange(S, OpPC, Ptr: ElemPtr, AK: AK_Read))
434 return false;
435
436 uint32_t Val;
437 FIXED_SIZE_INT_TYPE_SWITCH(
438 ElemT, { Val = static_cast<uint32_t>(ElemPtr.deref<T>()); });
439 if (Val == 0)
440 break;
441 }
442
443 pushInteger(S, Val: Len, QT: Call->getType());
444
445 return true;
446}
447
448static bool interp__builtin_nan(InterpState &S, CodePtr OpPC,
449 const InterpFrame *Frame, const CallExpr *Call,
450 bool Signaling) {
451 const Pointer &Arg = S.Stk.pop<Pointer>();
452
453 if (!CheckLoad(S, OpPC, Ptr: Arg))
454 return false;
455
456 // Convert the given string to an integer using StringRef's API.
457 llvm::APInt Fill;
458 if (Arg.isBlockPointer()) {
459 if (!Arg.getFieldDesc()->isPrimitiveArray())
460 return Invalid(S, OpPC);
461
462 std::string Str;
463 unsigned ArgLength = Arg.getNumElems();
464 bool FoundZero = false;
465 for (unsigned I = 0; I != ArgLength; ++I) {
466 if (!Arg.isElementInitialized(Index: I))
467 return false;
468
469 if (Arg.loadElem<int8_t>(I) == 0) {
470 FoundZero = true;
471 break;
472 }
473 Str += Arg.elem<char>(I);
474 }
475
476 // If we didn't find a NUL byte, diagnose as a one-past-the-end read.
477 if (!FoundZero)
478 return CheckRange(S, OpPC, Ptr: Arg.atIndex(Idx: ArgLength), AK: AK_Read);
479
480 // Treat empty strings as if they were zero.
481 if (Str.empty())
482 Fill = llvm::APInt(32, 0);
483 else if (StringRef(Str).getAsInteger(Radix: 0, Result&: Fill))
484 return false;
485 } else if (Arg.isStringPointer()) {
486 if (!Arg.asStringPointer().getLiteral()->isOrdinary())
487 return false;
488 StringRef Str = Arg.asStringPointer().getLiteral()->getString();
489 // Treat empty strings as if they were zero.
490 if (Str.empty())
491 Fill = llvm::APInt(32, 0);
492 else if (StringRef(Str).getAsInteger(Radix: 0, Result&: Fill))
493 return false;
494 } else {
495 return false;
496 }
497
498 const llvm::fltSemantics &TargetSemantics =
499 S.getASTContext().getFloatTypeSemantics(
500 T: Call->getDirectCallee()->getReturnType());
501
502 Floating Result = S.allocFloat(Sem: TargetSemantics);
503 if (S.getASTContext().getTargetInfo().isNan2008()) {
504 if (Signaling)
505 Result.copy(
506 F: llvm::APFloat::getSNaN(Sem: TargetSemantics, /*Negative=*/false, payload: &Fill));
507 else
508 Result.copy(
509 F: llvm::APFloat::getQNaN(Sem: TargetSemantics, /*Negative=*/false, payload: &Fill));
510 } else {
511 // Prior to IEEE 754-2008, architectures were allowed to choose whether
512 // the first bit of their significand was set for qNaN or sNaN. MIPS chose
513 // a different encoding to what became a standard in 2008, and for pre-
514 // 2008 revisions, MIPS interpreted sNaN-2008 as qNan and qNaN-2008 as
515 // sNaN. This is now known as "legacy NaN" encoding.
516 if (Signaling)
517 Result.copy(
518 F: llvm::APFloat::getQNaN(Sem: TargetSemantics, /*Negative=*/false, payload: &Fill));
519 else
520 Result.copy(
521 F: llvm::APFloat::getSNaN(Sem: TargetSemantics, /*Negative=*/false, payload: &Fill));
522 }
523
524 S.Stk.push<Floating>(Args&: Result);
525 return true;
526}
527
528static bool interp__builtin_inf(InterpState &S, CodePtr OpPC,
529 const InterpFrame *Frame,
530 const CallExpr *Call) {
531 const llvm::fltSemantics &TargetSemantics =
532 S.getASTContext().getFloatTypeSemantics(
533 T: Call->getDirectCallee()->getReturnType());
534
535 Floating Result = S.allocFloat(Sem: TargetSemantics);
536 Result.copy(F: APFloat::getInf(Sem: TargetSemantics));
537 S.Stk.push<Floating>(Args&: Result);
538 return true;
539}
540
541static bool interp__builtin_copysign(InterpState &S, CodePtr OpPC,
542 const InterpFrame *Frame) {
543 const Floating &Arg2 = S.Stk.pop<Floating>();
544 const Floating &Arg1 = S.Stk.pop<Floating>();
545 Floating Result = S.allocFloat(Sem: Arg1.getSemantics());
546
547 APFloat Copy = Arg1.getAPFloat();
548 Copy.copySign(RHS: Arg2.getAPFloat());
549 Result.copy(F: Copy);
550 S.Stk.push<Floating>(Args&: Result);
551
552 return true;
553}
554
555static bool interp__builtin_fmin(InterpState &S, CodePtr OpPC,
556 const InterpFrame *Frame, bool IsNumBuiltin) {
557 const Floating &RHS = S.Stk.pop<Floating>();
558 const Floating &LHS = S.Stk.pop<Floating>();
559 Floating Result = S.allocFloat(Sem: LHS.getSemantics());
560
561 if (IsNumBuiltin)
562 Result.copy(F: llvm::minimumnum(A: LHS.getAPFloat(), B: RHS.getAPFloat()));
563 else
564 Result.copy(F: minnum(A: LHS.getAPFloat(), B: RHS.getAPFloat()));
565 S.Stk.push<Floating>(Args&: Result);
566 return true;
567}
568
569static bool interp__builtin_fmax(InterpState &S, CodePtr OpPC,
570 const InterpFrame *Frame, bool IsNumBuiltin) {
571 const Floating &RHS = S.Stk.pop<Floating>();
572 const Floating &LHS = S.Stk.pop<Floating>();
573 Floating Result = S.allocFloat(Sem: LHS.getSemantics());
574
575 if (IsNumBuiltin)
576 Result.copy(F: llvm::maximumnum(A: LHS.getAPFloat(), B: RHS.getAPFloat()));
577 else
578 Result.copy(F: maxnum(A: LHS.getAPFloat(), B: RHS.getAPFloat()));
579 S.Stk.push<Floating>(Args&: Result);
580 return true;
581}
582
583/// Defined as __builtin_isnan(...), to accommodate the fact that it can
584/// take a float, double, long double, etc.
585/// But for us, that's all a Floating anyway.
586static bool interp__builtin_isnan(InterpState &S, CodePtr OpPC,
587 const InterpFrame *Frame,
588 const CallExpr *Call) {
589 const Floating &Arg = S.Stk.pop<Floating>();
590
591 pushInteger(S, Val: Arg.isNan(), QT: Call->getType());
592 return true;
593}
594
595static bool interp__builtin_issignaling(InterpState &S, CodePtr OpPC,
596 const InterpFrame *Frame,
597 const CallExpr *Call) {
598 const Floating &Arg = S.Stk.pop<Floating>();
599
600 pushInteger(S, Val: Arg.isSignaling(), QT: Call->getType());
601 return true;
602}
603
604static bool interp__builtin_isinf(InterpState &S, CodePtr OpPC,
605 const InterpFrame *Frame, bool CheckSign,
606 const CallExpr *Call) {
607 const Floating &Arg = S.Stk.pop<Floating>();
608 APFloat F = Arg.getAPFloat();
609 bool IsInf = F.isInfinity();
610
611 if (CheckSign)
612 pushInteger(S, Val: IsInf ? (F.isNegative() ? -1 : 1) : 0, QT: Call->getType());
613 else
614 pushInteger(S, Val: IsInf, QT: Call->getType());
615 return true;
616}
617
618static bool interp__builtin_isfinite(InterpState &S, CodePtr OpPC,
619 const InterpFrame *Frame,
620 const CallExpr *Call) {
621 const Floating &Arg = S.Stk.pop<Floating>();
622
623 pushInteger(S, Val: Arg.isFinite(), QT: Call->getType());
624 return true;
625}
626
627static bool interp__builtin_isnormal(InterpState &S, CodePtr OpPC,
628 const InterpFrame *Frame,
629 const CallExpr *Call) {
630 const Floating &Arg = S.Stk.pop<Floating>();
631
632 pushInteger(S, Val: Arg.isNormal(), QT: Call->getType());
633 return true;
634}
635
636static bool interp__builtin_issubnormal(InterpState &S, CodePtr OpPC,
637 const InterpFrame *Frame,
638 const CallExpr *Call) {
639 const Floating &Arg = S.Stk.pop<Floating>();
640
641 pushInteger(S, Val: Arg.isDenormal(), QT: Call->getType());
642 return true;
643}
644
645static bool interp__builtin_iszero(InterpState &S, CodePtr OpPC,
646 const InterpFrame *Frame,
647 const CallExpr *Call) {
648 const Floating &Arg = S.Stk.pop<Floating>();
649
650 pushInteger(S, Val: Arg.isZero(), QT: Call->getType());
651 return true;
652}
653
654static bool interp__builtin_signbit(InterpState &S, CodePtr OpPC,
655 const InterpFrame *Frame,
656 const CallExpr *Call) {
657 const Floating &Arg = S.Stk.pop<Floating>();
658
659 pushInteger(S, Val: Arg.isNegative(), QT: Call->getType());
660 return true;
661}
662
663static bool interp_floating_comparison(InterpState &S, CodePtr OpPC,
664 const CallExpr *Call, unsigned ID) {
665 const Floating &RHS = S.Stk.pop<Floating>();
666 const Floating &LHS = S.Stk.pop<Floating>();
667
668 pushInteger(
669 S,
670 Val: [&] {
671 switch (ID) {
672 case Builtin::BI__builtin_isgreater:
673 return LHS > RHS;
674 case Builtin::BI__builtin_isgreaterequal:
675 return LHS >= RHS;
676 case Builtin::BI__builtin_isless:
677 return LHS < RHS;
678 case Builtin::BI__builtin_islessequal:
679 return LHS <= RHS;
680 case Builtin::BI__builtin_islessgreater: {
681 ComparisonCategoryResult Cmp = LHS.compare(RHS);
682 return Cmp == ComparisonCategoryResult::Less ||
683 Cmp == ComparisonCategoryResult::Greater;
684 }
685 case Builtin::BI__builtin_isunordered:
686 return LHS.compare(RHS) == ComparisonCategoryResult::Unordered;
687 default:
688 llvm_unreachable("Unexpected builtin ID: Should be a floating point "
689 "comparison function");
690 }
691 }(),
692 QT: Call->getType());
693 return true;
694}
695
696/// First parameter to __builtin_isfpclass is the floating value, the
697/// second one is an integral value.
698static bool interp__builtin_isfpclass(InterpState &S, CodePtr OpPC,
699 const InterpFrame *Frame,
700 const CallExpr *Call) {
701 APSInt FPClassArg;
702 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: FPClassArg))
703 return false;
704 const Floating &F = S.Stk.pop<Floating>();
705
706 int32_t Result = static_cast<int32_t>(
707 (F.classify() & std::move(FPClassArg)).getZExtValue());
708 pushInteger(S, Val: Result, QT: Call->getType());
709
710 return true;
711}
712
713/// Five int values followed by one floating value.
714/// __builtin_fpclassify(int, int, int, int, int, float)
715static bool interp__builtin_fpclassify(InterpState &S, CodePtr OpPC,
716 const InterpFrame *Frame,
717 const CallExpr *Call) {
718 const Floating &Val = S.Stk.pop<Floating>();
719
720 PrimType IntT = *S.getContext().classify(E: Call->getArg(Arg: 0));
721 APSInt Values[5];
722 for (unsigned I = 0; I != 5; ++I) {
723 if (!popToAPSInt(Stk&: S.Stk, T: IntT, Out&: Values[4 - I]))
724 return false;
725 }
726
727 unsigned Index;
728 switch (Val.getCategory()) {
729 case APFloat::fcNaN:
730 Index = 0;
731 break;
732 case APFloat::fcInfinity:
733 Index = 1;
734 break;
735 case APFloat::fcNormal:
736 Index = Val.isDenormal() ? 3 : 2;
737 break;
738 case APFloat::fcZero:
739 Index = 4;
740 break;
741 }
742
743 // The last argument is first on the stack.
744 assert(Index <= 4);
745
746 pushInteger(S, Val: Values[Index], QT: Call->getType());
747 return true;
748}
749
750static inline Floating abs(InterpState &S, const Floating &In) {
751 if (!In.isNegative())
752 return In;
753
754 Floating Output = S.allocFloat(Sem: In.getSemantics());
755 APFloat New = In.getAPFloat();
756 New.changeSign();
757 Output.copy(F: New);
758 return Output;
759}
760
761// The C standard says "fabs raises no floating-point exceptions,
762// even if x is a signaling NaN. The returned value is independent of
763// the current rounding direction mode." Therefore constant folding can
764// proceed without regard to the floating point settings.
765// Reference, WG14 N2478 F.10.4.3
766static bool interp__builtin_fabs(InterpState &S, CodePtr OpPC,
767 const InterpFrame *Frame) {
768 const Floating &Val = S.Stk.pop<Floating>();
769 S.Stk.push<Floating>(Args: abs(S, In: Val));
770 return true;
771}
772
773static bool interp__builtin_abs(InterpState &S, CodePtr OpPC,
774 const InterpFrame *Frame,
775 const CallExpr *Call) {
776 APSInt Val;
777 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: Val))
778 return false;
779 if (Val ==
780 APSInt(APInt::getSignedMinValue(numBits: Val.getBitWidth()), /*IsUnsigned=*/false))
781 return false;
782 if (Val.isNegative())
783 Val.negate();
784 pushInteger(S, Val, QT: Call->getType());
785 return true;
786}
787
788static bool interp__builtin_popcount(InterpState &S, CodePtr OpPC,
789 const InterpFrame *Frame,
790 const CallExpr *Call) {
791 APSInt Val;
792 if (Call->getArg(Arg: 0)->getType()->isExtVectorBoolType()) {
793 const Pointer &Arg = S.Stk.pop<Pointer>();
794 Val = convertBoolVectorToInt(Val: Arg);
795 } else {
796 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: Val))
797 return false;
798 }
799 pushInteger(S, Val: Val.popcount(), QT: Call->getType());
800 return true;
801}
802
803static bool interp__builtin_ia32_crc32(InterpState &S, CodePtr OpPC,
804 const InterpFrame *Frame,
805 const CallExpr *Call,
806 unsigned DataBytes) {
807 uint64_t DataVal;
808 if (!popToUInt64(S, E: Call->getArg(Arg: 1), Out&: DataVal))
809 return false;
810 uint64_t CRCVal;
811 if (!popToUInt64(S, E: Call->getArg(Arg: 0), Out&: CRCVal))
812 return false;
813
814 // CRC32C polynomial (iSCSI polynomial, bit-reversed)
815 static const uint32_t CRC32C_POLY = 0x82F63B78;
816
817 // Process each byte
818 uint32_t Result = static_cast<uint32_t>(CRCVal);
819 for (unsigned I = 0; I != DataBytes; ++I) {
820 uint8_t Byte = static_cast<uint8_t>((DataVal >> (I * 8)) & 0xFF);
821 Result ^= Byte;
822 for (int J = 0; J != 8; ++J) {
823 Result = (Result >> 1) ^ ((Result & 1) ? CRC32C_POLY : 0);
824 }
825 }
826
827 pushInteger(S, Val: Result, QT: Call->getType());
828 return true;
829}
830
831static bool interp__builtin_classify_type(InterpState &S, CodePtr OpPC,
832 const InterpFrame *Frame,
833 const CallExpr *Call) {
834 // This is an unevaluated call, so there are no arguments on the stack.
835 assert(Call->getNumArgs() == 1);
836 const Expr *Arg = Call->getArg(Arg: 0);
837
838 GCCTypeClass ResultClass =
839 EvaluateBuiltinClassifyType(T: Arg->getType(), LangOpts: S.getLangOpts());
840 int32_t ReturnVal = static_cast<int32_t>(ResultClass);
841 pushInteger(S, Val: ReturnVal, QT: Call->getType());
842 return true;
843}
844
845// __builtin_expect(long, long)
846// __builtin_expect_with_probability(long, long, double)
847static bool interp__builtin_expect(InterpState &S, CodePtr OpPC,
848 const InterpFrame *Frame,
849 const CallExpr *Call) {
850 // The return value is simply the value of the first parameter.
851 // We ignore the probability.
852 unsigned NumArgs = Call->getNumArgs();
853 assert(NumArgs == 2 || NumArgs == 3);
854
855 PrimType ArgT = *S.getContext().classify(T: Call->getArg(Arg: 0)->getType());
856 if (NumArgs == 3)
857 S.Stk.discard<Floating>();
858 discard(Stk&: S.Stk, T: ArgT);
859
860 APSInt Val;
861 if (!popToAPSInt(Stk&: S.Stk, T: ArgT, Out&: Val))
862 return false;
863 pushInteger(S, Val, QT: Call->getType());
864 return true;
865}
866
867static bool interp__builtin_addressof(InterpState &S, CodePtr OpPC,
868 const InterpFrame *Frame,
869 const CallExpr *Call) {
870#ifndef NDEBUG
871 assert(Call->getArg(0)->isLValue());
872 PrimType PtrT = S.getContext().classify(Call->getArg(0)).value_or(PT_Ptr);
873 assert(PtrT == PT_Ptr &&
874 "Unsupported pointer type passed to __builtin_addressof()");
875#endif
876 return true;
877}
878
879static bool interp__builtin_move(InterpState &S, CodePtr OpPC,
880 const InterpFrame *Frame,
881 const CallExpr *Call) {
882 return Call->getDirectCallee()->isConstexpr();
883}
884
885static bool interp__builtin_eh_return_data_regno(InterpState &S, CodePtr OpPC,
886 const InterpFrame *Frame,
887 const CallExpr *Call) {
888 APSInt Arg;
889 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: Arg))
890 return false;
891
892 int Result = S.getASTContext().getTargetInfo().getEHDataRegisterNumber(
893 RegNo: Arg.getZExtValue());
894 pushInteger(S, Val: Result, QT: Call->getType());
895 return true;
896}
897
898// Two integral values followed by a pointer (lhs, rhs, resultOut)
899static bool interp__builtin_overflowop(InterpState &S, CodePtr OpPC,
900 const CallExpr *Call,
901 unsigned BuiltinOp) {
902 const Pointer &ResultPtr = S.Stk.pop<Pointer>();
903 if (ResultPtr.isDummy() || !ResultPtr.isBlockPointer())
904 return false;
905
906 PrimType RHST = *S.getContext().classify(T: Call->getArg(Arg: 1)->getType());
907 PrimType LHST = *S.getContext().classify(T: Call->getArg(Arg: 0)->getType());
908 APSInt RHS;
909 if (!popToAPSInt(Stk&: S.Stk, T: RHST, Out&: RHS))
910 return false;
911 APSInt LHS;
912 if (!popToAPSInt(Stk&: S.Stk, T: LHST, Out&: LHS))
913 return false;
914 QualType ResultType = Call->getArg(Arg: 2)->getType()->getPointeeType();
915 PrimType ResultT = *S.getContext().classify(T: ResultType);
916 bool Overflow;
917
918 APSInt Result;
919 if (BuiltinOp == Builtin::BI__builtin_add_overflow ||
920 BuiltinOp == Builtin::BI__builtin_sub_overflow ||
921 BuiltinOp == Builtin::BI__builtin_mul_overflow) {
922 bool IsSigned = LHS.isSigned() || RHS.isSigned() ||
923 ResultType->isSignedIntegerOrEnumerationType();
924 bool AllSigned = LHS.isSigned() && RHS.isSigned() &&
925 ResultType->isSignedIntegerOrEnumerationType();
926 uint64_t LHSSize = LHS.getBitWidth();
927 uint64_t RHSSize = RHS.getBitWidth();
928 uint64_t ResultSize = S.getASTContext().getIntWidth(T: ResultType);
929 uint64_t MaxBits = std::max(a: std::max(a: LHSSize, b: RHSSize), b: ResultSize);
930
931 // Add an additional bit if the signedness isn't uniformly agreed to. We
932 // could do this ONLY if there is a signed and an unsigned that both have
933 // MaxBits, but the code to check that is pretty nasty. The issue will be
934 // caught in the shrink-to-result later anyway.
935 if (IsSigned && !AllSigned)
936 ++MaxBits;
937
938 LHS = APSInt(LHS.extOrTrunc(width: MaxBits), !IsSigned);
939 RHS = APSInt(RHS.extOrTrunc(width: MaxBits), !IsSigned);
940 Result = APSInt(MaxBits, !IsSigned);
941 }
942
943 // Find largest int.
944 switch (BuiltinOp) {
945 default:
946 llvm_unreachable("Invalid value for BuiltinOp");
947 case Builtin::BI__builtin_add_overflow:
948 case Builtin::BI__builtin_sadd_overflow:
949 case Builtin::BI__builtin_saddl_overflow:
950 case Builtin::BI__builtin_saddll_overflow:
951 case Builtin::BI__builtin_uadd_overflow:
952 case Builtin::BI__builtin_uaddl_overflow:
953 case Builtin::BI__builtin_uaddll_overflow:
954 Result = LHS.isSigned() ? LHS.sadd_ov(RHS, Overflow)
955 : LHS.uadd_ov(RHS, Overflow);
956 break;
957 case Builtin::BI__builtin_sub_overflow:
958 case Builtin::BI__builtin_ssub_overflow:
959 case Builtin::BI__builtin_ssubl_overflow:
960 case Builtin::BI__builtin_ssubll_overflow:
961 case Builtin::BI__builtin_usub_overflow:
962 case Builtin::BI__builtin_usubl_overflow:
963 case Builtin::BI__builtin_usubll_overflow:
964 Result = LHS.isSigned() ? LHS.ssub_ov(RHS, Overflow)
965 : LHS.usub_ov(RHS, Overflow);
966 break;
967 case Builtin::BI__builtin_mul_overflow:
968 case Builtin::BI__builtin_smul_overflow:
969 case Builtin::BI__builtin_smull_overflow:
970 case Builtin::BI__builtin_smulll_overflow:
971 case Builtin::BI__builtin_umul_overflow:
972 case Builtin::BI__builtin_umull_overflow:
973 case Builtin::BI__builtin_umulll_overflow:
974 Result = LHS.isSigned() ? LHS.smul_ov(RHS, Overflow)
975 : LHS.umul_ov(RHS, Overflow);
976 break;
977 }
978
979 // In the case where multiple sizes are allowed, truncate and see if
980 // the values are the same.
981 if (BuiltinOp == Builtin::BI__builtin_add_overflow ||
982 BuiltinOp == Builtin::BI__builtin_sub_overflow ||
983 BuiltinOp == Builtin::BI__builtin_mul_overflow) {
984 // APSInt doesn't have a TruncOrSelf, so we use extOrTrunc instead,
985 // since it will give us the behavior of a TruncOrSelf in the case where
986 // its parameter <= its size. We previously set Result to be at least the
987 // integer width of the result, so getIntWidth(ResultType) <=
988 // Result.BitWidth
989 APSInt Temp = Result.extOrTrunc(width: S.getASTContext().getIntWidth(T: ResultType));
990 Temp.setIsSigned(ResultType->isSignedIntegerOrEnumerationType());
991
992 if (!APSInt::isSameValue(I1: Temp, I2: Result))
993 Overflow = true;
994 Result = std::move(Temp);
995 }
996
997 // Write Result to ResultPtr and put Overflow on the stack.
998 assignIntegral(S, Dest: ResultPtr, ValueT: ResultT, Value: Result);
999 if (ResultPtr.canBeInitialized())
1000 ResultPtr.initialize();
1001
1002 assert(Call->getDirectCallee()->getReturnType()->isBooleanType());
1003 S.Stk.push<Boolean>(Args&: Overflow);
1004 return true;
1005}
1006
1007/// Three integral values followed by a pointer (lhs, rhs, carry, carryOut).
1008static bool interp__builtin_carryop(InterpState &S, CodePtr OpPC,
1009 const InterpFrame *Frame,
1010 const CallExpr *Call, unsigned BuiltinOp) {
1011 const Pointer &CarryOutPtr = S.Stk.pop<Pointer>();
1012 PrimType LHST = *S.getContext().classify(T: Call->getArg(Arg: 0)->getType());
1013 PrimType RHST = *S.getContext().classify(T: Call->getArg(Arg: 1)->getType());
1014 APSInt CarryIn;
1015 if (!popToAPSInt(Stk&: S.Stk, T: LHST, Out&: CarryIn))
1016 return false;
1017 APSInt RHS;
1018 if (!popToAPSInt(Stk&: S.Stk, T: RHST, Out&: RHS))
1019 return false;
1020 APSInt LHS;
1021 if (!popToAPSInt(Stk&: S.Stk, T: LHST, Out&: LHS))
1022 return false;
1023
1024 if (!isReadable(P: CarryOutPtr))
1025 return false;
1026
1027 APSInt CarryOut;
1028
1029 APSInt Result;
1030 // Copy the number of bits and sign.
1031 Result = LHS;
1032 CarryOut = LHS;
1033
1034 bool FirstOverflowed = false;
1035 bool SecondOverflowed = false;
1036 switch (BuiltinOp) {
1037 default:
1038 llvm_unreachable("Invalid value for BuiltinOp");
1039 case Builtin::BI__builtin_addcb:
1040 case Builtin::BI__builtin_addcs:
1041 case Builtin::BI__builtin_addc:
1042 case Builtin::BI__builtin_addcl:
1043 case Builtin::BI__builtin_addcll:
1044 Result =
1045 LHS.uadd_ov(RHS, Overflow&: FirstOverflowed).uadd_ov(RHS: CarryIn, Overflow&: SecondOverflowed);
1046 break;
1047 case Builtin::BI__builtin_subcb:
1048 case Builtin::BI__builtin_subcs:
1049 case Builtin::BI__builtin_subc:
1050 case Builtin::BI__builtin_subcl:
1051 case Builtin::BI__builtin_subcll:
1052 Result =
1053 LHS.usub_ov(RHS, Overflow&: FirstOverflowed).usub_ov(RHS: CarryIn, Overflow&: SecondOverflowed);
1054 break;
1055 }
1056 // It is possible for both overflows to happen but CGBuiltin uses an OR so
1057 // this is consistent.
1058 CarryOut = (uint64_t)(FirstOverflowed | SecondOverflowed);
1059
1060 QualType CarryOutType = Call->getArg(Arg: 3)->getType()->getPointeeType();
1061 PrimType CarryOutT = *S.getContext().classify(T: CarryOutType);
1062 assignIntegral(S, Dest: CarryOutPtr, ValueT: CarryOutT, Value: CarryOut);
1063 if (CarryOutPtr.canBeInitialized())
1064 CarryOutPtr.initialize();
1065
1066 assert(S.getASTContext().hasSimilarType(Call->getType(),
1067 Call->getArg(0)->getType()));
1068 pushInteger(S, Val: Result, QT: Call->getType());
1069 return true;
1070}
1071
1072static bool interp__builtin_clz(InterpState &S, CodePtr OpPC,
1073 const InterpFrame *Frame, const CallExpr *Call,
1074 unsigned BuiltinOp) {
1075
1076 std::optional<APSInt> Fallback;
1077 if (BuiltinOp == Builtin::BI__builtin_clzg && Call->getNumArgs() == 2) {
1078 APSInt FallbackVal;
1079 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: FallbackVal))
1080 return false;
1081 Fallback = FallbackVal;
1082 }
1083
1084 APSInt Val;
1085 if (Call->getArg(Arg: 0)->getType()->isExtVectorBoolType()) {
1086 const Pointer &Arg = S.Stk.pop<Pointer>();
1087 Val = convertBoolVectorToInt(Val: Arg);
1088 } else {
1089 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: Val))
1090 return false;
1091 }
1092
1093 // When the argument is 0, the result of GCC builtins is undefined, whereas
1094 // for Microsoft intrinsics, the result is the bit-width of the argument.
1095 bool ZeroIsUndefined = BuiltinOp != Builtin::BI__lzcnt16 &&
1096 BuiltinOp != Builtin::BI__lzcnt &&
1097 BuiltinOp != Builtin::BI__lzcnt64;
1098
1099 if (Val == 0) {
1100 if (Fallback) {
1101 pushInteger(S, Val: *Fallback, QT: Call->getType());
1102 return true;
1103 }
1104
1105 if (ZeroIsUndefined)
1106 return false;
1107 }
1108
1109 pushInteger(S, Val: Val.countl_zero(), QT: Call->getType());
1110 return true;
1111}
1112
1113static bool interp__builtin_ctz(InterpState &S, CodePtr OpPC,
1114 const InterpFrame *Frame, const CallExpr *Call,
1115 unsigned BuiltinID) {
1116 std::optional<APSInt> Fallback;
1117 if (BuiltinID == Builtin::BI__builtin_ctzg && Call->getNumArgs() == 2) {
1118 APSInt FallbackVal;
1119 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: FallbackVal))
1120 return false;
1121 Fallback = FallbackVal;
1122 }
1123
1124 APSInt Val;
1125 if (Call->getArg(Arg: 0)->getType()->isExtVectorBoolType()) {
1126 const Pointer &Arg = S.Stk.pop<Pointer>();
1127 Val = convertBoolVectorToInt(Val: Arg);
1128 } else {
1129 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: Val))
1130 return false;
1131 }
1132
1133 if (Val == 0) {
1134 if (Fallback) {
1135 pushInteger(S, Val: *Fallback, QT: Call->getType());
1136 return true;
1137 }
1138 return false;
1139 }
1140
1141 pushInteger(S, Val: Val.countr_zero(), QT: Call->getType());
1142 return true;
1143}
1144
1145static bool interp__builtin_bswap(InterpState &S, CodePtr OpPC,
1146 const InterpFrame *Frame,
1147 const CallExpr *Call) {
1148 APSInt Val;
1149 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: Val))
1150 return false;
1151 if (Val.getBitWidth() == 8 || Val.getBitWidth() == 1)
1152 pushInteger(S, Val, QT: Call->getType());
1153 else
1154 pushInteger(S, Val: Val.byteSwap(), QT: Call->getType());
1155 return true;
1156}
1157
1158/// bool __atomic_always_lock_free(size_t, void const volatile*)
1159/// bool __atomic_is_lock_free(size_t, void const volatile*)
1160static bool interp__builtin_atomic_lock_free(InterpState &S, CodePtr OpPC,
1161 const InterpFrame *Frame,
1162 const CallExpr *Call,
1163 unsigned BuiltinOp) {
1164 auto returnBool = [&S](bool Value) -> bool {
1165 S.Stk.push<Boolean>(Args&: Value);
1166 return true;
1167 };
1168
1169 const Pointer &Ptr = S.Stk.pop<Pointer>();
1170 uint64_t SizeVal;
1171 if (!popToUInt64(S, E: Call->getArg(Arg: 0), Out&: SizeVal))
1172 return false;
1173
1174 // For __atomic_is_lock_free(sizeof(_Atomic(T))), if the size is a power
1175 // of two less than or equal to the maximum inline atomic width, we know it
1176 // is lock-free. If the size isn't a power of two, or greater than the
1177 // maximum alignment where we promote atomics, we know it is not lock-free
1178 // (at least not in the sense of atomic_is_lock_free). Otherwise,
1179 // the answer can only be determined at runtime; for example, 16-byte
1180 // atomics have lock-free implementations on some, but not all,
1181 // x86-64 processors.
1182
1183 // Check power-of-two.
1184 CharUnits Size = CharUnits::fromQuantity(Quantity: SizeVal);
1185 if (Size.isPowerOfTwo()) {
1186 // Check against inlining width.
1187 unsigned InlineWidthBits =
1188 S.getASTContext().getTargetInfo().getMaxAtomicInlineWidth();
1189 if (Size <= S.getASTContext().toCharUnitsFromBits(BitSize: InlineWidthBits)) {
1190
1191 // OK, we will inline appropriately-aligned operations of this size,
1192 // and _Atomic(T) is appropriately-aligned.
1193 if (Size == CharUnits::One())
1194 return returnBool(true);
1195
1196 // Same for null pointers.
1197 assert(BuiltinOp != Builtin::BI__c11_atomic_is_lock_free);
1198 if (Ptr.isZero())
1199 return returnBool(true);
1200
1201 if (Ptr.isIntegralPointer()) {
1202 uint64_t IntVal = Ptr.getIntegerRepresentation();
1203 if (APSInt(APInt(64, IntVal, false), true).isAligned(A: Size.getAsAlign()))
1204 return returnBool(true);
1205 }
1206
1207 const Expr *PtrArg = Call->getArg(Arg: 1);
1208 // Otherwise, check if the type's alignment against Size.
1209 if (const auto *ICE = dyn_cast<ImplicitCastExpr>(Val: PtrArg)) {
1210 // Drop the potential implicit-cast to 'const volatile void*', getting
1211 // the underlying type.
1212 if (ICE->getCastKind() == CK_BitCast)
1213 PtrArg = ICE->getSubExpr();
1214 }
1215
1216 if (const auto *PtrTy = PtrArg->getType()->getAs<PointerType>()) {
1217 QualType PointeeType = PtrTy->getPointeeType();
1218 if (!PointeeType->isIncompleteType() &&
1219 S.getASTContext().getTypeAlignInChars(T: PointeeType) >= Size) {
1220 // OK, we will inline operations on this object.
1221 return returnBool(true);
1222 }
1223 }
1224 }
1225 }
1226
1227 if (BuiltinOp == Builtin::BI__atomic_always_lock_free)
1228 return returnBool(false);
1229
1230 return Invalid(S, OpPC);
1231}
1232
1233/// bool __c11_atomic_is_lock_free(size_t)
1234static bool interp__builtin_c11_atomic_is_lock_free(InterpState &S,
1235 CodePtr OpPC,
1236 const InterpFrame *Frame,
1237 const CallExpr *Call) {
1238 uint64_t SizeVal;
1239 if (!popToUInt64(S, E: Call->getArg(Arg: 0), Out&: SizeVal))
1240 return false;
1241
1242 CharUnits Size = CharUnits::fromQuantity(Quantity: SizeVal);
1243 if (Size.isPowerOfTwo()) {
1244 // Check against inlining width.
1245 unsigned InlineWidthBits =
1246 S.getASTContext().getTargetInfo().getMaxAtomicInlineWidth();
1247 if (Size <= S.getASTContext().toCharUnitsFromBits(BitSize: InlineWidthBits)) {
1248 S.Stk.push<Boolean>(Args: true);
1249 return true;
1250 }
1251 }
1252
1253 return false; // returnBool(false);
1254}
1255
1256/// __builtin_complex(Float A, float B);
1257static bool interp__builtin_complex(InterpState &S, CodePtr OpPC,
1258 const InterpFrame *Frame,
1259 const CallExpr *Call) {
1260 const Floating &Arg2 = S.Stk.pop<Floating>();
1261 const Floating &Arg1 = S.Stk.pop<Floating>();
1262 Pointer &Result = S.Stk.peek<Pointer>();
1263
1264 Result.elem<Floating>(I: 0) = Arg1;
1265 Result.elem<Floating>(I: 1) = Arg2;
1266 Result.initializeAllElements();
1267
1268 return true;
1269}
1270
1271/// __builtin_is_aligned()
1272/// __builtin_align_up()
1273/// __builtin_align_down()
1274/// The first parameter is either an integer or a pointer.
1275/// The second parameter is the requested alignment as an integer.
1276static bool interp__builtin_is_aligned_up_down(InterpState &S, CodePtr OpPC,
1277 const InterpFrame *Frame,
1278 const CallExpr *Call,
1279 unsigned BuiltinOp) {
1280 APSInt Alignment;
1281 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: Alignment))
1282 return false;
1283
1284 if (Alignment < 0 || !Alignment.isPowerOf2()) {
1285 S.FFDiag(E: Call, DiagId: diag::note_constexpr_invalid_alignment) << Alignment;
1286 return false;
1287 }
1288 unsigned SrcWidth = S.getASTContext().getIntWidth(T: Call->getArg(Arg: 0)->getType());
1289 APSInt MaxValue(APInt::getOneBitSet(numBits: SrcWidth, BitNo: SrcWidth - 1));
1290 if (APSInt::compareValues(I1: Alignment, I2: MaxValue) > 0) {
1291 S.FFDiag(E: Call, DiagId: diag::note_constexpr_alignment_too_big)
1292 << MaxValue << Call->getArg(Arg: 0)->getType() << Alignment;
1293 return false;
1294 }
1295
1296 // The first parameter is either an integer or a pointer.
1297 PrimType FirstArgT = *S.Ctx.classify(E: Call->getArg(Arg: 0));
1298
1299 if (isIntegerType(T: FirstArgT)) {
1300 APSInt Src;
1301 if (!popToAPSInt(Stk&: S.Stk, T: FirstArgT, Out&: Src))
1302 return false;
1303 APInt AlignMinusOne = Alignment.extOrTrunc(width: Src.getBitWidth()) - 1;
1304 if (BuiltinOp == Builtin::BI__builtin_align_up) {
1305 APSInt AlignedVal =
1306 APSInt((Src + AlignMinusOne) & ~AlignMinusOne, Src.isUnsigned());
1307 pushInteger(S, Val: AlignedVal, QT: Call->getType());
1308 } else if (BuiltinOp == Builtin::BI__builtin_align_down) {
1309 APSInt AlignedVal = APSInt(Src & ~AlignMinusOne, Src.isUnsigned());
1310 pushInteger(S, Val: AlignedVal, QT: Call->getType());
1311 } else {
1312 assert(*S.Ctx.classify(Call->getType()) == PT_Bool);
1313 S.Stk.push<Boolean>(Args: (Src & AlignMinusOne) == 0);
1314 }
1315 return true;
1316 }
1317 assert(FirstArgT == PT_Ptr);
1318 const Pointer &Ptr = S.Stk.pop<Pointer>();
1319 if (!Ptr.isBlockPointer()) {
1320 S.FFDiag(E: Call->getArg(Arg: 0), DiagId: diag::note_constexpr_alignment_compute)
1321 << Alignment;
1322 return false;
1323 }
1324
1325 const ValueDecl *PtrDecl = Ptr.getDeclDesc()->asValueDecl();
1326 // We need a pointer for a declaration here.
1327 if (!PtrDecl) {
1328 if (BuiltinOp == Builtin::BI__builtin_is_aligned)
1329 S.FFDiag(E: Call->getArg(Arg: 0), DiagId: diag::note_constexpr_alignment_compute)
1330 << Alignment;
1331 else
1332 S.FFDiag(E: Call->getArg(Arg: 0), DiagId: diag::note_constexpr_alignment_adjust)
1333 << Alignment;
1334 return false;
1335 }
1336
1337 // For one-past-end pointers, we can't call getIndex() since it asserts.
1338 // Use getNumElems() instead which gives the correct index for past-end.
1339 unsigned PtrOffset =
1340 Ptr.isElementPastEnd() ? Ptr.getNumElems() : Ptr.getIndex();
1341 CharUnits BaseAlignment = S.getASTContext().getDeclAlign(D: PtrDecl);
1342 CharUnits PtrAlign =
1343 BaseAlignment.alignmentAtOffset(offset: CharUnits::fromQuantity(Quantity: PtrOffset));
1344
1345 if (BuiltinOp == Builtin::BI__builtin_is_aligned) {
1346 if (PtrAlign.getQuantity() >= Alignment) {
1347 S.Stk.push<Boolean>(Args: true);
1348 return true;
1349 }
1350 // If the alignment is not known to be sufficient, some cases could still
1351 // be aligned at run time. However, if the requested alignment is less or
1352 // equal to the base alignment and the offset is not aligned, we know that
1353 // the run-time value can never be aligned.
1354 if (BaseAlignment.getQuantity() >= Alignment &&
1355 PtrAlign.getQuantity() < Alignment) {
1356 S.Stk.push<Boolean>(Args: false);
1357 return true;
1358 }
1359
1360 S.FFDiag(E: Call->getArg(Arg: 0), DiagId: diag::note_constexpr_alignment_compute)
1361 << Alignment;
1362 return false;
1363 }
1364
1365 assert(BuiltinOp == Builtin::BI__builtin_align_down ||
1366 BuiltinOp == Builtin::BI__builtin_align_up);
1367
1368 // For align_up/align_down, we can return the same value if the alignment
1369 // is known to be greater or equal to the requested value.
1370 if (PtrAlign.getQuantity() >= Alignment) {
1371 S.Stk.push<Pointer>(Args: Ptr);
1372 return true;
1373 }
1374
1375 // The alignment could be greater than the minimum at run-time, so we cannot
1376 // infer much about the resulting pointer value. One case is possible:
1377 // For `_Alignas(32) char buf[N]; __builtin_align_down(&buf[idx], 32)` we
1378 // can infer the correct index if the requested alignment is smaller than
1379 // the base alignment so we can perform the computation on the offset.
1380 if (BaseAlignment.getQuantity() >= Alignment) {
1381 assert(Alignment.getBitWidth() <= 64 &&
1382 "Cannot handle > 64-bit address-space");
1383 uint64_t Alignment64 = Alignment.getZExtValue();
1384 CharUnits NewOffset =
1385 CharUnits::fromQuantity(Quantity: BuiltinOp == Builtin::BI__builtin_align_down
1386 ? llvm::alignDown(Value: PtrOffset, Align: Alignment64)
1387 : llvm::alignTo(Value: PtrOffset, Align: Alignment64));
1388
1389 S.Stk.push<Pointer>(Args: Ptr.atIndex(Idx: NewOffset.getQuantity()));
1390 return true;
1391 }
1392
1393 // Otherwise, we cannot constant-evaluate the result.
1394 S.FFDiag(E: Call->getArg(Arg: 0), DiagId: diag::note_constexpr_alignment_adjust) << Alignment;
1395 return false;
1396}
1397
1398/// __builtin_assume_aligned(Ptr, Alignment[, ExtraOffset])
1399static bool interp__builtin_assume_aligned(InterpState &S, CodePtr OpPC,
1400 const InterpFrame *Frame,
1401 const CallExpr *Call) {
1402 assert(Call->getNumArgs() == 2 || Call->getNumArgs() == 3);
1403
1404 std::optional<APSInt> ExtraOffset;
1405 if (Call->getNumArgs() == 3) {
1406 APSInt ExtraOffsetVal;
1407 if (!popToAPSInt(Stk&: S.Stk, T: *S.Ctx.classify(E: Call->getArg(Arg: 2)), Out&: ExtraOffsetVal))
1408 return false;
1409 ExtraOffset = ExtraOffsetVal;
1410 }
1411
1412 APSInt Alignment;
1413 if (!popToAPSInt(Stk&: S.Stk, T: *S.Ctx.classify(E: Call->getArg(Arg: 1)), Out&: Alignment))
1414 return false;
1415 const Pointer &Ptr = S.Stk.pop<Pointer>();
1416
1417 const ASTContext &ASTCtx = S.getASTContext();
1418 CharUnits Align = CharUnits::fromQuantity(Quantity: Alignment.getZExtValue());
1419
1420 // If there is a base object, then it must have the correct alignment.
1421 if (Ptr.isBlockPointer()) {
1422 CharUnits BaseAlignment;
1423 if (const auto *VD = Ptr.getDeclDesc()->asValueDecl())
1424 BaseAlignment = ASTCtx.getDeclAlign(D: VD);
1425 else if (const auto *E = Ptr.getRootExpr())
1426 BaseAlignment = GetAlignOfExpr(Ctx: ASTCtx, E, ExprKind: UETT_AlignOf);
1427
1428 if (BaseAlignment < Align) {
1429 S.CCEDiag(E: Call->getArg(Arg: 0),
1430 DiagId: diag::note_constexpr_baa_insufficient_alignment)
1431 << 0 << BaseAlignment.getQuantity() << Align.getQuantity();
1432 return false;
1433 }
1434 }
1435
1436 std::optional<size_t> LayoutOffset = Ptr.computeLayoutOffset(ASTCtx);
1437 if (!LayoutOffset)
1438 return false;
1439
1440 CharUnits AVOffset = CharUnits::fromQuantity(Quantity: *LayoutOffset);
1441 if (ExtraOffset)
1442 AVOffset -= CharUnits::fromQuantity(Quantity: ExtraOffset->getZExtValue());
1443 if (AVOffset.alignTo(Align) != AVOffset) {
1444 if (Ptr.isBlockPointer())
1445 S.CCEDiag(E: Call->getArg(Arg: 0),
1446 DiagId: diag::note_constexpr_baa_insufficient_alignment)
1447 << 1 << AVOffset.getQuantity() << Align.getQuantity();
1448 else
1449 S.CCEDiag(E: Call->getArg(Arg: 0),
1450 DiagId: diag::note_constexpr_baa_value_insufficient_alignment)
1451 << AVOffset.getQuantity() << Align.getQuantity();
1452 return false;
1453 }
1454
1455 S.Stk.push<Pointer>(Args: Ptr);
1456 return true;
1457}
1458
1459/// (CarryIn, LHS, RHS, Result)
1460static bool interp__builtin_ia32_addcarry_subborrow(InterpState &S,
1461 CodePtr OpPC,
1462 const InterpFrame *Frame,
1463 const CallExpr *Call,
1464 bool IsAdd) {
1465 if (Call->getNumArgs() != 4 || !Call->getArg(Arg: 0)->getType()->isIntegerType() ||
1466 !Call->getArg(Arg: 1)->getType()->isIntegerType() ||
1467 !Call->getArg(Arg: 2)->getType()->isIntegerType())
1468 return false;
1469
1470 const Pointer &CarryOutPtr = S.Stk.pop<Pointer>();
1471
1472 APSInt RHS;
1473 if (!popToAPSInt(S, E: Call->getArg(Arg: 2), Out&: RHS))
1474 return false;
1475 APSInt LHS;
1476 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: LHS))
1477 return false;
1478 APSInt CarryIn;
1479 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: CarryIn))
1480 return false;
1481
1482 unsigned BitWidth = LHS.getBitWidth();
1483 unsigned CarryInBit = CarryIn.ugt(RHS: 0) ? 1 : 0;
1484 APInt ExResult =
1485 IsAdd ? (LHS.zext(width: BitWidth + 1) + (RHS.zext(width: BitWidth + 1) + CarryInBit))
1486 : (LHS.zext(width: BitWidth + 1) - (RHS.zext(width: BitWidth + 1) + CarryInBit));
1487
1488 APInt Result = ExResult.extractBits(numBits: BitWidth, bitPosition: 0);
1489 APSInt CarryOut =
1490 APSInt(ExResult.extractBits(numBits: 1, bitPosition: BitWidth), /*IsUnsigned=*/true);
1491
1492 QualType CarryOutType = Call->getArg(Arg: 3)->getType()->getPointeeType();
1493 PrimType CarryOutT = *S.getContext().classify(T: CarryOutType);
1494 assignIntegral(S, Dest: CarryOutPtr, ValueT: CarryOutT, Value: APSInt(std::move(Result), true));
1495
1496 pushInteger(S, Val: CarryOut, QT: Call->getType());
1497
1498 return true;
1499}
1500
1501static bool interp__builtin_os_log_format_buffer_size(InterpState &S,
1502 CodePtr OpPC,
1503 const InterpFrame *Frame,
1504 const CallExpr *Call) {
1505 analyze_os_log::OSLogBufferLayout Layout;
1506 analyze_os_log::computeOSLogBufferLayout(Ctx&: S.getASTContext(), E: Call, layout&: Layout);
1507 pushInteger(S, Val: Layout.size().getQuantity(), QT: Call->getType());
1508 return true;
1509}
1510
1511static bool
1512interp__builtin_ptrauth_string_discriminator(InterpState &S, CodePtr OpPC,
1513 const InterpFrame *Frame,
1514 const CallExpr *Call) {
1515 const auto &Ptr = S.Stk.pop<Pointer>();
1516 if (!Ptr.isStringPointer())
1517 return false;
1518
1519 uint64_t Result = getPointerAuthStableSipHash(
1520 S: cast<StringLiteral>(Val: Ptr.getRootExpr())->getString());
1521 pushInteger(S, Val: Result, QT: Call->getType());
1522 return true;
1523}
1524
1525static bool interp__builtin_infer_alloc_token(InterpState &S, CodePtr OpPC,
1526 const InterpFrame *Frame,
1527 const CallExpr *Call) {
1528 const ASTContext &ASTCtx = S.getASTContext();
1529 uint64_t BitWidth = ASTCtx.getTypeSize(T: ASTCtx.getSizeType());
1530 auto Mode =
1531 ASTCtx.getLangOpts().AllocTokenMode.value_or(u: llvm::DefaultAllocTokenMode);
1532 auto MaxTokensOpt = ASTCtx.getLangOpts().AllocTokenMax;
1533 uint64_t MaxTokens =
1534 MaxTokensOpt.value_or(u: 0) ? *MaxTokensOpt : (~0ULL >> (64 - BitWidth));
1535
1536 // We do not read any of the arguments; discard them.
1537 for (int I = Call->getNumArgs() - 1; I >= 0; --I)
1538 discard(Stk&: S.Stk, T: S.getContext().classify(E: Call->getArg(Arg: I)).value_or(PT: PT_Ptr));
1539
1540 // Note: Type inference from a surrounding cast is not supported in
1541 // constexpr evaluation.
1542 QualType AllocType = infer_alloc::inferPossibleType(E: Call, Ctx: ASTCtx, CastE: nullptr);
1543 if (AllocType.isNull()) {
1544 S.CCEDiag(E: Call,
1545 DiagId: diag::note_constexpr_infer_alloc_token_type_inference_failed);
1546 return false;
1547 }
1548
1549 auto ATMD = infer_alloc::getAllocTokenMetadata(T: AllocType, Ctx: ASTCtx);
1550 if (!ATMD) {
1551 S.CCEDiag(E: Call, DiagId: diag::note_constexpr_infer_alloc_token_no_metadata);
1552 return false;
1553 }
1554
1555 auto MaybeToken = llvm::getAllocToken(Mode, Metadata: *ATMD, MaxTokens);
1556 if (!MaybeToken) {
1557 S.CCEDiag(E: Call, DiagId: diag::note_constexpr_infer_alloc_token_stateful_mode);
1558 return false;
1559 }
1560
1561 pushInteger(S, Val: llvm::APInt(BitWidth, *MaybeToken), QT: ASTCtx.getSizeType());
1562 return true;
1563}
1564
1565static bool interp__builtin_operator_new(InterpState &S, CodePtr OpPC,
1566 const InterpFrame *Frame,
1567 const CallExpr *Call) {
1568 // A call to __operator_new is only valid within std::allocate<>::allocate.
1569 // Walk up the call stack to find the appropriate caller and get the
1570 // element type from it.
1571 auto [NewCall, ElemType] = S.getStdAllocatorCaller(Name: "allocate");
1572
1573 if (ElemType.isNull()) {
1574 S.FFDiag(E: Call, DiagId: S.getLangOpts().CPlusPlus20
1575 ? diag::note_constexpr_new_untyped
1576 : diag::note_constexpr_new);
1577 return false;
1578 }
1579 assert(NewCall);
1580
1581 if (ElemType->isIncompleteType() || ElemType->isFunctionType()) {
1582 S.FFDiag(E: Call, DiagId: diag::note_constexpr_new_not_complete_object_type)
1583 << (ElemType->isIncompleteType() ? 0 : 1) << ElemType;
1584 return false;
1585 }
1586
1587 // We only care about the first parameter (the size), so discard all the
1588 // others.
1589 {
1590 unsigned NumArgs = Call->getNumArgs();
1591 assert(NumArgs >= 1);
1592
1593 // The std::nothrow_t arg never gets put on the stack.
1594 if (Call->getArg(Arg: NumArgs - 1)->getType()->isNothrowT())
1595 --NumArgs;
1596 auto Args = ArrayRef(Call->getArgs(), Call->getNumArgs());
1597 // First arg is needed.
1598 Args = Args.drop_front();
1599
1600 // Discard the rest.
1601 for (const Expr *Arg : Args)
1602 discard(Stk&: S.Stk, T: *S.getContext().classify(E: Arg));
1603 }
1604
1605 APSInt Bytes;
1606 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: Bytes))
1607 return false;
1608 CharUnits ElemSize = S.getASTContext().getTypeSizeInChars(T: ElemType);
1609 assert(!ElemSize.isZero());
1610 // Divide the number of bytes by sizeof(ElemType), so we get the number of
1611 // elements we should allocate.
1612 APInt NumElems, Remainder;
1613 APInt ElemSizeAP(Bytes.getBitWidth(), ElemSize.getQuantity());
1614 APInt::udivrem(LHS: Bytes, RHS: ElemSizeAP, Quotient&: NumElems, Remainder);
1615 if (Remainder != 0) {
1616 // This likely indicates a bug in the implementation of 'std::allocator'.
1617 S.FFDiag(E: Call, DiagId: diag::note_constexpr_operator_new_bad_size)
1618 << Bytes << APSInt(ElemSizeAP, true) << ElemType;
1619 return false;
1620 }
1621
1622 // NB: The same check we're using in CheckArraySize()
1623 if (NumElems.getActiveBits() >
1624 ConstantArrayType::getMaxSizeBits(Context: S.getASTContext()) ||
1625 NumElems.ugt(RHS: Descriptor::MaxArrayElemBytes / ElemSize.getQuantity())) {
1626 // FIXME: NoThrow check?
1627 const SourceInfo &Loc = S.Current->getSource(PC: OpPC);
1628 S.FFDiag(SI: Loc, DiagId: diag::note_constexpr_new_too_large)
1629 << NumElems.getZExtValue();
1630 return false;
1631 }
1632
1633 if (!CheckArraySize(S, OpPC, NumElems: NumElems.getZExtValue()))
1634 return false;
1635
1636 bool IsArray = NumElems.ugt(RHS: 1);
1637 OptPrimType ElemT = S.getContext().classify(T: ElemType);
1638 DynamicAllocator &Allocator = S.getAllocator();
1639 if (ElemT) {
1640 Block *B =
1641 Allocator.allocate(Source: NewCall, T: *ElemT, NumElements: NumElems.getZExtValue(),
1642 EvalID: S.Ctx.getEvalID(), AllocForm: DynamicAllocator::Form::Operator);
1643 assert(B);
1644 S.Stk.push<Pointer>(Args: Pointer(B).atIndex(Idx: 0));
1645 return true;
1646 }
1647
1648 assert(!ElemT);
1649
1650 // Composite arrays
1651 if (IsArray) {
1652 const Descriptor *Desc =
1653 S.P.createDescriptor(D: NewCall, Ty: ElemType.getTypePtr());
1654 Block *B =
1655 Allocator.allocate(D: Desc, NumElements: NumElems.getZExtValue(), EvalID: S.Ctx.getEvalID(),
1656 AllocForm: DynamicAllocator::Form::Operator);
1657 assert(B);
1658 S.Stk.push<Pointer>(Args: Pointer(B).atIndex(Idx: 0).narrow());
1659 return true;
1660 }
1661
1662 // Records. Still allocate them as single-element arrays.
1663 QualType AllocType = S.getASTContext().getConstantArrayType(
1664 EltTy: ElemType, ArySize: NumElems, SizeExpr: nullptr, ASM: ArraySizeModifier::Normal, IndexTypeQuals: 0);
1665
1666 const Descriptor *Desc =
1667 S.P.createDescriptor(D: NewCall, Ty: AllocType.getTypePtr());
1668 Block *B = Allocator.allocate(D: Desc, EvalID: S.getContext().getEvalID(),
1669 AllocForm: DynamicAllocator::Form::Operator);
1670 assert(B);
1671 S.Stk.push<Pointer>(Args: Pointer(B).atIndex(Idx: 0).narrow());
1672 return true;
1673}
1674
1675static bool interp__builtin_operator_delete(InterpState &S, CodePtr OpPC,
1676 const InterpFrame *Frame,
1677 const CallExpr *Call) {
1678 const Expr *Source = nullptr;
1679 const Block *BlockToDelete = nullptr;
1680
1681 unsigned NumArgs = Call->getNumArgs();
1682 assert(NumArgs >= 1);
1683
1684 // Args are pushed in source order. The trailing sized/aligned delete
1685 // operands are above the pointer on the stack.
1686 for (unsigned I = NumArgs - 1; I != 0; --I)
1687 discard(Stk&: S.Stk, T: *S.getContext().classify(E: Call->getArg(Arg: I)));
1688
1689 if (S.checkingPotentialConstantExpression()) {
1690 S.Stk.discard<Pointer>();
1691 return false;
1692 }
1693
1694 // This is permitted only within a call to std::allocator<T>::deallocate.
1695 if (!S.getStdAllocatorCaller(Name: "deallocate")) {
1696 S.FFDiag(E: Call);
1697 S.Stk.discard<Pointer>();
1698 return true;
1699 }
1700
1701 {
1702 const Pointer &Ptr = S.Stk.pop<Pointer>();
1703
1704 if (Ptr.isZero()) {
1705 S.CCEDiag(E: Call, DiagId: diag::note_constexpr_deallocate_null);
1706 return true;
1707 }
1708
1709 Source = Ptr.getRootExpr();
1710 BlockToDelete = Ptr.block();
1711
1712 if (!BlockToDelete->isDynamic()) {
1713 S.FFDiag(E: Call, DiagId: diag::note_constexpr_delete_not_heap_alloc)
1714 << Ptr.toDiagnosticString(Ctx: S.getASTContext());
1715 if (const auto *D = Ptr.getFieldDesc()->asDecl())
1716 S.Note(Loc: D->getLocation(), DiagId: diag::note_declared_at);
1717 }
1718 }
1719 assert(BlockToDelete);
1720
1721 DynamicAllocator &Allocator = S.getAllocator();
1722 const Descriptor *BlockDesc = BlockToDelete->getDescriptor();
1723 std::optional<DynamicAllocator::Form> AllocForm =
1724 Allocator.getAllocationForm(Source);
1725
1726 if (!Allocator.deallocate(Source, BlockToDelete)) {
1727 // Nothing has been deallocated, this must be a double-delete.
1728 const SourceInfo &Loc = S.Current->getSource(PC: OpPC);
1729 S.FFDiag(SI: Loc, DiagId: diag::note_constexpr_double_delete);
1730 return false;
1731 }
1732 assert(AllocForm);
1733
1734 return CheckNewDeleteForms(
1735 S, OpPC, AllocForm: *AllocForm, DeleteForm: DynamicAllocator::Form::Operator, D: BlockDesc, NewExpr: Source);
1736}
1737
1738static bool interp__builtin_arithmetic_fence(InterpState &S, CodePtr OpPC,
1739 const InterpFrame *Frame,
1740 const CallExpr *Call) {
1741 const Floating &Arg0 = S.Stk.pop<Floating>();
1742 S.Stk.push<Floating>(Args: Arg0);
1743 return true;
1744}
1745
1746static bool interp__builtin_vector_reduce(InterpState &S, CodePtr OpPC,
1747 const CallExpr *Call, unsigned ID) {
1748 const Pointer &Arg = S.Stk.pop<Pointer>();
1749 assert(Arg.getFieldDesc()->isPrimitiveArray());
1750
1751 QualType ElemType = Arg.getFieldDesc()->getElemQualType();
1752 assert(Call->getType() == ElemType);
1753 PrimType ElemT = *S.getContext().classify(T: ElemType);
1754 unsigned NumElems = Arg.getNumElems();
1755
1756 if (!isIntegerType(T: ElemT))
1757 return false;
1758
1759 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
1760 T Result = Arg.elem<T>(0);
1761 unsigned BitWidth = Result.bitWidth();
1762 for (unsigned I = 1; I != NumElems; ++I) {
1763 T Elem = Arg.elem<T>(I);
1764 T PrevResult = Result;
1765
1766 if (ID == Builtin::BI__builtin_reduce_add) {
1767 if (T::add(Result, Elem, BitWidth, &Result)) {
1768 unsigned OverflowBits = BitWidth + 1;
1769 (void)handleOverflow(S, OpPC,
1770 (PrevResult.toAPSInt(OverflowBits) +
1771 Elem.toAPSInt(OverflowBits)));
1772 return false;
1773 }
1774 } else if (ID == Builtin::BI__builtin_reduce_mul) {
1775 if (T::mul(Result, Elem, BitWidth, &Result)) {
1776 unsigned OverflowBits = BitWidth * 2;
1777 (void)handleOverflow(S, OpPC,
1778 (PrevResult.toAPSInt(OverflowBits) *
1779 Elem.toAPSInt(OverflowBits)));
1780 return false;
1781 }
1782
1783 } else if (ID == Builtin::BI__builtin_reduce_and) {
1784 (void)T::bitAnd(Result, Elem, BitWidth, &Result);
1785 } else if (ID == Builtin::BI__builtin_reduce_or) {
1786 (void)T::bitOr(Result, Elem, BitWidth, &Result);
1787 } else if (ID == Builtin::BI__builtin_reduce_xor) {
1788 (void)T::bitXor(Result, Elem, BitWidth, &Result);
1789 } else if (ID == Builtin::BI__builtin_reduce_min) {
1790 if (Elem < Result)
1791 Result = Elem;
1792 } else if (ID == Builtin::BI__builtin_reduce_max) {
1793 if (Elem > Result)
1794 Result = Elem;
1795 } else {
1796 llvm_unreachable("Unhandled vector reduce builtin");
1797 }
1798 }
1799 pushInteger(S, Result.toAPSInt(), Call->getType());
1800 });
1801
1802 return true;
1803}
1804
1805static bool interp__builtin_elementwise_abs(InterpState &S, CodePtr OpPC,
1806 const InterpFrame *Frame,
1807 const CallExpr *Call,
1808 unsigned BuiltinID) {
1809 assert(Call->getNumArgs() == 1);
1810 QualType Ty = Call->getArg(Arg: 0)->getType();
1811 if (Ty->isIntegerType()) {
1812 APSInt Val;
1813 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: Val))
1814 return false;
1815 pushInteger(S, Val: Val.abs(), QT: Call->getType());
1816 return true;
1817 }
1818
1819 if (Ty->isFloatingType()) {
1820 Floating Val = S.Stk.pop<Floating>();
1821 Floating Result = abs(S, In: Val);
1822 S.Stk.push<Floating>(Args&: Result);
1823 return true;
1824 }
1825
1826 // Otherwise, the argument must be a vector.
1827 assert(Call->getArg(0)->getType()->isVectorType());
1828 const Pointer &Arg = S.Stk.pop<Pointer>();
1829 assert(Arg.getFieldDesc()->isPrimitiveArray());
1830 const Pointer &Dst = S.Stk.peek<Pointer>();
1831 assert(Dst.getFieldDesc()->isPrimitiveArray());
1832 assert(Arg.getFieldDesc()->getNumElems() ==
1833 Dst.getFieldDesc()->getNumElems());
1834
1835 QualType ElemType = Arg.getFieldDesc()->getElemQualType();
1836 PrimType ElemT = *S.getContext().classify(T: ElemType);
1837 unsigned NumElems = Arg.getNumElems();
1838 // we can either have a vector of integer or a vector of floating point
1839 for (unsigned I = 0; I != NumElems; ++I) {
1840 if (ElemType->isIntegerType()) {
1841 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
1842 Dst.elem<T>(I) = T::from(static_cast<T>(
1843 APSInt(Arg.elem<T>(I).toAPSInt().abs(),
1844 ElemType->isUnsignedIntegerOrEnumerationType())));
1845 });
1846 } else {
1847 Floating Val = Arg.elem<Floating>(I);
1848 Dst.elem<Floating>(I) = abs(S, In: Val);
1849 }
1850 }
1851 Dst.initializeAllElements();
1852
1853 return true;
1854}
1855
1856/// Can be called with an integer or vector as the first and only parameter.
1857static bool interp__builtin_elementwise_countzeroes(InterpState &S,
1858 CodePtr OpPC,
1859 const InterpFrame *Frame,
1860 const CallExpr *Call,
1861 unsigned BuiltinID) {
1862 bool HasZeroArg = Call->getNumArgs() == 2;
1863 bool IsCTTZ = BuiltinID == Builtin::BI__builtin_elementwise_ctzg;
1864 assert(Call->getNumArgs() == 1 || HasZeroArg);
1865 if (Call->getArg(Arg: 0)->getType()->isIntegerType()) {
1866 PrimType ArgT = *S.getContext().classify(T: Call->getArg(Arg: 0)->getType());
1867 APSInt Val;
1868 if (!popToAPSInt(Stk&: S.Stk, T: ArgT, Out&: Val))
1869 return false;
1870 std::optional<APSInt> ZeroVal;
1871 if (HasZeroArg) {
1872 ZeroVal = Val;
1873 if (!popToAPSInt(Stk&: S.Stk, T: ArgT, Out&: Val))
1874 return false;
1875 }
1876
1877 if (Val.isZero()) {
1878 if (ZeroVal) {
1879 pushInteger(S, Val: *ZeroVal, QT: Call->getType());
1880 return true;
1881 }
1882 // If we haven't been provided the second argument, the result is
1883 // undefined
1884 S.FFDiag(SI: S.Current->getSource(PC: OpPC),
1885 DiagId: diag::note_constexpr_countzeroes_zero)
1886 << /*IsTrailing=*/IsCTTZ;
1887 return false;
1888 }
1889
1890 if (BuiltinID == Builtin::BI__builtin_elementwise_clzg) {
1891 pushInteger(S, Val: Val.countLeadingZeros(), QT: Call->getType());
1892 } else {
1893 pushInteger(S, Val: Val.countTrailingZeros(), QT: Call->getType());
1894 }
1895 return true;
1896 }
1897 // Otherwise, the argument must be a vector.
1898 const ASTContext &ASTCtx = S.getASTContext();
1899 Pointer ZeroArg;
1900 if (HasZeroArg) {
1901 assert(Call->getArg(1)->getType()->isVectorType() &&
1902 ASTCtx.hasSameUnqualifiedType(Call->getArg(0)->getType(),
1903 Call->getArg(1)->getType()));
1904 (void)ASTCtx;
1905 ZeroArg = S.Stk.pop<Pointer>();
1906 assert(ZeroArg.getFieldDesc()->isPrimitiveArray());
1907 }
1908 assert(Call->getArg(0)->getType()->isVectorType());
1909 const Pointer &Arg = S.Stk.pop<Pointer>();
1910 assert(Arg.getFieldDesc()->isPrimitiveArray());
1911 const Pointer &Dst = S.Stk.peek<Pointer>();
1912 assert(Dst.getFieldDesc()->isPrimitiveArray());
1913 assert(Arg.getFieldDesc()->getNumElems() ==
1914 Dst.getFieldDesc()->getNumElems());
1915
1916 QualType ElemType = Arg.getFieldDesc()->getElemQualType();
1917 PrimType ElemT = *S.getContext().classify(T: ElemType);
1918 unsigned NumElems = Arg.getNumElems();
1919
1920 // FIXME: Reading from uninitialized vector elements?
1921 for (unsigned I = 0; I != NumElems; ++I) {
1922 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
1923 APInt EltVal = Arg.atIndex(I).deref<T>().toAPSInt();
1924 if (EltVal.isZero()) {
1925 if (HasZeroArg) {
1926 Dst.atIndex(I).deref<T>() = ZeroArg.atIndex(I).deref<T>();
1927 } else {
1928 // If we haven't been provided the second argument, the result is
1929 // undefined
1930 S.FFDiag(S.Current->getSource(OpPC),
1931 diag::note_constexpr_countzeroes_zero)
1932 << /*IsTrailing=*/IsCTTZ;
1933 return false;
1934 }
1935 } else if (IsCTTZ) {
1936 Dst.atIndex(I).deref<T>() = T::from(EltVal.countTrailingZeros());
1937 } else {
1938 Dst.atIndex(I).deref<T>() = T::from(EltVal.countLeadingZeros());
1939 }
1940 Dst.atIndex(I).initialize();
1941 });
1942 }
1943
1944 return true;
1945}
1946
1947static bool interp__builtin_memcpy(InterpState &S, CodePtr OpPC,
1948 const InterpFrame *Frame,
1949 const CallExpr *Call, unsigned ID) {
1950 assert(Call->getNumArgs() == 3);
1951 const ASTContext &ASTCtx = S.getASTContext();
1952 uint64_t Size;
1953 if (!popToUInt64(S, E: Call->getArg(Arg: 2), Out&: Size))
1954 return false;
1955 Pointer SrcPtr = S.Stk.pop<Pointer>().expand();
1956 Pointer DestPtr = S.Stk.pop<Pointer>().expand();
1957
1958 if (ID == Builtin::BImemcpy || ID == Builtin::BImemmove)
1959 diagnoseNonConstexprBuiltin(S, OpPC, ID);
1960
1961 bool Move =
1962 (ID == Builtin::BI__builtin_memmove || ID == Builtin::BImemmove ||
1963 ID == Builtin::BI__builtin_wmemmove || ID == Builtin::BIwmemmove);
1964 bool WChar = ID == Builtin::BIwmemcpy || ID == Builtin::BIwmemmove ||
1965 ID == Builtin::BI__builtin_wmemcpy ||
1966 ID == Builtin::BI__builtin_wmemmove;
1967
1968 // If the size is zero, we treat this as always being a valid no-op.
1969 if (Size == 0) {
1970 S.Stk.push<Pointer>(Args&: DestPtr);
1971 return true;
1972 }
1973
1974 if (SrcPtr.isZero() || DestPtr.isZero()) {
1975 Pointer DiagPtr = (SrcPtr.isZero() ? SrcPtr : DestPtr);
1976 S.FFDiag(SI: S.Current->getSource(PC: OpPC), DiagId: diag::note_constexpr_memcpy_null)
1977 << /*IsMove=*/Move << /*IsWchar=*/WChar << !SrcPtr.isZero()
1978 << DiagPtr.toDiagnosticString(Ctx: ASTCtx);
1979 return false;
1980 }
1981
1982 // Diagnose integral src/dest pointers specially.
1983 if (SrcPtr.isIntegralPointer() || DestPtr.isIntegralPointer()) {
1984 std::string DiagVal = "(void *)";
1985 DiagVal += SrcPtr.isIntegralPointer()
1986 ? std::to_string(val: SrcPtr.getIntegerRepresentation())
1987 : std::to_string(val: DestPtr.getIntegerRepresentation());
1988 S.FFDiag(SI: S.Current->getSource(PC: OpPC), DiagId: diag::note_constexpr_memcpy_null)
1989 << Move << WChar << DestPtr.isIntegralPointer() << DiagVal;
1990 return false;
1991 }
1992
1993 if (!isReadable(P: DestPtr) || !isReadable(P: SrcPtr))
1994 return false;
1995
1996 if (DestPtr.getType()->isIncompleteType()) {
1997 S.FFDiag(SI: S.Current->getSource(PC: OpPC),
1998 DiagId: diag::note_constexpr_memcpy_incomplete_type)
1999 << Move << DestPtr.getType();
2000 return false;
2001 }
2002 if (SrcPtr.getType()->isIncompleteType()) {
2003 S.FFDiag(SI: S.Current->getSource(PC: OpPC),
2004 DiagId: diag::note_constexpr_memcpy_incomplete_type)
2005 << Move << SrcPtr.getType();
2006 return false;
2007 }
2008
2009 QualType DestElemType = getElemType(P: DestPtr);
2010 if (DestElemType->isIncompleteType()) {
2011 S.FFDiag(SI: S.Current->getSource(PC: OpPC),
2012 DiagId: diag::note_constexpr_memcpy_incomplete_type)
2013 << Move << DestElemType;
2014 return false;
2015 }
2016
2017 size_t RemainingDestElems;
2018 if (DestPtr.inArray()) {
2019 RemainingDestElems = DestPtr.isUnknownSizeArray()
2020 ? 0
2021 : (DestPtr.getNumElems() - DestPtr.getIndex());
2022 } else {
2023 RemainingDestElems = 1;
2024 }
2025 unsigned DestElemSize = ASTCtx.getTypeSizeInChars(T: DestElemType).getQuantity();
2026
2027 if (WChar) {
2028 uint64_t WCharSize =
2029 ASTCtx.getTypeSizeInChars(T: ASTCtx.getWCharType()).getQuantity();
2030 Size *= WCharSize;
2031 }
2032
2033 if (Size % DestElemSize != 0) {
2034 S.FFDiag(SI: S.Current->getSource(PC: OpPC),
2035 DiagId: diag::note_constexpr_memcpy_unsupported)
2036 << Move << WChar << 0 << DestElemType << Size << DestElemSize;
2037 return false;
2038 }
2039
2040 QualType SrcElemType = getElemType(P: SrcPtr);
2041 size_t RemainingSrcElems;
2042 if (SrcPtr.inArray()) {
2043 RemainingSrcElems = SrcPtr.isUnknownSizeArray()
2044 ? 0
2045 : (SrcPtr.getNumElems() - SrcPtr.getIndex());
2046 } else {
2047 RemainingSrcElems = 1;
2048 }
2049 unsigned SrcElemSize = ASTCtx.getTypeSizeInChars(T: SrcElemType).getQuantity();
2050
2051 if (!ASTCtx.hasSameUnqualifiedType(T1: DestElemType, T2: SrcElemType)) {
2052 S.FFDiag(SI: S.Current->getSource(PC: OpPC), DiagId: diag::note_constexpr_memcpy_type_pun)
2053 << Move << SrcElemType << DestElemType;
2054 return false;
2055 }
2056
2057 if (!DestElemType.isTriviallyCopyableType(Context: ASTCtx)) {
2058 S.FFDiag(SI: S.Current->getSource(PC: OpPC), DiagId: diag::note_constexpr_memcpy_nontrivial)
2059 << Move << DestElemType;
2060 return false;
2061 }
2062
2063 // Check if we have enough elements to read from and write to.
2064 size_t RemainingDestBytes = RemainingDestElems * DestElemSize;
2065 size_t RemainingSrcBytes = RemainingSrcElems * SrcElemSize;
2066 if (Size > RemainingDestBytes || Size > RemainingSrcBytes) {
2067 APInt N = APInt(64, Size / DestElemSize);
2068 S.FFDiag(SI: S.Current->getSource(PC: OpPC),
2069 DiagId: diag::note_constexpr_memcpy_unsupported)
2070 << Move << WChar << (Size > RemainingSrcBytes ? 1 : 2) << DestElemType
2071 << toString(I: N, Radix: 10, /*Signed=*/false);
2072 return false;
2073 }
2074
2075 // Check for overlapping memory regions.
2076 if (!Move && Pointer::pointToSameBlock(A: SrcPtr, B: DestPtr)) {
2077 // Remove base casts.
2078 Pointer SrcP = SrcPtr.stripBaseCasts();
2079 Pointer DestP = DestPtr.stripBaseCasts();
2080
2081 unsigned SrcIndex = SrcP.expand().getIndex() * SrcElemSize;
2082 unsigned DstIndex = DestP.expand().getIndex() * DestElemSize;
2083
2084 if ((SrcIndex <= DstIndex && (SrcIndex + Size) > DstIndex) ||
2085 (DstIndex <= SrcIndex && (DstIndex + Size) > SrcIndex)) {
2086 S.FFDiag(SI: S.Current->getSource(PC: OpPC), DiagId: diag::note_constexpr_memcpy_overlap)
2087 << /*IsWChar=*/false;
2088 return false;
2089 }
2090 }
2091
2092 assert(Size % DestElemSize == 0);
2093 if (!DoMemcpy(S, OpPC, SrcPtr, DestPtr, Size: Bytes(Size).toBits()))
2094 return false;
2095
2096 S.Stk.push<Pointer>(Args&: DestPtr);
2097 return true;
2098}
2099
2100/// Determine if T is a character type for which we guarantee that
2101/// sizeof(T) == 1.
2102static bool isOneByteCharacterType(QualType T) {
2103 return T->isCharType() || T->isChar8Type();
2104}
2105
2106static bool interp__builtin_memcmp(InterpState &S, CodePtr OpPC,
2107 const InterpFrame *Frame,
2108 const CallExpr *Call, unsigned ID) {
2109 assert(Call->getNumArgs() == 3);
2110 uint64_t Size;
2111 if (!popToUInt64(S, E: Call->getArg(Arg: 2), Out&: Size))
2112 return false;
2113 const Pointer &PtrB = S.Stk.pop<Pointer>();
2114 const Pointer &PtrA = S.Stk.pop<Pointer>();
2115
2116 if (ID == Builtin::BImemcmp || ID == Builtin::BIbcmp ||
2117 ID == Builtin::BIwmemcmp)
2118 diagnoseNonConstexprBuiltin(S, OpPC, ID);
2119
2120 if (Size == 0) {
2121 pushInteger(S, Val: 0, QT: Call->getType());
2122 return true;
2123 }
2124
2125 if (!PtrA.isReadablePointerType() || !PtrB.isReadablePointerType())
2126 return false;
2127
2128 bool IsWide =
2129 (ID == Builtin::BIwmemcmp || ID == Builtin::BI__builtin_wmemcmp);
2130
2131 const ASTContext &ASTCtx = S.getASTContext();
2132 QualType ElemTypeA = getElemType(P: PtrA);
2133 QualType ElemTypeB = getElemType(P: PtrB);
2134 // FIXME: This is an arbitrary limitation the current constant interpreter
2135 // had. We could remove this.
2136 if (!IsWide && (!isOneByteCharacterType(T: ElemTypeA) ||
2137 !isOneByteCharacterType(T: ElemTypeB))) {
2138 S.FFDiag(SI: S.Current->getSource(PC: OpPC),
2139 DiagId: diag::note_constexpr_memcmp_unsupported)
2140 << ASTCtx.BuiltinInfo.getQuotedName(ID) << PtrA.getType()
2141 << PtrB.getType();
2142 return false;
2143 }
2144
2145 if (!CheckLoad(S, OpPC, Ptr: PtrA, AK: AK_Read) || !CheckLoad(S, OpPC, Ptr: PtrB, AK: AK_Read))
2146 return false;
2147
2148 // Now, read both pointers to a buffer and compare those.
2149 BitcastBuffer BufferA(
2150 Bits(ASTCtx.getTypeSize(T: ElemTypeA) * PtrA.getNumElems()));
2151 readPointerToBuffer(Ctx: S.getContext(), FromPtr: PtrA, Buffer&: BufferA, /*ReturnOnUninit=*/false);
2152
2153 // FIXME: The swapping here is UNDOING something we do when reading the
2154 // data into the buffer.
2155 if (ASTCtx.getTargetInfo().isBigEndian())
2156 swapBytes(M: BufferA.Data.get(), N: BufferA.byteSize().getQuantity());
2157
2158 BitcastBuffer BufferB(
2159 Bits(ASTCtx.getTypeSize(T: ElemTypeB) * PtrB.getNumElems()));
2160 readPointerToBuffer(Ctx: S.getContext(), FromPtr: PtrB, Buffer&: BufferB, /*ReturnOnUninit=*/false);
2161 // FIXME: The swapping here is UNDOING something we do when reading the
2162 // data into the buffer.
2163 if (ASTCtx.getTargetInfo().isBigEndian())
2164 swapBytes(M: BufferB.Data.get(), N: BufferB.byteSize().getQuantity());
2165
2166 size_t MinBufferSize = std::min(a: BufferA.byteSize().getQuantity(),
2167 b: BufferB.byteSize().getQuantity());
2168
2169 unsigned ElemSize = 1;
2170 if (IsWide)
2171 ElemSize = ASTCtx.getTypeSizeInChars(T: ASTCtx.getWCharType()).getQuantity();
2172 // The Size given for the wide variants is in wide-char units. Convert it
2173 // to bytes.
2174 size_t ByteSize = Size * ElemSize;
2175 size_t CmpSize = std::min(a: MinBufferSize, b: ByteSize);
2176
2177 for (size_t I = 0; I != CmpSize; I += ElemSize) {
2178 if (IsWide) {
2179 FIXED_SIZE_INT_TYPE_SWITCH(
2180 *S.getContext().classify(ASTCtx.getWCharType()), {
2181 T A = T::bitcastFromMemory(BufferA.atByte(I), T::bitWidth());
2182 T B = T::bitcastFromMemory(BufferB.atByte(I), T::bitWidth());
2183 if (A < B) {
2184 pushInteger(S, -1, Call->getType());
2185 return true;
2186 }
2187 if (A > B) {
2188 pushInteger(S, 1, Call->getType());
2189 return true;
2190 }
2191 });
2192 } else {
2193 auto A = BufferA.deref<std::byte>(Offset: Bytes(I));
2194 auto B = BufferB.deref<std::byte>(Offset: Bytes(I));
2195
2196 if (A < B) {
2197 pushInteger(S, Val: -1, QT: Call->getType());
2198 return true;
2199 }
2200 if (A > B) {
2201 pushInteger(S, Val: 1, QT: Call->getType());
2202 return true;
2203 }
2204 }
2205 }
2206
2207 // We compared CmpSize bytes above. If the limiting factor was the Size
2208 // passed, we're done and the result is equality (0).
2209 if (ByteSize <= CmpSize) {
2210 pushInteger(S, Val: 0, QT: Call->getType());
2211 return true;
2212 }
2213
2214 // However, if we read all the available bytes but were instructed to read
2215 // even more, diagnose this as a "read of dereferenced one-past-the-end
2216 // pointer". This is what would happen if we called CheckLoad() on every array
2217 // element.
2218 S.FFDiag(SI: S.Current->getSource(PC: OpPC), DiagId: diag::note_constexpr_access_past_end)
2219 << AK_Read << S.Current->getRange(PC: OpPC);
2220 return false;
2221}
2222
2223// __builtin_memchr(ptr, int, int)
2224// __builtin_strchr(ptr, int)
2225static bool interp__builtin_memchr(InterpState &S, CodePtr OpPC,
2226 const CallExpr *Call, unsigned ID) {
2227 if (ID == Builtin::BImemchr || ID == Builtin::BIwcschr ||
2228 ID == Builtin::BIstrchr || ID == Builtin::BIwmemchr)
2229 diagnoseNonConstexprBuiltin(S, OpPC, ID);
2230
2231 std::optional<APSInt> MaxLength;
2232 if (Call->getNumArgs() == 3) {
2233 APSInt MaxLengthVal;
2234 if (!popToAPSInt(S, E: Call->getArg(Arg: 2), Out&: MaxLengthVal))
2235 return false;
2236 MaxLength = MaxLengthVal;
2237 }
2238
2239 APSInt Desired;
2240 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: Desired))
2241 return false;
2242 const Pointer &Ptr = S.Stk.pop<Pointer>();
2243
2244 if (MaxLength && MaxLength->isZero()) {
2245 S.Stk.push<Pointer>();
2246 return true;
2247 }
2248
2249 if (Ptr.isDummy()) {
2250 if (Ptr.getType()->isIncompleteType())
2251 S.FFDiag(SI: S.Current->getSource(PC: OpPC),
2252 DiagId: diag::note_constexpr_ltor_incomplete_type)
2253 << Ptr.getType();
2254 return false;
2255 }
2256
2257 // Null is only okay if the given size is 0.
2258 if (Ptr.isZero()) {
2259 S.FFDiag(SI: S.Current->getSource(PC: OpPC), DiagId: diag::note_constexpr_access_null)
2260 << AK_Read;
2261 return false;
2262 }
2263
2264 if (!Ptr.isReadablePointerType())
2265 return false;
2266
2267 QualType ElemTy = getElemType(P: Ptr);
2268 bool IsRawByte = ID == Builtin::BImemchr || ID == Builtin::BI__builtin_memchr;
2269
2270 // Give up on byte-oriented matching against multibyte elements.
2271 if (IsRawByte && !isOneByteCharacterType(T: ElemTy)) {
2272 S.FFDiag(SI: S.Current->getSource(PC: OpPC),
2273 DiagId: diag::note_constexpr_memchr_unsupported)
2274 << S.getASTContext().BuiltinInfo.getQuotedName(ID) << ElemTy;
2275 return false;
2276 }
2277
2278 if (!isReadable(P: Ptr))
2279 return false;
2280
2281 if (ID == Builtin::BIstrchr || ID == Builtin::BI__builtin_strchr) {
2282 int64_t DesiredTrunc;
2283 if (S.getASTContext().CharTy->isSignedIntegerType())
2284 DesiredTrunc =
2285 Desired.trunc(width: S.getASTContext().getCharWidth()).getSExtValue();
2286 else
2287 DesiredTrunc =
2288 Desired.trunc(width: S.getASTContext().getCharWidth()).getZExtValue();
2289 // strchr compares directly to the passed integer, and therefore
2290 // always fails if given an int that is not a char.
2291 if (Desired != DesiredTrunc) {
2292 S.Stk.push<Pointer>();
2293 return true;
2294 }
2295 }
2296
2297 uint64_t DesiredVal;
2298 if (ID == Builtin::BIwmemchr || ID == Builtin::BI__builtin_wmemchr ||
2299 ID == Builtin::BIwcschr || ID == Builtin::BI__builtin_wcschr) {
2300 // wcschr and wmemchr are given a wchar_t to look for. Just use it.
2301 DesiredVal = Desired.getZExtValue();
2302 } else {
2303 DesiredVal = Desired.trunc(width: S.getASTContext().getCharWidth()).getZExtValue();
2304 }
2305
2306 bool StopAtZero =
2307 (ID == Builtin::BIstrchr || ID == Builtin::BI__builtin_strchr ||
2308 ID == Builtin::BIwcschr || ID == Builtin::BI__builtin_wcschr);
2309
2310 PrimType ElemT =
2311 IsRawByte ? PT_Sint8 : *S.getContext().classify(T: getElemType(P: Ptr));
2312
2313 size_t Index = Ptr.getIndex();
2314 size_t Step = 0;
2315 for (;;) {
2316 const Pointer &ElemPtr =
2317 (Index + Step) > 0 ? Ptr.atIndex(Idx: Index + Step) : Ptr;
2318
2319 if (!CheckLoad(S, OpPC, Ptr: ElemPtr))
2320 return false;
2321
2322 uint64_t V;
2323 INT_TYPE_SWITCH_NO_BOOL(
2324 ElemT, { V = static_cast<uint64_t>(ElemPtr.load<T>().toUnsigned()); });
2325
2326 if (V == DesiredVal) {
2327 S.Stk.push<Pointer>(Args: ElemPtr);
2328 return true;
2329 }
2330
2331 if (StopAtZero && V == 0)
2332 break;
2333
2334 ++Step;
2335 if (MaxLength && Step == MaxLength->getZExtValue())
2336 break;
2337 }
2338
2339 S.Stk.push<Pointer>();
2340 return true;
2341}
2342
2343static std::optional<unsigned> computeFullDescSize(const ASTContext &ASTCtx,
2344 const Descriptor *Desc) {
2345 if (Desc->isPrimitive() || Desc->isArray())
2346 return ASTCtx.getTypeSizeInChars(T: Desc->getType()).getQuantity();
2347
2348 if (Desc->isRecord()) {
2349 // Can't use Descriptor::getType() as that may return a pointer type. Look
2350 // at the decl directly.
2351 return ASTCtx
2352 .getTypeSizeInChars(
2353 T: ASTCtx.getCanonicalTagType(TD: Desc->ElemRecord->getDecl()))
2354 .getQuantity();
2355 }
2356
2357 return std::nullopt;
2358}
2359
2360/// Compute the byte offset of \p Ptr in the full declaration.
2361static unsigned computePointerOffset(const ASTContext &ASTCtx,
2362 const Pointer &Ptr) {
2363 unsigned Result = 0;
2364
2365 Pointer P = Ptr;
2366 while (P.isField() || P.isArrayElement()) {
2367 P = P.expand();
2368 const Descriptor *D = P.getFieldDesc();
2369
2370 if (P.isArrayElement()) {
2371 unsigned ElemSize =
2372 ASTCtx.getTypeSizeInChars(T: D->getElemQualType()).getQuantity();
2373 if (P.isOnePastEnd())
2374 Result += ElemSize * P.getNumElems();
2375 else
2376 Result += ElemSize * P.getIndex();
2377 P = P.expand().getArray();
2378 } else if (P.isBaseClass()) {
2379 const auto *RD = cast<CXXRecordDecl>(Val: D->asDecl());
2380 bool IsVirtual = Ptr.isVirtualBaseClass();
2381 P = P.getBase();
2382 const Record *BaseRecord = P.getRecord();
2383
2384 const ASTRecordLayout &Layout =
2385 ASTCtx.getASTRecordLayout(D: cast<CXXRecordDecl>(Val: BaseRecord->getDecl()));
2386 if (IsVirtual)
2387 Result += Layout.getVBaseClassOffset(VBase: RD).getQuantity();
2388 else
2389 Result += Layout.getBaseClassOffset(Base: RD).getQuantity();
2390 } else if (P.isField()) {
2391 const FieldDecl *FD = P.getField();
2392 const ASTRecordLayout &Layout =
2393 ASTCtx.getASTRecordLayout(D: FD->getParent());
2394 unsigned FieldIndex = FD->getFieldIndex();
2395 uint64_t FieldOffset =
2396 ASTCtx.toCharUnitsFromBits(BitSize: Layout.getFieldOffset(FieldNo: FieldIndex))
2397 .getQuantity();
2398 Result += FieldOffset;
2399 P = P.getBase();
2400 } else
2401 llvm_unreachable("Unhandled descriptor type");
2402 }
2403
2404 return Result;
2405}
2406
2407/// Does Ptr point to the last subobject?
2408static bool pointsToLastObject(const Pointer &Ptr) {
2409 Pointer P = Ptr;
2410 while (!P.isRoot()) {
2411
2412 if (P.isArrayElement()) {
2413 P = P.expand().getArray();
2414 continue;
2415 }
2416 if (P.isBaseClass()) {
2417 if (P.getRecord()->getNumFields() > 0)
2418 return false;
2419 P = P.getBase();
2420 continue;
2421 }
2422
2423 Pointer Base = P.getBase();
2424 if (const Record *R = Base.getRecord()) {
2425 assert(P.getField());
2426 if (P.getField()->getFieldIndex() != R->getNumFields() - 1)
2427 return false;
2428 }
2429 P = Base;
2430 }
2431
2432 return true;
2433}
2434
2435/// Does Ptr point to the last object AND to a flexible array member?
2436static bool isUserWritingOffTheEnd(const ASTContext &Ctx, const Pointer &Ptr,
2437 bool InvalidBase) {
2438 auto isFlexibleArrayMember = [&](const Descriptor *FieldDesc) {
2439 using FAMKind = LangOptions::StrictFlexArraysLevelKind;
2440 FAMKind StrictFlexArraysLevel =
2441 Ctx.getLangOpts().getStrictFlexArraysLevel();
2442
2443 if (StrictFlexArraysLevel == FAMKind::Default)
2444 return true;
2445
2446 unsigned NumElems = FieldDesc->getNumElems();
2447 if (NumElems == 0 && StrictFlexArraysLevel != FAMKind::IncompleteOnly)
2448 return true;
2449
2450 if (NumElems == 1 && StrictFlexArraysLevel == FAMKind::OneZeroOrIncomplete)
2451 return true;
2452 return false;
2453 };
2454
2455 const Descriptor *FieldDesc = Ptr.getFieldDesc();
2456 if (!FieldDesc->isArray())
2457 return false;
2458
2459 return InvalidBase && pointsToLastObject(Ptr) &&
2460 isFlexibleArrayMember(FieldDesc);
2461}
2462
2463UnsignedOrNone evaluateBuiltinObjectSize(const ASTContext &ASTCtx,
2464 unsigned Kind, Pointer &Ptr) {
2465 if (Ptr.isZero() || !Ptr.isBlockPointer())
2466 return std::nullopt;
2467
2468 if (Ptr.isDummy() && Ptr.getType()->isPointerType())
2469 return std::nullopt;
2470
2471 bool InvalidBase = false;
2472
2473 if (Ptr.isDummy()) {
2474 if (const VarDecl *VD = Ptr.getDeclDesc()->asVarDecl();
2475 VD && VD->getType()->isPointerType())
2476 InvalidBase = true;
2477 }
2478
2479 // According to the GCC documentation, we want the size of the subobject
2480 // denoted by the pointer. But that's not quite right -- what we actually
2481 // want is the size of the immediately-enclosing array, if there is one.
2482 if (Ptr.isArrayElement())
2483 Ptr = Ptr.expand();
2484
2485 bool DetermineForCompleteObject = Ptr.getFieldDesc() == Ptr.getDeclDesc();
2486 const Descriptor *DeclDesc = Ptr.getDeclDesc();
2487 assert(DeclDesc);
2488
2489 bool UseFieldDesc = (Kind & 1u);
2490 bool ReportMinimum = (Kind & 2u);
2491 if (!UseFieldDesc || DetermineForCompleteObject) {
2492 // Can't read beyond the pointer decl desc.
2493 if (!ReportMinimum && DeclDesc->getType()->isPointerType())
2494 return std::nullopt;
2495
2496 if (InvalidBase)
2497 return std::nullopt;
2498 } else {
2499 if (isUserWritingOffTheEnd(Ctx: ASTCtx, Ptr, InvalidBase)) {
2500 // If we cannot determine the size of the initial allocation, then we
2501 // can't given an accurate upper-bound. However, we are still able to give
2502 // conservative lower-bounds for Type=3.
2503 if (Kind == 1)
2504 return std::nullopt;
2505 }
2506 // For Type=1, defer to the runtime path on a true incomplete-array
2507 // flexible array member (e.g. 'char fam[]') even when the base is a
2508 // concrete local/global. Without this, the bytecode interpreter would
2509 // happily fold &af.fam to 'NumElems * elemSize = 0' below; the default
2510 // const-evaluator avoids the same trap, and CGBuiltin emits
2511 // @llvm.objectsize for the correct layout-derived answer (matching
2512 // GCC's __bos/__bdos on '&af.fam').
2513 if (Kind == 1 && pointsToLastObject(Ptr) && Ptr.getFieldDesc()->isArray() &&
2514 Ptr.getFieldDesc()->getType()->isIncompleteArrayType())
2515 return std::nullopt;
2516 }
2517
2518 // The "closest surrounding subobject" is NOT a base class,
2519 // so strip the base class casts.
2520 if (UseFieldDesc && Ptr.isBaseClass())
2521 Ptr = Ptr.stripBaseCasts();
2522
2523 const Descriptor *Desc = UseFieldDesc ? Ptr.getFieldDesc() : DeclDesc;
2524 assert(Desc);
2525
2526 std::optional<unsigned> FullSize = computeFullDescSize(ASTCtx, Desc);
2527 if (!FullSize)
2528 return std::nullopt;
2529
2530 unsigned ByteOffset;
2531 if (UseFieldDesc) {
2532 if (Ptr.isBaseClass()) {
2533 assert(computePointerOffset(ASTCtx, Ptr.getBase()) <=
2534 computePointerOffset(ASTCtx, Ptr));
2535 ByteOffset = computePointerOffset(ASTCtx, Ptr: Ptr.getBase()) -
2536 computePointerOffset(ASTCtx, Ptr);
2537 } else {
2538 if (Ptr.inArray())
2539 ByteOffset =
2540 computePointerOffset(ASTCtx, Ptr) -
2541 computePointerOffset(ASTCtx, Ptr: Ptr.expand().atIndex(Idx: 0).narrow());
2542 else
2543 ByteOffset = 0;
2544 }
2545 } else
2546 ByteOffset = computePointerOffset(ASTCtx, Ptr);
2547
2548 assert(ByteOffset <= *FullSize);
2549 return *FullSize - ByteOffset;
2550}
2551
2552static bool interp__builtin_object_size(InterpState &S, CodePtr OpPC,
2553 const InterpFrame *Frame,
2554 const CallExpr *Call) {
2555 const ASTContext &ASTCtx = S.getASTContext();
2556 // From the GCC docs:
2557 // Kind is an integer constant from 0 to 3. If the least significant bit is
2558 // clear, objects are whole variables. If it is set, a closest surrounding
2559 // subobject is considered the object a pointer points to. The second bit
2560 // determines if maximum or minimum of remaining bytes is computed.
2561 uint64_t Kind;
2562 if (!popToUInt64(S, E: Call->getArg(Arg: 1), Out&: Kind))
2563 return false;
2564 assert(Kind <= 3 && "unexpected kind");
2565 Pointer Ptr = S.Stk.pop<Pointer>();
2566
2567 if (Call->getArg(Arg: 0)->HasSideEffects(Ctx: ASTCtx)) {
2568 // "If there are any side effects in them, it returns (size_t) -1
2569 // for type 0 or 1 and (size_t) 0 for type 2 or 3."
2570 pushInteger(S, Val: Kind <= 1 ? -1 : 0, QT: Call->getType());
2571 return true;
2572 }
2573
2574 if (auto Result = evaluateBuiltinObjectSize(ASTCtx, Kind, Ptr)) {
2575 pushInteger(S, Val: *Result, QT: Call->getType());
2576 return true;
2577 }
2578 return false;
2579}
2580
2581static bool interp__builtin_is_within_lifetime(InterpState &S, CodePtr OpPC,
2582 const CallExpr *Call) {
2583
2584 if (!S.inConstantContext())
2585 return false;
2586
2587 const Pointer &Ptr = S.Stk.pop<Pointer>();
2588
2589 auto Error = [&](int Diag) {
2590 bool CalledFromStd = false;
2591 const auto *Callee = S.Current->getCallee();
2592 if (Callee && Callee->isInStdNamespace()) {
2593 const IdentifierInfo *Identifier = Callee->getIdentifier();
2594 CalledFromStd = Identifier && Identifier->isStr(Str: "is_within_lifetime");
2595 }
2596 S.CCEDiag(SI: CalledFromStd
2597 ? S.Current->Caller->getSource(PC: S.Current->getRetPC())
2598 : S.Current->getSource(PC: OpPC),
2599 DiagId: diag::err_invalid_is_within_lifetime)
2600 << (CalledFromStd ? "std::is_within_lifetime"
2601 : "__builtin_is_within_lifetime")
2602 << Diag;
2603 return false;
2604 };
2605
2606 if (Ptr.isZero())
2607 return Error(0);
2608 if (Ptr.isOnePastEnd())
2609 return Error(1);
2610
2611 bool Result = Ptr.getLifetime() != Lifetime::Ended;
2612 if (!Ptr.isActive()) {
2613 Result = false;
2614 } else {
2615 if (!CheckLive(S, OpPC, Ptr, AK: AK_Read))
2616 return false;
2617 if (!CheckMutable(S, OpPC, Ptr))
2618 return false;
2619 if (!CheckDummy(S, OpPC, B: Ptr.block(), AK: AK_Read))
2620 return false;
2621 }
2622
2623 // Check if we're currently running an initializer.
2624 if (S.initializingBlock(B: Ptr.block()))
2625 return Error(2);
2626 if (S.EvaluatingDecl && Ptr.getDeclDesc()->asVarDecl() == S.EvaluatingDecl)
2627 return Error(2);
2628
2629 pushInteger(S, Val: Result, QT: Call->getType());
2630 return true;
2631}
2632
2633static bool interp__builtin_elementwise_int_unaryop(
2634 InterpState &S, CodePtr OpPC, const CallExpr *Call,
2635 llvm::function_ref<APInt(const APSInt &)> Fn) {
2636 assert(Call->getNumArgs() == 1);
2637
2638 // Single integer case.
2639 if (!Call->getArg(Arg: 0)->getType()->isVectorType()) {
2640 assert(Call->getType()->isIntegerType());
2641 APSInt Src;
2642 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: Src))
2643 return false;
2644 APInt Result = Fn(Src);
2645 pushInteger(S, Val: APSInt(std::move(Result), !Src.isSigned()), QT: Call->getType());
2646 return true;
2647 }
2648
2649 // Vector case.
2650 const Pointer &Arg = S.Stk.pop<Pointer>();
2651 assert(Arg.getFieldDesc()->isPrimitiveArray());
2652 const Pointer &Dst = S.Stk.peek<Pointer>();
2653 assert(Dst.getFieldDesc()->isPrimitiveArray());
2654 assert(Arg.getFieldDesc()->getNumElems() ==
2655 Dst.getFieldDesc()->getNumElems());
2656
2657 QualType ElemType = Arg.getFieldDesc()->getElemQualType();
2658 PrimType ElemT = *S.getContext().classify(T: ElemType);
2659 unsigned NumElems = Arg.getNumElems();
2660 bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
2661
2662 for (unsigned I = 0; I != NumElems; ++I) {
2663 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
2664 APSInt Src = Arg.elem<T>(I).toAPSInt();
2665 APInt Result = Fn(Src);
2666 Dst.elem<T>(I) = static_cast<T>(APSInt(std::move(Result), DestUnsigned));
2667 });
2668 }
2669 Dst.initializeAllElements();
2670
2671 return true;
2672}
2673
2674static bool interp__builtin_elementwise_fp_binop(
2675 InterpState &S, CodePtr OpPC, const CallExpr *Call,
2676 llvm::function_ref<std::optional<APFloat>(
2677 const APFloat &, const APFloat &, std::optional<APSInt> RoundingMode)>
2678 Fn,
2679 bool IsScalar = false) {
2680 assert((Call->getNumArgs() == 2) || (Call->getNumArgs() == 3));
2681 const auto *VT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
2682 assert(VT->getElementType()->isFloatingType());
2683 unsigned NumElems = VT->getNumElements();
2684
2685 // Vector case.
2686 assert(Call->getArg(0)->getType()->isVectorType() &&
2687 Call->getArg(1)->getType()->isVectorType());
2688 assert(VT->getElementType() ==
2689 Call->getArg(1)->getType()->castAs<VectorType>()->getElementType());
2690 assert(VT->getNumElements() ==
2691 Call->getArg(1)->getType()->castAs<VectorType>()->getNumElements());
2692
2693 std::optional<APSInt> RoundingMode = std::nullopt;
2694 if (Call->getNumArgs() == 3) {
2695 APSInt RoundingModeVal;
2696 if (!popToAPSInt(S, E: Call->getArg(Arg: 2), Out&: RoundingModeVal))
2697 return false;
2698 RoundingMode = RoundingModeVal;
2699 }
2700
2701 const Pointer &BPtr = S.Stk.pop<Pointer>();
2702 const Pointer &APtr = S.Stk.pop<Pointer>();
2703 const Pointer &Dst = S.Stk.peek<Pointer>();
2704 for (unsigned ElemIdx = 0; ElemIdx != NumElems; ++ElemIdx) {
2705 using T = PrimConv<PT_Float>::T;
2706 if (IsScalar && ElemIdx > 0) {
2707 Dst.elem<T>(I: ElemIdx) = APtr.elem<T>(I: ElemIdx);
2708 continue;
2709 }
2710 APFloat ElemA = APtr.elem<T>(I: ElemIdx).getAPFloat();
2711 APFloat ElemB = BPtr.elem<T>(I: ElemIdx).getAPFloat();
2712 std::optional<APFloat> Result = Fn(ElemA, ElemB, RoundingMode);
2713 if (!Result)
2714 return false;
2715 Dst.elem<T>(I: ElemIdx) = static_cast<T>(*Result);
2716 }
2717
2718 Dst.initializeAllElements();
2719
2720 return true;
2721}
2722
2723static bool interp__builtin_scalar_fp_round_mask_binop(
2724 InterpState &S, CodePtr OpPC, const CallExpr *Call,
2725 llvm::function_ref<std::optional<APFloat>(const APFloat &, const APFloat &,
2726 std::optional<APSInt>)>
2727 Fn) {
2728 assert(Call->getNumArgs() == 5);
2729 const auto *VT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
2730 unsigned NumElems = VT->getNumElements();
2731
2732 APSInt RoundingMode;
2733 if (!popToAPSInt(S, E: Call->getArg(Arg: 4), Out&: RoundingMode))
2734 return false;
2735 uint64_t MaskVal;
2736 if (!popToUInt64(S, E: Call->getArg(Arg: 3), Out&: MaskVal))
2737 return false;
2738 const Pointer &SrcPtr = S.Stk.pop<Pointer>();
2739 const Pointer &BPtr = S.Stk.pop<Pointer>();
2740 const Pointer &APtr = S.Stk.pop<Pointer>();
2741 const Pointer &Dst = S.Stk.peek<Pointer>();
2742
2743 using T = PrimConv<PT_Float>::T;
2744
2745 if (MaskVal & 1) {
2746 APFloat ElemA = APtr.elem<T>(I: 0).getAPFloat();
2747 APFloat ElemB = BPtr.elem<T>(I: 0).getAPFloat();
2748 std::optional<APFloat> Result = Fn(ElemA, ElemB, RoundingMode);
2749 if (!Result)
2750 return false;
2751 Dst.elem<T>(I: 0) = static_cast<T>(*Result);
2752 } else {
2753 Dst.elem<T>(I: 0) = SrcPtr.elem<T>(I: 0);
2754 }
2755
2756 for (unsigned I = 1; I < NumElems; ++I)
2757 Dst.elem<T>(I) = APtr.elem<T>(I);
2758
2759 Dst.initializeAllElements();
2760
2761 return true;
2762}
2763
2764static bool interp__builtin_elementwise_int_binop(
2765 InterpState &S, CodePtr OpPC, const CallExpr *Call,
2766 llvm::function_ref<APInt(const APSInt &, const APSInt &)> Fn) {
2767 assert(Call->getNumArgs() == 2);
2768
2769 // Single integer case.
2770 if (!Call->getArg(Arg: 0)->getType()->isVectorType()) {
2771 assert(!Call->getArg(1)->getType()->isVectorType());
2772 APSInt RHS;
2773 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: RHS))
2774 return false;
2775 APSInt LHS;
2776 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: LHS))
2777 return false;
2778 APInt Result = Fn(LHS, RHS);
2779 pushInteger(S, Val: APSInt(std::move(Result), !LHS.isSigned()), QT: Call->getType());
2780 return true;
2781 }
2782
2783 const auto *VT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
2784 assert(VT->getElementType()->isIntegralOrEnumerationType());
2785 PrimType ElemT = *S.getContext().classify(T: VT->getElementType());
2786 unsigned NumElems = VT->getNumElements();
2787 bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
2788
2789 // Vector + Scalar case.
2790 if (!Call->getArg(Arg: 1)->getType()->isVectorType()) {
2791 assert(Call->getArg(1)->getType()->isIntegralOrEnumerationType());
2792
2793 APSInt RHS;
2794 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: RHS))
2795 return false;
2796 const Pointer &LHS = S.Stk.pop<Pointer>();
2797 const Pointer &Dst = S.Stk.peek<Pointer>();
2798
2799 for (unsigned I = 0; I != NumElems; ++I) {
2800 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
2801 Dst.elem<T>(I) = static_cast<T>(
2802 APSInt(Fn(LHS.elem<T>(I).toAPSInt(), RHS), DestUnsigned));
2803 });
2804 }
2805 Dst.initializeAllElements();
2806 return true;
2807 }
2808
2809 // Vector case.
2810 assert(Call->getArg(0)->getType()->isVectorType() &&
2811 Call->getArg(1)->getType()->isVectorType());
2812 assert(VT->getElementType() ==
2813 Call->getArg(1)->getType()->castAs<VectorType>()->getElementType());
2814 assert(VT->getNumElements() ==
2815 Call->getArg(1)->getType()->castAs<VectorType>()->getNumElements());
2816 assert(VT->getElementType()->isIntegralOrEnumerationType());
2817
2818 const Pointer &RHS = S.Stk.pop<Pointer>();
2819 const Pointer &LHS = S.Stk.pop<Pointer>();
2820 const Pointer &Dst = S.Stk.peek<Pointer>();
2821 for (unsigned I = 0; I != NumElems; ++I) {
2822 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
2823 APSInt Elem1 = LHS.elem<T>(I).toAPSInt();
2824 APSInt Elem2 = RHS.elem<T>(I).toAPSInt();
2825 Dst.elem<T>(I) = static_cast<T>(APSInt(Fn(Elem1, Elem2), DestUnsigned));
2826 });
2827 }
2828 Dst.initializeAllElements();
2829
2830 return true;
2831}
2832
2833static bool
2834interp__builtin_ia32_pack(InterpState &S, CodePtr, const CallExpr *E,
2835 llvm::function_ref<APInt(const APSInt &)> PackFn) {
2836 const auto *VT0 = E->getArg(Arg: 0)->getType()->castAs<VectorType>();
2837 [[maybe_unused]] const auto *VT1 =
2838 E->getArg(Arg: 1)->getType()->castAs<VectorType>();
2839 assert(VT0 && VT1 && "pack builtin VT0 and VT1 must be VectorType");
2840 assert(VT0->getElementType() == VT1->getElementType() &&
2841 VT0->getNumElements() == VT1->getNumElements() &&
2842 "pack builtin VT0 and VT1 ElementType must be same");
2843
2844 const Pointer &RHS = S.Stk.pop<Pointer>();
2845 const Pointer &LHS = S.Stk.pop<Pointer>();
2846 const Pointer &Dst = S.Stk.peek<Pointer>();
2847
2848 const ASTContext &ASTCtx = S.getASTContext();
2849 unsigned SrcBits = ASTCtx.getIntWidth(T: VT0->getElementType());
2850 unsigned LHSVecLen = VT0->getNumElements();
2851 unsigned SrcPerLane = 128 / SrcBits;
2852 unsigned Lanes = LHSVecLen * SrcBits / 128;
2853
2854 PrimType SrcT = *S.getContext().classify(T: VT0->getElementType());
2855 PrimType DstT = *S.getContext().classify(T: getElemType(P: Dst));
2856 bool IsUnsigend = getElemType(P: Dst)->isUnsignedIntegerType();
2857
2858 for (unsigned Lane = 0; Lane != Lanes; ++Lane) {
2859 unsigned BaseSrc = Lane * SrcPerLane;
2860 unsigned BaseDst = Lane * (2 * SrcPerLane);
2861
2862 for (unsigned I = 0; I != SrcPerLane; ++I) {
2863 INT_TYPE_SWITCH_NO_BOOL(SrcT, {
2864 APSInt A = LHS.elem<T>(BaseSrc + I).toAPSInt();
2865 APSInt B = RHS.elem<T>(BaseSrc + I).toAPSInt();
2866
2867 assignIntegral(S, Dst.atIndex(BaseDst + I), DstT,
2868 APSInt(PackFn(A), IsUnsigend));
2869 assignIntegral(S, Dst.atIndex(BaseDst + SrcPerLane + I), DstT,
2870 APSInt(PackFn(B), IsUnsigend));
2871 });
2872 }
2873 }
2874
2875 Dst.initializeAllElements();
2876 return true;
2877}
2878
2879static bool interp__builtin_elementwise_maxmin(InterpState &S, CodePtr OpPC,
2880 const CallExpr *Call,
2881 unsigned BuiltinID) {
2882 assert(Call->getNumArgs() == 2);
2883
2884 QualType Arg0Type = Call->getArg(Arg: 0)->getType();
2885
2886 // TODO: Support floating-point types.
2887 if (!(Arg0Type->isIntegerType() ||
2888 (Arg0Type->isVectorType() &&
2889 Arg0Type->castAs<VectorType>()->getElementType()->isIntegerType())))
2890 return false;
2891
2892 if (!Arg0Type->isVectorType()) {
2893 assert(!Call->getArg(1)->getType()->isVectorType());
2894 APSInt RHS;
2895 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: RHS))
2896 return false;
2897 APSInt LHS;
2898 if (!popToAPSInt(S, T: Arg0Type, Out&: LHS))
2899 return false;
2900 APInt Result;
2901 if (BuiltinID == Builtin::BI__builtin_elementwise_max) {
2902 Result = std::max(a: LHS, b: RHS);
2903 } else if (BuiltinID == Builtin::BI__builtin_elementwise_min) {
2904 Result = std::min(a: LHS, b: RHS);
2905 } else {
2906 llvm_unreachable("Wrong builtin ID");
2907 }
2908
2909 pushInteger(S, Val: APSInt(Result, !LHS.isSigned()), QT: Call->getType());
2910 return true;
2911 }
2912
2913 // Vector case.
2914 assert(Call->getArg(0)->getType()->isVectorType() &&
2915 Call->getArg(1)->getType()->isVectorType());
2916 const auto *VT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
2917 assert(VT->getElementType() ==
2918 Call->getArg(1)->getType()->castAs<VectorType>()->getElementType());
2919 assert(VT->getNumElements() ==
2920 Call->getArg(1)->getType()->castAs<VectorType>()->getNumElements());
2921 assert(VT->getElementType()->isIntegralOrEnumerationType());
2922
2923 const Pointer &RHS = S.Stk.pop<Pointer>();
2924 const Pointer &LHS = S.Stk.pop<Pointer>();
2925 const Pointer &Dst = S.Stk.peek<Pointer>();
2926 PrimType ElemT = *S.getContext().classify(T: VT->getElementType());
2927 unsigned NumElems = VT->getNumElements();
2928 for (unsigned I = 0; I != NumElems; ++I) {
2929 APSInt Elem1;
2930 APSInt Elem2;
2931 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
2932 Elem1 = LHS.elem<T>(I).toAPSInt();
2933 Elem2 = RHS.elem<T>(I).toAPSInt();
2934 });
2935
2936 APSInt Result;
2937 if (BuiltinID == Builtin::BI__builtin_elementwise_max) {
2938 Result = APSInt(std::max(a: Elem1, b: Elem2),
2939 Call->getType()->isUnsignedIntegerOrEnumerationType());
2940 } else if (BuiltinID == Builtin::BI__builtin_elementwise_min) {
2941 Result = APSInt(std::min(a: Elem1, b: Elem2),
2942 Call->getType()->isUnsignedIntegerOrEnumerationType());
2943 } else {
2944 llvm_unreachable("Wrong builtin ID");
2945 }
2946
2947 INT_TYPE_SWITCH_NO_BOOL(ElemT,
2948 { Dst.elem<T>(I) = static_cast<T>(Result); });
2949 }
2950 Dst.initializeAllElements();
2951
2952 return true;
2953}
2954
2955static bool interp__builtin_ia32_pmul(
2956 InterpState &S, CodePtr OpPC, const CallExpr *Call,
2957 llvm::function_ref<APInt(const APSInt &, const APSInt &, const APSInt &,
2958 const APSInt &)>
2959 Fn) {
2960 assert(Call->getArg(0)->getType()->isVectorType() &&
2961 Call->getArg(1)->getType()->isVectorType());
2962 const Pointer &RHS = S.Stk.pop<Pointer>();
2963 const Pointer &LHS = S.Stk.pop<Pointer>();
2964 const Pointer &Dst = S.Stk.peek<Pointer>();
2965
2966 const auto *VT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
2967 PrimType ElemT = *S.getContext().classify(T: VT->getElementType());
2968 unsigned NumElems = VT->getNumElements();
2969 const auto *DestVT = Call->getType()->castAs<VectorType>();
2970 PrimType DestElemT = *S.getContext().classify(T: DestVT->getElementType());
2971 bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
2972
2973 unsigned DstElem = 0;
2974 for (unsigned I = 0; I != NumElems; I += 2) {
2975 APSInt Result;
2976 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
2977 APSInt LoLHS = LHS.elem<T>(I).toAPSInt();
2978 APSInt HiLHS = LHS.elem<T>(I + 1).toAPSInt();
2979 APSInt LoRHS = RHS.elem<T>(I).toAPSInt();
2980 APSInt HiRHS = RHS.elem<T>(I + 1).toAPSInt();
2981 Result = APSInt(Fn(LoLHS, HiLHS, LoRHS, HiRHS), DestUnsigned);
2982 });
2983
2984 INT_TYPE_SWITCH_NO_BOOL(DestElemT,
2985 { Dst.elem<T>(DstElem) = static_cast<T>(Result); });
2986 ++DstElem;
2987 }
2988
2989 Dst.initializeAllElements();
2990 return true;
2991}
2992
2993static bool interp__builtin_ia32_psadbw(InterpState &S, CodePtr OpPC,
2994 const CallExpr *Call) {
2995 assert(Call->getNumArgs() == 2);
2996
2997 const Pointer &RHS = S.Stk.pop<Pointer>();
2998 const Pointer &LHS = S.Stk.pop<Pointer>();
2999 const Pointer &Dst = S.Stk.peek<Pointer>();
3000
3001 const auto *SrcVT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
3002 PrimType SrcElemT = *S.getContext().classify(T: SrcVT->getElementType());
3003 unsigned SourceLen = SrcVT->getNumElements();
3004 assert((SourceLen % 8) == 0);
3005
3006 const auto *DestVT = Call->getType()->castAs<VectorType>();
3007 PrimType DestElemT = *S.getContext().classify(T: DestVT->getElementType());
3008 bool DestUnsigned =
3009 DestVT->getElementType()->isUnsignedIntegerOrEnumerationType();
3010
3011 unsigned DstElem = 0;
3012 for (unsigned Lane = 0; Lane != SourceLen; Lane += 8) {
3013 APInt Sum(64, 0);
3014 for (unsigned I = 0; I != 8; ++I) {
3015 INT_TYPE_SWITCH_NO_BOOL(SrcElemT, {
3016 APSInt L = LHS.elem<T>(Lane + I).toAPSInt();
3017 APSInt R = RHS.elem<T>(Lane + I).toAPSInt();
3018 Sum += llvm::APIntOps::abdu(L.extOrTrunc(8), R.extOrTrunc(8)).zext(64);
3019 });
3020 }
3021
3022 INT_TYPE_SWITCH_NO_BOOL(DestElemT, {
3023 Dst.elem<T>(DstElem) = static_cast<T>(APSInt(Sum, DestUnsigned));
3024 });
3025 ++DstElem;
3026 }
3027
3028 Dst.initializeAllElements();
3029 return true;
3030}
3031
3032static bool interp__builtin_ia32_dbpsadbw(InterpState &S, CodePtr OpPC,
3033 const CallExpr *Call) {
3034 assert(Call->getNumArgs() == 3);
3035 uint64_t Imm;
3036 if (!popToUInt64(S, E: Call->getArg(Arg: 2), Out&: Imm))
3037 return false;
3038
3039 const Pointer &Src2 = S.Stk.pop<Pointer>();
3040 const Pointer &Src1 = S.Stk.pop<Pointer>();
3041 const Pointer &Dst = S.Stk.peek<Pointer>();
3042
3043 const auto *SrcVT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
3044 PrimType SrcElemT = *S.getContext().classify(T: SrcVT->getElementType());
3045 unsigned SourceLen = SrcVT->getNumElements();
3046
3047 const auto *DestVT = Call->getType()->castAs<VectorType>();
3048 PrimType DestElemT = *S.getContext().classify(T: DestVT->getElementType());
3049 bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
3050
3051 constexpr unsigned LaneSize = 16; // 128-bit lane = 16 bytes
3052
3053 // Phase 1: Shuffle Src2 using all four 2-bit fields of imm8.
3054 // Within each 128-bit lane, for group j (0..3), select a 4-byte block
3055 // from Src2 based on bits [2*j+1:2*j] of imm8.
3056 SmallVector<uint8_t, 64> Shuffled(SourceLen);
3057 for (unsigned I = 0; I < SourceLen; I += LaneSize) {
3058 for (unsigned J = 0; J < 4; ++J) {
3059 unsigned Part = (Imm >> (2 * J)) & 3;
3060 for (unsigned K = 0; K < 4; ++K) {
3061 INT_TYPE_SWITCH_NO_BOOL(SrcElemT, {
3062 Shuffled[I + 4 * J + K] =
3063 static_cast<uint8_t>(Src2.elem<T>(I + 4 * Part + K));
3064 });
3065 }
3066 }
3067 }
3068
3069 // Phase 2: Sliding SAD computation.
3070 // For every group of 4 output u16 values, compute absolute differences
3071 // using overlapping windows into Src1 and the shuffled array.
3072 unsigned Size = SourceLen / 2; // number of output u16 elements
3073 for (unsigned I = 0; I < Size; I += 4) {
3074 unsigned Sad[4] = {0, 0, 0, 0};
3075 for (unsigned J = 0; J < 4; ++J) {
3076 uint8_t A1, A2;
3077 INT_TYPE_SWITCH_NO_BOOL(SrcElemT, {
3078 A1 = static_cast<uint8_t>(Src1.elem<T>(2 * I + J));
3079 A2 = static_cast<uint8_t>(Src1.elem<T>(2 * I + J + 4));
3080 });
3081 uint8_t B0 = Shuffled[2 * I + J];
3082 uint8_t B1 = Shuffled[2 * I + J + 1];
3083 uint8_t B2 = Shuffled[2 * I + J + 2];
3084 uint8_t B3 = Shuffled[2 * I + J + 3];
3085 Sad[0] += (A1 > B0) ? (A1 - B0) : (B0 - A1);
3086 Sad[1] += (A1 > B1) ? (A1 - B1) : (B1 - A1);
3087 Sad[2] += (A2 > B2) ? (A2 - B2) : (B2 - A2);
3088 Sad[3] += (A2 > B3) ? (A2 - B3) : (B3 - A2);
3089 }
3090 for (unsigned R = 0; R < 4; ++R) {
3091 INT_TYPE_SWITCH_NO_BOOL(DestElemT, {
3092 Dst.elem<T>(I + R) =
3093 static_cast<T>(APSInt(APInt(16, Sad[R]), DestUnsigned));
3094 });
3095 }
3096 }
3097
3098 Dst.initializeAllElements();
3099 return true;
3100}
3101
3102static bool interp__builtin_ia32_mpsadbw(InterpState &S, CodePtr OpPC,
3103 const CallExpr *Call) {
3104 assert(Call->getNumArgs() == 3);
3105 uint64_t Imm;
3106 if (!popToUInt64(S, E: Call->getArg(Arg: 2), Out&: Imm))
3107 return false;
3108
3109 const Pointer &Src2 = S.Stk.pop<Pointer>();
3110 const Pointer &Src1 = S.Stk.pop<Pointer>();
3111 const Pointer &Dst = S.Stk.peek<Pointer>();
3112
3113 const auto *SrcVT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
3114 PrimType SrcElemT = *S.getContext().classify(T: SrcVT->getElementType());
3115 unsigned SourceLen = SrcVT->getNumElements();
3116 assert((SourceLen == 16 || SourceLen == 32) &&
3117 "MPSADBW operates on 128-bit or 256-bit vectors");
3118
3119 const auto *DestVT = Call->getType()->castAs<VectorType>();
3120 PrimType DestElemT = *S.getContext().classify(T: DestVT->getElementType());
3121 bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
3122
3123 constexpr unsigned LaneSize = 16; // 128-bit lane = 16 bytes
3124 unsigned NumLanes = SourceLen / LaneSize;
3125
3126 for (unsigned Lane = 0; Lane != NumLanes; ++Lane) {
3127 unsigned Ctrl = (Imm >> (3 * Lane)) & 0x7;
3128 unsigned AOff = ((Ctrl >> 2) & 1) * 4;
3129 unsigned BOff = (Ctrl & 3) * 4;
3130 for (unsigned J = 0; J != 8; ++J) {
3131 uint16_t Sad = 0;
3132 for (unsigned K = 0; K != 4; ++K) {
3133 uint8_t A, B;
3134 INT_TYPE_SWITCH_NO_BOOL(SrcElemT, {
3135 A = static_cast<uint8_t>(
3136 Src1.elem<T>(Lane * LaneSize + AOff + J + K));
3137 B = static_cast<uint8_t>(Src2.elem<T>(Lane * LaneSize + BOff + K));
3138 });
3139 Sad += (A > B) ? (A - B) : (B - A);
3140 }
3141 INT_TYPE_SWITCH_NO_BOOL(DestElemT, {
3142 Dst.elem<T>(Lane * 8 + J) =
3143 static_cast<T>(APSInt(APInt(16, Sad), DestUnsigned));
3144 });
3145 }
3146 }
3147
3148 Dst.initializeAllElements();
3149 return true;
3150}
3151
3152static bool interp_builtin_horizontal_int_binop(
3153 InterpState &S, CodePtr OpPC, const CallExpr *Call,
3154 llvm::function_ref<APInt(const APSInt &, const APSInt &)> Fn) {
3155 const auto *VT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
3156 PrimType ElemT = *S.getContext().classify(T: VT->getElementType());
3157 bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
3158
3159 const Pointer &RHS = S.Stk.pop<Pointer>();
3160 const Pointer &LHS = S.Stk.pop<Pointer>();
3161 const Pointer &Dst = S.Stk.peek<Pointer>();
3162 unsigned NumElts = VT->getNumElements();
3163 unsigned EltBits = S.getASTContext().getIntWidth(T: VT->getElementType());
3164 unsigned EltsPerLane = 128 / EltBits;
3165 unsigned Lanes = NumElts * EltBits / 128;
3166 unsigned DestIndex = 0;
3167
3168 for (unsigned Lane = 0; Lane < Lanes; ++Lane) {
3169 unsigned LaneStart = Lane * EltsPerLane;
3170 for (unsigned I = 0; I < EltsPerLane; I += 2) {
3171 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
3172 APSInt Elem1 = LHS.elem<T>(LaneStart + I).toAPSInt();
3173 APSInt Elem2 = LHS.elem<T>(LaneStart + I + 1).toAPSInt();
3174 APSInt ResL = APSInt(Fn(Elem1, Elem2), DestUnsigned);
3175 Dst.elem<T>(DestIndex++) = static_cast<T>(ResL);
3176 });
3177 }
3178
3179 for (unsigned I = 0; I < EltsPerLane; I += 2) {
3180 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
3181 APSInt Elem1 = RHS.elem<T>(LaneStart + I).toAPSInt();
3182 APSInt Elem2 = RHS.elem<T>(LaneStart + I + 1).toAPSInt();
3183 APSInt ResR = APSInt(Fn(Elem1, Elem2), DestUnsigned);
3184 Dst.elem<T>(DestIndex++) = static_cast<T>(ResR);
3185 });
3186 }
3187 }
3188 Dst.initializeAllElements();
3189 return true;
3190}
3191
3192static bool interp_builtin_horizontal_fp_binop(
3193 InterpState &S, CodePtr OpPC, const CallExpr *Call,
3194 llvm::function_ref<APFloat(const APFloat &, const APFloat &,
3195 llvm::RoundingMode)>
3196 Fn) {
3197 const Pointer &RHS = S.Stk.pop<Pointer>();
3198 const Pointer &LHS = S.Stk.pop<Pointer>();
3199 const Pointer &Dst = S.Stk.peek<Pointer>();
3200 FPOptions FPO = Call->getFPFeaturesInEffect(LO: S.Ctx.getLangOpts());
3201 llvm::RoundingMode RM = getRoundingMode(FPO);
3202 const auto *VT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
3203
3204 unsigned NumElts = VT->getNumElements();
3205 unsigned EltBits = S.getASTContext().getTypeSize(T: VT->getElementType());
3206 unsigned NumLanes = NumElts * EltBits / 128;
3207 unsigned NumElemsPerLane = NumElts / NumLanes;
3208 unsigned HalfElemsPerLane = NumElemsPerLane / 2;
3209
3210 for (unsigned L = 0; L != NumElts; L += NumElemsPerLane) {
3211 using T = PrimConv<PT_Float>::T;
3212 for (unsigned E = 0; E != HalfElemsPerLane; ++E) {
3213 APFloat Elem1 = LHS.elem<T>(I: L + (2 * E) + 0).getAPFloat();
3214 APFloat Elem2 = LHS.elem<T>(I: L + (2 * E) + 1).getAPFloat();
3215 Dst.elem<T>(I: L + E) = static_cast<T>(Fn(Elem1, Elem2, RM));
3216 }
3217 for (unsigned E = 0; E != HalfElemsPerLane; ++E) {
3218 APFloat Elem1 = RHS.elem<T>(I: L + (2 * E) + 0).getAPFloat();
3219 APFloat Elem2 = RHS.elem<T>(I: L + (2 * E) + 1).getAPFloat();
3220 Dst.elem<T>(I: L + E + HalfElemsPerLane) =
3221 static_cast<T>(Fn(Elem1, Elem2, RM));
3222 }
3223 }
3224 Dst.initializeAllElements();
3225 return true;
3226}
3227
3228static bool interp__builtin_ia32_addsub(InterpState &S, CodePtr OpPC,
3229 const CallExpr *Call) {
3230 // Addsub: alternates between subtraction and addition
3231 // Result[i] = (i % 2 == 0) ? (a[i] - b[i]) : (a[i] + b[i])
3232 const Pointer &RHS = S.Stk.pop<Pointer>();
3233 const Pointer &LHS = S.Stk.pop<Pointer>();
3234 const Pointer &Dst = S.Stk.peek<Pointer>();
3235 FPOptions FPO = Call->getFPFeaturesInEffect(LO: S.Ctx.getLangOpts());
3236 llvm::RoundingMode RM = getRoundingMode(FPO);
3237 const auto *VT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
3238 unsigned NumElems = VT->getNumElements();
3239
3240 using T = PrimConv<PT_Float>::T;
3241 for (unsigned I = 0; I != NumElems; ++I) {
3242 APFloat LElem = LHS.elem<T>(I).getAPFloat();
3243 APFloat RElem = RHS.elem<T>(I).getAPFloat();
3244 if (I % 2 == 0) {
3245 // Even indices: subtract
3246 LElem.subtract(RHS: RElem, RM);
3247 } else {
3248 // Odd indices: add
3249 LElem.add(RHS: RElem, RM);
3250 }
3251 Dst.elem<T>(I) = static_cast<T>(LElem);
3252 }
3253 Dst.initializeAllElements();
3254 return true;
3255}
3256
3257static bool interp__builtin_ia32_pclmulqdq(InterpState &S, CodePtr OpPC,
3258 const CallExpr *Call) {
3259 // PCLMULQDQ: carry-less multiplication of selected 64-bit halves
3260 // imm8 bit 0: selects lower (0) or upper (1) 64 bits of first operand
3261 // imm8 bit 4: selects lower (0) or upper (1) 64 bits of second operand
3262 assert(Call->getArg(0)->getType()->isVectorType() &&
3263 Call->getArg(1)->getType()->isVectorType());
3264
3265 // Extract imm8 argument
3266 APSInt Imm8;
3267 if (!popToAPSInt(S, E: Call->getArg(Arg: 2), Out&: Imm8))
3268 return false;
3269 bool SelectUpperA = (Imm8 & 0x01) != 0;
3270 bool SelectUpperB = (Imm8 & 0x10) != 0;
3271
3272 const Pointer &RHS = S.Stk.pop<Pointer>();
3273 const Pointer &LHS = S.Stk.pop<Pointer>();
3274 const Pointer &Dst = S.Stk.peek<Pointer>();
3275
3276 const auto *VT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
3277 PrimType ElemT = *S.getContext().classify(T: VT->getElementType());
3278 unsigned NumElems = VT->getNumElements();
3279 const auto *DestVT = Call->getType()->castAs<VectorType>();
3280 PrimType DestElemT = *S.getContext().classify(T: DestVT->getElementType());
3281 bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
3282
3283 // Process each 128-bit lane (2 elements at a time)
3284 for (unsigned Lane = 0; Lane < NumElems; Lane += 2) {
3285 APSInt A0, A1, B0, B1;
3286 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
3287 A0 = LHS.elem<T>(Lane + 0).toAPSInt();
3288 A1 = LHS.elem<T>(Lane + 1).toAPSInt();
3289 B0 = RHS.elem<T>(Lane + 0).toAPSInt();
3290 B1 = RHS.elem<T>(Lane + 1).toAPSInt();
3291 });
3292
3293 // Select the appropriate 64-bit values based on imm8
3294 APInt A = SelectUpperA ? A1 : A0;
3295 APInt B = SelectUpperB ? B1 : B0;
3296
3297 // Extend both operands to 128 bits for carry-less multiplication
3298 APInt A128 = A.zext(width: 128);
3299 APInt B128 = B.zext(width: 128);
3300
3301 // Use APIntOps::clmul for carry-less multiplication
3302 APInt Result = llvm::APIntOps::clmul(LHS: A128, RHS: B128);
3303
3304 // Split the 128-bit result into two 64-bit halves
3305 APSInt ResultLow(Result.extractBits(numBits: 64, bitPosition: 0), DestUnsigned);
3306 APSInt ResultHigh(Result.extractBits(numBits: 64, bitPosition: 64), DestUnsigned);
3307
3308 INT_TYPE_SWITCH_NO_BOOL(DestElemT, {
3309 Dst.elem<T>(Lane + 0) = static_cast<T>(ResultLow);
3310 Dst.elem<T>(Lane + 1) = static_cast<T>(ResultHigh);
3311 });
3312 }
3313
3314 Dst.initializeAllElements();
3315 return true;
3316}
3317
3318static bool interp__builtin_elementwise_triop_fp(
3319 InterpState &S, CodePtr OpPC, const CallExpr *Call,
3320 llvm::function_ref<APFloat(const APFloat &, const APFloat &,
3321 const APFloat &, llvm::RoundingMode)>
3322 Fn) {
3323 assert(Call->getNumArgs() == 3);
3324
3325 FPOptions FPO = Call->getFPFeaturesInEffect(LO: S.Ctx.getLangOpts());
3326 llvm::RoundingMode RM = getRoundingMode(FPO);
3327 QualType Arg1Type = Call->getArg(Arg: 0)->getType();
3328 QualType Arg2Type = Call->getArg(Arg: 1)->getType();
3329 QualType Arg3Type = Call->getArg(Arg: 2)->getType();
3330
3331 // Non-vector floating point types.
3332 if (!Arg1Type->isVectorType()) {
3333 assert(!Arg2Type->isVectorType());
3334 assert(!Arg3Type->isVectorType());
3335 (void)Arg2Type;
3336 (void)Arg3Type;
3337
3338 const Floating &Z = S.Stk.pop<Floating>();
3339 const Floating &Y = S.Stk.pop<Floating>();
3340 const Floating &X = S.Stk.pop<Floating>();
3341 APFloat F = Fn(X.getAPFloat(), Y.getAPFloat(), Z.getAPFloat(), RM);
3342 Floating Result = S.allocFloat(Sem: X.getSemantics());
3343 Result.copy(F);
3344 S.Stk.push<Floating>(Args&: Result);
3345 return true;
3346 }
3347
3348 // Vector type.
3349 assert(Arg1Type->isVectorType() && Arg2Type->isVectorType() &&
3350 Arg3Type->isVectorType());
3351
3352 const VectorType *VecTy = Arg1Type->castAs<VectorType>();
3353 QualType ElemQT = VecTy->getElementType();
3354 unsigned NumElems = VecTy->getNumElements();
3355
3356 assert(ElemQT == Arg2Type->castAs<VectorType>()->getElementType() &&
3357 ElemQT == Arg3Type->castAs<VectorType>()->getElementType());
3358 assert(NumElems == Arg2Type->castAs<VectorType>()->getNumElements() &&
3359 NumElems == Arg3Type->castAs<VectorType>()->getNumElements());
3360 assert(ElemQT->isRealFloatingType());
3361 (void)ElemQT;
3362
3363 const Pointer &VZ = S.Stk.pop<Pointer>();
3364 const Pointer &VY = S.Stk.pop<Pointer>();
3365 const Pointer &VX = S.Stk.pop<Pointer>();
3366 const Pointer &Dst = S.Stk.peek<Pointer>();
3367 for (unsigned I = 0; I != NumElems; ++I) {
3368 using T = PrimConv<PT_Float>::T;
3369 APFloat X = VX.elem<T>(I).getAPFloat();
3370 APFloat Y = VY.elem<T>(I).getAPFloat();
3371 APFloat Z = VZ.elem<T>(I).getAPFloat();
3372 APFloat F = Fn(X, Y, Z, RM);
3373 Dst.elem<Floating>(I) = Floating(F);
3374 }
3375 Dst.initializeAllElements();
3376 return true;
3377}
3378
3379/// AVX512 predicated move: "Result = Mask[] ? LHS[] : RHS[]".
3380static bool interp__builtin_ia32_select(InterpState &S, CodePtr OpPC,
3381 const CallExpr *Call) {
3382 const Pointer &RHS = S.Stk.pop<Pointer>();
3383 const Pointer &LHS = S.Stk.pop<Pointer>();
3384 APSInt Mask;
3385 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: Mask))
3386 return false;
3387 const Pointer &Dst = S.Stk.peek<Pointer>();
3388
3389 assert(LHS.getNumElems() == RHS.getNumElems());
3390 assert(LHS.getNumElems() == Dst.getNumElems());
3391 unsigned NumElems = LHS.getNumElems();
3392 PrimType ElemT = LHS.getFieldDesc()->getPrimType();
3393 PrimType DstElemT = Dst.getFieldDesc()->getPrimType();
3394
3395 for (unsigned I = 0; I != NumElems; ++I) {
3396 if (ElemT == PT_Float) {
3397 assert(DstElemT == PT_Float);
3398 Dst.elem<Floating>(I) =
3399 Mask[I] ? LHS.elem<Floating>(I) : RHS.elem<Floating>(I);
3400 } else {
3401 APSInt Elem;
3402 INT_TYPE_SWITCH(ElemT, {
3403 Elem = Mask[I] ? LHS.elem<T>(I).toAPSInt() : RHS.elem<T>(I).toAPSInt();
3404 });
3405 INT_TYPE_SWITCH_NO_BOOL(DstElemT,
3406 { Dst.elem<T>(I) = static_cast<T>(Elem); });
3407 }
3408 }
3409 Dst.initializeAllElements();
3410
3411 return true;
3412}
3413
3414/// Scalar variant of AVX512 predicated select:
3415/// Result[i] = (Mask bit 0) ? LHS[i] : RHS[i], but only element 0 may change.
3416/// All other elements are taken from RHS.
3417static bool interp__builtin_ia32_select_scalar(InterpState &S,
3418 const CallExpr *Call) {
3419 unsigned N =
3420 Call->getArg(Arg: 1)->getType()->castAs<VectorType>()->getNumElements();
3421
3422 const Pointer &W = S.Stk.pop<Pointer>();
3423 const Pointer &A = S.Stk.pop<Pointer>();
3424 APSInt U;
3425 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: U))
3426 return false;
3427 const Pointer &Dst = S.Stk.peek<Pointer>();
3428
3429 bool TakeA0 = U.getZExtValue() & 1ULL;
3430
3431 for (unsigned I = TakeA0; I != N; ++I)
3432 Dst.elem<Floating>(I) = W.elem<Floating>(I);
3433 if (TakeA0)
3434 Dst.elem<Floating>(I: 0) = A.elem<Floating>(I: 0);
3435
3436 Dst.initializeAllElements();
3437 return true;
3438}
3439
3440static bool interp__builtin_ia32_test_op(
3441 InterpState &S, CodePtr OpPC, const CallExpr *Call,
3442 llvm::function_ref<bool(const APInt &A, const APInt &B)> Fn) {
3443 const Pointer &RHS = S.Stk.pop<Pointer>();
3444 const Pointer &LHS = S.Stk.pop<Pointer>();
3445
3446 assert(LHS.getNumElems() == RHS.getNumElems());
3447
3448 unsigned SourceLen = LHS.getNumElems();
3449 QualType ElemQT = getElemType(P: LHS);
3450 OptPrimType ElemPT = S.getContext().classify(T: ElemQT);
3451 unsigned LaneWidth = S.getASTContext().getTypeSize(T: ElemQT);
3452
3453 APInt AWide(LaneWidth * SourceLen, 0);
3454 APInt BWide(LaneWidth * SourceLen, 0);
3455
3456 for (unsigned I = 0; I != SourceLen; ++I) {
3457 APInt ALane;
3458 APInt BLane;
3459
3460 if (ElemQT->isIntegerType()) { // Get value.
3461 INT_TYPE_SWITCH_NO_BOOL(*ElemPT, {
3462 ALane = LHS.elem<T>(I).toAPSInt();
3463 BLane = RHS.elem<T>(I).toAPSInt();
3464 });
3465 } else if (ElemQT->isFloatingType()) { // Get only sign bit.
3466 using T = PrimConv<PT_Float>::T;
3467 ALane = LHS.elem<T>(I).getAPFloat().bitcastToAPInt().isNegative();
3468 BLane = RHS.elem<T>(I).getAPFloat().bitcastToAPInt().isNegative();
3469 } else { // Must be integer or floating type.
3470 return false;
3471 }
3472 AWide.insertBits(SubBits: ALane, bitPosition: I * LaneWidth);
3473 BWide.insertBits(SubBits: BLane, bitPosition: I * LaneWidth);
3474 }
3475 pushInteger(S, Val: Fn(AWide, BWide), QT: Call->getType());
3476 return true;
3477}
3478
3479static bool interp__builtin_ia32_movmsk_op(InterpState &S, CodePtr OpPC,
3480 const CallExpr *Call) {
3481 assert(Call->getNumArgs() == 1);
3482
3483 const Pointer &Source = S.Stk.pop<Pointer>();
3484
3485 unsigned SourceLen = Source.getNumElems();
3486 QualType ElemQT = getElemType(P: Source);
3487 OptPrimType ElemT = S.getContext().classify(T: ElemQT);
3488 unsigned ResultLen =
3489 S.getASTContext().getTypeSize(T: Call->getType()); // Always 32-bit integer.
3490 APInt Result(ResultLen, 0);
3491
3492 for (unsigned I = 0; I != SourceLen; ++I) {
3493 APInt Elem;
3494 if (ElemQT->isIntegerType()) {
3495 INT_TYPE_SWITCH_NO_BOOL(*ElemT, { Elem = Source.elem<T>(I).toAPSInt(); });
3496 } else if (ElemQT->isRealFloatingType()) {
3497 using T = PrimConv<PT_Float>::T;
3498 Elem = Source.elem<T>(I).getAPFloat().bitcastToAPInt();
3499 } else {
3500 return false;
3501 }
3502 Result.setBitVal(BitPosition: I, BitValue: Elem.isNegative());
3503 }
3504 pushInteger(S, Val: Result, QT: Call->getType());
3505 return true;
3506}
3507
3508static bool interp__builtin_elementwise_triop(
3509 InterpState &S, CodePtr OpPC, const CallExpr *Call,
3510 llvm::function_ref<APInt(const APSInt &, const APSInt &, const APSInt &)>
3511 Fn) {
3512 assert(Call->getNumArgs() == 3);
3513
3514 QualType Arg0Type = Call->getArg(Arg: 0)->getType();
3515 QualType Arg2Type = Call->getArg(Arg: 2)->getType();
3516 // Non-vector integer types.
3517 if (!Arg0Type->isVectorType()) {
3518 APSInt Op2;
3519 if (!popToAPSInt(S, T: Arg2Type, Out&: Op2))
3520 return false;
3521 APSInt Op1;
3522 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: Op1))
3523 return false;
3524 APSInt Op0;
3525 if (!popToAPSInt(S, T: Arg0Type, Out&: Op0))
3526 return false;
3527 APSInt Result = APSInt(Fn(Op0, Op1, Op2), Op0.isUnsigned());
3528 pushInteger(S, Val: Result, QT: Call->getType());
3529 return true;
3530 }
3531
3532 const auto *VecT = Arg0Type->castAs<VectorType>();
3533 PrimType ElemT = *S.getContext().classify(T: VecT->getElementType());
3534 unsigned NumElems = VecT->getNumElements();
3535 bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
3536
3537 // Vector + Vector + Scalar case.
3538 if (!Arg2Type->isVectorType()) {
3539 APSInt Op2;
3540 if (!popToAPSInt(S, T: Arg2Type, Out&: Op2))
3541 return false;
3542
3543 const Pointer &Op1 = S.Stk.pop<Pointer>();
3544 const Pointer &Op0 = S.Stk.pop<Pointer>();
3545 const Pointer &Dst = S.Stk.peek<Pointer>();
3546 for (unsigned I = 0; I != NumElems; ++I) {
3547 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
3548 Dst.elem<T>(I) = static_cast<T>(APSInt(
3549 Fn(Op0.elem<T>(I).toAPSInt(), Op1.elem<T>(I).toAPSInt(), Op2),
3550 DestUnsigned));
3551 });
3552 }
3553 Dst.initializeAllElements();
3554
3555 return true;
3556 }
3557
3558 // Vector type.
3559 const Pointer &Op2 = S.Stk.pop<Pointer>();
3560 const Pointer &Op1 = S.Stk.pop<Pointer>();
3561 const Pointer &Op0 = S.Stk.pop<Pointer>();
3562 const Pointer &Dst = S.Stk.peek<Pointer>();
3563 for (unsigned I = 0; I != NumElems; ++I) {
3564 APSInt Val0, Val1, Val2;
3565 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
3566 Val0 = Op0.elem<T>(I).toAPSInt();
3567 Val1 = Op1.elem<T>(I).toAPSInt();
3568 Val2 = Op2.elem<T>(I).toAPSInt();
3569 });
3570 APSInt Result = APSInt(Fn(Val0, Val1, Val2), Val0.isUnsigned());
3571 INT_TYPE_SWITCH_NO_BOOL(ElemT,
3572 { Dst.elem<T>(I) = static_cast<T>(Result); });
3573 }
3574 Dst.initializeAllElements();
3575
3576 return true;
3577}
3578
3579static bool interp__builtin_ia32_extract_vector(InterpState &S, CodePtr OpPC,
3580 const CallExpr *Call,
3581 unsigned ID) {
3582 assert(Call->getNumArgs() == 2);
3583
3584 APSInt ImmAPS;
3585 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: ImmAPS))
3586 return false;
3587 uint64_t Index = ImmAPS.getZExtValue();
3588
3589 const Pointer &Src = S.Stk.pop<Pointer>();
3590 if (!Src.getFieldDesc()->isPrimitiveArray())
3591 return false;
3592
3593 const Pointer &Dst = S.Stk.peek<Pointer>();
3594 if (!Dst.getFieldDesc()->isPrimitiveArray())
3595 return false;
3596
3597 unsigned SrcElems = Src.getNumElems();
3598 unsigned DstElems = Dst.getNumElems();
3599
3600 unsigned NumLanes = SrcElems / DstElems;
3601 unsigned Lane = static_cast<unsigned>(Index % NumLanes);
3602 unsigned ExtractPos = Lane * DstElems;
3603
3604 PrimType ElemT = Src.getFieldDesc()->getPrimType();
3605
3606 TYPE_SWITCH(ElemT, {
3607 for (unsigned I = 0; I != DstElems; ++I) {
3608 Dst.elem<T>(I) = Src.elem<T>(ExtractPos + I);
3609 }
3610 });
3611
3612 Dst.initializeAllElements();
3613 return true;
3614}
3615
3616static bool interp__builtin_ia32_extract_vector_masked(InterpState &S,
3617 CodePtr OpPC,
3618 const CallExpr *Call,
3619 unsigned ID) {
3620 assert(Call->getNumArgs() == 4);
3621
3622 APSInt MaskAPS;
3623 if (!popToAPSInt(S, E: Call->getArg(Arg: 3), Out&: MaskAPS))
3624 return false;
3625 const Pointer &Merge = S.Stk.pop<Pointer>();
3626 APSInt ImmAPS;
3627 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: ImmAPS))
3628 return false;
3629 const Pointer &Src = S.Stk.pop<Pointer>();
3630
3631 if (!Src.getFieldDesc()->isPrimitiveArray() ||
3632 !Merge.getFieldDesc()->isPrimitiveArray())
3633 return false;
3634
3635 const Pointer &Dst = S.Stk.peek<Pointer>();
3636 if (!Dst.getFieldDesc()->isPrimitiveArray())
3637 return false;
3638
3639 unsigned SrcElems = Src.getNumElems();
3640 unsigned DstElems = Dst.getNumElems();
3641
3642 unsigned NumLanes = SrcElems / DstElems;
3643 unsigned Lane = static_cast<unsigned>(ImmAPS.getZExtValue() % NumLanes);
3644 unsigned Base = Lane * DstElems;
3645
3646 PrimType ElemT = Src.getFieldDesc()->getPrimType();
3647
3648 TYPE_SWITCH(ElemT, {
3649 for (unsigned I = 0; I != DstElems; ++I) {
3650 if (MaskAPS[I])
3651 Dst.elem<T>(I) = Src.elem<T>(Base + I);
3652 else
3653 Dst.elem<T>(I) = Merge.elem<T>(I);
3654 }
3655 });
3656
3657 Dst.initializeAllElements();
3658 return true;
3659}
3660
3661static bool interp__builtin_ia32_insert_subvector(InterpState &S, CodePtr OpPC,
3662 const CallExpr *Call,
3663 unsigned ID) {
3664 assert(Call->getNumArgs() == 3);
3665
3666 APSInt ImmAPS;
3667 if (!popToAPSInt(S, E: Call->getArg(Arg: 2), Out&: ImmAPS))
3668 return false;
3669 uint64_t Index = ImmAPS.getZExtValue();
3670
3671 const Pointer &SubVec = S.Stk.pop<Pointer>();
3672 if (!SubVec.getFieldDesc()->isPrimitiveArray())
3673 return false;
3674
3675 const Pointer &BaseVec = S.Stk.pop<Pointer>();
3676 if (!BaseVec.getFieldDesc()->isPrimitiveArray())
3677 return false;
3678
3679 const Pointer &Dst = S.Stk.peek<Pointer>();
3680
3681 unsigned BaseElements = BaseVec.getNumElems();
3682 unsigned SubElements = SubVec.getNumElems();
3683
3684 assert(SubElements != 0 && BaseElements != 0 &&
3685 (BaseElements % SubElements) == 0);
3686
3687 unsigned NumLanes = BaseElements / SubElements;
3688 unsigned Lane = static_cast<unsigned>(Index % NumLanes);
3689 unsigned InsertPos = Lane * SubElements;
3690
3691 PrimType ElemT = BaseVec.getFieldDesc()->getPrimType();
3692
3693 TYPE_SWITCH(ElemT, {
3694 for (unsigned I = 0; I != BaseElements; ++I)
3695 Dst.elem<T>(I) = BaseVec.elem<T>(I);
3696 for (unsigned I = 0; I != SubElements; ++I)
3697 Dst.elem<T>(InsertPos + I) = SubVec.elem<T>(I);
3698 });
3699
3700 Dst.initializeAllElements();
3701 return true;
3702}
3703
3704static bool interp__builtin_ia32_phminposuw(InterpState &S, CodePtr OpPC,
3705 const CallExpr *Call) {
3706 assert(Call->getNumArgs() == 1);
3707
3708 const Pointer &Source = S.Stk.pop<Pointer>();
3709 const Pointer &Dest = S.Stk.peek<Pointer>();
3710
3711 unsigned SourceLen = Source.getNumElems();
3712 QualType ElemQT = getElemType(P: Source);
3713 OptPrimType ElemT = S.getContext().classify(T: ElemQT);
3714 unsigned ElemBitWidth = S.getASTContext().getTypeSize(T: ElemQT);
3715
3716 bool DestUnsigned = Call->getCallReturnType(Ctx: S.getASTContext())
3717 ->castAs<VectorType>()
3718 ->getElementType()
3719 ->isUnsignedIntegerOrEnumerationType();
3720
3721 INT_TYPE_SWITCH_NO_BOOL(*ElemT, {
3722 APSInt MinIndex(ElemBitWidth, DestUnsigned);
3723 APSInt MinVal = Source.elem<T>(0).toAPSInt();
3724
3725 for (unsigned I = 1; I != SourceLen; ++I) {
3726 APSInt Val = Source.elem<T>(I).toAPSInt();
3727 if (MinVal.ugt(Val)) {
3728 MinVal = Val;
3729 MinIndex = I;
3730 }
3731 }
3732
3733 Dest.elem<T>(0) = static_cast<T>(MinVal);
3734 Dest.elem<T>(1) = static_cast<T>(MinIndex);
3735 for (unsigned I = 2; I != SourceLen; ++I) {
3736 Dest.elem<T>(I) = static_cast<T>(APSInt(ElemBitWidth, DestUnsigned));
3737 }
3738 });
3739 Dest.initializeAllElements();
3740 return true;
3741}
3742
3743static bool interp__builtin_ia32_pternlog(InterpState &S, CodePtr OpPC,
3744 const CallExpr *Call, bool MaskZ) {
3745 assert(Call->getNumArgs() == 5);
3746
3747 APSInt UVal;
3748 if (!popToAPSInt(S, E: Call->getArg(Arg: 4), Out&: UVal))
3749 return false;
3750 APInt U = UVal; // Lane mask
3751 APSInt ImmVal;
3752 if (!popToAPSInt(S, E: Call->getArg(Arg: 3), Out&: ImmVal))
3753 return false;
3754 APInt Imm = ImmVal; // Ternary truth table
3755 const Pointer &C = S.Stk.pop<Pointer>();
3756 const Pointer &B = S.Stk.pop<Pointer>();
3757 const Pointer &A = S.Stk.pop<Pointer>();
3758 const Pointer &Dst = S.Stk.peek<Pointer>();
3759
3760 unsigned DstLen = A.getNumElems();
3761 QualType ElemQT = getElemType(P: A);
3762 OptPrimType ElemT = S.getContext().classify(T: ElemQT);
3763 unsigned LaneWidth = S.getASTContext().getTypeSize(T: ElemQT);
3764 bool DstUnsigned = ElemQT->isUnsignedIntegerOrEnumerationType();
3765
3766 INT_TYPE_SWITCH_NO_BOOL(*ElemT, {
3767 for (unsigned I = 0; I != DstLen; ++I) {
3768 APInt ALane = A.elem<T>(I).toAPSInt();
3769 APInt BLane = B.elem<T>(I).toAPSInt();
3770 APInt CLane = C.elem<T>(I).toAPSInt();
3771 APInt RLane(LaneWidth, 0);
3772 if (U[I]) { // If lane not masked, compute ternary logic.
3773 for (unsigned Bit = 0; Bit != LaneWidth; ++Bit) {
3774 unsigned ABit = ALane[Bit];
3775 unsigned BBit = BLane[Bit];
3776 unsigned CBit = CLane[Bit];
3777 unsigned Idx = (ABit << 2) | (BBit << 1) | (CBit);
3778 RLane.setBitVal(Bit, Imm[Idx]);
3779 }
3780 Dst.elem<T>(I) = static_cast<T>(APSInt(RLane, DstUnsigned));
3781 } else if (MaskZ) { // If zero masked, zero the lane.
3782 Dst.elem<T>(I) = static_cast<T>(APSInt(RLane, DstUnsigned));
3783 } else { // Just masked, put in A lane.
3784 Dst.elem<T>(I) = static_cast<T>(APSInt(ALane, DstUnsigned));
3785 }
3786 }
3787 });
3788 Dst.initializeAllElements();
3789 return true;
3790}
3791
3792static bool interp__builtin_ia32_vec_ext(InterpState &S, CodePtr OpPC,
3793 const CallExpr *Call, unsigned ID) {
3794 assert(Call->getNumArgs() == 2);
3795
3796 APSInt ImmAPS;
3797 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: ImmAPS))
3798 return false;
3799 const Pointer &Vec = S.Stk.pop<Pointer>();
3800 if (!Vec.getFieldDesc()->isPrimitiveArray())
3801 return false;
3802
3803 unsigned NumElems = Vec.getNumElems();
3804 unsigned Index =
3805 static_cast<unsigned>(ImmAPS.getZExtValue() & (NumElems - 1));
3806
3807 PrimType ElemT = Vec.getFieldDesc()->getPrimType();
3808 // FIXME(#161685): Replace float+int split with a numeric-only type switch
3809 if (ElemT == PT_Float) {
3810 S.Stk.push<Floating>(Args&: Vec.elem<Floating>(I: Index));
3811 return true;
3812 }
3813 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
3814 APSInt V = Vec.elem<T>(Index).toAPSInt();
3815 pushInteger(S, V, Call->getType());
3816 });
3817
3818 return true;
3819}
3820
3821static bool interp__builtin_ia32_vec_set(InterpState &S, CodePtr OpPC,
3822 const CallExpr *Call, unsigned ID) {
3823 assert(Call->getNumArgs() == 3);
3824
3825 APSInt ImmAPS;
3826 if (!popToAPSInt(S, E: Call->getArg(Arg: 2), Out&: ImmAPS))
3827 return false;
3828 APSInt ValAPS;
3829 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: ValAPS))
3830 return false;
3831
3832 const Pointer &Base = S.Stk.pop<Pointer>();
3833 if (!Base.getFieldDesc()->isPrimitiveArray())
3834 return false;
3835
3836 const Pointer &Dst = S.Stk.peek<Pointer>();
3837
3838 unsigned NumElems = Base.getNumElems();
3839 unsigned Index =
3840 static_cast<unsigned>(ImmAPS.getZExtValue() & (NumElems - 1));
3841
3842 PrimType ElemT = Base.getFieldDesc()->getPrimType();
3843 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
3844 for (unsigned I = 0; I != NumElems; ++I)
3845 Dst.elem<T>(I) = Base.elem<T>(I);
3846 Dst.elem<T>(Index) = static_cast<T>(ValAPS);
3847 });
3848
3849 Dst.initializeAllElements();
3850 return true;
3851}
3852
3853static bool evalICmpImm(uint8_t Imm, const APSInt &A, const APSInt &B,
3854 bool IsUnsigned) {
3855 switch (Imm & 0x7) {
3856 case 0x00: // _MM_CMPINT_EQ
3857 return (A == B);
3858 case 0x01: // _MM_CMPINT_LT
3859 return IsUnsigned ? A.ult(RHS: B) : A.slt(RHS: B);
3860 case 0x02: // _MM_CMPINT_LE
3861 return IsUnsigned ? A.ule(RHS: B) : A.sle(RHS: B);
3862 case 0x03: // _MM_CMPINT_FALSE
3863 return false;
3864 case 0x04: // _MM_CMPINT_NE
3865 return (A != B);
3866 case 0x05: // _MM_CMPINT_NLT
3867 return IsUnsigned ? A.ugt(RHS: B) : A.sgt(RHS: B);
3868 case 0x06: // _MM_CMPINT_NLE
3869 return IsUnsigned ? A.uge(RHS: B) : A.sge(RHS: B);
3870 case 0x07: // _MM_CMPINT_TRUE
3871 return true;
3872 default:
3873 llvm_unreachable("Invalid Op");
3874 }
3875}
3876
3877static bool interp__builtin_ia32_cmp_mask(InterpState &S, CodePtr OpPC,
3878 const CallExpr *Call, unsigned ID,
3879 bool IsUnsigned) {
3880 assert(Call->getNumArgs() == 4);
3881
3882 APSInt Mask;
3883 if (!popToAPSInt(S, E: Call->getArg(Arg: 3), Out&: Mask))
3884 return false;
3885 APSInt Opcode;
3886 if (!popToAPSInt(S, E: Call->getArg(Arg: 2), Out&: Opcode))
3887 return false;
3888 unsigned CmpOp = static_cast<unsigned>(Opcode.getZExtValue());
3889 const Pointer &RHS = S.Stk.pop<Pointer>();
3890 const Pointer &LHS = S.Stk.pop<Pointer>();
3891
3892 assert(LHS.getNumElems() == RHS.getNumElems());
3893
3894 APInt RetMask = APInt::getZero(numBits: LHS.getNumElems());
3895 unsigned VectorLen = LHS.getNumElems();
3896 PrimType ElemT = LHS.getFieldDesc()->getPrimType();
3897
3898 for (unsigned ElemNum = 0; ElemNum < VectorLen; ++ElemNum) {
3899 APSInt A, B;
3900 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
3901 A = LHS.elem<T>(ElemNum).toAPSInt();
3902 B = RHS.elem<T>(ElemNum).toAPSInt();
3903 });
3904 RetMask.setBitVal(BitPosition: ElemNum,
3905 BitValue: Mask[ElemNum] && evalICmpImm(Imm: CmpOp, A, B, IsUnsigned));
3906 }
3907 pushInteger(S, Val: RetMask, QT: Call->getType());
3908 return true;
3909}
3910
3911static bool interp__builtin_ia32_vpconflict(InterpState &S, CodePtr OpPC,
3912 const CallExpr *Call) {
3913 assert(Call->getNumArgs() == 1);
3914
3915 QualType Arg0Type = Call->getArg(Arg: 0)->getType();
3916 const auto *VecT = Arg0Type->castAs<VectorType>();
3917 PrimType ElemT = *S.getContext().classify(T: VecT->getElementType());
3918 unsigned NumElems = VecT->getNumElements();
3919 bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
3920 const Pointer &Src = S.Stk.pop<Pointer>();
3921 const Pointer &Dst = S.Stk.peek<Pointer>();
3922
3923 for (unsigned I = 0; I != NumElems; ++I) {
3924 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
3925 APSInt ElemI = Src.elem<T>(I).toAPSInt();
3926 APInt ConflictMask(ElemI.getBitWidth(), 0);
3927 for (unsigned J = 0; J != I; ++J) {
3928 APSInt ElemJ = Src.elem<T>(J).toAPSInt();
3929 ConflictMask.setBitVal(J, ElemI == ElemJ);
3930 }
3931 Dst.elem<T>(I) = static_cast<T>(APSInt(ConflictMask, DestUnsigned));
3932 });
3933 }
3934 Dst.initializeAllElements();
3935 return true;
3936}
3937
3938static bool interp__builtin_ia32_cvt_vec2mask(InterpState &S, CodePtr OpPC,
3939 const CallExpr *Call,
3940 unsigned ID) {
3941 assert(Call->getNumArgs() == 1);
3942
3943 const Pointer &Vec = S.Stk.pop<Pointer>();
3944 unsigned RetWidth = S.getASTContext().getIntWidth(T: Call->getType());
3945 APInt RetMask(RetWidth, 0);
3946
3947 unsigned VectorLen = Vec.getNumElems();
3948 PrimType ElemT = Vec.getFieldDesc()->getPrimType();
3949
3950 for (unsigned ElemNum = 0; ElemNum != VectorLen; ++ElemNum) {
3951 APSInt A;
3952 INT_TYPE_SWITCH_NO_BOOL(ElemT, { A = Vec.elem<T>(ElemNum).toAPSInt(); });
3953 unsigned MSB = A[A.getBitWidth() - 1];
3954 RetMask.setBitVal(BitPosition: ElemNum, BitValue: MSB);
3955 }
3956 pushInteger(S, Val: RetMask, QT: Call->getType());
3957 return true;
3958}
3959
3960static bool interp__builtin_ia32_cvt_mask2vec(InterpState &S, CodePtr OpPC,
3961 const CallExpr *Call,
3962 unsigned ID) {
3963 assert(Call->getNumArgs() == 1);
3964
3965 APSInt Mask;
3966 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: Mask))
3967 return false;
3968
3969 const Pointer &Vec = S.Stk.peek<Pointer>();
3970 unsigned NumElems = Vec.getNumElems();
3971 PrimType ElemT = Vec.getFieldDesc()->getPrimType();
3972
3973 for (unsigned I = 0; I != NumElems; ++I) {
3974 bool BitSet = Mask[I];
3975
3976 INT_TYPE_SWITCH_NO_BOOL(
3977 ElemT, { Vec.elem<T>(I) = BitSet ? T::from(-1) : T::from(0); });
3978 }
3979
3980 Vec.initializeAllElements();
3981
3982 return true;
3983}
3984
3985static bool interp__builtin_ia32_cvtsd2ss(InterpState &S, CodePtr OpPC,
3986 const CallExpr *Call,
3987 bool HasRoundingMask) {
3988 APSInt Rounding, MaskInt;
3989 Pointer Src, B, A;
3990
3991 if (HasRoundingMask) {
3992 assert(Call->getNumArgs() == 5);
3993 if (!popToAPSInt(S, E: Call->getArg(Arg: 4), Out&: Rounding))
3994 return false;
3995 if (!popToAPSInt(S, E: Call->getArg(Arg: 3), Out&: MaskInt))
3996 return false;
3997 Src = S.Stk.pop<Pointer>();
3998 B = S.Stk.pop<Pointer>();
3999 A = S.Stk.pop<Pointer>();
4000 if (!CheckLoad(S, OpPC, Ptr: A) || !CheckLoad(S, OpPC, Ptr: B) ||
4001 !CheckLoad(S, OpPC, Ptr: Src))
4002 return false;
4003 } else {
4004 assert(Call->getNumArgs() == 2);
4005 B = S.Stk.pop<Pointer>();
4006 A = S.Stk.pop<Pointer>();
4007 if (!CheckLoad(S, OpPC, Ptr: A) || !CheckLoad(S, OpPC, Ptr: B))
4008 return false;
4009 }
4010
4011 const auto *DstVTy = Call->getType()->castAs<VectorType>();
4012 unsigned NumElems = DstVTy->getNumElements();
4013 const Pointer &Dst = S.Stk.peek<Pointer>();
4014
4015 // Copy all elements except lane 0 (overwritten below) from A to Dst.
4016 for (unsigned I = 1; I != NumElems; ++I)
4017 Dst.elem<Floating>(I) = A.elem<Floating>(I);
4018
4019 // Convert element 0 from double to float, or use Src if masked off.
4020 if (!HasRoundingMask || (MaskInt.getZExtValue() & 0x1)) {
4021 assert(S.getASTContext().FloatTy == DstVTy->getElementType() &&
4022 "cvtsd2ss requires float element type in destination vector");
4023
4024 Floating Conv = S.allocFloat(
4025 Sem: S.getASTContext().getFloatTypeSemantics(T: DstVTy->getElementType()));
4026 APFloat SrcVal = B.elem<Floating>(I: 0).getAPFloat();
4027 if (!convertDoubleToFloatStrict(Src: SrcVal, Dst&: Conv, S, DiagExpr: Call))
4028 return false;
4029 Dst.elem<Floating>(I: 0) = Conv;
4030 } else {
4031 Dst.elem<Floating>(I: 0) = Src.elem<Floating>(I: 0);
4032 }
4033
4034 Dst.initializeAllElements();
4035 return true;
4036}
4037
4038static bool interp__builtin_ia32_cvtpd2ps(InterpState &S, CodePtr OpPC,
4039 const CallExpr *Call, bool IsMasked,
4040 bool HasRounding) {
4041 APSInt MaskVal;
4042 Pointer PassThrough;
4043 Pointer Src;
4044 APSInt Rounding;
4045
4046 if (IsMasked) {
4047 // Pop in reverse order.
4048 if (HasRounding) {
4049 if (!popToAPSInt(S, E: Call->getArg(Arg: 3), Out&: Rounding))
4050 return false;
4051 if (!popToAPSInt(S, E: Call->getArg(Arg: 2), Out&: MaskVal))
4052 return false;
4053 PassThrough = S.Stk.pop<Pointer>();
4054 Src = S.Stk.pop<Pointer>();
4055 } else {
4056 if (!popToAPSInt(S, E: Call->getArg(Arg: 2), Out&: MaskVal))
4057 return false;
4058 PassThrough = S.Stk.pop<Pointer>();
4059 Src = S.Stk.pop<Pointer>();
4060 }
4061
4062 if (!CheckLoad(S, OpPC, Ptr: PassThrough))
4063 return false;
4064 } else {
4065 // Pop source only.
4066 Src = S.Stk.pop<Pointer>();
4067 }
4068
4069 if (!CheckLoad(S, OpPC, Ptr: Src))
4070 return false;
4071
4072 const auto *RetVTy = Call->getType()->castAs<VectorType>();
4073 unsigned RetElems = RetVTy->getNumElements();
4074 unsigned SrcElems = Src.getNumElems();
4075 const Pointer &Dst = S.Stk.peek<Pointer>();
4076
4077 // Initialize destination with passthrough or zeros.
4078 for (unsigned I = 0; I != RetElems; ++I)
4079 if (IsMasked)
4080 Dst.elem<Floating>(I) = PassThrough.elem<Floating>(I);
4081 else
4082 Dst.elem<Floating>(I) = Floating(APFloat(0.0f));
4083
4084 assert(S.getASTContext().FloatTy == RetVTy->getElementType() &&
4085 "cvtpd2ps requires float element type in return vector");
4086
4087 // Convert double to float for enabled elements (only process source elements
4088 // that exist).
4089 for (unsigned I = 0; I != SrcElems; ++I) {
4090 if (IsMasked && !MaskVal[I])
4091 continue;
4092
4093 APFloat SrcVal = Src.elem<Floating>(I).getAPFloat();
4094
4095 Floating Conv = S.allocFloat(
4096 Sem: S.getASTContext().getFloatTypeSemantics(T: RetVTy->getElementType()));
4097 if (!convertDoubleToFloatStrict(Src: SrcVal, Dst&: Conv, S, DiagExpr: Call))
4098 return false;
4099 Dst.elem<Floating>(I) = Conv;
4100 }
4101
4102 Dst.initializeAllElements();
4103 return true;
4104}
4105
4106static bool interp__builtin_ia32_shuffle_generic(
4107 InterpState &S, CodePtr OpPC, const CallExpr *Call,
4108 llvm::function_ref<std::pair<unsigned, int>(unsigned, const APInt &)>
4109 GetSourceIndex) {
4110
4111 assert(Call->getNumArgs() == 2 || Call->getNumArgs() == 3);
4112
4113 APInt ShuffleMask;
4114 Pointer A, MaskVector, B;
4115 bool IsVectorMask = false;
4116 bool IsSingleOperand = (Call->getNumArgs() == 2);
4117
4118 if (IsSingleOperand) {
4119 QualType MaskType = Call->getArg(Arg: 1)->getType();
4120 if (MaskType->isVectorType()) {
4121 IsVectorMask = true;
4122 MaskVector = S.Stk.pop<Pointer>();
4123 A = S.Stk.pop<Pointer>();
4124 B = A;
4125 } else if (MaskType->isIntegerType()) {
4126 APSInt MaskVal;
4127 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: MaskVal))
4128 return false;
4129 ShuffleMask = MaskVal;
4130 A = S.Stk.pop<Pointer>();
4131 B = A;
4132 } else {
4133 return false;
4134 }
4135 } else {
4136 QualType Arg2Type = Call->getArg(Arg: 2)->getType();
4137 if (Arg2Type->isVectorType()) {
4138 IsVectorMask = true;
4139 B = S.Stk.pop<Pointer>();
4140 MaskVector = S.Stk.pop<Pointer>();
4141 A = S.Stk.pop<Pointer>();
4142 } else if (Arg2Type->isIntegerType()) {
4143 APSInt MaskVal;
4144 if (!popToAPSInt(S, E: Call->getArg(Arg: 2), Out&: MaskVal))
4145 return false;
4146 ShuffleMask = MaskVal;
4147 B = S.Stk.pop<Pointer>();
4148 A = S.Stk.pop<Pointer>();
4149 } else {
4150 return false;
4151 }
4152 }
4153
4154 QualType Arg0Type = Call->getArg(Arg: 0)->getType();
4155 const auto *VecT = Arg0Type->castAs<VectorType>();
4156 PrimType ElemT = *S.getContext().classify(T: VecT->getElementType());
4157 unsigned NumElems = VecT->getNumElements();
4158
4159 const Pointer &Dst = S.Stk.peek<Pointer>();
4160
4161 PrimType MaskElemT = PT_Uint32;
4162 if (IsVectorMask) {
4163 QualType Arg1Type = Call->getArg(Arg: 1)->getType();
4164 const auto *MaskVecT = Arg1Type->castAs<VectorType>();
4165 QualType MaskElemType = MaskVecT->getElementType();
4166 MaskElemT = *S.getContext().classify(T: MaskElemType);
4167 }
4168
4169 for (unsigned DstIdx = 0; DstIdx != NumElems; ++DstIdx) {
4170 if (IsVectorMask) {
4171 INT_TYPE_SWITCH(MaskElemT,
4172 { ShuffleMask = MaskVector.elem<T>(DstIdx).toAPSInt(); });
4173 }
4174
4175 auto [SrcVecIdx, SrcIdx] = GetSourceIndex(DstIdx, ShuffleMask);
4176
4177 if (SrcIdx < 0) {
4178 // Zero out this element
4179 if (ElemT == PT_Float) {
4180 Dst.elem<Floating>(I: DstIdx) = Floating(
4181 S.getASTContext().getFloatTypeSemantics(T: VecT->getElementType()));
4182 } else {
4183 INT_TYPE_SWITCH_NO_BOOL(ElemT, { Dst.elem<T>(DstIdx) = T::from(0); });
4184 }
4185 } else {
4186 const Pointer &Src = (SrcVecIdx == 0) ? A : B;
4187 TYPE_SWITCH(ElemT, { Dst.elem<T>(DstIdx) = Src.elem<T>(SrcIdx); });
4188 }
4189 }
4190 Dst.initializeAllElements();
4191
4192 return true;
4193}
4194
4195static bool interp__builtin_ia32_shuffle_generic(
4196 InterpState &S, CodePtr OpPC, const CallExpr *Call,
4197 llvm::function_ref<std::pair<unsigned, int>(unsigned, unsigned)>
4198 GetSourceIndex) {
4199 return interp__builtin_ia32_shuffle_generic(
4200 S, OpPC, Call,
4201 GetSourceIndex: [&GetSourceIndex](unsigned DstIdx,
4202 const APInt &Mask) -> std::pair<unsigned, int> {
4203 return GetSourceIndex(DstIdx, Mask.getZExtValue());
4204 });
4205}
4206
4207static bool interp__builtin_ia32_shift_with_count(
4208 InterpState &S, CodePtr OpPC, const CallExpr *Call,
4209 llvm::function_ref<APInt(const APInt &, uint64_t)> ShiftOp,
4210 llvm::function_ref<APInt(const APInt &, unsigned)> OverflowOp) {
4211
4212 assert(Call->getNumArgs() == 2);
4213
4214 const Pointer &Count = S.Stk.pop<Pointer>();
4215 const Pointer &Source = S.Stk.pop<Pointer>();
4216
4217 QualType SourceType = Call->getArg(Arg: 0)->getType();
4218 QualType CountType = Call->getArg(Arg: 1)->getType();
4219 assert(SourceType->isVectorType() && CountType->isVectorType());
4220
4221 const auto *SourceVecT = SourceType->castAs<VectorType>();
4222 const auto *CountVecT = CountType->castAs<VectorType>();
4223 PrimType SourceElemT = *S.getContext().classify(T: SourceVecT->getElementType());
4224 PrimType CountElemT = *S.getContext().classify(T: CountVecT->getElementType());
4225
4226 const Pointer &Dst = S.Stk.peek<Pointer>();
4227
4228 unsigned DestEltWidth =
4229 S.getASTContext().getTypeSize(T: SourceVecT->getElementType());
4230 bool IsDestUnsigned = SourceVecT->getElementType()->isUnsignedIntegerType();
4231 unsigned DestLen = SourceVecT->getNumElements();
4232 unsigned CountEltWidth =
4233 S.getASTContext().getTypeSize(T: CountVecT->getElementType());
4234 unsigned NumBitsInQWord = 64;
4235 unsigned NumCountElts = NumBitsInQWord / CountEltWidth;
4236
4237 uint64_t CountLQWord = 0;
4238 for (unsigned EltIdx = 0; EltIdx != NumCountElts; ++EltIdx) {
4239 uint64_t Elt = 0;
4240 INT_TYPE_SWITCH(CountElemT,
4241 { Elt = static_cast<uint64_t>(Count.elem<T>(EltIdx)); });
4242 CountLQWord |= (Elt << (EltIdx * CountEltWidth));
4243 }
4244
4245 for (unsigned EltIdx = 0; EltIdx != DestLen; ++EltIdx) {
4246 APSInt Elt;
4247 INT_TYPE_SWITCH(SourceElemT, { Elt = Source.elem<T>(EltIdx).toAPSInt(); });
4248
4249 APInt Result;
4250 if (CountLQWord < DestEltWidth) {
4251 Result = ShiftOp(Elt, CountLQWord);
4252 } else {
4253 Result = OverflowOp(Elt, DestEltWidth);
4254 }
4255 if (IsDestUnsigned) {
4256 INT_TYPE_SWITCH(SourceElemT, {
4257 Dst.elem<T>(EltIdx) = T::from(Result.getZExtValue());
4258 });
4259 } else {
4260 INT_TYPE_SWITCH(SourceElemT, {
4261 Dst.elem<T>(EltIdx) = T::from(Result.getSExtValue());
4262 });
4263 }
4264 }
4265
4266 Dst.initializeAllElements();
4267 return true;
4268}
4269
4270static bool interp__builtin_ia32_shufbitqmb_mask(InterpState &S, CodePtr OpPC,
4271 const CallExpr *Call) {
4272
4273 assert(Call->getNumArgs() == 3);
4274
4275 QualType SourceType = Call->getArg(Arg: 0)->getType();
4276 QualType ShuffleMaskType = Call->getArg(Arg: 1)->getType();
4277 QualType ZeroMaskType = Call->getArg(Arg: 2)->getType();
4278 if (!SourceType->isVectorType() || !ShuffleMaskType->isVectorType() ||
4279 !ZeroMaskType->isIntegerType()) {
4280 return false;
4281 }
4282
4283 Pointer Source, ShuffleMask;
4284 APSInt ZeroMask;
4285 if (!popToAPSInt(S, E: Call->getArg(Arg: 2), Out&: ZeroMask))
4286 return false;
4287 ShuffleMask = S.Stk.pop<Pointer>();
4288 Source = S.Stk.pop<Pointer>();
4289
4290 const auto *SourceVecT = SourceType->castAs<VectorType>();
4291 const auto *ShuffleMaskVecT = ShuffleMaskType->castAs<VectorType>();
4292 assert(SourceVecT->getNumElements() == ShuffleMaskVecT->getNumElements());
4293 assert(ZeroMask.getBitWidth() == SourceVecT->getNumElements());
4294
4295 PrimType SourceElemT = *S.getContext().classify(T: SourceVecT->getElementType());
4296 PrimType ShuffleMaskElemT =
4297 *S.getContext().classify(T: ShuffleMaskVecT->getElementType());
4298
4299 unsigned NumBytesInQWord = 8;
4300 unsigned NumBitsInByte = 8;
4301 unsigned NumBytes = SourceVecT->getNumElements();
4302 unsigned NumQWords = NumBytes / NumBytesInQWord;
4303 unsigned RetWidth = ZeroMask.getBitWidth();
4304 APSInt RetMask(llvm::APInt(RetWidth, 0), /*isUnsigned=*/true);
4305
4306 for (unsigned QWordId = 0; QWordId != NumQWords; ++QWordId) {
4307 APInt SourceQWord(64, 0);
4308 for (unsigned ByteIdx = 0; ByteIdx != NumBytesInQWord; ++ByteIdx) {
4309 uint64_t Byte = 0;
4310 INT_TYPE_SWITCH(SourceElemT, {
4311 Byte = static_cast<uint64_t>(
4312 Source.elem<T>(QWordId * NumBytesInQWord + ByteIdx));
4313 });
4314 SourceQWord.insertBits(SubBits: APInt(8, Byte & 0xFF), bitPosition: ByteIdx * NumBitsInByte);
4315 }
4316
4317 for (unsigned ByteIdx = 0; ByteIdx != NumBytesInQWord; ++ByteIdx) {
4318 unsigned SelIdx = QWordId * NumBytesInQWord + ByteIdx;
4319 unsigned M = 0;
4320 INT_TYPE_SWITCH(ShuffleMaskElemT, {
4321 M = static_cast<unsigned>(ShuffleMask.elem<T>(SelIdx)) & 0x3F;
4322 });
4323
4324 if (ZeroMask[SelIdx]) {
4325 RetMask.setBitVal(BitPosition: SelIdx, BitValue: SourceQWord[M]);
4326 }
4327 }
4328 }
4329
4330 pushInteger(S, Val: RetMask, QT: Call->getType());
4331 return true;
4332}
4333
4334static bool interp__builtin_ia32_vcvtps2ph(InterpState &S, CodePtr OpPC,
4335 const CallExpr *Call) {
4336 // Arguments are: vector of floats, rounding immediate
4337 assert(Call->getNumArgs() == 2);
4338
4339 APSInt Imm;
4340 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: Imm))
4341 return false;
4342 const Pointer &Src = S.Stk.pop<Pointer>();
4343 const Pointer &Dst = S.Stk.peek<Pointer>();
4344
4345 assert(Src.getFieldDesc()->isPrimitiveArray());
4346 assert(Dst.getFieldDesc()->isPrimitiveArray());
4347
4348 const auto *SrcVTy = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
4349 unsigned SrcNumElems = SrcVTy->getNumElements();
4350 const auto *DstVTy = Call->getType()->castAs<VectorType>();
4351 unsigned DstNumElems = DstVTy->getNumElements();
4352
4353 const llvm::fltSemantics &HalfSem =
4354 S.getASTContext().getFloatTypeSemantics(T: S.getASTContext().HalfTy);
4355
4356 // imm[2] == 1 means use MXCSR rounding mode.
4357 // In that case, we can only evaluate if the conversion is exact.
4358 int ImmVal = Imm.getZExtValue();
4359 bool UseMXCSR = (ImmVal & 4) != 0;
4360 bool IsFPConstrained =
4361 Call->getFPFeaturesInEffect(LO: S.getASTContext().getLangOpts())
4362 .isFPConstrained();
4363
4364 llvm::RoundingMode RM;
4365 if (!UseMXCSR) {
4366 switch (ImmVal & 3) {
4367 case 0:
4368 RM = llvm::RoundingMode::NearestTiesToEven;
4369 break;
4370 case 1:
4371 RM = llvm::RoundingMode::TowardNegative;
4372 break;
4373 case 2:
4374 RM = llvm::RoundingMode::TowardPositive;
4375 break;
4376 case 3:
4377 RM = llvm::RoundingMode::TowardZero;
4378 break;
4379 default:
4380 llvm_unreachable("Invalid immediate rounding mode");
4381 }
4382 } else {
4383 // For MXCSR, we must check for exactness. We can use any rounding mode
4384 // for the trial conversion since the result is the same if it's exact.
4385 RM = llvm::RoundingMode::NearestTiesToEven;
4386 }
4387
4388 QualType DstElemQT = Dst.getFieldDesc()->getElemQualType();
4389 PrimType DstElemT = *S.getContext().classify(T: DstElemQT);
4390
4391 for (unsigned I = 0; I != SrcNumElems; ++I) {
4392 Floating SrcVal = Src.elem<Floating>(I);
4393 APFloat DstVal = SrcVal.getAPFloat();
4394
4395 bool LostInfo;
4396 APFloat::opStatus St = DstVal.convert(ToSemantics: HalfSem, RM, losesInfo: &LostInfo);
4397
4398 if (UseMXCSR && IsFPConstrained && St != APFloat::opOK) {
4399 S.FFDiag(SI: S.Current->getSource(PC: OpPC),
4400 DiagId: diag::note_constexpr_dynamic_rounding);
4401 return false;
4402 }
4403
4404 INT_TYPE_SWITCH_NO_BOOL(DstElemT, {
4405 // Convert the destination value's bit pattern to an unsigned integer,
4406 // then reconstruct the element using the target type's 'from' method.
4407 uint64_t RawBits = DstVal.bitcastToAPInt().getZExtValue();
4408 Dst.elem<T>(I) = T::from(RawBits);
4409 });
4410 }
4411
4412 // Zero out remaining elements if the destination has more elements
4413 // (e.g., vcvtps2ph converting 4 floats to 8 shorts).
4414 if (DstNumElems > SrcNumElems) {
4415 for (unsigned I = SrcNumElems; I != DstNumElems; ++I) {
4416 INT_TYPE_SWITCH_NO_BOOL(DstElemT, { Dst.elem<T>(I) = T::from(0); });
4417 }
4418 }
4419
4420 Dst.initializeAllElements();
4421 return true;
4422}
4423
4424static bool interp__builtin_ia32_multishiftqb(InterpState &S, CodePtr OpPC,
4425 const CallExpr *Call) {
4426 assert(Call->getNumArgs() == 2);
4427
4428 QualType ATy = Call->getArg(Arg: 0)->getType();
4429 QualType BTy = Call->getArg(Arg: 1)->getType();
4430 if (!ATy->isVectorType() || !BTy->isVectorType()) {
4431 return false;
4432 }
4433
4434 const Pointer &BPtr = S.Stk.pop<Pointer>();
4435 const Pointer &APtr = S.Stk.pop<Pointer>();
4436 const auto *AVecT = ATy->castAs<VectorType>();
4437 assert(AVecT->getNumElements() ==
4438 BTy->castAs<VectorType>()->getNumElements());
4439
4440 PrimType ElemT = *S.getContext().classify(T: AVecT->getElementType());
4441
4442 unsigned NumBytesInQWord = 8;
4443 unsigned NumBitsInByte = 8;
4444 unsigned NumBytes = AVecT->getNumElements();
4445 unsigned NumQWords = NumBytes / NumBytesInQWord;
4446 const Pointer &Dst = S.Stk.peek<Pointer>();
4447
4448 for (unsigned QWordId = 0; QWordId != NumQWords; ++QWordId) {
4449 APInt BQWord(64, 0);
4450 for (unsigned ByteIdx = 0; ByteIdx != NumBytesInQWord; ++ByteIdx) {
4451 unsigned Idx = QWordId * NumBytesInQWord + ByteIdx;
4452 INT_TYPE_SWITCH(ElemT, {
4453 uint64_t Byte = static_cast<uint64_t>(BPtr.elem<T>(Idx));
4454 BQWord.insertBits(APInt(8, Byte & 0xFF), ByteIdx * NumBitsInByte);
4455 });
4456 }
4457
4458 for (unsigned ByteIdx = 0; ByteIdx != NumBytesInQWord; ++ByteIdx) {
4459 unsigned Idx = QWordId * NumBytesInQWord + ByteIdx;
4460 uint64_t Ctrl = 0;
4461 INT_TYPE_SWITCH(
4462 ElemT, { Ctrl = static_cast<uint64_t>(APtr.elem<T>(Idx)) & 0x3F; });
4463
4464 APInt Byte(8, 0);
4465 for (unsigned BitIdx = 0; BitIdx != NumBitsInByte; ++BitIdx) {
4466 Byte.setBitVal(BitPosition: BitIdx, BitValue: BQWord[(Ctrl + BitIdx) & 0x3F]);
4467 }
4468 INT_TYPE_SWITCH(ElemT,
4469 { Dst.elem<T>(Idx) = T::from(Byte.getZExtValue()); });
4470 }
4471 }
4472
4473 Dst.initializeAllElements();
4474
4475 return true;
4476}
4477
4478static bool interp__builtin_ia32_gfni_affine(InterpState &S, CodePtr OpPC,
4479 const CallExpr *Call,
4480 bool Inverse) {
4481 assert(Call->getNumArgs() == 3);
4482 QualType XType = Call->getArg(Arg: 0)->getType();
4483 QualType AType = Call->getArg(Arg: 1)->getType();
4484 QualType ImmType = Call->getArg(Arg: 2)->getType();
4485 if (!XType->isVectorType() || !AType->isVectorType() ||
4486 !ImmType->isIntegerType()) {
4487 return false;
4488 }
4489
4490 Pointer X, A;
4491 APSInt Imm;
4492 if (!popToAPSInt(S, E: Call->getArg(Arg: 2), Out&: Imm))
4493 return false;
4494 A = S.Stk.pop<Pointer>();
4495 X = S.Stk.pop<Pointer>();
4496
4497 const Pointer &Dst = S.Stk.peek<Pointer>();
4498 const auto *AVecT = AType->castAs<VectorType>();
4499 assert(XType->castAs<VectorType>()->getNumElements() ==
4500 AVecT->getNumElements());
4501 unsigned NumBytesInQWord = 8;
4502 unsigned NumBytes = AVecT->getNumElements();
4503 unsigned NumBitsInQWord = 64;
4504 unsigned NumQWords = NumBytes / NumBytesInQWord;
4505 unsigned NumBitsInByte = 8;
4506 PrimType AElemT = *S.getContext().classify(T: AVecT->getElementType());
4507
4508 // computing A*X + Imm
4509 for (unsigned QWordIdx = 0; QWordIdx != NumQWords; ++QWordIdx) {
4510 // Extract the QWords from X, A
4511 APInt XQWord(NumBitsInQWord, 0);
4512 APInt AQWord(NumBitsInQWord, 0);
4513 for (unsigned ByteIdx = 0; ByteIdx != NumBytesInQWord; ++ByteIdx) {
4514 unsigned Idx = QWordIdx * NumBytesInQWord + ByteIdx;
4515 uint8_t XByte;
4516 uint8_t AByte;
4517 INT_TYPE_SWITCH(AElemT, {
4518 XByte = static_cast<uint8_t>(X.elem<T>(Idx));
4519 AByte = static_cast<uint8_t>(A.elem<T>(Idx));
4520 });
4521
4522 XQWord.insertBits(SubBits: APInt(NumBitsInByte, XByte), bitPosition: ByteIdx * NumBitsInByte);
4523 AQWord.insertBits(SubBits: APInt(NumBitsInByte, AByte), bitPosition: ByteIdx * NumBitsInByte);
4524 }
4525
4526 for (unsigned ByteIdx = 0; ByteIdx != NumBytesInQWord; ++ByteIdx) {
4527 unsigned Idx = QWordIdx * NumBytesInQWord + ByteIdx;
4528 uint8_t XByte =
4529 XQWord.lshr(shiftAmt: ByteIdx * NumBitsInByte).getLoBits(numBits: 8).getZExtValue();
4530 INT_TYPE_SWITCH(AElemT, {
4531 Dst.elem<T>(Idx) = T::from(GFNIAffine(XByte, AQWord, Imm, Inverse));
4532 });
4533 }
4534 }
4535 Dst.initializeAllElements();
4536 return true;
4537}
4538
4539static bool interp__builtin_ia32_gfni_mul(InterpState &S, CodePtr OpPC,
4540 const CallExpr *Call) {
4541 assert(Call->getNumArgs() == 2);
4542
4543 QualType AType = Call->getArg(Arg: 0)->getType();
4544 QualType BType = Call->getArg(Arg: 1)->getType();
4545 if (!AType->isVectorType() || !BType->isVectorType()) {
4546 return false;
4547 }
4548
4549 Pointer A, B;
4550 B = S.Stk.pop<Pointer>();
4551 A = S.Stk.pop<Pointer>();
4552
4553 const Pointer &Dst = S.Stk.peek<Pointer>();
4554 const auto *AVecT = AType->castAs<VectorType>();
4555 assert(AVecT->getNumElements() ==
4556 BType->castAs<VectorType>()->getNumElements());
4557
4558 PrimType AElemT = *S.getContext().classify(T: AVecT->getElementType());
4559 unsigned NumBytes = A.getNumElems();
4560
4561 for (unsigned ByteIdx = 0; ByteIdx != NumBytes; ++ByteIdx) {
4562 uint8_t AByte, BByte;
4563 INT_TYPE_SWITCH(AElemT, {
4564 AByte = static_cast<uint8_t>(A.elem<T>(ByteIdx));
4565 BByte = static_cast<uint8_t>(B.elem<T>(ByteIdx));
4566 Dst.elem<T>(ByteIdx) = T::from(GFNIMul(AByte, BByte));
4567 });
4568 }
4569
4570 Dst.initializeAllElements();
4571 return true;
4572}
4573
4574static bool interp__builtin_ia32_vpdp(InterpState &S, CodePtr OpPC,
4575 const CallExpr *Call, bool IsSaturating) {
4576 assert(Call->getNumArgs() == 3);
4577
4578 QualType SrcT = Call->getArg(Arg: 0)->getType();
4579 QualType OpAT = Call->getArg(Arg: 1)->getType();
4580 QualType OpBT = Call->getArg(Arg: 2)->getType();
4581 QualType DstT = Call->getType();
4582 if (!SrcT->isVectorType() || !OpAT->isVectorType() || !OpBT->isVectorType() ||
4583 !DstT->isVectorType())
4584 return false;
4585
4586 const auto *SrcVecT = SrcT->castAs<VectorType>();
4587 const auto *OpAVecT = OpAT->castAs<VectorType>();
4588 const auto *OpBVecT = OpBT->castAs<VectorType>();
4589 const auto *DstVecT = DstT->castAs<VectorType>();
4590
4591 assert(OpAVecT->getNumElements() == OpBVecT->getNumElements());
4592
4593 unsigned NumSrcElems = SrcVecT->getNumElements();
4594 unsigned NumOperandElems = OpAVecT->getNumElements();
4595 unsigned ElemsPerLane = NumOperandElems / NumSrcElems;
4596
4597 PrimType SrcElemT = *S.getContext().classify(T: SrcVecT->getElementType());
4598 PrimType OpAElemT = *S.getContext().classify(T: OpAVecT->getElementType());
4599 PrimType OpBElemT = *S.getContext().classify(T: OpBVecT->getElementType());
4600 PrimType DstElemT = *S.getContext().classify(T: DstVecT->getElementType());
4601
4602 assert(SrcElemT == DstElemT);
4603
4604 const Pointer &OpBPtr = S.Stk.pop<Pointer>();
4605 const Pointer &OpAPtr = S.Stk.pop<Pointer>();
4606 const Pointer &SrcPtr = S.Stk.pop<Pointer>();
4607 const Pointer &Dst = S.Stk.peek<Pointer>();
4608
4609 for (unsigned I = 0; I != NumSrcElems; ++I) {
4610 APSInt Acc;
4611 INT_TYPE_SWITCH_NO_BOOL(SrcElemT, { Acc = SrcPtr.elem<T>(I).toAPSInt(); });
4612 Acc = Acc.sext(width: 64);
4613 for (unsigned J = 0; J != ElemsPerLane; ++J) {
4614 APSInt OpA, OpB;
4615 INT_TYPE_SWITCH_NO_BOOL(
4616 OpAElemT, { OpA = OpAPtr.elem<T>(ElemsPerLane * I + J).toAPSInt(); });
4617 INT_TYPE_SWITCH_NO_BOOL(
4618 OpBElemT, { OpB = OpBPtr.elem<T>(ElemsPerLane * I + J).toAPSInt(); });
4619 OpA = APSInt(OpA.extend(width: 64), false);
4620 OpB = APSInt(OpB.extend(width: 64), false);
4621 Acc += OpA * OpB;
4622 }
4623 if (IsSaturating)
4624 Acc = APSInt(Acc.truncSSat(width: 32), false);
4625 else
4626 Acc = APSInt(Acc.trunc(width: 32), false);
4627 INT_TYPE_SWITCH_NO_BOOL(DstElemT,
4628 { Dst.elem<T>(I) = static_cast<T>(Acc); });
4629 }
4630 Dst.initializeAllElements();
4631 return true;
4632}
4633
4634// Bit Matrix Multiply and Accumulate (AVX512BMM). Each 256-bit lane holds a
4635// 16x16 bit matrix as 16 x i16 elements; element i is row i and bit j of that
4636// element is entry [i][j]. The accumulator (third argument, src1 in the AMD
4637// ISA) provides the initial value of each result bit, into which the bit-matrix
4638// product of the first two arguments (src2 * src3) is reduced with OR (vbmacor)
4639// or XOR (vbmacxor):
4640// for i in 0..15, j in 0..15:
4641// bit = C[16*i+j]
4642// for k in 0..15: bit OP= A[16*i+k] & B[16*k+j]
4643// dest[16*i+j] = bit
4644static bool interp__builtin_ia32_bmac(InterpState &S, CodePtr OpPC,
4645 const CallExpr *Call, bool IsXor) {
4646 assert(Call->getNumArgs() == 3);
4647
4648 // AST-based type checks before popping the stack.
4649 QualType AType = Call->getArg(Arg: 0)->getType();
4650 QualType BType = Call->getArg(Arg: 1)->getType();
4651 QualType CType = Call->getArg(Arg: 2)->getType();
4652 if (!AType->isVectorType() || !BType->isVectorType() ||
4653 !CType->isVectorType())
4654 return false;
4655
4656 const Pointer &C = S.Stk.pop<Pointer>();
4657 const Pointer &B = S.Stk.pop<Pointer>();
4658 const Pointer &A = S.Stk.pop<Pointer>();
4659 const Pointer &Dst = S.Stk.peek<Pointer>();
4660
4661 // check if all three primitive arrays are with 16-bit elements.
4662 auto isValid16BitArray = [](const Pointer &P) {
4663 const Descriptor *D = P.getFieldDesc();
4664 if (!D->isPrimitiveArray())
4665 return false;
4666 PrimType PT = D->getPrimType();
4667 return ((PT == PT_Sint16) || (PT == PT_Uint16));
4668 };
4669
4670 if (!isValid16BitArray(A) || !isValid16BitArray(B) || !isValid16BitArray(C))
4671 return false;
4672
4673 PrimType ElemT = A.getFieldDesc()->getPrimType();
4674 unsigned NumElems = A.getNumElems();
4675 assert(NumElems % 16 == 0 && "BMM operates on 256-bit lanes of 16 x i16");
4676 bool DstUnsigned = ElemT == PT_Uint16;
4677
4678 // Lanes are always 16-bit; gather them so the reduction below is untyped.
4679 SmallVector<uint16_t> AVals(NumElems), BVals(NumElems), Acc(NumElems);
4680 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
4681 for (unsigned I = 0; I != NumElems; ++I) {
4682 AVals[I] = (uint16_t)A.elem<T>(I).toAPSInt().getZExtValue();
4683 BVals[I] = (uint16_t)B.elem<T>(I).toAPSInt().getZExtValue();
4684 Acc[I] = (uint16_t)C.elem<T>(I).toAPSInt().getZExtValue();
4685 }
4686 });
4687
4688 for (unsigned Lane = 0; Lane != NumElems; Lane += 16) {
4689 for (unsigned I = 0; I != 16; ++I) {
4690 uint16_t AVal = AVals[Lane + I], DVal = Acc[Lane + I];
4691 for (unsigned J = 0; J != 16; ++J) {
4692 // Seed the reduction with the accumulator bit, then fold in each
4693 // product term with the same operator (OR for vbmacor, XOR for
4694 // vbmacxor).
4695 unsigned Bit = (DVal >> J) & 1u;
4696 for (unsigned K = 0; K != 16; ++K) {
4697 unsigned Product = ((AVal >> K) & 1u) & ((BVals[Lane + K] >> J) & 1u);
4698 Bit = IsXor ? (Bit ^ Product) : (Bit | Product);
4699 }
4700 DVal = (DVal & ~(uint16_t(1) << J)) | (uint16_t(Bit) << J);
4701 }
4702 Acc[Lane + I] = DVal;
4703 }
4704 }
4705
4706 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
4707 for (unsigned I = 0; I != NumElems; ++I)
4708 Dst.elem<T>(I) = static_cast<T>(APSInt(APInt(16, Acc[I]), DstUnsigned));
4709 });
4710 Dst.initializeAllElements();
4711 return true;
4712}
4713
4714static bool interp_builtin_ia32_cvt_scalar_to_int(InterpState &S, CodePtr OpPC,
4715 const CallExpr *E) {
4716 Pointer SrcVecPtr = S.Stk.pop<Pointer>();
4717 const Floating &FloatElem = SrcVecPtr.elem<Floating>(I: 0);
4718
4719 unsigned BitWidth = S.getASTContext().getIntWidth(T: E->getType());
4720 bool IsUnsigned = E->getType()->isUnsignedIntegerType();
4721
4722 llvm::APSInt IntResult(BitWidth, IsUnsigned);
4723 bool IsExact = false;
4724 // We only allow exact conversions so rounding mode does not matter for cvt*
4725 // and cvtt* builtins
4726 FloatElem.getAPFloat().convertToInteger(
4727 Result&: IntResult, RM: llvm::APFloat::rmTowardZero, IsExact: &IsExact);
4728 if (!IsExact)
4729 return false;
4730
4731 pushInteger(S, Val: IntResult, QT: E->getType());
4732 return true;
4733}
4734
4735static bool interp_builtin_ia32_cvt_vector_to_int(InterpState &S, CodePtr OpPC,
4736 const CallExpr *E) {
4737 Pointer SrcVecPtr = S.Stk.pop<Pointer>();
4738 const Pointer &Dst = S.Stk.peek<Pointer>();
4739
4740 unsigned NumSrcElems = SrcVecPtr.getNumElems();
4741 unsigned NumDstElems = Dst.getNumElems();
4742
4743 if (NumSrcElems > NumDstElems)
4744 return false;
4745
4746 QualType ElemType = Dst.getFieldDesc()->getElemQualType();
4747 unsigned BitWidth = S.getASTContext().getIntWidth(T: ElemType);
4748 bool IsUnsigned = ElemType->isUnsignedIntegerType();
4749
4750 PrimType ElemT = *S.getContext().classify(T: ElemType);
4751 for (unsigned I = 0; I != NumSrcElems; ++I) {
4752 const Floating &FloatElem = SrcVecPtr.elem<Floating>(I);
4753 llvm::APSInt IntResult(BitWidth, IsUnsigned);
4754
4755 bool IsExact = false;
4756 // We only allow exact conversions so rounding mode does not matter for
4757 // cvt* and cvtt* builtins
4758 FloatElem.getAPFloat().convertToInteger(
4759 Result&: IntResult, RM: llvm::APFloat::rmTowardZero, IsExact: &IsExact);
4760 if (!IsExact)
4761 return false;
4762 INT_TYPE_SWITCH_NO_BOOL(
4763 ElemT, { Dst.elem<T>(I) = T::from(IntResult.getZExtValue()); });
4764 }
4765
4766 // Zero out remaining elements if the destination has more elements
4767 // (e.g., cvtpd2dq converting 2 doubles(_m128d) to 2 ints stored in _m128i).
4768 for (unsigned I = NumSrcElems; I != NumDstElems; ++I)
4769 INT_TYPE_SWITCH_NO_BOOL(ElemT, { Dst.elem<T>(I) = T::from(0); });
4770
4771 Dst.initializeAllElements();
4772 return true;
4773}
4774
4775bool InterpretBuiltin(InterpState &S, CodePtr OpPC, const CallExpr *Call,
4776 uint32_t BuiltinID) {
4777 const ASTContext &ASTCtx = S.getASTContext();
4778
4779 // BuiltinID is the raw ID baked into the bytecode. The "is constant
4780 // evaluated" gate needs the raw ID so that auxiliary-target IDs resolve into
4781 // the correct (aux-target) builtin records.
4782 if (!ASTCtx.BuiltinInfo.isConstantEvaluated(ID: BuiltinID))
4783 return Invalid(S, OpPC);
4784
4785 // Convert an auxiliary x86 target builtin ID to its canonical X86::BI* value
4786 // so the target-specific cases below (and the handlers they call) match. This
4787 // is a cheap integer operation (a single comparison for the common,
4788 // target-independent case); we deliberately avoid re-deriving the ID from the
4789 // call expression, which is comparatively slow.
4790 BuiltinID = ConvertBuiltinIDToX86BuiltinID(Ctx: ASTCtx, BuiltinID);
4791
4792 const InterpFrame *Frame = S.Current;
4793 switch (BuiltinID) {
4794 case Builtin::BI__builtin_is_constant_evaluated:
4795 return interp__builtin_is_constant_evaluated(S, OpPC, Frame, Call);
4796
4797 case Builtin::BI__builtin_assume:
4798 case Builtin::BI__assume:
4799 return interp__builtin_assume(S, OpPC, Frame, Call);
4800
4801 case Builtin::BI__builtin_strcmp:
4802 case Builtin::BIstrcmp:
4803 case Builtin::BI__builtin_strncmp:
4804 case Builtin::BIstrncmp:
4805 case Builtin::BI__builtin_wcsncmp:
4806 case Builtin::BIwcsncmp:
4807 case Builtin::BI__builtin_wcscmp:
4808 case Builtin::BIwcscmp:
4809 return interp__builtin_strcmp(S, OpPC, Frame, Call, ID: BuiltinID);
4810
4811 case Builtin::BI__builtin_strlen:
4812 case Builtin::BIstrlen:
4813 case Builtin::BI__builtin_wcslen:
4814 case Builtin::BIwcslen:
4815 return interp__builtin_strlen(S, OpPC, Frame, Call, ID: BuiltinID);
4816
4817 case Builtin::BI__builtin_nan:
4818 case Builtin::BI__builtin_nanf:
4819 case Builtin::BI__builtin_nanl:
4820 case Builtin::BI__builtin_nanf16:
4821 case Builtin::BI__builtin_nanf128:
4822 return interp__builtin_nan(S, OpPC, Frame, Call, /*Signaling=*/false);
4823
4824 case Builtin::BI__builtin_nans:
4825 case Builtin::BI__builtin_nansf:
4826 case Builtin::BI__builtin_nansl:
4827 case Builtin::BI__builtin_nansf16:
4828 case Builtin::BI__builtin_nansf128:
4829 return interp__builtin_nan(S, OpPC, Frame, Call, /*Signaling=*/true);
4830
4831 case Builtin::BI__builtin_huge_val:
4832 case Builtin::BI__builtin_huge_valf:
4833 case Builtin::BI__builtin_huge_vall:
4834 case Builtin::BI__builtin_huge_valf16:
4835 case Builtin::BI__builtin_huge_valf128:
4836 case Builtin::BI__builtin_inf:
4837 case Builtin::BI__builtin_inff:
4838 case Builtin::BI__builtin_infl:
4839 case Builtin::BI__builtin_inff16:
4840 case Builtin::BI__builtin_inff128:
4841 return interp__builtin_inf(S, OpPC, Frame, Call);
4842
4843 case Builtin::BI__builtin_copysign:
4844 case Builtin::BI__builtin_copysignf:
4845 case Builtin::BI__builtin_copysignl:
4846 case Builtin::BI__builtin_copysignf128:
4847 return interp__builtin_copysign(S, OpPC, Frame);
4848
4849 case Builtin::BI__builtin_fmin:
4850 case Builtin::BI__builtin_fminf:
4851 case Builtin::BI__builtin_fminl:
4852 case Builtin::BI__builtin_fminf16:
4853 case Builtin::BI__builtin_fminf128:
4854 return interp__builtin_fmin(S, OpPC, Frame, /*IsNumBuiltin=*/false);
4855
4856 case Builtin::BI__builtin_fminimum_num:
4857 case Builtin::BI__builtin_fminimum_numf:
4858 case Builtin::BI__builtin_fminimum_numl:
4859 case Builtin::BI__builtin_fminimum_numf16:
4860 case Builtin::BI__builtin_fminimum_numf128:
4861 return interp__builtin_fmin(S, OpPC, Frame, /*IsNumBuiltin=*/true);
4862
4863 case Builtin::BI__builtin_fmax:
4864 case Builtin::BI__builtin_fmaxf:
4865 case Builtin::BI__builtin_fmaxl:
4866 case Builtin::BI__builtin_fmaxf16:
4867 case Builtin::BI__builtin_fmaxf128:
4868 return interp__builtin_fmax(S, OpPC, Frame, /*IsNumBuiltin=*/false);
4869
4870 case Builtin::BI__builtin_fmaximum_num:
4871 case Builtin::BI__builtin_fmaximum_numf:
4872 case Builtin::BI__builtin_fmaximum_numl:
4873 case Builtin::BI__builtin_fmaximum_numf16:
4874 case Builtin::BI__builtin_fmaximum_numf128:
4875 return interp__builtin_fmax(S, OpPC, Frame, /*IsNumBuiltin=*/true);
4876
4877 case Builtin::BI__builtin_isnan:
4878 return interp__builtin_isnan(S, OpPC, Frame, Call);
4879
4880 case Builtin::BI__builtin_issignaling:
4881 return interp__builtin_issignaling(S, OpPC, Frame, Call);
4882
4883 case Builtin::BI__builtin_isinf:
4884 return interp__builtin_isinf(S, OpPC, Frame, /*Sign=*/CheckSign: false, Call);
4885
4886 case Builtin::BI__builtin_isinf_sign:
4887 return interp__builtin_isinf(S, OpPC, Frame, /*Sign=*/CheckSign: true, Call);
4888
4889 case Builtin::BI__builtin_isfinite:
4890 return interp__builtin_isfinite(S, OpPC, Frame, Call);
4891
4892 case Builtin::BI__builtin_isnormal:
4893 return interp__builtin_isnormal(S, OpPC, Frame, Call);
4894
4895 case Builtin::BI__builtin_issubnormal:
4896 return interp__builtin_issubnormal(S, OpPC, Frame, Call);
4897
4898 case Builtin::BI__builtin_iszero:
4899 return interp__builtin_iszero(S, OpPC, Frame, Call);
4900
4901 case Builtin::BI__builtin_signbit:
4902 case Builtin::BI__builtin_signbitf:
4903 case Builtin::BI__builtin_signbitl:
4904 return interp__builtin_signbit(S, OpPC, Frame, Call);
4905
4906 case Builtin::BI__builtin_isgreater:
4907 case Builtin::BI__builtin_isgreaterequal:
4908 case Builtin::BI__builtin_isless:
4909 case Builtin::BI__builtin_islessequal:
4910 case Builtin::BI__builtin_islessgreater:
4911 case Builtin::BI__builtin_isunordered:
4912 return interp_floating_comparison(S, OpPC, Call, ID: BuiltinID);
4913
4914 case Builtin::BI__builtin_isfpclass:
4915 return interp__builtin_isfpclass(S, OpPC, Frame, Call);
4916
4917 case Builtin::BI__builtin_fpclassify:
4918 return interp__builtin_fpclassify(S, OpPC, Frame, Call);
4919
4920 case Builtin::BI__builtin_fabs:
4921 case Builtin::BI__builtin_fabsf:
4922 case Builtin::BI__builtin_fabsl:
4923 case Builtin::BI__builtin_fabsf128:
4924 return interp__builtin_fabs(S, OpPC, Frame);
4925
4926 case Builtin::BI__builtin_abs:
4927 case Builtin::BI__builtin_labs:
4928 case Builtin::BI__builtin_llabs:
4929 return interp__builtin_abs(S, OpPC, Frame, Call);
4930
4931 case Builtin::BI__builtin_popcount:
4932 case Builtin::BI__builtin_popcountl:
4933 case Builtin::BI__builtin_popcountll:
4934 case Builtin::BI__builtin_popcountg:
4935 case Builtin::BI__popcnt16: // Microsoft variants of popcount
4936 case Builtin::BI__popcnt:
4937 case Builtin::BI__popcnt64:
4938 return interp__builtin_popcount(S, OpPC, Frame, Call);
4939
4940 case Builtin::BI__builtin_parity:
4941 case Builtin::BI__builtin_parityl:
4942 case Builtin::BI__builtin_parityll:
4943 return interp__builtin_elementwise_int_unaryop(
4944 S, OpPC, Call, Fn: [](const APSInt &Val) {
4945 return APInt(Val.getBitWidth(), Val.popcount() % 2);
4946 });
4947 case Builtin::BI__builtin_clrsb:
4948 case Builtin::BI__builtin_clrsbl:
4949 case Builtin::BI__builtin_clrsbll:
4950 return interp__builtin_elementwise_int_unaryop(
4951 S, OpPC, Call, Fn: [](const APSInt &Val) {
4952 return APInt(Val.getBitWidth(),
4953 Val.getBitWidth() - Val.getSignificantBits());
4954 });
4955 case Builtin::BI__builtin_bitreverseg:
4956 case Builtin::BI__builtin_bitreverse8:
4957 case Builtin::BI__builtin_bitreverse16:
4958 case Builtin::BI__builtin_bitreverse32:
4959 case Builtin::BI__builtin_bitreverse64:
4960 return interp__builtin_elementwise_int_unaryop(
4961 S, OpPC, Call, Fn: [](const APSInt &Val) { return Val.reverseBits(); });
4962
4963 case Builtin::BI__builtin_classify_type:
4964 return interp__builtin_classify_type(S, OpPC, Frame, Call);
4965
4966 case Builtin::BI__builtin_expect:
4967 case Builtin::BI__builtin_expect_with_probability:
4968 return interp__builtin_expect(S, OpPC, Frame, Call);
4969
4970 case Builtin::BI__builtin_rotateleft8:
4971 case Builtin::BI__builtin_rotateleft16:
4972 case Builtin::BI__builtin_rotateleft32:
4973 case Builtin::BI__builtin_rotateleft64:
4974 case Builtin::BI__builtin_stdc_rotate_left:
4975 case Builtin::BIstdc_rotate_left_uc:
4976 case Builtin::BIstdc_rotate_left_us:
4977 case Builtin::BIstdc_rotate_left_ui:
4978 case Builtin::BIstdc_rotate_left_ul:
4979 case Builtin::BIstdc_rotate_left_ull:
4980 case Builtin::BI_rotl8: // Microsoft variants of rotate left
4981 case Builtin::BI_rotl16:
4982 case Builtin::BI_rotl:
4983 case Builtin::BI_lrotl:
4984 case Builtin::BI_rotl64:
4985 case Builtin::BI__builtin_rotateright8:
4986 case Builtin::BI__builtin_rotateright16:
4987 case Builtin::BI__builtin_rotateright32:
4988 case Builtin::BI__builtin_rotateright64:
4989 case Builtin::BI__builtin_stdc_rotate_right:
4990 case Builtin::BIstdc_rotate_right_uc:
4991 case Builtin::BIstdc_rotate_right_us:
4992 case Builtin::BIstdc_rotate_right_ui:
4993 case Builtin::BIstdc_rotate_right_ul:
4994 case Builtin::BIstdc_rotate_right_ull:
4995 case Builtin::BI_rotr8: // Microsoft variants of rotate right
4996 case Builtin::BI_rotr16:
4997 case Builtin::BI_rotr:
4998 case Builtin::BI_lrotr:
4999 case Builtin::BI_rotr64: {
5000 // Determine if this is a rotate right operation
5001 bool IsRotateRight;
5002 switch (BuiltinID) {
5003 case Builtin::BI__builtin_rotateright8:
5004 case Builtin::BI__builtin_rotateright16:
5005 case Builtin::BI__builtin_rotateright32:
5006 case Builtin::BI__builtin_rotateright64:
5007 case Builtin::BI__builtin_stdc_rotate_right:
5008 case Builtin::BIstdc_rotate_right_uc:
5009 case Builtin::BIstdc_rotate_right_us:
5010 case Builtin::BIstdc_rotate_right_ui:
5011 case Builtin::BIstdc_rotate_right_ul:
5012 case Builtin::BIstdc_rotate_right_ull:
5013 case Builtin::BI_rotr8:
5014 case Builtin::BI_rotr16:
5015 case Builtin::BI_rotr:
5016 case Builtin::BI_lrotr:
5017 case Builtin::BI_rotr64:
5018 IsRotateRight = true;
5019 break;
5020 default:
5021 IsRotateRight = false;
5022 break;
5023 }
5024
5025 return interp__builtin_elementwise_int_binop(
5026 S, OpPC, Call, Fn: [IsRotateRight](const APSInt &Value, APSInt Amount) {
5027 Amount = NormalizeRotateAmount(Value, Amount);
5028 return IsRotateRight ? Value.rotr(rotateAmt: Amount.getZExtValue())
5029 : Value.rotl(rotateAmt: Amount.getZExtValue());
5030 });
5031 }
5032
5033 case Builtin::BIstdc_leading_zeros_uc:
5034 case Builtin::BIstdc_leading_zeros_us:
5035 case Builtin::BIstdc_leading_zeros_ui:
5036 case Builtin::BIstdc_leading_zeros_ul:
5037 case Builtin::BIstdc_leading_zeros_ull:
5038 case Builtin::BI__builtin_stdc_leading_zeros: {
5039 unsigned ResWidth = S.getASTContext().getIntWidth(T: Call->getType());
5040 return interp__builtin_elementwise_int_unaryop(
5041 S, OpPC, Call, Fn: [ResWidth](const APSInt &Val) {
5042 return APInt(ResWidth, Val.countl_zero());
5043 });
5044 }
5045
5046 case Builtin::BIstdc_leading_ones_uc:
5047 case Builtin::BIstdc_leading_ones_us:
5048 case Builtin::BIstdc_leading_ones_ui:
5049 case Builtin::BIstdc_leading_ones_ul:
5050 case Builtin::BIstdc_leading_ones_ull:
5051 case Builtin::BI__builtin_stdc_leading_ones: {
5052 unsigned ResWidth = S.getASTContext().getIntWidth(T: Call->getType());
5053 return interp__builtin_elementwise_int_unaryop(
5054 S, OpPC, Call, Fn: [ResWidth](const APSInt &Val) {
5055 return APInt(ResWidth, Val.countl_one());
5056 });
5057 }
5058
5059 case Builtin::BIstdc_trailing_zeros_uc:
5060 case Builtin::BIstdc_trailing_zeros_us:
5061 case Builtin::BIstdc_trailing_zeros_ui:
5062 case Builtin::BIstdc_trailing_zeros_ul:
5063 case Builtin::BIstdc_trailing_zeros_ull:
5064 case Builtin::BI__builtin_stdc_trailing_zeros: {
5065 unsigned ResWidth = S.getASTContext().getIntWidth(T: Call->getType());
5066 return interp__builtin_elementwise_int_unaryop(
5067 S, OpPC, Call, Fn: [ResWidth](const APSInt &Val) {
5068 return APInt(ResWidth, Val.countr_zero());
5069 });
5070 }
5071
5072 case Builtin::BIstdc_trailing_ones_uc:
5073 case Builtin::BIstdc_trailing_ones_us:
5074 case Builtin::BIstdc_trailing_ones_ui:
5075 case Builtin::BIstdc_trailing_ones_ul:
5076 case Builtin::BIstdc_trailing_ones_ull:
5077 case Builtin::BI__builtin_stdc_trailing_ones: {
5078 unsigned ResWidth = S.getASTContext().getIntWidth(T: Call->getType());
5079 return interp__builtin_elementwise_int_unaryop(
5080 S, OpPC, Call, Fn: [ResWidth](const APSInt &Val) {
5081 return APInt(ResWidth, Val.countr_one());
5082 });
5083 }
5084
5085 case Builtin::BIstdc_first_leading_zero_uc:
5086 case Builtin::BIstdc_first_leading_zero_us:
5087 case Builtin::BIstdc_first_leading_zero_ui:
5088 case Builtin::BIstdc_first_leading_zero_ul:
5089 case Builtin::BIstdc_first_leading_zero_ull:
5090 case Builtin::BI__builtin_stdc_first_leading_zero: {
5091 unsigned ResWidth = S.getASTContext().getIntWidth(T: Call->getType());
5092 return interp__builtin_elementwise_int_unaryop(
5093 S, OpPC, Call, Fn: [ResWidth](const APSInt &Val) {
5094 return APInt(ResWidth, Val.isAllOnes() ? 0 : Val.countl_one() + 1);
5095 });
5096 }
5097
5098 case Builtin::BIstdc_first_leading_one_uc:
5099 case Builtin::BIstdc_first_leading_one_us:
5100 case Builtin::BIstdc_first_leading_one_ui:
5101 case Builtin::BIstdc_first_leading_one_ul:
5102 case Builtin::BIstdc_first_leading_one_ull:
5103 case Builtin::BI__builtin_stdc_first_leading_one: {
5104 unsigned ResWidth = S.getASTContext().getIntWidth(T: Call->getType());
5105 return interp__builtin_elementwise_int_unaryop(
5106 S, OpPC, Call, Fn: [ResWidth](const APSInt &Val) {
5107 return APInt(ResWidth, Val.isZero() ? 0 : Val.countl_zero() + 1);
5108 });
5109 }
5110
5111 case Builtin::BIstdc_first_trailing_zero_uc:
5112 case Builtin::BIstdc_first_trailing_zero_us:
5113 case Builtin::BIstdc_first_trailing_zero_ui:
5114 case Builtin::BIstdc_first_trailing_zero_ul:
5115 case Builtin::BIstdc_first_trailing_zero_ull:
5116 case Builtin::BI__builtin_stdc_first_trailing_zero: {
5117 unsigned ResWidth = S.getASTContext().getIntWidth(T: Call->getType());
5118 return interp__builtin_elementwise_int_unaryop(
5119 S, OpPC, Call, Fn: [ResWidth](const APSInt &Val) {
5120 return APInt(ResWidth, Val.isAllOnes() ? 0 : Val.countr_one() + 1);
5121 });
5122 }
5123
5124 case Builtin::BIstdc_first_trailing_one_uc:
5125 case Builtin::BIstdc_first_trailing_one_us:
5126 case Builtin::BIstdc_first_trailing_one_ui:
5127 case Builtin::BIstdc_first_trailing_one_ul:
5128 case Builtin::BIstdc_first_trailing_one_ull:
5129 case Builtin::BI__builtin_stdc_first_trailing_one: {
5130 unsigned ResWidth = S.getASTContext().getIntWidth(T: Call->getType());
5131 return interp__builtin_elementwise_int_unaryop(
5132 S, OpPC, Call, Fn: [ResWidth](const APSInt &Val) {
5133 return APInt(ResWidth, Val.isZero() ? 0 : Val.countr_zero() + 1);
5134 });
5135 }
5136
5137 case Builtin::BIstdc_count_zeros_uc:
5138 case Builtin::BIstdc_count_zeros_us:
5139 case Builtin::BIstdc_count_zeros_ui:
5140 case Builtin::BIstdc_count_zeros_ul:
5141 case Builtin::BIstdc_count_zeros_ull:
5142 case Builtin::BI__builtin_stdc_count_zeros: {
5143 unsigned ResWidth = S.getASTContext().getIntWidth(T: Call->getType());
5144 return interp__builtin_elementwise_int_unaryop(
5145 S, OpPC, Call, Fn: [ResWidth](const APSInt &Val) {
5146 unsigned BitWidth = Val.getBitWidth();
5147 return APInt(ResWidth, BitWidth - Val.popcount());
5148 });
5149 }
5150
5151 case Builtin::BIstdc_count_ones_uc:
5152 case Builtin::BIstdc_count_ones_us:
5153 case Builtin::BIstdc_count_ones_ui:
5154 case Builtin::BIstdc_count_ones_ul:
5155 case Builtin::BIstdc_count_ones_ull:
5156 case Builtin::BI__builtin_stdc_count_ones: {
5157 unsigned ResWidth = S.getASTContext().getIntWidth(T: Call->getType());
5158 return interp__builtin_elementwise_int_unaryop(
5159 S, OpPC, Call, Fn: [ResWidth](const APSInt &Val) {
5160 return APInt(ResWidth, Val.popcount());
5161 });
5162 }
5163
5164 case Builtin::BIstdc_has_single_bit_uc:
5165 case Builtin::BIstdc_has_single_bit_us:
5166 case Builtin::BIstdc_has_single_bit_ui:
5167 case Builtin::BIstdc_has_single_bit_ul:
5168 case Builtin::BIstdc_has_single_bit_ull:
5169 case Builtin::BI__builtin_stdc_has_single_bit: {
5170 unsigned ResWidth = S.getASTContext().getIntWidth(T: Call->getType());
5171 return interp__builtin_elementwise_int_unaryop(
5172 S, OpPC, Call, Fn: [ResWidth](const APSInt &Val) {
5173 return APInt(ResWidth, Val.popcount() == 1 ? 1 : 0);
5174 });
5175 }
5176
5177 case Builtin::BIstdc_bit_width_uc:
5178 case Builtin::BIstdc_bit_width_us:
5179 case Builtin::BIstdc_bit_width_ui:
5180 case Builtin::BIstdc_bit_width_ul:
5181 case Builtin::BIstdc_bit_width_ull:
5182 case Builtin::BI__builtin_stdc_bit_width: {
5183 unsigned ResWidth = S.getASTContext().getIntWidth(T: Call->getType());
5184 return interp__builtin_elementwise_int_unaryop(
5185 S, OpPC, Call, Fn: [ResWidth](const APSInt &Val) {
5186 unsigned BitWidth = Val.getBitWidth();
5187 return APInt(ResWidth, BitWidth - Val.countl_zero());
5188 });
5189 }
5190
5191 case Builtin::BIstdc_bit_floor_uc:
5192 case Builtin::BIstdc_bit_floor_us:
5193 case Builtin::BIstdc_bit_floor_ui:
5194 case Builtin::BIstdc_bit_floor_ul:
5195 case Builtin::BIstdc_bit_floor_ull:
5196 case Builtin::BI__builtin_stdc_bit_floor:
5197 return interp__builtin_elementwise_int_unaryop(
5198 S, OpPC, Call, Fn: [](const APSInt &Val) {
5199 unsigned BitWidth = Val.getBitWidth();
5200 if (Val.isZero())
5201 return APInt::getZero(numBits: BitWidth);
5202 return APInt::getOneBitSet(numBits: BitWidth,
5203 BitNo: BitWidth - Val.countl_zero() - 1);
5204 });
5205
5206 case Builtin::BIstdc_bit_ceil_uc:
5207 case Builtin::BIstdc_bit_ceil_us:
5208 case Builtin::BIstdc_bit_ceil_ui:
5209 case Builtin::BIstdc_bit_ceil_ul:
5210 case Builtin::BIstdc_bit_ceil_ull:
5211 case Builtin::BI__builtin_stdc_bit_ceil:
5212 return interp__builtin_elementwise_int_unaryop(
5213 S, OpPC, Call, Fn: [](const APSInt &Val) {
5214 unsigned BitWidth = Val.getBitWidth();
5215 if (Val.ule(RHS: 1))
5216 return APInt(BitWidth, 1);
5217 APInt V = Val;
5218 APInt ValMinusOne = V - 1;
5219 unsigned LeadingZeros = ValMinusOne.countl_zero();
5220 if (LeadingZeros == 0)
5221 return APInt(BitWidth, 0); // overflows; wrap to 0
5222 return APInt::getOneBitSet(numBits: BitWidth, BitNo: BitWidth - LeadingZeros);
5223 });
5224
5225 case Builtin::BI__builtin_ffs:
5226 case Builtin::BI__builtin_ffsl:
5227 case Builtin::BI__builtin_ffsll:
5228 return interp__builtin_elementwise_int_unaryop(
5229 S, OpPC, Call, Fn: [](const APSInt &Val) {
5230 return APInt(Val.getBitWidth(),
5231 Val.isZero() ? 0u : Val.countTrailingZeros() + 1u);
5232 });
5233
5234 case Builtin::BIaddressof:
5235 case Builtin::BI__addressof:
5236 case Builtin::BI__builtin_addressof:
5237 assert(isNoopBuiltin(BuiltinID));
5238 return interp__builtin_addressof(S, OpPC, Frame, Call);
5239
5240 case Builtin::BIas_const:
5241 case Builtin::BIforward:
5242 case Builtin::BIforward_like:
5243 case Builtin::BImove:
5244 case Builtin::BImove_if_noexcept:
5245 assert(isNoopBuiltin(BuiltinID));
5246 return interp__builtin_move(S, OpPC, Frame, Call);
5247
5248 case Builtin::BI__builtin_eh_return_data_regno:
5249 return interp__builtin_eh_return_data_regno(S, OpPC, Frame, Call);
5250
5251 case Builtin::BI__builtin_launder:
5252 assert(isNoopBuiltin(BuiltinID));
5253 return true;
5254
5255 case Builtin::BI__builtin_add_overflow:
5256 case Builtin::BI__builtin_sub_overflow:
5257 case Builtin::BI__builtin_mul_overflow:
5258 case Builtin::BI__builtin_sadd_overflow:
5259 case Builtin::BI__builtin_uadd_overflow:
5260 case Builtin::BI__builtin_uaddl_overflow:
5261 case Builtin::BI__builtin_uaddll_overflow:
5262 case Builtin::BI__builtin_usub_overflow:
5263 case Builtin::BI__builtin_usubl_overflow:
5264 case Builtin::BI__builtin_usubll_overflow:
5265 case Builtin::BI__builtin_umul_overflow:
5266 case Builtin::BI__builtin_umull_overflow:
5267 case Builtin::BI__builtin_umulll_overflow:
5268 case Builtin::BI__builtin_saddl_overflow:
5269 case Builtin::BI__builtin_saddll_overflow:
5270 case Builtin::BI__builtin_ssub_overflow:
5271 case Builtin::BI__builtin_ssubl_overflow:
5272 case Builtin::BI__builtin_ssubll_overflow:
5273 case Builtin::BI__builtin_smul_overflow:
5274 case Builtin::BI__builtin_smull_overflow:
5275 case Builtin::BI__builtin_smulll_overflow:
5276 return interp__builtin_overflowop(S, OpPC, Call, BuiltinOp: BuiltinID);
5277
5278 case Builtin::BI__builtin_addcb:
5279 case Builtin::BI__builtin_addcs:
5280 case Builtin::BI__builtin_addc:
5281 case Builtin::BI__builtin_addcl:
5282 case Builtin::BI__builtin_addcll:
5283 case Builtin::BI__builtin_subcb:
5284 case Builtin::BI__builtin_subcs:
5285 case Builtin::BI__builtin_subc:
5286 case Builtin::BI__builtin_subcl:
5287 case Builtin::BI__builtin_subcll:
5288 return interp__builtin_carryop(S, OpPC, Frame, Call, BuiltinOp: BuiltinID);
5289
5290 case Builtin::BI__builtin_clz:
5291 case Builtin::BI__builtin_clzl:
5292 case Builtin::BI__builtin_clzll:
5293 case Builtin::BI__builtin_clzs:
5294 case Builtin::BI__builtin_clzg:
5295 case Builtin::BI__lzcnt16: // Microsoft variants of count leading-zeroes
5296 case Builtin::BI__lzcnt:
5297 case Builtin::BI__lzcnt64:
5298 return interp__builtin_clz(S, OpPC, Frame, Call, BuiltinOp: BuiltinID);
5299
5300 case Builtin::BI__builtin_ctz:
5301 case Builtin::BI__builtin_ctzl:
5302 case Builtin::BI__builtin_ctzll:
5303 case Builtin::BI__builtin_ctzs:
5304 case Builtin::BI__builtin_ctzg:
5305 return interp__builtin_ctz(S, OpPC, Frame, Call, BuiltinID);
5306
5307 case Builtin::BI__builtin_elementwise_clzg:
5308 case Builtin::BI__builtin_elementwise_ctzg:
5309 return interp__builtin_elementwise_countzeroes(S, OpPC, Frame, Call,
5310 BuiltinID);
5311 case Builtin::BI__builtin_bswapg:
5312 case Builtin::BI__builtin_bswap16:
5313 case Builtin::BI__builtin_bswap32:
5314 case Builtin::BI__builtin_bswap64:
5315 case Builtin::BIstdc_memreverse8u8:
5316 case Builtin::BIstdc_memreverse8u16:
5317 case Builtin::BIstdc_memreverse8u32:
5318 case Builtin::BIstdc_memreverse8u64:
5319 return interp__builtin_bswap(S, OpPC, Frame, Call);
5320
5321 case Builtin::BI__atomic_always_lock_free:
5322 case Builtin::BI__atomic_is_lock_free:
5323 return interp__builtin_atomic_lock_free(S, OpPC, Frame, Call, BuiltinOp: BuiltinID);
5324
5325 case Builtin::BI__c11_atomic_is_lock_free:
5326 return interp__builtin_c11_atomic_is_lock_free(S, OpPC, Frame, Call);
5327
5328 case Builtin::BI__builtin_complex:
5329 return interp__builtin_complex(S, OpPC, Frame, Call);
5330
5331 case Builtin::BI__builtin_is_aligned:
5332 case Builtin::BI__builtin_align_up:
5333 case Builtin::BI__builtin_align_down:
5334 return interp__builtin_is_aligned_up_down(S, OpPC, Frame, Call, BuiltinOp: BuiltinID);
5335
5336 case Builtin::BI__builtin_assume_aligned:
5337 return interp__builtin_assume_aligned(S, OpPC, Frame, Call);
5338
5339 case clang::X86::BI__builtin_ia32_crc32qi:
5340 return interp__builtin_ia32_crc32(S, OpPC, Frame, Call, DataBytes: 1);
5341 case clang::X86::BI__builtin_ia32_crc32hi:
5342 return interp__builtin_ia32_crc32(S, OpPC, Frame, Call, DataBytes: 2);
5343 case clang::X86::BI__builtin_ia32_crc32si:
5344 return interp__builtin_ia32_crc32(S, OpPC, Frame, Call, DataBytes: 4);
5345 case clang::X86::BI__builtin_ia32_crc32di:
5346 return interp__builtin_ia32_crc32(S, OpPC, Frame, Call, DataBytes: 8);
5347
5348 case clang::X86::BI__builtin_ia32_bextr_u32:
5349 case clang::X86::BI__builtin_ia32_bextr_u64:
5350 case clang::X86::BI__builtin_ia32_bextri_u32:
5351 case clang::X86::BI__builtin_ia32_bextri_u64:
5352 return interp__builtin_elementwise_int_binop(
5353 S, OpPC, Call, Fn: [](const APSInt &Val, const APSInt &Idx) {
5354 unsigned BitWidth = Val.getBitWidth();
5355 uint64_t Shift = Idx.extractBitsAsZExtValue(numBits: 8, bitPosition: 0);
5356 uint64_t Length = Idx.extractBitsAsZExtValue(numBits: 8, bitPosition: 8);
5357 if (Length > BitWidth) {
5358 Length = BitWidth;
5359 }
5360
5361 // Handle out of bounds cases.
5362 if (Length == 0 || Shift >= BitWidth)
5363 return APInt(BitWidth, 0);
5364
5365 uint64_t Result = Val.getZExtValue() >> Shift;
5366 Result &= llvm::maskTrailingOnes<uint64_t>(N: Length);
5367 return APInt(BitWidth, Result);
5368 });
5369
5370 case clang::X86::BI__builtin_ia32_bzhi_si:
5371 case clang::X86::BI__builtin_ia32_bzhi_di:
5372 return interp__builtin_elementwise_int_binop(
5373 S, OpPC, Call, Fn: [](const APSInt &Val, const APSInt &Idx) {
5374 unsigned BitWidth = Val.getBitWidth();
5375 uint64_t Index = Idx.extractBitsAsZExtValue(numBits: 8, bitPosition: 0);
5376 APSInt Result = Val;
5377
5378 if (Index < BitWidth)
5379 Result.clearHighBits(hiBits: BitWidth - Index);
5380
5381 return Result;
5382 });
5383
5384 case clang::X86::BI__builtin_ia32_ktestcqi:
5385 case clang::X86::BI__builtin_ia32_ktestchi:
5386 case clang::X86::BI__builtin_ia32_ktestcsi:
5387 case clang::X86::BI__builtin_ia32_ktestcdi:
5388 return interp__builtin_elementwise_int_binop(
5389 S, OpPC, Call, Fn: [](const APSInt &A, const APSInt &B) {
5390 return APInt(sizeof(unsigned char) * 8, (~A & B) == 0);
5391 });
5392
5393 case clang::X86::BI__builtin_ia32_ktestzqi:
5394 case clang::X86::BI__builtin_ia32_ktestzhi:
5395 case clang::X86::BI__builtin_ia32_ktestzsi:
5396 case clang::X86::BI__builtin_ia32_ktestzdi:
5397 return interp__builtin_elementwise_int_binop(
5398 S, OpPC, Call, Fn: [](const APSInt &A, const APSInt &B) {
5399 return APInt(sizeof(unsigned char) * 8, (A & B) == 0);
5400 });
5401
5402 case clang::X86::BI__builtin_ia32_kortestcqi:
5403 case clang::X86::BI__builtin_ia32_kortestchi:
5404 case clang::X86::BI__builtin_ia32_kortestcsi:
5405 case clang::X86::BI__builtin_ia32_kortestcdi:
5406 return interp__builtin_elementwise_int_binop(
5407 S, OpPC, Call, Fn: [](const APSInt &A, const APSInt &B) {
5408 return APInt(sizeof(unsigned char) * 8, ~(A | B) == 0);
5409 });
5410
5411 case clang::X86::BI__builtin_ia32_kortestzqi:
5412 case clang::X86::BI__builtin_ia32_kortestzhi:
5413 case clang::X86::BI__builtin_ia32_kortestzsi:
5414 case clang::X86::BI__builtin_ia32_kortestzdi:
5415 return interp__builtin_elementwise_int_binop(
5416 S, OpPC, Call, Fn: [](const APSInt &A, const APSInt &B) {
5417 return APInt(sizeof(unsigned char) * 8, (A | B) == 0);
5418 });
5419
5420 case clang::X86::BI__builtin_ia32_kshiftliqi:
5421 case clang::X86::BI__builtin_ia32_kshiftlihi:
5422 case clang::X86::BI__builtin_ia32_kshiftlisi:
5423 case clang::X86::BI__builtin_ia32_kshiftlidi:
5424 return interp__builtin_elementwise_int_binop(
5425 S, OpPC, Call, Fn: [](const APSInt &LHS, const APSInt &RHS) {
5426 unsigned Amt = RHS.getZExtValue() & 0xFF;
5427 if (Amt >= LHS.getBitWidth())
5428 return APInt::getZero(numBits: LHS.getBitWidth());
5429 return LHS.shl(shiftAmt: Amt);
5430 });
5431
5432 case clang::X86::BI__builtin_ia32_kshiftriqi:
5433 case clang::X86::BI__builtin_ia32_kshiftrihi:
5434 case clang::X86::BI__builtin_ia32_kshiftrisi:
5435 case clang::X86::BI__builtin_ia32_kshiftridi:
5436 return interp__builtin_elementwise_int_binop(
5437 S, OpPC, Call, Fn: [](const APSInt &LHS, const APSInt &RHS) {
5438 unsigned Amt = RHS.getZExtValue() & 0xFF;
5439 if (Amt >= LHS.getBitWidth())
5440 return APInt::getZero(numBits: LHS.getBitWidth());
5441 return LHS.lshr(shiftAmt: Amt);
5442 });
5443
5444 case clang::X86::BI__builtin_ia32_lzcnt_u16:
5445 case clang::X86::BI__builtin_ia32_lzcnt_u32:
5446 case clang::X86::BI__builtin_ia32_lzcnt_u64:
5447 return interp__builtin_elementwise_int_unaryop(
5448 S, OpPC, Call, Fn: [](const APSInt &Src) {
5449 return APInt(Src.getBitWidth(), Src.countLeadingZeros());
5450 });
5451
5452 case clang::X86::BI__builtin_ia32_tzcnt_u16:
5453 case clang::X86::BI__builtin_ia32_tzcnt_u32:
5454 case clang::X86::BI__builtin_ia32_tzcnt_u64:
5455 return interp__builtin_elementwise_int_unaryop(
5456 S, OpPC, Call, Fn: [](const APSInt &Src) {
5457 return APInt(Src.getBitWidth(), Src.countTrailingZeros());
5458 });
5459
5460 case clang::X86::BI__builtin_ia32_pdep_si:
5461 case clang::X86::BI__builtin_ia32_pdep_di:
5462 case Builtin::BI__builtin_elementwise_pdep:
5463 return interp__builtin_elementwise_int_binop(S, OpPC, Call,
5464 Fn: llvm::APIntOps::pdep);
5465
5466 case clang::X86::BI__builtin_ia32_pext_si:
5467 case clang::X86::BI__builtin_ia32_pext_di:
5468 case Builtin::BI__builtin_elementwise_pext:
5469 return interp__builtin_elementwise_int_binop(S, OpPC, Call,
5470 Fn: llvm::APIntOps::pext);
5471
5472 case clang::X86::BI__builtin_ia32_addcarryx_u32:
5473 case clang::X86::BI__builtin_ia32_addcarryx_u64:
5474 return interp__builtin_ia32_addcarry_subborrow(S, OpPC, Frame, Call,
5475 /*IsAdd=*/true);
5476
5477 case clang::X86::BI__builtin_ia32_subborrow_u32:
5478 case clang::X86::BI__builtin_ia32_subborrow_u64:
5479 return interp__builtin_ia32_addcarry_subborrow(S, OpPC, Frame, Call,
5480 /*IsAdd=*/false);
5481
5482 case Builtin::BI__builtin_os_log_format_buffer_size:
5483 return interp__builtin_os_log_format_buffer_size(S, OpPC, Frame, Call);
5484
5485 case Builtin::BI__builtin_ptrauth_string_discriminator:
5486 return interp__builtin_ptrauth_string_discriminator(S, OpPC, Frame, Call);
5487
5488 case Builtin::BI__builtin_infer_alloc_token:
5489 return interp__builtin_infer_alloc_token(S, OpPC, Frame, Call);
5490
5491 case Builtin::BI__noop:
5492 pushInteger(S, Val: 0, QT: Call->getType());
5493 return true;
5494
5495 case Builtin::BI__builtin_operator_new:
5496 return interp__builtin_operator_new(S, OpPC, Frame, Call);
5497
5498 case Builtin::BI__builtin_operator_delete:
5499 return interp__builtin_operator_delete(S, OpPC, Frame, Call);
5500
5501 case Builtin::BI__arithmetic_fence:
5502 return interp__builtin_arithmetic_fence(S, OpPC, Frame, Call);
5503
5504 case Builtin::BI__builtin_reduce_add:
5505 case Builtin::BI__builtin_reduce_mul:
5506 case Builtin::BI__builtin_reduce_and:
5507 case Builtin::BI__builtin_reduce_or:
5508 case Builtin::BI__builtin_reduce_xor:
5509 case Builtin::BI__builtin_reduce_min:
5510 case Builtin::BI__builtin_reduce_max:
5511 return interp__builtin_vector_reduce(S, OpPC, Call, ID: BuiltinID);
5512
5513 case Builtin::BI__builtin_elementwise_popcount:
5514 return interp__builtin_elementwise_int_unaryop(
5515 S, OpPC, Call, Fn: [](const APSInt &Src) {
5516 return APInt(Src.getBitWidth(), Src.popcount());
5517 });
5518 case Builtin::BI__builtin_elementwise_bitreverse:
5519 return interp__builtin_elementwise_int_unaryop(
5520 S, OpPC, Call, Fn: [](const APSInt &Src) { return Src.reverseBits(); });
5521
5522 case Builtin::BI__builtin_elementwise_abs:
5523 return interp__builtin_elementwise_abs(S, OpPC, Frame, Call, BuiltinID);
5524
5525 case Builtin::BI__builtin_memcpy:
5526 case Builtin::BImemcpy:
5527 case Builtin::BI__builtin_wmemcpy:
5528 case Builtin::BIwmemcpy:
5529 case Builtin::BI__builtin_memmove:
5530 case Builtin::BImemmove:
5531 case Builtin::BI__builtin_wmemmove:
5532 case Builtin::BIwmemmove:
5533 return interp__builtin_memcpy(S, OpPC, Frame, Call, ID: BuiltinID);
5534
5535 case Builtin::BI__builtin_memcmp:
5536 case Builtin::BImemcmp:
5537 case Builtin::BI__builtin_bcmp:
5538 case Builtin::BIbcmp:
5539 case Builtin::BI__builtin_wmemcmp:
5540 case Builtin::BIwmemcmp:
5541 return interp__builtin_memcmp(S, OpPC, Frame, Call, ID: BuiltinID);
5542
5543 case Builtin::BImemchr:
5544 case Builtin::BI__builtin_memchr:
5545 case Builtin::BIstrchr:
5546 case Builtin::BI__builtin_strchr:
5547 case Builtin::BIwmemchr:
5548 case Builtin::BI__builtin_wmemchr:
5549 case Builtin::BIwcschr:
5550 case Builtin::BI__builtin_wcschr:
5551 case Builtin::BI__builtin_char_memchr:
5552 return interp__builtin_memchr(S, OpPC, Call, ID: BuiltinID);
5553
5554 case Builtin::BI__builtin_object_size:
5555 case Builtin::BI__builtin_dynamic_object_size:
5556 return interp__builtin_object_size(S, OpPC, Frame, Call);
5557
5558 case Builtin::BI__builtin_is_within_lifetime:
5559 return interp__builtin_is_within_lifetime(S, OpPC, Call);
5560
5561 case Builtin::BI__builtin_elementwise_add_sat:
5562 return interp__builtin_elementwise_int_binop(
5563 S, OpPC, Call, Fn: [](const APSInt &LHS, const APSInt &RHS) {
5564 return LHS.isSigned() ? LHS.sadd_sat(RHS) : LHS.uadd_sat(RHS);
5565 });
5566
5567 case Builtin::BI__builtin_elementwise_sub_sat:
5568 return interp__builtin_elementwise_int_binop(
5569 S, OpPC, Call, Fn: [](const APSInt &LHS, const APSInt &RHS) {
5570 return LHS.isSigned() ? LHS.ssub_sat(RHS) : LHS.usub_sat(RHS);
5571 });
5572 case X86::BI__builtin_ia32_extract128i256:
5573 case X86::BI__builtin_ia32_vextractf128_pd256:
5574 case X86::BI__builtin_ia32_vextractf128_ps256:
5575 case X86::BI__builtin_ia32_vextractf128_si256:
5576 return interp__builtin_ia32_extract_vector(S, OpPC, Call, ID: BuiltinID);
5577
5578 case X86::BI__builtin_ia32_extractf32x4_256_mask:
5579 case X86::BI__builtin_ia32_extractf32x4_mask:
5580 case X86::BI__builtin_ia32_extractf32x8_mask:
5581 case X86::BI__builtin_ia32_extractf64x2_256_mask:
5582 case X86::BI__builtin_ia32_extractf64x2_512_mask:
5583 case X86::BI__builtin_ia32_extractf64x4_mask:
5584 case X86::BI__builtin_ia32_extracti32x4_256_mask:
5585 case X86::BI__builtin_ia32_extracti32x4_mask:
5586 case X86::BI__builtin_ia32_extracti32x8_mask:
5587 case X86::BI__builtin_ia32_extracti64x2_256_mask:
5588 case X86::BI__builtin_ia32_extracti64x2_512_mask:
5589 case X86::BI__builtin_ia32_extracti64x4_mask:
5590 return interp__builtin_ia32_extract_vector_masked(S, OpPC, Call, ID: BuiltinID);
5591
5592 case clang::X86::BI__builtin_ia32_pmulhrsw128:
5593 case clang::X86::BI__builtin_ia32_pmulhrsw256:
5594 case clang::X86::BI__builtin_ia32_pmulhrsw512:
5595 return interp__builtin_elementwise_int_binop(
5596 S, OpPC, Call, Fn: [](const APSInt &LHS, const APSInt &RHS) {
5597 return (llvm::APIntOps::mulsExtended(C1: LHS, C2: RHS).ashr(ShiftAmt: 14) + 1)
5598 .extractBits(numBits: 16, bitPosition: 1);
5599 });
5600
5601 case clang::X86::BI__builtin_ia32_movmskps:
5602 case clang::X86::BI__builtin_ia32_movmskpd:
5603 case clang::X86::BI__builtin_ia32_pmovmskb128:
5604 case clang::X86::BI__builtin_ia32_pmovmskb256:
5605 case clang::X86::BI__builtin_ia32_movmskps256:
5606 case clang::X86::BI__builtin_ia32_movmskpd256: {
5607 return interp__builtin_ia32_movmsk_op(S, OpPC, Call);
5608 }
5609
5610 case X86::BI__builtin_ia32_psignb128:
5611 case X86::BI__builtin_ia32_psignb256:
5612 case X86::BI__builtin_ia32_psignw128:
5613 case X86::BI__builtin_ia32_psignw256:
5614 case X86::BI__builtin_ia32_psignd128:
5615 case X86::BI__builtin_ia32_psignd256:
5616 return interp__builtin_elementwise_int_binop(
5617 S, OpPC, Call, Fn: [](const APInt &AElem, const APInt &BElem) {
5618 if (BElem.isZero())
5619 return APInt::getZero(numBits: AElem.getBitWidth());
5620 if (BElem.isNegative())
5621 return -AElem;
5622 return AElem;
5623 });
5624
5625 case clang::X86::BI__builtin_ia32_pavgb128:
5626 case clang::X86::BI__builtin_ia32_pavgw128:
5627 case clang::X86::BI__builtin_ia32_pavgb256:
5628 case clang::X86::BI__builtin_ia32_pavgw256:
5629 case clang::X86::BI__builtin_ia32_pavgb512:
5630 case clang::X86::BI__builtin_ia32_pavgw512:
5631 return interp__builtin_elementwise_int_binop(S, OpPC, Call,
5632 Fn: llvm::APIntOps::avgCeilU);
5633
5634 case clang::X86::BI__builtin_ia32_pmaddubsw128:
5635 case clang::X86::BI__builtin_ia32_pmaddubsw256:
5636 case clang::X86::BI__builtin_ia32_pmaddubsw512:
5637 return interp__builtin_ia32_pmul(
5638 S, OpPC, Call,
5639 Fn: [](const APSInt &LoLHS, const APSInt &HiLHS, const APSInt &LoRHS,
5640 const APSInt &HiRHS) {
5641 unsigned BitWidth = 2 * LoLHS.getBitWidth();
5642 return (LoLHS.zext(width: BitWidth) * LoRHS.sext(width: BitWidth))
5643 .sadd_sat(RHS: (HiLHS.zext(width: BitWidth) * HiRHS.sext(width: BitWidth)));
5644 });
5645
5646 case clang::X86::BI__builtin_ia32_pmaddwd128:
5647 case clang::X86::BI__builtin_ia32_pmaddwd256:
5648 case clang::X86::BI__builtin_ia32_pmaddwd512:
5649 return interp__builtin_ia32_pmul(
5650 S, OpPC, Call,
5651 Fn: [](const APSInt &LoLHS, const APSInt &HiLHS, const APSInt &LoRHS,
5652 const APSInt &HiRHS) {
5653 unsigned BitWidth = 2 * LoLHS.getBitWidth();
5654 return (LoLHS.sext(width: BitWidth) * LoRHS.sext(width: BitWidth)) +
5655 (HiLHS.sext(width: BitWidth) * HiRHS.sext(width: BitWidth));
5656 });
5657
5658 case clang::X86::BI__builtin_ia32_psadbw128:
5659 case clang::X86::BI__builtin_ia32_psadbw256:
5660 case clang::X86::BI__builtin_ia32_psadbw512:
5661 return interp__builtin_ia32_psadbw(S, OpPC, Call);
5662
5663 case clang::X86::BI__builtin_ia32_dbpsadbw128:
5664 case clang::X86::BI__builtin_ia32_dbpsadbw256:
5665 case clang::X86::BI__builtin_ia32_dbpsadbw512:
5666 return interp__builtin_ia32_dbpsadbw(S, OpPC, Call);
5667
5668 case clang::X86::BI__builtin_ia32_mpsadbw128:
5669 case clang::X86::BI__builtin_ia32_mpsadbw256:
5670 return interp__builtin_ia32_mpsadbw(S, OpPC, Call);
5671
5672 case clang::X86::BI__builtin_ia32_pmulhuw128:
5673 case clang::X86::BI__builtin_ia32_pmulhuw256:
5674 case clang::X86::BI__builtin_ia32_pmulhuw512:
5675 return interp__builtin_elementwise_int_binop(S, OpPC, Call,
5676 Fn: llvm::APIntOps::mulhu);
5677
5678 case clang::X86::BI__builtin_ia32_pmulhw128:
5679 case clang::X86::BI__builtin_ia32_pmulhw256:
5680 case clang::X86::BI__builtin_ia32_pmulhw512:
5681 return interp__builtin_elementwise_int_binop(S, OpPC, Call,
5682 Fn: llvm::APIntOps::mulhs);
5683
5684 case clang::X86::BI__builtin_ia32_psllv2di:
5685 case clang::X86::BI__builtin_ia32_psllv4di:
5686 case clang::X86::BI__builtin_ia32_psllv4si:
5687 case clang::X86::BI__builtin_ia32_psllv8di:
5688 case clang::X86::BI__builtin_ia32_psllv8hi:
5689 case clang::X86::BI__builtin_ia32_psllv8si:
5690 case clang::X86::BI__builtin_ia32_psllv16hi:
5691 case clang::X86::BI__builtin_ia32_psllv16si:
5692 case clang::X86::BI__builtin_ia32_psllv32hi:
5693 case clang::X86::BI__builtin_ia32_psllwi128:
5694 case clang::X86::BI__builtin_ia32_psllwi256:
5695 case clang::X86::BI__builtin_ia32_psllwi512:
5696 case clang::X86::BI__builtin_ia32_pslldi128:
5697 case clang::X86::BI__builtin_ia32_pslldi256:
5698 case clang::X86::BI__builtin_ia32_pslldi512:
5699 case clang::X86::BI__builtin_ia32_psllqi128:
5700 case clang::X86::BI__builtin_ia32_psllqi256:
5701 case clang::X86::BI__builtin_ia32_psllqi512:
5702 return interp__builtin_elementwise_int_binop(
5703 S, OpPC, Call, Fn: [](const APSInt &LHS, const APSInt &RHS) {
5704 if (RHS.uge(RHS: LHS.getBitWidth())) {
5705 return APInt::getZero(numBits: LHS.getBitWidth());
5706 }
5707 return LHS.shl(shiftAmt: RHS.getZExtValue());
5708 });
5709
5710 case clang::X86::BI__builtin_ia32_psrav4si:
5711 case clang::X86::BI__builtin_ia32_psrav8di:
5712 case clang::X86::BI__builtin_ia32_psrav8hi:
5713 case clang::X86::BI__builtin_ia32_psrav8si:
5714 case clang::X86::BI__builtin_ia32_psrav16hi:
5715 case clang::X86::BI__builtin_ia32_psrav16si:
5716 case clang::X86::BI__builtin_ia32_psrav32hi:
5717 case clang::X86::BI__builtin_ia32_psravq128:
5718 case clang::X86::BI__builtin_ia32_psravq256:
5719 case clang::X86::BI__builtin_ia32_psrawi128:
5720 case clang::X86::BI__builtin_ia32_psrawi256:
5721 case clang::X86::BI__builtin_ia32_psrawi512:
5722 case clang::X86::BI__builtin_ia32_psradi128:
5723 case clang::X86::BI__builtin_ia32_psradi256:
5724 case clang::X86::BI__builtin_ia32_psradi512:
5725 case clang::X86::BI__builtin_ia32_psraqi128:
5726 case clang::X86::BI__builtin_ia32_psraqi256:
5727 case clang::X86::BI__builtin_ia32_psraqi512:
5728 return interp__builtin_elementwise_int_binop(
5729 S, OpPC, Call, Fn: [](const APSInt &LHS, const APSInt &RHS) {
5730 if (RHS.uge(RHS: LHS.getBitWidth())) {
5731 return LHS.ashr(ShiftAmt: LHS.getBitWidth() - 1);
5732 }
5733 return LHS.ashr(ShiftAmt: RHS.getZExtValue());
5734 });
5735
5736 case clang::X86::BI__builtin_ia32_psrlv2di:
5737 case clang::X86::BI__builtin_ia32_psrlv4di:
5738 case clang::X86::BI__builtin_ia32_psrlv4si:
5739 case clang::X86::BI__builtin_ia32_psrlv8di:
5740 case clang::X86::BI__builtin_ia32_psrlv8hi:
5741 case clang::X86::BI__builtin_ia32_psrlv8si:
5742 case clang::X86::BI__builtin_ia32_psrlv16hi:
5743 case clang::X86::BI__builtin_ia32_psrlv16si:
5744 case clang::X86::BI__builtin_ia32_psrlv32hi:
5745 case clang::X86::BI__builtin_ia32_psrlwi128:
5746 case clang::X86::BI__builtin_ia32_psrlwi256:
5747 case clang::X86::BI__builtin_ia32_psrlwi512:
5748 case clang::X86::BI__builtin_ia32_psrldi128:
5749 case clang::X86::BI__builtin_ia32_psrldi256:
5750 case clang::X86::BI__builtin_ia32_psrldi512:
5751 case clang::X86::BI__builtin_ia32_psrlqi128:
5752 case clang::X86::BI__builtin_ia32_psrlqi256:
5753 case clang::X86::BI__builtin_ia32_psrlqi512:
5754 return interp__builtin_elementwise_int_binop(
5755 S, OpPC, Call, Fn: [](const APSInt &LHS, const APSInt &RHS) {
5756 if (RHS.uge(RHS: LHS.getBitWidth())) {
5757 return APInt::getZero(numBits: LHS.getBitWidth());
5758 }
5759 return LHS.lshr(shiftAmt: RHS.getZExtValue());
5760 });
5761 case clang::X86::BI__builtin_ia32_packsswb128:
5762 case clang::X86::BI__builtin_ia32_packsswb256:
5763 case clang::X86::BI__builtin_ia32_packsswb512:
5764 case clang::X86::BI__builtin_ia32_packssdw128:
5765 case clang::X86::BI__builtin_ia32_packssdw256:
5766 case clang::X86::BI__builtin_ia32_packssdw512:
5767 return interp__builtin_ia32_pack(S, OpPC, E: Call, PackFn: [](const APSInt &Src) {
5768 return APInt(Src).truncSSat(width: Src.getBitWidth() / 2);
5769 });
5770 case clang::X86::BI__builtin_ia32_packusdw128:
5771 case clang::X86::BI__builtin_ia32_packusdw256:
5772 case clang::X86::BI__builtin_ia32_packusdw512:
5773 case clang::X86::BI__builtin_ia32_packuswb128:
5774 case clang::X86::BI__builtin_ia32_packuswb256:
5775 case clang::X86::BI__builtin_ia32_packuswb512:
5776 return interp__builtin_ia32_pack(S, OpPC, E: Call, PackFn: [](const APSInt &Src) {
5777 return APInt(Src).truncSSatU(width: Src.getBitWidth() / 2);
5778 });
5779
5780 case clang::X86::BI__builtin_ia32_selectss_128:
5781 case clang::X86::BI__builtin_ia32_selectsd_128:
5782 case clang::X86::BI__builtin_ia32_selectsh_128:
5783 case clang::X86::BI__builtin_ia32_selectsbf_128:
5784 return interp__builtin_ia32_select_scalar(S, Call);
5785 case clang::X86::BI__builtin_ia32_vprotbi:
5786 case clang::X86::BI__builtin_ia32_vprotdi:
5787 case clang::X86::BI__builtin_ia32_vprotqi:
5788 case clang::X86::BI__builtin_ia32_vprotwi:
5789 case clang::X86::BI__builtin_ia32_prold128:
5790 case clang::X86::BI__builtin_ia32_prold256:
5791 case clang::X86::BI__builtin_ia32_prold512:
5792 case clang::X86::BI__builtin_ia32_prolq128:
5793 case clang::X86::BI__builtin_ia32_prolq256:
5794 case clang::X86::BI__builtin_ia32_prolq512:
5795 return interp__builtin_elementwise_int_binop(
5796 S, OpPC, Call,
5797 Fn: [](const APSInt &LHS, const APSInt &RHS) { return LHS.rotl(rotateAmt: RHS); });
5798
5799 case clang::X86::BI__builtin_ia32_prord128:
5800 case clang::X86::BI__builtin_ia32_prord256:
5801 case clang::X86::BI__builtin_ia32_prord512:
5802 case clang::X86::BI__builtin_ia32_prorq128:
5803 case clang::X86::BI__builtin_ia32_prorq256:
5804 case clang::X86::BI__builtin_ia32_prorq512:
5805 return interp__builtin_elementwise_int_binop(
5806 S, OpPC, Call,
5807 Fn: [](const APSInt &LHS, const APSInt &RHS) { return LHS.rotr(rotateAmt: RHS); });
5808
5809 case Builtin::BI__builtin_elementwise_max:
5810 case Builtin::BI__builtin_elementwise_min:
5811 return interp__builtin_elementwise_maxmin(S, OpPC, Call, BuiltinID);
5812
5813 case clang::X86::BI__builtin_ia32_phaddw128:
5814 case clang::X86::BI__builtin_ia32_phaddw256:
5815 case clang::X86::BI__builtin_ia32_phaddd128:
5816 case clang::X86::BI__builtin_ia32_phaddd256:
5817 return interp_builtin_horizontal_int_binop(
5818 S, OpPC, Call,
5819 Fn: [](const APSInt &LHS, const APSInt &RHS) { return LHS + RHS; });
5820 case clang::X86::BI__builtin_ia32_phaddsw128:
5821 case clang::X86::BI__builtin_ia32_phaddsw256:
5822 return interp_builtin_horizontal_int_binop(
5823 S, OpPC, Call,
5824 Fn: [](const APSInt &LHS, const APSInt &RHS) { return LHS.sadd_sat(RHS); });
5825 case clang::X86::BI__builtin_ia32_phsubw128:
5826 case clang::X86::BI__builtin_ia32_phsubw256:
5827 case clang::X86::BI__builtin_ia32_phsubd128:
5828 case clang::X86::BI__builtin_ia32_phsubd256:
5829 return interp_builtin_horizontal_int_binop(
5830 S, OpPC, Call,
5831 Fn: [](const APSInt &LHS, const APSInt &RHS) { return LHS - RHS; });
5832 case clang::X86::BI__builtin_ia32_phsubsw128:
5833 case clang::X86::BI__builtin_ia32_phsubsw256:
5834 return interp_builtin_horizontal_int_binop(
5835 S, OpPC, Call,
5836 Fn: [](const APSInt &LHS, const APSInt &RHS) { return LHS.ssub_sat(RHS); });
5837 case clang::X86::BI__builtin_ia32_haddpd:
5838 case clang::X86::BI__builtin_ia32_haddps:
5839 case clang::X86::BI__builtin_ia32_haddpd256:
5840 case clang::X86::BI__builtin_ia32_haddps256:
5841 return interp_builtin_horizontal_fp_binop(
5842 S, OpPC, Call,
5843 Fn: [](const APFloat &LHS, const APFloat &RHS, llvm::RoundingMode RM) {
5844 APFloat F = LHS;
5845 F.add(RHS, RM);
5846 return F;
5847 });
5848 case clang::X86::BI__builtin_ia32_hsubpd:
5849 case clang::X86::BI__builtin_ia32_hsubps:
5850 case clang::X86::BI__builtin_ia32_hsubpd256:
5851 case clang::X86::BI__builtin_ia32_hsubps256:
5852 return interp_builtin_horizontal_fp_binop(
5853 S, OpPC, Call,
5854 Fn: [](const APFloat &LHS, const APFloat &RHS, llvm::RoundingMode RM) {
5855 APFloat F = LHS;
5856 F.subtract(RHS, RM);
5857 return F;
5858 });
5859 case clang::X86::BI__builtin_ia32_addsubpd:
5860 case clang::X86::BI__builtin_ia32_addsubps:
5861 case clang::X86::BI__builtin_ia32_addsubpd256:
5862 case clang::X86::BI__builtin_ia32_addsubps256:
5863 return interp__builtin_ia32_addsub(S, OpPC, Call);
5864
5865 case clang::X86::BI__builtin_ia32_pmuldq128:
5866 case clang::X86::BI__builtin_ia32_pmuldq256:
5867 case clang::X86::BI__builtin_ia32_pmuldq512:
5868 return interp__builtin_ia32_pmul(
5869 S, OpPC, Call,
5870 Fn: [](const APSInt &LoLHS, const APSInt &HiLHS, const APSInt &LoRHS,
5871 const APSInt &HiRHS) {
5872 return llvm::APIntOps::mulsExtended(C1: LoLHS, C2: LoRHS);
5873 });
5874
5875 case clang::X86::BI__builtin_ia32_pmuludq128:
5876 case clang::X86::BI__builtin_ia32_pmuludq256:
5877 case clang::X86::BI__builtin_ia32_pmuludq512:
5878 return interp__builtin_ia32_pmul(
5879 S, OpPC, Call,
5880 Fn: [](const APSInt &LoLHS, const APSInt &HiLHS, const APSInt &LoRHS,
5881 const APSInt &HiRHS) {
5882 return llvm::APIntOps::muluExtended(C1: LoLHS, C2: LoRHS);
5883 });
5884
5885 case clang::X86::BI__builtin_ia32_pclmulqdq128:
5886 case clang::X86::BI__builtin_ia32_pclmulqdq256:
5887 case clang::X86::BI__builtin_ia32_pclmulqdq512:
5888 return interp__builtin_ia32_pclmulqdq(S, OpPC, Call);
5889 case Builtin::BI__builtin_elementwise_clmul:
5890 return interp__builtin_elementwise_int_binop(S, OpPC, Call,
5891 Fn: llvm::APIntOps::clmul);
5892
5893 case Builtin::BI__builtin_elementwise_fma:
5894 return interp__builtin_elementwise_triop_fp(
5895 S, OpPC, Call,
5896 Fn: [](const APFloat &X, const APFloat &Y, const APFloat &Z,
5897 llvm::RoundingMode RM) {
5898 APFloat F = X;
5899 F.fusedMultiplyAdd(Multiplicand: Y, Addend: Z, RM);
5900 return F;
5901 });
5902
5903 case X86::BI__builtin_ia32_vpmadd52luq128:
5904 case X86::BI__builtin_ia32_vpmadd52luq256:
5905 case X86::BI__builtin_ia32_vpmadd52luq512:
5906 return interp__builtin_elementwise_triop(
5907 S, OpPC, Call, Fn: [](const APSInt &A, const APSInt &B, const APSInt &C) {
5908 return A + (B.trunc(width: 52) * C.trunc(width: 52)).zext(width: 64);
5909 });
5910 case X86::BI__builtin_ia32_vpmadd52huq128:
5911 case X86::BI__builtin_ia32_vpmadd52huq256:
5912 case X86::BI__builtin_ia32_vpmadd52huq512:
5913 return interp__builtin_elementwise_triop(
5914 S, OpPC, Call, Fn: [](const APSInt &A, const APSInt &B, const APSInt &C) {
5915 return A + llvm::APIntOps::mulhu(C1: B.trunc(width: 52), C2: C.trunc(width: 52)).zext(width: 64);
5916 });
5917
5918 case X86::BI__builtin_ia32_vpshldd128:
5919 case X86::BI__builtin_ia32_vpshldd256:
5920 case X86::BI__builtin_ia32_vpshldd512:
5921 case X86::BI__builtin_ia32_vpshldq128:
5922 case X86::BI__builtin_ia32_vpshldq256:
5923 case X86::BI__builtin_ia32_vpshldq512:
5924 case X86::BI__builtin_ia32_vpshldw128:
5925 case X86::BI__builtin_ia32_vpshldw256:
5926 case X86::BI__builtin_ia32_vpshldw512:
5927 return interp__builtin_elementwise_triop(
5928 S, OpPC, Call,
5929 Fn: [](const APSInt &Hi, const APSInt &Lo, const APSInt &Amt) {
5930 return llvm::APIntOps::fshl(Hi, Lo, Shift: Amt);
5931 });
5932
5933 case X86::BI__builtin_ia32_vpshrdd128:
5934 case X86::BI__builtin_ia32_vpshrdd256:
5935 case X86::BI__builtin_ia32_vpshrdd512:
5936 case X86::BI__builtin_ia32_vpshrdq128:
5937 case X86::BI__builtin_ia32_vpshrdq256:
5938 case X86::BI__builtin_ia32_vpshrdq512:
5939 case X86::BI__builtin_ia32_vpshrdw128:
5940 case X86::BI__builtin_ia32_vpshrdw256:
5941 case X86::BI__builtin_ia32_vpshrdw512:
5942 // NOTE: Reversed Hi/Lo operands.
5943 return interp__builtin_elementwise_triop(
5944 S, OpPC, Call,
5945 Fn: [](const APSInt &Lo, const APSInt &Hi, const APSInt &Amt) {
5946 return llvm::APIntOps::fshr(Hi, Lo, Shift: Amt);
5947 });
5948 case X86::BI__builtin_ia32_vpconflictsi_128:
5949 case X86::BI__builtin_ia32_vpconflictsi_256:
5950 case X86::BI__builtin_ia32_vpconflictsi_512:
5951 case X86::BI__builtin_ia32_vpconflictdi_128:
5952 case X86::BI__builtin_ia32_vpconflictdi_256:
5953 case X86::BI__builtin_ia32_vpconflictdi_512:
5954 return interp__builtin_ia32_vpconflict(S, OpPC, Call);
5955 case X86::BI__builtin_ia32_compressdf128_mask:
5956 case X86::BI__builtin_ia32_compressdf256_mask:
5957 case X86::BI__builtin_ia32_compressdf512_mask:
5958 case X86::BI__builtin_ia32_compressdi128_mask:
5959 case X86::BI__builtin_ia32_compressdi256_mask:
5960 case X86::BI__builtin_ia32_compressdi512_mask:
5961 case X86::BI__builtin_ia32_compresshi128_mask:
5962 case X86::BI__builtin_ia32_compresshi256_mask:
5963 case X86::BI__builtin_ia32_compresshi512_mask:
5964 case X86::BI__builtin_ia32_compressqi128_mask:
5965 case X86::BI__builtin_ia32_compressqi256_mask:
5966 case X86::BI__builtin_ia32_compressqi512_mask:
5967 case X86::BI__builtin_ia32_compresssf128_mask:
5968 case X86::BI__builtin_ia32_compresssf256_mask:
5969 case X86::BI__builtin_ia32_compresssf512_mask:
5970 case X86::BI__builtin_ia32_compresssi128_mask:
5971 case X86::BI__builtin_ia32_compresssi256_mask:
5972 case X86::BI__builtin_ia32_compresssi512_mask: {
5973 unsigned NumElems =
5974 Call->getArg(Arg: 0)->getType()->castAs<VectorType>()->getNumElements();
5975 return interp__builtin_ia32_shuffle_generic(
5976 S, OpPC, Call, GetSourceIndex: [NumElems](unsigned DstIdx, const APInt &ShuffleMask) {
5977 APInt CompressMask = ShuffleMask.trunc(width: NumElems);
5978 if (DstIdx < CompressMask.popcount()) {
5979 while (DstIdx != 0) {
5980 CompressMask = CompressMask & (CompressMask - 1);
5981 DstIdx--;
5982 }
5983 return std::pair<unsigned, int>{
5984 0, static_cast<int>(CompressMask.countr_zero())};
5985 }
5986 return std::pair<unsigned, int>{1, static_cast<int>(DstIdx)};
5987 });
5988 }
5989 case X86::BI__builtin_ia32_expanddf128_mask:
5990 case X86::BI__builtin_ia32_expanddf256_mask:
5991 case X86::BI__builtin_ia32_expanddf512_mask:
5992 case X86::BI__builtin_ia32_expanddi128_mask:
5993 case X86::BI__builtin_ia32_expanddi256_mask:
5994 case X86::BI__builtin_ia32_expanddi512_mask:
5995 case X86::BI__builtin_ia32_expandhi128_mask:
5996 case X86::BI__builtin_ia32_expandhi256_mask:
5997 case X86::BI__builtin_ia32_expandhi512_mask:
5998 case X86::BI__builtin_ia32_expandqi128_mask:
5999 case X86::BI__builtin_ia32_expandqi256_mask:
6000 case X86::BI__builtin_ia32_expandqi512_mask:
6001 case X86::BI__builtin_ia32_expandsf128_mask:
6002 case X86::BI__builtin_ia32_expandsf256_mask:
6003 case X86::BI__builtin_ia32_expandsf512_mask:
6004 case X86::BI__builtin_ia32_expandsi128_mask:
6005 case X86::BI__builtin_ia32_expandsi256_mask:
6006 case X86::BI__builtin_ia32_expandsi512_mask: {
6007 return interp__builtin_ia32_shuffle_generic(
6008 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, const APInt &ShuffleMask) {
6009 // Trunc to the sub-mask for the dst index and count the number of
6010 // src elements used prior to that.
6011 APInt ExpandMask = ShuffleMask.trunc(width: DstIdx + 1);
6012 if (ExpandMask[DstIdx]) {
6013 int SrcIdx = ExpandMask.popcount() - 1;
6014 return std::pair<unsigned, int>{0, SrcIdx};
6015 }
6016 return std::pair<unsigned, int>{1, static_cast<int>(DstIdx)};
6017 });
6018 }
6019 case clang::X86::BI__builtin_ia32_blendpd:
6020 case clang::X86::BI__builtin_ia32_blendpd256:
6021 case clang::X86::BI__builtin_ia32_blendps:
6022 case clang::X86::BI__builtin_ia32_blendps256:
6023 case clang::X86::BI__builtin_ia32_pblendw128:
6024 case clang::X86::BI__builtin_ia32_pblendw256:
6025 case clang::X86::BI__builtin_ia32_pblendd128:
6026 case clang::X86::BI__builtin_ia32_pblendd256:
6027 return interp__builtin_ia32_shuffle_generic(
6028 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6029 // Bit index for mask.
6030 unsigned MaskBit = (ShuffleMask >> (DstIdx % 8)) & 0x1;
6031 unsigned SrcVecIdx = MaskBit ? 1 : 0; // 1 = TrueVec, 0 = FalseVec
6032 return std::pair<unsigned, int>{SrcVecIdx, static_cast<int>(DstIdx)};
6033 });
6034
6035
6036
6037 case clang::X86::BI__builtin_ia32_blendvpd:
6038 case clang::X86::BI__builtin_ia32_blendvpd256:
6039 case clang::X86::BI__builtin_ia32_blendvps:
6040 case clang::X86::BI__builtin_ia32_blendvps256:
6041 return interp__builtin_elementwise_triop_fp(
6042 S, OpPC, Call,
6043 Fn: [](const APFloat &F, const APFloat &T, const APFloat &C,
6044 llvm::RoundingMode) { return C.isNegative() ? T : F; });
6045
6046 case clang::X86::BI__builtin_ia32_pblendvb128:
6047 case clang::X86::BI__builtin_ia32_pblendvb256:
6048 return interp__builtin_elementwise_triop(
6049 S, OpPC, Call, Fn: [](const APSInt &F, const APSInt &T, const APSInt &C) {
6050 return ((APInt)C).isNegative() ? T : F;
6051 });
6052 case X86::BI__builtin_ia32_ptestz128:
6053 case X86::BI__builtin_ia32_ptestz256:
6054 case X86::BI__builtin_ia32_vtestzps:
6055 case X86::BI__builtin_ia32_vtestzps256:
6056 case X86::BI__builtin_ia32_vtestzpd:
6057 case X86::BI__builtin_ia32_vtestzpd256:
6058 return interp__builtin_ia32_test_op(
6059 S, OpPC, Call,
6060 Fn: [](const APInt &A, const APInt &B) { return (A & B) == 0; });
6061 case X86::BI__builtin_ia32_ptestc128:
6062 case X86::BI__builtin_ia32_ptestc256:
6063 case X86::BI__builtin_ia32_vtestcps:
6064 case X86::BI__builtin_ia32_vtestcps256:
6065 case X86::BI__builtin_ia32_vtestcpd:
6066 case X86::BI__builtin_ia32_vtestcpd256:
6067 return interp__builtin_ia32_test_op(
6068 S, OpPC, Call,
6069 Fn: [](const APInt &A, const APInt &B) { return (~A & B) == 0; });
6070 case X86::BI__builtin_ia32_ptestnzc128:
6071 case X86::BI__builtin_ia32_ptestnzc256:
6072 case X86::BI__builtin_ia32_vtestnzcps:
6073 case X86::BI__builtin_ia32_vtestnzcps256:
6074 case X86::BI__builtin_ia32_vtestnzcpd:
6075 case X86::BI__builtin_ia32_vtestnzcpd256:
6076 return interp__builtin_ia32_test_op(
6077 S, OpPC, Call, Fn: [](const APInt &A, const APInt &B) {
6078 return ((A & B) != 0) && ((~A & B) != 0);
6079 });
6080 case X86::BI__builtin_ia32_selectb_128:
6081 case X86::BI__builtin_ia32_selectb_256:
6082 case X86::BI__builtin_ia32_selectb_512:
6083 case X86::BI__builtin_ia32_selectw_128:
6084 case X86::BI__builtin_ia32_selectw_256:
6085 case X86::BI__builtin_ia32_selectw_512:
6086 case X86::BI__builtin_ia32_selectd_128:
6087 case X86::BI__builtin_ia32_selectd_256:
6088 case X86::BI__builtin_ia32_selectd_512:
6089 case X86::BI__builtin_ia32_selectq_128:
6090 case X86::BI__builtin_ia32_selectq_256:
6091 case X86::BI__builtin_ia32_selectq_512:
6092 case X86::BI__builtin_ia32_selectph_128:
6093 case X86::BI__builtin_ia32_selectph_256:
6094 case X86::BI__builtin_ia32_selectph_512:
6095 case X86::BI__builtin_ia32_selectpbf_128:
6096 case X86::BI__builtin_ia32_selectpbf_256:
6097 case X86::BI__builtin_ia32_selectpbf_512:
6098 case X86::BI__builtin_ia32_selectps_128:
6099 case X86::BI__builtin_ia32_selectps_256:
6100 case X86::BI__builtin_ia32_selectps_512:
6101 case X86::BI__builtin_ia32_selectpd_128:
6102 case X86::BI__builtin_ia32_selectpd_256:
6103 case X86::BI__builtin_ia32_selectpd_512:
6104 return interp__builtin_ia32_select(S, OpPC, Call);
6105
6106 case X86::BI__builtin_ia32_shufps:
6107 case X86::BI__builtin_ia32_shufps256:
6108 case X86::BI__builtin_ia32_shufps512:
6109 return interp__builtin_ia32_shuffle_generic(
6110 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6111 unsigned NumElemPerLane = 4;
6112 unsigned NumSelectableElems = NumElemPerLane / 2;
6113 unsigned BitsPerElem = 2;
6114 unsigned IndexMask = 0x3;
6115 unsigned MaskBits = 8;
6116 unsigned Lane = DstIdx / NumElemPerLane;
6117 unsigned ElemInLane = DstIdx % NumElemPerLane;
6118 unsigned LaneOffset = Lane * NumElemPerLane;
6119 unsigned SrcIdx = ElemInLane >= NumSelectableElems ? 1 : 0;
6120 unsigned BitIndex = (DstIdx * BitsPerElem) % MaskBits;
6121 unsigned Index = (ShuffleMask >> BitIndex) & IndexMask;
6122 return std::pair<unsigned, int>{SrcIdx,
6123 static_cast<int>(LaneOffset + Index)};
6124 });
6125 case X86::BI__builtin_ia32_shufpd:
6126 case X86::BI__builtin_ia32_shufpd256:
6127 case X86::BI__builtin_ia32_shufpd512:
6128 return interp__builtin_ia32_shuffle_generic(
6129 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6130 unsigned NumElemPerLane = 2;
6131 unsigned NumSelectableElems = NumElemPerLane / 2;
6132 unsigned BitsPerElem = 1;
6133 unsigned IndexMask = 0x1;
6134 unsigned MaskBits = 8;
6135 unsigned Lane = DstIdx / NumElemPerLane;
6136 unsigned ElemInLane = DstIdx % NumElemPerLane;
6137 unsigned LaneOffset = Lane * NumElemPerLane;
6138 unsigned SrcIdx = ElemInLane >= NumSelectableElems ? 1 : 0;
6139 unsigned BitIndex = (DstIdx * BitsPerElem) % MaskBits;
6140 unsigned Index = (ShuffleMask >> BitIndex) & IndexMask;
6141 return std::pair<unsigned, int>{SrcIdx,
6142 static_cast<int>(LaneOffset + Index)};
6143 });
6144
6145 case X86::BI__builtin_ia32_vgf2p8affineinvqb_v16qi:
6146 case X86::BI__builtin_ia32_vgf2p8affineinvqb_v32qi:
6147 case X86::BI__builtin_ia32_vgf2p8affineinvqb_v64qi:
6148 return interp__builtin_ia32_gfni_affine(S, OpPC, Call, Inverse: true);
6149 case X86::BI__builtin_ia32_vgf2p8affineqb_v16qi:
6150 case X86::BI__builtin_ia32_vgf2p8affineqb_v32qi:
6151 case X86::BI__builtin_ia32_vgf2p8affineqb_v64qi:
6152 return interp__builtin_ia32_gfni_affine(S, OpPC, Call, Inverse: false);
6153
6154 case X86::BI__builtin_ia32_vgf2p8mulb_v16qi:
6155 case X86::BI__builtin_ia32_vgf2p8mulb_v32qi:
6156 case X86::BI__builtin_ia32_vgf2p8mulb_v64qi:
6157 return interp__builtin_ia32_gfni_mul(S, OpPC, Call);
6158
6159 case X86::BI__builtin_ia32_bmacor16x16x16_v16hi:
6160 case X86::BI__builtin_ia32_bmacor16x16x16_v32hi:
6161 return interp__builtin_ia32_bmac(S, OpPC, Call, /*IsXor=*/false);
6162 case X86::BI__builtin_ia32_bmacxor16x16x16_v16hi:
6163 case X86::BI__builtin_ia32_bmacxor16x16x16_v32hi:
6164 return interp__builtin_ia32_bmac(S, OpPC, Call, /*IsXor=*/true);
6165
6166 case X86::BI__builtin_ia32_insertps128:
6167 return interp__builtin_ia32_shuffle_generic(
6168 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned Mask) {
6169 // Bits [3:0]: zero mask - if bit is set, zero this element
6170 if ((Mask & (1 << DstIdx)) != 0) {
6171 return std::pair<unsigned, int>{0, -1};
6172 }
6173 // Bits [7:6]: select element from source vector Y (0-3)
6174 // Bits [5:4]: select destination position (0-3)
6175 unsigned SrcElem = (Mask >> 6) & 0x3;
6176 unsigned DstElem = (Mask >> 4) & 0x3;
6177 if (DstIdx == DstElem) {
6178 // Insert element from source vector (B) at this position
6179 return std::pair<unsigned, int>{1, static_cast<int>(SrcElem)};
6180 } else {
6181 // Copy from destination vector (A)
6182 return std::pair<unsigned, int>{0, static_cast<int>(DstIdx)};
6183 }
6184 });
6185 case X86::BI__builtin_ia32_permvarsi256:
6186 case X86::BI__builtin_ia32_permvarsf256:
6187 case X86::BI__builtin_ia32_permvardf512:
6188 case X86::BI__builtin_ia32_permvardi512:
6189 case X86::BI__builtin_ia32_permvarhi128:
6190 return interp__builtin_ia32_shuffle_generic(
6191 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6192 int Offset = ShuffleMask & 0x7;
6193 return std::pair<unsigned, int>{0, Offset};
6194 });
6195 case X86::BI__builtin_ia32_permvarqi128:
6196 case X86::BI__builtin_ia32_permvarhi256:
6197 case X86::BI__builtin_ia32_permvarsi512:
6198 case X86::BI__builtin_ia32_permvarsf512:
6199 return interp__builtin_ia32_shuffle_generic(
6200 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6201 int Offset = ShuffleMask & 0xF;
6202 return std::pair<unsigned, int>{0, Offset};
6203 });
6204 case X86::BI__builtin_ia32_permvardi256:
6205 case X86::BI__builtin_ia32_permvardf256:
6206 return interp__builtin_ia32_shuffle_generic(
6207 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6208 int Offset = ShuffleMask & 0x3;
6209 return std::pair<unsigned, int>{0, Offset};
6210 });
6211 case X86::BI__builtin_ia32_permvarqi256:
6212 case X86::BI__builtin_ia32_permvarhi512:
6213 return interp__builtin_ia32_shuffle_generic(
6214 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6215 int Offset = ShuffleMask & 0x1F;
6216 return std::pair<unsigned, int>{0, Offset};
6217 });
6218 case X86::BI__builtin_ia32_permvarqi512:
6219 return interp__builtin_ia32_shuffle_generic(
6220 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6221 int Offset = ShuffleMask & 0x3F;
6222 return std::pair<unsigned, int>{0, Offset};
6223 });
6224 case X86::BI__builtin_ia32_vpermi2varq128:
6225 case X86::BI__builtin_ia32_vpermi2varpd128:
6226 return interp__builtin_ia32_shuffle_generic(
6227 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6228 int Offset = ShuffleMask & 0x1;
6229 unsigned SrcIdx = (ShuffleMask >> 1) & 0x1;
6230 return std::pair<unsigned, int>{SrcIdx, Offset};
6231 });
6232 case X86::BI__builtin_ia32_vpermi2vard128:
6233 case X86::BI__builtin_ia32_vpermi2varps128:
6234 case X86::BI__builtin_ia32_vpermi2varq256:
6235 case X86::BI__builtin_ia32_vpermi2varpd256:
6236 return interp__builtin_ia32_shuffle_generic(
6237 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6238 int Offset = ShuffleMask & 0x3;
6239 unsigned SrcIdx = (ShuffleMask >> 2) & 0x1;
6240 return std::pair<unsigned, int>{SrcIdx, Offset};
6241 });
6242 case X86::BI__builtin_ia32_vpermi2varhi128:
6243 case X86::BI__builtin_ia32_vpermi2vard256:
6244 case X86::BI__builtin_ia32_vpermi2varps256:
6245 case X86::BI__builtin_ia32_vpermi2varq512:
6246 case X86::BI__builtin_ia32_vpermi2varpd512:
6247 return interp__builtin_ia32_shuffle_generic(
6248 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6249 int Offset = ShuffleMask & 0x7;
6250 unsigned SrcIdx = (ShuffleMask >> 3) & 0x1;
6251 return std::pair<unsigned, int>{SrcIdx, Offset};
6252 });
6253 case X86::BI__builtin_ia32_vpermi2varqi128:
6254 case X86::BI__builtin_ia32_vpermi2varhi256:
6255 case X86::BI__builtin_ia32_vpermi2vard512:
6256 case X86::BI__builtin_ia32_vpermi2varps512:
6257 return interp__builtin_ia32_shuffle_generic(
6258 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6259 int Offset = ShuffleMask & 0xF;
6260 unsigned SrcIdx = (ShuffleMask >> 4) & 0x1;
6261 return std::pair<unsigned, int>{SrcIdx, Offset};
6262 });
6263 case X86::BI__builtin_ia32_vpermi2varqi256:
6264 case X86::BI__builtin_ia32_vpermi2varhi512:
6265 return interp__builtin_ia32_shuffle_generic(
6266 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6267 int Offset = ShuffleMask & 0x1F;
6268 unsigned SrcIdx = (ShuffleMask >> 5) & 0x1;
6269 return std::pair<unsigned, int>{SrcIdx, Offset};
6270 });
6271 case X86::BI__builtin_ia32_vpermi2varqi512:
6272 return interp__builtin_ia32_shuffle_generic(
6273 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6274 int Offset = ShuffleMask & 0x3F;
6275 unsigned SrcIdx = (ShuffleMask >> 6) & 0x1;
6276 return std::pair<unsigned, int>{SrcIdx, Offset};
6277 });
6278 case X86::BI__builtin_ia32_vperm2f128_pd256:
6279 case X86::BI__builtin_ia32_vperm2f128_ps256:
6280 case X86::BI__builtin_ia32_vperm2f128_si256:
6281 case X86::BI__builtin_ia32_permti256: {
6282 unsigned NumElements =
6283 Call->getArg(Arg: 0)->getType()->castAs<VectorType>()->getNumElements();
6284 unsigned PreservedBitsCnt = NumElements >> 2;
6285 return interp__builtin_ia32_shuffle_generic(
6286 S, OpPC, Call,
6287 GetSourceIndex: [PreservedBitsCnt](unsigned DstIdx, unsigned ShuffleMask) {
6288 unsigned ControlBitsCnt = DstIdx >> PreservedBitsCnt << 2;
6289 unsigned ControlBits = ShuffleMask >> ControlBitsCnt;
6290
6291 if (ControlBits & 0b1000)
6292 return std::make_pair(x: 0u, y: -1);
6293
6294 unsigned SrcVecIdx = (ControlBits & 0b10) >> 1;
6295 unsigned PreservedBitsMask = (1 << PreservedBitsCnt) - 1;
6296 int SrcIdx = ((ControlBits & 0b1) << PreservedBitsCnt) |
6297 (DstIdx & PreservedBitsMask);
6298 return std::make_pair(x&: SrcVecIdx, y&: SrcIdx);
6299 });
6300 }
6301 case X86::BI__builtin_ia32_pshufb128:
6302 case X86::BI__builtin_ia32_pshufb256:
6303 case X86::BI__builtin_ia32_pshufb512:
6304 return interp__builtin_ia32_shuffle_generic(
6305 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6306 uint8_t Ctlb = static_cast<uint8_t>(ShuffleMask);
6307 if (Ctlb & 0x80)
6308 return std::make_pair(x: 0, y: -1);
6309
6310 unsigned LaneBase = (DstIdx / 16) * 16;
6311 unsigned SrcOffset = Ctlb & 0x0F;
6312 unsigned SrcIdx = LaneBase + SrcOffset;
6313 return std::make_pair(x: 0, y: static_cast<int>(SrcIdx));
6314 });
6315
6316 case X86::BI__builtin_ia32_pshuflw:
6317 case X86::BI__builtin_ia32_pshuflw256:
6318 case X86::BI__builtin_ia32_pshuflw512:
6319 return interp__builtin_ia32_shuffle_generic(
6320 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6321 unsigned LaneBase = (DstIdx / 8) * 8;
6322 unsigned LaneIdx = DstIdx % 8;
6323 if (LaneIdx < 4) {
6324 unsigned Sel = (ShuffleMask >> (2 * LaneIdx)) & 0x3;
6325 return std::make_pair(x: 0, y: static_cast<int>(LaneBase + Sel));
6326 }
6327
6328 return std::make_pair(x: 0, y: static_cast<int>(DstIdx));
6329 });
6330
6331 case X86::BI__builtin_ia32_pshufhw:
6332 case X86::BI__builtin_ia32_pshufhw256:
6333 case X86::BI__builtin_ia32_pshufhw512:
6334 return interp__builtin_ia32_shuffle_generic(
6335 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6336 unsigned LaneBase = (DstIdx / 8) * 8;
6337 unsigned LaneIdx = DstIdx % 8;
6338 if (LaneIdx >= 4) {
6339 unsigned Sel = (ShuffleMask >> (2 * (LaneIdx - 4))) & 0x3;
6340 return std::make_pair(x: 0, y: static_cast<int>(LaneBase + 4 + Sel));
6341 }
6342
6343 return std::make_pair(x: 0, y: static_cast<int>(DstIdx));
6344 });
6345
6346 case X86::BI__builtin_ia32_pshufd:
6347 case X86::BI__builtin_ia32_pshufd256:
6348 case X86::BI__builtin_ia32_pshufd512:
6349 case X86::BI__builtin_ia32_vpermilps:
6350 case X86::BI__builtin_ia32_vpermilps256:
6351 case X86::BI__builtin_ia32_vpermilps512:
6352 return interp__builtin_ia32_shuffle_generic(
6353 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6354 unsigned LaneBase = (DstIdx / 4) * 4;
6355 unsigned LaneIdx = DstIdx % 4;
6356 unsigned Sel = (ShuffleMask >> (2 * LaneIdx)) & 0x3;
6357 return std::make_pair(x: 0, y: static_cast<int>(LaneBase + Sel));
6358 });
6359
6360 case X86::BI__builtin_ia32_vpermilvarpd:
6361 case X86::BI__builtin_ia32_vpermilvarpd256:
6362 case X86::BI__builtin_ia32_vpermilvarpd512:
6363 return interp__builtin_ia32_shuffle_generic(
6364 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6365 unsigned NumElemPerLane = 2;
6366 unsigned Lane = DstIdx / NumElemPerLane;
6367 unsigned Offset = ShuffleMask & 0b10 ? 1 : 0;
6368 return std::make_pair(
6369 x: 0, y: static_cast<int>(Lane * NumElemPerLane + Offset));
6370 });
6371
6372 case X86::BI__builtin_ia32_vpermilvarps:
6373 case X86::BI__builtin_ia32_vpermilvarps256:
6374 case X86::BI__builtin_ia32_vpermilvarps512:
6375 return interp__builtin_ia32_shuffle_generic(
6376 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6377 unsigned NumElemPerLane = 4;
6378 unsigned Lane = DstIdx / NumElemPerLane;
6379 unsigned Offset = ShuffleMask & 0b11;
6380 return std::make_pair(
6381 x: 0, y: static_cast<int>(Lane * NumElemPerLane + Offset));
6382 });
6383
6384 case X86::BI__builtin_ia32_vpermilpd:
6385 case X86::BI__builtin_ia32_vpermilpd256:
6386 case X86::BI__builtin_ia32_vpermilpd512:
6387 return interp__builtin_ia32_shuffle_generic(
6388 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned Control) {
6389 unsigned NumElemPerLane = 2;
6390 unsigned BitsPerElem = 1;
6391 unsigned MaskBits = 8;
6392 unsigned IndexMask = 0x1;
6393 unsigned Lane = DstIdx / NumElemPerLane;
6394 unsigned LaneOffset = Lane * NumElemPerLane;
6395 unsigned BitIndex = (DstIdx * BitsPerElem) % MaskBits;
6396 unsigned Index = (Control >> BitIndex) & IndexMask;
6397 return std::make_pair(x: 0, y: static_cast<int>(LaneOffset + Index));
6398 });
6399
6400 case X86::BI__builtin_ia32_permdf256:
6401 case X86::BI__builtin_ia32_permdi256:
6402 return interp__builtin_ia32_shuffle_generic(
6403 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned Control) {
6404 // permute4x64 operates on 4 64-bit elements
6405 // For element i (0-3), extract bits [2*i+1:2*i] from Control
6406 unsigned Index = (Control >> (2 * DstIdx)) & 0x3;
6407 return std::make_pair(x: 0, y: static_cast<int>(Index));
6408 });
6409
6410 case X86::BI__builtin_ia32_vpmultishiftqb128:
6411 case X86::BI__builtin_ia32_vpmultishiftqb256:
6412 case X86::BI__builtin_ia32_vpmultishiftqb512:
6413 return interp__builtin_ia32_multishiftqb(S, OpPC, Call);
6414 case X86::BI__builtin_ia32_kandqi:
6415 case X86::BI__builtin_ia32_kandhi:
6416 case X86::BI__builtin_ia32_kandsi:
6417 case X86::BI__builtin_ia32_kanddi:
6418 return interp__builtin_elementwise_int_binop(
6419 S, OpPC, Call,
6420 Fn: [](const APSInt &LHS, const APSInt &RHS) { return LHS & RHS; });
6421
6422 case X86::BI__builtin_ia32_kandnqi:
6423 case X86::BI__builtin_ia32_kandnhi:
6424 case X86::BI__builtin_ia32_kandnsi:
6425 case X86::BI__builtin_ia32_kandndi:
6426 return interp__builtin_elementwise_int_binop(
6427 S, OpPC, Call,
6428 Fn: [](const APSInt &LHS, const APSInt &RHS) { return ~LHS & RHS; });
6429
6430 case X86::BI__builtin_ia32_korqi:
6431 case X86::BI__builtin_ia32_korhi:
6432 case X86::BI__builtin_ia32_korsi:
6433 case X86::BI__builtin_ia32_kordi:
6434 return interp__builtin_elementwise_int_binop(
6435 S, OpPC, Call,
6436 Fn: [](const APSInt &LHS, const APSInt &RHS) { return LHS | RHS; });
6437
6438 case X86::BI__builtin_ia32_kxnorqi:
6439 case X86::BI__builtin_ia32_kxnorhi:
6440 case X86::BI__builtin_ia32_kxnorsi:
6441 case X86::BI__builtin_ia32_kxnordi:
6442 return interp__builtin_elementwise_int_binop(
6443 S, OpPC, Call,
6444 Fn: [](const APSInt &LHS, const APSInt &RHS) { return ~(LHS ^ RHS); });
6445
6446 case X86::BI__builtin_ia32_kxorqi:
6447 case X86::BI__builtin_ia32_kxorhi:
6448 case X86::BI__builtin_ia32_kxorsi:
6449 case X86::BI__builtin_ia32_kxordi:
6450 return interp__builtin_elementwise_int_binop(
6451 S, OpPC, Call,
6452 Fn: [](const APSInt &LHS, const APSInt &RHS) { return LHS ^ RHS; });
6453
6454 case X86::BI__builtin_ia32_knotqi:
6455 case X86::BI__builtin_ia32_knothi:
6456 case X86::BI__builtin_ia32_knotsi:
6457 case X86::BI__builtin_ia32_knotdi:
6458 return interp__builtin_elementwise_int_unaryop(
6459 S, OpPC, Call, Fn: [](const APSInt &Src) { return ~Src; });
6460
6461 case X86::BI__builtin_ia32_kaddqi:
6462 case X86::BI__builtin_ia32_kaddhi:
6463 case X86::BI__builtin_ia32_kaddsi:
6464 case X86::BI__builtin_ia32_kadddi:
6465 return interp__builtin_elementwise_int_binop(
6466 S, OpPC, Call,
6467 Fn: [](const APSInt &LHS, const APSInt &RHS) { return LHS + RHS; });
6468
6469 case X86::BI__builtin_ia32_kmovb:
6470 case X86::BI__builtin_ia32_kmovw:
6471 case X86::BI__builtin_ia32_kmovd:
6472 case X86::BI__builtin_ia32_kmovq:
6473 return interp__builtin_elementwise_int_unaryop(
6474 S, OpPC, Call, Fn: [](const APSInt &Src) { return Src; });
6475
6476 case X86::BI__builtin_ia32_kunpckhi:
6477 case X86::BI__builtin_ia32_kunpckdi:
6478 case X86::BI__builtin_ia32_kunpcksi:
6479 return interp__builtin_elementwise_int_binop(
6480 S, OpPC, Call, Fn: [](const APSInt &A, const APSInt &B) {
6481 // Generic kunpack: extract lower half of each operand and concatenate
6482 // Result = A[HalfWidth-1:0] concat B[HalfWidth-1:0]
6483 unsigned BW = A.getBitWidth();
6484 return APSInt(A.trunc(width: BW / 2).concat(NewLSB: B.trunc(width: BW / 2)),
6485 A.isUnsigned());
6486 });
6487
6488 case X86::BI__builtin_ia32_phminposuw128:
6489 return interp__builtin_ia32_phminposuw(S, OpPC, Call);
6490
6491 case X86::BI__builtin_ia32_psraq128:
6492 case X86::BI__builtin_ia32_psraq256:
6493 case X86::BI__builtin_ia32_psraq512:
6494 case X86::BI__builtin_ia32_psrad128:
6495 case X86::BI__builtin_ia32_psrad256:
6496 case X86::BI__builtin_ia32_psrad512:
6497 case X86::BI__builtin_ia32_psraw128:
6498 case X86::BI__builtin_ia32_psraw256:
6499 case X86::BI__builtin_ia32_psraw512:
6500 return interp__builtin_ia32_shift_with_count(
6501 S, OpPC, Call,
6502 ShiftOp: [](const APInt &Elt, uint64_t Count) { return Elt.ashr(ShiftAmt: Count); },
6503 OverflowOp: [](const APInt &Elt, unsigned Width) { return Elt.ashr(ShiftAmt: Width - 1); });
6504
6505 case X86::BI__builtin_ia32_psllq128:
6506 case X86::BI__builtin_ia32_psllq256:
6507 case X86::BI__builtin_ia32_psllq512:
6508 case X86::BI__builtin_ia32_pslld128:
6509 case X86::BI__builtin_ia32_pslld256:
6510 case X86::BI__builtin_ia32_pslld512:
6511 case X86::BI__builtin_ia32_psllw128:
6512 case X86::BI__builtin_ia32_psllw256:
6513 case X86::BI__builtin_ia32_psllw512:
6514 return interp__builtin_ia32_shift_with_count(
6515 S, OpPC, Call,
6516 ShiftOp: [](const APInt &Elt, uint64_t Count) { return Elt.shl(shiftAmt: Count); },
6517 OverflowOp: [](const APInt &Elt, unsigned Width) { return APInt::getZero(numBits: Width); });
6518
6519 case X86::BI__builtin_ia32_psrlq128:
6520 case X86::BI__builtin_ia32_psrlq256:
6521 case X86::BI__builtin_ia32_psrlq512:
6522 case X86::BI__builtin_ia32_psrld128:
6523 case X86::BI__builtin_ia32_psrld256:
6524 case X86::BI__builtin_ia32_psrld512:
6525 case X86::BI__builtin_ia32_psrlw128:
6526 case X86::BI__builtin_ia32_psrlw256:
6527 case X86::BI__builtin_ia32_psrlw512:
6528 return interp__builtin_ia32_shift_with_count(
6529 S, OpPC, Call,
6530 ShiftOp: [](const APInt &Elt, uint64_t Count) { return Elt.lshr(shiftAmt: Count); },
6531 OverflowOp: [](const APInt &Elt, unsigned Width) { return APInt::getZero(numBits: Width); });
6532
6533 case X86::BI__builtin_ia32_pternlogd128_mask:
6534 case X86::BI__builtin_ia32_pternlogd256_mask:
6535 case X86::BI__builtin_ia32_pternlogd512_mask:
6536 case X86::BI__builtin_ia32_pternlogq128_mask:
6537 case X86::BI__builtin_ia32_pternlogq256_mask:
6538 case X86::BI__builtin_ia32_pternlogq512_mask:
6539 return interp__builtin_ia32_pternlog(S, OpPC, Call, /*MaskZ=*/false);
6540 case X86::BI__builtin_ia32_pternlogd128_maskz:
6541 case X86::BI__builtin_ia32_pternlogd256_maskz:
6542 case X86::BI__builtin_ia32_pternlogd512_maskz:
6543 case X86::BI__builtin_ia32_pternlogq128_maskz:
6544 case X86::BI__builtin_ia32_pternlogq256_maskz:
6545 case X86::BI__builtin_ia32_pternlogq512_maskz:
6546 return interp__builtin_ia32_pternlog(S, OpPC, Call, /*MaskZ=*/true);
6547 case Builtin::BI__builtin_elementwise_fshl:
6548 return interp__builtin_elementwise_triop(S, OpPC, Call,
6549 Fn: llvm::APIntOps::fshl);
6550 case Builtin::BI__builtin_elementwise_fshr:
6551 return interp__builtin_elementwise_triop(S, OpPC, Call,
6552 Fn: llvm::APIntOps::fshr);
6553
6554 case X86::BI__builtin_ia32_shuf_f32x4_256:
6555 case X86::BI__builtin_ia32_shuf_i32x4_256:
6556 case X86::BI__builtin_ia32_shuf_f64x2_256:
6557 case X86::BI__builtin_ia32_shuf_i64x2_256:
6558 case X86::BI__builtin_ia32_shuf_f32x4:
6559 case X86::BI__builtin_ia32_shuf_i32x4:
6560 case X86::BI__builtin_ia32_shuf_f64x2:
6561 case X86::BI__builtin_ia32_shuf_i64x2: {
6562 // Destination and sources A, B all have the same type.
6563 QualType VecQT = Call->getArg(Arg: 0)->getType();
6564 const auto *VecT = VecQT->castAs<VectorType>();
6565 unsigned NumElems = VecT->getNumElements();
6566 unsigned ElemBits = S.getASTContext().getTypeSize(T: VecT->getElementType());
6567 unsigned LaneBits = 128u;
6568 unsigned NumLanes = (NumElems * ElemBits) / LaneBits;
6569 unsigned NumElemsPerLane = LaneBits / ElemBits;
6570
6571 return interp__builtin_ia32_shuffle_generic(
6572 S, OpPC, Call,
6573 GetSourceIndex: [NumLanes, NumElemsPerLane](unsigned DstIdx, unsigned ShuffleMask) {
6574 // DstIdx determines source. ShuffleMask selects lane in source.
6575 unsigned BitsPerElem = NumLanes / 2;
6576 unsigned IndexMask = (1u << BitsPerElem) - 1;
6577 unsigned Lane = DstIdx / NumElemsPerLane;
6578 unsigned SrcIdx = (Lane < NumLanes / 2) ? 0 : 1;
6579 unsigned BitIdx = BitsPerElem * Lane;
6580 unsigned SrcLaneIdx = (ShuffleMask >> BitIdx) & IndexMask;
6581 unsigned ElemInLane = DstIdx % NumElemsPerLane;
6582 unsigned IdxToPick = SrcLaneIdx * NumElemsPerLane + ElemInLane;
6583 return std::pair<unsigned, int>{SrcIdx, IdxToPick};
6584 });
6585 }
6586
6587 case X86::BI__builtin_ia32_insertf32x4_256:
6588 case X86::BI__builtin_ia32_inserti32x4_256:
6589 case X86::BI__builtin_ia32_insertf64x2_256:
6590 case X86::BI__builtin_ia32_inserti64x2_256:
6591 case X86::BI__builtin_ia32_insertf32x4:
6592 case X86::BI__builtin_ia32_inserti32x4:
6593 case X86::BI__builtin_ia32_insertf64x2_512:
6594 case X86::BI__builtin_ia32_inserti64x2_512:
6595 case X86::BI__builtin_ia32_insertf32x8:
6596 case X86::BI__builtin_ia32_inserti32x8:
6597 case X86::BI__builtin_ia32_insertf64x4:
6598 case X86::BI__builtin_ia32_inserti64x4:
6599 case X86::BI__builtin_ia32_vinsertf128_ps256:
6600 case X86::BI__builtin_ia32_vinsertf128_pd256:
6601 case X86::BI__builtin_ia32_vinsertf128_si256:
6602 case X86::BI__builtin_ia32_insert128i256:
6603 return interp__builtin_ia32_insert_subvector(S, OpPC, Call, ID: BuiltinID);
6604
6605 case clang::X86::BI__builtin_ia32_vcvtps2ph:
6606 case clang::X86::BI__builtin_ia32_vcvtps2ph256:
6607 return interp__builtin_ia32_vcvtps2ph(S, OpPC, Call);
6608
6609 case X86::BI__builtin_ia32_vec_ext_v4hi:
6610 case X86::BI__builtin_ia32_vec_ext_v16qi:
6611 case X86::BI__builtin_ia32_vec_ext_v8hi:
6612 case X86::BI__builtin_ia32_vec_ext_v4si:
6613 case X86::BI__builtin_ia32_vec_ext_v2di:
6614 case X86::BI__builtin_ia32_vec_ext_v32qi:
6615 case X86::BI__builtin_ia32_vec_ext_v16hi:
6616 case X86::BI__builtin_ia32_vec_ext_v8si:
6617 case X86::BI__builtin_ia32_vec_ext_v4di:
6618 case X86::BI__builtin_ia32_vec_ext_v4sf:
6619 return interp__builtin_ia32_vec_ext(S, OpPC, Call, ID: BuiltinID);
6620
6621 case X86::BI__builtin_ia32_vec_set_v4hi:
6622 case X86::BI__builtin_ia32_vec_set_v16qi:
6623 case X86::BI__builtin_ia32_vec_set_v8hi:
6624 case X86::BI__builtin_ia32_vec_set_v4si:
6625 case X86::BI__builtin_ia32_vec_set_v2di:
6626 case X86::BI__builtin_ia32_vec_set_v32qi:
6627 case X86::BI__builtin_ia32_vec_set_v16hi:
6628 case X86::BI__builtin_ia32_vec_set_v8si:
6629 case X86::BI__builtin_ia32_vec_set_v4di:
6630 return interp__builtin_ia32_vec_set(S, OpPC, Call, ID: BuiltinID);
6631
6632 case X86::BI__builtin_ia32_cvtb2mask128:
6633 case X86::BI__builtin_ia32_cvtb2mask256:
6634 case X86::BI__builtin_ia32_cvtb2mask512:
6635 case X86::BI__builtin_ia32_cvtw2mask128:
6636 case X86::BI__builtin_ia32_cvtw2mask256:
6637 case X86::BI__builtin_ia32_cvtw2mask512:
6638 case X86::BI__builtin_ia32_cvtd2mask128:
6639 case X86::BI__builtin_ia32_cvtd2mask256:
6640 case X86::BI__builtin_ia32_cvtd2mask512:
6641 case X86::BI__builtin_ia32_cvtq2mask128:
6642 case X86::BI__builtin_ia32_cvtq2mask256:
6643 case X86::BI__builtin_ia32_cvtq2mask512:
6644 return interp__builtin_ia32_cvt_vec2mask(S, OpPC, Call, ID: BuiltinID);
6645
6646 case X86::BI__builtin_ia32_cvtmask2b128:
6647 case X86::BI__builtin_ia32_cvtmask2b256:
6648 case X86::BI__builtin_ia32_cvtmask2b512:
6649 case X86::BI__builtin_ia32_cvtmask2w128:
6650 case X86::BI__builtin_ia32_cvtmask2w256:
6651 case X86::BI__builtin_ia32_cvtmask2w512:
6652 case X86::BI__builtin_ia32_cvtmask2d128:
6653 case X86::BI__builtin_ia32_cvtmask2d256:
6654 case X86::BI__builtin_ia32_cvtmask2d512:
6655 case X86::BI__builtin_ia32_cvtmask2q128:
6656 case X86::BI__builtin_ia32_cvtmask2q256:
6657 case X86::BI__builtin_ia32_cvtmask2q512:
6658 return interp__builtin_ia32_cvt_mask2vec(S, OpPC, Call, ID: BuiltinID);
6659
6660 case X86::BI__builtin_ia32_cvtsd2ss:
6661 return interp__builtin_ia32_cvtsd2ss(S, OpPC, Call, HasRoundingMask: false);
6662
6663 case X86::BI__builtin_ia32_cvtsd2ss_round_mask:
6664 return interp__builtin_ia32_cvtsd2ss(S, OpPC, Call, HasRoundingMask: true);
6665
6666 case X86::BI__builtin_ia32_cvtpd2ps:
6667 case X86::BI__builtin_ia32_cvtpd2ps256:
6668 return interp__builtin_ia32_cvtpd2ps(S, OpPC, Call, IsMasked: false, HasRounding: false);
6669 case X86::BI__builtin_ia32_cvtpd2ps_mask:
6670 return interp__builtin_ia32_cvtpd2ps(S, OpPC, Call, IsMasked: true, HasRounding: false);
6671 case X86::BI__builtin_ia32_cvtpd2ps512_mask:
6672 return interp__builtin_ia32_cvtpd2ps(S, OpPC, Call, IsMasked: true, HasRounding: true);
6673
6674 case X86::BI__builtin_ia32_cmpb128_mask:
6675 case X86::BI__builtin_ia32_cmpw128_mask:
6676 case X86::BI__builtin_ia32_cmpd128_mask:
6677 case X86::BI__builtin_ia32_cmpq128_mask:
6678 case X86::BI__builtin_ia32_cmpb256_mask:
6679 case X86::BI__builtin_ia32_cmpw256_mask:
6680 case X86::BI__builtin_ia32_cmpd256_mask:
6681 case X86::BI__builtin_ia32_cmpq256_mask:
6682 case X86::BI__builtin_ia32_cmpb512_mask:
6683 case X86::BI__builtin_ia32_cmpw512_mask:
6684 case X86::BI__builtin_ia32_cmpd512_mask:
6685 case X86::BI__builtin_ia32_cmpq512_mask:
6686 return interp__builtin_ia32_cmp_mask(S, OpPC, Call, ID: BuiltinID,
6687 /*IsUnsigned=*/false);
6688
6689 case X86::BI__builtin_ia32_ucmpb128_mask:
6690 case X86::BI__builtin_ia32_ucmpw128_mask:
6691 case X86::BI__builtin_ia32_ucmpd128_mask:
6692 case X86::BI__builtin_ia32_ucmpq128_mask:
6693 case X86::BI__builtin_ia32_ucmpb256_mask:
6694 case X86::BI__builtin_ia32_ucmpw256_mask:
6695 case X86::BI__builtin_ia32_ucmpd256_mask:
6696 case X86::BI__builtin_ia32_ucmpq256_mask:
6697 case X86::BI__builtin_ia32_ucmpb512_mask:
6698 case X86::BI__builtin_ia32_ucmpw512_mask:
6699 case X86::BI__builtin_ia32_ucmpd512_mask:
6700 case X86::BI__builtin_ia32_ucmpq512_mask:
6701 return interp__builtin_ia32_cmp_mask(S, OpPC, Call, ID: BuiltinID,
6702 /*IsUnsigned=*/true);
6703
6704 case X86::BI__builtin_ia32_vpshufbitqmb128_mask:
6705 case X86::BI__builtin_ia32_vpshufbitqmb256_mask:
6706 case X86::BI__builtin_ia32_vpshufbitqmb512_mask:
6707 return interp__builtin_ia32_shufbitqmb_mask(S, OpPC, Call);
6708
6709 case X86::BI__builtin_ia32_pslldqi128_byteshift:
6710 case X86::BI__builtin_ia32_pslldqi256_byteshift:
6711 case X86::BI__builtin_ia32_pslldqi512_byteshift:
6712 // These SLLDQ intrinsics always operate on byte elements (8 bits).
6713 // The lane width is hardcoded to 16 to match the SIMD register size,
6714 // but the algorithm processes one byte per iteration,
6715 // so APInt(8, ...) is correct and intentional.
6716 return interp__builtin_ia32_shuffle_generic(
6717 S, OpPC, Call,
6718 GetSourceIndex: [](unsigned DstIdx, unsigned Shift) -> std::pair<unsigned, int> {
6719 unsigned LaneBase = (DstIdx / 16) * 16;
6720 unsigned LaneIdx = DstIdx % 16;
6721 if (LaneIdx < Shift)
6722 return std::make_pair(x: 0, y: -1);
6723
6724 return std::make_pair(x: 0,
6725 y: static_cast<int>(LaneBase + LaneIdx - Shift));
6726 });
6727
6728 case X86::BI__builtin_ia32_psrldqi128_byteshift:
6729 case X86::BI__builtin_ia32_psrldqi256_byteshift:
6730 case X86::BI__builtin_ia32_psrldqi512_byteshift:
6731 // These SRLDQ intrinsics always operate on byte elements (8 bits).
6732 // The lane width is hardcoded to 16 to match the SIMD register size,
6733 // but the algorithm processes one byte per iteration,
6734 // so APInt(8, ...) is correct and intentional.
6735 return interp__builtin_ia32_shuffle_generic(
6736 S, OpPC, Call,
6737 GetSourceIndex: [](unsigned DstIdx, unsigned Shift) -> std::pair<unsigned, int> {
6738 unsigned LaneBase = (DstIdx / 16) * 16;
6739 unsigned LaneIdx = DstIdx % 16;
6740 if (LaneIdx + Shift < 16)
6741 return std::make_pair(x: 0,
6742 y: static_cast<int>(LaneBase + LaneIdx + Shift));
6743
6744 return std::make_pair(x: 0, y: -1);
6745 });
6746
6747 case X86::BI__builtin_ia32_palignr128:
6748 case X86::BI__builtin_ia32_palignr256:
6749 case X86::BI__builtin_ia32_palignr512:
6750 return interp__builtin_ia32_shuffle_generic(
6751 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned Shift) {
6752 // Default to -1 → zero-fill this destination element
6753 unsigned VecIdx = 1;
6754 int ElemIdx = -1;
6755
6756 int Lane = DstIdx / 16;
6757 int Offset = DstIdx % 16;
6758
6759 // Elements come from VecB first, then VecA after the shift boundary
6760 unsigned ShiftedIdx = Offset + (Shift & 0xFF);
6761 if (ShiftedIdx < 16) { // from VecB
6762 ElemIdx = ShiftedIdx + (Lane * 16);
6763 } else if (ShiftedIdx < 32) { // from VecA
6764 VecIdx = 0;
6765 ElemIdx = (ShiftedIdx - 16) + (Lane * 16);
6766 }
6767
6768 return std::pair<unsigned, int>{VecIdx, ElemIdx};
6769 });
6770
6771 case X86::BI__builtin_ia32_alignd128:
6772 case X86::BI__builtin_ia32_alignd256:
6773 case X86::BI__builtin_ia32_alignd512:
6774 case X86::BI__builtin_ia32_alignq128:
6775 case X86::BI__builtin_ia32_alignq256:
6776 case X86::BI__builtin_ia32_alignq512: {
6777 unsigned NumElems = Call->getType()->castAs<VectorType>()->getNumElements();
6778 return interp__builtin_ia32_shuffle_generic(
6779 S, OpPC, Call, GetSourceIndex: [NumElems](unsigned DstIdx, unsigned Shift) {
6780 unsigned Imm = Shift & 0xFF;
6781 unsigned EffectiveShift = Imm & (NumElems - 1);
6782 unsigned SourcePos = DstIdx + EffectiveShift;
6783 unsigned VecIdx = SourcePos < NumElems ? 1u : 0u;
6784 unsigned ElemIdx = SourcePos & (NumElems - 1);
6785 return std::pair<unsigned, int>{VecIdx, static_cast<int>(ElemIdx)};
6786 });
6787 }
6788
6789 case clang::X86::BI__builtin_ia32_minps:
6790 case clang::X86::BI__builtin_ia32_minpd:
6791 case clang::X86::BI__builtin_ia32_minph128:
6792 case clang::X86::BI__builtin_ia32_minph256:
6793 case clang::X86::BI__builtin_ia32_minps256:
6794 case clang::X86::BI__builtin_ia32_minpd256:
6795 case clang::X86::BI__builtin_ia32_minps512:
6796 case clang::X86::BI__builtin_ia32_minpd512:
6797 case clang::X86::BI__builtin_ia32_minph512:
6798 return interp__builtin_elementwise_fp_binop(
6799 S, OpPC, Call,
6800 Fn: [](const APFloat &A, const APFloat &B,
6801 std::optional<APSInt>) -> std::optional<APFloat> {
6802 if (A.isNaN() || A.isInfinity() || A.isDenormal() || B.isNaN() ||
6803 B.isInfinity() || B.isDenormal())
6804 return std::nullopt;
6805 if (A.isZero() && B.isZero())
6806 return B;
6807 return llvm::minimum(A, B);
6808 });
6809
6810 case clang::X86::BI__builtin_ia32_minss:
6811 case clang::X86::BI__builtin_ia32_minsd:
6812 return interp__builtin_elementwise_fp_binop(
6813 S, OpPC, Call,
6814 Fn: [](const APFloat &A, const APFloat &B,
6815 std::optional<APSInt> RoundingMode) -> std::optional<APFloat> {
6816 return EvalScalarMinMaxFp(A, B, RoundingMode, /*IsMin=*/true);
6817 },
6818 /*IsScalar=*/true);
6819
6820 case clang::X86::BI__builtin_ia32_minsd_round_mask:
6821 case clang::X86::BI__builtin_ia32_minss_round_mask:
6822 case clang::X86::BI__builtin_ia32_minsh_round_mask:
6823 case clang::X86::BI__builtin_ia32_maxsd_round_mask:
6824 case clang::X86::BI__builtin_ia32_maxss_round_mask:
6825 case clang::X86::BI__builtin_ia32_maxsh_round_mask: {
6826 bool IsMin = BuiltinID == clang::X86::BI__builtin_ia32_minsd_round_mask ||
6827 BuiltinID == clang::X86::BI__builtin_ia32_minss_round_mask ||
6828 BuiltinID == clang::X86::BI__builtin_ia32_minsh_round_mask;
6829 return interp__builtin_scalar_fp_round_mask_binop(
6830 S, OpPC, Call,
6831 Fn: [IsMin](const APFloat &A, const APFloat &B,
6832 std::optional<APSInt> RoundingMode) -> std::optional<APFloat> {
6833 return EvalScalarMinMaxFp(A, B, RoundingMode, IsMin);
6834 });
6835 }
6836
6837 case clang::X86::BI__builtin_ia32_maxps:
6838 case clang::X86::BI__builtin_ia32_maxpd:
6839 case clang::X86::BI__builtin_ia32_maxph128:
6840 case clang::X86::BI__builtin_ia32_maxph256:
6841 case clang::X86::BI__builtin_ia32_maxps256:
6842 case clang::X86::BI__builtin_ia32_maxpd256:
6843 case clang::X86::BI__builtin_ia32_maxps512:
6844 case clang::X86::BI__builtin_ia32_maxpd512:
6845 case clang::X86::BI__builtin_ia32_maxph512:
6846 return interp__builtin_elementwise_fp_binop(
6847 S, OpPC, Call,
6848 Fn: [](const APFloat &A, const APFloat &B,
6849 std::optional<APSInt>) -> std::optional<APFloat> {
6850 if (A.isNaN() || A.isInfinity() || A.isDenormal() || B.isNaN() ||
6851 B.isInfinity() || B.isDenormal())
6852 return std::nullopt;
6853 if (A.isZero() && B.isZero())
6854 return B;
6855 return llvm::maximum(A, B);
6856 });
6857
6858 case clang::X86::BI__builtin_ia32_maxss:
6859 case clang::X86::BI__builtin_ia32_maxsd:
6860 return interp__builtin_elementwise_fp_binop(
6861 S, OpPC, Call,
6862 Fn: [](const APFloat &A, const APFloat &B,
6863 std::optional<APSInt> RoundingMode) -> std::optional<APFloat> {
6864 return EvalScalarMinMaxFp(A, B, RoundingMode, /*IsMin=*/false);
6865 },
6866 /*IsScalar=*/true);
6867 case X86::BI__builtin_ia32_vpdpwssd128:
6868 case X86::BI__builtin_ia32_vpdpwssd256:
6869 case X86::BI__builtin_ia32_vpdpwssd512:
6870 case X86::BI__builtin_ia32_vpdpbusd128:
6871 case X86::BI__builtin_ia32_vpdpbusd256:
6872 case X86::BI__builtin_ia32_vpdpbusd512:
6873 return interp__builtin_ia32_vpdp(S, OpPC, Call, IsSaturating: false);
6874 case X86::BI__builtin_ia32_vpdpwssds128:
6875 case X86::BI__builtin_ia32_vpdpwssds256:
6876 case X86::BI__builtin_ia32_vpdpwssds512:
6877 case X86::BI__builtin_ia32_vpdpbusds128:
6878 case X86::BI__builtin_ia32_vpdpbusds256:
6879 case X86::BI__builtin_ia32_vpdpbusds512:
6880 return interp__builtin_ia32_vpdp(S, OpPC, Call, IsSaturating: true);
6881 case X86::BI__builtin_ia32_cvtss2si:
6882 case X86::BI__builtin_ia32_cvtsd2si:
6883 case X86::BI__builtin_ia32_cvttss2si:
6884 case X86::BI__builtin_ia32_cvttsd2si:
6885 case X86::BI__builtin_ia32_cvtss2si64:
6886 case X86::BI__builtin_ia32_cvtsd2si64:
6887 case X86::BI__builtin_ia32_cvttss2si64:
6888 case X86::BI__builtin_ia32_cvttsd2si64:
6889 return interp_builtin_ia32_cvt_scalar_to_int(S, OpPC, E: Call);
6890 case X86::BI__builtin_ia32_cvtpd2dq:
6891 case X86::BI__builtin_ia32_cvttpd2dq:
6892 case X86::BI__builtin_ia32_cvtps2dq:
6893 case X86::BI__builtin_ia32_cvtpd2dq256:
6894 case X86::BI__builtin_ia32_cvtps2dq256:
6895 case X86::BI__builtin_ia32_cvttps2dq:
6896 case X86::BI__builtin_ia32_cvttpd2dq256:
6897 case X86::BI__builtin_ia32_cvttps2dq256:
6898 return interp_builtin_ia32_cvt_vector_to_int(S, OpPC, E: Call);
6899 default:
6900 S.FFDiag(Loc: S.Current->getLocation(PC: OpPC),
6901 DiagId: diag::note_invalid_subexpr_in_const_expr)
6902 << S.Current->getRange(PC: OpPC);
6903
6904 return false;
6905 }
6906
6907 llvm_unreachable("Unhandled builtin ID");
6908}
6909
6910bool InterpretOffsetOf(InterpState &S, CodePtr OpPC, const OffsetOfExpr *E,
6911 ArrayRef<int64_t> ArrayIndices, int64_t &IntResult) {
6912 S.getASTContext().recordOffsetOfEvaluation(E);
6913 CharUnits Result;
6914 unsigned N = E->getNumComponents();
6915 assert(N > 0);
6916
6917 unsigned ArrayIndex = 0;
6918 QualType CurrentType = E->getTypeSourceInfo()->getType();
6919 for (unsigned I = 0; I != N; ++I) {
6920 const OffsetOfNode &Node = E->getComponent(Idx: I);
6921 switch (Node.getKind()) {
6922 case OffsetOfNode::Field: {
6923 const FieldDecl *MemberDecl = Node.getField();
6924 const auto *RD = CurrentType->getAsRecordDecl();
6925 if (!RD || RD->isInvalidDecl())
6926 return false;
6927 const ASTRecordLayout &RL = S.getASTContext().getASTRecordLayout(D: RD);
6928 unsigned FieldIndex = MemberDecl->getFieldIndex();
6929 assert(FieldIndex < RL.getFieldCount() && "offsetof field in wrong type");
6930 Result +=
6931 S.getASTContext().toCharUnitsFromBits(BitSize: RL.getFieldOffset(FieldNo: FieldIndex));
6932 CurrentType = MemberDecl->getType().getNonReferenceType();
6933 break;
6934 }
6935 case OffsetOfNode::Array: {
6936 // When generating bytecode, we put all the index expressions as Sint64 on
6937 // the stack.
6938 int64_t Index = ArrayIndices[ArrayIndex];
6939 if (Index < 0)
6940 return Invalid(S, OpPC);
6941 const ArrayType *AT = S.getASTContext().getAsArrayType(T: CurrentType);
6942 if (!AT)
6943 return false;
6944 CurrentType = AT->getElementType();
6945 CharUnits ElementSize = S.getASTContext().getTypeSizeInChars(T: CurrentType);
6946 int64_t ElemSize = ElementSize.getQuantity();
6947 if (Index != 0 && ElemSize > (llvm::maxIntN(N: 64) / Index)) {
6948 S.FFDiag(Loc: S.Current->getLocation(PC: OpPC),
6949 DiagId: diag::note_constexpr_offsetof_overflow)
6950 << S.Current->getRange(PC: OpPC);
6951 return false;
6952 }
6953 int64_t Offset = Index * ElemSize;
6954 if (Result.getQuantity() > llvm::maxIntN(N: 64) - Offset) {
6955 S.FFDiag(Loc: S.Current->getLocation(PC: OpPC),
6956 DiagId: diag::note_constexpr_offsetof_overflow)
6957 << S.Current->getRange(PC: OpPC);
6958 return false;
6959 }
6960 Result += CharUnits::fromQuantity(Quantity: Offset);
6961 ++ArrayIndex;
6962 break;
6963 }
6964 case OffsetOfNode::Base: {
6965 const CXXBaseSpecifier *BaseSpec = Node.getBase();
6966 if (BaseSpec->isVirtual())
6967 return false;
6968
6969 // Find the layout of the class whose base we are looking into.
6970 const auto *RD = CurrentType->getAsCXXRecordDecl();
6971 if (!RD || RD->isInvalidDecl())
6972 return false;
6973 const ASTRecordLayout &RL = S.getASTContext().getASTRecordLayout(D: RD);
6974
6975 // Find the base class itself.
6976 CurrentType = BaseSpec->getType();
6977 const auto *BaseRD = CurrentType->getAsCXXRecordDecl();
6978 if (!BaseRD)
6979 return false;
6980
6981 // Add the offset to the base.
6982 Result += RL.getBaseClassOffset(Base: BaseRD);
6983 break;
6984 }
6985 case OffsetOfNode::Identifier:
6986 llvm_unreachable("Dependent OffsetOfExpr?");
6987 }
6988 }
6989
6990 IntResult = Result.getQuantity();
6991
6992 return true;
6993}
6994
6995bool SetThreeWayComparisonField(InterpState &S, CodePtr OpPC,
6996 const Pointer &Ptr, const APSInt &IntValue) {
6997
6998 const Record *R = Ptr.getRecord();
6999 assert(R);
7000 assert(R->getNumFields() == 1);
7001
7002 unsigned FieldOffset = R->getField(I: 0u)->Offset;
7003 PtrView FieldPtr = Ptr.view().atField(Offset: FieldOffset);
7004 PrimType FieldT = FieldPtr.getFieldDesc()->getPrimType();
7005
7006 INT_TYPE_SWITCH(FieldT,
7007 FieldPtr.deref<T>() = T::from(IntValue.getSExtValue()));
7008 FieldPtr.initialize();
7009 return true;
7010}
7011
7012static void zeroAll(PtrView Dest) {
7013 const Descriptor *Desc = Dest.getFieldDesc();
7014
7015 if (Desc->isPrimitive()) {
7016 TYPE_SWITCH(Desc->getPrimType(), {
7017 Dest.deref<T>().~T();
7018 new (&Dest.deref<T>()) T();
7019 });
7020 return;
7021 }
7022
7023 if (Desc->isRecord()) {
7024 const Record *R = Desc->ElemRecord;
7025 for (const Record::Field &F : R->fields()) {
7026 PtrView FieldPtr = Dest.atField(Offset: F.Offset);
7027 zeroAll(Dest: FieldPtr);
7028 }
7029 return;
7030 }
7031
7032 if (Desc->isPrimitiveArray()) {
7033 for (unsigned I = 0, N = Desc->getNumElems(); I != N; ++I) {
7034 TYPE_SWITCH(Desc->getPrimType(), {
7035 Dest.deref<T>().~T();
7036 new (&Dest.deref<T>()) T();
7037 });
7038 }
7039 return;
7040 }
7041
7042 if (Desc->isCompositeArray()) {
7043 for (unsigned I = 0, N = Desc->getNumElems(); I != N; ++I) {
7044 PtrView ElemPtr = Dest.atIndex(Idx: I).narrow();
7045 zeroAll(Dest: ElemPtr);
7046 }
7047 return;
7048 }
7049}
7050
7051static bool copyComposite(InterpState &S, CodePtr OpPC, PtrView Src,
7052 PtrView Dest, bool Activate);
7053static bool copyRecord(InterpState &S, CodePtr OpPC, PtrView Src, PtrView Dest,
7054 bool Activate = false) {
7055 [[maybe_unused]] const Descriptor *SrcDesc = Src.getFieldDesc();
7056 const Descriptor *DestDesc = Dest.getFieldDesc();
7057
7058 auto copyField = [&](const Record::Field &F, bool Activate) -> bool {
7059 PtrView DestField = Dest.atField(Offset: F.Offset);
7060 if (OptPrimType FT = S.Ctx.classify(T: F.Decl->getType())) {
7061 TYPE_SWITCH(*FT, {
7062 DestField.deref<T>() = Src.atField(F.Offset).deref<T>();
7063 if (Src.atField(F.Offset).isInitialized())
7064 DestField.initialize();
7065 if (Activate)
7066 DestField.activate();
7067 });
7068 return true;
7069 }
7070 // Composite field.
7071 return copyComposite(S, OpPC, Src: Src.atField(Offset: F.Offset), Dest: DestField, Activate);
7072 };
7073
7074 assert(SrcDesc->isRecord());
7075 assert(SrcDesc->ElemRecord == DestDesc->ElemRecord);
7076 const Record *R = DestDesc->ElemRecord;
7077 for (const Record::Field &F : R->fields()) {
7078 PtrView FP = Src.atField(Offset: F.Offset);
7079
7080 if (!CheckMutable(S, OpPC, Ptr: FP))
7081 return false;
7082
7083 if (R->isUnion()) {
7084 // For unions, only copy the active field. Zero all others.
7085 if (FP.isActive()) {
7086 if (!copyField(F, /*Activate=*/true))
7087 return false;
7088 } else {
7089 PtrView DestField = Dest.atField(Offset: F.Offset);
7090 zeroAll(Dest: DestField);
7091 }
7092 } else {
7093 if (!copyField(F, Activate))
7094 return false;
7095 }
7096 }
7097
7098 for (const Record::Base &B : R->bases()) {
7099 PtrView DestBase = Dest.atField(Offset: B.Offset);
7100 if (!copyRecord(S, OpPC, Src: Src.atField(Offset: B.Offset), Dest: DestBase, Activate))
7101 return false;
7102 }
7103
7104 Dest.initialize();
7105 return true;
7106}
7107
7108static bool copyComposite(InterpState &S, CodePtr OpPC, PtrView Src,
7109 PtrView Dest, bool Activate = false) {
7110 assert(Src.isLive() && Dest.isLive());
7111
7112 [[maybe_unused]] const Descriptor *SrcDesc = Src.getFieldDesc();
7113 const Descriptor *DestDesc = Dest.getFieldDesc();
7114
7115 assert(!DestDesc->isPrimitive() && !SrcDesc->isPrimitive());
7116
7117 if (DestDesc->isPrimitiveArray()) {
7118 if (!SrcDesc->isPrimitiveArray())
7119 return false;
7120 // For floating types, check the actual QualType so we don't accidentally
7121 // mix up semantics.
7122 if (SrcDesc->getPrimType() == PT_Float) {
7123 if (!S.getASTContext().hasSimilarType(T1: SrcDesc->getElemQualType(),
7124 T2: DestDesc->getElemQualType()))
7125 return false;
7126 }
7127
7128 assert(SrcDesc->isPrimitiveArray());
7129 assert(SrcDesc->getNumElems() == DestDesc->getNumElems());
7130 assert(SrcDesc->getPrimType() == DestDesc->getPrimType());
7131 PrimType ET = DestDesc->getPrimType();
7132 for (unsigned I = 0, N = DestDesc->getNumElems(); I != N; ++I) {
7133 PtrView DestElem = Dest.atIndex(Idx: I);
7134 TYPE_SWITCH(ET, { DestElem.deref<T>() = Src.elem<T>(I); });
7135 DestElem.initializeElement(Index: I);
7136 }
7137 return true;
7138 }
7139
7140 if (DestDesc->isCompositeArray()) {
7141 if (!SrcDesc->isCompositeArray())
7142 return false;
7143 assert(SrcDesc->isCompositeArray());
7144 assert(SrcDesc->getNumElems() == DestDesc->getNumElems());
7145 for (unsigned I = 0, N = DestDesc->getNumElems(); I != N; ++I) {
7146 PtrView SrcElem = Src.atIndex(Idx: I).narrow();
7147 PtrView DestElem = Dest.atIndex(Idx: I).narrow();
7148 if (!copyComposite(S, OpPC, Src: SrcElem, Dest: DestElem, Activate))
7149 return false;
7150 }
7151 return true;
7152 }
7153
7154 if (DestDesc->isRecord()) {
7155 if (!SrcDesc->isRecord())
7156 return false;
7157 return copyRecord(S, OpPC, Src, Dest, Activate);
7158 }
7159 return Invalid(S, OpPC);
7160}
7161
7162bool DoMemcpy(InterpState &S, CodePtr OpPC, const Pointer &Src, Pointer &Dest) {
7163 if (!Src.isBlockPointer() || Src.getFieldDesc()->isPrimitive())
7164 return false;
7165 if (!Dest.isBlockPointer() || Dest.getFieldDesc()->isPrimitive())
7166 return false;
7167
7168 return copyComposite(S, OpPC, Src: Src.view(), Dest: Dest.view());
7169}
7170
7171} // namespace interp
7172} // namespace clang
7173