1//===--- InterpBuiltin.cpp - Interpreter for the constexpr VM ---*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8#include "../ExprConstShared.h"
9#include "Boolean.h"
10#include "Char.h"
11#include "EvalEmitter.h"
12#include "InterpBuiltinBitCast.h"
13#include "InterpHelpers.h"
14#include "PrimType.h"
15#include "Program.h"
16#include "clang/AST/InferAlloc.h"
17#include "clang/AST/OSLog.h"
18#include "clang/AST/RecordLayout.h"
19#include "clang/Basic/Builtins.h"
20#include "clang/Basic/TargetBuiltins.h"
21#include "clang/Basic/TargetInfo.h"
22#include "llvm/ADT/StringExtras.h"
23#include "llvm/Support/AllocToken.h"
24#include "llvm/Support/ErrorHandling.h"
25#include "llvm/Support/SipHash.h"
26
27namespace clang {
28namespace interp {
29
30[[maybe_unused]] static bool isNoopBuiltin(unsigned ID) {
31 switch (ID) {
32 case Builtin::BIas_const:
33 case Builtin::BIforward:
34 case Builtin::BIforward_like:
35 case Builtin::BImove:
36 case Builtin::BImove_if_noexcept:
37 case Builtin::BIaddressof:
38 case Builtin::BI__addressof:
39 case Builtin::BI__builtin_addressof:
40 case Builtin::BI__builtin_launder:
41 return true;
42 default:
43 return false;
44 }
45 return false;
46}
47
48static void discard(InterpStack &Stk, PrimType T) {
49 TYPE_SWITCH(T, { Stk.discard<T>(); });
50}
51
52static bool popToUInt64(const InterpState &S, const Expr *E, uint64_t &Out) {
53 INT_TYPE_SWITCH(*S.getContext().classify(E->getType()), {
54 const auto &Val = S.Stk.pop<T>();
55 if (!Val.isNumber())
56 return false;
57 Out = static_cast<uint64_t>(Val);
58 return true;
59 });
60}
61
62static bool popToAPSInt(InterpStack &Stk, PrimType T, APSInt &Out) {
63 INT_TYPE_SWITCH(T, {
64 const auto &Val = Stk.pop<T>();
65 if (!Val.isNumber())
66 return false;
67 Out = Val.toAPSInt();
68 return true;
69 });
70}
71
72static bool popToAPSInt(InterpState &S, const Expr *E, APSInt &Out) {
73 return popToAPSInt(Stk&: S.Stk, T: *S.getContext().classify(T: E->getType()), Out);
74}
75static bool popToAPSInt(InterpState &S, QualType T, APSInt &Out) {
76 return popToAPSInt(Stk&: S.Stk, T: *S.getContext().classify(T), Out);
77}
78
79/// Check for common reasons a pointer can't be read from, which
80/// are usually not diagnosed in a builtin function.
81static bool isReadable(const Pointer &P) {
82 if (P.isDummy())
83 return false;
84 if (!P.isReadablePointerType())
85 return false;
86 if (!P.isLive())
87 return false;
88 if (P.isOnePastEnd())
89 return false;
90 return true;
91}
92
93/// Pushes \p Val on the stack as the type given by \p QT.
94static void pushInteger(InterpState &S, const APSInt &Val, QualType QT) {
95 assert(QT->isSignedIntegerOrEnumerationType() ||
96 QT->isUnsignedIntegerOrEnumerationType());
97 OptPrimType T = *S.getContext().classify(T: QT);
98 assert(T);
99
100 if (T == PT_IntAPS) {
101 unsigned BitWidth = S.getASTContext().getIntWidth(T: QT);
102 auto Result = S.allocAP<IntegralAP<true>>(BitWidth);
103 Result.copy(V: Val.extOrTrunc(width: BitWidth));
104 S.Stk.push<IntegralAP<true>>(Args&: Result);
105 return;
106 }
107
108 if (T == PT_IntAP) {
109 unsigned BitWidth = S.getASTContext().getIntWidth(T: QT);
110 auto Result = S.allocAP<IntegralAP<false>>(BitWidth);
111 Result.copy(V: Val.extOrTrunc(width: BitWidth));
112 S.Stk.push<IntegralAP<false>>(Args&: Result);
113 return;
114 }
115
116 if (isSignedType(T: *T)) {
117 int64_t V = Val.getSExtValue();
118 INT_TYPE_SWITCH(*T, { S.Stk.push<T>(T::from(V)); });
119 } else {
120 assert(QT->isUnsignedIntegerOrEnumerationType());
121 uint64_t V = Val.getZExtValue();
122 INT_TYPE_SWITCH(*T, { S.Stk.push<T>(T::from(V)); });
123 }
124}
125
126template <typename T>
127static void pushInteger(InterpState &S, T Val, QualType QT) {
128 if constexpr (std::is_same_v<T, APInt>)
129 pushInteger(S, Val: APSInt(Val, !std::is_signed_v<T>), QT);
130 else if constexpr (std::is_same_v<T, APSInt>)
131 pushInteger(S, Val, QT);
132 else
133 pushInteger(S,
134 Val: APSInt(APInt(sizeof(T) * 8, static_cast<uint64_t>(Val),
135 std::is_signed_v<T>),
136 !std::is_signed_v<T>),
137 QT);
138}
139
140static void assignIntegral(InterpState &S, const Pointer &Dest, PrimType ValueT,
141 const APSInt &Value) {
142
143 if (ValueT == PT_IntAPS) {
144 Dest.deref<IntegralAP<true>>() =
145 S.allocAP<IntegralAP<true>>(BitWidth: Value.getBitWidth());
146 Dest.deref<IntegralAP<true>>().copy(V: Value);
147 } else if (ValueT == PT_IntAP) {
148 Dest.deref<IntegralAP<false>>() =
149 S.allocAP<IntegralAP<false>>(BitWidth: Value.getBitWidth());
150 Dest.deref<IntegralAP<false>>().copy(V: Value);
151 } else if (ValueT == PT_Bool) {
152 Dest.deref<Boolean>() = Boolean::from(Value: !Value.isZero());
153 } else {
154 INT_TYPE_SWITCH_NO_BOOL(
155 ValueT, { Dest.deref<T>() = T::from(static_cast<T>(Value)); });
156 }
157}
158
159static QualType getElemType(const Pointer &P) {
160 if (P.isStringPointer()) {
161 return P.asStringPointer()
162 .getLiteral()
163 ->getType()
164 ->getAsArrayTypeUnsafe()
165 ->getElementType();
166 }
167
168 if (P.isOpaquePointer() || P.isIntegralPointer())
169 return P.getType();
170
171 const Descriptor *Desc = P.getFieldDesc();
172 QualType T = Desc->getType();
173 if (Desc->isPrimitive())
174 return T;
175 if (T->isPointerType())
176 return T->castAs<PointerType>()->getPointeeType();
177 if (Desc->isArray())
178 return Desc->getElemQualType();
179 if (const auto *AT = T->getAsArrayTypeUnsafe())
180 return AT->getElementType();
181 return T;
182}
183
184static void diagnoseNonConstexprBuiltin(InterpState &S, CodePtr OpPC,
185 unsigned ID) {
186 if (!S.diagnosing())
187 return;
188
189 auto Loc = S.Current->getSource(PC: OpPC);
190 if (S.getLangOpts().CPlusPlus11)
191 S.CCEDiag(SI: Loc, DiagId: diag::note_constexpr_invalid_function)
192 << /*isConstexpr=*/0 << /*isConstructor=*/0
193 << S.getASTContext().BuiltinInfo.getQuotedName(ID);
194 else
195 S.CCEDiag(SI: Loc, DiagId: diag::note_invalid_subexpr_in_const_expr);
196}
197
198static llvm::APSInt convertBoolVectorToInt(const Pointer &Val) {
199 assert(Val.getFieldDesc()->isPrimitiveArray() &&
200 Val.getFieldDesc()->getElemQualType()->isBooleanType() &&
201 "Not a boolean vector");
202 unsigned NumElems = Val.getNumElems();
203
204 // Each element is one bit, so create an integer with NumElts bits.
205 llvm::APSInt Result(NumElems, 0);
206 for (unsigned I = 0; I != NumElems; ++I) {
207 if (Val.elem<bool>(I))
208 Result.setBit(I);
209 }
210
211 return Result;
212}
213
214// Strict double -> float conversion used for X86 PD2PS/cvtsd2ss intrinsics.
215// Reject NaN/Inf/Subnormal inputs and any lossy/inexact conversions.
216static bool convertDoubleToFloatStrict(const APFloat &Src, Floating &Dst,
217 InterpState &S, const Expr *DiagExpr) {
218 if (Src.isInfinity()) {
219 if (S.diagnosing())
220 S.CCEDiag(E: DiagExpr, DiagId: diag::note_constexpr_float_arithmetic) << 0;
221 return false;
222 }
223 if (Src.isNaN()) {
224 if (S.diagnosing())
225 S.CCEDiag(E: DiagExpr, DiagId: diag::note_constexpr_float_arithmetic) << 1;
226 return false;
227 }
228 APFloat Val = Src;
229 bool LosesInfo = false;
230 APFloat::opStatus Status = Val.convert(
231 ToSemantics: APFloat::IEEEsingle(), RM: APFloat::rmNearestTiesToEven, losesInfo: &LosesInfo);
232 if (LosesInfo || Val.isDenormal()) {
233 if (S.diagnosing())
234 S.CCEDiag(E: DiagExpr, DiagId: diag::note_constexpr_float_arithmetic_strict);
235 return false;
236 }
237 if (Status != APFloat::opOK) {
238 if (S.diagnosing())
239 S.CCEDiag(E: DiagExpr, DiagId: diag::note_invalid_subexpr_in_const_expr);
240 return false;
241 }
242 Dst.copy(F: Val);
243 return true;
244}
245
246static bool interp__builtin_is_constant_evaluated(InterpState &S, CodePtr OpPC,
247 const InterpFrame *Frame,
248 const CallExpr *Call) {
249 unsigned Depth = S.Current->getDepth();
250 auto isStdCall = [](const FunctionDecl *F) -> bool {
251 return F && F->isInStdNamespace() && F->getIdentifier() &&
252 F->getIdentifier()->isStr(Str: "is_constant_evaluated");
253 };
254 const InterpFrame *Caller = Frame->Caller;
255 // The current frame is the one for __builtin_is_constant_evaluated.
256 // The one above that, potentially the one for std::is_constant_evaluated().
257 if (S.inConstantContext() && !S.checkingPotentialConstantExpression() &&
258 S.getEvalStatus().Diag &&
259 (Depth == 0 || (Depth == 1 && isStdCall(Frame->getCallee())))) {
260 if (Caller && isStdCall(Frame->getCallee())) {
261 const Expr *E = Caller->getExpr(PC: Caller->getRetPC());
262 S.report(Loc: E->getExprLoc(),
263 DiagId: diag::warn_is_constant_evaluated_always_true_constexpr)
264 << "std::is_constant_evaluated" << E->getSourceRange();
265 } else {
266 S.report(Loc: Call->getExprLoc(),
267 DiagId: diag::warn_is_constant_evaluated_always_true_constexpr)
268 << "__builtin_is_constant_evaluated" << Call->getSourceRange();
269 }
270 }
271
272 S.Stk.push<Boolean>(Args: Boolean::from(Value: S.inConstantContext()));
273 return true;
274}
275
276// __builtin_assume
277// __assume (MS extension)
278static bool interp__builtin_assume(InterpState &S, CodePtr OpPC,
279 const InterpFrame *Frame,
280 const CallExpr *Call) {
281 // Nothing to be done here since the argument is NOT evaluated.
282 assert(Call->getNumArgs() == 1);
283 return true;
284}
285
286static bool interp__builtin_strcmp(InterpState &S, CodePtr OpPC,
287 const InterpFrame *Frame,
288 const CallExpr *Call, unsigned ID) {
289 uint64_t Limit = ~static_cast<uint64_t>(0);
290 if (ID == Builtin::BIstrncmp || ID == Builtin::BI__builtin_strncmp ||
291 ID == Builtin::BIwcsncmp || ID == Builtin::BI__builtin_wcsncmp) {
292 if (!popToUInt64(S, E: Call->getArg(Arg: 2), Out&: Limit))
293 return false;
294 }
295
296 const Pointer &B = S.Stk.pop<Pointer>();
297 const Pointer &A = S.Stk.pop<Pointer>();
298 if (ID == Builtin::BIstrcmp || ID == Builtin::BIstrncmp ||
299 ID == Builtin::BIwcscmp || ID == Builtin::BIwcsncmp)
300 diagnoseNonConstexprBuiltin(S, OpPC, ID);
301
302 if (Limit == 0) {
303 pushInteger(S, Val: 0, QT: Call->getType());
304 return true;
305 }
306
307 if (!CheckLive(S, OpPC, Ptr: A, AK: AK_Read) || !CheckLive(S, OpPC, Ptr: B, AK: AK_Read))
308 return false;
309
310 if (!A.isReadablePointerType() || !B.isReadablePointerType())
311 return false;
312
313 if (A.isDummy() || B.isDummy() || A.isUnknownSizeArray() ||
314 B.isUnknownSizeArray())
315 return false;
316
317 bool IsWide = ID == Builtin::BIwcscmp || ID == Builtin::BIwcsncmp ||
318 ID == Builtin::BI__builtin_wcscmp ||
319 ID == Builtin::BI__builtin_wcsncmp;
320
321 QualType ElemTy = getElemType(P: A);
322 // Different element types shouldn't happen, but with casts they can.
323 if (!S.getASTContext().hasSameUnqualifiedType(T1: ElemTy, T2: getElemType(P: B)))
324 return false;
325
326 PrimType ElemT = *S.getContext().classify(T: ElemTy);
327
328 auto returnResult = [&](int V) -> bool {
329 pushInteger(S, Val: V, QT: Call->getType());
330 return true;
331 };
332
333 unsigned IndexA = A.getIndex();
334 unsigned IndexB = B.getIndex();
335 unsigned NumElemsA = A.getNumElems();
336 unsigned NumElemsB = B.getNumElems();
337 uint64_t Steps = 0;
338 for (;; ++IndexA, ++IndexB, ++Steps) {
339
340 if (Steps >= Limit)
341 break;
342
343 // Diagnose this as a read of one-past-the-end.
344 if (IndexA >= NumElemsA || IndexB >= NumElemsB) {
345 S.FFDiag(SI: S.Current->getSource(PC: OpPC), DiagId: diag::note_constexpr_access_past_end)
346 << AK_Read << S.Current->getRange(PC: OpPC);
347 return false;
348 }
349
350 if (IsWide) {
351 INT_TYPE_SWITCH(ElemT, {
352 T CA = A.loadElem<T>(IndexA);
353 T CB = B.loadElem<T>(IndexB);
354 if (CA > CB)
355 return returnResult(1);
356 if (CA < CB)
357 return returnResult(-1);
358 if (CA.isZero() || CB.isZero())
359 return returnResult(0);
360 });
361 continue;
362 }
363
364 uint8_t CA = A.loadElem<uint8_t>(I: IndexA);
365 uint8_t CB = B.loadElem<uint8_t>(I: IndexB);
366
367 if (CA > CB)
368 return returnResult(1);
369 if (CA < CB)
370 return returnResult(-1);
371 if (CA == 0 || CB == 0)
372 return returnResult(0);
373 }
374
375 return returnResult(0);
376}
377
378static bool interp__builtin_strlen(InterpState &S, CodePtr OpPC,
379 const InterpFrame *Frame,
380 const CallExpr *Call, unsigned ID) {
381 const Pointer &StrPtr = S.Stk.pop<Pointer>().expand();
382
383 if (ID == Builtin::BIstrlen || ID == Builtin::BIwcslen)
384 diagnoseNonConstexprBuiltin(S, OpPC, ID);
385
386 if (StrPtr.isConstexprUnknown())
387 return false;
388
389 if (!CheckArray(S, OpPC, Ptr: StrPtr))
390 return false;
391
392 if (!CheckLive(S, OpPC, Ptr: StrPtr, AK: AK_Read))
393 return false;
394
395 // For string literal pointers, this is pretty simple.
396 if (StrPtr.isStringPointer()) {
397 if (StrPtr.isOnePastEnd())
398 return CheckRange(S, OpPC, Ptr: StrPtr, AK: AK_Read);
399
400 const auto *Lit = StrPtr.asStringPointer().getLiteral();
401 int64_t Off = StrPtr.getByteOffset();
402 if (Off < 0)
403 return false;
404
405 UnsignedOrNone ZeroIndex = Lit->findZeroCodeUnit(StartIndex: Off);
406 if (!ZeroIndex)
407 return false;
408 pushInteger(S, Val: *ZeroIndex, QT: Call->getType());
409 return true;
410 }
411
412 if (!StrPtr.isBlockPointer())
413 return false;
414
415 if (!CheckDummy(S, OpPC, Ptr: StrPtr, AK: AK_Read))
416 return false;
417
418 if (!StrPtr.getFieldDesc()->isPrimitiveArray())
419 return false;
420
421 assert(StrPtr.getFieldDesc()->isPrimitiveArray());
422 PrimType ElemT = StrPtr.getFieldDesc()->getPrimType();
423 unsigned ElemSize = StrPtr.getFieldDesc()->getElemDataSize();
424 if (ElemSize != 1 && ElemSize != 2 && ElemSize != 4)
425 return Invalid(S, OpPC);
426
427 if (ID == Builtin::BI__builtin_wcslen || ID == Builtin::BIwcslen) {
428 const ASTContext &AC = S.getASTContext();
429 unsigned WCharSize = AC.getTypeSizeInChars(T: AC.getWCharType()).getQuantity();
430 if (StrPtr.getFieldDesc()->getElemDataSize() != WCharSize)
431 return false;
432 }
433
434 size_t Len = 0;
435 for (size_t I = StrPtr.getIndex();; ++I, ++Len) {
436 PtrView ElemPtr = StrPtr.view().atIndex(Idx: I);
437
438 if (!CheckRange(S, OpPC, Ptr: ElemPtr, AK: AK_Read))
439 return false;
440
441 uint32_t Val;
442 FIXED_SIZE_INT_TYPE_SWITCH(
443 ElemT, { Val = static_cast<uint32_t>(ElemPtr.deref<T>()); });
444 if (Val == 0)
445 break;
446 }
447
448 pushInteger(S, Val: Len, QT: Call->getType());
449
450 return true;
451}
452
453static bool interp__builtin_nan(InterpState &S, CodePtr OpPC,
454 const InterpFrame *Frame, const CallExpr *Call,
455 bool Signaling) {
456 const Pointer &Arg = S.Stk.pop<Pointer>();
457
458 if (!CheckLoad(S, OpPC, Ptr: Arg))
459 return false;
460
461 // Convert the given string to an integer using StringRef's API.
462 llvm::APInt Fill;
463 if (Arg.isBlockPointer()) {
464 if (!Arg.getFieldDesc()->isPrimitiveArray())
465 return Invalid(S, OpPC);
466
467 std::string Str;
468 unsigned ArgLength = Arg.getNumElems();
469 bool FoundZero = false;
470 for (unsigned I = 0; I != ArgLength; ++I) {
471 if (!Arg.isElementInitialized(Index: I))
472 return false;
473
474 if (Arg.loadElem<int8_t>(I) == 0) {
475 FoundZero = true;
476 break;
477 }
478 Str += Arg.elem<char>(I);
479 }
480
481 // If we didn't find a NUL byte, diagnose as a one-past-the-end read.
482 if (!FoundZero)
483 return CheckRange(S, OpPC, Ptr: Arg.atIndex(Idx: ArgLength), AK: AK_Read);
484
485 // Treat empty strings as if they were zero.
486 if (Str.empty())
487 Fill = llvm::APInt(32, 0);
488 else if (StringRef(Str).getAsInteger(Radix: 0, Result&: Fill))
489 return false;
490 } else if (Arg.isStringPointer()) {
491 if (!Arg.asStringPointer().getLiteral()->isOrdinary())
492 return false;
493 StringRef Str = Arg.asStringPointer().getLiteral()->getString();
494 // Treat empty strings as if they were zero.
495 if (Str.empty())
496 Fill = llvm::APInt(32, 0);
497 else if (StringRef(Str).getAsInteger(Radix: 0, Result&: Fill))
498 return false;
499 } else {
500 return false;
501 }
502
503 const llvm::fltSemantics &TargetSemantics =
504 S.getASTContext().getFloatTypeSemantics(
505 T: Call->getDirectCallee()->getReturnType());
506
507 Floating Result = S.allocFloat(Sem: TargetSemantics);
508 if (S.getASTContext().getTargetInfo().isNan2008()) {
509 if (Signaling)
510 Result.copy(
511 F: llvm::APFloat::getSNaN(Sem: TargetSemantics, /*Negative=*/false, payload: &Fill));
512 else
513 Result.copy(
514 F: llvm::APFloat::getQNaN(Sem: TargetSemantics, /*Negative=*/false, payload: &Fill));
515 } else {
516 // Prior to IEEE 754-2008, architectures were allowed to choose whether
517 // the first bit of their significand was set for qNaN or sNaN. MIPS chose
518 // a different encoding to what became a standard in 2008, and for pre-
519 // 2008 revisions, MIPS interpreted sNaN-2008 as qNan and qNaN-2008 as
520 // sNaN. This is now known as "legacy NaN" encoding.
521 if (Signaling)
522 Result.copy(
523 F: llvm::APFloat::getQNaN(Sem: TargetSemantics, /*Negative=*/false, payload: &Fill));
524 else
525 Result.copy(
526 F: llvm::APFloat::getSNaN(Sem: TargetSemantics, /*Negative=*/false, payload: &Fill));
527 }
528
529 S.Stk.push<Floating>(Args&: Result);
530 return true;
531}
532
533static bool interp__builtin_inf(InterpState &S, CodePtr OpPC,
534 const InterpFrame *Frame,
535 const CallExpr *Call) {
536 const llvm::fltSemantics &TargetSemantics =
537 S.getASTContext().getFloatTypeSemantics(
538 T: Call->getDirectCallee()->getReturnType());
539
540 Floating Result = S.allocFloat(Sem: TargetSemantics);
541 Result.copy(F: APFloat::getInf(Sem: TargetSemantics));
542 S.Stk.push<Floating>(Args&: Result);
543 return true;
544}
545
546static bool interp__builtin_copysign(InterpState &S, CodePtr OpPC,
547 const InterpFrame *Frame) {
548 const Floating &Arg2 = S.Stk.pop<Floating>();
549 const Floating &Arg1 = S.Stk.pop<Floating>();
550 Floating Result = S.allocFloat(Sem: Arg1.getSemantics());
551
552 APFloat Copy = Arg1.getAPFloat();
553 Copy.copySign(RHS: Arg2.getAPFloat());
554 Result.copy(F: Copy);
555 S.Stk.push<Floating>(Args&: Result);
556
557 return true;
558}
559
560static bool interp__builtin_fmin(InterpState &S, CodePtr OpPC,
561 const InterpFrame *Frame, bool IsNumBuiltin) {
562 const Floating &RHS = S.Stk.pop<Floating>();
563 const Floating &LHS = S.Stk.pop<Floating>();
564 Floating Result = S.allocFloat(Sem: LHS.getSemantics());
565
566 if (IsNumBuiltin)
567 Result.copy(F: llvm::minimumnum(A: LHS.getAPFloat(), B: RHS.getAPFloat()));
568 else
569 Result.copy(F: minnum(A: LHS.getAPFloat(), B: RHS.getAPFloat()));
570 S.Stk.push<Floating>(Args&: Result);
571 return true;
572}
573
574static bool interp__builtin_fmax(InterpState &S, CodePtr OpPC,
575 const InterpFrame *Frame, bool IsNumBuiltin) {
576 const Floating &RHS = S.Stk.pop<Floating>();
577 const Floating &LHS = S.Stk.pop<Floating>();
578 Floating Result = S.allocFloat(Sem: LHS.getSemantics());
579
580 if (IsNumBuiltin)
581 Result.copy(F: llvm::maximumnum(A: LHS.getAPFloat(), B: RHS.getAPFloat()));
582 else
583 Result.copy(F: maxnum(A: LHS.getAPFloat(), B: RHS.getAPFloat()));
584 S.Stk.push<Floating>(Args&: Result);
585 return true;
586}
587
588/// Defined as __builtin_isnan(...), to accommodate the fact that it can
589/// take a float, double, long double, etc.
590/// But for us, that's all a Floating anyway.
591static bool interp__builtin_isnan(InterpState &S, CodePtr OpPC,
592 const InterpFrame *Frame,
593 const CallExpr *Call) {
594 const Floating &Arg = S.Stk.pop<Floating>();
595
596 pushInteger(S, Val: Arg.isNan(), QT: Call->getType());
597 return true;
598}
599
600static bool interp__builtin_issignaling(InterpState &S, CodePtr OpPC,
601 const InterpFrame *Frame,
602 const CallExpr *Call) {
603 const Floating &Arg = S.Stk.pop<Floating>();
604
605 pushInteger(S, Val: Arg.isSignaling(), QT: Call->getType());
606 return true;
607}
608
609static bool interp__builtin_isinf(InterpState &S, CodePtr OpPC,
610 const InterpFrame *Frame, bool CheckSign,
611 const CallExpr *Call) {
612 const Floating &Arg = S.Stk.pop<Floating>();
613 APFloat F = Arg.getAPFloat();
614 bool IsInf = F.isInfinity();
615
616 if (CheckSign)
617 pushInteger(S, Val: IsInf ? (F.isNegative() ? -1 : 1) : 0, QT: Call->getType());
618 else
619 pushInteger(S, Val: IsInf, QT: Call->getType());
620 return true;
621}
622
623static bool interp__builtin_isfinite(InterpState &S, CodePtr OpPC,
624 const InterpFrame *Frame,
625 const CallExpr *Call) {
626 const Floating &Arg = S.Stk.pop<Floating>();
627
628 pushInteger(S, Val: Arg.isFinite(), QT: Call->getType());
629 return true;
630}
631
632static bool interp__builtin_isnormal(InterpState &S, CodePtr OpPC,
633 const InterpFrame *Frame,
634 const CallExpr *Call) {
635 const Floating &Arg = S.Stk.pop<Floating>();
636
637 pushInteger(S, Val: Arg.isNormal(), QT: Call->getType());
638 return true;
639}
640
641static bool interp__builtin_issubnormal(InterpState &S, CodePtr OpPC,
642 const InterpFrame *Frame,
643 const CallExpr *Call) {
644 const Floating &Arg = S.Stk.pop<Floating>();
645
646 pushInteger(S, Val: Arg.isDenormal(), QT: Call->getType());
647 return true;
648}
649
650static bool interp__builtin_iszero(InterpState &S, CodePtr OpPC,
651 const InterpFrame *Frame,
652 const CallExpr *Call) {
653 const Floating &Arg = S.Stk.pop<Floating>();
654
655 pushInteger(S, Val: Arg.isZero(), QT: Call->getType());
656 return true;
657}
658
659static bool interp__builtin_signbit(InterpState &S, CodePtr OpPC,
660 const InterpFrame *Frame,
661 const CallExpr *Call) {
662 const Floating &Arg = S.Stk.pop<Floating>();
663
664 pushInteger(S, Val: Arg.isNegative(), QT: Call->getType());
665 return true;
666}
667
668static bool interp_floating_comparison(InterpState &S, CodePtr OpPC,
669 const CallExpr *Call, unsigned ID) {
670 const Floating &RHS = S.Stk.pop<Floating>();
671 const Floating &LHS = S.Stk.pop<Floating>();
672
673 pushInteger(
674 S,
675 Val: [&] {
676 switch (ID) {
677 case Builtin::BI__builtin_isgreater:
678 return LHS > RHS;
679 case Builtin::BI__builtin_isgreaterequal:
680 return LHS >= RHS;
681 case Builtin::BI__builtin_isless:
682 return LHS < RHS;
683 case Builtin::BI__builtin_islessequal:
684 return LHS <= RHS;
685 case Builtin::BI__builtin_islessgreater: {
686 ComparisonCategoryResult Cmp = LHS.compare(RHS);
687 return Cmp == ComparisonCategoryResult::Less ||
688 Cmp == ComparisonCategoryResult::Greater;
689 }
690 case Builtin::BI__builtin_isunordered:
691 return LHS.compare(RHS) == ComparisonCategoryResult::Unordered;
692 default:
693 llvm_unreachable("Unexpected builtin ID: Should be a floating point "
694 "comparison function");
695 }
696 }(),
697 QT: Call->getType());
698 return true;
699}
700
701/// First parameter to __builtin_isfpclass is the floating value, the
702/// second one is an integral value.
703static bool interp__builtin_isfpclass(InterpState &S, CodePtr OpPC,
704 const InterpFrame *Frame,
705 const CallExpr *Call) {
706 APSInt FPClassArg;
707 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: FPClassArg))
708 return false;
709 const Floating &F = S.Stk.pop<Floating>();
710
711 int32_t Result = static_cast<int32_t>(
712 (F.classify() & std::move(FPClassArg)).getZExtValue());
713 pushInteger(S, Val: Result, QT: Call->getType());
714
715 return true;
716}
717
718/// Five int values followed by one floating value.
719/// __builtin_fpclassify(int, int, int, int, int, float)
720static bool interp__builtin_fpclassify(InterpState &S, CodePtr OpPC,
721 const InterpFrame *Frame,
722 const CallExpr *Call) {
723 const Floating &Val = S.Stk.pop<Floating>();
724
725 PrimType IntT = *S.getContext().classify(E: Call->getArg(Arg: 0));
726 APSInt Values[5];
727 for (unsigned I = 0; I != 5; ++I) {
728 if (!popToAPSInt(Stk&: S.Stk, T: IntT, Out&: Values[4 - I]))
729 return false;
730 }
731
732 unsigned Index;
733 switch (Val.getCategory()) {
734 case APFloat::fcNaN:
735 Index = 0;
736 break;
737 case APFloat::fcInfinity:
738 Index = 1;
739 break;
740 case APFloat::fcNormal:
741 Index = Val.isDenormal() ? 3 : 2;
742 break;
743 case APFloat::fcZero:
744 Index = 4;
745 break;
746 }
747
748 // The last argument is first on the stack.
749 assert(Index <= 4);
750
751 pushInteger(S, Val: Values[Index], QT: Call->getType());
752 return true;
753}
754
755static inline Floating abs(InterpState &S, const Floating &In) {
756 if (!In.isNegative())
757 return In;
758
759 Floating Output = S.allocFloat(Sem: In.getSemantics());
760 APFloat New = In.getAPFloat();
761 New.changeSign();
762 Output.copy(F: New);
763 return Output;
764}
765
766// The C standard says "fabs raises no floating-point exceptions,
767// even if x is a signaling NaN. The returned value is independent of
768// the current rounding direction mode." Therefore constant folding can
769// proceed without regard to the floating point settings.
770// Reference, WG14 N2478 F.10.4.3
771static bool interp__builtin_fabs(InterpState &S, CodePtr OpPC,
772 const InterpFrame *Frame) {
773 const Floating &Val = S.Stk.pop<Floating>();
774 S.Stk.push<Floating>(Args: abs(S, In: Val));
775 return true;
776}
777
778static bool interp__builtin_abs(InterpState &S, CodePtr OpPC,
779 const InterpFrame *Frame,
780 const CallExpr *Call) {
781 APSInt Val;
782 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: Val))
783 return false;
784 if (Val ==
785 APSInt(APInt::getSignedMinValue(numBits: Val.getBitWidth()), /*IsUnsigned=*/false))
786 return false;
787 if (Val.isNegative())
788 Val.negate();
789 pushInteger(S, Val, QT: Call->getType());
790 return true;
791}
792
793static bool interp__builtin_popcount(InterpState &S, CodePtr OpPC,
794 const InterpFrame *Frame,
795 const CallExpr *Call) {
796 APSInt Val;
797 if (Call->getArg(Arg: 0)->getType()->isExtVectorBoolType()) {
798 const Pointer &Arg = S.Stk.pop<Pointer>();
799 Val = convertBoolVectorToInt(Val: Arg);
800 } else {
801 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: Val))
802 return false;
803 }
804 pushInteger(S, Val: Val.popcount(), QT: Call->getType());
805 return true;
806}
807
808static bool interp__builtin_ia32_crc32(InterpState &S, CodePtr OpPC,
809 const InterpFrame *Frame,
810 const CallExpr *Call,
811 unsigned DataBytes) {
812 uint64_t DataVal;
813 if (!popToUInt64(S, E: Call->getArg(Arg: 1), Out&: DataVal))
814 return false;
815 uint64_t CRCVal;
816 if (!popToUInt64(S, E: Call->getArg(Arg: 0), Out&: CRCVal))
817 return false;
818
819 // CRC32C polynomial (iSCSI polynomial, bit-reversed)
820 static const uint32_t CRC32C_POLY = 0x82F63B78;
821
822 // Process each byte
823 uint32_t Result = static_cast<uint32_t>(CRCVal);
824 for (unsigned I = 0; I != DataBytes; ++I) {
825 uint8_t Byte = static_cast<uint8_t>((DataVal >> (I * 8)) & 0xFF);
826 Result ^= Byte;
827 for (int J = 0; J != 8; ++J) {
828 Result = (Result >> 1) ^ ((Result & 1) ? CRC32C_POLY : 0);
829 }
830 }
831
832 pushInteger(S, Val: Result, QT: Call->getType());
833 return true;
834}
835
836static bool interp__builtin_classify_type(InterpState &S, CodePtr OpPC,
837 const InterpFrame *Frame,
838 const CallExpr *Call) {
839 // This is an unevaluated call, so there are no arguments on the stack.
840 assert(Call->getNumArgs() == 1);
841 const Expr *Arg = Call->getArg(Arg: 0);
842
843 GCCTypeClass ResultClass =
844 EvaluateBuiltinClassifyType(T: Arg->getType(), LangOpts: S.getLangOpts());
845 int32_t ReturnVal = static_cast<int32_t>(ResultClass);
846 pushInteger(S, Val: ReturnVal, QT: Call->getType());
847 return true;
848}
849
850// __builtin_expect(long, long)
851// __builtin_expect_with_probability(long, long, double)
852static bool interp__builtin_expect(InterpState &S, CodePtr OpPC,
853 const InterpFrame *Frame,
854 const CallExpr *Call) {
855 // The return value is simply the value of the first parameter.
856 // We ignore the probability.
857 unsigned NumArgs = Call->getNumArgs();
858 assert(NumArgs == 2 || NumArgs == 3);
859
860 PrimType ArgT = *S.getContext().classify(T: Call->getArg(Arg: 0)->getType());
861 if (NumArgs == 3)
862 S.Stk.discard<Floating>();
863 discard(Stk&: S.Stk, T: ArgT);
864 // Top of the stack is now the first paramter. Leave it there as the return
865 // value.
866
867 return true;
868}
869
870static bool interp__builtin_addressof(InterpState &S, CodePtr OpPC,
871 const InterpFrame *Frame,
872 const CallExpr *Call) {
873#ifndef NDEBUG
874 assert(Call->getArg(0)->isLValue());
875 PrimType PtrT = S.getContext().classify(Call->getArg(0)).value_or(PT_Ptr);
876 assert(PtrT == PT_Ptr &&
877 "Unsupported pointer type passed to __builtin_addressof()");
878#endif
879 return true;
880}
881
882static bool interp__builtin_move(InterpState &S, CodePtr OpPC,
883 const InterpFrame *Frame,
884 const CallExpr *Call) {
885 return Call->getDirectCallee()->isConstexpr();
886}
887
888static bool interp__builtin_eh_return_data_regno(InterpState &S, CodePtr OpPC,
889 const InterpFrame *Frame,
890 const CallExpr *Call) {
891 APSInt Arg;
892 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: Arg))
893 return false;
894
895 int Result = S.getASTContext().getTargetInfo().getEHDataRegisterNumber(
896 RegNo: Arg.getZExtValue());
897 pushInteger(S, Val: Result, QT: Call->getType());
898 return true;
899}
900
901// Two integral values followed by a pointer (lhs, rhs, resultOut)
902static bool interp__builtin_overflowop(InterpState &S, CodePtr OpPC,
903 const CallExpr *Call,
904 unsigned BuiltinOp) {
905 const Pointer &ResultPtr = S.Stk.pop<Pointer>();
906 if (ResultPtr.isDummy() || !ResultPtr.isBlockPointer())
907 return false;
908
909 PrimType RHST = *S.getContext().classify(T: Call->getArg(Arg: 1)->getType());
910 PrimType LHST = *S.getContext().classify(T: Call->getArg(Arg: 0)->getType());
911 APSInt RHS;
912 if (!popToAPSInt(Stk&: S.Stk, T: RHST, Out&: RHS))
913 return false;
914 APSInt LHS;
915 if (!popToAPSInt(Stk&: S.Stk, T: LHST, Out&: LHS))
916 return false;
917 QualType ResultType = Call->getArg(Arg: 2)->getType()->getPointeeType();
918 PrimType ResultT = *S.getContext().classify(T: ResultType);
919 bool Overflow;
920
921 APSInt Result;
922 if (BuiltinOp == Builtin::BI__builtin_add_overflow ||
923 BuiltinOp == Builtin::BI__builtin_sub_overflow ||
924 BuiltinOp == Builtin::BI__builtin_mul_overflow) {
925 bool IsSigned = LHS.isSigned() || RHS.isSigned() ||
926 ResultType->isSignedIntegerOrEnumerationType();
927 bool AllSigned = LHS.isSigned() && RHS.isSigned() &&
928 ResultType->isSignedIntegerOrEnumerationType();
929 uint64_t LHSSize = LHS.getBitWidth();
930 uint64_t RHSSize = RHS.getBitWidth();
931 uint64_t ResultSize = S.getASTContext().getIntWidth(T: ResultType);
932 uint64_t MaxBits = std::max(a: std::max(a: LHSSize, b: RHSSize), b: ResultSize);
933
934 // Add an additional bit if the signedness isn't uniformly agreed to. We
935 // could do this ONLY if there is a signed and an unsigned that both have
936 // MaxBits, but the code to check that is pretty nasty. The issue will be
937 // caught in the shrink-to-result later anyway.
938 if (IsSigned && !AllSigned)
939 ++MaxBits;
940
941 LHS = APSInt(LHS.extOrTrunc(width: MaxBits), !IsSigned);
942 RHS = APSInt(RHS.extOrTrunc(width: MaxBits), !IsSigned);
943 Result = APSInt(MaxBits, !IsSigned);
944 }
945
946 // Find largest int.
947 switch (BuiltinOp) {
948 default:
949 llvm_unreachable("Invalid value for BuiltinOp");
950 case Builtin::BI__builtin_add_overflow:
951 case Builtin::BI__builtin_sadd_overflow:
952 case Builtin::BI__builtin_saddl_overflow:
953 case Builtin::BI__builtin_saddll_overflow:
954 case Builtin::BI__builtin_uadd_overflow:
955 case Builtin::BI__builtin_uaddl_overflow:
956 case Builtin::BI__builtin_uaddll_overflow:
957 Result = LHS.isSigned() ? LHS.sadd_ov(RHS, Overflow)
958 : LHS.uadd_ov(RHS, Overflow);
959 break;
960 case Builtin::BI__builtin_sub_overflow:
961 case Builtin::BI__builtin_ssub_overflow:
962 case Builtin::BI__builtin_ssubl_overflow:
963 case Builtin::BI__builtin_ssubll_overflow:
964 case Builtin::BI__builtin_usub_overflow:
965 case Builtin::BI__builtin_usubl_overflow:
966 case Builtin::BI__builtin_usubll_overflow:
967 Result = LHS.isSigned() ? LHS.ssub_ov(RHS, Overflow)
968 : LHS.usub_ov(RHS, Overflow);
969 break;
970 case Builtin::BI__builtin_mul_overflow:
971 case Builtin::BI__builtin_smul_overflow:
972 case Builtin::BI__builtin_smull_overflow:
973 case Builtin::BI__builtin_smulll_overflow:
974 case Builtin::BI__builtin_umul_overflow:
975 case Builtin::BI__builtin_umull_overflow:
976 case Builtin::BI__builtin_umulll_overflow:
977 Result = LHS.isSigned() ? LHS.smul_ov(RHS, Overflow)
978 : LHS.umul_ov(RHS, Overflow);
979 break;
980 }
981
982 // In the case where multiple sizes are allowed, truncate and see if
983 // the values are the same.
984 if (BuiltinOp == Builtin::BI__builtin_add_overflow ||
985 BuiltinOp == Builtin::BI__builtin_sub_overflow ||
986 BuiltinOp == Builtin::BI__builtin_mul_overflow) {
987 // APSInt doesn't have a TruncOrSelf, so we use extOrTrunc instead,
988 // since it will give us the behavior of a TruncOrSelf in the case where
989 // its parameter <= its size. We previously set Result to be at least the
990 // integer width of the result, so getIntWidth(ResultType) <=
991 // Result.BitWidth
992 APSInt Temp = Result.extOrTrunc(width: S.getASTContext().getIntWidth(T: ResultType));
993 Temp.setIsSigned(ResultType->isSignedIntegerOrEnumerationType());
994
995 if (!APSInt::isSameValue(I1: Temp, I2: Result))
996 Overflow = true;
997 Result = std::move(Temp);
998 }
999
1000 // Write Result to ResultPtr and put Overflow on the stack.
1001 assignIntegral(S, Dest: ResultPtr, ValueT: ResultT, Value: Result);
1002 if (ResultPtr.canBeInitialized())
1003 ResultPtr.initialize();
1004
1005 assert(Call->getDirectCallee()->getReturnType()->isBooleanType());
1006 S.Stk.push<Boolean>(Args&: Overflow);
1007 return true;
1008}
1009
1010/// Three integral values followed by a pointer (lhs, rhs, carry, carryOut).
1011static bool interp__builtin_carryop(InterpState &S, CodePtr OpPC,
1012 const InterpFrame *Frame,
1013 const CallExpr *Call, unsigned BuiltinOp) {
1014 const Pointer &CarryOutPtr = S.Stk.pop<Pointer>();
1015 PrimType LHST = *S.getContext().classify(T: Call->getArg(Arg: 0)->getType());
1016 PrimType RHST = *S.getContext().classify(T: Call->getArg(Arg: 1)->getType());
1017 APSInt CarryIn;
1018 if (!popToAPSInt(Stk&: S.Stk, T: LHST, Out&: CarryIn))
1019 return false;
1020 APSInt RHS;
1021 if (!popToAPSInt(Stk&: S.Stk, T: RHST, Out&: RHS))
1022 return false;
1023 APSInt LHS;
1024 if (!popToAPSInt(Stk&: S.Stk, T: LHST, Out&: LHS))
1025 return false;
1026
1027 if (!isReadable(P: CarryOutPtr))
1028 return false;
1029
1030 APSInt CarryOut;
1031
1032 APSInt Result;
1033 // Copy the number of bits and sign.
1034 Result = LHS;
1035 CarryOut = LHS;
1036
1037 bool FirstOverflowed = false;
1038 bool SecondOverflowed = false;
1039 switch (BuiltinOp) {
1040 default:
1041 llvm_unreachable("Invalid value for BuiltinOp");
1042 case Builtin::BI__builtin_addcb:
1043 case Builtin::BI__builtin_addcs:
1044 case Builtin::BI__builtin_addc:
1045 case Builtin::BI__builtin_addcl:
1046 case Builtin::BI__builtin_addcll:
1047 Result =
1048 LHS.uadd_ov(RHS, Overflow&: FirstOverflowed).uadd_ov(RHS: CarryIn, Overflow&: SecondOverflowed);
1049 break;
1050 case Builtin::BI__builtin_subcb:
1051 case Builtin::BI__builtin_subcs:
1052 case Builtin::BI__builtin_subc:
1053 case Builtin::BI__builtin_subcl:
1054 case Builtin::BI__builtin_subcll:
1055 Result =
1056 LHS.usub_ov(RHS, Overflow&: FirstOverflowed).usub_ov(RHS: CarryIn, Overflow&: SecondOverflowed);
1057 break;
1058 }
1059 // It is possible for both overflows to happen but CGBuiltin uses an OR so
1060 // this is consistent.
1061 CarryOut = (uint64_t)(FirstOverflowed | SecondOverflowed);
1062
1063 QualType CarryOutType = Call->getArg(Arg: 3)->getType()->getPointeeType();
1064 PrimType CarryOutT = *S.getContext().classify(T: CarryOutType);
1065 assignIntegral(S, Dest: CarryOutPtr, ValueT: CarryOutT, Value: CarryOut);
1066 if (CarryOutPtr.canBeInitialized())
1067 CarryOutPtr.initialize();
1068
1069 assert(S.getASTContext().hasSimilarType(Call->getType(),
1070 Call->getArg(0)->getType()));
1071 pushInteger(S, Val: Result, QT: Call->getType());
1072 return true;
1073}
1074
1075static bool interp__builtin_clz(InterpState &S, CodePtr OpPC,
1076 const InterpFrame *Frame, const CallExpr *Call,
1077 unsigned BuiltinOp) {
1078
1079 std::optional<APSInt> Fallback;
1080 if (BuiltinOp == Builtin::BI__builtin_clzg && Call->getNumArgs() == 2) {
1081 APSInt FallbackVal;
1082 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: FallbackVal))
1083 return false;
1084 Fallback = FallbackVal;
1085 }
1086
1087 APSInt Val;
1088 if (Call->getArg(Arg: 0)->getType()->isExtVectorBoolType()) {
1089 const Pointer &Arg = S.Stk.pop<Pointer>();
1090 Val = convertBoolVectorToInt(Val: Arg);
1091 } else {
1092 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: Val))
1093 return false;
1094 }
1095
1096 // When the argument is 0, the result of GCC builtins is undefined, whereas
1097 // for Microsoft intrinsics, the result is the bit-width of the argument.
1098 bool ZeroIsUndefined = BuiltinOp != Builtin::BI__lzcnt16 &&
1099 BuiltinOp != Builtin::BI__lzcnt &&
1100 BuiltinOp != Builtin::BI__lzcnt64;
1101
1102 if (Val == 0) {
1103 if (Fallback) {
1104 pushInteger(S, Val: *Fallback, QT: Call->getType());
1105 return true;
1106 }
1107
1108 if (ZeroIsUndefined)
1109 return false;
1110 }
1111
1112 pushInteger(S, Val: Val.countl_zero(), QT: Call->getType());
1113 return true;
1114}
1115
1116static bool interp__builtin_ctz(InterpState &S, CodePtr OpPC,
1117 const InterpFrame *Frame, const CallExpr *Call,
1118 unsigned BuiltinID) {
1119 std::optional<APSInt> Fallback;
1120 if (BuiltinID == Builtin::BI__builtin_ctzg && Call->getNumArgs() == 2) {
1121 APSInt FallbackVal;
1122 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: FallbackVal))
1123 return false;
1124 Fallback = FallbackVal;
1125 }
1126
1127 APSInt Val;
1128 if (Call->getArg(Arg: 0)->getType()->isExtVectorBoolType()) {
1129 const Pointer &Arg = S.Stk.pop<Pointer>();
1130 Val = convertBoolVectorToInt(Val: Arg);
1131 } else {
1132 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: Val))
1133 return false;
1134 }
1135
1136 if (Val == 0) {
1137 if (Fallback) {
1138 pushInteger(S, Val: *Fallback, QT: Call->getType());
1139 return true;
1140 }
1141 return false;
1142 }
1143
1144 pushInteger(S, Val: Val.countr_zero(), QT: Call->getType());
1145 return true;
1146}
1147
1148static bool interp__builtin_bswap(InterpState &S, CodePtr OpPC,
1149 const InterpFrame *Frame,
1150 const CallExpr *Call) {
1151 APSInt Val;
1152 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: Val))
1153 return false;
1154 if (Val.getBitWidth() == 8 || Val.getBitWidth() == 1)
1155 pushInteger(S, Val, QT: Call->getType());
1156 else
1157 pushInteger(S, Val: Val.byteSwap(), QT: Call->getType());
1158 return true;
1159}
1160
1161/// bool __atomic_always_lock_free(size_t, void const volatile*)
1162/// bool __atomic_is_lock_free(size_t, void const volatile*)
1163static bool interp__builtin_atomic_lock_free(InterpState &S, CodePtr OpPC,
1164 const InterpFrame *Frame,
1165 const CallExpr *Call,
1166 unsigned BuiltinOp) {
1167 auto returnBool = [&S](bool Value) -> bool {
1168 S.Stk.push<Boolean>(Args&: Value);
1169 return true;
1170 };
1171
1172 const Pointer &Ptr = S.Stk.pop<Pointer>();
1173 uint64_t SizeVal;
1174 if (!popToUInt64(S, E: Call->getArg(Arg: 0), Out&: SizeVal))
1175 return false;
1176
1177 // For __atomic_is_lock_free(sizeof(_Atomic(T))), if the size is a power
1178 // of two less than or equal to the maximum inline atomic width, we know it
1179 // is lock-free. If the size isn't a power of two, or greater than the
1180 // maximum alignment where we promote atomics, we know it is not lock-free
1181 // (at least not in the sense of atomic_is_lock_free). Otherwise,
1182 // the answer can only be determined at runtime; for example, 16-byte
1183 // atomics have lock-free implementations on some, but not all,
1184 // x86-64 processors.
1185
1186 // Check power-of-two.
1187 CharUnits Size = CharUnits::fromQuantity(Quantity: SizeVal);
1188 if (Size.isPowerOfTwo()) {
1189 // Check against inlining width.
1190 unsigned InlineWidthBits =
1191 S.getASTContext().getTargetInfo().getMaxAtomicInlineWidth();
1192 if (Size <= S.getASTContext().toCharUnitsFromBits(BitSize: InlineWidthBits)) {
1193
1194 // OK, we will inline appropriately-aligned operations of this size,
1195 // and _Atomic(T) is appropriately-aligned.
1196 if (Size == CharUnits::One())
1197 return returnBool(true);
1198
1199 // Same for null pointers.
1200 assert(BuiltinOp != Builtin::BI__c11_atomic_is_lock_free);
1201 if (Ptr.isZero())
1202 return returnBool(true);
1203
1204 if (Ptr.isIntegralPointer()) {
1205 uint64_t IntVal = Ptr.getIntegerRepresentation();
1206 if (APSInt(APInt(64, IntVal, false), true).isAligned(A: Size.getAsAlign()))
1207 return returnBool(true);
1208 }
1209
1210 const Expr *PtrArg = Call->getArg(Arg: 1);
1211 // Otherwise, check if the type's alignment against Size.
1212 if (const auto *ICE = dyn_cast<ImplicitCastExpr>(Val: PtrArg)) {
1213 // Drop the potential implicit-cast to 'const volatile void*', getting
1214 // the underlying type.
1215 if (ICE->getCastKind() == CK_BitCast)
1216 PtrArg = ICE->getSubExpr();
1217 }
1218
1219 if (const auto *PtrTy = PtrArg->getType()->getAs<PointerType>()) {
1220 QualType PointeeType = PtrTy->getPointeeType();
1221 if (!PointeeType->isIncompleteType() &&
1222 S.getASTContext().getTypeAlignInChars(T: PointeeType) >= Size) {
1223 // OK, we will inline operations on this object.
1224 return returnBool(true);
1225 }
1226 }
1227 }
1228 }
1229
1230 if (BuiltinOp == Builtin::BI__atomic_always_lock_free)
1231 return returnBool(false);
1232
1233 return Invalid(S, OpPC);
1234}
1235
1236/// bool __c11_atomic_is_lock_free(size_t)
1237static bool interp__builtin_c11_atomic_is_lock_free(InterpState &S,
1238 CodePtr OpPC,
1239 const InterpFrame *Frame,
1240 const CallExpr *Call) {
1241 uint64_t SizeVal;
1242 if (!popToUInt64(S, E: Call->getArg(Arg: 0), Out&: SizeVal))
1243 return false;
1244
1245 CharUnits Size = CharUnits::fromQuantity(Quantity: SizeVal);
1246 if (Size.isPowerOfTwo()) {
1247 // Check against inlining width.
1248 unsigned InlineWidthBits =
1249 S.getASTContext().getTargetInfo().getMaxAtomicInlineWidth();
1250 if (Size <= S.getASTContext().toCharUnitsFromBits(BitSize: InlineWidthBits)) {
1251 S.Stk.push<Boolean>(Args: true);
1252 return true;
1253 }
1254 }
1255
1256 return false; // returnBool(false);
1257}
1258
1259/// __builtin_complex(Float A, float B);
1260static bool interp__builtin_complex(InterpState &S, CodePtr OpPC,
1261 const InterpFrame *Frame,
1262 const CallExpr *Call) {
1263 const Floating &Arg2 = S.Stk.pop<Floating>();
1264 const Floating &Arg1 = S.Stk.pop<Floating>();
1265 Pointer &Result = S.Stk.peek<Pointer>();
1266
1267 Result.elem<Floating>(I: 0) = Arg1;
1268 Result.elem<Floating>(I: 1) = Arg2;
1269 Result.initializeAllElements();
1270
1271 return true;
1272}
1273
1274/// __builtin_is_aligned()
1275/// __builtin_align_up()
1276/// __builtin_align_down()
1277/// The first parameter is either an integer or a pointer.
1278/// The second parameter is the requested alignment as an integer.
1279static bool interp__builtin_is_aligned_up_down(InterpState &S, CodePtr OpPC,
1280 const InterpFrame *Frame,
1281 const CallExpr *Call,
1282 unsigned BuiltinOp) {
1283 APSInt Alignment;
1284 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: Alignment))
1285 return false;
1286
1287 if (Alignment < 0 || !Alignment.isPowerOf2()) {
1288 S.FFDiag(E: Call, DiagId: diag::note_constexpr_invalid_alignment) << Alignment;
1289 return false;
1290 }
1291 unsigned SrcWidth = S.getASTContext().getIntWidth(T: Call->getArg(Arg: 0)->getType());
1292 APSInt MaxValue(APInt::getOneBitSet(numBits: SrcWidth, BitNo: SrcWidth - 1));
1293 if (APSInt::compareValues(I1: Alignment, I2: MaxValue) > 0) {
1294 S.FFDiag(E: Call, DiagId: diag::note_constexpr_alignment_too_big)
1295 << MaxValue << Call->getArg(Arg: 0)->getType() << Alignment;
1296 return false;
1297 }
1298
1299 // The first parameter is either an integer or a pointer.
1300 PrimType FirstArgT = *S.Ctx.classify(E: Call->getArg(Arg: 0));
1301
1302 if (isIntegerType(T: FirstArgT)) {
1303 APSInt Src;
1304 if (!popToAPSInt(Stk&: S.Stk, T: FirstArgT, Out&: Src))
1305 return false;
1306 APInt AlignMinusOne = Alignment.extOrTrunc(width: Src.getBitWidth()) - 1;
1307 if (BuiltinOp == Builtin::BI__builtin_align_up) {
1308 APSInt AlignedVal =
1309 APSInt((Src + AlignMinusOne) & ~AlignMinusOne, Src.isUnsigned());
1310 pushInteger(S, Val: AlignedVal, QT: Call->getType());
1311 } else if (BuiltinOp == Builtin::BI__builtin_align_down) {
1312 APSInt AlignedVal = APSInt(Src & ~AlignMinusOne, Src.isUnsigned());
1313 pushInteger(S, Val: AlignedVal, QT: Call->getType());
1314 } else {
1315 assert(*S.Ctx.classify(Call->getType()) == PT_Bool);
1316 S.Stk.push<Boolean>(Args: (Src & AlignMinusOne) == 0);
1317 }
1318 return true;
1319 }
1320 assert(FirstArgT == PT_Ptr);
1321 const Pointer &Ptr = S.Stk.pop<Pointer>();
1322 if (!Ptr.isBlockPointer() && !Ptr.isOpaquePointer()) {
1323 S.FFDiag(E: Call->getArg(Arg: 0), DiagId: diag::note_constexpr_alignment_compute)
1324 << Alignment;
1325 return false;
1326 }
1327
1328 const VarDecl *PtrDecl = Ptr.getRootVarDecl();
1329 // We need a pointer for a declaration here.
1330 if (!PtrDecl) {
1331 if (BuiltinOp == Builtin::BI__builtin_is_aligned)
1332 S.FFDiag(E: Call->getArg(Arg: 0), DiagId: diag::note_constexpr_alignment_compute)
1333 << Alignment;
1334 else
1335 S.FFDiag(E: Call->getArg(Arg: 0), DiagId: diag::note_constexpr_alignment_adjust)
1336 << Alignment;
1337 return false;
1338 }
1339
1340 unsigned PtrOffset;
1341 if (Ptr.isBlockPointer()) {
1342 // For one-past-end pointers, we can't call getIndex() since it asserts.
1343 // Use getNumElems() instead which gives the correct index for past-end.
1344 PtrOffset = Ptr.isElementPastEnd() ? Ptr.getNumElems() : Ptr.getIndex();
1345 } else {
1346 if (std::optional<size_t> PtrOff =
1347 Ptr.computeLayoutOffset(ASTCtx: S.getASTContext()))
1348 PtrOffset = *PtrOff;
1349 else
1350 return false;
1351 }
1352
1353 CharUnits BaseAlignment = S.getASTContext().getDeclAlign(D: PtrDecl);
1354 CharUnits PtrAlign =
1355 BaseAlignment.alignmentAtOffset(offset: CharUnits::fromQuantity(Quantity: PtrOffset));
1356
1357 if (BuiltinOp == Builtin::BI__builtin_is_aligned) {
1358 if (PtrAlign.getQuantity() >= Alignment) {
1359 S.Stk.push<Boolean>(Args: true);
1360 return true;
1361 }
1362 // If the alignment is not known to be sufficient, some cases could still
1363 // be aligned at run time. However, if the requested alignment is less or
1364 // equal to the base alignment and the offset is not aligned, we know that
1365 // the run-time value can never be aligned.
1366 if (BaseAlignment.getQuantity() >= Alignment &&
1367 PtrAlign.getQuantity() < Alignment) {
1368 S.Stk.push<Boolean>(Args: false);
1369 return true;
1370 }
1371
1372 S.FFDiag(E: Call->getArg(Arg: 0), DiagId: diag::note_constexpr_alignment_compute)
1373 << Alignment;
1374 return false;
1375 }
1376
1377 assert(BuiltinOp == Builtin::BI__builtin_align_down ||
1378 BuiltinOp == Builtin::BI__builtin_align_up);
1379
1380 // For align_up/align_down, we can return the same value if the alignment
1381 // is known to be greater or equal to the requested value.
1382 if (PtrAlign.getQuantity() >= Alignment) {
1383 S.Stk.push<Pointer>(Args: Ptr);
1384 return true;
1385 }
1386
1387 // The alignment could be greater than the minimum at run-time, so we cannot
1388 // infer much about the resulting pointer value. One case is possible:
1389 // For `_Alignas(32) char buf[N]; __builtin_align_down(&buf[idx], 32)` we
1390 // can infer the correct index if the requested alignment is smaller than
1391 // the base alignment so we can perform the computation on the offset.
1392 if (BaseAlignment.getQuantity() >= Alignment) {
1393 assert(Alignment.getBitWidth() <= 64 &&
1394 "Cannot handle > 64-bit address-space");
1395 uint64_t Alignment64 = Alignment.getZExtValue();
1396 CharUnits NewOffset =
1397 CharUnits::fromQuantity(Quantity: BuiltinOp == Builtin::BI__builtin_align_down
1398 ? llvm::alignDown(Value: PtrOffset, Align: Alignment64)
1399 : llvm::alignTo(Value: PtrOffset, Align: Alignment64));
1400
1401 if (Ptr.isBlockPointer()) {
1402 S.Stk.push<Pointer>(Args: Ptr.atIndex(Idx: NewOffset.getQuantity()));
1403 return true;
1404 }
1405
1406 assert(Ptr.isOpaquePointer());
1407
1408 APSInt APOffset =
1409 APSInt(APInt(64, NewOffset.getQuantity(), /*IsSigned=*/true),
1410 /*IsUnsigned=*/false);
1411 return arrayElemPtrOpaque(S, OpPC, Ptr, Index: std::move(APOffset),
1412 /*AllocReplace=*/AllowReplace: true);
1413 }
1414
1415 // Otherwise, we cannot constant-evaluate the result.
1416 S.FFDiag(E: Call->getArg(Arg: 0), DiagId: diag::note_constexpr_alignment_adjust) << Alignment;
1417 return false;
1418}
1419
1420/// __builtin_assume_aligned(Ptr, Alignment[, ExtraOffset])
1421static bool interp__builtin_assume_aligned(InterpState &S, CodePtr OpPC,
1422 const InterpFrame *Frame,
1423 const CallExpr *Call) {
1424 assert(Call->getNumArgs() == 2 || Call->getNumArgs() == 3);
1425
1426 std::optional<APSInt> ExtraOffset;
1427 if (Call->getNumArgs() == 3) {
1428 APSInt ExtraOffsetVal;
1429 if (!popToAPSInt(Stk&: S.Stk, T: *S.Ctx.classify(E: Call->getArg(Arg: 2)), Out&: ExtraOffsetVal))
1430 return false;
1431 ExtraOffset = ExtraOffsetVal;
1432 }
1433
1434 APSInt Alignment;
1435 if (!popToAPSInt(Stk&: S.Stk, T: *S.Ctx.classify(E: Call->getArg(Arg: 1)), Out&: Alignment))
1436 return false;
1437 const Pointer &Ptr = S.Stk.pop<Pointer>();
1438
1439 const ASTContext &ASTCtx = S.getASTContext();
1440 CharUnits Align = CharUnits::fromQuantity(Quantity: Alignment.getZExtValue());
1441
1442 // If there is a base object, then it must have the correct alignment.
1443 if (Ptr.isBlockPointer() || Ptr.isOpaquePointer()) {
1444 CharUnits BaseAlignment;
1445 if (const auto *VD = Ptr.getRootVarDecl())
1446 BaseAlignment = ASTCtx.getDeclAlign(D: VD);
1447 else if (const auto *E = Ptr.getRootExpr())
1448 BaseAlignment = GetAlignOfExpr(Ctx: ASTCtx, E, ExprKind: UETT_AlignOf);
1449
1450 if (BaseAlignment < Align) {
1451 S.CCEDiag(E: Call->getArg(Arg: 0),
1452 DiagId: diag::note_constexpr_baa_insufficient_alignment)
1453 << 0 << BaseAlignment.getQuantity() << Align.getQuantity();
1454 return false;
1455 }
1456 }
1457
1458 std::optional<size_t> LayoutOffset = Ptr.computeLayoutOffset(ASTCtx);
1459 if (!LayoutOffset)
1460 return false;
1461
1462 CharUnits AVOffset = CharUnits::fromQuantity(Quantity: *LayoutOffset);
1463 if (ExtraOffset)
1464 AVOffset -= CharUnits::fromQuantity(Quantity: ExtraOffset->getZExtValue());
1465 if (AVOffset.alignTo(Align) != AVOffset) {
1466 if (Ptr.isBlockPointer() || Ptr.isOpaquePointer())
1467 S.CCEDiag(E: Call->getArg(Arg: 0),
1468 DiagId: diag::note_constexpr_baa_insufficient_alignment)
1469 << 1 << AVOffset.getQuantity() << Align.getQuantity();
1470 else
1471 S.CCEDiag(E: Call->getArg(Arg: 0),
1472 DiagId: diag::note_constexpr_baa_value_insufficient_alignment)
1473 << AVOffset.getQuantity() << Align.getQuantity();
1474 return false;
1475 }
1476
1477 S.Stk.push<Pointer>(Args: Ptr);
1478 return true;
1479}
1480
1481/// (CarryIn, LHS, RHS, Result)
1482static bool interp__builtin_ia32_addcarry_subborrow(InterpState &S,
1483 CodePtr OpPC,
1484 const InterpFrame *Frame,
1485 const CallExpr *Call,
1486 bool IsAdd) {
1487 if (Call->getNumArgs() != 4 || !Call->getArg(Arg: 0)->getType()->isIntegerType() ||
1488 !Call->getArg(Arg: 1)->getType()->isIntegerType() ||
1489 !Call->getArg(Arg: 2)->getType()->isIntegerType())
1490 return false;
1491
1492 const Pointer &CarryOutPtr = S.Stk.pop<Pointer>();
1493
1494 APSInt RHS;
1495 if (!popToAPSInt(S, E: Call->getArg(Arg: 2), Out&: RHS))
1496 return false;
1497 APSInt LHS;
1498 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: LHS))
1499 return false;
1500 APSInt CarryIn;
1501 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: CarryIn))
1502 return false;
1503
1504 unsigned BitWidth = LHS.getBitWidth();
1505 unsigned CarryInBit = CarryIn.ugt(RHS: 0) ? 1 : 0;
1506 APInt ExResult =
1507 IsAdd ? (LHS.zext(width: BitWidth + 1) + (RHS.zext(width: BitWidth + 1) + CarryInBit))
1508 : (LHS.zext(width: BitWidth + 1) - (RHS.zext(width: BitWidth + 1) + CarryInBit));
1509
1510 APInt Result = ExResult.extractBits(numBits: BitWidth, bitPosition: 0);
1511 APSInt CarryOut =
1512 APSInt(ExResult.extractBits(numBits: 1, bitPosition: BitWidth), /*IsUnsigned=*/true);
1513
1514 QualType CarryOutType = Call->getArg(Arg: 3)->getType()->getPointeeType();
1515 PrimType CarryOutT = *S.getContext().classify(T: CarryOutType);
1516 assignIntegral(S, Dest: CarryOutPtr, ValueT: CarryOutT, Value: APSInt(std::move(Result), true));
1517
1518 pushInteger(S, Val: CarryOut, QT: Call->getType());
1519
1520 return true;
1521}
1522
1523static bool interp__builtin_os_log_format_buffer_size(InterpState &S,
1524 CodePtr OpPC,
1525 const InterpFrame *Frame,
1526 const CallExpr *Call) {
1527 analyze_os_log::OSLogBufferLayout Layout;
1528 analyze_os_log::computeOSLogBufferLayout(Ctx&: S.getASTContext(), E: Call, layout&: Layout);
1529 pushInteger(S, Val: Layout.size().getQuantity(), QT: Call->getType());
1530 return true;
1531}
1532
1533static bool
1534interp__builtin_ptrauth_string_discriminator(InterpState &S, CodePtr OpPC,
1535 const InterpFrame *Frame,
1536 const CallExpr *Call) {
1537 const auto &Ptr = S.Stk.pop<Pointer>();
1538 if (!Ptr.isStringPointer())
1539 return false;
1540
1541 uint64_t Result = getPointerAuthStableSipHash(
1542 S: cast<StringLiteral>(Val: Ptr.getRootExpr())->getString());
1543 pushInteger(S, Val: Result, QT: Call->getType());
1544 return true;
1545}
1546
1547static bool interp__builtin_infer_alloc_token(InterpState &S, CodePtr OpPC,
1548 const InterpFrame *Frame,
1549 const CallExpr *Call) {
1550 const ASTContext &ASTCtx = S.getASTContext();
1551 uint64_t BitWidth = ASTCtx.getTypeSize(T: ASTCtx.getSizeType());
1552 auto Mode =
1553 ASTCtx.getLangOpts().AllocTokenMode.value_or(u: llvm::DefaultAllocTokenMode);
1554 auto MaxTokensOpt = ASTCtx.getLangOpts().AllocTokenMax;
1555 uint64_t MaxTokens =
1556 MaxTokensOpt.value_or(u: 0) ? *MaxTokensOpt : (~0ULL >> (64 - BitWidth));
1557
1558 // We do not read any of the arguments; discard them.
1559 for (int I = Call->getNumArgs() - 1; I >= 0; --I)
1560 discard(Stk&: S.Stk, T: S.getContext().classify(E: Call->getArg(Arg: I)).value_or(PT: PT_Ptr));
1561
1562 // Note: Type inference from a surrounding cast is not supported in
1563 // constexpr evaluation.
1564 QualType AllocType = infer_alloc::inferPossibleType(E: Call, Ctx: ASTCtx, CastE: nullptr);
1565 if (AllocType.isNull()) {
1566 S.CCEDiag(E: Call,
1567 DiagId: diag::note_constexpr_infer_alloc_token_type_inference_failed);
1568 return false;
1569 }
1570
1571 auto ATMD = infer_alloc::getAllocTokenMetadata(T: AllocType, Ctx: ASTCtx);
1572 if (!ATMD) {
1573 S.CCEDiag(E: Call, DiagId: diag::note_constexpr_infer_alloc_token_no_metadata);
1574 return false;
1575 }
1576
1577 auto MaybeToken = llvm::getAllocToken(Mode, Metadata: *ATMD, MaxTokens);
1578 if (!MaybeToken) {
1579 S.CCEDiag(E: Call, DiagId: diag::note_constexpr_infer_alloc_token_stateful_mode);
1580 return false;
1581 }
1582
1583 pushInteger(S, Val: llvm::APInt(BitWidth, *MaybeToken), QT: ASTCtx.getSizeType());
1584 return true;
1585}
1586
1587static bool interp__builtin_operator_new(InterpState &S, CodePtr OpPC,
1588 const InterpFrame *Frame,
1589 const CallExpr *Call) {
1590 // A call to __operator_new is only valid within std::allocate<>::allocate.
1591 // Walk up the call stack to find the appropriate caller and get the
1592 // element type from it.
1593 auto [NewCall, ElemType] = S.getStdAllocatorCaller(Name: "allocate");
1594
1595 if (ElemType.isNull()) {
1596 S.FFDiag(E: Call, DiagId: S.getLangOpts().CPlusPlus20
1597 ? diag::note_constexpr_new_untyped
1598 : diag::note_constexpr_new);
1599 return false;
1600 }
1601 assert(NewCall);
1602
1603 if (ElemType->isIncompleteType() || ElemType->isFunctionType()) {
1604 S.FFDiag(E: Call, DiagId: diag::note_constexpr_new_not_complete_object_type)
1605 << (ElemType->isIncompleteType() ? 0 : 1) << ElemType;
1606 return false;
1607 }
1608
1609 // We only care about the first parameter (the size), so discard all the
1610 // others.
1611 {
1612 unsigned NumArgs = Call->getNumArgs();
1613 assert(NumArgs >= 1);
1614
1615 // The std::nothrow_t arg never gets put on the stack.
1616 if (Call->getArg(Arg: NumArgs - 1)->getType()->isNothrowT())
1617 --NumArgs;
1618 auto Args = ArrayRef(Call->getArgs(), Call->getNumArgs());
1619 // First arg is needed.
1620 Args = Args.drop_front();
1621
1622 // Discard the rest.
1623 for (const Expr *Arg : Args)
1624 discard(Stk&: S.Stk, T: *S.getContext().classify(E: Arg));
1625 }
1626
1627 APSInt Bytes;
1628 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: Bytes))
1629 return false;
1630 CharUnits ElemSize = S.getASTContext().getTypeSizeInChars(T: ElemType);
1631 assert(!ElemSize.isZero());
1632 // Divide the number of bytes by sizeof(ElemType), so we get the number of
1633 // elements we should allocate.
1634 APInt NumElems, Remainder;
1635 APInt ElemSizeAP(Bytes.getBitWidth(), ElemSize.getQuantity());
1636 APInt::udivrem(LHS: Bytes, RHS: ElemSizeAP, Quotient&: NumElems, Remainder);
1637 if (Remainder != 0) {
1638 // This likely indicates a bug in the implementation of 'std::allocator'.
1639 S.FFDiag(E: Call, DiagId: diag::note_constexpr_operator_new_bad_size)
1640 << Bytes << APSInt(ElemSizeAP, true) << ElemType;
1641 return false;
1642 }
1643
1644 // NB: The same check we're using in CheckArraySize()
1645 if (NumElems.getActiveBits() >
1646 ConstantArrayType::getMaxSizeBits(Context: S.getASTContext()) ||
1647 NumElems.ugt(RHS: Descriptor::MaxArrayElemBytes / ElemSize.getQuantity())) {
1648 // FIXME: NoThrow check?
1649 const SourceInfo &Loc = S.Current->getSource(PC: OpPC);
1650 S.FFDiag(SI: Loc, DiagId: diag::note_constexpr_new_too_large)
1651 << NumElems.getZExtValue();
1652 return false;
1653 }
1654
1655 if (!CheckArraySize(S, OpPC, NumElems: NumElems.getZExtValue()))
1656 return false;
1657
1658 bool IsArray = NumElems.ugt(RHS: 1);
1659 OptPrimType ElemT = S.getContext().classify(T: ElemType);
1660 DynamicAllocator &Allocator = S.getAllocator();
1661 if (ElemT) {
1662 Block *B =
1663 Allocator.allocate(Source: NewCall, T: *ElemT, NumElements: NumElems.getZExtValue(),
1664 EvalID: S.Ctx.getEvalID(), AllocForm: DynamicAllocator::Form::Operator);
1665 assert(B);
1666 S.Stk.push<Pointer>(Args: Pointer(B).atIndex(Idx: 0));
1667 return true;
1668 }
1669
1670 assert(!ElemT);
1671
1672 // Composite arrays
1673 if (IsArray) {
1674 const Descriptor *Desc =
1675 S.P.createDescriptor(D: NewCall, Ty: ElemType.getTypePtr());
1676 Block *B =
1677 Allocator.allocate(D: Desc, NumElements: NumElems.getZExtValue(), EvalID: S.Ctx.getEvalID(),
1678 AllocForm: DynamicAllocator::Form::Operator);
1679 assert(B);
1680 S.Stk.push<Pointer>(Args: Pointer(B).atIndex(Idx: 0).narrow());
1681 return true;
1682 }
1683
1684 // Records. Still allocate them as single-element arrays.
1685 QualType AllocType = S.getASTContext().getConstantArrayType(
1686 EltTy: ElemType, ArySize: NumElems, SizeExpr: nullptr, ASM: ArraySizeModifier::Normal, IndexTypeQuals: 0);
1687
1688 const Descriptor *Desc =
1689 S.P.createDescriptor(D: NewCall, Ty: AllocType.getTypePtr());
1690 Block *B = Allocator.allocate(D: Desc, EvalID: S.getContext().getEvalID(),
1691 AllocForm: DynamicAllocator::Form::Operator);
1692 assert(B);
1693 S.Stk.push<Pointer>(Args: Pointer(B).atIndex(Idx: 0).narrow());
1694 return true;
1695}
1696
1697static bool interp__builtin_operator_delete(InterpState &S, CodePtr OpPC,
1698 const InterpFrame *Frame,
1699 const CallExpr *Call) {
1700 const Expr *Source = nullptr;
1701 const Block *BlockToDelete = nullptr;
1702
1703 unsigned NumArgs = Call->getNumArgs();
1704 assert(NumArgs >= 1);
1705
1706 // Args are pushed in source order. The trailing sized/aligned delete
1707 // operands are above the pointer on the stack.
1708 for (unsigned I = NumArgs - 1; I != 0; --I)
1709 discard(Stk&: S.Stk, T: *S.getContext().classify(E: Call->getArg(Arg: I)));
1710
1711 if (S.checkingPotentialConstantExpression()) {
1712 S.Stk.discard<Pointer>();
1713 return false;
1714 }
1715
1716 // This is permitted only within a call to std::allocator<T>::deallocate.
1717 if (!S.getStdAllocatorCaller(Name: "deallocate")) {
1718 S.FFDiag(E: Call);
1719 S.Stk.discard<Pointer>();
1720 return true;
1721 }
1722
1723 {
1724 const Pointer &Ptr = S.Stk.pop<Pointer>();
1725
1726 if (Ptr.isZero()) {
1727 S.CCEDiag(E: Call, DiagId: diag::note_constexpr_deallocate_null);
1728 return true;
1729 }
1730
1731 Source = Ptr.getRootExpr();
1732 BlockToDelete = Ptr.block();
1733
1734 if (!BlockToDelete->isDynamic()) {
1735 S.FFDiag(E: Call, DiagId: diag::note_constexpr_delete_not_heap_alloc)
1736 << Ptr.toDiagnosticString(Ctx: S.getASTContext());
1737 if (const auto *D = Ptr.getFieldDesc()->asDecl())
1738 S.Note(Loc: D->getLocation(), DiagId: diag::note_declared_at);
1739 }
1740 }
1741 assert(BlockToDelete);
1742
1743 DynamicAllocator &Allocator = S.getAllocator();
1744 const Descriptor *BlockDesc = BlockToDelete->getDescriptor();
1745 std::optional<DynamicAllocator::Form> AllocForm =
1746 Allocator.getAllocationForm(Source);
1747
1748 if (!Allocator.deallocate(Source, BlockToDelete)) {
1749 // Nothing has been deallocated, this must be a double-delete.
1750 const SourceInfo &Loc = S.Current->getSource(PC: OpPC);
1751 S.FFDiag(SI: Loc, DiagId: diag::note_constexpr_double_delete);
1752 return false;
1753 }
1754 assert(AllocForm);
1755
1756 return CheckNewDeleteForms(
1757 S, OpPC, AllocForm: *AllocForm, DeleteForm: DynamicAllocator::Form::Operator, D: BlockDesc, NewExpr: Source);
1758}
1759
1760static bool interp__builtin_arithmetic_fence(InterpState &S, CodePtr OpPC,
1761 const InterpFrame *Frame,
1762 const CallExpr *Call) {
1763 const Floating &Arg0 = S.Stk.pop<Floating>();
1764 S.Stk.push<Floating>(Args: Arg0);
1765 return true;
1766}
1767
1768static bool interp__builtin_vector_reduce(InterpState &S, CodePtr OpPC,
1769 const CallExpr *Call, unsigned ID) {
1770 const Pointer &Arg = S.Stk.pop<Pointer>();
1771 assert(Arg.getFieldDesc()->isPrimitiveArray());
1772
1773 QualType ElemType = Arg.getFieldDesc()->getElemQualType();
1774 assert(Call->getType() == ElemType);
1775 PrimType ElemT = *S.getContext().classify(T: ElemType);
1776 unsigned NumElems = Arg.getNumElems();
1777
1778 if (!isIntegerType(T: ElemT))
1779 return false;
1780
1781 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
1782 T Result = Arg.elem<T>(0);
1783 unsigned BitWidth = Result.bitWidth();
1784 for (unsigned I = 1; I != NumElems; ++I) {
1785 T Elem = Arg.elem<T>(I);
1786 T PrevResult = Result;
1787
1788 if (ID == Builtin::BI__builtin_reduce_add) {
1789 if (T::add(Result, Elem, BitWidth, &Result)) {
1790 unsigned OverflowBits = BitWidth + 1;
1791 (void)handleOverflow(S, OpPC,
1792 (PrevResult.toAPSInt(OverflowBits) +
1793 Elem.toAPSInt(OverflowBits)));
1794 return false;
1795 }
1796 } else if (ID == Builtin::BI__builtin_reduce_mul) {
1797 if (T::mul(Result, Elem, BitWidth, &Result)) {
1798 unsigned OverflowBits = BitWidth * 2;
1799 (void)handleOverflow(S, OpPC,
1800 (PrevResult.toAPSInt(OverflowBits) *
1801 Elem.toAPSInt(OverflowBits)));
1802 return false;
1803 }
1804
1805 } else if (ID == Builtin::BI__builtin_reduce_and) {
1806 (void)T::bitAnd(Result, Elem, BitWidth, &Result);
1807 } else if (ID == Builtin::BI__builtin_reduce_or) {
1808 (void)T::bitOr(Result, Elem, BitWidth, &Result);
1809 } else if (ID == Builtin::BI__builtin_reduce_xor) {
1810 (void)T::bitXor(Result, Elem, BitWidth, &Result);
1811 } else if (ID == Builtin::BI__builtin_reduce_min) {
1812 if (Elem < Result)
1813 Result = Elem;
1814 } else if (ID == Builtin::BI__builtin_reduce_max) {
1815 if (Elem > Result)
1816 Result = Elem;
1817 } else {
1818 llvm_unreachable("Unhandled vector reduce builtin");
1819 }
1820 }
1821 pushInteger(S, Result.toAPSInt(), Call->getType());
1822 });
1823
1824 return true;
1825}
1826
1827static bool interp__builtin_elementwise_abs(InterpState &S, CodePtr OpPC,
1828 const InterpFrame *Frame,
1829 const CallExpr *Call,
1830 unsigned BuiltinID) {
1831 assert(Call->getNumArgs() == 1);
1832 QualType Ty = Call->getArg(Arg: 0)->getType();
1833 if (Ty->isIntegerType()) {
1834 APSInt Val;
1835 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: Val))
1836 return false;
1837 pushInteger(S, Val: Val.abs(), QT: Call->getType());
1838 return true;
1839 }
1840
1841 if (Ty->isFloatingType()) {
1842 Floating Val = S.Stk.pop<Floating>();
1843 Floating Result = abs(S, In: Val);
1844 S.Stk.push<Floating>(Args&: Result);
1845 return true;
1846 }
1847
1848 // Otherwise, the argument must be a vector.
1849 assert(Call->getArg(0)->getType()->isVectorType());
1850 const Pointer &Arg = S.Stk.pop<Pointer>();
1851 assert(Arg.getFieldDesc()->isPrimitiveArray());
1852 const Pointer &Dst = S.Stk.peek<Pointer>();
1853 assert(Dst.getFieldDesc()->isPrimitiveArray());
1854 assert(Arg.getFieldDesc()->getNumElems() ==
1855 Dst.getFieldDesc()->getNumElems());
1856
1857 QualType ElemType = Arg.getFieldDesc()->getElemQualType();
1858 PrimType ElemT = *S.getContext().classify(T: ElemType);
1859 unsigned NumElems = Arg.getNumElems();
1860 // we can either have a vector of integer or a vector of floating point
1861 for (unsigned I = 0; I != NumElems; ++I) {
1862 if (ElemType->isIntegerType()) {
1863 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
1864 Dst.elem<T>(I) = T::from(static_cast<T>(
1865 APSInt(Arg.elem<T>(I).toAPSInt().abs(),
1866 ElemType->isUnsignedIntegerOrEnumerationType())));
1867 });
1868 } else {
1869 Floating Val = Arg.elem<Floating>(I);
1870 Dst.elem<Floating>(I) = abs(S, In: Val);
1871 }
1872 }
1873 Dst.initializeAllElements();
1874
1875 return true;
1876}
1877
1878/// Can be called with an integer or vector as the first and only parameter.
1879static bool interp__builtin_elementwise_countzeroes(InterpState &S,
1880 CodePtr OpPC,
1881 const InterpFrame *Frame,
1882 const CallExpr *Call,
1883 unsigned BuiltinID) {
1884 bool HasZeroArg = Call->getNumArgs() == 2;
1885 bool IsCTTZ = BuiltinID == Builtin::BI__builtin_elementwise_ctzg;
1886 assert(Call->getNumArgs() == 1 || HasZeroArg);
1887 if (Call->getArg(Arg: 0)->getType()->isIntegerType()) {
1888 PrimType ArgT = *S.getContext().classify(T: Call->getArg(Arg: 0)->getType());
1889 APSInt Val;
1890 if (!popToAPSInt(Stk&: S.Stk, T: ArgT, Out&: Val))
1891 return false;
1892 std::optional<APSInt> ZeroVal;
1893 if (HasZeroArg) {
1894 ZeroVal = Val;
1895 if (!popToAPSInt(Stk&: S.Stk, T: ArgT, Out&: Val))
1896 return false;
1897 }
1898
1899 if (Val.isZero()) {
1900 if (ZeroVal) {
1901 pushInteger(S, Val: *ZeroVal, QT: Call->getType());
1902 return true;
1903 }
1904 // If we haven't been provided the second argument, the result is
1905 // undefined
1906 S.FFDiag(SI: S.Current->getSource(PC: OpPC),
1907 DiagId: diag::note_constexpr_countzeroes_zero)
1908 << /*IsTrailing=*/IsCTTZ;
1909 return false;
1910 }
1911
1912 if (BuiltinID == Builtin::BI__builtin_elementwise_clzg) {
1913 pushInteger(S, Val: Val.countLeadingZeros(), QT: Call->getType());
1914 } else {
1915 pushInteger(S, Val: Val.countTrailingZeros(), QT: Call->getType());
1916 }
1917 return true;
1918 }
1919 // Otherwise, the argument must be a vector.
1920 const ASTContext &ASTCtx = S.getASTContext();
1921 Pointer ZeroArg;
1922 if (HasZeroArg) {
1923 assert(Call->getArg(1)->getType()->isVectorType() &&
1924 ASTCtx.hasSameUnqualifiedType(Call->getArg(0)->getType(),
1925 Call->getArg(1)->getType()));
1926 (void)ASTCtx;
1927 ZeroArg = S.Stk.pop<Pointer>();
1928 assert(ZeroArg.getFieldDesc()->isPrimitiveArray());
1929 }
1930 assert(Call->getArg(0)->getType()->isVectorType());
1931 const Pointer &Arg = S.Stk.pop<Pointer>();
1932 assert(Arg.getFieldDesc()->isPrimitiveArray());
1933 const Pointer &Dst = S.Stk.peek<Pointer>();
1934 assert(Dst.getFieldDesc()->isPrimitiveArray());
1935 assert(Arg.getFieldDesc()->getNumElems() ==
1936 Dst.getFieldDesc()->getNumElems());
1937
1938 QualType ElemType = Arg.getFieldDesc()->getElemQualType();
1939 PrimType ElemT = *S.getContext().classify(T: ElemType);
1940 unsigned NumElems = Arg.getNumElems();
1941
1942 // FIXME: Reading from uninitialized vector elements?
1943 for (unsigned I = 0; I != NumElems; ++I) {
1944 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
1945 APInt EltVal = Arg.atIndex(I).deref<T>().toAPSInt();
1946 if (EltVal.isZero()) {
1947 if (HasZeroArg) {
1948 Dst.atIndex(I).deref<T>() = ZeroArg.atIndex(I).deref<T>();
1949 } else {
1950 // If we haven't been provided the second argument, the result is
1951 // undefined
1952 S.FFDiag(S.Current->getSource(OpPC),
1953 diag::note_constexpr_countzeroes_zero)
1954 << /*IsTrailing=*/IsCTTZ;
1955 return false;
1956 }
1957 } else if (IsCTTZ) {
1958 Dst.atIndex(I).deref<T>() = T::from(EltVal.countTrailingZeros());
1959 } else {
1960 Dst.atIndex(I).deref<T>() = T::from(EltVal.countLeadingZeros());
1961 }
1962 Dst.atIndex(I).initialize();
1963 });
1964 }
1965
1966 return true;
1967}
1968
1969static bool interp__builtin_memcpy(InterpState &S, CodePtr OpPC,
1970 const InterpFrame *Frame,
1971 const CallExpr *Call, unsigned ID) {
1972 assert(Call->getNumArgs() == 3);
1973 const ASTContext &ASTCtx = S.getASTContext();
1974 uint64_t Size;
1975 if (!popToUInt64(S, E: Call->getArg(Arg: 2), Out&: Size))
1976 return false;
1977 Pointer SrcPtr = S.Stk.pop<Pointer>().expand();
1978 Pointer DestPtr = S.Stk.pop<Pointer>().expand();
1979
1980 if (ID == Builtin::BImemcpy || ID == Builtin::BImemmove)
1981 diagnoseNonConstexprBuiltin(S, OpPC, ID);
1982
1983 bool Move =
1984 (ID == Builtin::BI__builtin_memmove || ID == Builtin::BImemmove ||
1985 ID == Builtin::BI__builtin_wmemmove || ID == Builtin::BIwmemmove);
1986 bool WChar = ID == Builtin::BIwmemcpy || ID == Builtin::BIwmemmove ||
1987 ID == Builtin::BI__builtin_wmemcpy ||
1988 ID == Builtin::BI__builtin_wmemmove;
1989
1990 // If the size is zero, we treat this as always being a valid no-op.
1991 if (Size == 0) {
1992 S.Stk.push<Pointer>(Args&: DestPtr);
1993 return true;
1994 }
1995
1996 if (SrcPtr.isZero() || DestPtr.isZero()) {
1997 Pointer DiagPtr = (SrcPtr.isZero() ? SrcPtr : DestPtr);
1998 S.FFDiag(SI: S.Current->getSource(PC: OpPC), DiagId: diag::note_constexpr_memcpy_null)
1999 << /*IsMove=*/Move << /*IsWchar=*/WChar << !SrcPtr.isZero()
2000 << DiagPtr.toDiagnosticString(Ctx: ASTCtx);
2001 return false;
2002 }
2003
2004 // Diagnose integral src/dest pointers specially.
2005 if (SrcPtr.isIntegralPointer() || DestPtr.isIntegralPointer()) {
2006 std::string DiagVal = "(void *)";
2007 DiagVal += SrcPtr.isIntegralPointer()
2008 ? std::to_string(val: SrcPtr.getIntegerRepresentation())
2009 : std::to_string(val: DestPtr.getIntegerRepresentation());
2010 S.FFDiag(SI: S.Current->getSource(PC: OpPC), DiagId: diag::note_constexpr_memcpy_null)
2011 << Move << WChar << DestPtr.isIntegralPointer() << DiagVal;
2012 return false;
2013 }
2014
2015 if (!isReadable(P: DestPtr) || !isReadable(P: SrcPtr))
2016 return false;
2017
2018 if (DestPtr.getType()->isIncompleteType()) {
2019 S.FFDiag(SI: S.Current->getSource(PC: OpPC),
2020 DiagId: diag::note_constexpr_memcpy_incomplete_type)
2021 << Move << DestPtr.getType();
2022 return false;
2023 }
2024 if (SrcPtr.getType()->isIncompleteType()) {
2025 S.FFDiag(SI: S.Current->getSource(PC: OpPC),
2026 DiagId: diag::note_constexpr_memcpy_incomplete_type)
2027 << Move << SrcPtr.getType();
2028 return false;
2029 }
2030
2031 QualType DestElemType = getElemType(P: DestPtr);
2032 if (DestElemType->isIncompleteType()) {
2033 S.FFDiag(SI: S.Current->getSource(PC: OpPC),
2034 DiagId: diag::note_constexpr_memcpy_incomplete_type)
2035 << Move << DestElemType;
2036 return false;
2037 }
2038
2039 size_t RemainingDestElems;
2040 if (DestPtr.inArray()) {
2041 RemainingDestElems = DestPtr.isUnknownSizeArray()
2042 ? 0
2043 : (DestPtr.getNumElems() - DestPtr.getIndex());
2044 } else {
2045 RemainingDestElems = 1;
2046 }
2047 unsigned DestElemSize = ASTCtx.getTypeSizeInChars(T: DestElemType).getQuantity();
2048
2049 if (WChar) {
2050 uint64_t WCharSize =
2051 ASTCtx.getTypeSizeInChars(T: ASTCtx.getWCharType()).getQuantity();
2052 Size *= WCharSize;
2053 }
2054
2055 if (Size % DestElemSize != 0) {
2056 S.FFDiag(SI: S.Current->getSource(PC: OpPC),
2057 DiagId: diag::note_constexpr_memcpy_unsupported)
2058 << Move << WChar << 0 << DestElemType << Size << DestElemSize;
2059 return false;
2060 }
2061
2062 QualType SrcElemType = getElemType(P: SrcPtr);
2063 size_t RemainingSrcElems;
2064 if (SrcPtr.inArray()) {
2065 RemainingSrcElems = SrcPtr.isUnknownSizeArray()
2066 ? 0
2067 : (SrcPtr.getNumElems() - SrcPtr.getIndex());
2068 } else {
2069 RemainingSrcElems = 1;
2070 }
2071 unsigned SrcElemSize = ASTCtx.getTypeSizeInChars(T: SrcElemType).getQuantity();
2072
2073 if (!ASTCtx.hasSameUnqualifiedType(T1: DestElemType, T2: SrcElemType)) {
2074 S.FFDiag(SI: S.Current->getSource(PC: OpPC), DiagId: diag::note_constexpr_memcpy_type_pun)
2075 << Move << SrcElemType << DestElemType;
2076 return false;
2077 }
2078
2079 if (!DestElemType.isTriviallyCopyableType(Context: ASTCtx)) {
2080 S.FFDiag(SI: S.Current->getSource(PC: OpPC), DiagId: diag::note_constexpr_memcpy_nontrivial)
2081 << Move << DestElemType;
2082 return false;
2083 }
2084
2085 // Check if we have enough elements to read from and write to.
2086 size_t RemainingDestBytes = RemainingDestElems * DestElemSize;
2087 size_t RemainingSrcBytes = RemainingSrcElems * SrcElemSize;
2088 if (Size > RemainingDestBytes || Size > RemainingSrcBytes) {
2089 APInt N = APInt(64, Size / DestElemSize);
2090 S.FFDiag(SI: S.Current->getSource(PC: OpPC),
2091 DiagId: diag::note_constexpr_memcpy_unsupported)
2092 << Move << WChar << (Size > RemainingSrcBytes ? 1 : 2) << DestElemType
2093 << toString(I: N, Radix: 10, /*Signed=*/false);
2094 return false;
2095 }
2096
2097 // Check for overlapping memory regions.
2098 if (!Move && Pointer::pointToSameBlock(A: SrcPtr, B: DestPtr)) {
2099 // Remove base casts.
2100 Pointer SrcP = SrcPtr.stripBaseCasts();
2101 Pointer DestP = DestPtr.stripBaseCasts();
2102
2103 unsigned SrcIndex = SrcP.expand().getIndex() * SrcElemSize;
2104 unsigned DstIndex = DestP.expand().getIndex() * DestElemSize;
2105
2106 if ((SrcIndex <= DstIndex && (SrcIndex + Size) > DstIndex) ||
2107 (DstIndex <= SrcIndex && (DstIndex + Size) > SrcIndex)) {
2108 S.FFDiag(SI: S.Current->getSource(PC: OpPC), DiagId: diag::note_constexpr_memcpy_overlap)
2109 << /*IsWChar=*/false;
2110 return false;
2111 }
2112 }
2113
2114 assert(Size % DestElemSize == 0);
2115 if (!DoMemcpy(S, OpPC, SrcPtr, DestPtr, Size: Bytes(Size).toBits()))
2116 return false;
2117
2118 S.Stk.push<Pointer>(Args&: DestPtr);
2119 return true;
2120}
2121
2122/// Determine if T is a character type for which we guarantee that
2123/// sizeof(T) == 1.
2124static bool isOneByteCharacterType(QualType T) {
2125 return T->isCharType() || T->isChar8Type();
2126}
2127
2128static bool interp__builtin_memcmp(InterpState &S, CodePtr OpPC,
2129 const InterpFrame *Frame,
2130 const CallExpr *Call, unsigned ID) {
2131 assert(Call->getNumArgs() == 3);
2132 uint64_t Size;
2133 if (!popToUInt64(S, E: Call->getArg(Arg: 2), Out&: Size))
2134 return false;
2135 const Pointer &PtrB = S.Stk.pop<Pointer>();
2136 const Pointer &PtrA = S.Stk.pop<Pointer>();
2137
2138 if (ID == Builtin::BImemcmp || ID == Builtin::BIbcmp ||
2139 ID == Builtin::BIwmemcmp)
2140 diagnoseNonConstexprBuiltin(S, OpPC, ID);
2141
2142 if (Size == 0) {
2143 pushInteger(S, Val: 0, QT: Call->getType());
2144 return true;
2145 }
2146 bool IsWide =
2147 (ID == Builtin::BIwmemcmp || ID == Builtin::BI__builtin_wmemcmp);
2148
2149 const ASTContext &ASTCtx = S.getASTContext();
2150 QualType ElemTypeA = getElemType(P: PtrA);
2151 QualType ElemTypeB = getElemType(P: PtrB);
2152 // FIXME: This is an arbitrary limitation the current constant interpreter
2153 // had. We could remove this.
2154 if (!IsWide && (!isOneByteCharacterType(T: ElemTypeA) ||
2155 !isOneByteCharacterType(T: ElemTypeB))) {
2156 S.FFDiag(SI: S.Current->getSource(PC: OpPC),
2157 DiagId: diag::note_constexpr_memcmp_unsupported)
2158 << ASTCtx.BuiltinInfo.getQuotedName(ID) << PtrA.getType()
2159 << PtrB.getType();
2160 return false;
2161 }
2162
2163 if (!PtrA.isReadablePointerType() || !PtrB.isReadablePointerType())
2164 return false;
2165
2166 if (!CheckLoad(S, OpPC, Ptr: PtrA, AK: AK_Read) || !CheckLoad(S, OpPC, Ptr: PtrB, AK: AK_Read))
2167 return false;
2168
2169 // Now, read both pointers to a buffer and compare those.
2170 BitcastBuffer BufferA(
2171 Bits(ASTCtx.getTypeSize(T: ElemTypeA) * PtrA.getNumElems()));
2172 readPointerToBuffer(Ctx: S.getContext(), FromPtr: PtrA, Buffer&: BufferA, /*ReturnOnUninit=*/false);
2173
2174 // FIXME: The swapping here is UNDOING something we do when reading the
2175 // data into the buffer.
2176 if (ASTCtx.getTargetInfo().isBigEndian())
2177 swapBytes(M: BufferA.Data.get(), N: BufferA.byteSize().getQuantity());
2178
2179 BitcastBuffer BufferB(
2180 Bits(ASTCtx.getTypeSize(T: ElemTypeB) * PtrB.getNumElems()));
2181 readPointerToBuffer(Ctx: S.getContext(), FromPtr: PtrB, Buffer&: BufferB, /*ReturnOnUninit=*/false);
2182 // FIXME: The swapping here is UNDOING something we do when reading the
2183 // data into the buffer.
2184 if (ASTCtx.getTargetInfo().isBigEndian())
2185 swapBytes(M: BufferB.Data.get(), N: BufferB.byteSize().getQuantity());
2186
2187 size_t MinBufferSize = std::min(a: BufferA.byteSize().getQuantity(),
2188 b: BufferB.byteSize().getQuantity());
2189
2190 unsigned ElemSize = 1;
2191 if (IsWide)
2192 ElemSize = ASTCtx.getTypeSizeInChars(T: ASTCtx.getWCharType()).getQuantity();
2193 // The Size given for the wide variants is in wide-char units. Convert it
2194 // to bytes.
2195 size_t ByteSize = Size * ElemSize;
2196 size_t CmpSize = std::min(a: MinBufferSize, b: ByteSize);
2197
2198 for (size_t I = 0; I != CmpSize; I += ElemSize) {
2199 if (IsWide) {
2200 FIXED_SIZE_INT_TYPE_SWITCH(
2201 *S.getContext().classify(ASTCtx.getWCharType()), {
2202 T A = T::bitcastFromMemory(BufferA.atByte(I), T::bitWidth());
2203 T B = T::bitcastFromMemory(BufferB.atByte(I), T::bitWidth());
2204 if (A < B) {
2205 pushInteger(S, -1, Call->getType());
2206 return true;
2207 }
2208 if (A > B) {
2209 pushInteger(S, 1, Call->getType());
2210 return true;
2211 }
2212 });
2213 } else {
2214 auto A = BufferA.deref<std::byte>(Offset: Bytes(I));
2215 auto B = BufferB.deref<std::byte>(Offset: Bytes(I));
2216
2217 if (A < B) {
2218 pushInteger(S, Val: -1, QT: Call->getType());
2219 return true;
2220 }
2221 if (A > B) {
2222 pushInteger(S, Val: 1, QT: Call->getType());
2223 return true;
2224 }
2225 }
2226 }
2227
2228 // We compared CmpSize bytes above. If the limiting factor was the Size
2229 // passed, we're done and the result is equality (0).
2230 if (ByteSize <= CmpSize) {
2231 pushInteger(S, Val: 0, QT: Call->getType());
2232 return true;
2233 }
2234
2235 // However, if we read all the available bytes but were instructed to read
2236 // even more, diagnose this as a "read of dereferenced one-past-the-end
2237 // pointer". This is what would happen if we called CheckLoad() on every array
2238 // element.
2239 S.FFDiag(SI: S.Current->getSource(PC: OpPC), DiagId: diag::note_constexpr_access_past_end)
2240 << AK_Read << S.Current->getRange(PC: OpPC);
2241 return false;
2242}
2243
2244// __builtin_memchr(ptr, int, int)
2245// __builtin_strchr(ptr, int)
2246static bool interp__builtin_memchr(InterpState &S, CodePtr OpPC,
2247 const CallExpr *Call, unsigned ID) {
2248 if (ID == Builtin::BImemchr || ID == Builtin::BIwcschr ||
2249 ID == Builtin::BIstrchr || ID == Builtin::BIwmemchr)
2250 diagnoseNonConstexprBuiltin(S, OpPC, ID);
2251
2252 std::optional<APSInt> MaxLength;
2253 if (Call->getNumArgs() == 3) {
2254 APSInt MaxLengthVal;
2255 if (!popToAPSInt(S, E: Call->getArg(Arg: 2), Out&: MaxLengthVal))
2256 return false;
2257 MaxLength = MaxLengthVal;
2258 }
2259
2260 APSInt Desired;
2261 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: Desired))
2262 return false;
2263 const Pointer &Ptr = S.Stk.pop<Pointer>();
2264
2265 if (MaxLength && MaxLength->isZero()) {
2266 S.Stk.push<Pointer>();
2267 return true;
2268 }
2269
2270 if (Ptr.isDummy()) {
2271 if (Ptr.getType()->isIncompleteType())
2272 S.FFDiag(SI: S.Current->getSource(PC: OpPC),
2273 DiagId: diag::note_constexpr_ltor_incomplete_type)
2274 << Ptr.getType();
2275 return false;
2276 }
2277
2278 // Null is only okay if the given size is 0.
2279 if (Ptr.isZero()) {
2280 S.FFDiag(SI: S.Current->getSource(PC: OpPC), DiagId: diag::note_constexpr_access_null)
2281 << AK_Read;
2282 return false;
2283 }
2284
2285 if (!Ptr.isReadablePointerType())
2286 return false;
2287
2288 QualType ElemTy = getElemType(P: Ptr);
2289 bool IsRawByte = ID == Builtin::BImemchr || ID == Builtin::BI__builtin_memchr;
2290
2291 // Give up on byte-oriented matching against multibyte elements.
2292 if (IsRawByte && !isOneByteCharacterType(T: ElemTy)) {
2293 S.FFDiag(SI: S.Current->getSource(PC: OpPC),
2294 DiagId: diag::note_constexpr_memchr_unsupported)
2295 << S.getASTContext().BuiltinInfo.getQuotedName(ID) << ElemTy;
2296 return false;
2297 }
2298
2299 if (!isReadable(P: Ptr))
2300 return false;
2301
2302 if (ID == Builtin::BIstrchr || ID == Builtin::BI__builtin_strchr) {
2303 int64_t DesiredTrunc;
2304 if (S.getASTContext().CharTy->isSignedIntegerType())
2305 DesiredTrunc =
2306 Desired.trunc(width: S.getASTContext().getCharWidth()).getSExtValue();
2307 else
2308 DesiredTrunc =
2309 Desired.trunc(width: S.getASTContext().getCharWidth()).getZExtValue();
2310 // strchr compares directly to the passed integer, and therefore
2311 // always fails if given an int that is not a char.
2312 if (Desired != DesiredTrunc) {
2313 S.Stk.push<Pointer>();
2314 return true;
2315 }
2316 }
2317
2318 uint64_t DesiredVal;
2319 if (ID == Builtin::BIwmemchr || ID == Builtin::BI__builtin_wmemchr ||
2320 ID == Builtin::BIwcschr || ID == Builtin::BI__builtin_wcschr) {
2321 // wcschr and wmemchr are given a wchar_t to look for. Just use it.
2322 DesiredVal = Desired.getZExtValue();
2323 } else {
2324 DesiredVal = Desired.trunc(width: S.getASTContext().getCharWidth()).getZExtValue();
2325 }
2326
2327 bool StopAtZero =
2328 (ID == Builtin::BIstrchr || ID == Builtin::BI__builtin_strchr ||
2329 ID == Builtin::BIwcschr || ID == Builtin::BI__builtin_wcschr);
2330
2331 PrimType ElemT =
2332 IsRawByte ? PT_Sint8 : *S.getContext().classify(T: getElemType(P: Ptr));
2333
2334 size_t Index = Ptr.getIndex();
2335 size_t Step = 0;
2336 for (;;) {
2337 const Pointer &ElemPtr =
2338 (Index + Step) > 0 ? Ptr.atIndex(Idx: Index + Step) : Ptr;
2339
2340 if (!CheckLoad(S, OpPC, Ptr: ElemPtr))
2341 return false;
2342
2343 uint64_t V;
2344 INT_TYPE_SWITCH_NO_BOOL(
2345 ElemT, { V = static_cast<uint64_t>(ElemPtr.load<T>().toUnsigned()); });
2346
2347 if (V == DesiredVal) {
2348 S.Stk.push<Pointer>(Args: ElemPtr);
2349 return true;
2350 }
2351
2352 if (StopAtZero && V == 0)
2353 break;
2354
2355 ++Step;
2356 if (MaxLength && Step == MaxLength->getZExtValue())
2357 break;
2358 }
2359
2360 S.Stk.push<Pointer>();
2361 return true;
2362}
2363
2364static bool interp__builtin_object_size(InterpState &S, CodePtr OpPC,
2365 const InterpFrame *Frame,
2366 const CallExpr *Call, bool IsDynamic) {
2367 const ASTContext &ASTCtx = S.getASTContext();
2368 // From the GCC docs:
2369 // Kind is an integer constant from 0 to 3. If the least significant bit is
2370 // clear, objects are whole variables. If it is set, a closest surrounding
2371 // subobject is considered the object a pointer points to. The second bit
2372 // determines if maximum or minimum of remaining bytes is computed.
2373 uint64_t Kind;
2374 if (!popToUInt64(S, E: Call->getArg(Arg: 1), Out&: Kind))
2375 return false;
2376 assert(Kind <= 3 && "unexpected kind");
2377 Pointer Ptr = S.Stk.pop<Pointer>();
2378
2379 if (auto Result = evaluateBuiltinObjectSize(ASTCtx, Kind, Ptr,
2380 E: Call->getArg(Arg: 0), IsDynamic)) {
2381 pushInteger(S, Val: *Result, QT: Call->getType());
2382 return true;
2383 }
2384
2385 if (Call->getArg(Arg: 0)->HasSideEffects(Ctx: ASTCtx)) {
2386 // "If there are any side effects in them, it returns (size_t) -1
2387 // for type 0 or 1 and (size_t) 0 for type 2 or 3."
2388 pushInteger(S, Val: Kind <= 1 ? (size_t)-1 : (size_t)0, QT: Call->getType());
2389 return true;
2390 }
2391
2392 switch (S.EvalMode) {
2393 case EvaluationMode::ConstantExpression:
2394 case EvaluationMode::ConstantFold:
2395 case EvaluationMode::IgnoreSideEffects:
2396 // Leave it to IR generation.
2397 return Invalid(S, OpPC);
2398 case EvaluationMode::ConstantExpressionUnevaluated:
2399 // Reduce it to a constant now.
2400 pushInteger(S, Val: ((Kind & 2u) ? (size_t)0 : (size_t)-1), QT: Call->getType());
2401 return true;
2402 }
2403
2404 return false;
2405}
2406
2407static bool interp__builtin_is_within_lifetime(InterpState &S, CodePtr OpPC,
2408 const CallExpr *Call) {
2409
2410 if (!S.inConstantContext())
2411 return false;
2412
2413 const Pointer &Ptr = S.Stk.pop<Pointer>();
2414
2415 auto Error = [&](int Diag) {
2416 bool CalledFromStd = false;
2417 const auto *Callee = S.Current->getCallee();
2418 if (Callee && Callee->isInStdNamespace()) {
2419 const IdentifierInfo *Identifier = Callee->getIdentifier();
2420 CalledFromStd = Identifier && Identifier->isStr(Str: "is_within_lifetime");
2421 }
2422 S.CCEDiag(SI: CalledFromStd
2423 ? S.Current->Caller->getSource(PC: S.Current->getRetPC())
2424 : S.Current->getSource(PC: OpPC),
2425 DiagId: diag::err_invalid_is_within_lifetime)
2426 << (CalledFromStd ? "std::is_within_lifetime"
2427 : "__builtin_is_within_lifetime")
2428 << Diag;
2429 return false;
2430 };
2431
2432 if (Ptr.isZero())
2433 return Error(0);
2434 if (Ptr.isOnePastEnd())
2435 return Error(1);
2436
2437 bool Result = Ptr.getLifetime() != Lifetime::Ended;
2438 if (!Ptr.isActive()) {
2439 Result = false;
2440 } else {
2441 if (!CheckLive(S, OpPC, Ptr, AK: AK_Read))
2442 return false;
2443 if (!CheckMutable(S, OpPC, Ptr))
2444 return false;
2445 if (!CheckDummy(S, OpPC, Ptr, AK: AK_Read))
2446 return false;
2447 }
2448
2449 // Check if we're currently running an initializer.
2450 if (S.initializingBlock(B: Ptr.block()))
2451 return Error(2);
2452 if (S.EvaluatingDecl && Ptr.getRootVarDecl() == S.EvaluatingDecl)
2453 return Error(2);
2454
2455 pushInteger(S, Val: Result, QT: Call->getType());
2456 return true;
2457}
2458
2459static bool interp__builtin_elementwise_int_unaryop(
2460 InterpState &S, CodePtr OpPC, const CallExpr *Call,
2461 llvm::function_ref<APInt(const APSInt &)> Fn) {
2462 assert(Call->getNumArgs() == 1);
2463
2464 // Single integer case.
2465 if (!Call->getArg(Arg: 0)->getType()->isVectorType()) {
2466 assert(Call->getType()->isIntegerType());
2467 APSInt Src;
2468 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: Src))
2469 return false;
2470 APInt Result = Fn(Src);
2471 pushInteger(S, Val: APSInt(std::move(Result), !Src.isSigned()), QT: Call->getType());
2472 return true;
2473 }
2474
2475 // Vector case.
2476 const Pointer &Arg = S.Stk.pop<Pointer>();
2477 assert(Arg.getFieldDesc()->isPrimitiveArray());
2478 const Pointer &Dst = S.Stk.peek<Pointer>();
2479 assert(Dst.getFieldDesc()->isPrimitiveArray());
2480 assert(Arg.getFieldDesc()->getNumElems() ==
2481 Dst.getFieldDesc()->getNumElems());
2482
2483 QualType ElemType = Arg.getFieldDesc()->getElemQualType();
2484 PrimType ElemT = *S.getContext().classify(T: ElemType);
2485 unsigned NumElems = Arg.getNumElems();
2486 bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
2487
2488 for (unsigned I = 0; I != NumElems; ++I) {
2489 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
2490 APSInt Src = Arg.elem<T>(I).toAPSInt();
2491 APInt Result = Fn(Src);
2492 Dst.elem<T>(I) = static_cast<T>(APSInt(std::move(Result), DestUnsigned));
2493 });
2494 }
2495 Dst.initializeAllElements();
2496
2497 return true;
2498}
2499
2500static bool interp__builtin_elementwise_fp_binop(
2501 InterpState &S, CodePtr OpPC, const CallExpr *Call,
2502 llvm::function_ref<std::optional<APFloat>(
2503 const APFloat &, const APFloat &, std::optional<APSInt> RoundingMode)>
2504 Fn,
2505 bool IsScalar = false) {
2506 assert((Call->getNumArgs() == 2) || (Call->getNumArgs() == 3));
2507 const auto *VT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
2508 assert(VT->getElementType()->isFloatingType());
2509 unsigned NumElems = VT->getNumElements();
2510
2511 // Vector case.
2512 assert(Call->getArg(0)->getType()->isVectorType() &&
2513 Call->getArg(1)->getType()->isVectorType());
2514 assert(VT->getElementType() ==
2515 Call->getArg(1)->getType()->castAs<VectorType>()->getElementType());
2516 assert(VT->getNumElements() ==
2517 Call->getArg(1)->getType()->castAs<VectorType>()->getNumElements());
2518
2519 std::optional<APSInt> RoundingMode = std::nullopt;
2520 if (Call->getNumArgs() == 3) {
2521 APSInt RoundingModeVal;
2522 if (!popToAPSInt(S, E: Call->getArg(Arg: 2), Out&: RoundingModeVal))
2523 return false;
2524 RoundingMode = RoundingModeVal;
2525 }
2526
2527 const Pointer &BPtr = S.Stk.pop<Pointer>();
2528 const Pointer &APtr = S.Stk.pop<Pointer>();
2529 const Pointer &Dst = S.Stk.peek<Pointer>();
2530 for (unsigned ElemIdx = 0; ElemIdx != NumElems; ++ElemIdx) {
2531 using T = PrimConv<PT_Float>::T;
2532 if (IsScalar && ElemIdx > 0) {
2533 Dst.elem<T>(I: ElemIdx) = APtr.elem<T>(I: ElemIdx);
2534 continue;
2535 }
2536 APFloat ElemA = APtr.elem<T>(I: ElemIdx).getAPFloat();
2537 APFloat ElemB = BPtr.elem<T>(I: ElemIdx).getAPFloat();
2538 std::optional<APFloat> Result = Fn(ElemA, ElemB, RoundingMode);
2539 if (!Result)
2540 return false;
2541 Dst.elem<T>(I: ElemIdx) = static_cast<T>(*Result);
2542 }
2543
2544 Dst.initializeAllElements();
2545
2546 return true;
2547}
2548
2549static bool interp__builtin_scalar_fp_round_mask_binop(
2550 InterpState &S, CodePtr OpPC, const CallExpr *Call,
2551 llvm::function_ref<std::optional<APFloat>(const APFloat &, const APFloat &,
2552 std::optional<APSInt>)>
2553 Fn) {
2554 assert(Call->getNumArgs() == 5);
2555 const auto *VT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
2556 unsigned NumElems = VT->getNumElements();
2557
2558 APSInt RoundingMode;
2559 if (!popToAPSInt(S, E: Call->getArg(Arg: 4), Out&: RoundingMode))
2560 return false;
2561 uint64_t MaskVal;
2562 if (!popToUInt64(S, E: Call->getArg(Arg: 3), Out&: MaskVal))
2563 return false;
2564 const Pointer &SrcPtr = S.Stk.pop<Pointer>();
2565 const Pointer &BPtr = S.Stk.pop<Pointer>();
2566 const Pointer &APtr = S.Stk.pop<Pointer>();
2567 const Pointer &Dst = S.Stk.peek<Pointer>();
2568
2569 using T = PrimConv<PT_Float>::T;
2570
2571 if (MaskVal & 1) {
2572 APFloat ElemA = APtr.elem<T>(I: 0).getAPFloat();
2573 APFloat ElemB = BPtr.elem<T>(I: 0).getAPFloat();
2574 std::optional<APFloat> Result = Fn(ElemA, ElemB, RoundingMode);
2575 if (!Result)
2576 return false;
2577 Dst.elem<T>(I: 0) = static_cast<T>(*Result);
2578 } else {
2579 Dst.elem<T>(I: 0) = SrcPtr.elem<T>(I: 0);
2580 }
2581
2582 for (unsigned I = 1; I < NumElems; ++I)
2583 Dst.elem<T>(I) = APtr.elem<T>(I);
2584
2585 Dst.initializeAllElements();
2586
2587 return true;
2588}
2589
2590static bool interp__builtin_elementwise_int_binop(
2591 InterpState &S, CodePtr OpPC, const CallExpr *Call,
2592 llvm::function_ref<APInt(const APSInt &, const APSInt &)> Fn) {
2593 assert(Call->getNumArgs() == 2);
2594
2595 // Single integer case.
2596 if (!Call->getArg(Arg: 0)->getType()->isVectorType()) {
2597 assert(!Call->getArg(1)->getType()->isVectorType());
2598 APSInt RHS;
2599 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: RHS))
2600 return false;
2601 APSInt LHS;
2602 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: LHS))
2603 return false;
2604 APInt Result = Fn(LHS, RHS);
2605 pushInteger(S, Val: APSInt(std::move(Result), !LHS.isSigned()), QT: Call->getType());
2606 return true;
2607 }
2608
2609 const auto *VT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
2610 assert(VT->getElementType()->isIntegralOrEnumerationType());
2611 PrimType ElemT = *S.getContext().classify(T: VT->getElementType());
2612 unsigned NumElems = VT->getNumElements();
2613 bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
2614
2615 // Vector + Scalar case.
2616 if (!Call->getArg(Arg: 1)->getType()->isVectorType()) {
2617 assert(Call->getArg(1)->getType()->isIntegralOrEnumerationType());
2618
2619 APSInt RHS;
2620 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: RHS))
2621 return false;
2622 const Pointer &LHS = S.Stk.pop<Pointer>();
2623 const Pointer &Dst = S.Stk.peek<Pointer>();
2624
2625 for (unsigned I = 0; I != NumElems; ++I) {
2626 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
2627 Dst.elem<T>(I) = static_cast<T>(
2628 APSInt(Fn(LHS.elem<T>(I).toAPSInt(), RHS), DestUnsigned));
2629 });
2630 }
2631 Dst.initializeAllElements();
2632 return true;
2633 }
2634
2635 // Vector case.
2636 assert(Call->getArg(0)->getType()->isVectorType() &&
2637 Call->getArg(1)->getType()->isVectorType());
2638 assert(VT->getElementType() ==
2639 Call->getArg(1)->getType()->castAs<VectorType>()->getElementType());
2640 assert(VT->getNumElements() ==
2641 Call->getArg(1)->getType()->castAs<VectorType>()->getNumElements());
2642 assert(VT->getElementType()->isIntegralOrEnumerationType());
2643
2644 const Pointer &RHS = S.Stk.pop<Pointer>();
2645 const Pointer &LHS = S.Stk.pop<Pointer>();
2646 const Pointer &Dst = S.Stk.peek<Pointer>();
2647 for (unsigned I = 0; I != NumElems; ++I) {
2648 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
2649 APSInt Elem1 = LHS.elem<T>(I).toAPSInt();
2650 APSInt Elem2 = RHS.elem<T>(I).toAPSInt();
2651 Dst.elem<T>(I) = static_cast<T>(APSInt(Fn(Elem1, Elem2), DestUnsigned));
2652 });
2653 }
2654 Dst.initializeAllElements();
2655
2656 return true;
2657}
2658
2659static bool
2660interp__builtin_ia32_pack(InterpState &S, CodePtr, const CallExpr *E,
2661 llvm::function_ref<APInt(const APSInt &)> PackFn) {
2662 const auto *VT0 = E->getArg(Arg: 0)->getType()->castAs<VectorType>();
2663 [[maybe_unused]] const auto *VT1 =
2664 E->getArg(Arg: 1)->getType()->castAs<VectorType>();
2665 assert(VT0 && VT1 && "pack builtin VT0 and VT1 must be VectorType");
2666 assert(VT0->getElementType() == VT1->getElementType() &&
2667 VT0->getNumElements() == VT1->getNumElements() &&
2668 "pack builtin VT0 and VT1 ElementType must be same");
2669
2670 const Pointer &RHS = S.Stk.pop<Pointer>();
2671 const Pointer &LHS = S.Stk.pop<Pointer>();
2672 const Pointer &Dst = S.Stk.peek<Pointer>();
2673
2674 const ASTContext &ASTCtx = S.getASTContext();
2675 unsigned SrcBits = ASTCtx.getIntWidth(T: VT0->getElementType());
2676 unsigned LHSVecLen = VT0->getNumElements();
2677 unsigned SrcPerLane = 128 / SrcBits;
2678 unsigned Lanes = LHSVecLen * SrcBits / 128;
2679
2680 PrimType SrcT = *S.getContext().classify(T: VT0->getElementType());
2681 PrimType DstT = *S.getContext().classify(T: getElemType(P: Dst));
2682 bool IsUnsigend = getElemType(P: Dst)->isUnsignedIntegerType();
2683
2684 for (unsigned Lane = 0; Lane != Lanes; ++Lane) {
2685 unsigned BaseSrc = Lane * SrcPerLane;
2686 unsigned BaseDst = Lane * (2 * SrcPerLane);
2687
2688 for (unsigned I = 0; I != SrcPerLane; ++I) {
2689 INT_TYPE_SWITCH_NO_BOOL(SrcT, {
2690 APSInt A = LHS.elem<T>(BaseSrc + I).toAPSInt();
2691 APSInt B = RHS.elem<T>(BaseSrc + I).toAPSInt();
2692
2693 assignIntegral(S, Dst.atIndex(BaseDst + I), DstT,
2694 APSInt(PackFn(A), IsUnsigend));
2695 assignIntegral(S, Dst.atIndex(BaseDst + SrcPerLane + I), DstT,
2696 APSInt(PackFn(B), IsUnsigend));
2697 });
2698 }
2699 }
2700
2701 Dst.initializeAllElements();
2702 return true;
2703}
2704
2705static bool interp__builtin_elementwise_maxmin(InterpState &S, CodePtr OpPC,
2706 const CallExpr *Call,
2707 unsigned BuiltinID) {
2708 assert(Call->getNumArgs() == 2);
2709
2710 QualType Arg0Type = Call->getArg(Arg: 0)->getType();
2711
2712 // TODO: Support floating-point types.
2713 if (!(Arg0Type->isIntegerType() ||
2714 (Arg0Type->isVectorType() &&
2715 Arg0Type->castAs<VectorType>()->getElementType()->isIntegerType())))
2716 return false;
2717
2718 if (!Arg0Type->isVectorType()) {
2719 assert(!Call->getArg(1)->getType()->isVectorType());
2720 APSInt RHS;
2721 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: RHS))
2722 return false;
2723 APSInt LHS;
2724 if (!popToAPSInt(S, T: Arg0Type, Out&: LHS))
2725 return false;
2726 APInt Result;
2727 if (BuiltinID == Builtin::BI__builtin_elementwise_max) {
2728 Result = std::max(a: LHS, b: RHS);
2729 } else if (BuiltinID == Builtin::BI__builtin_elementwise_min) {
2730 Result = std::min(a: LHS, b: RHS);
2731 } else {
2732 llvm_unreachable("Wrong builtin ID");
2733 }
2734
2735 pushInteger(S, Val: APSInt(Result, !LHS.isSigned()), QT: Call->getType());
2736 return true;
2737 }
2738
2739 // Vector case.
2740 assert(Call->getArg(0)->getType()->isVectorType() &&
2741 Call->getArg(1)->getType()->isVectorType());
2742 const auto *VT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
2743 assert(VT->getElementType() ==
2744 Call->getArg(1)->getType()->castAs<VectorType>()->getElementType());
2745 assert(VT->getNumElements() ==
2746 Call->getArg(1)->getType()->castAs<VectorType>()->getNumElements());
2747 assert(VT->getElementType()->isIntegralOrEnumerationType());
2748
2749 const Pointer &RHS = S.Stk.pop<Pointer>();
2750 const Pointer &LHS = S.Stk.pop<Pointer>();
2751 const Pointer &Dst = S.Stk.peek<Pointer>();
2752 PrimType ElemT = *S.getContext().classify(T: VT->getElementType());
2753 unsigned NumElems = VT->getNumElements();
2754 for (unsigned I = 0; I != NumElems; ++I) {
2755 APSInt Elem1;
2756 APSInt Elem2;
2757 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
2758 Elem1 = LHS.elem<T>(I).toAPSInt();
2759 Elem2 = RHS.elem<T>(I).toAPSInt();
2760 });
2761
2762 APSInt Result;
2763 if (BuiltinID == Builtin::BI__builtin_elementwise_max) {
2764 Result = APSInt(std::max(a: Elem1, b: Elem2),
2765 Call->getType()->isUnsignedIntegerOrEnumerationType());
2766 } else if (BuiltinID == Builtin::BI__builtin_elementwise_min) {
2767 Result = APSInt(std::min(a: Elem1, b: Elem2),
2768 Call->getType()->isUnsignedIntegerOrEnumerationType());
2769 } else {
2770 llvm_unreachable("Wrong builtin ID");
2771 }
2772
2773 INT_TYPE_SWITCH_NO_BOOL(ElemT,
2774 { Dst.elem<T>(I) = static_cast<T>(Result); });
2775 }
2776 Dst.initializeAllElements();
2777
2778 return true;
2779}
2780
2781static bool interp__builtin_ia32_pmul(
2782 InterpState &S, CodePtr OpPC, const CallExpr *Call,
2783 llvm::function_ref<APInt(const APSInt &, const APSInt &, const APSInt &,
2784 const APSInt &)>
2785 Fn) {
2786 assert(Call->getArg(0)->getType()->isVectorType() &&
2787 Call->getArg(1)->getType()->isVectorType());
2788 const Pointer &RHS = S.Stk.pop<Pointer>();
2789 const Pointer &LHS = S.Stk.pop<Pointer>();
2790 const Pointer &Dst = S.Stk.peek<Pointer>();
2791
2792 const auto *VT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
2793 PrimType ElemT = *S.getContext().classify(T: VT->getElementType());
2794 unsigned NumElems = VT->getNumElements();
2795 const auto *DestVT = Call->getType()->castAs<VectorType>();
2796 PrimType DestElemT = *S.getContext().classify(T: DestVT->getElementType());
2797 bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
2798
2799 unsigned DstElem = 0;
2800 for (unsigned I = 0; I != NumElems; I += 2) {
2801 APSInt Result;
2802 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
2803 APSInt LoLHS = LHS.elem<T>(I).toAPSInt();
2804 APSInt HiLHS = LHS.elem<T>(I + 1).toAPSInt();
2805 APSInt LoRHS = RHS.elem<T>(I).toAPSInt();
2806 APSInt HiRHS = RHS.elem<T>(I + 1).toAPSInt();
2807 Result = APSInt(Fn(LoLHS, HiLHS, LoRHS, HiRHS), DestUnsigned);
2808 });
2809
2810 INT_TYPE_SWITCH_NO_BOOL(DestElemT,
2811 { Dst.elem<T>(DstElem) = static_cast<T>(Result); });
2812 ++DstElem;
2813 }
2814
2815 Dst.initializeAllElements();
2816 return true;
2817}
2818
2819static bool interp__builtin_ia32_psadbw(InterpState &S, CodePtr OpPC,
2820 const CallExpr *Call) {
2821 assert(Call->getNumArgs() == 2);
2822
2823 const Pointer &RHS = S.Stk.pop<Pointer>();
2824 const Pointer &LHS = S.Stk.pop<Pointer>();
2825 const Pointer &Dst = S.Stk.peek<Pointer>();
2826
2827 const auto *SrcVT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
2828 PrimType SrcElemT = *S.getContext().classify(T: SrcVT->getElementType());
2829 unsigned SourceLen = SrcVT->getNumElements();
2830 assert((SourceLen % 8) == 0);
2831
2832 const auto *DestVT = Call->getType()->castAs<VectorType>();
2833 PrimType DestElemT = *S.getContext().classify(T: DestVT->getElementType());
2834 bool DestUnsigned =
2835 DestVT->getElementType()->isUnsignedIntegerOrEnumerationType();
2836
2837 unsigned DstElem = 0;
2838 for (unsigned Lane = 0; Lane != SourceLen; Lane += 8) {
2839 APInt Sum(64, 0);
2840 for (unsigned I = 0; I != 8; ++I) {
2841 INT_TYPE_SWITCH_NO_BOOL(SrcElemT, {
2842 APSInt L = LHS.elem<T>(Lane + I).toAPSInt();
2843 APSInt R = RHS.elem<T>(Lane + I).toAPSInt();
2844 Sum += llvm::APIntOps::abdu(L.extOrTrunc(8), R.extOrTrunc(8)).zext(64);
2845 });
2846 }
2847
2848 INT_TYPE_SWITCH_NO_BOOL(DestElemT, {
2849 Dst.elem<T>(DstElem) = static_cast<T>(APSInt(Sum, DestUnsigned));
2850 });
2851 ++DstElem;
2852 }
2853
2854 Dst.initializeAllElements();
2855 return true;
2856}
2857
2858static bool interp__builtin_ia32_dbpsadbw(InterpState &S, CodePtr OpPC,
2859 const CallExpr *Call) {
2860 assert(Call->getNumArgs() == 3);
2861 uint64_t Imm;
2862 if (!popToUInt64(S, E: Call->getArg(Arg: 2), Out&: Imm))
2863 return false;
2864
2865 const Pointer &Src2 = S.Stk.pop<Pointer>();
2866 const Pointer &Src1 = S.Stk.pop<Pointer>();
2867 const Pointer &Dst = S.Stk.peek<Pointer>();
2868
2869 const auto *SrcVT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
2870 PrimType SrcElemT = *S.getContext().classify(T: SrcVT->getElementType());
2871 unsigned SourceLen = SrcVT->getNumElements();
2872
2873 const auto *DestVT = Call->getType()->castAs<VectorType>();
2874 PrimType DestElemT = *S.getContext().classify(T: DestVT->getElementType());
2875 bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
2876
2877 constexpr unsigned LaneSize = 16; // 128-bit lane = 16 bytes
2878
2879 // Phase 1: Shuffle Src2 using all four 2-bit fields of imm8.
2880 // Within each 128-bit lane, for group j (0..3), select a 4-byte block
2881 // from Src2 based on bits [2*j+1:2*j] of imm8.
2882 SmallVector<uint8_t, 64> Shuffled(SourceLen);
2883 for (unsigned I = 0; I < SourceLen; I += LaneSize) {
2884 for (unsigned J = 0; J < 4; ++J) {
2885 unsigned Part = (Imm >> (2 * J)) & 3;
2886 for (unsigned K = 0; K < 4; ++K) {
2887 INT_TYPE_SWITCH_NO_BOOL(SrcElemT, {
2888 Shuffled[I + 4 * J + K] =
2889 static_cast<uint8_t>(Src2.elem<T>(I + 4 * Part + K));
2890 });
2891 }
2892 }
2893 }
2894
2895 // Phase 2: Sliding SAD computation.
2896 // For every group of 4 output u16 values, compute absolute differences
2897 // using overlapping windows into Src1 and the shuffled array.
2898 unsigned Size = SourceLen / 2; // number of output u16 elements
2899 for (unsigned I = 0; I < Size; I += 4) {
2900 unsigned Sad[4] = {0, 0, 0, 0};
2901 for (unsigned J = 0; J < 4; ++J) {
2902 uint8_t A1, A2;
2903 INT_TYPE_SWITCH_NO_BOOL(SrcElemT, {
2904 A1 = static_cast<uint8_t>(Src1.elem<T>(2 * I + J));
2905 A2 = static_cast<uint8_t>(Src1.elem<T>(2 * I + J + 4));
2906 });
2907 uint8_t B0 = Shuffled[2 * I + J];
2908 uint8_t B1 = Shuffled[2 * I + J + 1];
2909 uint8_t B2 = Shuffled[2 * I + J + 2];
2910 uint8_t B3 = Shuffled[2 * I + J + 3];
2911 Sad[0] += (A1 > B0) ? (A1 - B0) : (B0 - A1);
2912 Sad[1] += (A1 > B1) ? (A1 - B1) : (B1 - A1);
2913 Sad[2] += (A2 > B2) ? (A2 - B2) : (B2 - A2);
2914 Sad[3] += (A2 > B3) ? (A2 - B3) : (B3 - A2);
2915 }
2916 for (unsigned R = 0; R < 4; ++R) {
2917 INT_TYPE_SWITCH_NO_BOOL(DestElemT, {
2918 Dst.elem<T>(I + R) =
2919 static_cast<T>(APSInt(APInt(16, Sad[R]), DestUnsigned));
2920 });
2921 }
2922 }
2923
2924 Dst.initializeAllElements();
2925 return true;
2926}
2927
2928static bool interp__builtin_ia32_mpsadbw(InterpState &S, CodePtr OpPC,
2929 const CallExpr *Call) {
2930 assert(Call->getNumArgs() == 3);
2931 uint64_t Imm;
2932 if (!popToUInt64(S, E: Call->getArg(Arg: 2), Out&: Imm))
2933 return false;
2934
2935 const Pointer &Src2 = S.Stk.pop<Pointer>();
2936 const Pointer &Src1 = S.Stk.pop<Pointer>();
2937 const Pointer &Dst = S.Stk.peek<Pointer>();
2938
2939 const auto *SrcVT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
2940 PrimType SrcElemT = *S.getContext().classify(T: SrcVT->getElementType());
2941 unsigned SourceLen = SrcVT->getNumElements();
2942 assert((SourceLen == 16 || SourceLen == 32) &&
2943 "MPSADBW operates on 128-bit or 256-bit vectors");
2944
2945 const auto *DestVT = Call->getType()->castAs<VectorType>();
2946 PrimType DestElemT = *S.getContext().classify(T: DestVT->getElementType());
2947 bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
2948
2949 constexpr unsigned LaneSize = 16; // 128-bit lane = 16 bytes
2950 unsigned NumLanes = SourceLen / LaneSize;
2951
2952 for (unsigned Lane = 0; Lane != NumLanes; ++Lane) {
2953 unsigned Ctrl = (Imm >> (3 * Lane)) & 0x7;
2954 unsigned AOff = ((Ctrl >> 2) & 1) * 4;
2955 unsigned BOff = (Ctrl & 3) * 4;
2956 for (unsigned J = 0; J != 8; ++J) {
2957 uint16_t Sad = 0;
2958 for (unsigned K = 0; K != 4; ++K) {
2959 uint8_t A, B;
2960 INT_TYPE_SWITCH_NO_BOOL(SrcElemT, {
2961 A = static_cast<uint8_t>(
2962 Src1.elem<T>(Lane * LaneSize + AOff + J + K));
2963 B = static_cast<uint8_t>(Src2.elem<T>(Lane * LaneSize + BOff + K));
2964 });
2965 Sad += (A > B) ? (A - B) : (B - A);
2966 }
2967 INT_TYPE_SWITCH_NO_BOOL(DestElemT, {
2968 Dst.elem<T>(Lane * 8 + J) =
2969 static_cast<T>(APSInt(APInt(16, Sad), DestUnsigned));
2970 });
2971 }
2972 }
2973
2974 Dst.initializeAllElements();
2975 return true;
2976}
2977
2978static bool interp_builtin_horizontal_int_binop(
2979 InterpState &S, CodePtr OpPC, const CallExpr *Call,
2980 llvm::function_ref<APInt(const APSInt &, const APSInt &)> Fn) {
2981 const auto *VT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
2982 PrimType ElemT = *S.getContext().classify(T: VT->getElementType());
2983 bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
2984
2985 const Pointer &RHS = S.Stk.pop<Pointer>();
2986 const Pointer &LHS = S.Stk.pop<Pointer>();
2987 const Pointer &Dst = S.Stk.peek<Pointer>();
2988 unsigned NumElts = VT->getNumElements();
2989 unsigned EltBits = S.getASTContext().getIntWidth(T: VT->getElementType());
2990 unsigned EltsPerLane = 128 / EltBits;
2991 unsigned Lanes = NumElts * EltBits / 128;
2992 unsigned DestIndex = 0;
2993
2994 for (unsigned Lane = 0; Lane < Lanes; ++Lane) {
2995 unsigned LaneStart = Lane * EltsPerLane;
2996 for (unsigned I = 0; I < EltsPerLane; I += 2) {
2997 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
2998 APSInt Elem1 = LHS.elem<T>(LaneStart + I).toAPSInt();
2999 APSInt Elem2 = LHS.elem<T>(LaneStart + I + 1).toAPSInt();
3000 APSInt ResL = APSInt(Fn(Elem1, Elem2), DestUnsigned);
3001 Dst.elem<T>(DestIndex++) = static_cast<T>(ResL);
3002 });
3003 }
3004
3005 for (unsigned I = 0; I < EltsPerLane; I += 2) {
3006 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
3007 APSInt Elem1 = RHS.elem<T>(LaneStart + I).toAPSInt();
3008 APSInt Elem2 = RHS.elem<T>(LaneStart + I + 1).toAPSInt();
3009 APSInt ResR = APSInt(Fn(Elem1, Elem2), DestUnsigned);
3010 Dst.elem<T>(DestIndex++) = static_cast<T>(ResR);
3011 });
3012 }
3013 }
3014 Dst.initializeAllElements();
3015 return true;
3016}
3017
3018static bool interp_builtin_horizontal_fp_binop(
3019 InterpState &S, CodePtr OpPC, const CallExpr *Call,
3020 llvm::function_ref<APFloat(const APFloat &, const APFloat &,
3021 llvm::RoundingMode)>
3022 Fn) {
3023 const Pointer &RHS = S.Stk.pop<Pointer>();
3024 const Pointer &LHS = S.Stk.pop<Pointer>();
3025 const Pointer &Dst = S.Stk.peek<Pointer>();
3026 FPOptions FPO = Call->getFPFeaturesInEffect(LO: S.Ctx.getLangOpts());
3027 llvm::RoundingMode RM = getRoundingMode(FPO);
3028 const auto *VT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
3029
3030 unsigned NumElts = VT->getNumElements();
3031 unsigned EltBits = S.getASTContext().getTypeSize(T: VT->getElementType());
3032 unsigned NumLanes = NumElts * EltBits / 128;
3033 unsigned NumElemsPerLane = NumElts / NumLanes;
3034 unsigned HalfElemsPerLane = NumElemsPerLane / 2;
3035
3036 for (unsigned L = 0; L != NumElts; L += NumElemsPerLane) {
3037 using T = PrimConv<PT_Float>::T;
3038 for (unsigned E = 0; E != HalfElemsPerLane; ++E) {
3039 APFloat Elem1 = LHS.elem<T>(I: L + (2 * E) + 0).getAPFloat();
3040 APFloat Elem2 = LHS.elem<T>(I: L + (2 * E) + 1).getAPFloat();
3041 Dst.elem<T>(I: L + E) = static_cast<T>(Fn(Elem1, Elem2, RM));
3042 }
3043 for (unsigned E = 0; E != HalfElemsPerLane; ++E) {
3044 APFloat Elem1 = RHS.elem<T>(I: L + (2 * E) + 0).getAPFloat();
3045 APFloat Elem2 = RHS.elem<T>(I: L + (2 * E) + 1).getAPFloat();
3046 Dst.elem<T>(I: L + E + HalfElemsPerLane) =
3047 static_cast<T>(Fn(Elem1, Elem2, RM));
3048 }
3049 }
3050 Dst.initializeAllElements();
3051 return true;
3052}
3053
3054static bool interp__builtin_ia32_addsub(InterpState &S, CodePtr OpPC,
3055 const CallExpr *Call) {
3056 // Addsub: alternates between subtraction and addition
3057 // Result[i] = (i % 2 == 0) ? (a[i] - b[i]) : (a[i] + b[i])
3058 const Pointer &RHS = S.Stk.pop<Pointer>();
3059 const Pointer &LHS = S.Stk.pop<Pointer>();
3060 const Pointer &Dst = S.Stk.peek<Pointer>();
3061 FPOptions FPO = Call->getFPFeaturesInEffect(LO: S.Ctx.getLangOpts());
3062 llvm::RoundingMode RM = getRoundingMode(FPO);
3063 const auto *VT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
3064 unsigned NumElems = VT->getNumElements();
3065
3066 using T = PrimConv<PT_Float>::T;
3067 for (unsigned I = 0; I != NumElems; ++I) {
3068 APFloat LElem = LHS.elem<T>(I).getAPFloat();
3069 APFloat RElem = RHS.elem<T>(I).getAPFloat();
3070 if (I % 2 == 0) {
3071 // Even indices: subtract
3072 LElem.subtract(RHS: RElem, RM);
3073 } else {
3074 // Odd indices: add
3075 LElem.add(RHS: RElem, RM);
3076 }
3077 Dst.elem<T>(I) = static_cast<T>(LElem);
3078 }
3079 Dst.initializeAllElements();
3080 return true;
3081}
3082
3083static bool interp__builtin_ia32_pclmulqdq(InterpState &S, CodePtr OpPC,
3084 const CallExpr *Call) {
3085 // PCLMULQDQ: carry-less multiplication of selected 64-bit halves
3086 // imm8 bit 0: selects lower (0) or upper (1) 64 bits of first operand
3087 // imm8 bit 4: selects lower (0) or upper (1) 64 bits of second operand
3088 assert(Call->getArg(0)->getType()->isVectorType() &&
3089 Call->getArg(1)->getType()->isVectorType());
3090
3091 // Extract imm8 argument
3092 APSInt Imm8;
3093 if (!popToAPSInt(S, E: Call->getArg(Arg: 2), Out&: Imm8))
3094 return false;
3095 bool SelectUpperA = (Imm8 & 0x01) != 0;
3096 bool SelectUpperB = (Imm8 & 0x10) != 0;
3097
3098 const Pointer &RHS = S.Stk.pop<Pointer>();
3099 const Pointer &LHS = S.Stk.pop<Pointer>();
3100 const Pointer &Dst = S.Stk.peek<Pointer>();
3101
3102 const auto *VT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
3103 PrimType ElemT = *S.getContext().classify(T: VT->getElementType());
3104 unsigned NumElems = VT->getNumElements();
3105 const auto *DestVT = Call->getType()->castAs<VectorType>();
3106 PrimType DestElemT = *S.getContext().classify(T: DestVT->getElementType());
3107 bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
3108
3109 // Process each 128-bit lane (2 elements at a time)
3110 for (unsigned Lane = 0; Lane < NumElems; Lane += 2) {
3111 APSInt A0, A1, B0, B1;
3112 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
3113 A0 = LHS.elem<T>(Lane + 0).toAPSInt();
3114 A1 = LHS.elem<T>(Lane + 1).toAPSInt();
3115 B0 = RHS.elem<T>(Lane + 0).toAPSInt();
3116 B1 = RHS.elem<T>(Lane + 1).toAPSInt();
3117 });
3118
3119 // Select the appropriate 64-bit values based on imm8
3120 APInt A = SelectUpperA ? A1 : A0;
3121 APInt B = SelectUpperB ? B1 : B0;
3122
3123 // Extend both operands to 128 bits for carry-less multiplication
3124 APInt A128 = A.zext(width: 128);
3125 APInt B128 = B.zext(width: 128);
3126
3127 // Use APIntOps::clmul for carry-less multiplication
3128 APInt Result = llvm::APIntOps::clmul(LHS: A128, RHS: B128);
3129
3130 // Split the 128-bit result into two 64-bit halves
3131 APSInt ResultLow(Result.extractBits(numBits: 64, bitPosition: 0), DestUnsigned);
3132 APSInt ResultHigh(Result.extractBits(numBits: 64, bitPosition: 64), DestUnsigned);
3133
3134 INT_TYPE_SWITCH_NO_BOOL(DestElemT, {
3135 Dst.elem<T>(Lane + 0) = static_cast<T>(ResultLow);
3136 Dst.elem<T>(Lane + 1) = static_cast<T>(ResultHigh);
3137 });
3138 }
3139
3140 Dst.initializeAllElements();
3141 return true;
3142}
3143
3144static bool interp__builtin_elementwise_triop_fp(
3145 InterpState &S, CodePtr OpPC, const CallExpr *Call,
3146 llvm::function_ref<APFloat(const APFloat &, const APFloat &,
3147 const APFloat &, llvm::RoundingMode)>
3148 Fn) {
3149 assert(Call->getNumArgs() == 3);
3150
3151 FPOptions FPO = Call->getFPFeaturesInEffect(LO: S.Ctx.getLangOpts());
3152 llvm::RoundingMode RM = getRoundingMode(FPO);
3153 QualType Arg1Type = Call->getArg(Arg: 0)->getType();
3154 QualType Arg2Type = Call->getArg(Arg: 1)->getType();
3155 QualType Arg3Type = Call->getArg(Arg: 2)->getType();
3156
3157 // Non-vector floating point types.
3158 if (!Arg1Type->isVectorType()) {
3159 assert(!Arg2Type->isVectorType());
3160 assert(!Arg3Type->isVectorType());
3161 (void)Arg2Type;
3162 (void)Arg3Type;
3163
3164 const Floating &Z = S.Stk.pop<Floating>();
3165 const Floating &Y = S.Stk.pop<Floating>();
3166 const Floating &X = S.Stk.pop<Floating>();
3167 APFloat F = Fn(X.getAPFloat(), Y.getAPFloat(), Z.getAPFloat(), RM);
3168 Floating Result = S.allocFloat(Sem: X.getSemantics());
3169 Result.copy(F);
3170 S.Stk.push<Floating>(Args&: Result);
3171 return true;
3172 }
3173
3174 // Vector type.
3175 assert(Arg1Type->isVectorType() && Arg2Type->isVectorType() &&
3176 Arg3Type->isVectorType());
3177
3178 const VectorType *VecTy = Arg1Type->castAs<VectorType>();
3179 QualType ElemQT = VecTy->getElementType();
3180 unsigned NumElems = VecTy->getNumElements();
3181
3182 assert(ElemQT == Arg2Type->castAs<VectorType>()->getElementType() &&
3183 ElemQT == Arg3Type->castAs<VectorType>()->getElementType());
3184 assert(NumElems == Arg2Type->castAs<VectorType>()->getNumElements() &&
3185 NumElems == Arg3Type->castAs<VectorType>()->getNumElements());
3186 assert(ElemQT->isRealFloatingType());
3187 (void)ElemQT;
3188
3189 const Pointer &VZ = S.Stk.pop<Pointer>();
3190 const Pointer &VY = S.Stk.pop<Pointer>();
3191 const Pointer &VX = S.Stk.pop<Pointer>();
3192 const Pointer &Dst = S.Stk.peek<Pointer>();
3193 for (unsigned I = 0; I != NumElems; ++I) {
3194 using T = PrimConv<PT_Float>::T;
3195 APFloat X = VX.elem<T>(I).getAPFloat();
3196 APFloat Y = VY.elem<T>(I).getAPFloat();
3197 APFloat Z = VZ.elem<T>(I).getAPFloat();
3198 APFloat F = Fn(X, Y, Z, RM);
3199 Dst.elem<Floating>(I) = Floating(F);
3200 }
3201 Dst.initializeAllElements();
3202 return true;
3203}
3204
3205/// AVX512 predicated move: "Result = Mask[] ? LHS[] : RHS[]".
3206static bool interp__builtin_ia32_select(InterpState &S, CodePtr OpPC,
3207 const CallExpr *Call) {
3208 const Pointer &RHS = S.Stk.pop<Pointer>();
3209 const Pointer &LHS = S.Stk.pop<Pointer>();
3210 APSInt Mask;
3211 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: Mask))
3212 return false;
3213 const Pointer &Dst = S.Stk.peek<Pointer>();
3214
3215 assert(LHS.getNumElems() == RHS.getNumElems());
3216 assert(LHS.getNumElems() == Dst.getNumElems());
3217 unsigned NumElems = LHS.getNumElems();
3218 PrimType ElemT = LHS.getFieldDesc()->getPrimType();
3219 PrimType DstElemT = Dst.getFieldDesc()->getPrimType();
3220
3221 for (unsigned I = 0; I != NumElems; ++I) {
3222 if (ElemT == PT_Float) {
3223 assert(DstElemT == PT_Float);
3224 Dst.elem<Floating>(I) =
3225 Mask[I] ? LHS.elem<Floating>(I) : RHS.elem<Floating>(I);
3226 } else {
3227 APSInt Elem;
3228 INT_TYPE_SWITCH(ElemT, {
3229 Elem = Mask[I] ? LHS.elem<T>(I).toAPSInt() : RHS.elem<T>(I).toAPSInt();
3230 });
3231 INT_TYPE_SWITCH_NO_BOOL(DstElemT,
3232 { Dst.elem<T>(I) = static_cast<T>(Elem); });
3233 }
3234 }
3235 Dst.initializeAllElements();
3236
3237 return true;
3238}
3239
3240/// Scalar variant of AVX512 predicated select:
3241/// Result[i] = (Mask bit 0) ? LHS[i] : RHS[i], but only element 0 may change.
3242/// All other elements are taken from RHS.
3243static bool interp__builtin_ia32_select_scalar(InterpState &S,
3244 const CallExpr *Call) {
3245 unsigned N =
3246 Call->getArg(Arg: 1)->getType()->castAs<VectorType>()->getNumElements();
3247
3248 const Pointer &W = S.Stk.pop<Pointer>();
3249 const Pointer &A = S.Stk.pop<Pointer>();
3250 APSInt U;
3251 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: U))
3252 return false;
3253 const Pointer &Dst = S.Stk.peek<Pointer>();
3254
3255 bool TakeA0 = U.getZExtValue() & 1ULL;
3256
3257 for (unsigned I = TakeA0; I != N; ++I)
3258 Dst.elem<Floating>(I) = W.elem<Floating>(I);
3259 if (TakeA0)
3260 Dst.elem<Floating>(I: 0) = A.elem<Floating>(I: 0);
3261
3262 Dst.initializeAllElements();
3263 return true;
3264}
3265
3266static bool interp__builtin_ia32_test_op(
3267 InterpState &S, CodePtr OpPC, const CallExpr *Call,
3268 llvm::function_ref<bool(const APInt &A, const APInt &B)> Fn) {
3269 const Pointer &RHS = S.Stk.pop<Pointer>();
3270 const Pointer &LHS = S.Stk.pop<Pointer>();
3271
3272 assert(LHS.getNumElems() == RHS.getNumElems());
3273
3274 unsigned SourceLen = LHS.getNumElems();
3275 QualType ElemQT = getElemType(P: LHS);
3276 OptPrimType ElemPT = S.getContext().classify(T: ElemQT);
3277 unsigned LaneWidth = S.getASTContext().getTypeSize(T: ElemQT);
3278
3279 APInt AWide(LaneWidth * SourceLen, 0);
3280 APInt BWide(LaneWidth * SourceLen, 0);
3281
3282 for (unsigned I = 0; I != SourceLen; ++I) {
3283 APInt ALane;
3284 APInt BLane;
3285
3286 if (ElemQT->isIntegerType()) { // Get value.
3287 INT_TYPE_SWITCH_NO_BOOL(*ElemPT, {
3288 ALane = LHS.elem<T>(I).toAPSInt();
3289 BLane = RHS.elem<T>(I).toAPSInt();
3290 });
3291 } else if (ElemQT->isFloatingType()) { // Get only sign bit.
3292 using T = PrimConv<PT_Float>::T;
3293 ALane = LHS.elem<T>(I).getAPFloat().bitcastToAPInt().isNegative();
3294 BLane = RHS.elem<T>(I).getAPFloat().bitcastToAPInt().isNegative();
3295 } else { // Must be integer or floating type.
3296 return false;
3297 }
3298 AWide.insertBits(SubBits: ALane, bitPosition: I * LaneWidth);
3299 BWide.insertBits(SubBits: BLane, bitPosition: I * LaneWidth);
3300 }
3301 pushInteger(S, Val: Fn(AWide, BWide), QT: Call->getType());
3302 return true;
3303}
3304
3305static bool interp__builtin_ia32_movmsk_op(InterpState &S, CodePtr OpPC,
3306 const CallExpr *Call) {
3307 assert(Call->getNumArgs() == 1);
3308
3309 const Pointer &Source = S.Stk.pop<Pointer>();
3310
3311 unsigned SourceLen = Source.getNumElems();
3312 QualType ElemQT = getElemType(P: Source);
3313 OptPrimType ElemT = S.getContext().classify(T: ElemQT);
3314 unsigned ResultLen =
3315 S.getASTContext().getTypeSize(T: Call->getType()); // Always 32-bit integer.
3316 APInt Result(ResultLen, 0);
3317
3318 for (unsigned I = 0; I != SourceLen; ++I) {
3319 APInt Elem;
3320 if (ElemQT->isIntegerType()) {
3321 INT_TYPE_SWITCH_NO_BOOL(*ElemT, { Elem = Source.elem<T>(I).toAPSInt(); });
3322 } else if (ElemQT->isRealFloatingType()) {
3323 using T = PrimConv<PT_Float>::T;
3324 Elem = Source.elem<T>(I).getAPFloat().bitcastToAPInt();
3325 } else {
3326 return false;
3327 }
3328 Result.setBitVal(BitPosition: I, BitValue: Elem.isNegative());
3329 }
3330 pushInteger(S, Val: Result, QT: Call->getType());
3331 return true;
3332}
3333
3334static bool interp__builtin_elementwise_triop(
3335 InterpState &S, CodePtr OpPC, const CallExpr *Call,
3336 llvm::function_ref<APInt(const APSInt &, const APSInt &, const APSInt &)>
3337 Fn) {
3338 assert(Call->getNumArgs() == 3);
3339
3340 QualType Arg0Type = Call->getArg(Arg: 0)->getType();
3341 QualType Arg2Type = Call->getArg(Arg: 2)->getType();
3342 // Non-vector integer types.
3343 if (!Arg0Type->isVectorType()) {
3344 APSInt Op2;
3345 if (!popToAPSInt(S, T: Arg2Type, Out&: Op2))
3346 return false;
3347 APSInt Op1;
3348 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: Op1))
3349 return false;
3350 APSInt Op0;
3351 if (!popToAPSInt(S, T: Arg0Type, Out&: Op0))
3352 return false;
3353 APSInt Result = APSInt(Fn(Op0, Op1, Op2), Op0.isUnsigned());
3354 pushInteger(S, Val: Result, QT: Call->getType());
3355 return true;
3356 }
3357
3358 const auto *VecT = Arg0Type->castAs<VectorType>();
3359 PrimType ElemT = *S.getContext().classify(T: VecT->getElementType());
3360 unsigned NumElems = VecT->getNumElements();
3361 bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
3362
3363 // Vector + Vector + Scalar case.
3364 if (!Arg2Type->isVectorType()) {
3365 APSInt Op2;
3366 if (!popToAPSInt(S, T: Arg2Type, Out&: Op2))
3367 return false;
3368
3369 const Pointer &Op1 = S.Stk.pop<Pointer>();
3370 const Pointer &Op0 = S.Stk.pop<Pointer>();
3371 const Pointer &Dst = S.Stk.peek<Pointer>();
3372 for (unsigned I = 0; I != NumElems; ++I) {
3373 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
3374 Dst.elem<T>(I) = static_cast<T>(APSInt(
3375 Fn(Op0.elem<T>(I).toAPSInt(), Op1.elem<T>(I).toAPSInt(), Op2),
3376 DestUnsigned));
3377 });
3378 }
3379 Dst.initializeAllElements();
3380
3381 return true;
3382 }
3383
3384 // Vector type.
3385 const Pointer &Op2 = S.Stk.pop<Pointer>();
3386 const Pointer &Op1 = S.Stk.pop<Pointer>();
3387 const Pointer &Op0 = S.Stk.pop<Pointer>();
3388 const Pointer &Dst = S.Stk.peek<Pointer>();
3389 for (unsigned I = 0; I != NumElems; ++I) {
3390 APSInt Val0, Val1, Val2;
3391 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
3392 Val0 = Op0.elem<T>(I).toAPSInt();
3393 Val1 = Op1.elem<T>(I).toAPSInt();
3394 Val2 = Op2.elem<T>(I).toAPSInt();
3395 });
3396 APSInt Result = APSInt(Fn(Val0, Val1, Val2), Val0.isUnsigned());
3397 INT_TYPE_SWITCH_NO_BOOL(ElemT,
3398 { Dst.elem<T>(I) = static_cast<T>(Result); });
3399 }
3400 Dst.initializeAllElements();
3401
3402 return true;
3403}
3404
3405static bool interp__builtin_ia32_extract_vector(InterpState &S, CodePtr OpPC,
3406 const CallExpr *Call,
3407 unsigned ID) {
3408 assert(Call->getNumArgs() == 2);
3409
3410 APSInt ImmAPS;
3411 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: ImmAPS))
3412 return false;
3413 uint64_t Index = ImmAPS.getZExtValue();
3414
3415 const Pointer &Src = S.Stk.pop<Pointer>();
3416 if (!Src.getFieldDesc()->isPrimitiveArray())
3417 return false;
3418
3419 const Pointer &Dst = S.Stk.peek<Pointer>();
3420 if (!Dst.getFieldDesc()->isPrimitiveArray())
3421 return false;
3422
3423 unsigned SrcElems = Src.getNumElems();
3424 unsigned DstElems = Dst.getNumElems();
3425
3426 unsigned NumLanes = SrcElems / DstElems;
3427 unsigned Lane = static_cast<unsigned>(Index % NumLanes);
3428 unsigned ExtractPos = Lane * DstElems;
3429
3430 PrimType ElemT = Src.getFieldDesc()->getPrimType();
3431
3432 TYPE_SWITCH(ElemT, {
3433 for (unsigned I = 0; I != DstElems; ++I) {
3434 Dst.elem<T>(I) = Src.elem<T>(ExtractPos + I);
3435 }
3436 });
3437
3438 Dst.initializeAllElements();
3439 return true;
3440}
3441
3442static bool interp__builtin_ia32_extract_vector_masked(InterpState &S,
3443 CodePtr OpPC,
3444 const CallExpr *Call,
3445 unsigned ID) {
3446 assert(Call->getNumArgs() == 4);
3447
3448 APSInt MaskAPS;
3449 if (!popToAPSInt(S, E: Call->getArg(Arg: 3), Out&: MaskAPS))
3450 return false;
3451 const Pointer &Merge = S.Stk.pop<Pointer>();
3452 APSInt ImmAPS;
3453 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: ImmAPS))
3454 return false;
3455 const Pointer &Src = S.Stk.pop<Pointer>();
3456
3457 if (!Src.getFieldDesc()->isPrimitiveArray() ||
3458 !Merge.getFieldDesc()->isPrimitiveArray())
3459 return false;
3460
3461 const Pointer &Dst = S.Stk.peek<Pointer>();
3462 if (!Dst.getFieldDesc()->isPrimitiveArray())
3463 return false;
3464
3465 unsigned SrcElems = Src.getNumElems();
3466 unsigned DstElems = Dst.getNumElems();
3467
3468 unsigned NumLanes = SrcElems / DstElems;
3469 unsigned Lane = static_cast<unsigned>(ImmAPS.getZExtValue() % NumLanes);
3470 unsigned Base = Lane * DstElems;
3471
3472 PrimType ElemT = Src.getFieldDesc()->getPrimType();
3473
3474 TYPE_SWITCH(ElemT, {
3475 for (unsigned I = 0; I != DstElems; ++I) {
3476 if (MaskAPS[I])
3477 Dst.elem<T>(I) = Src.elem<T>(Base + I);
3478 else
3479 Dst.elem<T>(I) = Merge.elem<T>(I);
3480 }
3481 });
3482
3483 Dst.initializeAllElements();
3484 return true;
3485}
3486
3487static bool interp__builtin_ia32_insert_subvector(InterpState &S, CodePtr OpPC,
3488 const CallExpr *Call,
3489 unsigned ID) {
3490 assert(Call->getNumArgs() == 3);
3491
3492 APSInt ImmAPS;
3493 if (!popToAPSInt(S, E: Call->getArg(Arg: 2), Out&: ImmAPS))
3494 return false;
3495 uint64_t Index = ImmAPS.getZExtValue();
3496
3497 const Pointer &SubVec = S.Stk.pop<Pointer>();
3498 if (!SubVec.getFieldDesc()->isPrimitiveArray())
3499 return false;
3500
3501 const Pointer &BaseVec = S.Stk.pop<Pointer>();
3502 if (!BaseVec.getFieldDesc()->isPrimitiveArray())
3503 return false;
3504
3505 const Pointer &Dst = S.Stk.peek<Pointer>();
3506
3507 unsigned BaseElements = BaseVec.getNumElems();
3508 unsigned SubElements = SubVec.getNumElems();
3509
3510 assert(SubElements != 0 && BaseElements != 0 &&
3511 (BaseElements % SubElements) == 0);
3512
3513 unsigned NumLanes = BaseElements / SubElements;
3514 unsigned Lane = static_cast<unsigned>(Index % NumLanes);
3515 unsigned InsertPos = Lane * SubElements;
3516
3517 PrimType ElemT = BaseVec.getFieldDesc()->getPrimType();
3518
3519 TYPE_SWITCH(ElemT, {
3520 for (unsigned I = 0; I != BaseElements; ++I)
3521 Dst.elem<T>(I) = BaseVec.elem<T>(I);
3522 for (unsigned I = 0; I != SubElements; ++I)
3523 Dst.elem<T>(InsertPos + I) = SubVec.elem<T>(I);
3524 });
3525
3526 Dst.initializeAllElements();
3527 return true;
3528}
3529
3530static bool interp__builtin_ia32_phminposuw(InterpState &S, CodePtr OpPC,
3531 const CallExpr *Call) {
3532 assert(Call->getNumArgs() == 1);
3533
3534 const Pointer &Source = S.Stk.pop<Pointer>();
3535 const Pointer &Dest = S.Stk.peek<Pointer>();
3536
3537 unsigned SourceLen = Source.getNumElems();
3538 QualType ElemQT = getElemType(P: Source);
3539 OptPrimType ElemT = S.getContext().classify(T: ElemQT);
3540 unsigned ElemBitWidth = S.getASTContext().getTypeSize(T: ElemQT);
3541
3542 bool DestUnsigned = Call->getCallReturnType(Ctx: S.getASTContext())
3543 ->castAs<VectorType>()
3544 ->getElementType()
3545 ->isUnsignedIntegerOrEnumerationType();
3546
3547 INT_TYPE_SWITCH_NO_BOOL(*ElemT, {
3548 APSInt MinIndex(ElemBitWidth, DestUnsigned);
3549 APSInt MinVal = Source.elem<T>(0).toAPSInt();
3550
3551 for (unsigned I = 1; I != SourceLen; ++I) {
3552 APSInt Val = Source.elem<T>(I).toAPSInt();
3553 if (MinVal.ugt(Val)) {
3554 MinVal = Val;
3555 MinIndex = I;
3556 }
3557 }
3558
3559 Dest.elem<T>(0) = static_cast<T>(MinVal);
3560 Dest.elem<T>(1) = static_cast<T>(MinIndex);
3561 for (unsigned I = 2; I != SourceLen; ++I) {
3562 Dest.elem<T>(I) = static_cast<T>(APSInt(ElemBitWidth, DestUnsigned));
3563 }
3564 });
3565 Dest.initializeAllElements();
3566 return true;
3567}
3568
3569static bool interp__builtin_ia32_pternlog(InterpState &S, CodePtr OpPC,
3570 const CallExpr *Call, bool MaskZ) {
3571 assert(Call->getNumArgs() == 5);
3572
3573 APSInt UVal;
3574 if (!popToAPSInt(S, E: Call->getArg(Arg: 4), Out&: UVal))
3575 return false;
3576 APInt U = UVal; // Lane mask
3577 APSInt ImmVal;
3578 if (!popToAPSInt(S, E: Call->getArg(Arg: 3), Out&: ImmVal))
3579 return false;
3580 APInt Imm = ImmVal; // Ternary truth table
3581 const Pointer &C = S.Stk.pop<Pointer>();
3582 const Pointer &B = S.Stk.pop<Pointer>();
3583 const Pointer &A = S.Stk.pop<Pointer>();
3584 const Pointer &Dst = S.Stk.peek<Pointer>();
3585
3586 unsigned DstLen = A.getNumElems();
3587 QualType ElemQT = getElemType(P: A);
3588 OptPrimType ElemT = S.getContext().classify(T: ElemQT);
3589 unsigned LaneWidth = S.getASTContext().getTypeSize(T: ElemQT);
3590 bool DstUnsigned = ElemQT->isUnsignedIntegerOrEnumerationType();
3591
3592 INT_TYPE_SWITCH_NO_BOOL(*ElemT, {
3593 for (unsigned I = 0; I != DstLen; ++I) {
3594 APInt ALane = A.elem<T>(I).toAPSInt();
3595 APInt BLane = B.elem<T>(I).toAPSInt();
3596 APInt CLane = C.elem<T>(I).toAPSInt();
3597 APInt RLane(LaneWidth, 0);
3598 if (U[I]) { // If lane not masked, compute ternary logic.
3599 for (unsigned Bit = 0; Bit != LaneWidth; ++Bit) {
3600 unsigned ABit = ALane[Bit];
3601 unsigned BBit = BLane[Bit];
3602 unsigned CBit = CLane[Bit];
3603 unsigned Idx = (ABit << 2) | (BBit << 1) | (CBit);
3604 RLane.setBitVal(Bit, Imm[Idx]);
3605 }
3606 Dst.elem<T>(I) = static_cast<T>(APSInt(RLane, DstUnsigned));
3607 } else if (MaskZ) { // If zero masked, zero the lane.
3608 Dst.elem<T>(I) = static_cast<T>(APSInt(RLane, DstUnsigned));
3609 } else { // Just masked, put in A lane.
3610 Dst.elem<T>(I) = static_cast<T>(APSInt(ALane, DstUnsigned));
3611 }
3612 }
3613 });
3614 Dst.initializeAllElements();
3615 return true;
3616}
3617
3618static bool interp__builtin_ia32_vec_ext(InterpState &S, CodePtr OpPC,
3619 const CallExpr *Call, unsigned ID) {
3620 assert(Call->getNumArgs() == 2);
3621
3622 APSInt ImmAPS;
3623 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: ImmAPS))
3624 return false;
3625 const Pointer &Vec = S.Stk.pop<Pointer>();
3626 if (!Vec.getFieldDesc()->isPrimitiveArray())
3627 return false;
3628
3629 unsigned NumElems = Vec.getNumElems();
3630 unsigned Index =
3631 static_cast<unsigned>(ImmAPS.getZExtValue() & (NumElems - 1));
3632
3633 PrimType ElemT = Vec.getFieldDesc()->getPrimType();
3634 // FIXME(#161685): Replace float+int split with a numeric-only type switch
3635 if (ElemT == PT_Float) {
3636 S.Stk.push<Floating>(Args&: Vec.elem<Floating>(I: Index));
3637 return true;
3638 }
3639 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
3640 APSInt V = Vec.elem<T>(Index).toAPSInt();
3641 pushInteger(S, V, Call->getType());
3642 });
3643
3644 return true;
3645}
3646
3647static bool interp__builtin_ia32_vec_set(InterpState &S, CodePtr OpPC,
3648 const CallExpr *Call, unsigned ID) {
3649 assert(Call->getNumArgs() == 3);
3650
3651 APSInt ImmAPS;
3652 if (!popToAPSInt(S, E: Call->getArg(Arg: 2), Out&: ImmAPS))
3653 return false;
3654 APSInt ValAPS;
3655 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: ValAPS))
3656 return false;
3657
3658 const Pointer &Base = S.Stk.pop<Pointer>();
3659 if (!Base.getFieldDesc()->isPrimitiveArray())
3660 return false;
3661
3662 const Pointer &Dst = S.Stk.peek<Pointer>();
3663
3664 unsigned NumElems = Base.getNumElems();
3665 unsigned Index =
3666 static_cast<unsigned>(ImmAPS.getZExtValue() & (NumElems - 1));
3667
3668 PrimType ElemT = Base.getFieldDesc()->getPrimType();
3669 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
3670 for (unsigned I = 0; I != NumElems; ++I)
3671 Dst.elem<T>(I) = Base.elem<T>(I);
3672 Dst.elem<T>(Index) = static_cast<T>(ValAPS);
3673 });
3674
3675 Dst.initializeAllElements();
3676 return true;
3677}
3678
3679static bool evalICmpImm(uint8_t Imm, const APSInt &A, const APSInt &B,
3680 bool IsUnsigned) {
3681 switch (Imm & 0x7) {
3682 case 0x00: // _MM_CMPINT_EQ
3683 return (A == B);
3684 case 0x01: // _MM_CMPINT_LT
3685 return IsUnsigned ? A.ult(RHS: B) : A.slt(RHS: B);
3686 case 0x02: // _MM_CMPINT_LE
3687 return IsUnsigned ? A.ule(RHS: B) : A.sle(RHS: B);
3688 case 0x03: // _MM_CMPINT_FALSE
3689 return false;
3690 case 0x04: // _MM_CMPINT_NE
3691 return (A != B);
3692 case 0x05: // _MM_CMPINT_NLT
3693 return IsUnsigned ? A.ugt(RHS: B) : A.sgt(RHS: B);
3694 case 0x06: // _MM_CMPINT_NLE
3695 return IsUnsigned ? A.uge(RHS: B) : A.sge(RHS: B);
3696 case 0x07: // _MM_CMPINT_TRUE
3697 return true;
3698 default:
3699 llvm_unreachable("Invalid Op");
3700 }
3701}
3702
3703static bool interp__builtin_ia32_cmp_mask(InterpState &S, CodePtr OpPC,
3704 const CallExpr *Call, unsigned ID,
3705 bool IsUnsigned) {
3706 assert(Call->getNumArgs() == 4);
3707
3708 APSInt Mask;
3709 if (!popToAPSInt(S, E: Call->getArg(Arg: 3), Out&: Mask))
3710 return false;
3711 APSInt Opcode;
3712 if (!popToAPSInt(S, E: Call->getArg(Arg: 2), Out&: Opcode))
3713 return false;
3714 unsigned CmpOp = static_cast<unsigned>(Opcode.getZExtValue());
3715 const Pointer &RHS = S.Stk.pop<Pointer>();
3716 const Pointer &LHS = S.Stk.pop<Pointer>();
3717
3718 assert(LHS.getNumElems() == RHS.getNumElems());
3719
3720 APInt RetMask = APInt::getZero(numBits: LHS.getNumElems());
3721 unsigned VectorLen = LHS.getNumElems();
3722 PrimType ElemT = LHS.getFieldDesc()->getPrimType();
3723
3724 for (unsigned ElemNum = 0; ElemNum < VectorLen; ++ElemNum) {
3725 APSInt A, B;
3726 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
3727 A = LHS.elem<T>(ElemNum).toAPSInt();
3728 B = RHS.elem<T>(ElemNum).toAPSInt();
3729 });
3730 RetMask.setBitVal(BitPosition: ElemNum,
3731 BitValue: Mask[ElemNum] && evalICmpImm(Imm: CmpOp, A, B, IsUnsigned));
3732 }
3733 pushInteger(S, Val: RetMask, QT: Call->getType());
3734 return true;
3735}
3736
3737static bool interp__builtin_ia32_vpconflict(InterpState &S, CodePtr OpPC,
3738 const CallExpr *Call) {
3739 assert(Call->getNumArgs() == 1);
3740
3741 QualType Arg0Type = Call->getArg(Arg: 0)->getType();
3742 const auto *VecT = Arg0Type->castAs<VectorType>();
3743 PrimType ElemT = *S.getContext().classify(T: VecT->getElementType());
3744 unsigned NumElems = VecT->getNumElements();
3745 bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
3746 const Pointer &Src = S.Stk.pop<Pointer>();
3747 const Pointer &Dst = S.Stk.peek<Pointer>();
3748
3749 for (unsigned I = 0; I != NumElems; ++I) {
3750 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
3751 APSInt ElemI = Src.elem<T>(I).toAPSInt();
3752 APInt ConflictMask(ElemI.getBitWidth(), 0);
3753 for (unsigned J = 0; J != I; ++J) {
3754 APSInt ElemJ = Src.elem<T>(J).toAPSInt();
3755 ConflictMask.setBitVal(J, ElemI == ElemJ);
3756 }
3757 Dst.elem<T>(I) = static_cast<T>(APSInt(ConflictMask, DestUnsigned));
3758 });
3759 }
3760 Dst.initializeAllElements();
3761 return true;
3762}
3763
3764static bool interp__builtin_ia32_cvt_vec2mask(InterpState &S, CodePtr OpPC,
3765 const CallExpr *Call,
3766 unsigned ID) {
3767 assert(Call->getNumArgs() == 1);
3768
3769 const Pointer &Vec = S.Stk.pop<Pointer>();
3770 unsigned RetWidth = S.getASTContext().getIntWidth(T: Call->getType());
3771 APInt RetMask(RetWidth, 0);
3772
3773 unsigned VectorLen = Vec.getNumElems();
3774 PrimType ElemT = Vec.getFieldDesc()->getPrimType();
3775
3776 for (unsigned ElemNum = 0; ElemNum != VectorLen; ++ElemNum) {
3777 APSInt A;
3778 INT_TYPE_SWITCH_NO_BOOL(ElemT, { A = Vec.elem<T>(ElemNum).toAPSInt(); });
3779 unsigned MSB = A[A.getBitWidth() - 1];
3780 RetMask.setBitVal(BitPosition: ElemNum, BitValue: MSB);
3781 }
3782 pushInteger(S, Val: RetMask, QT: Call->getType());
3783 return true;
3784}
3785
3786static bool interp__builtin_ia32_cvt_mask2vec(InterpState &S, CodePtr OpPC,
3787 const CallExpr *Call,
3788 unsigned ID) {
3789 assert(Call->getNumArgs() == 1);
3790
3791 APSInt Mask;
3792 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: Mask))
3793 return false;
3794
3795 const Pointer &Vec = S.Stk.peek<Pointer>();
3796 unsigned NumElems = Vec.getNumElems();
3797 PrimType ElemT = Vec.getFieldDesc()->getPrimType();
3798
3799 for (unsigned I = 0; I != NumElems; ++I) {
3800 bool BitSet = Mask[I];
3801
3802 INT_TYPE_SWITCH_NO_BOOL(
3803 ElemT, { Vec.elem<T>(I) = BitSet ? T::from(-1) : T::from(0); });
3804 }
3805
3806 Vec.initializeAllElements();
3807
3808 return true;
3809}
3810
3811static bool interp__builtin_ia32_cvtsd2ss(InterpState &S, CodePtr OpPC,
3812 const CallExpr *Call,
3813 bool HasRoundingMask) {
3814 APSInt Rounding, MaskInt;
3815 Pointer Src, B, A;
3816
3817 if (HasRoundingMask) {
3818 assert(Call->getNumArgs() == 5);
3819 if (!popToAPSInt(S, E: Call->getArg(Arg: 4), Out&: Rounding))
3820 return false;
3821 if (!popToAPSInt(S, E: Call->getArg(Arg: 3), Out&: MaskInt))
3822 return false;
3823 Src = S.Stk.pop<Pointer>();
3824 B = S.Stk.pop<Pointer>();
3825 A = S.Stk.pop<Pointer>();
3826 if (!CheckLoad(S, OpPC, Ptr: A) || !CheckLoad(S, OpPC, Ptr: B) ||
3827 !CheckLoad(S, OpPC, Ptr: Src))
3828 return false;
3829 } else {
3830 assert(Call->getNumArgs() == 2);
3831 B = S.Stk.pop<Pointer>();
3832 A = S.Stk.pop<Pointer>();
3833 if (!CheckLoad(S, OpPC, Ptr: A) || !CheckLoad(S, OpPC, Ptr: B))
3834 return false;
3835 }
3836
3837 const auto *DstVTy = Call->getType()->castAs<VectorType>();
3838 unsigned NumElems = DstVTy->getNumElements();
3839 const Pointer &Dst = S.Stk.peek<Pointer>();
3840
3841 // Copy all elements except lane 0 (overwritten below) from A to Dst.
3842 for (unsigned I = 1; I != NumElems; ++I)
3843 Dst.elem<Floating>(I) = A.elem<Floating>(I);
3844
3845 // Convert element 0 from double to float, or use Src if masked off.
3846 if (!HasRoundingMask || (MaskInt.getZExtValue() & 0x1)) {
3847 assert(S.getASTContext().FloatTy == DstVTy->getElementType() &&
3848 "cvtsd2ss requires float element type in destination vector");
3849
3850 Floating Conv = S.allocFloat(
3851 Sem: S.getASTContext().getFloatTypeSemantics(T: DstVTy->getElementType()));
3852 APFloat SrcVal = B.elem<Floating>(I: 0).getAPFloat();
3853 if (!convertDoubleToFloatStrict(Src: SrcVal, Dst&: Conv, S, DiagExpr: Call))
3854 return false;
3855 Dst.elem<Floating>(I: 0) = Conv;
3856 } else {
3857 Dst.elem<Floating>(I: 0) = Src.elem<Floating>(I: 0);
3858 }
3859
3860 Dst.initializeAllElements();
3861 return true;
3862}
3863
3864static bool interp__builtin_ia32_cvtpd2ps(InterpState &S, CodePtr OpPC,
3865 const CallExpr *Call, bool IsMasked,
3866 bool HasRounding) {
3867 APSInt MaskVal;
3868 Pointer PassThrough;
3869 Pointer Src;
3870 APSInt Rounding;
3871
3872 if (IsMasked) {
3873 // Pop in reverse order.
3874 if (HasRounding) {
3875 if (!popToAPSInt(S, E: Call->getArg(Arg: 3), Out&: Rounding))
3876 return false;
3877 if (!popToAPSInt(S, E: Call->getArg(Arg: 2), Out&: MaskVal))
3878 return false;
3879 PassThrough = S.Stk.pop<Pointer>();
3880 Src = S.Stk.pop<Pointer>();
3881 } else {
3882 if (!popToAPSInt(S, E: Call->getArg(Arg: 2), Out&: MaskVal))
3883 return false;
3884 PassThrough = S.Stk.pop<Pointer>();
3885 Src = S.Stk.pop<Pointer>();
3886 }
3887
3888 if (!CheckLoad(S, OpPC, Ptr: PassThrough))
3889 return false;
3890 } else {
3891 // Pop source only.
3892 Src = S.Stk.pop<Pointer>();
3893 }
3894
3895 if (!CheckLoad(S, OpPC, Ptr: Src))
3896 return false;
3897
3898 const auto *RetVTy = Call->getType()->castAs<VectorType>();
3899 unsigned RetElems = RetVTy->getNumElements();
3900 unsigned SrcElems = Src.getNumElems();
3901 const Pointer &Dst = S.Stk.peek<Pointer>();
3902
3903 // Initialize destination with passthrough or zeros.
3904 for (unsigned I = 0; I != RetElems; ++I)
3905 if (IsMasked)
3906 Dst.elem<Floating>(I) = PassThrough.elem<Floating>(I);
3907 else
3908 Dst.elem<Floating>(I) = Floating(APFloat(0.0f));
3909
3910 assert(S.getASTContext().FloatTy == RetVTy->getElementType() &&
3911 "cvtpd2ps requires float element type in return vector");
3912
3913 // Convert double to float for enabled elements (only process source elements
3914 // that exist).
3915 for (unsigned I = 0; I != SrcElems; ++I) {
3916 if (IsMasked && !MaskVal[I])
3917 continue;
3918
3919 APFloat SrcVal = Src.elem<Floating>(I).getAPFloat();
3920
3921 Floating Conv = S.allocFloat(
3922 Sem: S.getASTContext().getFloatTypeSemantics(T: RetVTy->getElementType()));
3923 if (!convertDoubleToFloatStrict(Src: SrcVal, Dst&: Conv, S, DiagExpr: Call))
3924 return false;
3925 Dst.elem<Floating>(I) = Conv;
3926 }
3927
3928 Dst.initializeAllElements();
3929 return true;
3930}
3931
3932static bool interp__builtin_ia32_shuffle_generic(
3933 InterpState &S, CodePtr OpPC, const CallExpr *Call,
3934 llvm::function_ref<std::pair<unsigned, int>(unsigned, const APInt &)>
3935 GetSourceIndex) {
3936
3937 assert(Call->getNumArgs() == 2 || Call->getNumArgs() == 3);
3938
3939 APInt ShuffleMask;
3940 Pointer A, MaskVector, B;
3941 bool IsVectorMask = false;
3942 bool IsSingleOperand = (Call->getNumArgs() == 2);
3943
3944 if (IsSingleOperand) {
3945 QualType MaskType = Call->getArg(Arg: 1)->getType();
3946 if (MaskType->isVectorType()) {
3947 IsVectorMask = true;
3948 MaskVector = S.Stk.pop<Pointer>();
3949 A = S.Stk.pop<Pointer>();
3950 B = A;
3951 } else if (MaskType->isIntegerType()) {
3952 APSInt MaskVal;
3953 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: MaskVal))
3954 return false;
3955 ShuffleMask = MaskVal;
3956 A = S.Stk.pop<Pointer>();
3957 B = A;
3958 } else {
3959 return false;
3960 }
3961 } else {
3962 QualType Arg2Type = Call->getArg(Arg: 2)->getType();
3963 if (Arg2Type->isVectorType()) {
3964 IsVectorMask = true;
3965 B = S.Stk.pop<Pointer>();
3966 MaskVector = S.Stk.pop<Pointer>();
3967 A = S.Stk.pop<Pointer>();
3968 } else if (Arg2Type->isIntegerType()) {
3969 APSInt MaskVal;
3970 if (!popToAPSInt(S, E: Call->getArg(Arg: 2), Out&: MaskVal))
3971 return false;
3972 ShuffleMask = MaskVal;
3973 B = S.Stk.pop<Pointer>();
3974 A = S.Stk.pop<Pointer>();
3975 } else {
3976 return false;
3977 }
3978 }
3979
3980 QualType Arg0Type = Call->getArg(Arg: 0)->getType();
3981 const auto *VecT = Arg0Type->castAs<VectorType>();
3982 PrimType ElemT = *S.getContext().classify(T: VecT->getElementType());
3983 unsigned NumElems = VecT->getNumElements();
3984
3985 const Pointer &Dst = S.Stk.peek<Pointer>();
3986
3987 PrimType MaskElemT = PT_Uint32;
3988 if (IsVectorMask) {
3989 QualType Arg1Type = Call->getArg(Arg: 1)->getType();
3990 const auto *MaskVecT = Arg1Type->castAs<VectorType>();
3991 QualType MaskElemType = MaskVecT->getElementType();
3992 MaskElemT = *S.getContext().classify(T: MaskElemType);
3993 }
3994
3995 for (unsigned DstIdx = 0; DstIdx != NumElems; ++DstIdx) {
3996 if (IsVectorMask) {
3997 INT_TYPE_SWITCH(MaskElemT,
3998 { ShuffleMask = MaskVector.elem<T>(DstIdx).toAPSInt(); });
3999 }
4000
4001 auto [SrcVecIdx, SrcIdx] = GetSourceIndex(DstIdx, ShuffleMask);
4002
4003 if (SrcIdx < 0) {
4004 // Zero out this element
4005 if (ElemT == PT_Float) {
4006 Dst.elem<Floating>(I: DstIdx) = Floating(
4007 S.getASTContext().getFloatTypeSemantics(T: VecT->getElementType()));
4008 } else {
4009 INT_TYPE_SWITCH_NO_BOOL(ElemT, { Dst.elem<T>(DstIdx) = T::from(0); });
4010 }
4011 } else {
4012 const Pointer &Src = (SrcVecIdx == 0) ? A : B;
4013 TYPE_SWITCH(ElemT, { Dst.elem<T>(DstIdx) = Src.elem<T>(SrcIdx); });
4014 }
4015 }
4016 Dst.initializeAllElements();
4017
4018 return true;
4019}
4020
4021static bool interp__builtin_ia32_shuffle_generic(
4022 InterpState &S, CodePtr OpPC, const CallExpr *Call,
4023 llvm::function_ref<std::pair<unsigned, int>(unsigned, unsigned)>
4024 GetSourceIndex) {
4025 return interp__builtin_ia32_shuffle_generic(
4026 S, OpPC, Call,
4027 GetSourceIndex: [&GetSourceIndex](unsigned DstIdx,
4028 const APInt &Mask) -> std::pair<unsigned, int> {
4029 return GetSourceIndex(DstIdx, Mask.getZExtValue());
4030 });
4031}
4032
4033static bool interp__builtin_ia32_shift_with_count(
4034 InterpState &S, CodePtr OpPC, const CallExpr *Call,
4035 llvm::function_ref<APInt(const APInt &, uint64_t)> ShiftOp,
4036 llvm::function_ref<APInt(const APInt &, unsigned)> OverflowOp) {
4037
4038 assert(Call->getNumArgs() == 2);
4039
4040 const Pointer &Count = S.Stk.pop<Pointer>();
4041 const Pointer &Source = S.Stk.pop<Pointer>();
4042
4043 QualType SourceType = Call->getArg(Arg: 0)->getType();
4044 QualType CountType = Call->getArg(Arg: 1)->getType();
4045 assert(SourceType->isVectorType() && CountType->isVectorType());
4046
4047 const auto *SourceVecT = SourceType->castAs<VectorType>();
4048 const auto *CountVecT = CountType->castAs<VectorType>();
4049 PrimType SourceElemT = *S.getContext().classify(T: SourceVecT->getElementType());
4050 PrimType CountElemT = *S.getContext().classify(T: CountVecT->getElementType());
4051
4052 const Pointer &Dst = S.Stk.peek<Pointer>();
4053
4054 unsigned DestEltWidth =
4055 S.getASTContext().getTypeSize(T: SourceVecT->getElementType());
4056 bool IsDestUnsigned = SourceVecT->getElementType()->isUnsignedIntegerType();
4057 unsigned DestLen = SourceVecT->getNumElements();
4058 unsigned CountEltWidth =
4059 S.getASTContext().getTypeSize(T: CountVecT->getElementType());
4060 unsigned NumBitsInQWord = 64;
4061 unsigned NumCountElts = NumBitsInQWord / CountEltWidth;
4062
4063 uint64_t CountLQWord = 0;
4064 for (unsigned EltIdx = 0; EltIdx != NumCountElts; ++EltIdx) {
4065 uint64_t Elt = 0;
4066 INT_TYPE_SWITCH(CountElemT,
4067 { Elt = static_cast<uint64_t>(Count.elem<T>(EltIdx)); });
4068 CountLQWord |= (Elt << (EltIdx * CountEltWidth));
4069 }
4070
4071 for (unsigned EltIdx = 0; EltIdx != DestLen; ++EltIdx) {
4072 APSInt Elt;
4073 INT_TYPE_SWITCH(SourceElemT, { Elt = Source.elem<T>(EltIdx).toAPSInt(); });
4074
4075 APInt Result;
4076 if (CountLQWord < DestEltWidth) {
4077 Result = ShiftOp(Elt, CountLQWord);
4078 } else {
4079 Result = OverflowOp(Elt, DestEltWidth);
4080 }
4081 if (IsDestUnsigned) {
4082 INT_TYPE_SWITCH(SourceElemT, {
4083 Dst.elem<T>(EltIdx) = T::from(Result.getZExtValue());
4084 });
4085 } else {
4086 INT_TYPE_SWITCH(SourceElemT, {
4087 Dst.elem<T>(EltIdx) = T::from(Result.getSExtValue());
4088 });
4089 }
4090 }
4091
4092 Dst.initializeAllElements();
4093 return true;
4094}
4095
4096static bool interp__builtin_ia32_shufbitqmb_mask(InterpState &S, CodePtr OpPC,
4097 const CallExpr *Call) {
4098
4099 assert(Call->getNumArgs() == 3);
4100
4101 QualType SourceType = Call->getArg(Arg: 0)->getType();
4102 QualType ShuffleMaskType = Call->getArg(Arg: 1)->getType();
4103 QualType ZeroMaskType = Call->getArg(Arg: 2)->getType();
4104 if (!SourceType->isVectorType() || !ShuffleMaskType->isVectorType() ||
4105 !ZeroMaskType->isIntegerType()) {
4106 return false;
4107 }
4108
4109 Pointer Source, ShuffleMask;
4110 APSInt ZeroMask;
4111 if (!popToAPSInt(S, E: Call->getArg(Arg: 2), Out&: ZeroMask))
4112 return false;
4113 ShuffleMask = S.Stk.pop<Pointer>();
4114 Source = S.Stk.pop<Pointer>();
4115
4116 const auto *SourceVecT = SourceType->castAs<VectorType>();
4117 const auto *ShuffleMaskVecT = ShuffleMaskType->castAs<VectorType>();
4118 assert(SourceVecT->getNumElements() == ShuffleMaskVecT->getNumElements());
4119 assert(ZeroMask.getBitWidth() == SourceVecT->getNumElements());
4120
4121 PrimType SourceElemT = *S.getContext().classify(T: SourceVecT->getElementType());
4122 PrimType ShuffleMaskElemT =
4123 *S.getContext().classify(T: ShuffleMaskVecT->getElementType());
4124
4125 unsigned NumBytesInQWord = 8;
4126 unsigned NumBitsInByte = 8;
4127 unsigned NumBytes = SourceVecT->getNumElements();
4128 unsigned NumQWords = NumBytes / NumBytesInQWord;
4129 unsigned RetWidth = ZeroMask.getBitWidth();
4130 APSInt RetMask(llvm::APInt(RetWidth, 0), /*isUnsigned=*/true);
4131
4132 for (unsigned QWordId = 0; QWordId != NumQWords; ++QWordId) {
4133 APInt SourceQWord(64, 0);
4134 for (unsigned ByteIdx = 0; ByteIdx != NumBytesInQWord; ++ByteIdx) {
4135 uint64_t Byte = 0;
4136 INT_TYPE_SWITCH(SourceElemT, {
4137 Byte = static_cast<uint64_t>(
4138 Source.elem<T>(QWordId * NumBytesInQWord + ByteIdx));
4139 });
4140 SourceQWord.insertBits(SubBits: APInt(8, Byte & 0xFF), bitPosition: ByteIdx * NumBitsInByte);
4141 }
4142
4143 for (unsigned ByteIdx = 0; ByteIdx != NumBytesInQWord; ++ByteIdx) {
4144 unsigned SelIdx = QWordId * NumBytesInQWord + ByteIdx;
4145 unsigned M = 0;
4146 INT_TYPE_SWITCH(ShuffleMaskElemT, {
4147 M = static_cast<unsigned>(ShuffleMask.elem<T>(SelIdx)) & 0x3F;
4148 });
4149
4150 if (ZeroMask[SelIdx]) {
4151 RetMask.setBitVal(BitPosition: SelIdx, BitValue: SourceQWord[M]);
4152 }
4153 }
4154 }
4155
4156 pushInteger(S, Val: RetMask, QT: Call->getType());
4157 return true;
4158}
4159
4160static bool interp__builtin_ia32_vcvtps2ph(InterpState &S, CodePtr OpPC,
4161 const CallExpr *Call) {
4162 // Arguments are: vector of floats, rounding immediate
4163 assert(Call->getNumArgs() == 2);
4164
4165 APSInt Imm;
4166 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: Imm))
4167 return false;
4168 const Pointer &Src = S.Stk.pop<Pointer>();
4169 const Pointer &Dst = S.Stk.peek<Pointer>();
4170
4171 assert(Src.getFieldDesc()->isPrimitiveArray());
4172 assert(Dst.getFieldDesc()->isPrimitiveArray());
4173
4174 const auto *SrcVTy = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
4175 unsigned SrcNumElems = SrcVTy->getNumElements();
4176 const auto *DstVTy = Call->getType()->castAs<VectorType>();
4177 unsigned DstNumElems = DstVTy->getNumElements();
4178
4179 const llvm::fltSemantics &HalfSem =
4180 S.getASTContext().getFloatTypeSemantics(T: S.getASTContext().HalfTy);
4181
4182 // imm[2] == 1 means use MXCSR rounding mode.
4183 // In that case, we can only evaluate if the conversion is exact.
4184 int ImmVal = Imm.getZExtValue();
4185 bool UseMXCSR = (ImmVal & 4) != 0;
4186 bool IsFPConstrained =
4187 Call->getFPFeaturesInEffect(LO: S.getASTContext().getLangOpts())
4188 .isFPConstrained();
4189
4190 llvm::RoundingMode RM;
4191 if (!UseMXCSR) {
4192 switch (ImmVal & 3) {
4193 case 0:
4194 RM = llvm::RoundingMode::NearestTiesToEven;
4195 break;
4196 case 1:
4197 RM = llvm::RoundingMode::TowardNegative;
4198 break;
4199 case 2:
4200 RM = llvm::RoundingMode::TowardPositive;
4201 break;
4202 case 3:
4203 RM = llvm::RoundingMode::TowardZero;
4204 break;
4205 default:
4206 llvm_unreachable("Invalid immediate rounding mode");
4207 }
4208 } else {
4209 // For MXCSR, we must check for exactness. We can use any rounding mode
4210 // for the trial conversion since the result is the same if it's exact.
4211 RM = llvm::RoundingMode::NearestTiesToEven;
4212 }
4213
4214 QualType DstElemQT = Dst.getFieldDesc()->getElemQualType();
4215 PrimType DstElemT = *S.getContext().classify(T: DstElemQT);
4216
4217 for (unsigned I = 0; I != SrcNumElems; ++I) {
4218 Floating SrcVal = Src.elem<Floating>(I);
4219 APFloat DstVal = SrcVal.getAPFloat();
4220
4221 bool LostInfo;
4222 APFloat::opStatus St = DstVal.convert(ToSemantics: HalfSem, RM, losesInfo: &LostInfo);
4223
4224 if (UseMXCSR && IsFPConstrained && St != APFloat::opOK) {
4225 S.FFDiag(SI: S.Current->getSource(PC: OpPC),
4226 DiagId: diag::note_constexpr_dynamic_rounding);
4227 return false;
4228 }
4229
4230 INT_TYPE_SWITCH_NO_BOOL(DstElemT, {
4231 // Convert the destination value's bit pattern to an unsigned integer,
4232 // then reconstruct the element using the target type's 'from' method.
4233 uint64_t RawBits = DstVal.bitcastToAPInt().getZExtValue();
4234 Dst.elem<T>(I) = T::from(RawBits);
4235 });
4236 }
4237
4238 // Zero out remaining elements if the destination has more elements
4239 // (e.g., vcvtps2ph converting 4 floats to 8 shorts).
4240 if (DstNumElems > SrcNumElems) {
4241 for (unsigned I = SrcNumElems; I != DstNumElems; ++I) {
4242 INT_TYPE_SWITCH_NO_BOOL(DstElemT, { Dst.elem<T>(I) = T::from(0); });
4243 }
4244 }
4245
4246 Dst.initializeAllElements();
4247 return true;
4248}
4249
4250static bool interp__builtin_ia32_multishiftqb(InterpState &S, CodePtr OpPC,
4251 const CallExpr *Call) {
4252 assert(Call->getNumArgs() == 2);
4253
4254 QualType ATy = Call->getArg(Arg: 0)->getType();
4255 QualType BTy = Call->getArg(Arg: 1)->getType();
4256 if (!ATy->isVectorType() || !BTy->isVectorType()) {
4257 return false;
4258 }
4259
4260 const Pointer &BPtr = S.Stk.pop<Pointer>();
4261 const Pointer &APtr = S.Stk.pop<Pointer>();
4262 const auto *AVecT = ATy->castAs<VectorType>();
4263 assert(AVecT->getNumElements() ==
4264 BTy->castAs<VectorType>()->getNumElements());
4265
4266 PrimType ElemT = *S.getContext().classify(T: AVecT->getElementType());
4267
4268 unsigned NumBytesInQWord = 8;
4269 unsigned NumBitsInByte = 8;
4270 unsigned NumBytes = AVecT->getNumElements();
4271 unsigned NumQWords = NumBytes / NumBytesInQWord;
4272 const Pointer &Dst = S.Stk.peek<Pointer>();
4273
4274 for (unsigned QWordId = 0; QWordId != NumQWords; ++QWordId) {
4275 APInt BQWord(64, 0);
4276 for (unsigned ByteIdx = 0; ByteIdx != NumBytesInQWord; ++ByteIdx) {
4277 unsigned Idx = QWordId * NumBytesInQWord + ByteIdx;
4278 INT_TYPE_SWITCH(ElemT, {
4279 uint64_t Byte = static_cast<uint64_t>(BPtr.elem<T>(Idx));
4280 BQWord.insertBits(APInt(8, Byte & 0xFF), ByteIdx * NumBitsInByte);
4281 });
4282 }
4283
4284 for (unsigned ByteIdx = 0; ByteIdx != NumBytesInQWord; ++ByteIdx) {
4285 unsigned Idx = QWordId * NumBytesInQWord + ByteIdx;
4286 uint64_t Ctrl = 0;
4287 INT_TYPE_SWITCH(
4288 ElemT, { Ctrl = static_cast<uint64_t>(APtr.elem<T>(Idx)) & 0x3F; });
4289
4290 APInt Byte(8, 0);
4291 for (unsigned BitIdx = 0; BitIdx != NumBitsInByte; ++BitIdx) {
4292 Byte.setBitVal(BitPosition: BitIdx, BitValue: BQWord[(Ctrl + BitIdx) & 0x3F]);
4293 }
4294 INT_TYPE_SWITCH(ElemT,
4295 { Dst.elem<T>(Idx) = T::from(Byte.getZExtValue()); });
4296 }
4297 }
4298
4299 Dst.initializeAllElements();
4300
4301 return true;
4302}
4303
4304static bool interp__builtin_ia32_gfni_affine(InterpState &S, CodePtr OpPC,
4305 const CallExpr *Call,
4306 bool Inverse) {
4307 assert(Call->getNumArgs() == 3);
4308 QualType XType = Call->getArg(Arg: 0)->getType();
4309 QualType AType = Call->getArg(Arg: 1)->getType();
4310 QualType ImmType = Call->getArg(Arg: 2)->getType();
4311 if (!XType->isVectorType() || !AType->isVectorType() ||
4312 !ImmType->isIntegerType()) {
4313 return false;
4314 }
4315
4316 Pointer X, A;
4317 APSInt Imm;
4318 if (!popToAPSInt(S, E: Call->getArg(Arg: 2), Out&: Imm))
4319 return false;
4320 A = S.Stk.pop<Pointer>();
4321 X = S.Stk.pop<Pointer>();
4322
4323 const Pointer &Dst = S.Stk.peek<Pointer>();
4324 const auto *AVecT = AType->castAs<VectorType>();
4325 assert(XType->castAs<VectorType>()->getNumElements() ==
4326 AVecT->getNumElements());
4327 unsigned NumBytesInQWord = 8;
4328 unsigned NumBytes = AVecT->getNumElements();
4329 unsigned NumBitsInQWord = 64;
4330 unsigned NumQWords = NumBytes / NumBytesInQWord;
4331 unsigned NumBitsInByte = 8;
4332 PrimType AElemT = *S.getContext().classify(T: AVecT->getElementType());
4333
4334 // computing A*X + Imm
4335 for (unsigned QWordIdx = 0; QWordIdx != NumQWords; ++QWordIdx) {
4336 // Extract the QWords from X, A
4337 APInt XQWord(NumBitsInQWord, 0);
4338 APInt AQWord(NumBitsInQWord, 0);
4339 for (unsigned ByteIdx = 0; ByteIdx != NumBytesInQWord; ++ByteIdx) {
4340 unsigned Idx = QWordIdx * NumBytesInQWord + ByteIdx;
4341 uint8_t XByte;
4342 uint8_t AByte;
4343 INT_TYPE_SWITCH(AElemT, {
4344 XByte = static_cast<uint8_t>(X.elem<T>(Idx));
4345 AByte = static_cast<uint8_t>(A.elem<T>(Idx));
4346 });
4347
4348 XQWord.insertBits(SubBits: APInt(NumBitsInByte, XByte), bitPosition: ByteIdx * NumBitsInByte);
4349 AQWord.insertBits(SubBits: APInt(NumBitsInByte, AByte), bitPosition: ByteIdx * NumBitsInByte);
4350 }
4351
4352 for (unsigned ByteIdx = 0; ByteIdx != NumBytesInQWord; ++ByteIdx) {
4353 unsigned Idx = QWordIdx * NumBytesInQWord + ByteIdx;
4354 uint8_t XByte =
4355 XQWord.lshr(shiftAmt: ByteIdx * NumBitsInByte).getLoBits(numBits: 8).getZExtValue();
4356 INT_TYPE_SWITCH(AElemT, {
4357 Dst.elem<T>(Idx) = T::from(GFNIAffine(XByte, AQWord, Imm, Inverse));
4358 });
4359 }
4360 }
4361 Dst.initializeAllElements();
4362 return true;
4363}
4364
4365static bool interp__builtin_ia32_gfni_mul(InterpState &S, CodePtr OpPC,
4366 const CallExpr *Call) {
4367 assert(Call->getNumArgs() == 2);
4368
4369 QualType AType = Call->getArg(Arg: 0)->getType();
4370 QualType BType = Call->getArg(Arg: 1)->getType();
4371 if (!AType->isVectorType() || !BType->isVectorType()) {
4372 return false;
4373 }
4374
4375 Pointer A, B;
4376 B = S.Stk.pop<Pointer>();
4377 A = S.Stk.pop<Pointer>();
4378
4379 const Pointer &Dst = S.Stk.peek<Pointer>();
4380 const auto *AVecT = AType->castAs<VectorType>();
4381 assert(AVecT->getNumElements() ==
4382 BType->castAs<VectorType>()->getNumElements());
4383
4384 PrimType AElemT = *S.getContext().classify(T: AVecT->getElementType());
4385 unsigned NumBytes = A.getNumElems();
4386
4387 for (unsigned ByteIdx = 0; ByteIdx != NumBytes; ++ByteIdx) {
4388 uint8_t AByte, BByte;
4389 INT_TYPE_SWITCH(AElemT, {
4390 AByte = static_cast<uint8_t>(A.elem<T>(ByteIdx));
4391 BByte = static_cast<uint8_t>(B.elem<T>(ByteIdx));
4392 Dst.elem<T>(ByteIdx) = T::from(GFNIMul(AByte, BByte));
4393 });
4394 }
4395
4396 Dst.initializeAllElements();
4397 return true;
4398}
4399
4400static bool interp__builtin_ia32_vpdp(InterpState &S, CodePtr OpPC,
4401 const CallExpr *Call, bool IsSaturating) {
4402 assert(Call->getNumArgs() == 3);
4403
4404 QualType SrcT = Call->getArg(Arg: 0)->getType();
4405 QualType OpAT = Call->getArg(Arg: 1)->getType();
4406 QualType OpBT = Call->getArg(Arg: 2)->getType();
4407 QualType DstT = Call->getType();
4408 if (!SrcT->isVectorType() || !OpAT->isVectorType() || !OpBT->isVectorType() ||
4409 !DstT->isVectorType())
4410 return false;
4411
4412 const auto *SrcVecT = SrcT->castAs<VectorType>();
4413 const auto *OpAVecT = OpAT->castAs<VectorType>();
4414 const auto *OpBVecT = OpBT->castAs<VectorType>();
4415 const auto *DstVecT = DstT->castAs<VectorType>();
4416
4417 assert(OpAVecT->getNumElements() == OpBVecT->getNumElements());
4418
4419 unsigned NumSrcElems = SrcVecT->getNumElements();
4420 unsigned NumOperandElems = OpAVecT->getNumElements();
4421 unsigned ElemsPerLane = NumOperandElems / NumSrcElems;
4422
4423 PrimType SrcElemT = *S.getContext().classify(T: SrcVecT->getElementType());
4424 PrimType OpAElemT = *S.getContext().classify(T: OpAVecT->getElementType());
4425 PrimType OpBElemT = *S.getContext().classify(T: OpBVecT->getElementType());
4426 PrimType DstElemT = *S.getContext().classify(T: DstVecT->getElementType());
4427
4428 assert(SrcElemT == DstElemT);
4429
4430 const Pointer &OpBPtr = S.Stk.pop<Pointer>();
4431 const Pointer &OpAPtr = S.Stk.pop<Pointer>();
4432 const Pointer &SrcPtr = S.Stk.pop<Pointer>();
4433 const Pointer &Dst = S.Stk.peek<Pointer>();
4434
4435 for (unsigned I = 0; I != NumSrcElems; ++I) {
4436 APSInt Acc;
4437 INT_TYPE_SWITCH_NO_BOOL(SrcElemT, { Acc = SrcPtr.elem<T>(I).toAPSInt(); });
4438 Acc = Acc.sext(width: 64);
4439 for (unsigned J = 0; J != ElemsPerLane; ++J) {
4440 APSInt OpA, OpB;
4441 INT_TYPE_SWITCH_NO_BOOL(
4442 OpAElemT, { OpA = OpAPtr.elem<T>(ElemsPerLane * I + J).toAPSInt(); });
4443 INT_TYPE_SWITCH_NO_BOOL(
4444 OpBElemT, { OpB = OpBPtr.elem<T>(ElemsPerLane * I + J).toAPSInt(); });
4445 OpA = APSInt(OpA.extend(width: 64), false);
4446 OpB = APSInt(OpB.extend(width: 64), false);
4447 Acc += OpA * OpB;
4448 }
4449 if (IsSaturating)
4450 Acc = APSInt(Acc.truncSSat(width: 32), false);
4451 else
4452 Acc = APSInt(Acc.trunc(width: 32), false);
4453 INT_TYPE_SWITCH_NO_BOOL(DstElemT,
4454 { Dst.elem<T>(I) = static_cast<T>(Acc); });
4455 }
4456 Dst.initializeAllElements();
4457 return true;
4458}
4459
4460// Bit Matrix Multiply and Accumulate (AVX512BMM). Each 256-bit lane holds a
4461// 16x16 bit matrix as 16 x i16 elements; element i is row i and bit j of that
4462// element is entry [i][j]. The accumulator (third argument, src1 in the AMD
4463// ISA) provides the initial value of each result bit, into which the bit-matrix
4464// product of the first two arguments (src2 * src3) is reduced with OR (vbmacor)
4465// or XOR (vbmacxor):
4466// for i in 0..15, j in 0..15:
4467// bit = C[16*i+j]
4468// for k in 0..15: bit OP= A[16*i+k] & B[16*k+j]
4469// dest[16*i+j] = bit
4470static bool interp__builtin_ia32_bmac(InterpState &S, CodePtr OpPC,
4471 const CallExpr *Call, bool IsXor) {
4472 assert(Call->getNumArgs() == 3);
4473
4474 // AST-based type checks before popping the stack.
4475 QualType AType = Call->getArg(Arg: 0)->getType();
4476 QualType BType = Call->getArg(Arg: 1)->getType();
4477 QualType CType = Call->getArg(Arg: 2)->getType();
4478 if (!AType->isVectorType() || !BType->isVectorType() ||
4479 !CType->isVectorType())
4480 return false;
4481
4482 const Pointer &C = S.Stk.pop<Pointer>();
4483 const Pointer &B = S.Stk.pop<Pointer>();
4484 const Pointer &A = S.Stk.pop<Pointer>();
4485 const Pointer &Dst = S.Stk.peek<Pointer>();
4486
4487 // check if all three primitive arrays are with 16-bit elements.
4488 auto isValid16BitArray = [](const Pointer &P) {
4489 const Descriptor *D = P.getFieldDesc();
4490 if (!D->isPrimitiveArray())
4491 return false;
4492 PrimType PT = D->getPrimType();
4493 return ((PT == PT_Sint16) || (PT == PT_Uint16));
4494 };
4495
4496 if (!isValid16BitArray(A) || !isValid16BitArray(B) || !isValid16BitArray(C))
4497 return false;
4498
4499 PrimType ElemT = A.getFieldDesc()->getPrimType();
4500 unsigned NumElems = A.getNumElems();
4501 assert(NumElems % 16 == 0 && "BMM operates on 256-bit lanes of 16 x i16");
4502 bool DstUnsigned = ElemT == PT_Uint16;
4503
4504 // Lanes are always 16-bit; gather them so the reduction below is untyped.
4505 SmallVector<uint16_t> AVals(NumElems), BVals(NumElems), Acc(NumElems);
4506 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
4507 for (unsigned I = 0; I != NumElems; ++I) {
4508 AVals[I] = (uint16_t)A.elem<T>(I).toAPSInt().getZExtValue();
4509 BVals[I] = (uint16_t)B.elem<T>(I).toAPSInt().getZExtValue();
4510 Acc[I] = (uint16_t)C.elem<T>(I).toAPSInt().getZExtValue();
4511 }
4512 });
4513
4514 for (unsigned Lane = 0; Lane != NumElems; Lane += 16) {
4515 for (unsigned I = 0; I != 16; ++I) {
4516 uint16_t AVal = AVals[Lane + I], DVal = Acc[Lane + I];
4517 for (unsigned J = 0; J != 16; ++J) {
4518 // Seed the reduction with the accumulator bit, then fold in each
4519 // product term with the same operator (OR for vbmacor, XOR for
4520 // vbmacxor).
4521 unsigned Bit = (DVal >> J) & 1u;
4522 for (unsigned K = 0; K != 16; ++K) {
4523 unsigned Product = ((AVal >> K) & 1u) & ((BVals[Lane + K] >> J) & 1u);
4524 Bit = IsXor ? (Bit ^ Product) : (Bit | Product);
4525 }
4526 DVal = (DVal & ~(uint16_t(1) << J)) | (uint16_t(Bit) << J);
4527 }
4528 Acc[Lane + I] = DVal;
4529 }
4530 }
4531
4532 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
4533 for (unsigned I = 0; I != NumElems; ++I)
4534 Dst.elem<T>(I) = static_cast<T>(APSInt(APInt(16, Acc[I]), DstUnsigned));
4535 });
4536 Dst.initializeAllElements();
4537 return true;
4538}
4539
4540static bool interp_builtin_ia32_cvt_scalar_to_int(InterpState &S, CodePtr OpPC,
4541 const CallExpr *E) {
4542 Pointer SrcVecPtr = S.Stk.pop<Pointer>();
4543 const Floating &FloatElem = SrcVecPtr.elem<Floating>(I: 0);
4544
4545 unsigned BitWidth = S.getASTContext().getIntWidth(T: E->getType());
4546 bool IsUnsigned = E->getType()->isUnsignedIntegerType();
4547
4548 llvm::APSInt IntResult(BitWidth, IsUnsigned);
4549 bool IsExact = false;
4550 // We only allow exact conversions so rounding mode does not matter for cvt*
4551 // and cvtt* builtins
4552 FloatElem.getAPFloat().convertToInteger(
4553 Result&: IntResult, RM: llvm::APFloat::rmTowardZero, IsExact: &IsExact);
4554 if (!IsExact)
4555 return false;
4556
4557 pushInteger(S, Val: IntResult, QT: E->getType());
4558 return true;
4559}
4560
4561static bool interp_builtin_ia32_cvt_vector_to_int(InterpState &S, CodePtr OpPC,
4562 const CallExpr *E) {
4563 Pointer SrcVecPtr = S.Stk.pop<Pointer>();
4564 const Pointer &Dst = S.Stk.peek<Pointer>();
4565
4566 unsigned NumSrcElems = SrcVecPtr.getNumElems();
4567 unsigned NumDstElems = Dst.getNumElems();
4568
4569 if (NumSrcElems > NumDstElems)
4570 return false;
4571
4572 QualType ElemType = Dst.getFieldDesc()->getElemQualType();
4573 unsigned BitWidth = S.getASTContext().getIntWidth(T: ElemType);
4574 bool IsUnsigned = ElemType->isUnsignedIntegerType();
4575
4576 PrimType ElemT = *S.getContext().classify(T: ElemType);
4577 for (unsigned I = 0; I != NumSrcElems; ++I) {
4578 const Floating &FloatElem = SrcVecPtr.elem<Floating>(I);
4579 llvm::APSInt IntResult(BitWidth, IsUnsigned);
4580
4581 bool IsExact = false;
4582 // We only allow exact conversions so rounding mode does not matter for
4583 // cvt* and cvtt* builtins
4584 FloatElem.getAPFloat().convertToInteger(
4585 Result&: IntResult, RM: llvm::APFloat::rmTowardZero, IsExact: &IsExact);
4586 if (!IsExact)
4587 return false;
4588 INT_TYPE_SWITCH_NO_BOOL(
4589 ElemT, { Dst.elem<T>(I) = T::from(IntResult.getZExtValue()); });
4590 }
4591
4592 // Zero out remaining elements if the destination has more elements
4593 // (e.g., cvtpd2dq converting 2 doubles(_m128d) to 2 ints stored in _m128i).
4594 for (unsigned I = NumSrcElems; I != NumDstElems; ++I)
4595 INT_TYPE_SWITCH_NO_BOOL(ElemT, { Dst.elem<T>(I) = T::from(0); });
4596
4597 Dst.initializeAllElements();
4598 return true;
4599}
4600
4601bool InterpretBuiltin(InterpState &S, CodePtr OpPC, const CallExpr *Call,
4602 uint32_t BuiltinID) {
4603 const ASTContext &ASTCtx = S.getASTContext();
4604
4605 // BuiltinID is the raw ID baked into the bytecode. The "is constant
4606 // evaluated" gate needs the raw ID so that auxiliary-target IDs resolve into
4607 // the correct (aux-target) builtin records.
4608 if (!ASTCtx.BuiltinInfo.isConstantEvaluated(ID: BuiltinID))
4609 return Invalid(S, OpPC);
4610
4611 // Convert an auxiliary x86 target builtin ID to its canonical X86::BI* value
4612 // so the target-specific cases below (and the handlers they call) match. This
4613 // is a cheap integer operation (a single comparison for the common,
4614 // target-independent case); we deliberately avoid re-deriving the ID from the
4615 // call expression, which is comparatively slow.
4616 BuiltinID = ConvertBuiltinIDToX86BuiltinID(Ctx: ASTCtx, BuiltinID);
4617
4618 const InterpFrame *Frame = S.Current;
4619 switch (BuiltinID) {
4620 case Builtin::BI__builtin_is_constant_evaluated:
4621 return interp__builtin_is_constant_evaluated(S, OpPC, Frame, Call);
4622
4623 case Builtin::BI__builtin_assume:
4624 case Builtin::BI__assume:
4625 return interp__builtin_assume(S, OpPC, Frame, Call);
4626
4627 case Builtin::BI__builtin_strcmp:
4628 case Builtin::BIstrcmp:
4629 case Builtin::BI__builtin_strncmp:
4630 case Builtin::BIstrncmp:
4631 case Builtin::BI__builtin_wcsncmp:
4632 case Builtin::BIwcsncmp:
4633 case Builtin::BI__builtin_wcscmp:
4634 case Builtin::BIwcscmp:
4635 return interp__builtin_strcmp(S, OpPC, Frame, Call, ID: BuiltinID);
4636
4637 case Builtin::BI__builtin_strlen:
4638 case Builtin::BIstrlen:
4639 case Builtin::BI__builtin_wcslen:
4640 case Builtin::BIwcslen:
4641 return interp__builtin_strlen(S, OpPC, Frame, Call, ID: BuiltinID);
4642
4643 case Builtin::BI__builtin_nan:
4644 case Builtin::BI__builtin_nanf:
4645 case Builtin::BI__builtin_nanl:
4646 case Builtin::BI__builtin_nanf16:
4647 case Builtin::BI__builtin_nanf128:
4648 return interp__builtin_nan(S, OpPC, Frame, Call, /*Signaling=*/false);
4649
4650 case Builtin::BI__builtin_nans:
4651 case Builtin::BI__builtin_nansf:
4652 case Builtin::BI__builtin_nansl:
4653 case Builtin::BI__builtin_nansf16:
4654 case Builtin::BI__builtin_nansf128:
4655 return interp__builtin_nan(S, OpPC, Frame, Call, /*Signaling=*/true);
4656
4657 case Builtin::BI__builtin_huge_val:
4658 case Builtin::BI__builtin_huge_valf:
4659 case Builtin::BI__builtin_huge_vall:
4660 case Builtin::BI__builtin_huge_valf16:
4661 case Builtin::BI__builtin_huge_valf128:
4662 case Builtin::BI__builtin_inf:
4663 case Builtin::BI__builtin_inff:
4664 case Builtin::BI__builtin_infl:
4665 case Builtin::BI__builtin_inff16:
4666 case Builtin::BI__builtin_inff128:
4667 return interp__builtin_inf(S, OpPC, Frame, Call);
4668
4669 case Builtin::BI__builtin_copysign:
4670 case Builtin::BI__builtin_copysignf:
4671 case Builtin::BI__builtin_copysignl:
4672 case Builtin::BI__builtin_copysignf128:
4673 return interp__builtin_copysign(S, OpPC, Frame);
4674
4675 case Builtin::BI__builtin_fmin:
4676 case Builtin::BI__builtin_fminf:
4677 case Builtin::BI__builtin_fminl:
4678 case Builtin::BI__builtin_fminf16:
4679 case Builtin::BI__builtin_fminf128:
4680 return interp__builtin_fmin(S, OpPC, Frame, /*IsNumBuiltin=*/false);
4681
4682 case Builtin::BI__builtin_fminimum_num:
4683 case Builtin::BI__builtin_fminimum_numf:
4684 case Builtin::BI__builtin_fminimum_numl:
4685 case Builtin::BI__builtin_fminimum_numf16:
4686 case Builtin::BI__builtin_fminimum_numf128:
4687 return interp__builtin_fmin(S, OpPC, Frame, /*IsNumBuiltin=*/true);
4688
4689 case Builtin::BI__builtin_fmax:
4690 case Builtin::BI__builtin_fmaxf:
4691 case Builtin::BI__builtin_fmaxl:
4692 case Builtin::BI__builtin_fmaxf16:
4693 case Builtin::BI__builtin_fmaxf128:
4694 return interp__builtin_fmax(S, OpPC, Frame, /*IsNumBuiltin=*/false);
4695
4696 case Builtin::BI__builtin_fmaximum_num:
4697 case Builtin::BI__builtin_fmaximum_numf:
4698 case Builtin::BI__builtin_fmaximum_numl:
4699 case Builtin::BI__builtin_fmaximum_numf16:
4700 case Builtin::BI__builtin_fmaximum_numf128:
4701 return interp__builtin_fmax(S, OpPC, Frame, /*IsNumBuiltin=*/true);
4702
4703 case Builtin::BI__builtin_isnan:
4704 return interp__builtin_isnan(S, OpPC, Frame, Call);
4705
4706 case Builtin::BI__builtin_issignaling:
4707 return interp__builtin_issignaling(S, OpPC, Frame, Call);
4708
4709 case Builtin::BI__builtin_isinf:
4710 return interp__builtin_isinf(S, OpPC, Frame, /*Sign=*/CheckSign: false, Call);
4711
4712 case Builtin::BI__builtin_isinf_sign:
4713 return interp__builtin_isinf(S, OpPC, Frame, /*Sign=*/CheckSign: true, Call);
4714
4715 case Builtin::BI__builtin_isfinite:
4716 return interp__builtin_isfinite(S, OpPC, Frame, Call);
4717
4718 case Builtin::BI__builtin_isnormal:
4719 return interp__builtin_isnormal(S, OpPC, Frame, Call);
4720
4721 case Builtin::BI__builtin_issubnormal:
4722 return interp__builtin_issubnormal(S, OpPC, Frame, Call);
4723
4724 case Builtin::BI__builtin_iszero:
4725 return interp__builtin_iszero(S, OpPC, Frame, Call);
4726
4727 case Builtin::BI__builtin_signbit:
4728 case Builtin::BI__builtin_signbitf:
4729 case Builtin::BI__builtin_signbitl:
4730 return interp__builtin_signbit(S, OpPC, Frame, Call);
4731
4732 case Builtin::BI__builtin_isgreater:
4733 case Builtin::BI__builtin_isgreaterequal:
4734 case Builtin::BI__builtin_isless:
4735 case Builtin::BI__builtin_islessequal:
4736 case Builtin::BI__builtin_islessgreater:
4737 case Builtin::BI__builtin_isunordered:
4738 return interp_floating_comparison(S, OpPC, Call, ID: BuiltinID);
4739
4740 case Builtin::BI__builtin_isfpclass:
4741 return interp__builtin_isfpclass(S, OpPC, Frame, Call);
4742
4743 case Builtin::BI__builtin_fpclassify:
4744 return interp__builtin_fpclassify(S, OpPC, Frame, Call);
4745
4746 case Builtin::BI__builtin_fabs:
4747 case Builtin::BI__builtin_fabsf:
4748 case Builtin::BI__builtin_fabsl:
4749 case Builtin::BI__builtin_fabsf128:
4750 return interp__builtin_fabs(S, OpPC, Frame);
4751
4752 case Builtin::BI__builtin_abs:
4753 case Builtin::BI__builtin_labs:
4754 case Builtin::BI__builtin_llabs:
4755 return interp__builtin_abs(S, OpPC, Frame, Call);
4756
4757 case Builtin::BI__builtin_popcount:
4758 case Builtin::BI__builtin_popcountl:
4759 case Builtin::BI__builtin_popcountll:
4760 case Builtin::BI__builtin_popcountg:
4761 case Builtin::BI__popcnt16: // Microsoft variants of popcount
4762 case Builtin::BI__popcnt:
4763 case Builtin::BI__popcnt64:
4764 return interp__builtin_popcount(S, OpPC, Frame, Call);
4765
4766 case Builtin::BI__builtin_parity:
4767 case Builtin::BI__builtin_parityl:
4768 case Builtin::BI__builtin_parityll:
4769 return interp__builtin_elementwise_int_unaryop(
4770 S, OpPC, Call, Fn: [](const APSInt &Val) {
4771 return APInt(Val.getBitWidth(), Val.popcount() % 2);
4772 });
4773 case Builtin::BI__builtin_clrsb:
4774 case Builtin::BI__builtin_clrsbl:
4775 case Builtin::BI__builtin_clrsbll:
4776 return interp__builtin_elementwise_int_unaryop(
4777 S, OpPC, Call, Fn: [](const APSInt &Val) {
4778 return APInt(Val.getBitWidth(),
4779 Val.getBitWidth() - Val.getSignificantBits());
4780 });
4781 case Builtin::BI__builtin_bitreverseg:
4782 case Builtin::BI__builtin_bitreverse8:
4783 case Builtin::BI__builtin_bitreverse16:
4784 case Builtin::BI__builtin_bitreverse32:
4785 case Builtin::BI__builtin_bitreverse64:
4786 return interp__builtin_elementwise_int_unaryop(
4787 S, OpPC, Call, Fn: [](const APSInt &Val) { return Val.reverseBits(); });
4788
4789 case Builtin::BI__builtin_classify_type:
4790 return interp__builtin_classify_type(S, OpPC, Frame, Call);
4791
4792 case Builtin::BI__builtin_expect:
4793 case Builtin::BI__builtin_expect_with_probability:
4794 return interp__builtin_expect(S, OpPC, Frame, Call);
4795
4796 case Builtin::BI__builtin_rotateleft8:
4797 case Builtin::BI__builtin_rotateleft16:
4798 case Builtin::BI__builtin_rotateleft32:
4799 case Builtin::BI__builtin_rotateleft64:
4800 case Builtin::BI__builtin_stdc_rotate_left:
4801 case Builtin::BIstdc_rotate_left_uc:
4802 case Builtin::BIstdc_rotate_left_us:
4803 case Builtin::BIstdc_rotate_left_ui:
4804 case Builtin::BIstdc_rotate_left_ul:
4805 case Builtin::BIstdc_rotate_left_ull:
4806 case Builtin::BI_rotl8: // Microsoft variants of rotate left
4807 case Builtin::BI_rotl16:
4808 case Builtin::BI_rotl:
4809 case Builtin::BI_lrotl:
4810 case Builtin::BI_rotl64:
4811 case Builtin::BI__builtin_rotateright8:
4812 case Builtin::BI__builtin_rotateright16:
4813 case Builtin::BI__builtin_rotateright32:
4814 case Builtin::BI__builtin_rotateright64:
4815 case Builtin::BI__builtin_stdc_rotate_right:
4816 case Builtin::BIstdc_rotate_right_uc:
4817 case Builtin::BIstdc_rotate_right_us:
4818 case Builtin::BIstdc_rotate_right_ui:
4819 case Builtin::BIstdc_rotate_right_ul:
4820 case Builtin::BIstdc_rotate_right_ull:
4821 case Builtin::BI_rotr8: // Microsoft variants of rotate right
4822 case Builtin::BI_rotr16:
4823 case Builtin::BI_rotr:
4824 case Builtin::BI_lrotr:
4825 case Builtin::BI_rotr64: {
4826 // Determine if this is a rotate right operation
4827 bool IsRotateRight;
4828 switch (BuiltinID) {
4829 case Builtin::BI__builtin_rotateright8:
4830 case Builtin::BI__builtin_rotateright16:
4831 case Builtin::BI__builtin_rotateright32:
4832 case Builtin::BI__builtin_rotateright64:
4833 case Builtin::BI__builtin_stdc_rotate_right:
4834 case Builtin::BIstdc_rotate_right_uc:
4835 case Builtin::BIstdc_rotate_right_us:
4836 case Builtin::BIstdc_rotate_right_ui:
4837 case Builtin::BIstdc_rotate_right_ul:
4838 case Builtin::BIstdc_rotate_right_ull:
4839 case Builtin::BI_rotr8:
4840 case Builtin::BI_rotr16:
4841 case Builtin::BI_rotr:
4842 case Builtin::BI_lrotr:
4843 case Builtin::BI_rotr64:
4844 IsRotateRight = true;
4845 break;
4846 default:
4847 IsRotateRight = false;
4848 break;
4849 }
4850
4851 return interp__builtin_elementwise_int_binop(
4852 S, OpPC, Call, Fn: [IsRotateRight](const APSInt &Value, APSInt Amount) {
4853 Amount = NormalizeRotateAmount(Value, Amount);
4854 return IsRotateRight ? Value.rotr(rotateAmt: Amount.getZExtValue())
4855 : Value.rotl(rotateAmt: Amount.getZExtValue());
4856 });
4857 }
4858
4859 case Builtin::BIstdc_leading_zeros_uc:
4860 case Builtin::BIstdc_leading_zeros_us:
4861 case Builtin::BIstdc_leading_zeros_ui:
4862 case Builtin::BIstdc_leading_zeros_ul:
4863 case Builtin::BIstdc_leading_zeros_ull:
4864 case Builtin::BI__builtin_stdc_leading_zeros: {
4865 unsigned ResWidth = S.getASTContext().getIntWidth(T: Call->getType());
4866 return interp__builtin_elementwise_int_unaryop(
4867 S, OpPC, Call, Fn: [ResWidth](const APSInt &Val) {
4868 return APInt(ResWidth, Val.countl_zero());
4869 });
4870 }
4871
4872 case Builtin::BIstdc_leading_ones_uc:
4873 case Builtin::BIstdc_leading_ones_us:
4874 case Builtin::BIstdc_leading_ones_ui:
4875 case Builtin::BIstdc_leading_ones_ul:
4876 case Builtin::BIstdc_leading_ones_ull:
4877 case Builtin::BI__builtin_stdc_leading_ones: {
4878 unsigned ResWidth = S.getASTContext().getIntWidth(T: Call->getType());
4879 return interp__builtin_elementwise_int_unaryop(
4880 S, OpPC, Call, Fn: [ResWidth](const APSInt &Val) {
4881 return APInt(ResWidth, Val.countl_one());
4882 });
4883 }
4884
4885 case Builtin::BIstdc_trailing_zeros_uc:
4886 case Builtin::BIstdc_trailing_zeros_us:
4887 case Builtin::BIstdc_trailing_zeros_ui:
4888 case Builtin::BIstdc_trailing_zeros_ul:
4889 case Builtin::BIstdc_trailing_zeros_ull:
4890 case Builtin::BI__builtin_stdc_trailing_zeros: {
4891 unsigned ResWidth = S.getASTContext().getIntWidth(T: Call->getType());
4892 return interp__builtin_elementwise_int_unaryop(
4893 S, OpPC, Call, Fn: [ResWidth](const APSInt &Val) {
4894 return APInt(ResWidth, Val.countr_zero());
4895 });
4896 }
4897
4898 case Builtin::BIstdc_trailing_ones_uc:
4899 case Builtin::BIstdc_trailing_ones_us:
4900 case Builtin::BIstdc_trailing_ones_ui:
4901 case Builtin::BIstdc_trailing_ones_ul:
4902 case Builtin::BIstdc_trailing_ones_ull:
4903 case Builtin::BI__builtin_stdc_trailing_ones: {
4904 unsigned ResWidth = S.getASTContext().getIntWidth(T: Call->getType());
4905 return interp__builtin_elementwise_int_unaryop(
4906 S, OpPC, Call, Fn: [ResWidth](const APSInt &Val) {
4907 return APInt(ResWidth, Val.countr_one());
4908 });
4909 }
4910
4911 case Builtin::BIstdc_first_leading_zero_uc:
4912 case Builtin::BIstdc_first_leading_zero_us:
4913 case Builtin::BIstdc_first_leading_zero_ui:
4914 case Builtin::BIstdc_first_leading_zero_ul:
4915 case Builtin::BIstdc_first_leading_zero_ull:
4916 case Builtin::BI__builtin_stdc_first_leading_zero: {
4917 unsigned ResWidth = S.getASTContext().getIntWidth(T: Call->getType());
4918 return interp__builtin_elementwise_int_unaryop(
4919 S, OpPC, Call, Fn: [ResWidth](const APSInt &Val) {
4920 return APInt(ResWidth, Val.isAllOnes() ? 0 : Val.countl_one() + 1);
4921 });
4922 }
4923
4924 case Builtin::BIstdc_first_leading_one_uc:
4925 case Builtin::BIstdc_first_leading_one_us:
4926 case Builtin::BIstdc_first_leading_one_ui:
4927 case Builtin::BIstdc_first_leading_one_ul:
4928 case Builtin::BIstdc_first_leading_one_ull:
4929 case Builtin::BI__builtin_stdc_first_leading_one: {
4930 unsigned ResWidth = S.getASTContext().getIntWidth(T: Call->getType());
4931 return interp__builtin_elementwise_int_unaryop(
4932 S, OpPC, Call, Fn: [ResWidth](const APSInt &Val) {
4933 return APInt(ResWidth, Val.isZero() ? 0 : Val.countl_zero() + 1);
4934 });
4935 }
4936
4937 case Builtin::BIstdc_first_trailing_zero_uc:
4938 case Builtin::BIstdc_first_trailing_zero_us:
4939 case Builtin::BIstdc_first_trailing_zero_ui:
4940 case Builtin::BIstdc_first_trailing_zero_ul:
4941 case Builtin::BIstdc_first_trailing_zero_ull:
4942 case Builtin::BI__builtin_stdc_first_trailing_zero: {
4943 unsigned ResWidth = S.getASTContext().getIntWidth(T: Call->getType());
4944 return interp__builtin_elementwise_int_unaryop(
4945 S, OpPC, Call, Fn: [ResWidth](const APSInt &Val) {
4946 return APInt(ResWidth, Val.isAllOnes() ? 0 : Val.countr_one() + 1);
4947 });
4948 }
4949
4950 case Builtin::BIstdc_first_trailing_one_uc:
4951 case Builtin::BIstdc_first_trailing_one_us:
4952 case Builtin::BIstdc_first_trailing_one_ui:
4953 case Builtin::BIstdc_first_trailing_one_ul:
4954 case Builtin::BIstdc_first_trailing_one_ull:
4955 case Builtin::BI__builtin_stdc_first_trailing_one: {
4956 unsigned ResWidth = S.getASTContext().getIntWidth(T: Call->getType());
4957 return interp__builtin_elementwise_int_unaryop(
4958 S, OpPC, Call, Fn: [ResWidth](const APSInt &Val) {
4959 return APInt(ResWidth, Val.isZero() ? 0 : Val.countr_zero() + 1);
4960 });
4961 }
4962
4963 case Builtin::BIstdc_count_zeros_uc:
4964 case Builtin::BIstdc_count_zeros_us:
4965 case Builtin::BIstdc_count_zeros_ui:
4966 case Builtin::BIstdc_count_zeros_ul:
4967 case Builtin::BIstdc_count_zeros_ull:
4968 case Builtin::BI__builtin_stdc_count_zeros: {
4969 unsigned ResWidth = S.getASTContext().getIntWidth(T: Call->getType());
4970 return interp__builtin_elementwise_int_unaryop(
4971 S, OpPC, Call, Fn: [ResWidth](const APSInt &Val) {
4972 unsigned BitWidth = Val.getBitWidth();
4973 return APInt(ResWidth, BitWidth - Val.popcount());
4974 });
4975 }
4976
4977 case Builtin::BIstdc_count_ones_uc:
4978 case Builtin::BIstdc_count_ones_us:
4979 case Builtin::BIstdc_count_ones_ui:
4980 case Builtin::BIstdc_count_ones_ul:
4981 case Builtin::BIstdc_count_ones_ull:
4982 case Builtin::BI__builtin_stdc_count_ones: {
4983 unsigned ResWidth = S.getASTContext().getIntWidth(T: Call->getType());
4984 return interp__builtin_elementwise_int_unaryop(
4985 S, OpPC, Call, Fn: [ResWidth](const APSInt &Val) {
4986 return APInt(ResWidth, Val.popcount());
4987 });
4988 }
4989
4990 case Builtin::BIstdc_has_single_bit_uc:
4991 case Builtin::BIstdc_has_single_bit_us:
4992 case Builtin::BIstdc_has_single_bit_ui:
4993 case Builtin::BIstdc_has_single_bit_ul:
4994 case Builtin::BIstdc_has_single_bit_ull:
4995 case Builtin::BI__builtin_stdc_has_single_bit: {
4996 unsigned ResWidth = S.getASTContext().getIntWidth(T: Call->getType());
4997 return interp__builtin_elementwise_int_unaryop(
4998 S, OpPC, Call, Fn: [ResWidth](const APSInt &Val) {
4999 return APInt(ResWidth, Val.popcount() == 1 ? 1 : 0);
5000 });
5001 }
5002
5003 case Builtin::BIstdc_bit_width_uc:
5004 case Builtin::BIstdc_bit_width_us:
5005 case Builtin::BIstdc_bit_width_ui:
5006 case Builtin::BIstdc_bit_width_ul:
5007 case Builtin::BIstdc_bit_width_ull:
5008 case Builtin::BI__builtin_stdc_bit_width: {
5009 unsigned ResWidth = S.getASTContext().getIntWidth(T: Call->getType());
5010 return interp__builtin_elementwise_int_unaryop(
5011 S, OpPC, Call, Fn: [ResWidth](const APSInt &Val) {
5012 unsigned BitWidth = Val.getBitWidth();
5013 return APInt(ResWidth, BitWidth - Val.countl_zero());
5014 });
5015 }
5016
5017 case Builtin::BIstdc_bit_floor_uc:
5018 case Builtin::BIstdc_bit_floor_us:
5019 case Builtin::BIstdc_bit_floor_ui:
5020 case Builtin::BIstdc_bit_floor_ul:
5021 case Builtin::BIstdc_bit_floor_ull:
5022 case Builtin::BI__builtin_stdc_bit_floor:
5023 return interp__builtin_elementwise_int_unaryop(
5024 S, OpPC, Call, Fn: [](const APSInt &Val) {
5025 unsigned BitWidth = Val.getBitWidth();
5026 if (Val.isZero())
5027 return APInt::getZero(numBits: BitWidth);
5028 return APInt::getOneBitSet(numBits: BitWidth,
5029 BitNo: BitWidth - Val.countl_zero() - 1);
5030 });
5031
5032 case Builtin::BIstdc_bit_ceil_uc:
5033 case Builtin::BIstdc_bit_ceil_us:
5034 case Builtin::BIstdc_bit_ceil_ui:
5035 case Builtin::BIstdc_bit_ceil_ul:
5036 case Builtin::BIstdc_bit_ceil_ull:
5037 case Builtin::BI__builtin_stdc_bit_ceil:
5038 return interp__builtin_elementwise_int_unaryop(
5039 S, OpPC, Call, Fn: [](const APSInt &Val) {
5040 unsigned BitWidth = Val.getBitWidth();
5041 if (Val.ule(RHS: 1))
5042 return APInt(BitWidth, 1);
5043 APInt V = Val;
5044 APInt ValMinusOne = V - 1;
5045 unsigned LeadingZeros = ValMinusOne.countl_zero();
5046 if (LeadingZeros == 0)
5047 return APInt(BitWidth, 0); // overflows; wrap to 0
5048 return APInt::getOneBitSet(numBits: BitWidth, BitNo: BitWidth - LeadingZeros);
5049 });
5050
5051 case Builtin::BI__builtin_ffs:
5052 case Builtin::BI__builtin_ffsl:
5053 case Builtin::BI__builtin_ffsll:
5054 return interp__builtin_elementwise_int_unaryop(
5055 S, OpPC, Call, Fn: [](const APSInt &Val) {
5056 return APInt(Val.getBitWidth(),
5057 Val.isZero() ? 0u : Val.countTrailingZeros() + 1u);
5058 });
5059
5060 case Builtin::BIaddressof:
5061 case Builtin::BI__addressof:
5062 case Builtin::BI__builtin_addressof:
5063 assert(isNoopBuiltin(BuiltinID));
5064 return interp__builtin_addressof(S, OpPC, Frame, Call);
5065
5066 case Builtin::BIas_const:
5067 case Builtin::BIforward:
5068 case Builtin::BIforward_like:
5069 case Builtin::BImove:
5070 case Builtin::BImove_if_noexcept:
5071 assert(isNoopBuiltin(BuiltinID));
5072 return interp__builtin_move(S, OpPC, Frame, Call);
5073
5074 case Builtin::BI__builtin_eh_return_data_regno:
5075 return interp__builtin_eh_return_data_regno(S, OpPC, Frame, Call);
5076
5077 case Builtin::BI__builtin_launder:
5078 assert(isNoopBuiltin(BuiltinID));
5079 return true;
5080
5081 case Builtin::BI__builtin_add_overflow:
5082 case Builtin::BI__builtin_sub_overflow:
5083 case Builtin::BI__builtin_mul_overflow:
5084 case Builtin::BI__builtin_sadd_overflow:
5085 case Builtin::BI__builtin_uadd_overflow:
5086 case Builtin::BI__builtin_uaddl_overflow:
5087 case Builtin::BI__builtin_uaddll_overflow:
5088 case Builtin::BI__builtin_usub_overflow:
5089 case Builtin::BI__builtin_usubl_overflow:
5090 case Builtin::BI__builtin_usubll_overflow:
5091 case Builtin::BI__builtin_umul_overflow:
5092 case Builtin::BI__builtin_umull_overflow:
5093 case Builtin::BI__builtin_umulll_overflow:
5094 case Builtin::BI__builtin_saddl_overflow:
5095 case Builtin::BI__builtin_saddll_overflow:
5096 case Builtin::BI__builtin_ssub_overflow:
5097 case Builtin::BI__builtin_ssubl_overflow:
5098 case Builtin::BI__builtin_ssubll_overflow:
5099 case Builtin::BI__builtin_smul_overflow:
5100 case Builtin::BI__builtin_smull_overflow:
5101 case Builtin::BI__builtin_smulll_overflow:
5102 return interp__builtin_overflowop(S, OpPC, Call, BuiltinOp: BuiltinID);
5103
5104 case Builtin::BI__builtin_addcb:
5105 case Builtin::BI__builtin_addcs:
5106 case Builtin::BI__builtin_addc:
5107 case Builtin::BI__builtin_addcl:
5108 case Builtin::BI__builtin_addcll:
5109 case Builtin::BI__builtin_subcb:
5110 case Builtin::BI__builtin_subcs:
5111 case Builtin::BI__builtin_subc:
5112 case Builtin::BI__builtin_subcl:
5113 case Builtin::BI__builtin_subcll:
5114 return interp__builtin_carryop(S, OpPC, Frame, Call, BuiltinOp: BuiltinID);
5115
5116 case Builtin::BI__builtin_clz:
5117 case Builtin::BI__builtin_clzl:
5118 case Builtin::BI__builtin_clzll:
5119 case Builtin::BI__builtin_clzs:
5120 case Builtin::BI__builtin_clzg:
5121 case Builtin::BI__lzcnt16: // Microsoft variants of count leading-zeroes
5122 case Builtin::BI__lzcnt:
5123 case Builtin::BI__lzcnt64:
5124 return interp__builtin_clz(S, OpPC, Frame, Call, BuiltinOp: BuiltinID);
5125
5126 case Builtin::BI__builtin_ctz:
5127 case Builtin::BI__builtin_ctzl:
5128 case Builtin::BI__builtin_ctzll:
5129 case Builtin::BI__builtin_ctzs:
5130 case Builtin::BI__builtin_ctzg:
5131 return interp__builtin_ctz(S, OpPC, Frame, Call, BuiltinID);
5132
5133 case Builtin::BI__builtin_elementwise_clzg:
5134 case Builtin::BI__builtin_elementwise_ctzg:
5135 return interp__builtin_elementwise_countzeroes(S, OpPC, Frame, Call,
5136 BuiltinID);
5137 case Builtin::BI__builtin_bswapg:
5138 case Builtin::BI__builtin_bswap16:
5139 case Builtin::BI__builtin_bswap32:
5140 case Builtin::BI__builtin_bswap64:
5141 case Builtin::BIstdc_memreverse8u8:
5142 case Builtin::BIstdc_memreverse8u16:
5143 case Builtin::BIstdc_memreverse8u32:
5144 case Builtin::BIstdc_memreverse8u64:
5145 return interp__builtin_bswap(S, OpPC, Frame, Call);
5146
5147 case Builtin::BI__atomic_always_lock_free:
5148 case Builtin::BI__atomic_is_lock_free:
5149 return interp__builtin_atomic_lock_free(S, OpPC, Frame, Call, BuiltinOp: BuiltinID);
5150
5151 case Builtin::BI__c11_atomic_is_lock_free:
5152 return interp__builtin_c11_atomic_is_lock_free(S, OpPC, Frame, Call);
5153
5154 case Builtin::BI__builtin_complex:
5155 return interp__builtin_complex(S, OpPC, Frame, Call);
5156
5157 case Builtin::BI__builtin_is_aligned:
5158 case Builtin::BI__builtin_align_up:
5159 case Builtin::BI__builtin_align_down:
5160 return interp__builtin_is_aligned_up_down(S, OpPC, Frame, Call, BuiltinOp: BuiltinID);
5161
5162 case Builtin::BI__builtin_assume_aligned:
5163 return interp__builtin_assume_aligned(S, OpPC, Frame, Call);
5164
5165 case clang::X86::BI__builtin_ia32_crc32qi:
5166 return interp__builtin_ia32_crc32(S, OpPC, Frame, Call, DataBytes: 1);
5167 case clang::X86::BI__builtin_ia32_crc32hi:
5168 return interp__builtin_ia32_crc32(S, OpPC, Frame, Call, DataBytes: 2);
5169 case clang::X86::BI__builtin_ia32_crc32si:
5170 return interp__builtin_ia32_crc32(S, OpPC, Frame, Call, DataBytes: 4);
5171 case clang::X86::BI__builtin_ia32_crc32di:
5172 return interp__builtin_ia32_crc32(S, OpPC, Frame, Call, DataBytes: 8);
5173
5174 case clang::X86::BI__builtin_ia32_bextr_u32:
5175 case clang::X86::BI__builtin_ia32_bextr_u64:
5176 case clang::X86::BI__builtin_ia32_bextri_u32:
5177 case clang::X86::BI__builtin_ia32_bextri_u64:
5178 return interp__builtin_elementwise_int_binop(
5179 S, OpPC, Call, Fn: [](const APSInt &Val, const APSInt &Idx) {
5180 unsigned BitWidth = Val.getBitWidth();
5181 uint64_t Shift = Idx.extractBitsAsZExtValue(numBits: 8, bitPosition: 0);
5182 uint64_t Length = Idx.extractBitsAsZExtValue(numBits: 8, bitPosition: 8);
5183 if (Length > BitWidth) {
5184 Length = BitWidth;
5185 }
5186
5187 // Handle out of bounds cases.
5188 if (Length == 0 || Shift >= BitWidth)
5189 return APInt(BitWidth, 0);
5190
5191 uint64_t Result = Val.getZExtValue() >> Shift;
5192 Result &= llvm::maskTrailingOnes<uint64_t>(N: Length);
5193 return APInt(BitWidth, Result);
5194 });
5195
5196 case clang::X86::BI__builtin_ia32_bzhi_si:
5197 case clang::X86::BI__builtin_ia32_bzhi_di:
5198 return interp__builtin_elementwise_int_binop(
5199 S, OpPC, Call, Fn: [](const APSInt &Val, const APSInt &Idx) {
5200 unsigned BitWidth = Val.getBitWidth();
5201 uint64_t Index = Idx.extractBitsAsZExtValue(numBits: 8, bitPosition: 0);
5202 APSInt Result = Val;
5203
5204 if (Index < BitWidth)
5205 Result.clearHighBits(hiBits: BitWidth - Index);
5206
5207 return Result;
5208 });
5209
5210 case clang::X86::BI__builtin_ia32_ktestcqi:
5211 case clang::X86::BI__builtin_ia32_ktestchi:
5212 case clang::X86::BI__builtin_ia32_ktestcsi:
5213 case clang::X86::BI__builtin_ia32_ktestcdi:
5214 return interp__builtin_elementwise_int_binop(
5215 S, OpPC, Call, Fn: [](const APSInt &A, const APSInt &B) {
5216 return APInt(sizeof(unsigned char) * 8, (~A & B) == 0);
5217 });
5218
5219 case clang::X86::BI__builtin_ia32_ktestzqi:
5220 case clang::X86::BI__builtin_ia32_ktestzhi:
5221 case clang::X86::BI__builtin_ia32_ktestzsi:
5222 case clang::X86::BI__builtin_ia32_ktestzdi:
5223 return interp__builtin_elementwise_int_binop(
5224 S, OpPC, Call, Fn: [](const APSInt &A, const APSInt &B) {
5225 return APInt(sizeof(unsigned char) * 8, (A & B) == 0);
5226 });
5227
5228 case clang::X86::BI__builtin_ia32_kortestcqi:
5229 case clang::X86::BI__builtin_ia32_kortestchi:
5230 case clang::X86::BI__builtin_ia32_kortestcsi:
5231 case clang::X86::BI__builtin_ia32_kortestcdi:
5232 return interp__builtin_elementwise_int_binop(
5233 S, OpPC, Call, Fn: [](const APSInt &A, const APSInt &B) {
5234 return APInt(sizeof(unsigned char) * 8, ~(A | B) == 0);
5235 });
5236
5237 case clang::X86::BI__builtin_ia32_kortestzqi:
5238 case clang::X86::BI__builtin_ia32_kortestzhi:
5239 case clang::X86::BI__builtin_ia32_kortestzsi:
5240 case clang::X86::BI__builtin_ia32_kortestzdi:
5241 return interp__builtin_elementwise_int_binop(
5242 S, OpPC, Call, Fn: [](const APSInt &A, const APSInt &B) {
5243 return APInt(sizeof(unsigned char) * 8, (A | B) == 0);
5244 });
5245
5246 case clang::X86::BI__builtin_ia32_kshiftliqi:
5247 case clang::X86::BI__builtin_ia32_kshiftlihi:
5248 case clang::X86::BI__builtin_ia32_kshiftlisi:
5249 case clang::X86::BI__builtin_ia32_kshiftlidi:
5250 return interp__builtin_elementwise_int_binop(
5251 S, OpPC, Call, Fn: [](const APSInt &LHS, const APSInt &RHS) {
5252 unsigned Amt = RHS.getZExtValue() & 0xFF;
5253 if (Amt >= LHS.getBitWidth())
5254 return APInt::getZero(numBits: LHS.getBitWidth());
5255 return LHS.shl(shiftAmt: Amt);
5256 });
5257
5258 case clang::X86::BI__builtin_ia32_kshiftriqi:
5259 case clang::X86::BI__builtin_ia32_kshiftrihi:
5260 case clang::X86::BI__builtin_ia32_kshiftrisi:
5261 case clang::X86::BI__builtin_ia32_kshiftridi:
5262 return interp__builtin_elementwise_int_binop(
5263 S, OpPC, Call, Fn: [](const APSInt &LHS, const APSInt &RHS) {
5264 unsigned Amt = RHS.getZExtValue() & 0xFF;
5265 if (Amt >= LHS.getBitWidth())
5266 return APInt::getZero(numBits: LHS.getBitWidth());
5267 return LHS.lshr(shiftAmt: Amt);
5268 });
5269
5270 case clang::X86::BI__builtin_ia32_lzcnt_u16:
5271 case clang::X86::BI__builtin_ia32_lzcnt_u32:
5272 case clang::X86::BI__builtin_ia32_lzcnt_u64:
5273 return interp__builtin_elementwise_int_unaryop(
5274 S, OpPC, Call, Fn: [](const APSInt &Src) {
5275 return APInt(Src.getBitWidth(), Src.countLeadingZeros());
5276 });
5277
5278 case clang::X86::BI__builtin_ia32_tzcnt_u16:
5279 case clang::X86::BI__builtin_ia32_tzcnt_u32:
5280 case clang::X86::BI__builtin_ia32_tzcnt_u64:
5281 return interp__builtin_elementwise_int_unaryop(
5282 S, OpPC, Call, Fn: [](const APSInt &Src) {
5283 return APInt(Src.getBitWidth(), Src.countTrailingZeros());
5284 });
5285
5286 case clang::X86::BI__builtin_ia32_addcarryx_u32:
5287 case clang::X86::BI__builtin_ia32_addcarryx_u64:
5288 return interp__builtin_ia32_addcarry_subborrow(S, OpPC, Frame, Call,
5289 /*IsAdd=*/true);
5290
5291 case clang::X86::BI__builtin_ia32_subborrow_u32:
5292 case clang::X86::BI__builtin_ia32_subborrow_u64:
5293 return interp__builtin_ia32_addcarry_subborrow(S, OpPC, Frame, Call,
5294 /*IsAdd=*/false);
5295
5296 case Builtin::BI__builtin_os_log_format_buffer_size:
5297 return interp__builtin_os_log_format_buffer_size(S, OpPC, Frame, Call);
5298
5299 case Builtin::BI__builtin_ptrauth_string_discriminator:
5300 return interp__builtin_ptrauth_string_discriminator(S, OpPC, Frame, Call);
5301
5302 case Builtin::BI__builtin_infer_alloc_token:
5303 return interp__builtin_infer_alloc_token(S, OpPC, Frame, Call);
5304
5305 case Builtin::BI__noop:
5306 pushInteger(S, Val: 0, QT: Call->getType());
5307 return true;
5308
5309 case Builtin::BI__builtin_operator_new:
5310 return interp__builtin_operator_new(S, OpPC, Frame, Call);
5311
5312 case Builtin::BI__builtin_operator_delete:
5313 return interp__builtin_operator_delete(S, OpPC, Frame, Call);
5314
5315 case Builtin::BI__arithmetic_fence:
5316 return interp__builtin_arithmetic_fence(S, OpPC, Frame, Call);
5317
5318 case Builtin::BI__builtin_reduce_add:
5319 case Builtin::BI__builtin_reduce_mul:
5320 case Builtin::BI__builtin_reduce_and:
5321 case Builtin::BI__builtin_reduce_or:
5322 case Builtin::BI__builtin_reduce_xor:
5323 case Builtin::BI__builtin_reduce_min:
5324 case Builtin::BI__builtin_reduce_max:
5325 return interp__builtin_vector_reduce(S, OpPC, Call, ID: BuiltinID);
5326
5327 case Builtin::BI__builtin_elementwise_popcount:
5328 return interp__builtin_elementwise_int_unaryop(
5329 S, OpPC, Call, Fn: [](const APSInt &Src) {
5330 return APInt(Src.getBitWidth(), Src.popcount());
5331 });
5332 case Builtin::BI__builtin_elementwise_bitreverse:
5333 return interp__builtin_elementwise_int_unaryop(
5334 S, OpPC, Call, Fn: [](const APSInt &Src) { return Src.reverseBits(); });
5335
5336 case Builtin::BI__builtin_elementwise_abs:
5337 return interp__builtin_elementwise_abs(S, OpPC, Frame, Call, BuiltinID);
5338
5339 case Builtin::BI__builtin_memcpy:
5340 case Builtin::BImemcpy:
5341 case Builtin::BI__builtin_wmemcpy:
5342 case Builtin::BIwmemcpy:
5343 case Builtin::BI__builtin_memmove:
5344 case Builtin::BImemmove:
5345 case Builtin::BI__builtin_wmemmove:
5346 case Builtin::BIwmemmove:
5347 return interp__builtin_memcpy(S, OpPC, Frame, Call, ID: BuiltinID);
5348
5349 case Builtin::BI__builtin_memcmp:
5350 case Builtin::BImemcmp:
5351 case Builtin::BI__builtin_bcmp:
5352 case Builtin::BIbcmp:
5353 case Builtin::BI__builtin_wmemcmp:
5354 case Builtin::BIwmemcmp:
5355 return interp__builtin_memcmp(S, OpPC, Frame, Call, ID: BuiltinID);
5356
5357 case Builtin::BImemchr:
5358 case Builtin::BI__builtin_memchr:
5359 case Builtin::BIstrchr:
5360 case Builtin::BI__builtin_strchr:
5361 case Builtin::BIwmemchr:
5362 case Builtin::BI__builtin_wmemchr:
5363 case Builtin::BIwcschr:
5364 case Builtin::BI__builtin_wcschr:
5365 case Builtin::BI__builtin_char_memchr:
5366 return interp__builtin_memchr(S, OpPC, Call, ID: BuiltinID);
5367
5368 case Builtin::BI__builtin_object_size:
5369 return interp__builtin_object_size(S, OpPC, Frame, Call,
5370 /*IsDynamic=*/false);
5371 case Builtin::BI__builtin_dynamic_object_size:
5372 return interp__builtin_object_size(S, OpPC, Frame, Call,
5373 /*IsDynamic=*/true);
5374
5375 case Builtin::BI__builtin_is_within_lifetime:
5376 return interp__builtin_is_within_lifetime(S, OpPC, Call);
5377
5378 case Builtin::BI__builtin_elementwise_add_sat:
5379 return interp__builtin_elementwise_int_binop(
5380 S, OpPC, Call, Fn: [](const APSInt &LHS, const APSInt &RHS) {
5381 return LHS.isSigned() ? LHS.sadd_sat(RHS) : LHS.uadd_sat(RHS);
5382 });
5383
5384 case Builtin::BI__builtin_elementwise_sub_sat:
5385 return interp__builtin_elementwise_int_binop(
5386 S, OpPC, Call, Fn: [](const APSInt &LHS, const APSInt &RHS) {
5387 return LHS.isSigned() ? LHS.ssub_sat(RHS) : LHS.usub_sat(RHS);
5388 });
5389
5390 case Builtin::BI__builtin_elementwise_pdep:
5391 return interp__builtin_elementwise_int_binop(S, OpPC, Call,
5392 Fn: llvm::APIntOps::pdep);
5393
5394 case Builtin::BI__builtin_elementwise_pext:
5395 return interp__builtin_elementwise_int_binop(S, OpPC, Call,
5396 Fn: llvm::APIntOps::pext);
5397
5398 case X86::BI__builtin_ia32_extract128i256:
5399 case X86::BI__builtin_ia32_vextractf128_pd256:
5400 case X86::BI__builtin_ia32_vextractf128_ps256:
5401 case X86::BI__builtin_ia32_vextractf128_si256:
5402 return interp__builtin_ia32_extract_vector(S, OpPC, Call, ID: BuiltinID);
5403
5404 case X86::BI__builtin_ia32_extractf32x4_256_mask:
5405 case X86::BI__builtin_ia32_extractf32x4_mask:
5406 case X86::BI__builtin_ia32_extractf32x8_mask:
5407 case X86::BI__builtin_ia32_extractf64x2_256_mask:
5408 case X86::BI__builtin_ia32_extractf64x2_512_mask:
5409 case X86::BI__builtin_ia32_extractf64x4_mask:
5410 case X86::BI__builtin_ia32_extracti32x4_256_mask:
5411 case X86::BI__builtin_ia32_extracti32x4_mask:
5412 case X86::BI__builtin_ia32_extracti32x8_mask:
5413 case X86::BI__builtin_ia32_extracti64x2_256_mask:
5414 case X86::BI__builtin_ia32_extracti64x2_512_mask:
5415 case X86::BI__builtin_ia32_extracti64x4_mask:
5416 return interp__builtin_ia32_extract_vector_masked(S, OpPC, Call, ID: BuiltinID);
5417
5418 case clang::X86::BI__builtin_ia32_pmulhrsw128:
5419 case clang::X86::BI__builtin_ia32_pmulhrsw256:
5420 case clang::X86::BI__builtin_ia32_pmulhrsw512:
5421 return interp__builtin_elementwise_int_binop(
5422 S, OpPC, Call, Fn: [](const APSInt &LHS, const APSInt &RHS) {
5423 return (llvm::APIntOps::mulsExtended(C1: LHS, C2: RHS).ashr(ShiftAmt: 14) + 1)
5424 .extractBits(numBits: 16, bitPosition: 1);
5425 });
5426
5427 case clang::X86::BI__builtin_ia32_movmskps:
5428 case clang::X86::BI__builtin_ia32_movmskpd:
5429 case clang::X86::BI__builtin_ia32_pmovmskb128:
5430 case clang::X86::BI__builtin_ia32_pmovmskb256:
5431 case clang::X86::BI__builtin_ia32_movmskps256:
5432 case clang::X86::BI__builtin_ia32_movmskpd256: {
5433 return interp__builtin_ia32_movmsk_op(S, OpPC, Call);
5434 }
5435
5436 case X86::BI__builtin_ia32_psignb128:
5437 case X86::BI__builtin_ia32_psignb256:
5438 case X86::BI__builtin_ia32_psignw128:
5439 case X86::BI__builtin_ia32_psignw256:
5440 case X86::BI__builtin_ia32_psignd128:
5441 case X86::BI__builtin_ia32_psignd256:
5442 return interp__builtin_elementwise_int_binop(
5443 S, OpPC, Call, Fn: [](const APInt &AElem, const APInt &BElem) {
5444 if (BElem.isZero())
5445 return APInt::getZero(numBits: AElem.getBitWidth());
5446 if (BElem.isNegative())
5447 return -AElem;
5448 return AElem;
5449 });
5450
5451 case clang::X86::BI__builtin_ia32_pavgb128:
5452 case clang::X86::BI__builtin_ia32_pavgw128:
5453 case clang::X86::BI__builtin_ia32_pavgb256:
5454 case clang::X86::BI__builtin_ia32_pavgw256:
5455 case clang::X86::BI__builtin_ia32_pavgb512:
5456 case clang::X86::BI__builtin_ia32_pavgw512:
5457 return interp__builtin_elementwise_int_binop(S, OpPC, Call,
5458 Fn: llvm::APIntOps::avgCeilU);
5459
5460 case clang::X86::BI__builtin_ia32_pmaddubsw128:
5461 case clang::X86::BI__builtin_ia32_pmaddubsw256:
5462 case clang::X86::BI__builtin_ia32_pmaddubsw512:
5463 return interp__builtin_ia32_pmul(
5464 S, OpPC, Call,
5465 Fn: [](const APSInt &LoLHS, const APSInt &HiLHS, const APSInt &LoRHS,
5466 const APSInt &HiRHS) {
5467 unsigned BitWidth = 2 * LoLHS.getBitWidth();
5468 return (LoLHS.zext(width: BitWidth) * LoRHS.sext(width: BitWidth))
5469 .sadd_sat(RHS: (HiLHS.zext(width: BitWidth) * HiRHS.sext(width: BitWidth)));
5470 });
5471
5472 case clang::X86::BI__builtin_ia32_pmaddwd128:
5473 case clang::X86::BI__builtin_ia32_pmaddwd256:
5474 case clang::X86::BI__builtin_ia32_pmaddwd512:
5475 return interp__builtin_ia32_pmul(
5476 S, OpPC, Call,
5477 Fn: [](const APSInt &LoLHS, const APSInt &HiLHS, const APSInt &LoRHS,
5478 const APSInt &HiRHS) {
5479 unsigned BitWidth = 2 * LoLHS.getBitWidth();
5480 return (LoLHS.sext(width: BitWidth) * LoRHS.sext(width: BitWidth)) +
5481 (HiLHS.sext(width: BitWidth) * HiRHS.sext(width: BitWidth));
5482 });
5483
5484 case clang::X86::BI__builtin_ia32_psadbw128:
5485 case clang::X86::BI__builtin_ia32_psadbw256:
5486 case clang::X86::BI__builtin_ia32_psadbw512:
5487 return interp__builtin_ia32_psadbw(S, OpPC, Call);
5488
5489 case clang::X86::BI__builtin_ia32_dbpsadbw128:
5490 case clang::X86::BI__builtin_ia32_dbpsadbw256:
5491 case clang::X86::BI__builtin_ia32_dbpsadbw512:
5492 return interp__builtin_ia32_dbpsadbw(S, OpPC, Call);
5493
5494 case clang::X86::BI__builtin_ia32_mpsadbw128:
5495 case clang::X86::BI__builtin_ia32_mpsadbw256:
5496 return interp__builtin_ia32_mpsadbw(S, OpPC, Call);
5497
5498 case clang::X86::BI__builtin_ia32_pmulhuw128:
5499 case clang::X86::BI__builtin_ia32_pmulhuw256:
5500 case clang::X86::BI__builtin_ia32_pmulhuw512:
5501 return interp__builtin_elementwise_int_binop(S, OpPC, Call,
5502 Fn: llvm::APIntOps::mulhu);
5503
5504 case clang::X86::BI__builtin_ia32_pmulhw128:
5505 case clang::X86::BI__builtin_ia32_pmulhw256:
5506 case clang::X86::BI__builtin_ia32_pmulhw512:
5507 return interp__builtin_elementwise_int_binop(S, OpPC, Call,
5508 Fn: llvm::APIntOps::mulhs);
5509
5510 case clang::X86::BI__builtin_ia32_psllv2di:
5511 case clang::X86::BI__builtin_ia32_psllv4di:
5512 case clang::X86::BI__builtin_ia32_psllv4si:
5513 case clang::X86::BI__builtin_ia32_psllv8di:
5514 case clang::X86::BI__builtin_ia32_psllv8hi:
5515 case clang::X86::BI__builtin_ia32_psllv8si:
5516 case clang::X86::BI__builtin_ia32_psllv16hi:
5517 case clang::X86::BI__builtin_ia32_psllv16si:
5518 case clang::X86::BI__builtin_ia32_psllv32hi:
5519 case clang::X86::BI__builtin_ia32_psllwi128:
5520 case clang::X86::BI__builtin_ia32_psllwi256:
5521 case clang::X86::BI__builtin_ia32_psllwi512:
5522 case clang::X86::BI__builtin_ia32_pslldi128:
5523 case clang::X86::BI__builtin_ia32_pslldi256:
5524 case clang::X86::BI__builtin_ia32_pslldi512:
5525 case clang::X86::BI__builtin_ia32_psllqi128:
5526 case clang::X86::BI__builtin_ia32_psllqi256:
5527 case clang::X86::BI__builtin_ia32_psllqi512:
5528 return interp__builtin_elementwise_int_binop(
5529 S, OpPC, Call, Fn: [](const APSInt &LHS, const APSInt &RHS) {
5530 if (RHS.uge(RHS: LHS.getBitWidth())) {
5531 return APInt::getZero(numBits: LHS.getBitWidth());
5532 }
5533 return LHS.shl(shiftAmt: RHS.getZExtValue());
5534 });
5535
5536 case clang::X86::BI__builtin_ia32_psrav4si:
5537 case clang::X86::BI__builtin_ia32_psrav8di:
5538 case clang::X86::BI__builtin_ia32_psrav8hi:
5539 case clang::X86::BI__builtin_ia32_psrav8si:
5540 case clang::X86::BI__builtin_ia32_psrav16hi:
5541 case clang::X86::BI__builtin_ia32_psrav16si:
5542 case clang::X86::BI__builtin_ia32_psrav32hi:
5543 case clang::X86::BI__builtin_ia32_psravq128:
5544 case clang::X86::BI__builtin_ia32_psravq256:
5545 case clang::X86::BI__builtin_ia32_psrawi128:
5546 case clang::X86::BI__builtin_ia32_psrawi256:
5547 case clang::X86::BI__builtin_ia32_psrawi512:
5548 case clang::X86::BI__builtin_ia32_psradi128:
5549 case clang::X86::BI__builtin_ia32_psradi256:
5550 case clang::X86::BI__builtin_ia32_psradi512:
5551 case clang::X86::BI__builtin_ia32_psraqi128:
5552 case clang::X86::BI__builtin_ia32_psraqi256:
5553 case clang::X86::BI__builtin_ia32_psraqi512:
5554 return interp__builtin_elementwise_int_binop(
5555 S, OpPC, Call, Fn: [](const APSInt &LHS, const APSInt &RHS) {
5556 if (RHS.uge(RHS: LHS.getBitWidth())) {
5557 return LHS.ashr(ShiftAmt: LHS.getBitWidth() - 1);
5558 }
5559 return LHS.ashr(ShiftAmt: RHS.getZExtValue());
5560 });
5561
5562 case clang::X86::BI__builtin_ia32_psrlv2di:
5563 case clang::X86::BI__builtin_ia32_psrlv4di:
5564 case clang::X86::BI__builtin_ia32_psrlv4si:
5565 case clang::X86::BI__builtin_ia32_psrlv8di:
5566 case clang::X86::BI__builtin_ia32_psrlv8hi:
5567 case clang::X86::BI__builtin_ia32_psrlv8si:
5568 case clang::X86::BI__builtin_ia32_psrlv16hi:
5569 case clang::X86::BI__builtin_ia32_psrlv16si:
5570 case clang::X86::BI__builtin_ia32_psrlv32hi:
5571 case clang::X86::BI__builtin_ia32_psrlwi128:
5572 case clang::X86::BI__builtin_ia32_psrlwi256:
5573 case clang::X86::BI__builtin_ia32_psrlwi512:
5574 case clang::X86::BI__builtin_ia32_psrldi128:
5575 case clang::X86::BI__builtin_ia32_psrldi256:
5576 case clang::X86::BI__builtin_ia32_psrldi512:
5577 case clang::X86::BI__builtin_ia32_psrlqi128:
5578 case clang::X86::BI__builtin_ia32_psrlqi256:
5579 case clang::X86::BI__builtin_ia32_psrlqi512:
5580 return interp__builtin_elementwise_int_binop(
5581 S, OpPC, Call, Fn: [](const APSInt &LHS, const APSInt &RHS) {
5582 if (RHS.uge(RHS: LHS.getBitWidth())) {
5583 return APInt::getZero(numBits: LHS.getBitWidth());
5584 }
5585 return LHS.lshr(shiftAmt: RHS.getZExtValue());
5586 });
5587 case clang::X86::BI__builtin_ia32_packsswb128:
5588 case clang::X86::BI__builtin_ia32_packsswb256:
5589 case clang::X86::BI__builtin_ia32_packsswb512:
5590 case clang::X86::BI__builtin_ia32_packssdw128:
5591 case clang::X86::BI__builtin_ia32_packssdw256:
5592 case clang::X86::BI__builtin_ia32_packssdw512:
5593 return interp__builtin_ia32_pack(S, OpPC, E: Call, PackFn: [](const APSInt &Src) {
5594 return APInt(Src).truncSSat(width: Src.getBitWidth() / 2);
5595 });
5596 case clang::X86::BI__builtin_ia32_packusdw128:
5597 case clang::X86::BI__builtin_ia32_packusdw256:
5598 case clang::X86::BI__builtin_ia32_packusdw512:
5599 case clang::X86::BI__builtin_ia32_packuswb128:
5600 case clang::X86::BI__builtin_ia32_packuswb256:
5601 case clang::X86::BI__builtin_ia32_packuswb512:
5602 return interp__builtin_ia32_pack(S, OpPC, E: Call, PackFn: [](const APSInt &Src) {
5603 return APInt(Src).truncSSatU(width: Src.getBitWidth() / 2);
5604 });
5605
5606 case clang::X86::BI__builtin_ia32_selectss_128:
5607 case clang::X86::BI__builtin_ia32_selectsd_128:
5608 case clang::X86::BI__builtin_ia32_selectsh_128:
5609 case clang::X86::BI__builtin_ia32_selectsbf_128:
5610 return interp__builtin_ia32_select_scalar(S, Call);
5611 case clang::X86::BI__builtin_ia32_vprotbi:
5612 case clang::X86::BI__builtin_ia32_vprotdi:
5613 case clang::X86::BI__builtin_ia32_vprotqi:
5614 case clang::X86::BI__builtin_ia32_vprotwi:
5615 case clang::X86::BI__builtin_ia32_prold128:
5616 case clang::X86::BI__builtin_ia32_prold256:
5617 case clang::X86::BI__builtin_ia32_prold512:
5618 case clang::X86::BI__builtin_ia32_prolq128:
5619 case clang::X86::BI__builtin_ia32_prolq256:
5620 case clang::X86::BI__builtin_ia32_prolq512:
5621 return interp__builtin_elementwise_int_binop(
5622 S, OpPC, Call,
5623 Fn: [](const APSInt &LHS, const APSInt &RHS) { return LHS.rotl(rotateAmt: RHS); });
5624
5625 case clang::X86::BI__builtin_ia32_prord128:
5626 case clang::X86::BI__builtin_ia32_prord256:
5627 case clang::X86::BI__builtin_ia32_prord512:
5628 case clang::X86::BI__builtin_ia32_prorq128:
5629 case clang::X86::BI__builtin_ia32_prorq256:
5630 case clang::X86::BI__builtin_ia32_prorq512:
5631 return interp__builtin_elementwise_int_binop(
5632 S, OpPC, Call,
5633 Fn: [](const APSInt &LHS, const APSInt &RHS) { return LHS.rotr(rotateAmt: RHS); });
5634
5635 case Builtin::BI__builtin_elementwise_max:
5636 case Builtin::BI__builtin_elementwise_min:
5637 return interp__builtin_elementwise_maxmin(S, OpPC, Call, BuiltinID);
5638
5639 case clang::X86::BI__builtin_ia32_phaddw128:
5640 case clang::X86::BI__builtin_ia32_phaddw256:
5641 case clang::X86::BI__builtin_ia32_phaddd128:
5642 case clang::X86::BI__builtin_ia32_phaddd256:
5643 return interp_builtin_horizontal_int_binop(
5644 S, OpPC, Call,
5645 Fn: [](const APSInt &LHS, const APSInt &RHS) { return LHS + RHS; });
5646 case clang::X86::BI__builtin_ia32_phaddsw128:
5647 case clang::X86::BI__builtin_ia32_phaddsw256:
5648 return interp_builtin_horizontal_int_binop(
5649 S, OpPC, Call,
5650 Fn: [](const APSInt &LHS, const APSInt &RHS) { return LHS.sadd_sat(RHS); });
5651 case clang::X86::BI__builtin_ia32_phsubw128:
5652 case clang::X86::BI__builtin_ia32_phsubw256:
5653 case clang::X86::BI__builtin_ia32_phsubd128:
5654 case clang::X86::BI__builtin_ia32_phsubd256:
5655 return interp_builtin_horizontal_int_binop(
5656 S, OpPC, Call,
5657 Fn: [](const APSInt &LHS, const APSInt &RHS) { return LHS - RHS; });
5658 case clang::X86::BI__builtin_ia32_phsubsw128:
5659 case clang::X86::BI__builtin_ia32_phsubsw256:
5660 return interp_builtin_horizontal_int_binop(
5661 S, OpPC, Call,
5662 Fn: [](const APSInt &LHS, const APSInt &RHS) { return LHS.ssub_sat(RHS); });
5663 case clang::X86::BI__builtin_ia32_haddpd:
5664 case clang::X86::BI__builtin_ia32_haddps:
5665 case clang::X86::BI__builtin_ia32_haddpd256:
5666 case clang::X86::BI__builtin_ia32_haddps256:
5667 return interp_builtin_horizontal_fp_binop(
5668 S, OpPC, Call,
5669 Fn: [](const APFloat &LHS, const APFloat &RHS, llvm::RoundingMode RM) {
5670 APFloat F = LHS;
5671 F.add(RHS, RM);
5672 return F;
5673 });
5674 case clang::X86::BI__builtin_ia32_hsubpd:
5675 case clang::X86::BI__builtin_ia32_hsubps:
5676 case clang::X86::BI__builtin_ia32_hsubpd256:
5677 case clang::X86::BI__builtin_ia32_hsubps256:
5678 return interp_builtin_horizontal_fp_binop(
5679 S, OpPC, Call,
5680 Fn: [](const APFloat &LHS, const APFloat &RHS, llvm::RoundingMode RM) {
5681 APFloat F = LHS;
5682 F.subtract(RHS, RM);
5683 return F;
5684 });
5685 case clang::X86::BI__builtin_ia32_addsubpd:
5686 case clang::X86::BI__builtin_ia32_addsubps:
5687 case clang::X86::BI__builtin_ia32_addsubpd256:
5688 case clang::X86::BI__builtin_ia32_addsubps256:
5689 return interp__builtin_ia32_addsub(S, OpPC, Call);
5690
5691 case clang::X86::BI__builtin_ia32_pmuldq128:
5692 case clang::X86::BI__builtin_ia32_pmuldq256:
5693 case clang::X86::BI__builtin_ia32_pmuldq512:
5694 return interp__builtin_ia32_pmul(
5695 S, OpPC, Call,
5696 Fn: [](const APSInt &LoLHS, const APSInt &HiLHS, const APSInt &LoRHS,
5697 const APSInt &HiRHS) {
5698 return llvm::APIntOps::mulsExtended(C1: LoLHS, C2: LoRHS);
5699 });
5700
5701 case clang::X86::BI__builtin_ia32_pmuludq128:
5702 case clang::X86::BI__builtin_ia32_pmuludq256:
5703 case clang::X86::BI__builtin_ia32_pmuludq512:
5704 return interp__builtin_ia32_pmul(
5705 S, OpPC, Call,
5706 Fn: [](const APSInt &LoLHS, const APSInt &HiLHS, const APSInt &LoRHS,
5707 const APSInt &HiRHS) {
5708 return llvm::APIntOps::muluExtended(C1: LoLHS, C2: LoRHS);
5709 });
5710
5711 case clang::X86::BI__builtin_ia32_pclmulqdq128:
5712 case clang::X86::BI__builtin_ia32_pclmulqdq256:
5713 case clang::X86::BI__builtin_ia32_pclmulqdq512:
5714 return interp__builtin_ia32_pclmulqdq(S, OpPC, Call);
5715 case Builtin::BI__builtin_elementwise_clmul:
5716 return interp__builtin_elementwise_int_binop(S, OpPC, Call,
5717 Fn: llvm::APIntOps::clmul);
5718
5719 case Builtin::BI__builtin_elementwise_fma:
5720 return interp__builtin_elementwise_triop_fp(
5721 S, OpPC, Call,
5722 Fn: [](const APFloat &X, const APFloat &Y, const APFloat &Z,
5723 llvm::RoundingMode RM) {
5724 APFloat F = X;
5725 F.fusedMultiplyAdd(Multiplicand: Y, Addend: Z, RM);
5726 return F;
5727 });
5728
5729 case X86::BI__builtin_ia32_vpmadd52luq128:
5730 case X86::BI__builtin_ia32_vpmadd52luq256:
5731 case X86::BI__builtin_ia32_vpmadd52luq512:
5732 return interp__builtin_elementwise_triop(
5733 S, OpPC, Call, Fn: [](const APSInt &A, const APSInt &B, const APSInt &C) {
5734 return A + (B.trunc(width: 52) * C.trunc(width: 52)).zext(width: 64);
5735 });
5736 case X86::BI__builtin_ia32_vpmadd52huq128:
5737 case X86::BI__builtin_ia32_vpmadd52huq256:
5738 case X86::BI__builtin_ia32_vpmadd52huq512:
5739 return interp__builtin_elementwise_triop(
5740 S, OpPC, Call, Fn: [](const APSInt &A, const APSInt &B, const APSInt &C) {
5741 return A + llvm::APIntOps::mulhu(C1: B.trunc(width: 52), C2: C.trunc(width: 52)).zext(width: 64);
5742 });
5743
5744 case X86::BI__builtin_ia32_vpshldd128:
5745 case X86::BI__builtin_ia32_vpshldd256:
5746 case X86::BI__builtin_ia32_vpshldd512:
5747 case X86::BI__builtin_ia32_vpshldq128:
5748 case X86::BI__builtin_ia32_vpshldq256:
5749 case X86::BI__builtin_ia32_vpshldq512:
5750 case X86::BI__builtin_ia32_vpshldw128:
5751 case X86::BI__builtin_ia32_vpshldw256:
5752 case X86::BI__builtin_ia32_vpshldw512:
5753 return interp__builtin_elementwise_triop(
5754 S, OpPC, Call,
5755 Fn: [](const APSInt &Hi, const APSInt &Lo, const APSInt &Amt) {
5756 return llvm::APIntOps::fshl(Hi, Lo, Shift: Amt);
5757 });
5758
5759 case X86::BI__builtin_ia32_vpshrdd128:
5760 case X86::BI__builtin_ia32_vpshrdd256:
5761 case X86::BI__builtin_ia32_vpshrdd512:
5762 case X86::BI__builtin_ia32_vpshrdq128:
5763 case X86::BI__builtin_ia32_vpshrdq256:
5764 case X86::BI__builtin_ia32_vpshrdq512:
5765 case X86::BI__builtin_ia32_vpshrdw128:
5766 case X86::BI__builtin_ia32_vpshrdw256:
5767 case X86::BI__builtin_ia32_vpshrdw512:
5768 // NOTE: Reversed Hi/Lo operands.
5769 return interp__builtin_elementwise_triop(
5770 S, OpPC, Call,
5771 Fn: [](const APSInt &Lo, const APSInt &Hi, const APSInt &Amt) {
5772 return llvm::APIntOps::fshr(Hi, Lo, Shift: Amt);
5773 });
5774 case X86::BI__builtin_ia32_vpconflictsi_128:
5775 case X86::BI__builtin_ia32_vpconflictsi_256:
5776 case X86::BI__builtin_ia32_vpconflictsi_512:
5777 case X86::BI__builtin_ia32_vpconflictdi_128:
5778 case X86::BI__builtin_ia32_vpconflictdi_256:
5779 case X86::BI__builtin_ia32_vpconflictdi_512:
5780 return interp__builtin_ia32_vpconflict(S, OpPC, Call);
5781 case X86::BI__builtin_ia32_compressdf128_mask:
5782 case X86::BI__builtin_ia32_compressdf256_mask:
5783 case X86::BI__builtin_ia32_compressdf512_mask:
5784 case X86::BI__builtin_ia32_compressdi128_mask:
5785 case X86::BI__builtin_ia32_compressdi256_mask:
5786 case X86::BI__builtin_ia32_compressdi512_mask:
5787 case X86::BI__builtin_ia32_compresshi128_mask:
5788 case X86::BI__builtin_ia32_compresshi256_mask:
5789 case X86::BI__builtin_ia32_compresshi512_mask:
5790 case X86::BI__builtin_ia32_compressqi128_mask:
5791 case X86::BI__builtin_ia32_compressqi256_mask:
5792 case X86::BI__builtin_ia32_compressqi512_mask:
5793 case X86::BI__builtin_ia32_compresssf128_mask:
5794 case X86::BI__builtin_ia32_compresssf256_mask:
5795 case X86::BI__builtin_ia32_compresssf512_mask:
5796 case X86::BI__builtin_ia32_compresssi128_mask:
5797 case X86::BI__builtin_ia32_compresssi256_mask:
5798 case X86::BI__builtin_ia32_compresssi512_mask: {
5799 unsigned NumElems =
5800 Call->getArg(Arg: 0)->getType()->castAs<VectorType>()->getNumElements();
5801 return interp__builtin_ia32_shuffle_generic(
5802 S, OpPC, Call, GetSourceIndex: [NumElems](unsigned DstIdx, const APInt &ShuffleMask) {
5803 APInt CompressMask = ShuffleMask.trunc(width: NumElems);
5804 if (DstIdx < CompressMask.popcount()) {
5805 while (DstIdx != 0) {
5806 CompressMask = CompressMask & (CompressMask - 1);
5807 DstIdx--;
5808 }
5809 return std::pair<unsigned, int>{
5810 0, static_cast<int>(CompressMask.countr_zero())};
5811 }
5812 return std::pair<unsigned, int>{1, static_cast<int>(DstIdx)};
5813 });
5814 }
5815 case X86::BI__builtin_ia32_expanddf128_mask:
5816 case X86::BI__builtin_ia32_expanddf256_mask:
5817 case X86::BI__builtin_ia32_expanddf512_mask:
5818 case X86::BI__builtin_ia32_expanddi128_mask:
5819 case X86::BI__builtin_ia32_expanddi256_mask:
5820 case X86::BI__builtin_ia32_expanddi512_mask:
5821 case X86::BI__builtin_ia32_expandhi128_mask:
5822 case X86::BI__builtin_ia32_expandhi256_mask:
5823 case X86::BI__builtin_ia32_expandhi512_mask:
5824 case X86::BI__builtin_ia32_expandqi128_mask:
5825 case X86::BI__builtin_ia32_expandqi256_mask:
5826 case X86::BI__builtin_ia32_expandqi512_mask:
5827 case X86::BI__builtin_ia32_expandsf128_mask:
5828 case X86::BI__builtin_ia32_expandsf256_mask:
5829 case X86::BI__builtin_ia32_expandsf512_mask:
5830 case X86::BI__builtin_ia32_expandsi128_mask:
5831 case X86::BI__builtin_ia32_expandsi256_mask:
5832 case X86::BI__builtin_ia32_expandsi512_mask: {
5833 return interp__builtin_ia32_shuffle_generic(
5834 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, const APInt &ShuffleMask) {
5835 // Trunc to the sub-mask for the dst index and count the number of
5836 // src elements used prior to that.
5837 APInt ExpandMask = ShuffleMask.trunc(width: DstIdx + 1);
5838 if (ExpandMask[DstIdx]) {
5839 int SrcIdx = ExpandMask.popcount() - 1;
5840 return std::pair<unsigned, int>{0, SrcIdx};
5841 }
5842 return std::pair<unsigned, int>{1, static_cast<int>(DstIdx)};
5843 });
5844 }
5845 case clang::X86::BI__builtin_ia32_blendpd:
5846 case clang::X86::BI__builtin_ia32_blendpd256:
5847 case clang::X86::BI__builtin_ia32_blendps:
5848 case clang::X86::BI__builtin_ia32_blendps256:
5849 case clang::X86::BI__builtin_ia32_pblendw128:
5850 case clang::X86::BI__builtin_ia32_pblendw256:
5851 case clang::X86::BI__builtin_ia32_pblendd128:
5852 case clang::X86::BI__builtin_ia32_pblendd256:
5853 return interp__builtin_ia32_shuffle_generic(
5854 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
5855 // Bit index for mask.
5856 unsigned MaskBit = (ShuffleMask >> (DstIdx % 8)) & 0x1;
5857 unsigned SrcVecIdx = MaskBit ? 1 : 0; // 1 = TrueVec, 0 = FalseVec
5858 return std::pair<unsigned, int>{SrcVecIdx, static_cast<int>(DstIdx)};
5859 });
5860
5861
5862
5863 case clang::X86::BI__builtin_ia32_blendvpd:
5864 case clang::X86::BI__builtin_ia32_blendvpd256:
5865 case clang::X86::BI__builtin_ia32_blendvps:
5866 case clang::X86::BI__builtin_ia32_blendvps256:
5867 return interp__builtin_elementwise_triop_fp(
5868 S, OpPC, Call,
5869 Fn: [](const APFloat &F, const APFloat &T, const APFloat &C,
5870 llvm::RoundingMode) { return C.isNegative() ? T : F; });
5871
5872 case clang::X86::BI__builtin_ia32_pblendvb128:
5873 case clang::X86::BI__builtin_ia32_pblendvb256:
5874 return interp__builtin_elementwise_triop(
5875 S, OpPC, Call, Fn: [](const APSInt &F, const APSInt &T, const APSInt &C) {
5876 return ((APInt)C).isNegative() ? T : F;
5877 });
5878 case X86::BI__builtin_ia32_ptestz128:
5879 case X86::BI__builtin_ia32_ptestz256:
5880 case X86::BI__builtin_ia32_vtestzps:
5881 case X86::BI__builtin_ia32_vtestzps256:
5882 case X86::BI__builtin_ia32_vtestzpd:
5883 case X86::BI__builtin_ia32_vtestzpd256:
5884 return interp__builtin_ia32_test_op(
5885 S, OpPC, Call,
5886 Fn: [](const APInt &A, const APInt &B) { return (A & B) == 0; });
5887 case X86::BI__builtin_ia32_ptestc128:
5888 case X86::BI__builtin_ia32_ptestc256:
5889 case X86::BI__builtin_ia32_vtestcps:
5890 case X86::BI__builtin_ia32_vtestcps256:
5891 case X86::BI__builtin_ia32_vtestcpd:
5892 case X86::BI__builtin_ia32_vtestcpd256:
5893 return interp__builtin_ia32_test_op(
5894 S, OpPC, Call,
5895 Fn: [](const APInt &A, const APInt &B) { return (~A & B) == 0; });
5896 case X86::BI__builtin_ia32_ptestnzc128:
5897 case X86::BI__builtin_ia32_ptestnzc256:
5898 case X86::BI__builtin_ia32_vtestnzcps:
5899 case X86::BI__builtin_ia32_vtestnzcps256:
5900 case X86::BI__builtin_ia32_vtestnzcpd:
5901 case X86::BI__builtin_ia32_vtestnzcpd256:
5902 return interp__builtin_ia32_test_op(
5903 S, OpPC, Call, Fn: [](const APInt &A, const APInt &B) {
5904 return ((A & B) != 0) && ((~A & B) != 0);
5905 });
5906 case X86::BI__builtin_ia32_selectb_128:
5907 case X86::BI__builtin_ia32_selectb_256:
5908 case X86::BI__builtin_ia32_selectb_512:
5909 case X86::BI__builtin_ia32_selectw_128:
5910 case X86::BI__builtin_ia32_selectw_256:
5911 case X86::BI__builtin_ia32_selectw_512:
5912 case X86::BI__builtin_ia32_selectd_128:
5913 case X86::BI__builtin_ia32_selectd_256:
5914 case X86::BI__builtin_ia32_selectd_512:
5915 case X86::BI__builtin_ia32_selectq_128:
5916 case X86::BI__builtin_ia32_selectq_256:
5917 case X86::BI__builtin_ia32_selectq_512:
5918 case X86::BI__builtin_ia32_selectph_128:
5919 case X86::BI__builtin_ia32_selectph_256:
5920 case X86::BI__builtin_ia32_selectph_512:
5921 case X86::BI__builtin_ia32_selectpbf_128:
5922 case X86::BI__builtin_ia32_selectpbf_256:
5923 case X86::BI__builtin_ia32_selectpbf_512:
5924 case X86::BI__builtin_ia32_selectps_128:
5925 case X86::BI__builtin_ia32_selectps_256:
5926 case X86::BI__builtin_ia32_selectps_512:
5927 case X86::BI__builtin_ia32_selectpd_128:
5928 case X86::BI__builtin_ia32_selectpd_256:
5929 case X86::BI__builtin_ia32_selectpd_512:
5930 return interp__builtin_ia32_select(S, OpPC, Call);
5931
5932 case X86::BI__builtin_ia32_shufps:
5933 case X86::BI__builtin_ia32_shufps256:
5934 case X86::BI__builtin_ia32_shufps512:
5935 return interp__builtin_ia32_shuffle_generic(
5936 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
5937 unsigned NumElemPerLane = 4;
5938 unsigned NumSelectableElems = NumElemPerLane / 2;
5939 unsigned BitsPerElem = 2;
5940 unsigned IndexMask = 0x3;
5941 unsigned MaskBits = 8;
5942 unsigned Lane = DstIdx / NumElemPerLane;
5943 unsigned ElemInLane = DstIdx % NumElemPerLane;
5944 unsigned LaneOffset = Lane * NumElemPerLane;
5945 unsigned SrcIdx = ElemInLane >= NumSelectableElems ? 1 : 0;
5946 unsigned BitIndex = (DstIdx * BitsPerElem) % MaskBits;
5947 unsigned Index = (ShuffleMask >> BitIndex) & IndexMask;
5948 return std::pair<unsigned, int>{SrcIdx,
5949 static_cast<int>(LaneOffset + Index)};
5950 });
5951 case X86::BI__builtin_ia32_shufpd:
5952 case X86::BI__builtin_ia32_shufpd256:
5953 case X86::BI__builtin_ia32_shufpd512:
5954 return interp__builtin_ia32_shuffle_generic(
5955 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
5956 unsigned NumElemPerLane = 2;
5957 unsigned NumSelectableElems = NumElemPerLane / 2;
5958 unsigned BitsPerElem = 1;
5959 unsigned IndexMask = 0x1;
5960 unsigned MaskBits = 8;
5961 unsigned Lane = DstIdx / NumElemPerLane;
5962 unsigned ElemInLane = DstIdx % NumElemPerLane;
5963 unsigned LaneOffset = Lane * NumElemPerLane;
5964 unsigned SrcIdx = ElemInLane >= NumSelectableElems ? 1 : 0;
5965 unsigned BitIndex = (DstIdx * BitsPerElem) % MaskBits;
5966 unsigned Index = (ShuffleMask >> BitIndex) & IndexMask;
5967 return std::pair<unsigned, int>{SrcIdx,
5968 static_cast<int>(LaneOffset + Index)};
5969 });
5970
5971 case X86::BI__builtin_ia32_vgf2p8affineinvqb_v16qi:
5972 case X86::BI__builtin_ia32_vgf2p8affineinvqb_v32qi:
5973 case X86::BI__builtin_ia32_vgf2p8affineinvqb_v64qi:
5974 return interp__builtin_ia32_gfni_affine(S, OpPC, Call, Inverse: true);
5975 case X86::BI__builtin_ia32_vgf2p8affineqb_v16qi:
5976 case X86::BI__builtin_ia32_vgf2p8affineqb_v32qi:
5977 case X86::BI__builtin_ia32_vgf2p8affineqb_v64qi:
5978 return interp__builtin_ia32_gfni_affine(S, OpPC, Call, Inverse: false);
5979
5980 case X86::BI__builtin_ia32_vgf2p8mulb_v16qi:
5981 case X86::BI__builtin_ia32_vgf2p8mulb_v32qi:
5982 case X86::BI__builtin_ia32_vgf2p8mulb_v64qi:
5983 return interp__builtin_ia32_gfni_mul(S, OpPC, Call);
5984
5985 case X86::BI__builtin_ia32_bmacor16x16x16_v16hi:
5986 case X86::BI__builtin_ia32_bmacor16x16x16_v32hi:
5987 return interp__builtin_ia32_bmac(S, OpPC, Call, /*IsXor=*/false);
5988 case X86::BI__builtin_ia32_bmacxor16x16x16_v16hi:
5989 case X86::BI__builtin_ia32_bmacxor16x16x16_v32hi:
5990 return interp__builtin_ia32_bmac(S, OpPC, Call, /*IsXor=*/true);
5991
5992 case X86::BI__builtin_ia32_insertps128:
5993 return interp__builtin_ia32_shuffle_generic(
5994 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned Mask) {
5995 // Bits [3:0]: zero mask - if bit is set, zero this element
5996 if ((Mask & (1 << DstIdx)) != 0) {
5997 return std::pair<unsigned, int>{0, -1};
5998 }
5999 // Bits [7:6]: select element from source vector Y (0-3)
6000 // Bits [5:4]: select destination position (0-3)
6001 unsigned SrcElem = (Mask >> 6) & 0x3;
6002 unsigned DstElem = (Mask >> 4) & 0x3;
6003 if (DstIdx == DstElem) {
6004 // Insert element from source vector (B) at this position
6005 return std::pair<unsigned, int>{1, static_cast<int>(SrcElem)};
6006 } else {
6007 // Copy from destination vector (A)
6008 return std::pair<unsigned, int>{0, static_cast<int>(DstIdx)};
6009 }
6010 });
6011 case X86::BI__builtin_ia32_permvarsi256:
6012 case X86::BI__builtin_ia32_permvarsf256:
6013 case X86::BI__builtin_ia32_permvardf512:
6014 case X86::BI__builtin_ia32_permvardi512:
6015 case X86::BI__builtin_ia32_permvarhi128:
6016 return interp__builtin_ia32_shuffle_generic(
6017 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6018 int Offset = ShuffleMask & 0x7;
6019 return std::pair<unsigned, int>{0, Offset};
6020 });
6021 case X86::BI__builtin_ia32_permvarqi128:
6022 case X86::BI__builtin_ia32_permvarhi256:
6023 case X86::BI__builtin_ia32_permvarsi512:
6024 case X86::BI__builtin_ia32_permvarsf512:
6025 return interp__builtin_ia32_shuffle_generic(
6026 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6027 int Offset = ShuffleMask & 0xF;
6028 return std::pair<unsigned, int>{0, Offset};
6029 });
6030 case X86::BI__builtin_ia32_permvardi256:
6031 case X86::BI__builtin_ia32_permvardf256:
6032 return interp__builtin_ia32_shuffle_generic(
6033 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6034 int Offset = ShuffleMask & 0x3;
6035 return std::pair<unsigned, int>{0, Offset};
6036 });
6037 case X86::BI__builtin_ia32_permvarqi256:
6038 case X86::BI__builtin_ia32_permvarhi512:
6039 return interp__builtin_ia32_shuffle_generic(
6040 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6041 int Offset = ShuffleMask & 0x1F;
6042 return std::pair<unsigned, int>{0, Offset};
6043 });
6044 case X86::BI__builtin_ia32_permvarqi512:
6045 return interp__builtin_ia32_shuffle_generic(
6046 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6047 int Offset = ShuffleMask & 0x3F;
6048 return std::pair<unsigned, int>{0, Offset};
6049 });
6050 case X86::BI__builtin_ia32_vpermi2varq128:
6051 case X86::BI__builtin_ia32_vpermi2varpd128:
6052 return interp__builtin_ia32_shuffle_generic(
6053 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6054 int Offset = ShuffleMask & 0x1;
6055 unsigned SrcIdx = (ShuffleMask >> 1) & 0x1;
6056 return std::pair<unsigned, int>{SrcIdx, Offset};
6057 });
6058 case X86::BI__builtin_ia32_vpermi2vard128:
6059 case X86::BI__builtin_ia32_vpermi2varps128:
6060 case X86::BI__builtin_ia32_vpermi2varq256:
6061 case X86::BI__builtin_ia32_vpermi2varpd256:
6062 return interp__builtin_ia32_shuffle_generic(
6063 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6064 int Offset = ShuffleMask & 0x3;
6065 unsigned SrcIdx = (ShuffleMask >> 2) & 0x1;
6066 return std::pair<unsigned, int>{SrcIdx, Offset};
6067 });
6068 case X86::BI__builtin_ia32_vpermi2varhi128:
6069 case X86::BI__builtin_ia32_vpermi2vard256:
6070 case X86::BI__builtin_ia32_vpermi2varps256:
6071 case X86::BI__builtin_ia32_vpermi2varq512:
6072 case X86::BI__builtin_ia32_vpermi2varpd512:
6073 return interp__builtin_ia32_shuffle_generic(
6074 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6075 int Offset = ShuffleMask & 0x7;
6076 unsigned SrcIdx = (ShuffleMask >> 3) & 0x1;
6077 return std::pair<unsigned, int>{SrcIdx, Offset};
6078 });
6079 case X86::BI__builtin_ia32_vpermi2varqi128:
6080 case X86::BI__builtin_ia32_vpermi2varhi256:
6081 case X86::BI__builtin_ia32_vpermi2vard512:
6082 case X86::BI__builtin_ia32_vpermi2varps512:
6083 return interp__builtin_ia32_shuffle_generic(
6084 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6085 int Offset = ShuffleMask & 0xF;
6086 unsigned SrcIdx = (ShuffleMask >> 4) & 0x1;
6087 return std::pair<unsigned, int>{SrcIdx, Offset};
6088 });
6089 case X86::BI__builtin_ia32_vpermi2varqi256:
6090 case X86::BI__builtin_ia32_vpermi2varhi512:
6091 return interp__builtin_ia32_shuffle_generic(
6092 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6093 int Offset = ShuffleMask & 0x1F;
6094 unsigned SrcIdx = (ShuffleMask >> 5) & 0x1;
6095 return std::pair<unsigned, int>{SrcIdx, Offset};
6096 });
6097 case X86::BI__builtin_ia32_vpermi2varqi512:
6098 return interp__builtin_ia32_shuffle_generic(
6099 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6100 int Offset = ShuffleMask & 0x3F;
6101 unsigned SrcIdx = (ShuffleMask >> 6) & 0x1;
6102 return std::pair<unsigned, int>{SrcIdx, Offset};
6103 });
6104 case X86::BI__builtin_ia32_vperm2f128_pd256:
6105 case X86::BI__builtin_ia32_vperm2f128_ps256:
6106 case X86::BI__builtin_ia32_vperm2f128_si256:
6107 case X86::BI__builtin_ia32_permti256: {
6108 unsigned NumElements =
6109 Call->getArg(Arg: 0)->getType()->castAs<VectorType>()->getNumElements();
6110 unsigned PreservedBitsCnt = NumElements >> 2;
6111 return interp__builtin_ia32_shuffle_generic(
6112 S, OpPC, Call,
6113 GetSourceIndex: [PreservedBitsCnt](unsigned DstIdx, unsigned ShuffleMask) {
6114 unsigned ControlBitsCnt = DstIdx >> PreservedBitsCnt << 2;
6115 unsigned ControlBits = ShuffleMask >> ControlBitsCnt;
6116
6117 if (ControlBits & 0b1000)
6118 return std::make_pair(x: 0u, y: -1);
6119
6120 unsigned SrcVecIdx = (ControlBits & 0b10) >> 1;
6121 unsigned PreservedBitsMask = (1 << PreservedBitsCnt) - 1;
6122 int SrcIdx = ((ControlBits & 0b1) << PreservedBitsCnt) |
6123 (DstIdx & PreservedBitsMask);
6124 return std::make_pair(x&: SrcVecIdx, y&: SrcIdx);
6125 });
6126 }
6127 case X86::BI__builtin_ia32_pshufb128:
6128 case X86::BI__builtin_ia32_pshufb256:
6129 case X86::BI__builtin_ia32_pshufb512:
6130 return interp__builtin_ia32_shuffle_generic(
6131 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6132 uint8_t Ctlb = static_cast<uint8_t>(ShuffleMask);
6133 if (Ctlb & 0x80)
6134 return std::make_pair(x: 0, y: -1);
6135
6136 unsigned LaneBase = (DstIdx / 16) * 16;
6137 unsigned SrcOffset = Ctlb & 0x0F;
6138 unsigned SrcIdx = LaneBase + SrcOffset;
6139 return std::make_pair(x: 0, y: static_cast<int>(SrcIdx));
6140 });
6141
6142 case X86::BI__builtin_ia32_pshuflw:
6143 case X86::BI__builtin_ia32_pshuflw256:
6144 case X86::BI__builtin_ia32_pshuflw512:
6145 return interp__builtin_ia32_shuffle_generic(
6146 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6147 unsigned LaneBase = (DstIdx / 8) * 8;
6148 unsigned LaneIdx = DstIdx % 8;
6149 if (LaneIdx < 4) {
6150 unsigned Sel = (ShuffleMask >> (2 * LaneIdx)) & 0x3;
6151 return std::make_pair(x: 0, y: static_cast<int>(LaneBase + Sel));
6152 }
6153
6154 return std::make_pair(x: 0, y: static_cast<int>(DstIdx));
6155 });
6156
6157 case X86::BI__builtin_ia32_pshufhw:
6158 case X86::BI__builtin_ia32_pshufhw256:
6159 case X86::BI__builtin_ia32_pshufhw512:
6160 return interp__builtin_ia32_shuffle_generic(
6161 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6162 unsigned LaneBase = (DstIdx / 8) * 8;
6163 unsigned LaneIdx = DstIdx % 8;
6164 if (LaneIdx >= 4) {
6165 unsigned Sel = (ShuffleMask >> (2 * (LaneIdx - 4))) & 0x3;
6166 return std::make_pair(x: 0, y: static_cast<int>(LaneBase + 4 + Sel));
6167 }
6168
6169 return std::make_pair(x: 0, y: static_cast<int>(DstIdx));
6170 });
6171
6172 case X86::BI__builtin_ia32_pshufd:
6173 case X86::BI__builtin_ia32_pshufd256:
6174 case X86::BI__builtin_ia32_pshufd512:
6175 case X86::BI__builtin_ia32_vpermilps:
6176 case X86::BI__builtin_ia32_vpermilps256:
6177 case X86::BI__builtin_ia32_vpermilps512:
6178 return interp__builtin_ia32_shuffle_generic(
6179 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6180 unsigned LaneBase = (DstIdx / 4) * 4;
6181 unsigned LaneIdx = DstIdx % 4;
6182 unsigned Sel = (ShuffleMask >> (2 * LaneIdx)) & 0x3;
6183 return std::make_pair(x: 0, y: static_cast<int>(LaneBase + Sel));
6184 });
6185
6186 case X86::BI__builtin_ia32_vpermilvarpd:
6187 case X86::BI__builtin_ia32_vpermilvarpd256:
6188 case X86::BI__builtin_ia32_vpermilvarpd512:
6189 return interp__builtin_ia32_shuffle_generic(
6190 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6191 unsigned NumElemPerLane = 2;
6192 unsigned Lane = DstIdx / NumElemPerLane;
6193 unsigned Offset = ShuffleMask & 0b10 ? 1 : 0;
6194 return std::make_pair(
6195 x: 0, y: static_cast<int>(Lane * NumElemPerLane + Offset));
6196 });
6197
6198 case X86::BI__builtin_ia32_vpermilvarps:
6199 case X86::BI__builtin_ia32_vpermilvarps256:
6200 case X86::BI__builtin_ia32_vpermilvarps512:
6201 return interp__builtin_ia32_shuffle_generic(
6202 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6203 unsigned NumElemPerLane = 4;
6204 unsigned Lane = DstIdx / NumElemPerLane;
6205 unsigned Offset = ShuffleMask & 0b11;
6206 return std::make_pair(
6207 x: 0, y: static_cast<int>(Lane * NumElemPerLane + Offset));
6208 });
6209
6210 case X86::BI__builtin_ia32_vpermilpd:
6211 case X86::BI__builtin_ia32_vpermilpd256:
6212 case X86::BI__builtin_ia32_vpermilpd512:
6213 return interp__builtin_ia32_shuffle_generic(
6214 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned Control) {
6215 unsigned NumElemPerLane = 2;
6216 unsigned BitsPerElem = 1;
6217 unsigned MaskBits = 8;
6218 unsigned IndexMask = 0x1;
6219 unsigned Lane = DstIdx / NumElemPerLane;
6220 unsigned LaneOffset = Lane * NumElemPerLane;
6221 unsigned BitIndex = (DstIdx * BitsPerElem) % MaskBits;
6222 unsigned Index = (Control >> BitIndex) & IndexMask;
6223 return std::make_pair(x: 0, y: static_cast<int>(LaneOffset + Index));
6224 });
6225
6226 case X86::BI__builtin_ia32_permdf256:
6227 case X86::BI__builtin_ia32_permdi256:
6228 return interp__builtin_ia32_shuffle_generic(
6229 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned Control) {
6230 // permute4x64 operates on 4 64-bit elements
6231 // For element i (0-3), extract bits [2*i+1:2*i] from Control
6232 unsigned Index = (Control >> (2 * DstIdx)) & 0x3;
6233 return std::make_pair(x: 0, y: static_cast<int>(Index));
6234 });
6235
6236 case X86::BI__builtin_ia32_vpmultishiftqb128:
6237 case X86::BI__builtin_ia32_vpmultishiftqb256:
6238 case X86::BI__builtin_ia32_vpmultishiftqb512:
6239 return interp__builtin_ia32_multishiftqb(S, OpPC, Call);
6240 case X86::BI__builtin_ia32_kandqi:
6241 case X86::BI__builtin_ia32_kandhi:
6242 case X86::BI__builtin_ia32_kandsi:
6243 case X86::BI__builtin_ia32_kanddi:
6244 return interp__builtin_elementwise_int_binop(
6245 S, OpPC, Call,
6246 Fn: [](const APSInt &LHS, const APSInt &RHS) { return LHS & RHS; });
6247
6248 case X86::BI__builtin_ia32_kandnqi:
6249 case X86::BI__builtin_ia32_kandnhi:
6250 case X86::BI__builtin_ia32_kandnsi:
6251 case X86::BI__builtin_ia32_kandndi:
6252 return interp__builtin_elementwise_int_binop(
6253 S, OpPC, Call,
6254 Fn: [](const APSInt &LHS, const APSInt &RHS) { return ~LHS & RHS; });
6255
6256 case X86::BI__builtin_ia32_korqi:
6257 case X86::BI__builtin_ia32_korhi:
6258 case X86::BI__builtin_ia32_korsi:
6259 case X86::BI__builtin_ia32_kordi:
6260 return interp__builtin_elementwise_int_binop(
6261 S, OpPC, Call,
6262 Fn: [](const APSInt &LHS, const APSInt &RHS) { return LHS | RHS; });
6263
6264 case X86::BI__builtin_ia32_kxnorqi:
6265 case X86::BI__builtin_ia32_kxnorhi:
6266 case X86::BI__builtin_ia32_kxnorsi:
6267 case X86::BI__builtin_ia32_kxnordi:
6268 return interp__builtin_elementwise_int_binop(
6269 S, OpPC, Call,
6270 Fn: [](const APSInt &LHS, const APSInt &RHS) { return ~(LHS ^ RHS); });
6271
6272 case X86::BI__builtin_ia32_kxorqi:
6273 case X86::BI__builtin_ia32_kxorhi:
6274 case X86::BI__builtin_ia32_kxorsi:
6275 case X86::BI__builtin_ia32_kxordi:
6276 return interp__builtin_elementwise_int_binop(
6277 S, OpPC, Call,
6278 Fn: [](const APSInt &LHS, const APSInt &RHS) { return LHS ^ RHS; });
6279
6280 case X86::BI__builtin_ia32_knotqi:
6281 case X86::BI__builtin_ia32_knothi:
6282 case X86::BI__builtin_ia32_knotsi:
6283 case X86::BI__builtin_ia32_knotdi:
6284 return interp__builtin_elementwise_int_unaryop(
6285 S, OpPC, Call, Fn: [](const APSInt &Src) { return ~Src; });
6286
6287 case X86::BI__builtin_ia32_kaddqi:
6288 case X86::BI__builtin_ia32_kaddhi:
6289 case X86::BI__builtin_ia32_kaddsi:
6290 case X86::BI__builtin_ia32_kadddi:
6291 return interp__builtin_elementwise_int_binop(
6292 S, OpPC, Call,
6293 Fn: [](const APSInt &LHS, const APSInt &RHS) { return LHS + RHS; });
6294
6295 case X86::BI__builtin_ia32_kmovb:
6296 case X86::BI__builtin_ia32_kmovw:
6297 case X86::BI__builtin_ia32_kmovd:
6298 case X86::BI__builtin_ia32_kmovq:
6299 return interp__builtin_elementwise_int_unaryop(
6300 S, OpPC, Call, Fn: [](const APSInt &Src) { return Src; });
6301
6302 case X86::BI__builtin_ia32_kunpckhi:
6303 case X86::BI__builtin_ia32_kunpckdi:
6304 case X86::BI__builtin_ia32_kunpcksi:
6305 return interp__builtin_elementwise_int_binop(
6306 S, OpPC, Call, Fn: [](const APSInt &A, const APSInt &B) {
6307 // Generic kunpack: extract lower half of each operand and concatenate
6308 // Result = A[HalfWidth-1:0] concat B[HalfWidth-1:0]
6309 unsigned BW = A.getBitWidth();
6310 return APSInt(A.trunc(width: BW / 2).concat(NewLSB: B.trunc(width: BW / 2)),
6311 A.isUnsigned());
6312 });
6313
6314 case X86::BI__builtin_ia32_phminposuw128:
6315 return interp__builtin_ia32_phminposuw(S, OpPC, Call);
6316
6317 case X86::BI__builtin_ia32_psraq128:
6318 case X86::BI__builtin_ia32_psraq256:
6319 case X86::BI__builtin_ia32_psraq512:
6320 case X86::BI__builtin_ia32_psrad128:
6321 case X86::BI__builtin_ia32_psrad256:
6322 case X86::BI__builtin_ia32_psrad512:
6323 case X86::BI__builtin_ia32_psraw128:
6324 case X86::BI__builtin_ia32_psraw256:
6325 case X86::BI__builtin_ia32_psraw512:
6326 return interp__builtin_ia32_shift_with_count(
6327 S, OpPC, Call,
6328 ShiftOp: [](const APInt &Elt, uint64_t Count) { return Elt.ashr(ShiftAmt: Count); },
6329 OverflowOp: [](const APInt &Elt, unsigned Width) { return Elt.ashr(ShiftAmt: Width - 1); });
6330
6331 case X86::BI__builtin_ia32_psllq128:
6332 case X86::BI__builtin_ia32_psllq256:
6333 case X86::BI__builtin_ia32_psllq512:
6334 case X86::BI__builtin_ia32_pslld128:
6335 case X86::BI__builtin_ia32_pslld256:
6336 case X86::BI__builtin_ia32_pslld512:
6337 case X86::BI__builtin_ia32_psllw128:
6338 case X86::BI__builtin_ia32_psllw256:
6339 case X86::BI__builtin_ia32_psllw512:
6340 return interp__builtin_ia32_shift_with_count(
6341 S, OpPC, Call,
6342 ShiftOp: [](const APInt &Elt, uint64_t Count) { return Elt.shl(shiftAmt: Count); },
6343 OverflowOp: [](const APInt &Elt, unsigned Width) { return APInt::getZero(numBits: Width); });
6344
6345 case X86::BI__builtin_ia32_psrlq128:
6346 case X86::BI__builtin_ia32_psrlq256:
6347 case X86::BI__builtin_ia32_psrlq512:
6348 case X86::BI__builtin_ia32_psrld128:
6349 case X86::BI__builtin_ia32_psrld256:
6350 case X86::BI__builtin_ia32_psrld512:
6351 case X86::BI__builtin_ia32_psrlw128:
6352 case X86::BI__builtin_ia32_psrlw256:
6353 case X86::BI__builtin_ia32_psrlw512:
6354 return interp__builtin_ia32_shift_with_count(
6355 S, OpPC, Call,
6356 ShiftOp: [](const APInt &Elt, uint64_t Count) { return Elt.lshr(shiftAmt: Count); },
6357 OverflowOp: [](const APInt &Elt, unsigned Width) { return APInt::getZero(numBits: Width); });
6358
6359 case X86::BI__builtin_ia32_pternlogd128_mask:
6360 case X86::BI__builtin_ia32_pternlogd256_mask:
6361 case X86::BI__builtin_ia32_pternlogd512_mask:
6362 case X86::BI__builtin_ia32_pternlogq128_mask:
6363 case X86::BI__builtin_ia32_pternlogq256_mask:
6364 case X86::BI__builtin_ia32_pternlogq512_mask:
6365 return interp__builtin_ia32_pternlog(S, OpPC, Call, /*MaskZ=*/false);
6366 case X86::BI__builtin_ia32_pternlogd128_maskz:
6367 case X86::BI__builtin_ia32_pternlogd256_maskz:
6368 case X86::BI__builtin_ia32_pternlogd512_maskz:
6369 case X86::BI__builtin_ia32_pternlogq128_maskz:
6370 case X86::BI__builtin_ia32_pternlogq256_maskz:
6371 case X86::BI__builtin_ia32_pternlogq512_maskz:
6372 return interp__builtin_ia32_pternlog(S, OpPC, Call, /*MaskZ=*/true);
6373 case Builtin::BI__builtin_elementwise_fshl:
6374 return interp__builtin_elementwise_triop(S, OpPC, Call,
6375 Fn: llvm::APIntOps::fshl);
6376 case Builtin::BI__builtin_elementwise_fshr:
6377 return interp__builtin_elementwise_triop(S, OpPC, Call,
6378 Fn: llvm::APIntOps::fshr);
6379
6380 case X86::BI__builtin_ia32_shuf_f32x4_256:
6381 case X86::BI__builtin_ia32_shuf_i32x4_256:
6382 case X86::BI__builtin_ia32_shuf_f64x2_256:
6383 case X86::BI__builtin_ia32_shuf_i64x2_256:
6384 case X86::BI__builtin_ia32_shuf_f32x4:
6385 case X86::BI__builtin_ia32_shuf_i32x4:
6386 case X86::BI__builtin_ia32_shuf_f64x2:
6387 case X86::BI__builtin_ia32_shuf_i64x2: {
6388 // Destination and sources A, B all have the same type.
6389 QualType VecQT = Call->getArg(Arg: 0)->getType();
6390 const auto *VecT = VecQT->castAs<VectorType>();
6391 unsigned NumElems = VecT->getNumElements();
6392 unsigned ElemBits = S.getASTContext().getTypeSize(T: VecT->getElementType());
6393 unsigned LaneBits = 128u;
6394 unsigned NumLanes = (NumElems * ElemBits) / LaneBits;
6395 unsigned NumElemsPerLane = LaneBits / ElemBits;
6396
6397 return interp__builtin_ia32_shuffle_generic(
6398 S, OpPC, Call,
6399 GetSourceIndex: [NumLanes, NumElemsPerLane](unsigned DstIdx, unsigned ShuffleMask) {
6400 // DstIdx determines source. ShuffleMask selects lane in source.
6401 unsigned BitsPerElem = NumLanes / 2;
6402 unsigned IndexMask = (1u << BitsPerElem) - 1;
6403 unsigned Lane = DstIdx / NumElemsPerLane;
6404 unsigned SrcIdx = (Lane < NumLanes / 2) ? 0 : 1;
6405 unsigned BitIdx = BitsPerElem * Lane;
6406 unsigned SrcLaneIdx = (ShuffleMask >> BitIdx) & IndexMask;
6407 unsigned ElemInLane = DstIdx % NumElemsPerLane;
6408 unsigned IdxToPick = SrcLaneIdx * NumElemsPerLane + ElemInLane;
6409 return std::pair<unsigned, int>{SrcIdx, IdxToPick};
6410 });
6411 }
6412
6413 case X86::BI__builtin_ia32_insertf32x4_256:
6414 case X86::BI__builtin_ia32_inserti32x4_256:
6415 case X86::BI__builtin_ia32_insertf64x2_256:
6416 case X86::BI__builtin_ia32_inserti64x2_256:
6417 case X86::BI__builtin_ia32_insertf32x4:
6418 case X86::BI__builtin_ia32_inserti32x4:
6419 case X86::BI__builtin_ia32_insertf64x2_512:
6420 case X86::BI__builtin_ia32_inserti64x2_512:
6421 case X86::BI__builtin_ia32_insertf32x8:
6422 case X86::BI__builtin_ia32_inserti32x8:
6423 case X86::BI__builtin_ia32_insertf64x4:
6424 case X86::BI__builtin_ia32_inserti64x4:
6425 case X86::BI__builtin_ia32_vinsertf128_ps256:
6426 case X86::BI__builtin_ia32_vinsertf128_pd256:
6427 case X86::BI__builtin_ia32_vinsertf128_si256:
6428 case X86::BI__builtin_ia32_insert128i256:
6429 return interp__builtin_ia32_insert_subvector(S, OpPC, Call, ID: BuiltinID);
6430
6431 case clang::X86::BI__builtin_ia32_vcvtps2ph:
6432 case clang::X86::BI__builtin_ia32_vcvtps2ph256:
6433 return interp__builtin_ia32_vcvtps2ph(S, OpPC, Call);
6434
6435 case X86::BI__builtin_ia32_vec_ext_v4hi:
6436 case X86::BI__builtin_ia32_vec_ext_v16qi:
6437 case X86::BI__builtin_ia32_vec_ext_v8hi:
6438 case X86::BI__builtin_ia32_vec_ext_v4si:
6439 case X86::BI__builtin_ia32_vec_ext_v2di:
6440 case X86::BI__builtin_ia32_vec_ext_v32qi:
6441 case X86::BI__builtin_ia32_vec_ext_v16hi:
6442 case X86::BI__builtin_ia32_vec_ext_v8si:
6443 case X86::BI__builtin_ia32_vec_ext_v4di:
6444 case X86::BI__builtin_ia32_vec_ext_v4sf:
6445 return interp__builtin_ia32_vec_ext(S, OpPC, Call, ID: BuiltinID);
6446
6447 case X86::BI__builtin_ia32_vec_set_v4hi:
6448 case X86::BI__builtin_ia32_vec_set_v16qi:
6449 case X86::BI__builtin_ia32_vec_set_v8hi:
6450 case X86::BI__builtin_ia32_vec_set_v4si:
6451 case X86::BI__builtin_ia32_vec_set_v2di:
6452 case X86::BI__builtin_ia32_vec_set_v32qi:
6453 case X86::BI__builtin_ia32_vec_set_v16hi:
6454 case X86::BI__builtin_ia32_vec_set_v8si:
6455 case X86::BI__builtin_ia32_vec_set_v4di:
6456 return interp__builtin_ia32_vec_set(S, OpPC, Call, ID: BuiltinID);
6457
6458 case X86::BI__builtin_ia32_cvtb2mask128:
6459 case X86::BI__builtin_ia32_cvtb2mask256:
6460 case X86::BI__builtin_ia32_cvtb2mask512:
6461 case X86::BI__builtin_ia32_cvtw2mask128:
6462 case X86::BI__builtin_ia32_cvtw2mask256:
6463 case X86::BI__builtin_ia32_cvtw2mask512:
6464 case X86::BI__builtin_ia32_cvtd2mask128:
6465 case X86::BI__builtin_ia32_cvtd2mask256:
6466 case X86::BI__builtin_ia32_cvtd2mask512:
6467 case X86::BI__builtin_ia32_cvtq2mask128:
6468 case X86::BI__builtin_ia32_cvtq2mask256:
6469 case X86::BI__builtin_ia32_cvtq2mask512:
6470 return interp__builtin_ia32_cvt_vec2mask(S, OpPC, Call, ID: BuiltinID);
6471
6472 case X86::BI__builtin_ia32_cvtmask2b128:
6473 case X86::BI__builtin_ia32_cvtmask2b256:
6474 case X86::BI__builtin_ia32_cvtmask2b512:
6475 case X86::BI__builtin_ia32_cvtmask2w128:
6476 case X86::BI__builtin_ia32_cvtmask2w256:
6477 case X86::BI__builtin_ia32_cvtmask2w512:
6478 case X86::BI__builtin_ia32_cvtmask2d128:
6479 case X86::BI__builtin_ia32_cvtmask2d256:
6480 case X86::BI__builtin_ia32_cvtmask2d512:
6481 case X86::BI__builtin_ia32_cvtmask2q128:
6482 case X86::BI__builtin_ia32_cvtmask2q256:
6483 case X86::BI__builtin_ia32_cvtmask2q512:
6484 return interp__builtin_ia32_cvt_mask2vec(S, OpPC, Call, ID: BuiltinID);
6485
6486 case X86::BI__builtin_ia32_cvtsd2ss:
6487 return interp__builtin_ia32_cvtsd2ss(S, OpPC, Call, HasRoundingMask: false);
6488
6489 case X86::BI__builtin_ia32_cvtsd2ss_round_mask:
6490 return interp__builtin_ia32_cvtsd2ss(S, OpPC, Call, HasRoundingMask: true);
6491
6492 case X86::BI__builtin_ia32_cvtpd2ps:
6493 case X86::BI__builtin_ia32_cvtpd2ps256:
6494 return interp__builtin_ia32_cvtpd2ps(S, OpPC, Call, IsMasked: false, HasRounding: false);
6495 case X86::BI__builtin_ia32_cvtpd2ps_mask:
6496 return interp__builtin_ia32_cvtpd2ps(S, OpPC, Call, IsMasked: true, HasRounding: false);
6497 case X86::BI__builtin_ia32_cvtpd2ps512_mask:
6498 return interp__builtin_ia32_cvtpd2ps(S, OpPC, Call, IsMasked: true, HasRounding: true);
6499
6500 case X86::BI__builtin_ia32_cmpb128_mask:
6501 case X86::BI__builtin_ia32_cmpw128_mask:
6502 case X86::BI__builtin_ia32_cmpd128_mask:
6503 case X86::BI__builtin_ia32_cmpq128_mask:
6504 case X86::BI__builtin_ia32_cmpb256_mask:
6505 case X86::BI__builtin_ia32_cmpw256_mask:
6506 case X86::BI__builtin_ia32_cmpd256_mask:
6507 case X86::BI__builtin_ia32_cmpq256_mask:
6508 case X86::BI__builtin_ia32_cmpb512_mask:
6509 case X86::BI__builtin_ia32_cmpw512_mask:
6510 case X86::BI__builtin_ia32_cmpd512_mask:
6511 case X86::BI__builtin_ia32_cmpq512_mask:
6512 return interp__builtin_ia32_cmp_mask(S, OpPC, Call, ID: BuiltinID,
6513 /*IsUnsigned=*/false);
6514
6515 case X86::BI__builtin_ia32_ucmpb128_mask:
6516 case X86::BI__builtin_ia32_ucmpw128_mask:
6517 case X86::BI__builtin_ia32_ucmpd128_mask:
6518 case X86::BI__builtin_ia32_ucmpq128_mask:
6519 case X86::BI__builtin_ia32_ucmpb256_mask:
6520 case X86::BI__builtin_ia32_ucmpw256_mask:
6521 case X86::BI__builtin_ia32_ucmpd256_mask:
6522 case X86::BI__builtin_ia32_ucmpq256_mask:
6523 case X86::BI__builtin_ia32_ucmpb512_mask:
6524 case X86::BI__builtin_ia32_ucmpw512_mask:
6525 case X86::BI__builtin_ia32_ucmpd512_mask:
6526 case X86::BI__builtin_ia32_ucmpq512_mask:
6527 return interp__builtin_ia32_cmp_mask(S, OpPC, Call, ID: BuiltinID,
6528 /*IsUnsigned=*/true);
6529
6530 case X86::BI__builtin_ia32_vpshufbitqmb128_mask:
6531 case X86::BI__builtin_ia32_vpshufbitqmb256_mask:
6532 case X86::BI__builtin_ia32_vpshufbitqmb512_mask:
6533 return interp__builtin_ia32_shufbitqmb_mask(S, OpPC, Call);
6534
6535 case X86::BI__builtin_ia32_pslldqi128_byteshift:
6536 case X86::BI__builtin_ia32_pslldqi256_byteshift:
6537 case X86::BI__builtin_ia32_pslldqi512_byteshift:
6538 // These SLLDQ intrinsics always operate on byte elements (8 bits).
6539 // The lane width is hardcoded to 16 to match the SIMD register size,
6540 // but the algorithm processes one byte per iteration,
6541 // so APInt(8, ...) is correct and intentional.
6542 return interp__builtin_ia32_shuffle_generic(
6543 S, OpPC, Call,
6544 GetSourceIndex: [](unsigned DstIdx, unsigned Shift) -> std::pair<unsigned, int> {
6545 unsigned LaneBase = (DstIdx / 16) * 16;
6546 unsigned LaneIdx = DstIdx % 16;
6547 if (LaneIdx < Shift)
6548 return std::make_pair(x: 0, y: -1);
6549
6550 return std::make_pair(x: 0,
6551 y: static_cast<int>(LaneBase + LaneIdx - Shift));
6552 });
6553
6554 case X86::BI__builtin_ia32_psrldqi128_byteshift:
6555 case X86::BI__builtin_ia32_psrldqi256_byteshift:
6556 case X86::BI__builtin_ia32_psrldqi512_byteshift:
6557 // These SRLDQ intrinsics always operate on byte elements (8 bits).
6558 // The lane width is hardcoded to 16 to match the SIMD register size,
6559 // but the algorithm processes one byte per iteration,
6560 // so APInt(8, ...) is correct and intentional.
6561 return interp__builtin_ia32_shuffle_generic(
6562 S, OpPC, Call,
6563 GetSourceIndex: [](unsigned DstIdx, unsigned Shift) -> std::pair<unsigned, int> {
6564 unsigned LaneBase = (DstIdx / 16) * 16;
6565 unsigned LaneIdx = DstIdx % 16;
6566 if (LaneIdx + Shift < 16)
6567 return std::make_pair(x: 0,
6568 y: static_cast<int>(LaneBase + LaneIdx + Shift));
6569
6570 return std::make_pair(x: 0, y: -1);
6571 });
6572
6573 case X86::BI__builtin_ia32_palignr128:
6574 case X86::BI__builtin_ia32_palignr256:
6575 case X86::BI__builtin_ia32_palignr512:
6576 return interp__builtin_ia32_shuffle_generic(
6577 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned Shift) {
6578 // Default to -1 → zero-fill this destination element
6579 unsigned VecIdx = 1;
6580 int ElemIdx = -1;
6581
6582 int Lane = DstIdx / 16;
6583 int Offset = DstIdx % 16;
6584
6585 // Elements come from VecB first, then VecA after the shift boundary
6586 unsigned ShiftedIdx = Offset + (Shift & 0xFF);
6587 if (ShiftedIdx < 16) { // from VecB
6588 ElemIdx = ShiftedIdx + (Lane * 16);
6589 } else if (ShiftedIdx < 32) { // from VecA
6590 VecIdx = 0;
6591 ElemIdx = (ShiftedIdx - 16) + (Lane * 16);
6592 }
6593
6594 return std::pair<unsigned, int>{VecIdx, ElemIdx};
6595 });
6596
6597 case X86::BI__builtin_ia32_alignd128:
6598 case X86::BI__builtin_ia32_alignd256:
6599 case X86::BI__builtin_ia32_alignd512:
6600 case X86::BI__builtin_ia32_alignq128:
6601 case X86::BI__builtin_ia32_alignq256:
6602 case X86::BI__builtin_ia32_alignq512: {
6603 unsigned NumElems = Call->getType()->castAs<VectorType>()->getNumElements();
6604 return interp__builtin_ia32_shuffle_generic(
6605 S, OpPC, Call, GetSourceIndex: [NumElems](unsigned DstIdx, unsigned Shift) {
6606 unsigned Imm = Shift & 0xFF;
6607 unsigned EffectiveShift = Imm & (NumElems - 1);
6608 unsigned SourcePos = DstIdx + EffectiveShift;
6609 unsigned VecIdx = SourcePos < NumElems ? 1u : 0u;
6610 unsigned ElemIdx = SourcePos & (NumElems - 1);
6611 return std::pair<unsigned, int>{VecIdx, static_cast<int>(ElemIdx)};
6612 });
6613 }
6614
6615 case clang::X86::BI__builtin_ia32_minps:
6616 case clang::X86::BI__builtin_ia32_minpd:
6617 case clang::X86::BI__builtin_ia32_minph128:
6618 case clang::X86::BI__builtin_ia32_minph256:
6619 case clang::X86::BI__builtin_ia32_minps256:
6620 case clang::X86::BI__builtin_ia32_minpd256:
6621 case clang::X86::BI__builtin_ia32_minps512:
6622 case clang::X86::BI__builtin_ia32_minpd512:
6623 case clang::X86::BI__builtin_ia32_minph512:
6624 return interp__builtin_elementwise_fp_binop(
6625 S, OpPC, Call,
6626 Fn: [](const APFloat &A, const APFloat &B,
6627 std::optional<APSInt>) -> std::optional<APFloat> {
6628 if (A.isNaN() || A.isInfinity() || A.isDenormal() || B.isNaN() ||
6629 B.isInfinity() || B.isDenormal())
6630 return std::nullopt;
6631 if (A.isZero() && B.isZero())
6632 return B;
6633 return llvm::minimum(A, B);
6634 });
6635
6636 case clang::X86::BI__builtin_ia32_minss:
6637 case clang::X86::BI__builtin_ia32_minsd:
6638 return interp__builtin_elementwise_fp_binop(
6639 S, OpPC, Call,
6640 Fn: [](const APFloat &A, const APFloat &B,
6641 std::optional<APSInt> RoundingMode) -> std::optional<APFloat> {
6642 return EvalScalarMinMaxFp(A, B, RoundingMode, /*IsMin=*/true);
6643 },
6644 /*IsScalar=*/true);
6645
6646 case clang::X86::BI__builtin_ia32_minsd_round_mask:
6647 case clang::X86::BI__builtin_ia32_minss_round_mask:
6648 case clang::X86::BI__builtin_ia32_minsh_round_mask:
6649 case clang::X86::BI__builtin_ia32_maxsd_round_mask:
6650 case clang::X86::BI__builtin_ia32_maxss_round_mask:
6651 case clang::X86::BI__builtin_ia32_maxsh_round_mask: {
6652 bool IsMin = BuiltinID == clang::X86::BI__builtin_ia32_minsd_round_mask ||
6653 BuiltinID == clang::X86::BI__builtin_ia32_minss_round_mask ||
6654 BuiltinID == clang::X86::BI__builtin_ia32_minsh_round_mask;
6655 return interp__builtin_scalar_fp_round_mask_binop(
6656 S, OpPC, Call,
6657 Fn: [IsMin](const APFloat &A, const APFloat &B,
6658 std::optional<APSInt> RoundingMode) -> std::optional<APFloat> {
6659 return EvalScalarMinMaxFp(A, B, RoundingMode, IsMin);
6660 });
6661 }
6662
6663 case clang::X86::BI__builtin_ia32_maxps:
6664 case clang::X86::BI__builtin_ia32_maxpd:
6665 case clang::X86::BI__builtin_ia32_maxph128:
6666 case clang::X86::BI__builtin_ia32_maxph256:
6667 case clang::X86::BI__builtin_ia32_maxps256:
6668 case clang::X86::BI__builtin_ia32_maxpd256:
6669 case clang::X86::BI__builtin_ia32_maxps512:
6670 case clang::X86::BI__builtin_ia32_maxpd512:
6671 case clang::X86::BI__builtin_ia32_maxph512:
6672 return interp__builtin_elementwise_fp_binop(
6673 S, OpPC, Call,
6674 Fn: [](const APFloat &A, const APFloat &B,
6675 std::optional<APSInt>) -> std::optional<APFloat> {
6676 if (A.isNaN() || A.isInfinity() || A.isDenormal() || B.isNaN() ||
6677 B.isInfinity() || B.isDenormal())
6678 return std::nullopt;
6679 if (A.isZero() && B.isZero())
6680 return B;
6681 return llvm::maximum(A, B);
6682 });
6683
6684 case clang::X86::BI__builtin_ia32_maxss:
6685 case clang::X86::BI__builtin_ia32_maxsd:
6686 return interp__builtin_elementwise_fp_binop(
6687 S, OpPC, Call,
6688 Fn: [](const APFloat &A, const APFloat &B,
6689 std::optional<APSInt> RoundingMode) -> std::optional<APFloat> {
6690 return EvalScalarMinMaxFp(A, B, RoundingMode, /*IsMin=*/false);
6691 },
6692 /*IsScalar=*/true);
6693 case X86::BI__builtin_ia32_vpdpwssd128:
6694 case X86::BI__builtin_ia32_vpdpwssd256:
6695 case X86::BI__builtin_ia32_vpdpwssd512:
6696 case X86::BI__builtin_ia32_vpdpbusd128:
6697 case X86::BI__builtin_ia32_vpdpbusd256:
6698 case X86::BI__builtin_ia32_vpdpbusd512:
6699 return interp__builtin_ia32_vpdp(S, OpPC, Call, IsSaturating: false);
6700 case X86::BI__builtin_ia32_vpdpwssds128:
6701 case X86::BI__builtin_ia32_vpdpwssds256:
6702 case X86::BI__builtin_ia32_vpdpwssds512:
6703 case X86::BI__builtin_ia32_vpdpbusds128:
6704 case X86::BI__builtin_ia32_vpdpbusds256:
6705 case X86::BI__builtin_ia32_vpdpbusds512:
6706 return interp__builtin_ia32_vpdp(S, OpPC, Call, IsSaturating: true);
6707 case X86::BI__builtin_ia32_cvtss2si:
6708 case X86::BI__builtin_ia32_cvtsd2si:
6709 case X86::BI__builtin_ia32_cvttss2si:
6710 case X86::BI__builtin_ia32_cvttsd2si:
6711 case X86::BI__builtin_ia32_cvtss2si64:
6712 case X86::BI__builtin_ia32_cvtsd2si64:
6713 case X86::BI__builtin_ia32_cvttss2si64:
6714 case X86::BI__builtin_ia32_cvttsd2si64:
6715 return interp_builtin_ia32_cvt_scalar_to_int(S, OpPC, E: Call);
6716 case X86::BI__builtin_ia32_cvtpd2dq:
6717 case X86::BI__builtin_ia32_cvttpd2dq:
6718 case X86::BI__builtin_ia32_cvtps2dq:
6719 case X86::BI__builtin_ia32_cvtpd2dq256:
6720 case X86::BI__builtin_ia32_cvtps2dq256:
6721 case X86::BI__builtin_ia32_cvttps2dq:
6722 case X86::BI__builtin_ia32_cvttpd2dq256:
6723 case X86::BI__builtin_ia32_cvttps2dq256:
6724 return interp_builtin_ia32_cvt_vector_to_int(S, OpPC, E: Call);
6725 default:
6726 S.FFDiag(Loc: S.Current->getLocation(PC: OpPC),
6727 DiagId: diag::note_invalid_subexpr_in_const_expr)
6728 << S.Current->getRange(PC: OpPC);
6729
6730 return false;
6731 }
6732
6733 llvm_unreachable("Unhandled builtin ID");
6734}
6735
6736bool InterpretOffsetOf(InterpState &S, CodePtr OpPC, const OffsetOfExpr *E,
6737 ArrayRef<int64_t> ArrayIndices, int64_t &IntResult) {
6738 S.getASTContext().recordOffsetOfEvaluation(E);
6739 CharUnits Result;
6740 unsigned N = E->getNumComponents();
6741 assert(N > 0);
6742
6743 unsigned ArrayIndex = 0;
6744 QualType CurrentType = E->getTypeSourceInfo()->getType();
6745 for (unsigned I = 0; I != N; ++I) {
6746 const OffsetOfNode &Node = E->getComponent(Idx: I);
6747 switch (Node.getKind()) {
6748 case OffsetOfNode::Field: {
6749 const FieldDecl *MemberDecl = Node.getField();
6750 const auto *RD = CurrentType->getAsRecordDecl();
6751 if (!RD || RD->isInvalidDecl())
6752 return false;
6753 const ASTRecordLayout &RL = S.getASTContext().getASTRecordLayout(D: RD);
6754 unsigned FieldIndex = MemberDecl->getFieldIndex();
6755 assert(FieldIndex < RL.getFieldCount() && "offsetof field in wrong type");
6756 Result +=
6757 S.getASTContext().toCharUnitsFromBits(BitSize: RL.getFieldOffset(FieldNo: FieldIndex));
6758 CurrentType = MemberDecl->getType().getNonReferenceType();
6759 break;
6760 }
6761 case OffsetOfNode::Array: {
6762 // When generating bytecode, we put all the index expressions as Sint64 on
6763 // the stack.
6764 int64_t Index = ArrayIndices[ArrayIndex];
6765 if (Index < 0)
6766 return Invalid(S, OpPC);
6767 const ArrayType *AT = S.getASTContext().getAsArrayType(T: CurrentType);
6768 if (!AT)
6769 return false;
6770 CurrentType = AT->getElementType();
6771 CharUnits ElementSize = S.getASTContext().getTypeSizeInChars(T: CurrentType);
6772 int64_t ElemSize = ElementSize.getQuantity();
6773 if (Index != 0 && ElemSize > (llvm::maxIntN(N: 64) / Index)) {
6774 S.FFDiag(Loc: S.Current->getLocation(PC: OpPC),
6775 DiagId: diag::note_constexpr_offsetof_overflow)
6776 << S.Current->getRange(PC: OpPC);
6777 return false;
6778 }
6779 int64_t Offset = Index * ElemSize;
6780 if (Result.getQuantity() > llvm::maxIntN(N: 64) - Offset) {
6781 S.FFDiag(Loc: S.Current->getLocation(PC: OpPC),
6782 DiagId: diag::note_constexpr_offsetof_overflow)
6783 << S.Current->getRange(PC: OpPC);
6784 return false;
6785 }
6786 Result += CharUnits::fromQuantity(Quantity: Offset);
6787 ++ArrayIndex;
6788 break;
6789 }
6790 case OffsetOfNode::Base: {
6791 const CXXBaseSpecifier *BaseSpec = Node.getBase();
6792 if (BaseSpec->isVirtual())
6793 return false;
6794
6795 // Find the layout of the class whose base we are looking into.
6796 const auto *RD = CurrentType->getAsCXXRecordDecl();
6797 if (!RD || RD->isInvalidDecl())
6798 return false;
6799 const ASTRecordLayout &RL = S.getASTContext().getASTRecordLayout(D: RD);
6800
6801 // Find the base class itself.
6802 CurrentType = BaseSpec->getType();
6803 const auto *BaseRD = CurrentType->getAsCXXRecordDecl();
6804 if (!BaseRD)
6805 return false;
6806
6807 // Add the offset to the base.
6808 Result += RL.getBaseClassOffset(Base: BaseRD);
6809 break;
6810 }
6811 case OffsetOfNode::Identifier:
6812 llvm_unreachable("Dependent OffsetOfExpr?");
6813 }
6814 }
6815
6816 IntResult = Result.getQuantity();
6817
6818 return true;
6819}
6820
6821bool SetThreeWayComparisonField(InterpState &S, CodePtr OpPC,
6822 const Pointer &Ptr, const APSInt &IntValue) {
6823
6824 const Record *R = Ptr.getRecord();
6825 assert(R);
6826 assert(R->getNumFields() == 1);
6827
6828 unsigned FieldOffset = R->getField(I: 0u)->Offset;
6829 PtrView FieldPtr = Ptr.view().atField(Offset: FieldOffset);
6830 PrimType FieldT = FieldPtr.getFieldDesc()->getPrimType();
6831
6832 INT_TYPE_SWITCH(FieldT,
6833 FieldPtr.deref<T>() = T::from(IntValue.getSExtValue()));
6834 FieldPtr.initialize();
6835 return true;
6836}
6837
6838static void zeroAll(PtrView Dest) {
6839 const Descriptor *Desc = Dest.getFieldDesc();
6840
6841 if (Desc->isPrimitive()) {
6842 TYPE_SWITCH(Desc->getPrimType(), {
6843 Dest.deref<T>().~T();
6844 new (&Dest.deref<T>()) T();
6845 });
6846 return;
6847 }
6848
6849 if (Desc->isRecord()) {
6850 const Record *R = Desc->ElemRecord;
6851 for (const Record::Field &F : R->fields()) {
6852 PtrView FieldPtr = Dest.atField(Offset: F.Offset);
6853 zeroAll(Dest: FieldPtr);
6854 }
6855 return;
6856 }
6857
6858 if (Desc->isPrimitiveArray()) {
6859 for (unsigned I = 0, N = Desc->getNumElems(); I != N; ++I) {
6860 TYPE_SWITCH(Desc->getPrimType(), {
6861 Dest.deref<T>().~T();
6862 new (&Dest.deref<T>()) T();
6863 });
6864 }
6865 return;
6866 }
6867
6868 if (Desc->isCompositeArray()) {
6869 for (unsigned I = 0, N = Desc->getNumElems(); I != N; ++I) {
6870 PtrView ElemPtr = Dest.atIndex(Idx: I).narrow();
6871 zeroAll(Dest: ElemPtr);
6872 }
6873 return;
6874 }
6875}
6876
6877static bool copyComposite(InterpState &S, CodePtr OpPC, PtrView Src,
6878 PtrView Dest, bool Activate, bool Diagnose);
6879static bool copyRecord(InterpState &S, CodePtr OpPC, PtrView Src, PtrView Dest,
6880 bool Activate = false, bool Diagnose = true) {
6881 [[maybe_unused]] const Descriptor *SrcDesc = Src.getFieldDesc();
6882 const Descriptor *DestDesc = Dest.getFieldDesc();
6883
6884 auto copyField = [&](const Record::Field &F, bool Activate) -> bool {
6885 PtrView DestField = Dest.atField(Offset: F.Offset);
6886 PtrView SrcField = Src.atField(Offset: F.Offset);
6887
6888 if (OptPrimType FT = F.T) {
6889 if (!SrcField.isInitialized()) {
6890 if (Diagnose)
6891 return diagnoseUninitialized(S, OpPC, Extern: false, B: SrcField.block(),
6892 LT: SrcField.getLifetime(), AK: AK_Read);
6893 // Just skip.
6894 return true;
6895 }
6896
6897 TYPE_SWITCH(*FT, DestField.deref<T>() = SrcField.deref<T>(););
6898 if (DestField.canBeInitialized())
6899 DestField.initialize();
6900 if (Activate)
6901 DestField.activate();
6902 return true;
6903 }
6904
6905 return copyComposite(S, OpPC, Src: SrcField, Dest: DestField, Activate, Diagnose);
6906 };
6907
6908 assert(SrcDesc->isRecord());
6909 assert(SrcDesc->ElemRecord == DestDesc->ElemRecord);
6910 const Record *R = DestDesc->ElemRecord;
6911 for (const Record::Field &F : R->fields()) {
6912 PtrView FP = Src.atField(Offset: F.Offset);
6913
6914 if (!CheckMutable(S, OpPC, Ptr: FP))
6915 return false;
6916
6917 if (R->isUnion()) {
6918 // For unions, only copy the active field. Zero all others.
6919 if (FP.isActive()) {
6920 if (!copyField(F, /*Activate=*/true))
6921 return false;
6922 } else {
6923 PtrView DestField = Dest.atField(Offset: F.Offset);
6924 zeroAll(Dest: DestField);
6925 }
6926 } else {
6927 if (!copyField(F, Activate))
6928 return false;
6929 }
6930 }
6931
6932 for (const Record::Base &B : R->bases()) {
6933 PtrView DestBase = Dest.atField(Offset: B.Offset);
6934 if (!copyRecord(S, OpPC, Src: Src.atField(Offset: B.Offset), Dest: DestBase, Activate,
6935 Diagnose))
6936 return false;
6937 }
6938
6939 Dest.initialize();
6940 if (Activate)
6941 Dest.activate();
6942 return true;
6943}
6944
6945static bool copyComposite(InterpState &S, CodePtr OpPC, PtrView Src,
6946 PtrView Dest, bool Activate = false,
6947 bool Diagnose = false) {
6948 assert(Src.isLive() && Dest.isLive());
6949
6950 [[maybe_unused]] const Descriptor *SrcDesc = Src.getFieldDesc();
6951 const Descriptor *DestDesc = Dest.getFieldDesc();
6952
6953 assert(!DestDesc->isPrimitive() && !SrcDesc->isPrimitive());
6954
6955 if (DestDesc->isPrimitiveArray()) {
6956 if (!SrcDesc->isPrimitiveArray())
6957 return false;
6958 // For floating types, check the actual QualType so we don't accidentally
6959 // mix up semantics.
6960 if (SrcDesc->getPrimType() == PT_Float) {
6961 if (!S.getASTContext().hasSimilarType(T1: SrcDesc->getElemQualType(),
6962 T2: DestDesc->getElemQualType()))
6963 return false;
6964 }
6965
6966 assert(SrcDesc->isPrimitiveArray());
6967 assert(SrcDesc->getNumElems() == DestDesc->getNumElems());
6968 assert(SrcDesc->getPrimType() == DestDesc->getPrimType());
6969 PrimType ET = DestDesc->getPrimType();
6970 for (unsigned I = 0, N = DestDesc->getNumElems(); I != N; ++I) {
6971 PtrView DestElem = Dest.atIndex(Idx: I);
6972 TYPE_SWITCH(ET, { DestElem.deref<T>() = Src.elem<T>(I); });
6973 DestElem.initializeElement(Index: I);
6974 }
6975 return true;
6976 }
6977
6978 if (DestDesc->isCompositeArray()) {
6979 if (!SrcDesc->isCompositeArray())
6980 return false;
6981 assert(SrcDesc->isCompositeArray());
6982 assert(SrcDesc->getNumElems() == DestDesc->getNumElems());
6983 for (unsigned I = 0, N = DestDesc->getNumElems(); I != N; ++I) {
6984 PtrView SrcElem = Src.atIndex(Idx: I).narrow();
6985 PtrView DestElem = Dest.atIndex(Idx: I).narrow();
6986 if (!copyComposite(S, OpPC, Src: SrcElem, Dest: DestElem, Activate))
6987 return false;
6988 }
6989 return true;
6990 }
6991
6992 if (DestDesc->isRecord()) {
6993 if (!SrcDesc->isRecord())
6994 return false;
6995 return copyRecord(S, OpPC, Src, Dest, Activate, Diagnose);
6996 }
6997 return Invalid(S, OpPC);
6998}
6999
7000bool DoMemcpy(InterpState &S, CodePtr OpPC, const Pointer &Src, Pointer &Dest,
7001 bool Activate, bool Diagnose) {
7002 if (!Src.isBlockPointer() || Src.getFieldDesc()->isPrimitive())
7003 return false;
7004 if (!Dest.isBlockPointer() || Dest.getFieldDesc()->isPrimitive())
7005 return false;
7006
7007 return copyComposite(S, OpPC, Src: Src.view(), Dest: Dest.view(), Activate, Diagnose);
7008}
7009
7010} // namespace interp
7011} // namespace clang
7012