1//===--- InterpBuiltin.cpp - Interpreter for the constexpr VM ---*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8#include "../ExprConstShared.h"
9#include "Boolean.h"
10#include "Char.h"
11#include "EvalEmitter.h"
12#include "Interp.h"
13#include "InterpBuiltinBitCast.h"
14#include "InterpHelpers.h"
15#include "PrimType.h"
16#include "Program.h"
17#include "clang/AST/ASTContext.h"
18#include "clang/AST/ExprCXX.h"
19#include "clang/AST/InferAlloc.h"
20#include "clang/AST/OSLog.h"
21#include "clang/AST/RecordLayout.h"
22#include "clang/Basic/Builtins.h"
23#include "clang/Basic/TargetBuiltins.h"
24#include "clang/Basic/TargetInfo.h"
25#include "llvm/ADT/StringExtras.h"
26#include "llvm/Support/AllocToken.h"
27#include "llvm/Support/CRC.h"
28#include "llvm/Support/ErrorHandling.h"
29#include "llvm/Support/MathExtras.h"
30#include "llvm/Support/SipHash.h"
31
32namespace clang {
33namespace interp {
34
35[[maybe_unused]] static bool isNoopBuiltin(unsigned ID) {
36 switch (ID) {
37 case Builtin::BIas_const:
38 case Builtin::BIforward:
39 case Builtin::BIforward_like:
40 case Builtin::BImove:
41 case Builtin::BImove_if_noexcept:
42 case Builtin::BIaddressof:
43 case Builtin::BI__addressof:
44 case Builtin::BI__builtin_addressof:
45 case Builtin::BI__builtin_launder:
46 return true;
47 default:
48 return false;
49 }
50 return false;
51}
52
53static bool isFixedSizeIntegralType(PrimType T) {
54 switch (T) {
55 case PT_Sint8:
56 case PT_Uint8:
57 case PT_Sint16:
58 case PT_Uint16:
59 case PT_Sint32:
60 case PT_Uint32:
61 case PT_Sint64:
62 case PT_Uint64:
63 case PT_Bool:
64 return true;
65 default:
66 return false;
67 }
68}
69
70static void discard(InterpStack &Stk, PrimType T) {
71 TYPE_SWITCH(T, { Stk.discard<T>(); });
72}
73
74static bool popToUInt64(const InterpState &S, const Expr *E, uint64_t &Out) {
75 INT_TYPE_SWITCH(*S.getContext().classify(E->getType()), {
76 const auto &Val = S.Stk.pop<T>();
77 if (!Val.isNumber())
78 return false;
79 Out = static_cast<uint64_t>(Val);
80 return true;
81 });
82}
83
84static bool popToAPSInt(InterpStack &Stk, PrimType T, APSInt &Out) {
85 INT_TYPE_SWITCH(T, {
86 const auto &Val = Stk.pop<T>();
87 if (!Val.isNumber())
88 return false;
89 Out = Val.toAPSInt();
90 return true;
91 });
92}
93
94static bool popToAPSInt(InterpState &S, const Expr *E, APSInt &Out) {
95 return popToAPSInt(Stk&: S.Stk, T: *S.getContext().classify(T: E->getType()), Out);
96}
97static bool popToAPSInt(InterpState &S, QualType T, APSInt &Out) {
98 return popToAPSInt(Stk&: S.Stk, T: *S.getContext().classify(T), Out);
99}
100
101/// Check for common reasons a pointer can't be read from, which
102/// are usually not diagnosed in a builtin function.
103static bool isReadable(const Pointer &P) {
104 if (P.isDummy())
105 return false;
106 if (!P.isReadablePointerType())
107 return false;
108 if (!P.isLive())
109 return false;
110 if (P.isOnePastEnd())
111 return false;
112 return true;
113}
114
115/// Pushes \p Val on the stack as the type given by \p QT.
116static void pushInteger(InterpState &S, const APSInt &Val, QualType QT) {
117 assert(QT->isSignedIntegerOrEnumerationType() ||
118 QT->isUnsignedIntegerOrEnumerationType());
119 OptPrimType T = *S.getContext().classify(T: QT);
120 assert(T);
121
122 if (T == PT_IntAPS) {
123 unsigned BitWidth = S.getASTContext().getIntWidth(T: QT);
124 auto Result = S.allocAP<IntegralAP<true>>(BitWidth);
125 Result.copy(V: Val.extOrTrunc(width: BitWidth));
126 S.Stk.push<IntegralAP<true>>(Args&: Result);
127 return;
128 }
129
130 if (T == PT_IntAP) {
131 unsigned BitWidth = S.getASTContext().getIntWidth(T: QT);
132 auto Result = S.allocAP<IntegralAP<false>>(BitWidth);
133 Result.copy(V: Val.extOrTrunc(width: BitWidth));
134 S.Stk.push<IntegralAP<false>>(Args&: Result);
135 return;
136 }
137
138 if (isSignedType(T: *T)) {
139 int64_t V = Val.getSExtValue();
140 INT_TYPE_SWITCH(*T, { S.Stk.push<T>(T::from(V)); });
141 } else {
142 assert(QT->isUnsignedIntegerOrEnumerationType());
143 uint64_t V = Val.getZExtValue();
144 INT_TYPE_SWITCH(*T, { S.Stk.push<T>(T::from(V)); });
145 }
146}
147
148template <typename T>
149static void pushInteger(InterpState &S, T Val, QualType QT) {
150 if constexpr (std::is_same_v<T, APInt>)
151 pushInteger(S, Val: APSInt(Val, !std::is_signed_v<T>), QT);
152 else if constexpr (std::is_same_v<T, APSInt>)
153 pushInteger(S, Val, QT);
154 else
155 pushInteger(S,
156 Val: APSInt(APInt(sizeof(T) * 8, static_cast<uint64_t>(Val),
157 std::is_signed_v<T>),
158 !std::is_signed_v<T>),
159 QT);
160}
161
162static void assignIntegral(InterpState &S, const Pointer &Dest, PrimType ValueT,
163 const APSInt &Value) {
164
165 if (ValueT == PT_IntAPS) {
166 Dest.deref<IntegralAP<true>>() =
167 S.allocAP<IntegralAP<true>>(BitWidth: Value.getBitWidth());
168 Dest.deref<IntegralAP<true>>().copy(V: Value);
169 } else if (ValueT == PT_IntAP) {
170 Dest.deref<IntegralAP<false>>() =
171 S.allocAP<IntegralAP<false>>(BitWidth: Value.getBitWidth());
172 Dest.deref<IntegralAP<false>>().copy(V: Value);
173 } else if (ValueT == PT_Bool) {
174 Dest.deref<Boolean>() = Boolean::from(Value: !Value.isZero());
175 } else {
176 INT_TYPE_SWITCH_NO_BOOL(
177 ValueT, { Dest.deref<T>() = T::from(static_cast<T>(Value)); });
178 }
179}
180
181static QualType getElemType(const Pointer &P) {
182 if (P.isStringPointer()) {
183 return P.asStringPointer()
184 .getLiteral()
185 ->getType()
186 ->getAsArrayTypeUnsafe()
187 ->getElementType();
188 }
189
190 if (P.isOpaquePointer() || P.isIntegralPointer())
191 return P.getType();
192
193 const Descriptor *Desc = P.getFieldDesc();
194 QualType T = Desc->getType();
195 if (Desc->isPrimitive())
196 return T;
197 if (T->isPointerType())
198 return T->castAs<PointerType>()->getPointeeType();
199 if (Desc->isArray())
200 return Desc->getElemQualType();
201 if (const auto *AT = T->getAsArrayTypeUnsafe())
202 return AT->getElementType();
203 return T;
204}
205
206static void diagnoseNonConstexprBuiltin(InterpState &S, CodePtr OpPC,
207 unsigned ID) {
208 if (!S.diagnosing())
209 return;
210
211 auto Loc = S.Current->getSource(PC: OpPC);
212 if (S.getLangOpts().CPlusPlus11)
213 S.CCEDiag(SI: Loc, DiagId: diag::note_constexpr_invalid_function)
214 << /*isConstexpr=*/0 << /*isConstructor=*/0
215 << S.getASTContext().BuiltinInfo.getQuotedName(ID);
216 else
217 S.CCEDiag(SI: Loc, DiagId: diag::note_invalid_subexpr_in_const_expr);
218}
219
220static bool diagnoseUnreadableObject(InterpState &S, CodePtr OpPC,
221 const Pointer &Ptr, AccessKinds AK) {
222 S.FFDiag(SI: S.Current->getSource(PC: OpPC),
223 DiagId: diag::note_constexpr_access_unreadable_object)
224 << AK << Ptr.toDiagnosticString(Ctx: S.getASTContext());
225 noteValueLocation(S, Ptr);
226 return false;
227}
228
229static llvm::APSInt convertBoolVectorToInt(const Pointer &Val) {
230 assert(Val.getFieldDesc()->isPrimitiveArray() &&
231 Val.getFieldDesc()->getElemQualType()->isBooleanType() &&
232 "Not a boolean vector");
233 unsigned NumElems = Val.getNumElems();
234
235 // Each element is one bit, so create an integer with NumElts bits.
236 llvm::APSInt Result(NumElems, 0);
237 for (unsigned I = 0; I != NumElems; ++I) {
238 if (Val.elem<bool>(I))
239 Result.setBit(I);
240 }
241
242 return Result;
243}
244
245// Strict double -> float conversion used for X86 PD2PS/cvtsd2ss intrinsics.
246// Reject NaN/Inf/Subnormal inputs and any lossy/inexact conversions.
247static bool convertDoubleToFloatStrict(const APFloat &Src, Floating &Dst,
248 InterpState &S, const Expr *DiagExpr) {
249 if (Src.isInfinity()) {
250 if (S.diagnosing())
251 S.CCEDiag(E: DiagExpr, DiagId: diag::note_constexpr_float_arithmetic) << 0;
252 return false;
253 }
254 if (Src.isNaN()) {
255 if (S.diagnosing())
256 S.CCEDiag(E: DiagExpr, DiagId: diag::note_constexpr_float_arithmetic) << 1;
257 return false;
258 }
259 APFloat Val = Src;
260 bool LosesInfo = false;
261 APFloat::opStatus Status = Val.convert(
262 ToSemantics: APFloat::IEEEsingle(), RM: APFloat::rmNearestTiesToEven, losesInfo: &LosesInfo);
263 if (LosesInfo || Val.isDenormal()) {
264 if (S.diagnosing())
265 S.CCEDiag(E: DiagExpr, DiagId: diag::note_constexpr_float_arithmetic_strict);
266 return false;
267 }
268 if (Status != APFloat::opOK) {
269 if (S.diagnosing())
270 S.CCEDiag(E: DiagExpr, DiagId: diag::note_invalid_subexpr_in_const_expr);
271 return false;
272 }
273 Dst.copy(F: Val);
274 return true;
275}
276
277static bool interp__builtin_is_constant_evaluated(InterpState &S, CodePtr OpPC,
278 const InterpFrame *Frame,
279 const CallExpr *Call) {
280 unsigned Depth = S.FrameDepth;
281 auto isStdCall = [](const FunctionDecl *F) -> bool {
282 return F && F->isInStdNamespace() && F->getIdentifier() &&
283 F->getIdentifier()->isStr(Str: "is_constant_evaluated");
284 };
285 const InterpFrame *Caller = Frame->Caller;
286 // The current frame is the one for __builtin_is_constant_evaluated.
287 // The one above that, potentially the one for std::is_constant_evaluated().
288 if (S.inConstantContext() && !S.checkingPotentialConstantExpression() &&
289 S.getEvalStatus().Diag &&
290 (Depth == 0 || (Depth == 1 && isStdCall(Frame->getCallee())))) {
291 if (Caller && isStdCall(Frame->getCallee())) {
292 const Expr *E = Caller->getExpr(PC: Caller->getRetPC());
293 S.report(Loc: E->getExprLoc(),
294 DiagId: diag::warn_is_constant_evaluated_always_true_constexpr)
295 << "std::is_constant_evaluated" << E->getSourceRange();
296 } else {
297 S.report(Loc: Call->getExprLoc(),
298 DiagId: diag::warn_is_constant_evaluated_always_true_constexpr)
299 << "__builtin_is_constant_evaluated" << Call->getSourceRange();
300 }
301 }
302
303 S.Stk.push<Boolean>(Args: Boolean::from(Value: S.inConstantContext()));
304 return true;
305}
306
307// __builtin_assume
308// __assume (MS extension)
309static bool interp__builtin_assume(InterpState &S, CodePtr OpPC,
310 const InterpFrame *Frame,
311 const CallExpr *Call) {
312 // Nothing to be done here since the argument is NOT evaluated.
313 assert(Call->getNumArgs() == 1);
314 return true;
315}
316
317static bool interp__builtin_strcmp(InterpState &S, CodePtr OpPC,
318 const InterpFrame *Frame,
319 const CallExpr *Call, unsigned ID) {
320 uint64_t Limit = ~static_cast<uint64_t>(0);
321 if (ID == Builtin::BIstrncmp || ID == Builtin::BI__builtin_strncmp ||
322 ID == Builtin::BIwcsncmp || ID == Builtin::BI__builtin_wcsncmp) {
323 if (!popToUInt64(S, E: Call->getArg(Arg: 2), Out&: Limit))
324 return false;
325 }
326
327 const Pointer &B = S.Stk.pop<Pointer>();
328 const Pointer &A = S.Stk.pop<Pointer>();
329 if (ID == Builtin::BIstrcmp || ID == Builtin::BIstrncmp ||
330 ID == Builtin::BIwcscmp || ID == Builtin::BIwcsncmp)
331 diagnoseNonConstexprBuiltin(S, OpPC, ID);
332
333 if (Limit == 0) {
334 pushInteger(S, Val: 0, QT: Call->getType());
335 return true;
336 }
337
338 if (!CheckLive(S, OpPC, Ptr: A, AK: AK_Read) || !CheckLive(S, OpPC, Ptr: B, AK: AK_Read))
339 return false;
340
341 if (!A.isReadablePointerType() || !B.isReadablePointerType())
342 return false;
343
344 if (A.isDummy() || B.isDummy() || A.isUnknownSizeArray() ||
345 B.isUnknownSizeArray())
346 return false;
347
348 bool IsWide = ID == Builtin::BIwcscmp || ID == Builtin::BIwcsncmp ||
349 ID == Builtin::BI__builtin_wcscmp ||
350 ID == Builtin::BI__builtin_wcsncmp;
351
352 QualType ElemTy = getElemType(P: A);
353 // Different element types shouldn't happen, but with casts they can.
354 if (!S.getASTContext().hasSameUnqualifiedType(T1: ElemTy, T2: getElemType(P: B)) ||
355 !S.getContext().canClassify(T: ElemTy))
356 return false;
357
358 PrimType ElemT = *S.getContext().classify(T: ElemTy);
359
360 auto returnResult = [&](int V) -> bool {
361 pushInteger(S, Val: V, QT: Call->getType());
362 return true;
363 };
364
365 unsigned IndexA = A.getIndex();
366 unsigned IndexB = B.getIndex();
367 unsigned NumElemsA = A.getNumElems();
368 unsigned NumElemsB = B.getNumElems();
369 uint64_t Steps = 0;
370 for (;; ++IndexA, ++IndexB, ++Steps) {
371
372 if (Steps >= Limit)
373 break;
374
375 // Diagnose this as a read of one-past-the-end.
376 if (IndexA >= NumElemsA || IndexB >= NumElemsB) {
377 S.FFDiag(SI: S.Current->getSource(PC: OpPC), DiagId: diag::note_constexpr_access_past_end)
378 << AK_Read << S.Current->getRange(PC: OpPC);
379 return false;
380 }
381
382 if (IsWide) {
383 INT_TYPE_SWITCH(ElemT, {
384 T CA = A.loadElem<T>(IndexA);
385 T CB = B.loadElem<T>(IndexB);
386 if (CA > CB)
387 return returnResult(1);
388 if (CA < CB)
389 return returnResult(-1);
390 if (CA.isZero() || CB.isZero())
391 return returnResult(0);
392 });
393 continue;
394 }
395
396 uint8_t CA = A.loadElem<uint8_t>(I: IndexA);
397 uint8_t CB = B.loadElem<uint8_t>(I: IndexB);
398
399 if (CA > CB)
400 return returnResult(1);
401 if (CA < CB)
402 return returnResult(-1);
403 if (CA == 0 || CB == 0)
404 return returnResult(0);
405 }
406
407 return returnResult(0);
408}
409
410static bool interp__builtin_strlen(InterpState &S, CodePtr OpPC,
411 const InterpFrame *Frame,
412 const CallExpr *Call, unsigned ID) {
413 const Pointer &StrPtr = S.Stk.pop<Pointer>().expand();
414
415 if (ID == Builtin::BIstrlen || ID == Builtin::BIwcslen)
416 diagnoseNonConstexprBuiltin(S, OpPC, ID);
417
418 if (StrPtr.isConstexprUnknown())
419 return false;
420
421 if (!CheckArray(S, OpPC, Ptr: StrPtr))
422 return false;
423
424 if (!CheckLive(S, OpPC, Ptr: StrPtr, AK: AK_Read))
425 return false;
426
427 // For string literal pointers, this is pretty simple.
428 if (StrPtr.isStringPointer()) {
429 if (StrPtr.isOnePastEnd())
430 return CheckRange(S, OpPC, Ptr: StrPtr, AK: AK_Read);
431
432 const auto *Lit = StrPtr.asStringPointer().getLiteral();
433 int64_t Off = StrPtr.getByteOffset();
434 if (Off < 0)
435 return false;
436
437 UnsignedOrNone ZeroIndex = Lit->findZeroCodeUnit(StartIndex: Off);
438 if (!ZeroIndex)
439 return false;
440 pushInteger(S, Val: *ZeroIndex, QT: Call->getType());
441 return true;
442 }
443
444 if (!StrPtr.isBlockPointer())
445 return false;
446
447 if (!CheckDummy(S, OpPC, Ptr: StrPtr, AK: AK_Read))
448 return false;
449
450 if (!StrPtr.getFieldDesc()->isPrimitiveArray())
451 return false;
452
453 assert(StrPtr.getFieldDesc()->isPrimitiveArray());
454 PrimType ElemT = StrPtr.getFieldDesc()->getPrimType();
455 if (!isFixedSizeIntegralType(T: ElemT))
456 return false;
457
458 unsigned ElemSize = StrPtr.getFieldDesc()->getElemDataSize();
459 if (ElemSize != 1 && ElemSize != 2 && ElemSize != 4)
460 return Invalid(S, OpPC);
461
462 if (ID == Builtin::BI__builtin_wcslen || ID == Builtin::BIwcslen) {
463 const ASTContext &AC = S.getASTContext();
464 unsigned WCharSize = AC.getTypeSizeInChars(T: AC.getWCharType()).getQuantity();
465 if (ElemSize != WCharSize)
466 return false;
467 }
468
469 size_t Len = 0;
470 for (size_t I = StrPtr.getIndex();; ++I, ++Len) {
471 PtrView ElemPtr = StrPtr.view().atIndex(Idx: I);
472
473 if (!CheckRange(S, OpPC, Ptr: ElemPtr, AK: AK_Read))
474 return false;
475
476 uint32_t Val;
477 FIXED_SIZE_INT_TYPE_SWITCH(
478 ElemT, { Val = static_cast<uint32_t>(ElemPtr.deref<T>()); });
479 if (Val == 0)
480 break;
481 }
482
483 pushInteger(S, Val: Len, QT: Call->getType());
484
485 return true;
486}
487
488static bool interp__builtin_nan(InterpState &S, CodePtr OpPC,
489 const InterpFrame *Frame, const CallExpr *Call,
490 bool Signaling) {
491 const Pointer &Arg = S.Stk.pop<Pointer>();
492
493 if (!CheckLoad(S, OpPC, Ptr: Arg))
494 return false;
495
496 // Convert the given string to an integer using StringRef's API.
497 llvm::APInt Fill;
498 if (Arg.isBlockPointer()) {
499 if (!Arg.getFieldDesc()->isPrimitiveArray())
500 return Invalid(S, OpPC);
501
502 std::string Str;
503 unsigned ArgLength = Arg.getNumElems();
504 bool FoundZero = false;
505 for (unsigned I = 0; I != ArgLength; ++I) {
506 if (!Arg.isElementInitialized(Index: I))
507 return false;
508
509 if (Arg.loadElem<int8_t>(I) == 0) {
510 FoundZero = true;
511 break;
512 }
513 Str += Arg.elem<char>(I);
514 }
515
516 // If we didn't find a NUL byte, diagnose as a one-past-the-end read.
517 if (!FoundZero)
518 return CheckRange(S, OpPC, Ptr: Arg.atIndex(Idx: ArgLength), AK: AK_Read);
519
520 // Treat empty strings as if they were zero.
521 if (Str.empty())
522 Fill = llvm::APInt(32, 0);
523 else if (StringRef(Str).getAsInteger(Radix: 0, Result&: Fill))
524 return false;
525 } else if (Arg.isStringPointer()) {
526 if (!Arg.asStringPointer().getLiteral()->isOrdinary())
527 return false;
528 StringRef Str = Arg.asStringPointer().getLiteral()->getString();
529 // Treat empty strings as if they were zero.
530 if (Str.empty())
531 Fill = llvm::APInt(32, 0);
532 else if (StringRef(Str).getAsInteger(Radix: 0, Result&: Fill))
533 return false;
534 } else {
535 return false;
536 }
537
538 const llvm::fltSemantics &TargetSemantics =
539 S.getASTContext().getFloatTypeSemantics(
540 T: Call->getDirectCallee()->getReturnType());
541
542 Floating Result = S.allocFloat(Sem: TargetSemantics);
543 if (S.getASTContext().getTargetInfo().isNan2008()) {
544 if (Signaling)
545 Result.copy(
546 F: llvm::APFloat::getSNaN(Sem: TargetSemantics, /*Negative=*/false, payload: &Fill));
547 else
548 Result.copy(
549 F: llvm::APFloat::getQNaN(Sem: TargetSemantics, /*Negative=*/false, payload: &Fill));
550 } else {
551 // Prior to IEEE 754-2008, architectures were allowed to choose whether
552 // the first bit of their significand was set for qNaN or sNaN. MIPS chose
553 // a different encoding to what became a standard in 2008, and for pre-
554 // 2008 revisions, MIPS interpreted sNaN-2008 as qNan and qNaN-2008 as
555 // sNaN. This is now known as "legacy NaN" encoding.
556 if (Signaling)
557 Result.copy(
558 F: llvm::APFloat::getQNaN(Sem: TargetSemantics, /*Negative=*/false, payload: &Fill));
559 else
560 Result.copy(
561 F: llvm::APFloat::getSNaN(Sem: TargetSemantics, /*Negative=*/false, payload: &Fill));
562 }
563
564 S.Stk.push<Floating>(Args&: Result);
565 return true;
566}
567
568static bool interp__builtin_inf(InterpState &S, CodePtr OpPC,
569 const InterpFrame *Frame,
570 const CallExpr *Call) {
571 const llvm::fltSemantics &TargetSemantics =
572 S.getASTContext().getFloatTypeSemantics(
573 T: Call->getDirectCallee()->getReturnType());
574
575 Floating Result = S.allocFloat(Sem: TargetSemantics);
576 Result.copy(F: APFloat::getInf(Sem: TargetSemantics));
577 S.Stk.push<Floating>(Args&: Result);
578 return true;
579}
580
581static bool interp__builtin_copysign(InterpState &S, CodePtr OpPC,
582 const InterpFrame *Frame) {
583 const Floating &Arg2 = S.Stk.pop<Floating>();
584 const Floating &Arg1 = S.Stk.pop<Floating>();
585 Floating Result = S.allocFloat(Sem: Arg1.getSemantics());
586
587 APFloat Copy = Arg1.getAPFloat();
588 Copy.copySign(RHS: Arg2.getAPFloat());
589 Result.copy(F: Copy);
590 S.Stk.push<Floating>(Args&: Result);
591
592 return true;
593}
594
595static bool interp__builtin_fmin(InterpState &S, CodePtr OpPC,
596 const InterpFrame *Frame, bool IsNumBuiltin) {
597 const Floating &RHS = S.Stk.pop<Floating>();
598 const Floating &LHS = S.Stk.pop<Floating>();
599 Floating Result = S.allocFloat(Sem: LHS.getSemantics());
600
601 if (IsNumBuiltin)
602 Result.copy(F: llvm::minimumnum(A: LHS.getAPFloat(), B: RHS.getAPFloat()));
603 else
604 Result.copy(F: minnum(A: LHS.getAPFloat(), B: RHS.getAPFloat()));
605 S.Stk.push<Floating>(Args&: Result);
606 return true;
607}
608
609static bool interp__builtin_fmax(InterpState &S, CodePtr OpPC,
610 const InterpFrame *Frame, bool IsNumBuiltin) {
611 const Floating &RHS = S.Stk.pop<Floating>();
612 const Floating &LHS = S.Stk.pop<Floating>();
613 Floating Result = S.allocFloat(Sem: LHS.getSemantics());
614
615 if (IsNumBuiltin)
616 Result.copy(F: llvm::maximumnum(A: LHS.getAPFloat(), B: RHS.getAPFloat()));
617 else
618 Result.copy(F: maxnum(A: LHS.getAPFloat(), B: RHS.getAPFloat()));
619 S.Stk.push<Floating>(Args&: Result);
620 return true;
621}
622
623/// Defined as __builtin_isnan(...), to accommodate the fact that it can
624/// take a float, double, long double, etc.
625/// But for us, that's all a Floating anyway.
626static bool interp__builtin_isnan(InterpState &S, CodePtr OpPC,
627 const InterpFrame *Frame,
628 const CallExpr *Call) {
629 const Floating &Arg = S.Stk.pop<Floating>();
630
631 pushInteger(S, Val: Arg.isNan(), QT: Call->getType());
632 return true;
633}
634
635static bool interp__builtin_issignaling(InterpState &S, CodePtr OpPC,
636 const InterpFrame *Frame,
637 const CallExpr *Call) {
638 const Floating &Arg = S.Stk.pop<Floating>();
639
640 pushInteger(S, Val: Arg.isSignaling(), QT: Call->getType());
641 return true;
642}
643
644static bool interp__builtin_isinf(InterpState &S, CodePtr OpPC,
645 const InterpFrame *Frame, bool CheckSign,
646 const CallExpr *Call) {
647 const Floating &Arg = S.Stk.pop<Floating>();
648 APFloat F = Arg.getAPFloat();
649 bool IsInf = F.isInfinity();
650
651 if (CheckSign)
652 pushInteger(S, Val: IsInf ? (F.isNegative() ? -1 : 1) : 0, QT: Call->getType());
653 else
654 pushInteger(S, Val: IsInf, QT: Call->getType());
655 return true;
656}
657
658static bool interp__builtin_isfinite(InterpState &S, CodePtr OpPC,
659 const InterpFrame *Frame,
660 const CallExpr *Call) {
661 const Floating &Arg = S.Stk.pop<Floating>();
662
663 pushInteger(S, Val: Arg.isFinite(), QT: Call->getType());
664 return true;
665}
666
667static bool interp__builtin_isnormal(InterpState &S, CodePtr OpPC,
668 const InterpFrame *Frame,
669 const CallExpr *Call) {
670 const Floating &Arg = S.Stk.pop<Floating>();
671
672 pushInteger(S, Val: Arg.isNormal(), QT: Call->getType());
673 return true;
674}
675
676static bool interp__builtin_issubnormal(InterpState &S, CodePtr OpPC,
677 const InterpFrame *Frame,
678 const CallExpr *Call) {
679 const Floating &Arg = S.Stk.pop<Floating>();
680
681 pushInteger(S, Val: Arg.isDenormal(), QT: Call->getType());
682 return true;
683}
684
685static bool interp__builtin_iszero(InterpState &S, CodePtr OpPC,
686 const InterpFrame *Frame,
687 const CallExpr *Call) {
688 const Floating &Arg = S.Stk.pop<Floating>();
689
690 pushInteger(S, Val: Arg.isZero(), QT: Call->getType());
691 return true;
692}
693
694static bool interp__builtin_signbit(InterpState &S, CodePtr OpPC,
695 const InterpFrame *Frame,
696 const CallExpr *Call) {
697 const Floating &Arg = S.Stk.pop<Floating>();
698
699 pushInteger(S, Val: Arg.isNegative(), QT: Call->getType());
700 return true;
701}
702
703static bool interp_floating_comparison(InterpState &S, CodePtr OpPC,
704 const CallExpr *Call, unsigned ID) {
705 const Floating &RHS = S.Stk.pop<Floating>();
706 const Floating &LHS = S.Stk.pop<Floating>();
707
708 pushInteger(
709 S,
710 Val: [&] {
711 switch (ID) {
712 case Builtin::BI__builtin_isgreater:
713 return LHS > RHS;
714 case Builtin::BI__builtin_isgreaterequal:
715 return LHS >= RHS;
716 case Builtin::BI__builtin_isless:
717 return LHS < RHS;
718 case Builtin::BI__builtin_islessequal:
719 return LHS <= RHS;
720 case Builtin::BI__builtin_islessgreater: {
721 ComparisonCategoryResult Cmp = LHS.compare(RHS);
722 return Cmp == ComparisonCategoryResult::Less ||
723 Cmp == ComparisonCategoryResult::Greater;
724 }
725 case Builtin::BI__builtin_isunordered:
726 return LHS.compare(RHS) == ComparisonCategoryResult::Unordered;
727 default:
728 llvm_unreachable("Unexpected builtin ID: Should be a floating point "
729 "comparison function");
730 }
731 }(),
732 QT: Call->getType());
733 return true;
734}
735
736/// First parameter to __builtin_isfpclass is the floating value, the
737/// second one is an integral value.
738static bool interp__builtin_isfpclass(InterpState &S, CodePtr OpPC,
739 const InterpFrame *Frame,
740 const CallExpr *Call) {
741 APSInt FPClassArg;
742 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: FPClassArg))
743 return false;
744 const Floating &F = S.Stk.pop<Floating>();
745
746 int32_t Result = static_cast<int32_t>(
747 (F.classify() & std::move(FPClassArg)).getZExtValue());
748 pushInteger(S, Val: Result, QT: Call->getType());
749
750 return true;
751}
752
753/// Five int values followed by one floating value.
754/// __builtin_fpclassify(int, int, int, int, int, float)
755static bool interp__builtin_fpclassify(InterpState &S, CodePtr OpPC,
756 const InterpFrame *Frame,
757 const CallExpr *Call) {
758 const Floating &Val = S.Stk.pop<Floating>();
759
760 PrimType IntT = *S.getContext().classify(E: Call->getArg(Arg: 0));
761 APSInt Values[5];
762 for (unsigned I = 0; I != 5; ++I) {
763 if (!popToAPSInt(Stk&: S.Stk, T: IntT, Out&: Values[4 - I]))
764 return false;
765 }
766
767 unsigned Index;
768 switch (Val.getCategory()) {
769 case APFloat::fcNaN:
770 Index = 0;
771 break;
772 case APFloat::fcInfinity:
773 Index = 1;
774 break;
775 case APFloat::fcNormal:
776 Index = Val.isDenormal() ? 3 : 2;
777 break;
778 case APFloat::fcZero:
779 Index = 4;
780 break;
781 }
782
783 // The last argument is first on the stack.
784 assert(Index <= 4);
785
786 pushInteger(S, Val: Values[Index], QT: Call->getType());
787 return true;
788}
789
790static inline Floating abs(InterpState &S, const Floating &In) {
791 if (!In.isNegative())
792 return In;
793
794 Floating Output = S.allocFloat(Sem: In.getSemantics());
795 APFloat New = In.getAPFloat();
796 New.changeSign();
797 Output.copy(F: New);
798 return Output;
799}
800
801// The C standard says "fabs raises no floating-point exceptions,
802// even if x is a signaling NaN. The returned value is independent of
803// the current rounding direction mode." Therefore constant folding can
804// proceed without regard to the floating point settings.
805// Reference, WG14 N2478 F.10.4.3
806static bool interp__builtin_fabs(InterpState &S, CodePtr OpPC,
807 const InterpFrame *Frame) {
808 const Floating &Val = S.Stk.pop<Floating>();
809 S.Stk.push<Floating>(Args: abs(S, In: Val));
810 return true;
811}
812
813static bool interp__builtin_abs(InterpState &S, CodePtr OpPC,
814 const InterpFrame *Frame,
815 const CallExpr *Call) {
816 APSInt Val;
817 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: Val))
818 return false;
819 if (Val ==
820 APSInt(APInt::getSignedMinValue(numBits: Val.getBitWidth()), /*IsUnsigned=*/false))
821 return false;
822 if (Val.isNegative())
823 Val.negate();
824 pushInteger(S, Val, QT: Call->getType());
825 return true;
826}
827
828static bool interp__builtin_popcount(InterpState &S, CodePtr OpPC,
829 const InterpFrame *Frame,
830 const CallExpr *Call) {
831 APSInt Val;
832 if (Call->getArg(Arg: 0)->getType()->isExtVectorBoolType()) {
833 const Pointer &Arg = S.Stk.pop<Pointer>();
834 Val = convertBoolVectorToInt(Val: Arg);
835 } else {
836 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: Val))
837 return false;
838 }
839 pushInteger(S, Val: Val.popcount(), QT: Call->getType());
840 return true;
841}
842
843static bool interp__builtin_ia32_crc32(InterpState &S, CodePtr OpPC,
844 const InterpFrame *Frame,
845 const CallExpr *Call,
846 unsigned DataBytes) {
847 uint64_t DataVal;
848 if (!popToUInt64(S, E: Call->getArg(Arg: 1), Out&: DataVal))
849 return false;
850 uint64_t CRCVal;
851 if (!popToUInt64(S, E: Call->getArg(Arg: 0), Out&: CRCVal))
852 return false;
853
854 // CRC32C polynomial (iSCSI polynomial, bit-reversed)
855 static const uint32_t CRC32C_POLY = 0x82F63B78;
856
857 uint32_t Result = llvm::calculateReflectedCRC32(
858 Crc: static_cast<uint32_t>(CRCVal), Data: DataVal, DataBytes, Poly: CRC32C_POLY);
859
860 pushInteger(S, Val: Result, QT: Call->getType());
861 return true;
862}
863
864static bool interp__builtin_classify_type(InterpState &S, CodePtr OpPC,
865 const InterpFrame *Frame,
866 const CallExpr *Call) {
867 // This is an unevaluated call, so there are no arguments on the stack.
868 assert(Call->getNumArgs() == 1);
869 const Expr *Arg = Call->getArg(Arg: 0);
870
871 GCCTypeClass ResultClass =
872 EvaluateBuiltinClassifyType(T: Arg->getType(), LangOpts: S.getLangOpts());
873 int32_t ReturnVal = static_cast<int32_t>(ResultClass);
874 pushInteger(S, Val: ReturnVal, QT: Call->getType());
875 return true;
876}
877
878// __builtin_expect(long, long)
879// __builtin_expect_with_probability(long, long, double)
880static bool interp__builtin_expect(InterpState &S, CodePtr OpPC,
881 const InterpFrame *Frame,
882 const CallExpr *Call) {
883 // The return value is simply the value of the first parameter.
884 // We ignore the probability.
885 unsigned NumArgs = Call->getNumArgs();
886 assert(NumArgs == 2 || NumArgs == 3);
887
888 PrimType ArgT = *S.getContext().classify(T: Call->getArg(Arg: 0)->getType());
889 if (NumArgs == 3)
890 S.Stk.discard<Floating>();
891 discard(Stk&: S.Stk, T: ArgT);
892 // Top of the stack is now the first paramter. Leave it there as the return
893 // value.
894
895 return true;
896}
897
898static bool interp__builtin_addressof(InterpState &S, CodePtr OpPC,
899 const InterpFrame *Frame,
900 const CallExpr *Call) {
901#ifndef NDEBUG
902 assert(Call->getArg(0)->isLValue());
903 PrimType PtrT = S.getContext().classify(Call->getArg(0)).value_or(PT_Ptr);
904 assert(PtrT == PT_Ptr &&
905 "Unsupported pointer type passed to __builtin_addressof()");
906#endif
907 return true;
908}
909
910static bool interp__builtin_move(InterpState &S, CodePtr OpPC,
911 const InterpFrame *Frame,
912 const CallExpr *Call) {
913 return Call->getDirectCallee()->isConstexpr();
914}
915
916static bool interp__builtin_eh_return_data_regno(InterpState &S, CodePtr OpPC,
917 const InterpFrame *Frame,
918 const CallExpr *Call) {
919 APSInt Arg;
920 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: Arg))
921 return false;
922
923 int Result = S.getASTContext().getTargetInfo().getEHDataRegisterNumber(
924 RegNo: Arg.getZExtValue());
925 pushInteger(S, Val: Result, QT: Call->getType());
926 return true;
927}
928
929// Two integral values followed by a pointer (lhs, rhs, resultOut)
930static bool interp__builtin_overflowop(InterpState &S, CodePtr OpPC,
931 const CallExpr *Call,
932 unsigned BuiltinOp) {
933 const Pointer &ResultPtr = S.Stk.pop<Pointer>();
934 if (ResultPtr.isDummy() || !ResultPtr.isBlockPointer())
935 return false;
936
937 PrimType RHST = *S.getContext().classify(T: Call->getArg(Arg: 1)->getType());
938 PrimType LHST = *S.getContext().classify(T: Call->getArg(Arg: 0)->getType());
939 APSInt RHS;
940 if (!popToAPSInt(Stk&: S.Stk, T: RHST, Out&: RHS))
941 return false;
942 APSInt LHS;
943 if (!popToAPSInt(Stk&: S.Stk, T: LHST, Out&: LHS))
944 return false;
945 QualType ResultType = Call->getArg(Arg: 2)->getType()->getPointeeType();
946 PrimType ResultT = *S.getContext().classify(T: ResultType);
947 bool Overflow;
948
949 APSInt Result;
950 if (BuiltinOp == Builtin::BI__builtin_add_overflow ||
951 BuiltinOp == Builtin::BI__builtin_sub_overflow ||
952 BuiltinOp == Builtin::BI__builtin_mul_overflow) {
953 bool IsSigned = LHS.isSigned() || RHS.isSigned() ||
954 ResultType->isSignedIntegerOrEnumerationType();
955 bool AllSigned = LHS.isSigned() && RHS.isSigned() &&
956 ResultType->isSignedIntegerOrEnumerationType();
957 uint64_t LHSSize = LHS.getBitWidth();
958 uint64_t RHSSize = RHS.getBitWidth();
959 uint64_t ResultSize = S.getASTContext().getIntWidth(T: ResultType);
960 uint64_t MaxBits = std::max(a: std::max(a: LHSSize, b: RHSSize), b: ResultSize);
961
962 // Add an additional bit if the signedness isn't uniformly agreed to. We
963 // could do this ONLY if there is a signed and an unsigned that both have
964 // MaxBits, but the code to check that is pretty nasty. The issue will be
965 // caught in the shrink-to-result later anyway.
966 if (IsSigned && !AllSigned)
967 ++MaxBits;
968
969 LHS = APSInt(LHS.extOrTrunc(width: MaxBits), !IsSigned);
970 RHS = APSInt(RHS.extOrTrunc(width: MaxBits), !IsSigned);
971 Result = APSInt(MaxBits, !IsSigned);
972 }
973
974 // Find largest int.
975 switch (BuiltinOp) {
976 default:
977 llvm_unreachable("Invalid value for BuiltinOp");
978 case Builtin::BI__builtin_add_overflow:
979 case Builtin::BI__builtin_sadd_overflow:
980 case Builtin::BI__builtin_saddl_overflow:
981 case Builtin::BI__builtin_saddll_overflow:
982 case Builtin::BI__builtin_uadd_overflow:
983 case Builtin::BI__builtin_uaddl_overflow:
984 case Builtin::BI__builtin_uaddll_overflow:
985 Result = LHS.isSigned() ? LHS.sadd_ov(RHS, Overflow)
986 : LHS.uadd_ov(RHS, Overflow);
987 break;
988 case Builtin::BI__builtin_sub_overflow:
989 case Builtin::BI__builtin_ssub_overflow:
990 case Builtin::BI__builtin_ssubl_overflow:
991 case Builtin::BI__builtin_ssubll_overflow:
992 case Builtin::BI__builtin_usub_overflow:
993 case Builtin::BI__builtin_usubl_overflow:
994 case Builtin::BI__builtin_usubll_overflow:
995 Result = LHS.isSigned() ? LHS.ssub_ov(RHS, Overflow)
996 : LHS.usub_ov(RHS, Overflow);
997 break;
998 case Builtin::BI__builtin_mul_overflow:
999 case Builtin::BI__builtin_smul_overflow:
1000 case Builtin::BI__builtin_smull_overflow:
1001 case Builtin::BI__builtin_smulll_overflow:
1002 case Builtin::BI__builtin_umul_overflow:
1003 case Builtin::BI__builtin_umull_overflow:
1004 case Builtin::BI__builtin_umulll_overflow:
1005 Result = LHS.isSigned() ? LHS.smul_ov(RHS, Overflow)
1006 : LHS.umul_ov(RHS, Overflow);
1007 break;
1008 }
1009
1010 // In the case where multiple sizes are allowed, truncate and see if
1011 // the values are the same.
1012 if (BuiltinOp == Builtin::BI__builtin_add_overflow ||
1013 BuiltinOp == Builtin::BI__builtin_sub_overflow ||
1014 BuiltinOp == Builtin::BI__builtin_mul_overflow) {
1015 // APSInt doesn't have a TruncOrSelf, so we use extOrTrunc instead,
1016 // since it will give us the behavior of a TruncOrSelf in the case where
1017 // its parameter <= its size. We previously set Result to be at least the
1018 // integer width of the result, so getIntWidth(ResultType) <=
1019 // Result.BitWidth
1020 APSInt Temp = Result.extOrTrunc(width: S.getASTContext().getIntWidth(T: ResultType));
1021 Temp.setIsSigned(ResultType->isSignedIntegerOrEnumerationType());
1022
1023 if (!APSInt::isSameValue(I1: Temp, I2: Result))
1024 Overflow = true;
1025 Result = std::move(Temp);
1026 }
1027
1028 // Write Result to ResultPtr and put Overflow on the stack.
1029 assignIntegral(S, Dest: ResultPtr, ValueT: ResultT, Value: Result);
1030 ResultPtr.initialize();
1031
1032 assert(Call->getDirectCallee()->getReturnType()->isBooleanType());
1033 S.Stk.push<Boolean>(Args&: Overflow);
1034 return true;
1035}
1036
1037/// Three integral values followed by a pointer (lhs, rhs, carry, carryOut).
1038static bool interp__builtin_carryop(InterpState &S, CodePtr OpPC,
1039 const InterpFrame *Frame,
1040 const CallExpr *Call, unsigned BuiltinOp) {
1041 const Pointer &CarryOutPtr = S.Stk.pop<Pointer>();
1042 PrimType LHST = *S.getContext().classify(T: Call->getArg(Arg: 0)->getType());
1043 PrimType RHST = *S.getContext().classify(T: Call->getArg(Arg: 1)->getType());
1044 APSInt CarryIn;
1045 if (!popToAPSInt(Stk&: S.Stk, T: LHST, Out&: CarryIn))
1046 return false;
1047 APSInt RHS;
1048 if (!popToAPSInt(Stk&: S.Stk, T: RHST, Out&: RHS))
1049 return false;
1050 APSInt LHS;
1051 if (!popToAPSInt(Stk&: S.Stk, T: LHST, Out&: LHS))
1052 return false;
1053
1054 if (!isReadable(P: CarryOutPtr))
1055 return false;
1056
1057 APSInt CarryOut;
1058
1059 APSInt Result;
1060 // Copy the number of bits and sign.
1061 Result = LHS;
1062 CarryOut = LHS;
1063
1064 bool FirstOverflowed = false;
1065 bool SecondOverflowed = false;
1066 switch (BuiltinOp) {
1067 default:
1068 llvm_unreachable("Invalid value for BuiltinOp");
1069 case Builtin::BI__builtin_addcb:
1070 case Builtin::BI__builtin_addcs:
1071 case Builtin::BI__builtin_addc:
1072 case Builtin::BI__builtin_addcl:
1073 case Builtin::BI__builtin_addcll:
1074 Result =
1075 LHS.uadd_ov(RHS, Overflow&: FirstOverflowed).uadd_ov(RHS: CarryIn, Overflow&: SecondOverflowed);
1076 break;
1077 case Builtin::BI__builtin_subcb:
1078 case Builtin::BI__builtin_subcs:
1079 case Builtin::BI__builtin_subc:
1080 case Builtin::BI__builtin_subcl:
1081 case Builtin::BI__builtin_subcll:
1082 Result =
1083 LHS.usub_ov(RHS, Overflow&: FirstOverflowed).usub_ov(RHS: CarryIn, Overflow&: SecondOverflowed);
1084 break;
1085 }
1086 // It is possible for both overflows to happen but CGBuiltin uses an OR so
1087 // this is consistent.
1088 CarryOut = (uint64_t)(FirstOverflowed | SecondOverflowed);
1089
1090 QualType CarryOutType = Call->getArg(Arg: 3)->getType()->getPointeeType();
1091 PrimType CarryOutT = *S.getContext().classify(T: CarryOutType);
1092 assignIntegral(S, Dest: CarryOutPtr, ValueT: CarryOutT, Value: CarryOut);
1093 CarryOutPtr.initialize();
1094
1095 assert(S.getASTContext().hasSimilarType(Call->getType(),
1096 Call->getArg(0)->getType()));
1097 pushInteger(S, Val: Result, QT: Call->getType());
1098 return true;
1099}
1100
1101static bool interp__builtin_clz(InterpState &S, CodePtr OpPC,
1102 const InterpFrame *Frame, const CallExpr *Call,
1103 unsigned BuiltinOp) {
1104
1105 std::optional<APSInt> Fallback;
1106 if (BuiltinOp == Builtin::BI__builtin_clzg && Call->getNumArgs() == 2) {
1107 APSInt FallbackVal;
1108 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: FallbackVal))
1109 return false;
1110 Fallback = FallbackVal;
1111 }
1112
1113 APSInt Val;
1114 if (Call->getArg(Arg: 0)->getType()->isExtVectorBoolType()) {
1115 const Pointer &Arg = S.Stk.pop<Pointer>();
1116 Val = convertBoolVectorToInt(Val: Arg);
1117 } else {
1118 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: Val))
1119 return false;
1120 }
1121
1122 // When the argument is 0, the result of GCC builtins is undefined, whereas
1123 // for Microsoft intrinsics, the result is the bit-width of the argument.
1124 bool ZeroIsUndefined = BuiltinOp != Builtin::BI__lzcnt16 &&
1125 BuiltinOp != Builtin::BI__lzcnt &&
1126 BuiltinOp != Builtin::BI__lzcnt64;
1127
1128 if (Val == 0) {
1129 if (Fallback) {
1130 pushInteger(S, Val: *Fallback, QT: Call->getType());
1131 return true;
1132 }
1133
1134 if (ZeroIsUndefined)
1135 return false;
1136 }
1137
1138 pushInteger(S, Val: Val.countl_zero(), QT: Call->getType());
1139 return true;
1140}
1141
1142static bool interp__builtin_ctz(InterpState &S, CodePtr OpPC,
1143 const InterpFrame *Frame, const CallExpr *Call,
1144 unsigned BuiltinID) {
1145 std::optional<APSInt> Fallback;
1146 if (BuiltinID == Builtin::BI__builtin_ctzg && Call->getNumArgs() == 2) {
1147 APSInt FallbackVal;
1148 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: FallbackVal))
1149 return false;
1150 Fallback = FallbackVal;
1151 }
1152
1153 APSInt Val;
1154 if (Call->getArg(Arg: 0)->getType()->isExtVectorBoolType()) {
1155 const Pointer &Arg = S.Stk.pop<Pointer>();
1156 Val = convertBoolVectorToInt(Val: Arg);
1157 } else {
1158 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: Val))
1159 return false;
1160 }
1161
1162 if (Val == 0) {
1163 if (Fallback) {
1164 pushInteger(S, Val: *Fallback, QT: Call->getType());
1165 return true;
1166 }
1167 return false;
1168 }
1169
1170 pushInteger(S, Val: Val.countr_zero(), QT: Call->getType());
1171 return true;
1172}
1173
1174static bool interp__builtin_bswap(InterpState &S, CodePtr OpPC,
1175 const InterpFrame *Frame,
1176 const CallExpr *Call) {
1177 APSInt Val;
1178 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: Val))
1179 return false;
1180 if (Val.getBitWidth() == 8 || Val.getBitWidth() == 1)
1181 pushInteger(S, Val, QT: Call->getType());
1182 else
1183 pushInteger(S, Val: Val.byteSwap(), QT: Call->getType());
1184 return true;
1185}
1186
1187/// bool __atomic_always_lock_free(size_t, void const volatile*)
1188/// bool __atomic_is_lock_free(size_t, void const volatile*)
1189static bool interp__builtin_atomic_lock_free(InterpState &S, CodePtr OpPC,
1190 const InterpFrame *Frame,
1191 const CallExpr *Call,
1192 unsigned BuiltinOp) {
1193 auto returnBool = [&S](bool Value) -> bool {
1194 S.Stk.push<Boolean>(Args&: Value);
1195 return true;
1196 };
1197
1198 const Pointer &Ptr = S.Stk.pop<Pointer>();
1199 uint64_t SizeVal;
1200 if (!popToUInt64(S, E: Call->getArg(Arg: 0), Out&: SizeVal))
1201 return false;
1202
1203 // For __atomic_is_lock_free(sizeof(_Atomic(T))), if the size is a power
1204 // of two less than or equal to the maximum inline atomic width, we know it
1205 // is lock-free. If the size isn't a power of two, or greater than the
1206 // maximum alignment where we promote atomics, we know it is not lock-free
1207 // (at least not in the sense of atomic_is_lock_free). Otherwise,
1208 // the answer can only be determined at runtime; for example, 16-byte
1209 // atomics have lock-free implementations on some, but not all,
1210 // x86-64 processors.
1211
1212 // Check power-of-two.
1213 CharUnits Size = CharUnits::fromQuantity(Quantity: SizeVal);
1214 if (Size.isPowerOfTwo()) {
1215 // Check against inlining width.
1216 unsigned InlineWidthBits =
1217 S.getASTContext().getTargetInfo().getMaxAtomicInlineWidth();
1218 if (Size <= S.getASTContext().toCharUnitsFromBits(BitSize: InlineWidthBits)) {
1219
1220 // OK, we will inline appropriately-aligned operations of this size,
1221 // and _Atomic(T) is appropriately-aligned.
1222 if (Size == CharUnits::One())
1223 return returnBool(true);
1224
1225 // Same for null pointers.
1226 assert(BuiltinOp != Builtin::BI__c11_atomic_is_lock_free);
1227 if (Ptr.isZero())
1228 return returnBool(true);
1229
1230 if (Ptr.isIntegralPointer()) {
1231 uint64_t IntVal = Ptr.getIntegerRepresentation();
1232 if (APSInt(APInt(64, IntVal, false), true).isAligned(A: Size.getAsAlign()))
1233 return returnBool(true);
1234 }
1235
1236 const Expr *PtrArg = Call->getArg(Arg: 1);
1237 // Otherwise, check if the type's alignment against Size.
1238 if (const auto *ICE = dyn_cast<ImplicitCastExpr>(Val: PtrArg)) {
1239 // Drop the potential implicit-cast to 'const volatile void*', getting
1240 // the underlying type.
1241 if (ICE->getCastKind() == CK_BitCast)
1242 PtrArg = ICE->getSubExpr();
1243 }
1244
1245 if (const auto *PtrTy = PtrArg->getType()->getAs<PointerType>()) {
1246 QualType PointeeType = PtrTy->getPointeeType();
1247 if (!PointeeType->isIncompleteType() &&
1248 S.getASTContext().getTypeAlignInChars(T: PointeeType) >= Size) {
1249 // OK, we will inline operations on this object.
1250 return returnBool(true);
1251 }
1252 }
1253 }
1254 }
1255
1256 if (BuiltinOp == Builtin::BI__atomic_always_lock_free)
1257 return returnBool(false);
1258
1259 return Invalid(S, OpPC);
1260}
1261
1262/// bool __c11_atomic_is_lock_free(size_t)
1263static bool interp__builtin_c11_atomic_is_lock_free(InterpState &S,
1264 CodePtr OpPC,
1265 const InterpFrame *Frame,
1266 const CallExpr *Call) {
1267 uint64_t SizeVal;
1268 if (!popToUInt64(S, E: Call->getArg(Arg: 0), Out&: SizeVal))
1269 return false;
1270
1271 CharUnits Size = CharUnits::fromQuantity(Quantity: SizeVal);
1272 if (Size.isPowerOfTwo()) {
1273 // Check against inlining width.
1274 unsigned InlineWidthBits =
1275 S.getASTContext().getTargetInfo().getMaxAtomicInlineWidth();
1276 if (Size <= S.getASTContext().toCharUnitsFromBits(BitSize: InlineWidthBits)) {
1277 S.Stk.push<Boolean>(Args: true);
1278 return true;
1279 }
1280 }
1281
1282 return false; // returnBool(false);
1283}
1284
1285/// __builtin_complex(Float A, float B);
1286static bool interp__builtin_complex(InterpState &S, CodePtr OpPC,
1287 const InterpFrame *Frame,
1288 const CallExpr *Call) {
1289 const Floating &Arg2 = S.Stk.pop<Floating>();
1290 const Floating &Arg1 = S.Stk.pop<Floating>();
1291 Pointer &Result = S.Stk.peek<Pointer>();
1292
1293 Result.elem<Floating>(I: 0) = Arg1;
1294 Result.elem<Floating>(I: 1) = Arg2;
1295 Result.initializeAllElements();
1296
1297 return true;
1298}
1299
1300/// __builtin_is_aligned()
1301/// __builtin_align_up()
1302/// __builtin_align_down()
1303/// The first parameter is either an integer or a pointer.
1304/// The second parameter is the requested alignment as an integer.
1305static bool interp__builtin_is_aligned_up_down(InterpState &S, CodePtr OpPC,
1306 const InterpFrame *Frame,
1307 const CallExpr *Call,
1308 unsigned BuiltinOp) {
1309 APSInt Alignment;
1310 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: Alignment))
1311 return false;
1312
1313 if (Alignment < 0 || !Alignment.isPowerOf2()) {
1314 S.FFDiag(E: Call, DiagId: diag::note_constexpr_invalid_alignment) << Alignment;
1315 return false;
1316 }
1317 unsigned SrcWidth = S.getASTContext().getIntWidth(T: Call->getArg(Arg: 0)->getType());
1318 APSInt MaxValue(APInt::getOneBitSet(numBits: SrcWidth, BitNo: SrcWidth - 1));
1319 if (APSInt::compareValues(I1: Alignment, I2: MaxValue) > 0) {
1320 S.FFDiag(E: Call, DiagId: diag::note_constexpr_alignment_too_big)
1321 << MaxValue << Call->getArg(Arg: 0)->getType() << Alignment;
1322 return false;
1323 }
1324
1325 // The first parameter is either an integer or a pointer.
1326 PrimType FirstArgT = *S.Ctx.classify(E: Call->getArg(Arg: 0));
1327
1328 if (isIntegerType(T: FirstArgT)) {
1329 APSInt Src;
1330 if (!popToAPSInt(Stk&: S.Stk, T: FirstArgT, Out&: Src))
1331 return false;
1332 APInt AlignMinusOne = Alignment.extOrTrunc(width: Src.getBitWidth()) - 1;
1333 if (BuiltinOp == Builtin::BI__builtin_align_up) {
1334 APSInt AlignedVal =
1335 APSInt((Src + AlignMinusOne) & ~AlignMinusOne, Src.isUnsigned());
1336 pushInteger(S, Val: AlignedVal, QT: Call->getType());
1337 } else if (BuiltinOp == Builtin::BI__builtin_align_down) {
1338 APSInt AlignedVal = APSInt(Src & ~AlignMinusOne, Src.isUnsigned());
1339 pushInteger(S, Val: AlignedVal, QT: Call->getType());
1340 } else {
1341 assert(*S.Ctx.classify(Call->getType()) == PT_Bool);
1342 S.Stk.push<Boolean>(Args: (Src & AlignMinusOne) == 0);
1343 }
1344 return true;
1345 }
1346 assert(FirstArgT == PT_Ptr);
1347 const Pointer &Ptr = S.Stk.pop<Pointer>();
1348
1349 // Null pointers are always aligned. Preserve null pointers for
1350 // align_up/align_down and return true for is_aligned.
1351 if (Ptr.isZero()) {
1352 if (BuiltinOp == Builtin::BI__builtin_is_aligned) {
1353 S.Stk.push<Boolean>(Args: true);
1354 return true;
1355 }
1356
1357 assert(BuiltinOp == Builtin::BI__builtin_align_up ||
1358 BuiltinOp == Builtin::BI__builtin_align_down);
1359
1360 S.Stk.push<Pointer>(Args: Ptr);
1361 return true;
1362 }
1363
1364 if (!Ptr.isBlockPointer() && !Ptr.isOpaquePointer()) {
1365 S.FFDiag(E: Call->getArg(Arg: 0), DiagId: diag::note_constexpr_alignment_compute)
1366 << Alignment;
1367 return false;
1368 }
1369
1370 const VarDecl *PtrDecl = Ptr.getRootVarDecl();
1371 // We need a pointer for a declaration here.
1372 if (!PtrDecl) {
1373 if (BuiltinOp == Builtin::BI__builtin_is_aligned)
1374 S.FFDiag(E: Call->getArg(Arg: 0), DiagId: diag::note_constexpr_alignment_compute)
1375 << Alignment;
1376 else
1377 S.FFDiag(E: Call->getArg(Arg: 0), DiagId: diag::note_constexpr_alignment_adjust)
1378 << Alignment;
1379 return false;
1380 }
1381
1382 unsigned PtrOffset;
1383 if (Ptr.isBlockPointer()) {
1384 // For one-past-end pointers, we can't call getIndex() since it asserts.
1385 // Use getNumElems() instead which gives the correct index for past-end.
1386 PtrOffset = Ptr.isElementPastEnd() ? Ptr.getNumElems() : Ptr.getIndex();
1387 } else {
1388 if (std::optional<size_t> PtrOff =
1389 Ptr.computeLayoutOffset(ASTCtx: S.getASTContext()))
1390 PtrOffset = *PtrOff;
1391 else
1392 return false;
1393 }
1394
1395 CharUnits BaseAlignment = S.getASTContext().getDeclAlign(D: PtrDecl);
1396 CharUnits PtrAlign =
1397 BaseAlignment.alignmentAtOffset(offset: CharUnits::fromQuantity(Quantity: PtrOffset));
1398
1399 if (BuiltinOp == Builtin::BI__builtin_is_aligned) {
1400 if (PtrAlign.getQuantity() >= Alignment) {
1401 S.Stk.push<Boolean>(Args: true);
1402 return true;
1403 }
1404 // If the alignment is not known to be sufficient, some cases could still
1405 // be aligned at run time. However, if the requested alignment is less or
1406 // equal to the base alignment and the offset is not aligned, we know that
1407 // the run-time value can never be aligned.
1408 if (BaseAlignment.getQuantity() >= Alignment &&
1409 PtrAlign.getQuantity() < Alignment) {
1410 S.Stk.push<Boolean>(Args: false);
1411 return true;
1412 }
1413
1414 S.FFDiag(E: Call->getArg(Arg: 0), DiagId: diag::note_constexpr_alignment_compute)
1415 << Alignment;
1416 return false;
1417 }
1418
1419 assert(BuiltinOp == Builtin::BI__builtin_align_down ||
1420 BuiltinOp == Builtin::BI__builtin_align_up);
1421
1422 // For align_up/align_down, we can return the same value if the alignment
1423 // is known to be greater or equal to the requested value.
1424 if (PtrAlign.getQuantity() >= Alignment) {
1425 S.Stk.push<Pointer>(Args: Ptr);
1426 return true;
1427 }
1428
1429 // The alignment could be greater than the minimum at run-time, so we cannot
1430 // infer much about the resulting pointer value. One case is possible:
1431 // For `_Alignas(32) char buf[N]; __builtin_align_down(&buf[idx], 32)` we
1432 // can infer the correct index if the requested alignment is smaller than
1433 // the base alignment so we can perform the computation on the offset.
1434 if (BaseAlignment.getQuantity() >= Alignment) {
1435 assert(Alignment.getBitWidth() <= 64 &&
1436 "Cannot handle > 64-bit address-space");
1437 uint64_t Alignment64 = Alignment.getZExtValue();
1438 CharUnits NewOffset =
1439 CharUnits::fromQuantity(Quantity: BuiltinOp == Builtin::BI__builtin_align_down
1440 ? llvm::alignDown(Value: PtrOffset, Align: Alignment64)
1441 : llvm::alignTo(Value: PtrOffset, Align: Alignment64));
1442
1443 if (Ptr.isBlockPointer()) {
1444 S.Stk.push<Pointer>(Args: Ptr.atIndex(Idx: NewOffset.getQuantity()));
1445 return true;
1446 }
1447
1448 assert(Ptr.isOpaquePointer());
1449
1450 APSInt APOffset =
1451 APSInt(APInt(64, NewOffset.getQuantity(), /*IsSigned=*/true),
1452 /*IsUnsigned=*/false);
1453 return arrayElemPtrOpaque(S, OpPC, Ptr, Index: std::move(APOffset),
1454 /*AllocReplace=*/AllowReplace: true);
1455 }
1456
1457 // Otherwise, we cannot constant-evaluate the result.
1458 S.FFDiag(E: Call->getArg(Arg: 0), DiagId: diag::note_constexpr_alignment_adjust) << Alignment;
1459 return false;
1460}
1461
1462/// Returns the alignment of the object Ptr points into.
1463static CharUnits getBaseAlignment(const ASTContext &ASTCtx,
1464 const Pointer &Ptr) {
1465 CharUnits BaseAlignment;
1466 if (Ptr.isBlockPointer() && Ptr.block()->isDynamic())
1467 BaseAlignment = Ptr.getDeclDesc()->computeAlignForDynamicAlloc(Ctx: ASTCtx);
1468 else if (const auto *VD = Ptr.getRootVarDecl())
1469 BaseAlignment = ASTCtx.getDeclAlign(D: VD);
1470 else if (const auto *E = Ptr.getRootExpr())
1471 BaseAlignment = GetAlignOfExpr(Ctx: ASTCtx, E, ExprKind: UETT_AlignOf);
1472 return BaseAlignment;
1473}
1474
1475/// __builtin_assume_aligned(Ptr, Alignment[, ExtraOffset])
1476static bool interp__builtin_assume_aligned(InterpState &S, CodePtr OpPC,
1477 const InterpFrame *Frame,
1478 const CallExpr *Call) {
1479 assert(Call->getNumArgs() == 2 || Call->getNumArgs() == 3);
1480
1481 std::optional<APSInt> ExtraOffset;
1482 if (Call->getNumArgs() == 3) {
1483 APSInt ExtraOffsetVal;
1484 if (!popToAPSInt(Stk&: S.Stk, T: *S.Ctx.classify(E: Call->getArg(Arg: 2)), Out&: ExtraOffsetVal))
1485 return false;
1486 ExtraOffset = ExtraOffsetVal;
1487 }
1488
1489 APSInt Alignment;
1490 if (!popToAPSInt(Stk&: S.Stk, T: *S.Ctx.classify(E: Call->getArg(Arg: 1)), Out&: Alignment))
1491 return false;
1492 const Pointer &Ptr = S.Stk.pop<Pointer>();
1493
1494 const ASTContext &ASTCtx = S.getASTContext();
1495 CharUnits Align = CharUnits::fromQuantity(Quantity: Alignment.getZExtValue());
1496
1497 // If there is a base object, then it must have the correct alignment.
1498 if (Ptr.isBlockPointer() || Ptr.isOpaquePointer()) {
1499 CharUnits BaseAlignment = getBaseAlignment(ASTCtx, Ptr);
1500 if (BaseAlignment < Align) {
1501 S.CCEDiag(E: Call->getArg(Arg: 0),
1502 DiagId: diag::note_constexpr_baa_insufficient_alignment)
1503 << 0 << BaseAlignment.getQuantity() << Align.getQuantity();
1504 return false;
1505 }
1506 }
1507
1508 std::optional<size_t> LayoutOffset = Ptr.computeLayoutOffset(ASTCtx);
1509 if (!LayoutOffset)
1510 return false;
1511
1512 CharUnits AVOffset = CharUnits::fromQuantity(Quantity: *LayoutOffset);
1513 if (ExtraOffset)
1514 AVOffset -= CharUnits::fromQuantity(Quantity: ExtraOffset->getZExtValue());
1515 if (AVOffset.alignTo(Align) != AVOffset) {
1516 if (Ptr.isBlockPointer() || Ptr.isOpaquePointer())
1517 S.CCEDiag(E: Call->getArg(Arg: 0),
1518 DiagId: diag::note_constexpr_baa_insufficient_alignment)
1519 << 1 << AVOffset.getQuantity() << Align.getQuantity();
1520 else
1521 S.CCEDiag(E: Call->getArg(Arg: 0),
1522 DiagId: diag::note_constexpr_baa_value_insufficient_alignment)
1523 << AVOffset.getQuantity() << Align.getQuantity();
1524 return false;
1525 }
1526
1527 S.Stk.push<Pointer>(Args: Ptr);
1528 return true;
1529}
1530
1531/// (CarryIn, LHS, RHS, Result)
1532static bool interp__builtin_ia32_addcarry_subborrow(InterpState &S,
1533 CodePtr OpPC,
1534 const InterpFrame *Frame,
1535 const CallExpr *Call,
1536 bool IsAdd) {
1537 if (Call->getNumArgs() != 4 || !Call->getArg(Arg: 0)->getType()->isIntegerType() ||
1538 !Call->getArg(Arg: 1)->getType()->isIntegerType() ||
1539 !Call->getArg(Arg: 2)->getType()->isIntegerType())
1540 return false;
1541
1542 const Pointer &CarryOutPtr = S.Stk.pop<Pointer>();
1543
1544 APSInt RHS;
1545 if (!popToAPSInt(S, E: Call->getArg(Arg: 2), Out&: RHS))
1546 return false;
1547 APSInt LHS;
1548 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: LHS))
1549 return false;
1550 APSInt CarryIn;
1551 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: CarryIn))
1552 return false;
1553
1554 unsigned BitWidth = LHS.getBitWidth();
1555 unsigned CarryInBit = CarryIn.ugt(RHS: 0) ? 1 : 0;
1556 APInt ExResult =
1557 IsAdd ? (LHS.zext(width: BitWidth + 1) + (RHS.zext(width: BitWidth + 1) + CarryInBit))
1558 : (LHS.zext(width: BitWidth + 1) - (RHS.zext(width: BitWidth + 1) + CarryInBit));
1559
1560 APInt Result = ExResult.extractBits(numBits: BitWidth, bitPosition: 0);
1561 APSInt CarryOut =
1562 APSInt(ExResult.extractBits(numBits: 1, bitPosition: BitWidth), /*IsUnsigned=*/true);
1563
1564 QualType CarryOutType = Call->getArg(Arg: 3)->getType()->getPointeeType();
1565 PrimType CarryOutT = *S.getContext().classify(T: CarryOutType);
1566 assignIntegral(S, Dest: CarryOutPtr, ValueT: CarryOutT, Value: APSInt(std::move(Result), true));
1567
1568 pushInteger(S, Val: CarryOut, QT: Call->getType());
1569
1570 return true;
1571}
1572
1573static bool interp__builtin_os_log_format_buffer_size(InterpState &S,
1574 CodePtr OpPC,
1575 const InterpFrame *Frame,
1576 const CallExpr *Call) {
1577 analyze_os_log::OSLogBufferLayout Layout;
1578 analyze_os_log::computeOSLogBufferLayout(Ctx&: S.getASTContext(), E: Call, layout&: Layout);
1579 pushInteger(S, Val: Layout.size().getQuantity(), QT: Call->getType());
1580 return true;
1581}
1582
1583static bool
1584interp__builtin_ptrauth_string_discriminator(InterpState &S, CodePtr OpPC,
1585 const InterpFrame *Frame,
1586 const CallExpr *Call) {
1587 const auto &Ptr = S.Stk.pop<Pointer>();
1588 if (!Ptr.isStringPointer())
1589 return false;
1590
1591 uint64_t Result = getPointerAuthStableSipHash(
1592 S: cast<StringLiteral>(Val: Ptr.getRootExpr())->getString());
1593 pushInteger(S, Val: Result, QT: Call->getType());
1594 return true;
1595}
1596
1597static bool interp__builtin_infer_alloc_token(InterpState &S, CodePtr OpPC,
1598 const InterpFrame *Frame,
1599 const CallExpr *Call) {
1600 const ASTContext &ASTCtx = S.getASTContext();
1601 uint64_t BitWidth = ASTCtx.getTypeSize(T: ASTCtx.getSizeType());
1602 auto Mode =
1603 ASTCtx.getLangOpts().AllocTokenMode.value_or(u: llvm::DefaultAllocTokenMode);
1604 auto MaxTokensOpt = ASTCtx.getLangOpts().AllocTokenMax;
1605 uint64_t MaxTokens =
1606 MaxTokensOpt.value_or(u: 0) ? *MaxTokensOpt : (~0ULL >> (64 - BitWidth));
1607
1608 // We do not read any of the arguments; discard them.
1609 for (int I = Call->getNumArgs() - 1; I >= 0; --I)
1610 discard(Stk&: S.Stk, T: S.getContext().classify(E: Call->getArg(Arg: I)).value_or(PT: PT_Ptr));
1611
1612 // Note: Type inference from a surrounding cast is not supported in
1613 // constexpr evaluation.
1614 QualType AllocType = infer_alloc::inferPossibleType(E: Call, Ctx: ASTCtx, CastE: nullptr);
1615 if (AllocType.isNull()) {
1616 S.CCEDiag(E: Call,
1617 DiagId: diag::note_constexpr_infer_alloc_token_type_inference_failed);
1618 return false;
1619 }
1620
1621 auto ATMD = infer_alloc::getAllocTokenMetadata(T: AllocType, Ctx: ASTCtx);
1622 if (!ATMD) {
1623 S.CCEDiag(E: Call, DiagId: diag::note_constexpr_infer_alloc_token_no_metadata);
1624 return false;
1625 }
1626
1627 auto MaybeToken = llvm::getAllocToken(Mode, Metadata: *ATMD, MaxTokens);
1628 if (!MaybeToken) {
1629 S.CCEDiag(E: Call, DiagId: diag::note_constexpr_infer_alloc_token_unsupported_mode);
1630 return false;
1631 }
1632
1633 pushInteger(S, Val: llvm::APInt(BitWidth, *MaybeToken), QT: ASTCtx.getSizeType());
1634 return true;
1635}
1636
1637static bool interp__builtin_operator_new(InterpState &S, CodePtr OpPC,
1638 const InterpFrame *Frame,
1639 const CallExpr *Call) {
1640 // A call to __operator_new is only valid within std::allocate<>::allocate.
1641 // Walk up the call stack to find the appropriate caller and get the
1642 // element type from it.
1643 auto [NewCall, ElemType] = S.getStdAllocatorCaller(Name: "allocate");
1644
1645 if (ElemType.isNull()) {
1646 S.FFDiag(E: Call, DiagId: S.getLangOpts().CPlusPlus20
1647 ? diag::note_constexpr_new_untyped
1648 : diag::note_constexpr_new);
1649 return false;
1650 }
1651 assert(NewCall);
1652
1653 if (ElemType->isIncompleteType() || ElemType->isFunctionType()) {
1654 S.FFDiag(E: Call, DiagId: diag::note_constexpr_new_not_complete_object_type)
1655 << (ElemType->isIncompleteType() ? 0 : 1) << ElemType;
1656 return false;
1657 }
1658
1659 // We only care about the first parameter (the size), so discard all the
1660 // others.
1661 {
1662 unsigned NumArgs = Call->getNumArgs();
1663 assert(NumArgs >= 1);
1664
1665 // The std::nothrow_t arg never gets put on the stack.
1666 if (Call->getArg(Arg: NumArgs - 1)->getType()->isNothrowT())
1667 --NumArgs;
1668 auto Args = ArrayRef(Call->getArgs(), Call->getNumArgs());
1669 // First arg is needed.
1670 Args = Args.drop_front();
1671
1672 // Discard the rest.
1673 for (const Expr *Arg : Args)
1674 discard(Stk&: S.Stk, T: *S.getContext().classify(E: Arg));
1675 }
1676
1677 APSInt Bytes;
1678 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: Bytes))
1679 return false;
1680 CharUnits ElemSize = S.getASTContext().getTypeSizeInChars(T: ElemType);
1681 assert(!ElemSize.isZero());
1682 // Divide the number of bytes by sizeof(ElemType), so we get the number of
1683 // elements we should allocate.
1684 APInt NumElems, Remainder;
1685 APInt ElemSizeAP(Bytes.getBitWidth(), ElemSize.getQuantity());
1686 APInt::udivrem(LHS: Bytes, RHS: ElemSizeAP, Quotient&: NumElems, Remainder);
1687 if (Remainder != 0) {
1688 // This likely indicates a bug in the implementation of 'std::allocator'.
1689 S.FFDiag(E: Call, DiagId: diag::note_constexpr_operator_new_bad_size)
1690 << Bytes << APSInt(ElemSizeAP, true) << ElemType;
1691 return false;
1692 }
1693
1694 // NB: The same check we're using in CheckArraySize()
1695 if (NumElems.getActiveBits() >
1696 ConstantArrayType::getMaxSizeBits(Context: S.getASTContext()) ||
1697 NumElems.ugt(RHS: Descriptor::MaxArrayElemBytes / ElemSize.getQuantity())) {
1698 // FIXME: NoThrow check?
1699 const SourceInfo &Loc = S.Current->getSource(PC: OpPC);
1700 S.FFDiag(SI: Loc, DiagId: diag::note_constexpr_new_too_large)
1701 << NumElems.getZExtValue();
1702 return false;
1703 }
1704
1705 if (!CheckArraySize(S, OpPC, NumElems: NumElems.getZExtValue()))
1706 return false;
1707
1708 bool IsArray = NumElems.ugt(RHS: 1);
1709 OptPrimType ElemT = S.getContext().classify(T: ElemType);
1710 DynamicAllocator &Allocator = S.getAllocator();
1711 if (ElemT) {
1712 Block *B =
1713 Allocator.allocate(Source: NewCall, T: *ElemT, NumElements: NumElems.getZExtValue(),
1714 EvalID: S.Ctx.getEvalID(), AllocForm: DynamicAllocator::Form::Operator);
1715 assert(B);
1716 S.Stk.push<Pointer>(Args: Pointer(B).atIndex(Idx: 0));
1717 return true;
1718 }
1719
1720 assert(!ElemT);
1721
1722 // Composite arrays
1723 if (IsArray) {
1724 const Descriptor *Desc =
1725 S.P.createDescriptor(D: NewCall, Ty: ElemType.getTypePtr());
1726 Block *B =
1727 Allocator.allocate(D: Desc, NumElements: NumElems.getZExtValue(), EvalID: S.Ctx.getEvalID(),
1728 AllocForm: DynamicAllocator::Form::Operator);
1729 assert(B);
1730 S.Stk.push<Pointer>(Args: Pointer(B).atIndex(Idx: 0).narrow());
1731 return true;
1732 }
1733
1734 // Records. Still allocate them as single-element arrays.
1735 QualType AllocType = S.getASTContext().getConstantArrayType(
1736 EltTy: ElemType, ArySize: NumElems, SizeExpr: nullptr, ASM: ArraySizeModifier::Normal, IndexTypeQuals: 0);
1737
1738 const Descriptor *Desc =
1739 S.P.createDescriptor(D: NewCall, Ty: AllocType.getTypePtr());
1740 Block *B = Allocator.allocate(D: Desc, EvalID: S.getContext().getEvalID(),
1741 AllocForm: DynamicAllocator::Form::Operator);
1742 assert(B);
1743 S.Stk.push<Pointer>(Args: Pointer(B).atIndex(Idx: 0).narrow());
1744 return true;
1745}
1746
1747static bool interp__builtin_operator_delete(InterpState &S, CodePtr OpPC,
1748 const InterpFrame *Frame,
1749 const CallExpr *Call) {
1750 const Expr *Source = nullptr;
1751 const Block *BlockToDelete = nullptr;
1752
1753 unsigned NumArgs = Call->getNumArgs();
1754 assert(NumArgs >= 1);
1755
1756 // Args are pushed in source order. The trailing sized/aligned delete
1757 // operands are above the pointer on the stack.
1758 for (unsigned I = NumArgs - 1; I != 0; --I)
1759 discard(Stk&: S.Stk, T: *S.getContext().classify(E: Call->getArg(Arg: I)));
1760
1761 if (S.checkingPotentialConstantExpression()) {
1762 S.Stk.discard<Pointer>();
1763 return false;
1764 }
1765
1766 // This is permitted only within a call to std::allocator<T>::deallocate.
1767 if (!S.getStdAllocatorCaller(Name: "deallocate")) {
1768 S.FFDiag(E: Call);
1769 S.Stk.discard<Pointer>();
1770 return true;
1771 }
1772
1773 {
1774 const Pointer &Ptr = S.Stk.pop<Pointer>();
1775
1776 if (Ptr.isZero()) {
1777 S.CCEDiag(E: Call, DiagId: diag::note_constexpr_deallocate_null);
1778 return true;
1779 }
1780
1781 Source = Ptr.getRootExpr();
1782 BlockToDelete = Ptr.block();
1783
1784 if (!BlockToDelete->isDynamic()) {
1785 S.FFDiag(E: Call, DiagId: diag::note_constexpr_delete_not_heap_alloc)
1786 << Ptr.toDiagnosticString(Ctx: S.getASTContext());
1787 if (const auto *D = Ptr.getFieldDesc()->asDecl())
1788 S.Note(Loc: D->getLocation(), DiagId: diag::note_declared_at);
1789 }
1790 }
1791 assert(BlockToDelete);
1792
1793 DynamicAllocator &Allocator = S.getAllocator();
1794 const Descriptor *BlockDesc = BlockToDelete->getDescriptor();
1795 std::optional<DynamicAllocator::Form> AllocForm =
1796 Allocator.getAllocationForm(Source);
1797
1798 if (!Allocator.deallocate(Source, BlockToDelete)) {
1799 // Nothing has been deallocated, this must be a double-delete.
1800 const SourceInfo &Loc = S.Current->getSource(PC: OpPC);
1801 S.FFDiag(SI: Loc, DiagId: diag::note_constexpr_double_delete);
1802 return false;
1803 }
1804 assert(AllocForm);
1805
1806 return CheckNewDeleteForms(
1807 S, OpPC, AllocForm: *AllocForm, DeleteForm: DynamicAllocator::Form::Operator, D: BlockDesc, NewExpr: Source);
1808}
1809
1810static bool interp__builtin_arithmetic_fence(InterpState &S, CodePtr OpPC,
1811 const InterpFrame *Frame,
1812 const CallExpr *Call) {
1813 const Floating &Arg0 = S.Stk.pop<Floating>();
1814 S.Stk.push<Floating>(Args: Arg0);
1815 return true;
1816}
1817
1818static bool interp__builtin_vector_reduce(InterpState &S, CodePtr OpPC,
1819 const CallExpr *Call, unsigned ID) {
1820 const Pointer &Arg = S.Stk.pop<Pointer>();
1821 assert(Arg.getFieldDesc()->isPrimitiveArray());
1822
1823 QualType ElemType = Arg.getFieldDesc()->getElemQualType();
1824 assert(Call->getType() == ElemType);
1825 PrimType ElemT = *S.getContext().classify(T: ElemType);
1826 unsigned NumElems = Arg.getNumElems();
1827
1828 if (!isIntegerType(T: ElemT))
1829 return false;
1830
1831 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
1832 T Result = Arg.elem<T>(0);
1833 unsigned BitWidth = Result.bitWidth();
1834 for (unsigned I = 1; I != NumElems; ++I) {
1835 T Elem = Arg.elem<T>(I);
1836 T PrevResult = Result;
1837
1838 if (ID == Builtin::BI__builtin_reduce_add) {
1839 if (T::add(Result, Elem, BitWidth, &Result)) {
1840 unsigned OverflowBits = BitWidth + 1;
1841 (void)handleOverflow(S, OpPC,
1842 (PrevResult.toAPSInt(OverflowBits) +
1843 Elem.toAPSInt(OverflowBits)));
1844 return false;
1845 }
1846 } else if (ID == Builtin::BI__builtin_reduce_mul) {
1847 if (T::mul(Result, Elem, BitWidth, &Result)) {
1848 unsigned OverflowBits = BitWidth * 2;
1849 (void)handleOverflow(S, OpPC,
1850 (PrevResult.toAPSInt(OverflowBits) *
1851 Elem.toAPSInt(OverflowBits)));
1852 return false;
1853 }
1854
1855 } else if (ID == Builtin::BI__builtin_reduce_and) {
1856 (void)T::bitAnd(Result, Elem, BitWidth, &Result);
1857 } else if (ID == Builtin::BI__builtin_reduce_or) {
1858 (void)T::bitOr(Result, Elem, BitWidth, &Result);
1859 } else if (ID == Builtin::BI__builtin_reduce_xor) {
1860 (void)T::bitXor(Result, Elem, BitWidth, &Result);
1861 } else if (ID == Builtin::BI__builtin_reduce_min) {
1862 if (Elem < Result)
1863 Result = Elem;
1864 } else if (ID == Builtin::BI__builtin_reduce_max) {
1865 if (Elem > Result)
1866 Result = Elem;
1867 } else {
1868 llvm_unreachable("Unhandled vector reduce builtin");
1869 }
1870 }
1871 pushInteger(S, Result.toAPSInt(), Call->getType());
1872 });
1873
1874 return true;
1875}
1876
1877static bool interp__builtin_elementwise_abs(InterpState &S, CodePtr OpPC,
1878 const InterpFrame *Frame,
1879 const CallExpr *Call,
1880 unsigned BuiltinID) {
1881 assert(Call->getNumArgs() == 1);
1882 QualType Ty = Call->getArg(Arg: 0)->getType();
1883 if (Ty->isIntegerType()) {
1884 APSInt Val;
1885 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: Val))
1886 return false;
1887 pushInteger(S, Val: Val.abs(), QT: Call->getType());
1888 return true;
1889 }
1890
1891 if (Ty->isFloatingType()) {
1892 Floating Val = S.Stk.pop<Floating>();
1893 Floating Result = abs(S, In: Val);
1894 S.Stk.push<Floating>(Args&: Result);
1895 return true;
1896 }
1897
1898 // Otherwise, the argument must be a vector.
1899 assert(Call->getArg(0)->getType()->isVectorType());
1900 const Pointer &Arg = S.Stk.pop<Pointer>();
1901 assert(Arg.getFieldDesc()->isPrimitiveArray());
1902 const Pointer &Dst = S.Stk.peek<Pointer>();
1903 assert(Dst.getFieldDesc()->isPrimitiveArray());
1904 assert(Arg.getFieldDesc()->getNumElems() ==
1905 Dst.getFieldDesc()->getNumElems());
1906
1907 QualType ElemType = Arg.getFieldDesc()->getElemQualType();
1908 PrimType ElemT = *S.getContext().classify(T: ElemType);
1909 unsigned NumElems = Arg.getNumElems();
1910 // we can either have a vector of integer or a vector of floating point
1911 for (unsigned I = 0; I != NumElems; ++I) {
1912 if (ElemType->isIntegerType()) {
1913 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
1914 Dst.elem<T>(I) = T::from(static_cast<T>(
1915 APSInt(Arg.elem<T>(I).toAPSInt().abs(),
1916 ElemType->isUnsignedIntegerOrEnumerationType())));
1917 });
1918 } else {
1919 Floating Val = Arg.elem<Floating>(I);
1920 Dst.elem<Floating>(I) = abs(S, In: Val);
1921 }
1922 }
1923 Dst.initializeAllElements();
1924
1925 return true;
1926}
1927
1928/// Can be called with an integer or vector as the first and only parameter.
1929static bool interp__builtin_elementwise_countzeroes(InterpState &S,
1930 CodePtr OpPC,
1931 const InterpFrame *Frame,
1932 const CallExpr *Call,
1933 unsigned BuiltinID) {
1934 bool HasZeroArg = Call->getNumArgs() == 2;
1935 bool IsCTTZ = BuiltinID == Builtin::BI__builtin_elementwise_ctzg;
1936 assert(Call->getNumArgs() == 1 || HasZeroArg);
1937 if (Call->getArg(Arg: 0)->getType()->isIntegerType()) {
1938 PrimType ArgT = *S.getContext().classify(T: Call->getArg(Arg: 0)->getType());
1939 APSInt Val;
1940 if (!popToAPSInt(Stk&: S.Stk, T: ArgT, Out&: Val))
1941 return false;
1942 std::optional<APSInt> ZeroVal;
1943 if (HasZeroArg) {
1944 ZeroVal = Val;
1945 if (!popToAPSInt(Stk&: S.Stk, T: ArgT, Out&: Val))
1946 return false;
1947 }
1948
1949 if (Val.isZero()) {
1950 if (ZeroVal) {
1951 pushInteger(S, Val: *ZeroVal, QT: Call->getType());
1952 return true;
1953 }
1954 // If we haven't been provided the second argument, the result is
1955 // undefined
1956 S.FFDiag(SI: S.Current->getSource(PC: OpPC),
1957 DiagId: diag::note_constexpr_countzeroes_zero)
1958 << /*IsTrailing=*/IsCTTZ;
1959 return false;
1960 }
1961
1962 if (BuiltinID == Builtin::BI__builtin_elementwise_clzg) {
1963 pushInteger(S, Val: Val.countLeadingZeros(), QT: Call->getType());
1964 } else {
1965 pushInteger(S, Val: Val.countTrailingZeros(), QT: Call->getType());
1966 }
1967 return true;
1968 }
1969 // Otherwise, the argument must be a vector.
1970 const ASTContext &ASTCtx = S.getASTContext();
1971 Pointer ZeroArg;
1972 if (HasZeroArg) {
1973 assert(Call->getArg(1)->getType()->isVectorType() &&
1974 ASTCtx.hasSameUnqualifiedType(Call->getArg(0)->getType(),
1975 Call->getArg(1)->getType()));
1976 (void)ASTCtx;
1977 ZeroArg = S.Stk.pop<Pointer>();
1978 assert(ZeroArg.getFieldDesc()->isPrimitiveArray());
1979 }
1980 assert(Call->getArg(0)->getType()->isVectorType());
1981 const Pointer &Arg = S.Stk.pop<Pointer>();
1982 assert(Arg.getFieldDesc()->isPrimitiveArray());
1983 const Pointer &Dst = S.Stk.peek<Pointer>();
1984 assert(Dst.getFieldDesc()->isPrimitiveArray());
1985 assert(Arg.getFieldDesc()->getNumElems() ==
1986 Dst.getFieldDesc()->getNumElems());
1987
1988 QualType ElemType = Arg.getFieldDesc()->getElemQualType();
1989 PrimType ElemT = *S.getContext().classify(T: ElemType);
1990 unsigned NumElems = Arg.getNumElems();
1991
1992 // FIXME: Reading from uninitialized vector elements?
1993 for (unsigned I = 0; I != NumElems; ++I) {
1994 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
1995 APInt EltVal = Arg.atIndex(I).deref<T>().toAPSInt();
1996 if (EltVal.isZero()) {
1997 if (HasZeroArg) {
1998 Dst.atIndex(I).deref<T>() = ZeroArg.atIndex(I).deref<T>();
1999 } else {
2000 // If we haven't been provided the second argument, the result is
2001 // undefined
2002 S.FFDiag(S.Current->getSource(OpPC),
2003 diag::note_constexpr_countzeroes_zero)
2004 << /*IsTrailing=*/IsCTTZ;
2005 return false;
2006 }
2007 } else if (IsCTTZ) {
2008 Dst.atIndex(I).deref<T>() = T::from(EltVal.countTrailingZeros());
2009 } else {
2010 Dst.atIndex(I).deref<T>() = T::from(EltVal.countLeadingZeros());
2011 }
2012 Dst.atIndex(I).initialize();
2013 });
2014 }
2015
2016 return true;
2017}
2018
2019static bool interp__builtin_memcpy(InterpState &S, CodePtr OpPC,
2020 const InterpFrame *Frame,
2021 const CallExpr *Call, unsigned ID) {
2022 assert(Call->getNumArgs() == 3);
2023 const ASTContext &ASTCtx = S.getASTContext();
2024 uint64_t Size;
2025 if (!popToUInt64(S, E: Call->getArg(Arg: 2), Out&: Size))
2026 return false;
2027 Pointer SrcPtr = S.Stk.pop<Pointer>().expand();
2028 Pointer DestPtr = S.Stk.pop<Pointer>().expand();
2029
2030 if (ID == Builtin::BImemcpy || ID == Builtin::BImemmove)
2031 diagnoseNonConstexprBuiltin(S, OpPC, ID);
2032
2033 bool Move =
2034 (ID == Builtin::BI__builtin_memmove || ID == Builtin::BImemmove ||
2035 ID == Builtin::BI__builtin_wmemmove || ID == Builtin::BIwmemmove);
2036 bool WChar = ID == Builtin::BIwmemcpy || ID == Builtin::BIwmemmove ||
2037 ID == Builtin::BI__builtin_wmemcpy ||
2038 ID == Builtin::BI__builtin_wmemmove;
2039
2040 // If the size is zero, we treat this as always being a valid no-op.
2041 if (Size == 0) {
2042 S.Stk.push<Pointer>(Args&: DestPtr);
2043 return true;
2044 }
2045
2046 if (SrcPtr.isZero() || DestPtr.isZero()) {
2047 Pointer DiagPtr = (SrcPtr.isZero() ? SrcPtr : DestPtr);
2048 S.FFDiag(SI: S.Current->getSource(PC: OpPC), DiagId: diag::note_constexpr_memcpy_null)
2049 << /*IsMove=*/Move << /*IsWchar=*/WChar << !SrcPtr.isZero()
2050 << DiagPtr.toDiagnosticString(Ctx: ASTCtx);
2051 return false;
2052 }
2053
2054 // Diagnose integral src/dest pointers specially.
2055 if (SrcPtr.isIntegralPointer() || DestPtr.isIntegralPointer()) {
2056 std::string DiagVal = "(void *)";
2057 DiagVal += SrcPtr.isIntegralPointer()
2058 ? std::to_string(val: SrcPtr.getIntegerRepresentation())
2059 : std::to_string(val: DestPtr.getIntegerRepresentation());
2060 S.FFDiag(SI: S.Current->getSource(PC: OpPC), DiagId: diag::note_constexpr_memcpy_null)
2061 << Move << WChar << DestPtr.isIntegralPointer() << DiagVal;
2062 return false;
2063 }
2064
2065 if (!DestPtr.isBlockPointer())
2066 return diagnoseUnreadableObject(S, OpPC, Ptr: DestPtr, AK: AK_Assign);
2067
2068 if (!isReadable(P: SrcPtr))
2069 return false;
2070
2071 if (DestPtr.getType()->isIncompleteType()) {
2072 S.FFDiag(SI: S.Current->getSource(PC: OpPC),
2073 DiagId: diag::note_constexpr_memcpy_incomplete_type)
2074 << Move << DestPtr.getType();
2075 return false;
2076 }
2077 if (SrcPtr.getType()->isIncompleteType()) {
2078 S.FFDiag(SI: S.Current->getSource(PC: OpPC),
2079 DiagId: diag::note_constexpr_memcpy_incomplete_type)
2080 << Move << SrcPtr.getType();
2081 return false;
2082 }
2083
2084 QualType DestElemType = getElemType(P: DestPtr);
2085 if (DestElemType->isIncompleteType()) {
2086 S.FFDiag(SI: S.Current->getSource(PC: OpPC),
2087 DiagId: diag::note_constexpr_memcpy_incomplete_type)
2088 << Move << DestElemType;
2089 return false;
2090 }
2091
2092 size_t RemainingDestElems;
2093 if (DestPtr.inArray()) {
2094 RemainingDestElems = DestPtr.isUnknownSizeArray()
2095 ? 0
2096 : (DestPtr.getNumElems() - DestPtr.getIndex());
2097 } else {
2098 RemainingDestElems = 1;
2099 }
2100 unsigned DestElemSize = ASTCtx.getTypeSizeInChars(T: DestElemType).getQuantity();
2101
2102 if (WChar) {
2103 uint64_t WCharSize =
2104 ASTCtx.getTypeSizeInChars(T: ASTCtx.getWCharType()).getQuantity();
2105 Size *= WCharSize;
2106 }
2107
2108 if (Size % DestElemSize != 0) {
2109 S.FFDiag(SI: S.Current->getSource(PC: OpPC),
2110 DiagId: diag::note_constexpr_memcpy_unsupported)
2111 << Move << WChar << 0 << DestElemType << Size << DestElemSize;
2112 return false;
2113 }
2114
2115 QualType SrcElemType = getElemType(P: SrcPtr);
2116 size_t RemainingSrcElems;
2117 if (SrcPtr.inArray()) {
2118 RemainingSrcElems = SrcPtr.isUnknownSizeArray()
2119 ? 0
2120 : (SrcPtr.getNumElems() - SrcPtr.getIndex());
2121 } else {
2122 RemainingSrcElems = 1;
2123 }
2124 unsigned SrcElemSize = ASTCtx.getTypeSizeInChars(T: SrcElemType).getQuantity();
2125
2126 if (!ASTCtx.hasSameUnqualifiedType(T1: DestElemType, T2: SrcElemType)) {
2127 S.FFDiag(SI: S.Current->getSource(PC: OpPC), DiagId: diag::note_constexpr_memcpy_type_pun)
2128 << Move << SrcElemType << DestElemType;
2129 return false;
2130 }
2131
2132 if (!DestElemType.isTriviallyCopyableType(Context: ASTCtx)) {
2133 S.FFDiag(SI: S.Current->getSource(PC: OpPC), DiagId: diag::note_constexpr_memcpy_nontrivial)
2134 << Move << DestElemType;
2135 return false;
2136 }
2137
2138 // Check if we have enough elements to read from and write to.
2139 size_t RemainingDestBytes = RemainingDestElems * DestElemSize;
2140 size_t RemainingSrcBytes = RemainingSrcElems * SrcElemSize;
2141 if (Size > RemainingDestBytes || Size > RemainingSrcBytes) {
2142 APInt N = APInt(64, Size / DestElemSize);
2143 S.FFDiag(SI: S.Current->getSource(PC: OpPC),
2144 DiagId: diag::note_constexpr_memcpy_unsupported)
2145 << Move << WChar << (Size > RemainingSrcBytes ? 1 : 2) << DestElemType
2146 << toString(I: N, Radix: 10, /*Signed=*/false);
2147 return false;
2148 }
2149
2150 // Check for overlapping memory regions.
2151 if (!Move && Pointer::pointToSameBlock(A: SrcPtr, B: DestPtr)) {
2152 // Remove base casts.
2153 Pointer SrcP = SrcPtr.stripBaseCasts();
2154 Pointer DestP = DestPtr.stripBaseCasts();
2155
2156 unsigned SrcIndex = SrcP.expand().getIndex() * SrcElemSize;
2157 unsigned DstIndex = DestP.expand().getIndex() * DestElemSize;
2158
2159 if ((SrcIndex <= DstIndex && (SrcIndex + Size) > DstIndex) ||
2160 (DstIndex <= SrcIndex && (DstIndex + Size) > SrcIndex)) {
2161 S.FFDiag(SI: S.Current->getSource(PC: OpPC), DiagId: diag::note_constexpr_memcpy_overlap)
2162 << /*IsWChar=*/false;
2163 return false;
2164 }
2165 }
2166
2167 assert(Size % DestElemSize == 0);
2168 if (!DoMemcpy(S, OpPC, SrcPtr, DestPtr, Size: Bytes(Size).toBits()))
2169 return false;
2170
2171 S.Stk.push<Pointer>(Args&: DestPtr);
2172 return true;
2173}
2174
2175/// Determine if T is a character type for which we guarantee that
2176/// sizeof(T) == 1.
2177static bool isOneByteCharacterType(QualType T) {
2178 return T->isCharType() || T->isChar8Type();
2179}
2180
2181// stdc_memreverse8(size_t N, unsigned char *P)
2182static bool interp__builtin_stdc_memreverse8(InterpState &S, CodePtr OpPC,
2183 const InterpFrame *Frame,
2184 const CallExpr *Call) {
2185 Pointer Ptr = S.Stk.pop<Pointer>();
2186
2187 uint64_t NElems;
2188 if (!popToUInt64(S, E: Call->getArg(Arg: 0), Out&: NElems))
2189 return false;
2190
2191 if (Ptr.isZero()) {
2192 S.FFDiag(SI: S.Current->getSource(PC: OpPC), DiagId: diag::note_constexpr_access_null)
2193 << AK_Assign;
2194 return false;
2195 }
2196
2197 if (!isReadable(P: Ptr) && !Ptr.isOnePastEnd())
2198 return false;
2199
2200 const Descriptor *Desc = Ptr.getFieldDesc();
2201 bool IsArray = Desc->isArray();
2202 QualType ElemTy = IsArray ? Desc->getElemQualType() : Desc->getType();
2203
2204 if (IsArray)
2205 Ptr = Ptr.expand();
2206
2207 uint64_t BaseIdx = Ptr.getIndex();
2208 uint64_t ArraySize = Ptr.getNumElems();
2209 uint64_t RemainingElems = ArraySize - BaseIdx;
2210 if (NElems > RemainingElems) {
2211 uint64_t LastIndex = llvm::SaturatingAdd(X: BaseIdx, Y: NElems - 1);
2212 if (IsArray)
2213 S.FFDiag(SI: S.Current->getSource(PC: OpPC), DiagId: diag::note_constexpr_array_index)
2214 << LastIndex << /*array*/ 0 << ArraySize;
2215 else
2216 S.FFDiag(SI: S.Current->getSource(PC: OpPC), DiagId: diag::note_constexpr_array_index)
2217 << LastIndex << /*non-array*/ 1;
2218 return false;
2219 }
2220
2221 if (NElems <= 1)
2222 return true;
2223
2224 PrimType ElemT = *S.getContext().classify(T: ElemTy);
2225
2226 for (uint64_t I = 0, Half = NElems / 2; I < Half; ++I) {
2227 Pointer LoPtr = Ptr.atIndex(Idx: BaseIdx + I);
2228 Pointer HiPtr = Ptr.atIndex(Idx: BaseIdx + NElems - 1 - I);
2229
2230 if (!CheckLoad(S, OpPC, Ptr: LoPtr, AK: AK_Read) ||
2231 !CheckLoad(S, OpPC, Ptr: HiPtr, AK: AK_Read) || !CheckStore(S, OpPC, Ptr: LoPtr) ||
2232 !CheckStore(S, OpPC, Ptr: HiPtr))
2233 return false;
2234
2235 INT_TYPE_SWITCH_NO_BOOL(ElemT,
2236 { std::swap(LoPtr.deref<T>(), HiPtr.deref<T>()); });
2237 LoPtr.initialize();
2238 HiPtr.initialize();
2239 }
2240 return true;
2241}
2242
2243static bool interp__builtin_load8(InterpState &S, CodePtr OpPC,
2244 const InterpFrame *Frame,
2245 const CallExpr *Call, bool IsBigEndian,
2246 bool IsAligned) {
2247 Pointer Ptr = S.Stk.pop<Pointer>();
2248
2249 if (Ptr.isZero()) {
2250 S.FFDiag(SI: S.Current->getSource(PC: OpPC), DiagId: diag::note_constexpr_access_null)
2251 << AK_Read;
2252 return false;
2253 }
2254
2255 if (!isReadable(P: Ptr) && !Ptr.isOnePastEnd())
2256 return false;
2257
2258 if (!Ptr.isReadablePointerType())
2259 return false;
2260
2261 if (IsAligned) {
2262 CharUnits RequiredAlign =
2263 S.getASTContext().getTypeAlignInChars(T: Call->getType());
2264 CharUnits BaseAlignment = getBaseAlignment(ASTCtx: S.getASTContext(), Ptr);
2265 std::optional<size_t> LayoutOffset =
2266 Ptr.computeLayoutOffset(ASTCtx: S.getASTContext());
2267 if (!LayoutOffset)
2268 return false;
2269 CharUnits PtrOffset = CharUnits::fromQuantity(Quantity: *LayoutOffset);
2270 CharUnits PtrAlign = BaseAlignment.alignmentAtOffset(offset: PtrOffset);
2271 if (PtrAlign < RequiredAlign) {
2272 S.FFDiag(SI: S.Current->getSource(PC: OpPC), DiagId: diag::note_constexpr_load8_unaligned)
2273 << S.getASTContext().BuiltinInfo.getQuotedName(
2274 ID: Call->getBuiltinCallee())
2275 << RequiredAlign.getQuantity() << PtrAlign.getQuantity();
2276 return false;
2277 }
2278 }
2279
2280 // A string pointer has no Descriptor; treat it as an array of its
2281 // character type.
2282 bool IsArray = Ptr.isStringPointer() || Ptr.getFieldDesc()->isArray();
2283 QualType ElemTy = getElemType(P: Ptr);
2284
2285 if (IsArray)
2286 Ptr = Ptr.expand();
2287
2288 uint64_t BaseIdx = Ptr.getIndex();
2289 uint64_t ArraySize = Ptr.getNumElems();
2290 uint64_t RemainingElems = ArraySize - BaseIdx;
2291
2292 unsigned ByteWidth = S.getASTContext().getTypeSize(T: Call->getType()) / 8;
2293 if (ByteWidth > RemainingElems) {
2294 uint64_t LastIndex = llvm::SaturatingAdd(X: BaseIdx, Y: uint64_t(ByteWidth - 1));
2295 S.FFDiag(SI: S.Current->getSource(PC: OpPC), DiagId: diag::note_constexpr_array_index)
2296 << LastIndex << /*array*/ !IsArray << ArraySize;
2297 return false;
2298 }
2299
2300 OptPrimType ElemT = S.getContext().classify(T: ElemTy);
2301 if (!ElemT || !isIntegerOrBoolType(T: *ElemT))
2302 return false;
2303 uint64_t Result = 0;
2304
2305 if (IsArray) {
2306 // C2y §7.18.21: result = sum(b_index * 2^(8*index)) for index in [0, N/8)
2307 // where b_index = ptr[index] (LE) or ptr[N/8 - index - 1] (BE).
2308 for (unsigned I = 0; I != ByteWidth; ++I) {
2309 size_t SrcIdx = IsBigEndian ? (ByteWidth - I - 1) : I;
2310 Pointer BytePtr = Ptr.atIndex(Idx: BaseIdx + SrcIdx);
2311 if (!CheckLoad(S, OpPC, Ptr: BytePtr, AK: AK_Read))
2312 return false;
2313 INT_TYPE_SWITCH(*ElemT, {
2314 Result |= BytePtr.load<T>().toAPSInt().getZExtValue() << (8 * I);
2315 });
2316 }
2317 } else {
2318 assert(ByteWidth == 1 && "non-array pointer, expected a 1-byte load");
2319 if (!CheckLoad(S, OpPC, Ptr, AK: AK_Read))
2320 return false;
2321 INT_TYPE_SWITCH(*ElemT,
2322 { Result = Ptr.load<T>().toAPSInt().getZExtValue(); });
2323 }
2324
2325 bool IsSigned = Call->getType()->isSignedIntegerType();
2326 pushInteger(S, Val: APSInt(APInt(ByteWidth * 8, Result), !IsSigned),
2327 QT: Call->getType());
2328 return true;
2329}
2330
2331static bool interp__builtin_memcmp(InterpState &S, CodePtr OpPC,
2332 const InterpFrame *Frame,
2333 const CallExpr *Call, unsigned ID) {
2334 assert(Call->getNumArgs() == 3);
2335 uint64_t Size;
2336 if (!popToUInt64(S, E: Call->getArg(Arg: 2), Out&: Size))
2337 return false;
2338 const Pointer &PtrB = S.Stk.pop<Pointer>();
2339 const Pointer &PtrA = S.Stk.pop<Pointer>();
2340
2341 if (ID == Builtin::BImemcmp || ID == Builtin::BIbcmp ||
2342 ID == Builtin::BIwmemcmp)
2343 diagnoseNonConstexprBuiltin(S, OpPC, ID);
2344
2345 if (Size == 0) {
2346 pushInteger(S, Val: 0, QT: Call->getType());
2347 return true;
2348 }
2349
2350 if (PtrA.isFunctionPointer() || PtrB.isFunctionPointer())
2351 return false;
2352
2353 bool IsWide =
2354 (ID == Builtin::BIwmemcmp || ID == Builtin::BI__builtin_wmemcmp);
2355
2356 const ASTContext &ASTCtx = S.getASTContext();
2357 QualType ElemTypeA = getElemType(P: PtrA);
2358 QualType ElemTypeB = getElemType(P: PtrB);
2359 // FIXME: This is an arbitrary limitation the current constant interpreter
2360 // had. We could remove this.
2361 if (!IsWide && (!isOneByteCharacterType(T: ElemTypeA) ||
2362 !isOneByteCharacterType(T: ElemTypeB))) {
2363 S.FFDiag(SI: S.Current->getSource(PC: OpPC),
2364 DiagId: diag::note_constexpr_memcmp_unsupported)
2365 << ASTCtx.BuiltinInfo.getQuotedName(ID) << PtrA.getType()
2366 << PtrB.getType();
2367 return false;
2368 }
2369
2370 if (!PtrA.isReadablePointerType() || !PtrB.isReadablePointerType())
2371 return false;
2372
2373 if (!CheckLoad(S, OpPC, Ptr: PtrA, AK: AK_Read) || !CheckLoad(S, OpPC, Ptr: PtrB, AK: AK_Read))
2374 return false;
2375
2376 // Now, read both pointers to a buffer and compare those.
2377 BitcastBuffer BufferA(
2378 Bits(ASTCtx.getTypeSize(T: ElemTypeA) * PtrA.getNumElems()));
2379 readPointerToBuffer(Ctx: S.getContext(), FromPtr: PtrA, Buffer&: BufferA, /*ReturnOnUninit=*/false);
2380
2381 // FIXME: The swapping here is UNDOING something we do when reading the
2382 // data into the buffer.
2383 if (ASTCtx.getTargetInfo().isBigEndian())
2384 swapBytes(M: BufferA.Data.get(), N: BufferA.byteSize().getQuantity());
2385
2386 BitcastBuffer BufferB(
2387 Bits(ASTCtx.getTypeSize(T: ElemTypeB) * PtrB.getNumElems()));
2388 readPointerToBuffer(Ctx: S.getContext(), FromPtr: PtrB, Buffer&: BufferB, /*ReturnOnUninit=*/false);
2389 // FIXME: The swapping here is UNDOING something we do when reading the
2390 // data into the buffer.
2391 if (ASTCtx.getTargetInfo().isBigEndian())
2392 swapBytes(M: BufferB.Data.get(), N: BufferB.byteSize().getQuantity());
2393
2394 size_t MinBufferSize = std::min(a: BufferA.byteSize().getQuantity(),
2395 b: BufferB.byteSize().getQuantity());
2396
2397 unsigned ElemSize = 1;
2398 if (IsWide)
2399 ElemSize = ASTCtx.getTypeSizeInChars(T: ASTCtx.getWCharType()).getQuantity();
2400 // The Size given for the wide variants is in wide-char units. Convert it
2401 // to bytes.
2402 size_t ByteSize = Size * ElemSize;
2403 size_t CmpSize = std::min(a: MinBufferSize, b: ByteSize);
2404
2405 for (size_t I = 0; I != CmpSize; I += ElemSize) {
2406 if (IsWide) {
2407 FIXED_SIZE_INT_TYPE_SWITCH(
2408 *S.getContext().classify(ASTCtx.getWCharType()), {
2409 T A = T::bitcastFromMemory(BufferA.atByte(I), T::bitWidth());
2410 T B = T::bitcastFromMemory(BufferB.atByte(I), T::bitWidth());
2411 if (A < B) {
2412 pushInteger(S, -1, Call->getType());
2413 return true;
2414 }
2415 if (A > B) {
2416 pushInteger(S, 1, Call->getType());
2417 return true;
2418 }
2419 });
2420 } else {
2421 auto A = BufferA.deref<std::byte>(Offset: Bytes(I));
2422 auto B = BufferB.deref<std::byte>(Offset: Bytes(I));
2423
2424 if (A < B) {
2425 pushInteger(S, Val: -1, QT: Call->getType());
2426 return true;
2427 }
2428 if (A > B) {
2429 pushInteger(S, Val: 1, QT: Call->getType());
2430 return true;
2431 }
2432 }
2433 }
2434
2435 // We compared CmpSize bytes above. If the limiting factor was the Size
2436 // passed, we're done and the result is equality (0).
2437 if (ByteSize <= CmpSize) {
2438 pushInteger(S, Val: 0, QT: Call->getType());
2439 return true;
2440 }
2441
2442 // However, if we read all the available bytes but were instructed to read
2443 // even more, diagnose this as a "read of dereferenced one-past-the-end
2444 // pointer". This is what would happen if we called CheckLoad() on every array
2445 // element.
2446 S.FFDiag(SI: S.Current->getSource(PC: OpPC), DiagId: diag::note_constexpr_access_past_end)
2447 << AK_Read << S.Current->getRange(PC: OpPC);
2448 return false;
2449}
2450
2451// __builtin_memchr(ptr, int, int)
2452// __builtin_strchr(ptr, int)
2453static bool interp__builtin_memchr(InterpState &S, CodePtr OpPC,
2454 const CallExpr *Call, unsigned ID) {
2455 if (ID == Builtin::BImemchr || ID == Builtin::BIwcschr ||
2456 ID == Builtin::BIstrchr || ID == Builtin::BIwmemchr)
2457 diagnoseNonConstexprBuiltin(S, OpPC, ID);
2458
2459 std::optional<APSInt> MaxLength;
2460 if (Call->getNumArgs() == 3) {
2461 APSInt MaxLengthVal;
2462 if (!popToAPSInt(S, E: Call->getArg(Arg: 2), Out&: MaxLengthVal))
2463 return false;
2464 MaxLength = MaxLengthVal;
2465 }
2466
2467 APSInt Desired;
2468 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: Desired))
2469 return false;
2470 const Pointer &Ptr = S.Stk.pop<Pointer>();
2471
2472 if (MaxLength && MaxLength->isZero()) {
2473 S.Stk.push<Pointer>();
2474 return true;
2475 }
2476
2477 if (Ptr.isDummy()) {
2478 if (Ptr.getType()->isIncompleteType())
2479 S.FFDiag(SI: S.Current->getSource(PC: OpPC),
2480 DiagId: diag::note_constexpr_ltor_incomplete_type)
2481 << Ptr.getType();
2482 return false;
2483 }
2484
2485 // Null is only okay if the given size is 0.
2486 if (Ptr.isZero()) {
2487 S.FFDiag(SI: S.Current->getSource(PC: OpPC), DiagId: diag::note_constexpr_access_null)
2488 << AK_Read;
2489 return false;
2490 }
2491
2492 if (!Ptr.isReadablePointerType())
2493 return false;
2494
2495 QualType ElemTy = getElemType(P: Ptr);
2496 bool IsRawByte = ID == Builtin::BImemchr || ID == Builtin::BI__builtin_memchr;
2497
2498 // Give up on byte-oriented matching against multibyte elements.
2499 if (IsRawByte && !isOneByteCharacterType(T: ElemTy)) {
2500 S.FFDiag(SI: S.Current->getSource(PC: OpPC),
2501 DiagId: diag::note_constexpr_memchr_unsupported)
2502 << S.getASTContext().BuiltinInfo.getQuotedName(ID) << ElemTy;
2503 return false;
2504 }
2505
2506 if (!isReadable(P: Ptr))
2507 return false;
2508
2509 uint64_t DesiredVal;
2510 if (ID == Builtin::BIwmemchr || ID == Builtin::BI__builtin_wmemchr ||
2511 ID == Builtin::BIwcschr || ID == Builtin::BI__builtin_wcschr) {
2512 // wcschr and wmemchr are given a wchar_t to look for. Just use it.
2513 DesiredVal = Desired.getZExtValue();
2514 } else {
2515 DesiredVal = Desired.trunc(width: S.getASTContext().getCharWidth()).getZExtValue();
2516 }
2517
2518 bool StopAtZero =
2519 (ID == Builtin::BIstrchr || ID == Builtin::BI__builtin_strchr ||
2520 ID == Builtin::BIwcschr || ID == Builtin::BI__builtin_wcschr);
2521
2522 PrimType ElemT =
2523 IsRawByte ? PT_Sint8 : *S.getContext().classify(T: getElemType(P: Ptr));
2524
2525 size_t Index = Ptr.getIndex();
2526 size_t Step = 0;
2527 for (;;) {
2528 const Pointer &ElemPtr =
2529 (Index + Step) > 0 ? Ptr.atIndex(Idx: Index + Step) : Ptr;
2530
2531 if (!CheckLoad(S, OpPC, Ptr: ElemPtr))
2532 return false;
2533
2534 uint64_t V;
2535 INT_TYPE_SWITCH_NO_BOOL(
2536 ElemT, { V = static_cast<uint64_t>(ElemPtr.load<T>().toUnsigned()); });
2537
2538 if (V == DesiredVal) {
2539 S.Stk.push<Pointer>(Args: ElemPtr);
2540 return true;
2541 }
2542
2543 if (StopAtZero && V == 0)
2544 break;
2545
2546 ++Step;
2547 if (MaxLength && Step == MaxLength->getZExtValue())
2548 break;
2549 }
2550
2551 S.Stk.push<Pointer>();
2552 return true;
2553}
2554
2555static bool interp__builtin_object_size(InterpState &S, CodePtr OpPC,
2556 const InterpFrame *Frame,
2557 const CallExpr *Call, bool IsDynamic) {
2558 const ASTContext &ASTCtx = S.getASTContext();
2559 // From the GCC docs:
2560 // Kind is an integer constant from 0 to 3. If the least significant bit is
2561 // clear, objects are whole variables. If it is set, a closest surrounding
2562 // subobject is considered the object a pointer points to. The second bit
2563 // determines if maximum or minimum of remaining bytes is computed.
2564 APSInt Kind;
2565 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: Kind))
2566 return false;
2567 if (Kind < 0 || Kind > 3) {
2568 S.FFDiag(E: Call->getArg(Arg: 1), DiagId: diag::note_constexpr_builtin_arg_out_of_range)
2569 << Kind << 0 << 3;
2570 return false;
2571 }
2572 unsigned KindValue = Kind.getZExtValue();
2573 Pointer Ptr = S.Stk.pop<Pointer>();
2574
2575 if (auto Result = evaluateBuiltinObjectSize(ASTCtx, Kind: KindValue, Ptr,
2576 E: Call->getArg(Arg: 0), IsDynamic)) {
2577 pushInteger(S, Val: *Result, QT: Call->getType());
2578 return true;
2579 }
2580
2581 if (Call->getArg(Arg: 0)->HasSideEffects(Ctx: ASTCtx)) {
2582 // "If there are any side effects in them, it returns (size_t) -1
2583 // for type 0 or 1 and (size_t) 0 for type 2 or 3."
2584 pushInteger(S, Val: KindValue <= 1 ? (size_t)-1 : (size_t)0, QT: Call->getType());
2585 return true;
2586 }
2587
2588 switch (S.EvalMode) {
2589 case EvaluationMode::ConstantExpression:
2590 case EvaluationMode::ConstantFold:
2591 case EvaluationMode::IgnoreSideEffects:
2592 // Leave it to IR generation.
2593 return Invalid(S, OpPC);
2594 case EvaluationMode::ConstantExpressionUnevaluated:
2595 // Reduce it to a constant now.
2596 pushInteger(S, Val: ((KindValue & 2u) ? (size_t)0 : (size_t)-1),
2597 QT: Call->getType());
2598 return true;
2599 }
2600
2601 return false;
2602}
2603
2604static bool interp__builtin_is_within_lifetime(InterpState &S, CodePtr OpPC,
2605 const CallExpr *Call) {
2606
2607 if (!S.inConstantContext())
2608 return false;
2609
2610 const Pointer &Ptr = S.Stk.pop<Pointer>();
2611
2612 auto Error = [&](int Diag) {
2613 bool CalledFromStd = false;
2614 const auto *Callee = S.Current->getCallee();
2615 if (Callee && Callee->isInStdNamespace()) {
2616 const IdentifierInfo *Identifier = Callee->getIdentifier();
2617 CalledFromStd = Identifier && Identifier->isStr(Str: "is_within_lifetime");
2618 }
2619 S.CCEDiag(SI: CalledFromStd
2620 ? S.Current->Caller->getSource(PC: S.Current->getRetPC())
2621 : S.Current->getSource(PC: OpPC),
2622 DiagId: diag::err_invalid_is_within_lifetime)
2623 << (CalledFromStd ? "std::is_within_lifetime"
2624 : "__builtin_is_within_lifetime")
2625 << Diag;
2626 return false;
2627 };
2628
2629 if (Ptr.isZero())
2630 return Error(0);
2631 if (Ptr.isOnePastEnd())
2632 return Error(1);
2633
2634 bool Result = Ptr.getLifetime() != Lifetime::Ended;
2635 if (!Ptr.isActive()) {
2636 Result = false;
2637 } else {
2638 if (!CheckLive(S, OpPC, Ptr, AK: AK_Read))
2639 return false;
2640 if (!CheckMutable(S, OpPC, Ptr))
2641 return false;
2642 if (!CheckDummy(S, OpPC, Ptr, AK: AK_Read))
2643 return false;
2644 }
2645
2646 if (!Ptr.isBlockPointer())
2647 return diagnoseUnreadableObject(S, OpPC, Ptr, AK: AK_Read);
2648
2649 // Check if we're currently running an initializer.
2650 if (S.initializingBlock(B: Ptr.block()))
2651 return Error(2);
2652 if (S.EvaluatingDecl && Ptr.getRootVarDecl() == S.EvaluatingDecl)
2653 return Error(2);
2654
2655 pushInteger(S, Val: Result, QT: Call->getType());
2656 return true;
2657}
2658
2659static bool interp__builtin_elementwise_int_unaryop(
2660 InterpState &S, CodePtr OpPC, const CallExpr *Call,
2661 llvm::function_ref<APInt(const APSInt &)> Fn) {
2662 assert(Call->getNumArgs() == 1);
2663
2664 // Single integer case.
2665 if (!Call->getArg(Arg: 0)->getType()->isVectorType()) {
2666 assert(Call->getType()->isIntegerType());
2667 APSInt Src;
2668 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: Src))
2669 return false;
2670 APInt Result = Fn(Src);
2671 pushInteger(S, Val: APSInt(std::move(Result), !Src.isSigned()), QT: Call->getType());
2672 return true;
2673 }
2674
2675 // Vector case.
2676 const Pointer &Arg = S.Stk.pop<Pointer>();
2677 assert(Arg.getFieldDesc()->isPrimitiveArray());
2678 const Pointer &Dst = S.Stk.peek<Pointer>();
2679 assert(Dst.getFieldDesc()->isPrimitiveArray());
2680 assert(Arg.getFieldDesc()->getNumElems() ==
2681 Dst.getFieldDesc()->getNumElems());
2682
2683 QualType ElemType = Arg.getFieldDesc()->getElemQualType();
2684 PrimType ElemT = *S.getContext().classify(T: ElemType);
2685 unsigned NumElems = Arg.getNumElems();
2686 bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
2687
2688 for (unsigned I = 0; I != NumElems; ++I) {
2689 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
2690 APSInt Src = Arg.elem<T>(I).toAPSInt();
2691 APInt Result = Fn(Src);
2692 Dst.elem<T>(I) = static_cast<T>(APSInt(std::move(Result), DestUnsigned));
2693 });
2694 }
2695 Dst.initializeAllElements();
2696
2697 return true;
2698}
2699
2700static bool interp__builtin_elementwise_fp_binop(
2701 InterpState &S, CodePtr OpPC, const CallExpr *Call,
2702 llvm::function_ref<std::optional<APFloat>(
2703 const APFloat &, const APFloat &, std::optional<APSInt> RoundingMode)>
2704 Fn,
2705 bool IsScalar = false) {
2706 assert((Call->getNumArgs() == 2) || (Call->getNumArgs() == 3));
2707 const auto *VT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
2708 assert(VT->getElementType()->isFloatingType());
2709 unsigned NumElems = VT->getNumElements();
2710
2711 // Vector case.
2712 assert(Call->getArg(0)->getType()->isVectorType() &&
2713 Call->getArg(1)->getType()->isVectorType());
2714 assert(VT->getElementType() ==
2715 Call->getArg(1)->getType()->castAs<VectorType>()->getElementType());
2716 assert(VT->getNumElements() ==
2717 Call->getArg(1)->getType()->castAs<VectorType>()->getNumElements());
2718
2719 std::optional<APSInt> RoundingMode = std::nullopt;
2720 if (Call->getNumArgs() == 3) {
2721 APSInt RoundingModeVal;
2722 if (!popToAPSInt(S, E: Call->getArg(Arg: 2), Out&: RoundingModeVal))
2723 return false;
2724 RoundingMode = RoundingModeVal;
2725 }
2726
2727 const Pointer &BPtr = S.Stk.pop<Pointer>();
2728 const Pointer &APtr = S.Stk.pop<Pointer>();
2729 const Pointer &Dst = S.Stk.peek<Pointer>();
2730 for (unsigned ElemIdx = 0; ElemIdx != NumElems; ++ElemIdx) {
2731 using T = PrimConv<PT_Float>::T;
2732 if (IsScalar && ElemIdx > 0) {
2733 Dst.elem<T>(I: ElemIdx) = APtr.elem<T>(I: ElemIdx);
2734 continue;
2735 }
2736 APFloat ElemA = APtr.elem<T>(I: ElemIdx).getAPFloat();
2737 APFloat ElemB = BPtr.elem<T>(I: ElemIdx).getAPFloat();
2738 std::optional<APFloat> Result = Fn(ElemA, ElemB, RoundingMode);
2739 if (!Result)
2740 return false;
2741 Dst.elem<T>(I: ElemIdx) = static_cast<T>(*Result);
2742 }
2743
2744 Dst.initializeAllElements();
2745
2746 return true;
2747}
2748
2749static bool interp__builtin_scalar_fp_round_mask_binop(
2750 InterpState &S, CodePtr OpPC, const CallExpr *Call,
2751 llvm::function_ref<std::optional<APFloat>(const APFloat &, const APFloat &,
2752 std::optional<APSInt>)>
2753 Fn) {
2754 assert(Call->getNumArgs() == 5);
2755 const auto *VT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
2756 unsigned NumElems = VT->getNumElements();
2757
2758 APSInt RoundingMode;
2759 if (!popToAPSInt(S, E: Call->getArg(Arg: 4), Out&: RoundingMode))
2760 return false;
2761 uint64_t MaskVal;
2762 if (!popToUInt64(S, E: Call->getArg(Arg: 3), Out&: MaskVal))
2763 return false;
2764 const Pointer &SrcPtr = S.Stk.pop<Pointer>();
2765 const Pointer &BPtr = S.Stk.pop<Pointer>();
2766 const Pointer &APtr = S.Stk.pop<Pointer>();
2767 const Pointer &Dst = S.Stk.peek<Pointer>();
2768
2769 using T = PrimConv<PT_Float>::T;
2770
2771 if (MaskVal & 1) {
2772 APFloat ElemA = APtr.elem<T>(I: 0).getAPFloat();
2773 APFloat ElemB = BPtr.elem<T>(I: 0).getAPFloat();
2774 std::optional<APFloat> Result = Fn(ElemA, ElemB, RoundingMode);
2775 if (!Result)
2776 return false;
2777 Dst.elem<T>(I: 0) = static_cast<T>(*Result);
2778 } else {
2779 Dst.elem<T>(I: 0) = SrcPtr.elem<T>(I: 0);
2780 }
2781
2782 for (unsigned I = 1; I < NumElems; ++I)
2783 Dst.elem<T>(I) = APtr.elem<T>(I);
2784
2785 Dst.initializeAllElements();
2786
2787 return true;
2788}
2789
2790static bool interp__builtin_elementwise_int_binop(
2791 InterpState &S, CodePtr OpPC, const CallExpr *Call,
2792 llvm::function_ref<APInt(const APSInt &, const APSInt &)> Fn) {
2793 assert(Call->getNumArgs() == 2);
2794
2795 // Single integer case.
2796 if (!Call->getArg(Arg: 0)->getType()->isVectorType()) {
2797 assert(!Call->getArg(1)->getType()->isVectorType());
2798 APSInt RHS;
2799 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: RHS))
2800 return false;
2801 APSInt LHS;
2802 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: LHS))
2803 return false;
2804 APInt Result = Fn(LHS, RHS);
2805 pushInteger(S, Val: APSInt(std::move(Result), !LHS.isSigned()), QT: Call->getType());
2806 return true;
2807 }
2808
2809 const auto *VT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
2810 assert(VT->getElementType()->isIntegralOrEnumerationType());
2811 PrimType ElemT = *S.getContext().classify(T: VT->getElementType());
2812 unsigned NumElems = VT->getNumElements();
2813 bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
2814
2815 // Vector + Scalar case.
2816 if (!Call->getArg(Arg: 1)->getType()->isVectorType()) {
2817 assert(Call->getArg(1)->getType()->isIntegralOrEnumerationType());
2818
2819 APSInt RHS;
2820 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: RHS))
2821 return false;
2822 const Pointer &LHS = S.Stk.pop<Pointer>();
2823 const Pointer &Dst = S.Stk.peek<Pointer>();
2824
2825 for (unsigned I = 0; I != NumElems; ++I) {
2826 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
2827 Dst.elem<T>(I) = static_cast<T>(
2828 APSInt(Fn(LHS.elem<T>(I).toAPSInt(), RHS), DestUnsigned));
2829 });
2830 }
2831 Dst.initializeAllElements();
2832 return true;
2833 }
2834
2835 // Vector case.
2836 assert(Call->getArg(0)->getType()->isVectorType() &&
2837 Call->getArg(1)->getType()->isVectorType());
2838 assert(VT->getElementType() ==
2839 Call->getArg(1)->getType()->castAs<VectorType>()->getElementType());
2840 assert(VT->getNumElements() ==
2841 Call->getArg(1)->getType()->castAs<VectorType>()->getNumElements());
2842 assert(VT->getElementType()->isIntegralOrEnumerationType());
2843
2844 const Pointer &RHS = S.Stk.pop<Pointer>();
2845 const Pointer &LHS = S.Stk.pop<Pointer>();
2846 const Pointer &Dst = S.Stk.peek<Pointer>();
2847 for (unsigned I = 0; I != NumElems; ++I) {
2848 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
2849 APSInt Elem1 = LHS.elem<T>(I).toAPSInt();
2850 APSInt Elem2 = RHS.elem<T>(I).toAPSInt();
2851 Dst.elem<T>(I) = static_cast<T>(APSInt(Fn(Elem1, Elem2), DestUnsigned));
2852 });
2853 }
2854 Dst.initializeAllElements();
2855
2856 return true;
2857}
2858
2859static bool
2860interp__builtin_ia32_pack(InterpState &S, CodePtr, const CallExpr *E,
2861 llvm::function_ref<APInt(const APSInt &)> PackFn) {
2862 const auto *VT0 = E->getArg(Arg: 0)->getType()->castAs<VectorType>();
2863 [[maybe_unused]] const auto *VT1 =
2864 E->getArg(Arg: 1)->getType()->castAs<VectorType>();
2865 assert(VT0 && VT1 && "pack builtin VT0 and VT1 must be VectorType");
2866 assert(VT0->getElementType() == VT1->getElementType() &&
2867 VT0->getNumElements() == VT1->getNumElements() &&
2868 "pack builtin VT0 and VT1 ElementType must be same");
2869
2870 const Pointer &RHS = S.Stk.pop<Pointer>();
2871 const Pointer &LHS = S.Stk.pop<Pointer>();
2872 const Pointer &Dst = S.Stk.peek<Pointer>();
2873
2874 const ASTContext &ASTCtx = S.getASTContext();
2875 unsigned SrcBits = ASTCtx.getIntWidth(T: VT0->getElementType());
2876 unsigned LHSVecLen = VT0->getNumElements();
2877 unsigned SrcPerLane = 128 / SrcBits;
2878 unsigned Lanes = LHSVecLen * SrcBits / 128;
2879
2880 PrimType SrcT = *S.getContext().classify(T: VT0->getElementType());
2881 PrimType DstT = *S.getContext().classify(T: getElemType(P: Dst));
2882 bool IsUnsigend = getElemType(P: Dst)->isUnsignedIntegerType();
2883
2884 for (unsigned Lane = 0; Lane != Lanes; ++Lane) {
2885 unsigned BaseSrc = Lane * SrcPerLane;
2886 unsigned BaseDst = Lane * (2 * SrcPerLane);
2887
2888 for (unsigned I = 0; I != SrcPerLane; ++I) {
2889 INT_TYPE_SWITCH_NO_BOOL(SrcT, {
2890 APSInt A = LHS.elem<T>(BaseSrc + I).toAPSInt();
2891 APSInt B = RHS.elem<T>(BaseSrc + I).toAPSInt();
2892
2893 assignIntegral(S, Dst.atIndex(BaseDst + I), DstT,
2894 APSInt(PackFn(A), IsUnsigend));
2895 assignIntegral(S, Dst.atIndex(BaseDst + SrcPerLane + I), DstT,
2896 APSInt(PackFn(B), IsUnsigend));
2897 });
2898 }
2899 }
2900
2901 Dst.initializeAllElements();
2902 return true;
2903}
2904
2905static bool interp__builtin_elementwise_maxmin(InterpState &S, CodePtr OpPC,
2906 const CallExpr *Call,
2907 unsigned BuiltinID) {
2908 assert(Call->getNumArgs() == 2);
2909
2910 QualType Arg0Type = Call->getArg(Arg: 0)->getType();
2911
2912 // TODO: Support floating-point types.
2913 if (!(Arg0Type->isIntegerType() ||
2914 (Arg0Type->isVectorType() &&
2915 Arg0Type->castAs<VectorType>()->getElementType()->isIntegerType())))
2916 return false;
2917
2918 if (!Arg0Type->isVectorType()) {
2919 assert(!Call->getArg(1)->getType()->isVectorType());
2920 APSInt RHS;
2921 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: RHS))
2922 return false;
2923 APSInt LHS;
2924 if (!popToAPSInt(S, T: Arg0Type, Out&: LHS))
2925 return false;
2926 APInt Result;
2927 if (BuiltinID == Builtin::BI__builtin_elementwise_max) {
2928 Result = std::max(a: LHS, b: RHS);
2929 } else if (BuiltinID == Builtin::BI__builtin_elementwise_min) {
2930 Result = std::min(a: LHS, b: RHS);
2931 } else {
2932 llvm_unreachable("Wrong builtin ID");
2933 }
2934
2935 pushInteger(S, Val: APSInt(Result, !LHS.isSigned()), QT: Call->getType());
2936 return true;
2937 }
2938
2939 // Vector case.
2940 assert(Call->getArg(0)->getType()->isVectorType() &&
2941 Call->getArg(1)->getType()->isVectorType());
2942 const auto *VT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
2943 assert(VT->getElementType() ==
2944 Call->getArg(1)->getType()->castAs<VectorType>()->getElementType());
2945 assert(VT->getNumElements() ==
2946 Call->getArg(1)->getType()->castAs<VectorType>()->getNumElements());
2947 assert(VT->getElementType()->isIntegralOrEnumerationType());
2948
2949 const Pointer &RHS = S.Stk.pop<Pointer>();
2950 const Pointer &LHS = S.Stk.pop<Pointer>();
2951 const Pointer &Dst = S.Stk.peek<Pointer>();
2952 PrimType ElemT = *S.getContext().classify(T: VT->getElementType());
2953 unsigned NumElems = VT->getNumElements();
2954 for (unsigned I = 0; I != NumElems; ++I) {
2955 APSInt Elem1;
2956 APSInt Elem2;
2957 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
2958 Elem1 = LHS.elem<T>(I).toAPSInt();
2959 Elem2 = RHS.elem<T>(I).toAPSInt();
2960 });
2961
2962 APSInt Result;
2963 if (BuiltinID == Builtin::BI__builtin_elementwise_max) {
2964 Result = APSInt(std::max(a: Elem1, b: Elem2),
2965 Call->getType()->isUnsignedIntegerOrEnumerationType());
2966 } else if (BuiltinID == Builtin::BI__builtin_elementwise_min) {
2967 Result = APSInt(std::min(a: Elem1, b: Elem2),
2968 Call->getType()->isUnsignedIntegerOrEnumerationType());
2969 } else {
2970 llvm_unreachable("Wrong builtin ID");
2971 }
2972
2973 INT_TYPE_SWITCH_NO_BOOL(ElemT,
2974 { Dst.elem<T>(I) = static_cast<T>(Result); });
2975 }
2976 Dst.initializeAllElements();
2977
2978 return true;
2979}
2980
2981static bool interp__builtin_ia32_pmul(
2982 InterpState &S, CodePtr OpPC, const CallExpr *Call,
2983 llvm::function_ref<APInt(const APSInt &, const APSInt &, const APSInt &,
2984 const APSInt &)>
2985 Fn) {
2986 assert(Call->getArg(0)->getType()->isVectorType() &&
2987 Call->getArg(1)->getType()->isVectorType());
2988 const Pointer &RHS = S.Stk.pop<Pointer>();
2989 const Pointer &LHS = S.Stk.pop<Pointer>();
2990 const Pointer &Dst = S.Stk.peek<Pointer>();
2991
2992 const auto *VT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
2993 PrimType ElemT = *S.getContext().classify(T: VT->getElementType());
2994 unsigned NumElems = VT->getNumElements();
2995 const auto *DestVT = Call->getType()->castAs<VectorType>();
2996 PrimType DestElemT = *S.getContext().classify(T: DestVT->getElementType());
2997 bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
2998
2999 unsigned DstElem = 0;
3000 for (unsigned I = 0; I != NumElems; I += 2) {
3001 APSInt Result;
3002 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
3003 APSInt LoLHS = LHS.elem<T>(I).toAPSInt();
3004 APSInt HiLHS = LHS.elem<T>(I + 1).toAPSInt();
3005 APSInt LoRHS = RHS.elem<T>(I).toAPSInt();
3006 APSInt HiRHS = RHS.elem<T>(I + 1).toAPSInt();
3007 Result = APSInt(Fn(LoLHS, HiLHS, LoRHS, HiRHS), DestUnsigned);
3008 });
3009
3010 INT_TYPE_SWITCH_NO_BOOL(DestElemT,
3011 { Dst.elem<T>(DstElem) = static_cast<T>(Result); });
3012 ++DstElem;
3013 }
3014
3015 Dst.initializeAllElements();
3016 return true;
3017}
3018
3019static bool interp__builtin_ia32_psadbw(InterpState &S, CodePtr OpPC,
3020 const CallExpr *Call) {
3021 assert(Call->getNumArgs() == 2);
3022
3023 const Pointer &RHS = S.Stk.pop<Pointer>();
3024 const Pointer &LHS = S.Stk.pop<Pointer>();
3025 const Pointer &Dst = S.Stk.peek<Pointer>();
3026
3027 const auto *SrcVT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
3028 PrimType SrcElemT = *S.getContext().classify(T: SrcVT->getElementType());
3029 unsigned SourceLen = SrcVT->getNumElements();
3030 assert((SourceLen % 8) == 0);
3031
3032 const auto *DestVT = Call->getType()->castAs<VectorType>();
3033 PrimType DestElemT = *S.getContext().classify(T: DestVT->getElementType());
3034 bool DestUnsigned =
3035 DestVT->getElementType()->isUnsignedIntegerOrEnumerationType();
3036
3037 unsigned DstElem = 0;
3038 for (unsigned Lane = 0; Lane != SourceLen; Lane += 8) {
3039 APInt Sum(64, 0);
3040 for (unsigned I = 0; I != 8; ++I) {
3041 INT_TYPE_SWITCH_NO_BOOL(SrcElemT, {
3042 APSInt L = LHS.elem<T>(Lane + I).toAPSInt();
3043 APSInt R = RHS.elem<T>(Lane + I).toAPSInt();
3044 Sum += llvm::APIntOps::abdu(L.extOrTrunc(8), R.extOrTrunc(8)).zext(64);
3045 });
3046 }
3047
3048 INT_TYPE_SWITCH_NO_BOOL(DestElemT, {
3049 Dst.elem<T>(DstElem) = static_cast<T>(APSInt(Sum, DestUnsigned));
3050 });
3051 ++DstElem;
3052 }
3053
3054 Dst.initializeAllElements();
3055 return true;
3056}
3057
3058static bool interp__builtin_ia32_dbpsadbw(InterpState &S, CodePtr OpPC,
3059 const CallExpr *Call) {
3060 assert(Call->getNumArgs() == 3);
3061 uint64_t Imm;
3062 if (!popToUInt64(S, E: Call->getArg(Arg: 2), Out&: Imm))
3063 return false;
3064
3065 const Pointer &Src2 = S.Stk.pop<Pointer>();
3066 const Pointer &Src1 = S.Stk.pop<Pointer>();
3067 const Pointer &Dst = S.Stk.peek<Pointer>();
3068
3069 const auto *SrcVT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
3070 PrimType SrcElemT = *S.getContext().classify(T: SrcVT->getElementType());
3071 unsigned SourceLen = SrcVT->getNumElements();
3072
3073 const auto *DestVT = Call->getType()->castAs<VectorType>();
3074 PrimType DestElemT = *S.getContext().classify(T: DestVT->getElementType());
3075 bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
3076
3077 constexpr unsigned LaneSize = 16; // 128-bit lane = 16 bytes
3078
3079 // Phase 1: Shuffle Src2 using all four 2-bit fields of imm8.
3080 // Within each 128-bit lane, for group j (0..3), select a 4-byte block
3081 // from Src2 based on bits [2*j+1:2*j] of imm8.
3082 SmallVector<uint8_t, 64> Shuffled(SourceLen);
3083 for (unsigned I = 0; I < SourceLen; I += LaneSize) {
3084 for (unsigned J = 0; J < 4; ++J) {
3085 unsigned Part = (Imm >> (2 * J)) & 3;
3086 for (unsigned K = 0; K < 4; ++K) {
3087 INT_TYPE_SWITCH_NO_BOOL(SrcElemT, {
3088 Shuffled[I + 4 * J + K] =
3089 static_cast<uint8_t>(Src2.elem<T>(I + 4 * Part + K));
3090 });
3091 }
3092 }
3093 }
3094
3095 // Phase 2: Sliding SAD computation.
3096 // For every group of 4 output u16 values, compute absolute differences
3097 // using overlapping windows into Src1 and the shuffled array.
3098 unsigned Size = SourceLen / 2; // number of output u16 elements
3099 for (unsigned I = 0; I < Size; I += 4) {
3100 unsigned Sad[4] = {0, 0, 0, 0};
3101 for (unsigned J = 0; J < 4; ++J) {
3102 uint8_t A1, A2;
3103 INT_TYPE_SWITCH_NO_BOOL(SrcElemT, {
3104 A1 = static_cast<uint8_t>(Src1.elem<T>(2 * I + J));
3105 A2 = static_cast<uint8_t>(Src1.elem<T>(2 * I + J + 4));
3106 });
3107 uint8_t B0 = Shuffled[2 * I + J];
3108 uint8_t B1 = Shuffled[2 * I + J + 1];
3109 uint8_t B2 = Shuffled[2 * I + J + 2];
3110 uint8_t B3 = Shuffled[2 * I + J + 3];
3111 Sad[0] += (A1 > B0) ? (A1 - B0) : (B0 - A1);
3112 Sad[1] += (A1 > B1) ? (A1 - B1) : (B1 - A1);
3113 Sad[2] += (A2 > B2) ? (A2 - B2) : (B2 - A2);
3114 Sad[3] += (A2 > B3) ? (A2 - B3) : (B3 - A2);
3115 }
3116 for (unsigned R = 0; R < 4; ++R) {
3117 INT_TYPE_SWITCH_NO_BOOL(DestElemT, {
3118 Dst.elem<T>(I + R) =
3119 static_cast<T>(APSInt(APInt(16, Sad[R]), DestUnsigned));
3120 });
3121 }
3122 }
3123
3124 Dst.initializeAllElements();
3125 return true;
3126}
3127
3128static bool interp__builtin_ia32_mpsadbw(InterpState &S, CodePtr OpPC,
3129 const CallExpr *Call) {
3130 assert(Call->getNumArgs() == 3);
3131 uint64_t Imm;
3132 if (!popToUInt64(S, E: Call->getArg(Arg: 2), Out&: Imm))
3133 return false;
3134
3135 const Pointer &Src2 = S.Stk.pop<Pointer>();
3136 const Pointer &Src1 = S.Stk.pop<Pointer>();
3137 const Pointer &Dst = S.Stk.peek<Pointer>();
3138
3139 const auto *SrcVT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
3140 PrimType SrcElemT = *S.getContext().classify(T: SrcVT->getElementType());
3141 unsigned SourceLen = SrcVT->getNumElements();
3142 assert((SourceLen == 16 || SourceLen == 32) &&
3143 "MPSADBW operates on 128-bit or 256-bit vectors");
3144
3145 const auto *DestVT = Call->getType()->castAs<VectorType>();
3146 PrimType DestElemT = *S.getContext().classify(T: DestVT->getElementType());
3147 bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
3148
3149 constexpr unsigned LaneSize = 16; // 128-bit lane = 16 bytes
3150 unsigned NumLanes = SourceLen / LaneSize;
3151
3152 for (unsigned Lane = 0; Lane != NumLanes; ++Lane) {
3153 unsigned Ctrl = (Imm >> (3 * Lane)) & 0x7;
3154 unsigned AOff = ((Ctrl >> 2) & 1) * 4;
3155 unsigned BOff = (Ctrl & 3) * 4;
3156 for (unsigned J = 0; J != 8; ++J) {
3157 uint16_t Sad = 0;
3158 for (unsigned K = 0; K != 4; ++K) {
3159 uint8_t A, B;
3160 INT_TYPE_SWITCH_NO_BOOL(SrcElemT, {
3161 A = static_cast<uint8_t>(
3162 Src1.elem<T>(Lane * LaneSize + AOff + J + K));
3163 B = static_cast<uint8_t>(Src2.elem<T>(Lane * LaneSize + BOff + K));
3164 });
3165 Sad += (A > B) ? (A - B) : (B - A);
3166 }
3167 INT_TYPE_SWITCH_NO_BOOL(DestElemT, {
3168 Dst.elem<T>(Lane * 8 + J) =
3169 static_cast<T>(APSInt(APInt(16, Sad), DestUnsigned));
3170 });
3171 }
3172 }
3173
3174 Dst.initializeAllElements();
3175 return true;
3176}
3177
3178static bool interp_builtin_horizontal_int_binop(
3179 InterpState &S, CodePtr OpPC, const CallExpr *Call,
3180 llvm::function_ref<APInt(const APSInt &, const APSInt &)> Fn) {
3181 const auto *VT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
3182 PrimType ElemT = *S.getContext().classify(T: VT->getElementType());
3183 bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
3184
3185 const Pointer &RHS = S.Stk.pop<Pointer>();
3186 const Pointer &LHS = S.Stk.pop<Pointer>();
3187 const Pointer &Dst = S.Stk.peek<Pointer>();
3188 unsigned NumElts = VT->getNumElements();
3189 unsigned EltBits = S.getASTContext().getIntWidth(T: VT->getElementType());
3190 unsigned EltsPerLane = 128 / EltBits;
3191 unsigned Lanes = NumElts * EltBits / 128;
3192 unsigned DestIndex = 0;
3193
3194 for (unsigned Lane = 0; Lane < Lanes; ++Lane) {
3195 unsigned LaneStart = Lane * EltsPerLane;
3196 for (unsigned I = 0; I < EltsPerLane; I += 2) {
3197 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
3198 APSInt Elem1 = LHS.elem<T>(LaneStart + I).toAPSInt();
3199 APSInt Elem2 = LHS.elem<T>(LaneStart + I + 1).toAPSInt();
3200 APSInt ResL = APSInt(Fn(Elem1, Elem2), DestUnsigned);
3201 Dst.elem<T>(DestIndex++) = static_cast<T>(ResL);
3202 });
3203 }
3204
3205 for (unsigned I = 0; I < EltsPerLane; I += 2) {
3206 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
3207 APSInt Elem1 = RHS.elem<T>(LaneStart + I).toAPSInt();
3208 APSInt Elem2 = RHS.elem<T>(LaneStart + I + 1).toAPSInt();
3209 APSInt ResR = APSInt(Fn(Elem1, Elem2), DestUnsigned);
3210 Dst.elem<T>(DestIndex++) = static_cast<T>(ResR);
3211 });
3212 }
3213 }
3214 Dst.initializeAllElements();
3215 return true;
3216}
3217
3218static bool interp_builtin_horizontal_fp_binop(
3219 InterpState &S, CodePtr OpPC, const CallExpr *Call,
3220 llvm::function_ref<APFloat(const APFloat &, const APFloat &,
3221 llvm::RoundingMode)>
3222 Fn) {
3223 const Pointer &RHS = S.Stk.pop<Pointer>();
3224 const Pointer &LHS = S.Stk.pop<Pointer>();
3225 const Pointer &Dst = S.Stk.peek<Pointer>();
3226 FPOptions FPO = Call->getFPFeaturesInEffect(LO: S.Ctx.getLangOpts());
3227 llvm::RoundingMode RM = getRoundingMode(FPO);
3228 const auto *VT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
3229
3230 unsigned NumElts = VT->getNumElements();
3231 unsigned EltBits = S.getASTContext().getTypeSize(T: VT->getElementType());
3232 unsigned NumLanes = NumElts * EltBits / 128;
3233 unsigned NumElemsPerLane = NumElts / NumLanes;
3234 unsigned HalfElemsPerLane = NumElemsPerLane / 2;
3235
3236 for (unsigned L = 0; L != NumElts; L += NumElemsPerLane) {
3237 using T = PrimConv<PT_Float>::T;
3238 for (unsigned E = 0; E != HalfElemsPerLane; ++E) {
3239 APFloat Elem1 = LHS.elem<T>(I: L + (2 * E) + 0).getAPFloat();
3240 APFloat Elem2 = LHS.elem<T>(I: L + (2 * E) + 1).getAPFloat();
3241 Dst.elem<T>(I: L + E) = static_cast<T>(Fn(Elem1, Elem2, RM));
3242 }
3243 for (unsigned E = 0; E != HalfElemsPerLane; ++E) {
3244 APFloat Elem1 = RHS.elem<T>(I: L + (2 * E) + 0).getAPFloat();
3245 APFloat Elem2 = RHS.elem<T>(I: L + (2 * E) + 1).getAPFloat();
3246 Dst.elem<T>(I: L + E + HalfElemsPerLane) =
3247 static_cast<T>(Fn(Elem1, Elem2, RM));
3248 }
3249 }
3250 Dst.initializeAllElements();
3251 return true;
3252}
3253
3254static bool interp__builtin_ia32_addsub(InterpState &S, CodePtr OpPC,
3255 const CallExpr *Call) {
3256 // Addsub: alternates between subtraction and addition
3257 // Result[i] = (i % 2 == 0) ? (a[i] - b[i]) : (a[i] + b[i])
3258 const Pointer &RHS = S.Stk.pop<Pointer>();
3259 const Pointer &LHS = S.Stk.pop<Pointer>();
3260 const Pointer &Dst = S.Stk.peek<Pointer>();
3261 FPOptions FPO = Call->getFPFeaturesInEffect(LO: S.Ctx.getLangOpts());
3262 llvm::RoundingMode RM = getRoundingMode(FPO);
3263 const auto *VT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
3264 unsigned NumElems = VT->getNumElements();
3265
3266 using T = PrimConv<PT_Float>::T;
3267 for (unsigned I = 0; I != NumElems; ++I) {
3268 APFloat LElem = LHS.elem<T>(I).getAPFloat();
3269 APFloat RElem = RHS.elem<T>(I).getAPFloat();
3270 if (I % 2 == 0) {
3271 // Even indices: subtract
3272 LElem.subtract(RHS: RElem, RM);
3273 } else {
3274 // Odd indices: add
3275 LElem.add(RHS: RElem, RM);
3276 }
3277 Dst.elem<T>(I) = static_cast<T>(LElem);
3278 }
3279 Dst.initializeAllElements();
3280 return true;
3281}
3282
3283static bool interp__builtin_ia32_pclmulqdq(InterpState &S, CodePtr OpPC,
3284 const CallExpr *Call) {
3285 // PCLMULQDQ: carry-less multiplication of selected 64-bit halves
3286 // imm8 bit 0: selects lower (0) or upper (1) 64 bits of first operand
3287 // imm8 bit 4: selects lower (0) or upper (1) 64 bits of second operand
3288 assert(Call->getArg(0)->getType()->isVectorType() &&
3289 Call->getArg(1)->getType()->isVectorType());
3290
3291 // Extract imm8 argument
3292 APSInt Imm8;
3293 if (!popToAPSInt(S, E: Call->getArg(Arg: 2), Out&: Imm8))
3294 return false;
3295 bool SelectUpperA = (Imm8 & 0x01) != 0;
3296 bool SelectUpperB = (Imm8 & 0x10) != 0;
3297
3298 const Pointer &RHS = S.Stk.pop<Pointer>();
3299 const Pointer &LHS = S.Stk.pop<Pointer>();
3300 const Pointer &Dst = S.Stk.peek<Pointer>();
3301
3302 const auto *VT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
3303 PrimType ElemT = *S.getContext().classify(T: VT->getElementType());
3304 unsigned NumElems = VT->getNumElements();
3305 const auto *DestVT = Call->getType()->castAs<VectorType>();
3306 PrimType DestElemT = *S.getContext().classify(T: DestVT->getElementType());
3307 bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
3308
3309 // Process each 128-bit lane (2 elements at a time)
3310 for (unsigned Lane = 0; Lane < NumElems; Lane += 2) {
3311 APSInt A0, A1, B0, B1;
3312 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
3313 A0 = LHS.elem<T>(Lane + 0).toAPSInt();
3314 A1 = LHS.elem<T>(Lane + 1).toAPSInt();
3315 B0 = RHS.elem<T>(Lane + 0).toAPSInt();
3316 B1 = RHS.elem<T>(Lane + 1).toAPSInt();
3317 });
3318
3319 // Select the appropriate 64-bit values based on imm8
3320 APInt A = SelectUpperA ? A1 : A0;
3321 APInt B = SelectUpperB ? B1 : B0;
3322
3323 // Extend both operands to 128 bits for carry-less multiplication
3324 APInt A128 = A.zext(width: 128);
3325 APInt B128 = B.zext(width: 128);
3326
3327 // Use APIntOps::clmul for carry-less multiplication
3328 APInt Result = llvm::APIntOps::clmul(LHS: A128, RHS: B128);
3329
3330 // Split the 128-bit result into two 64-bit halves
3331 APSInt ResultLow(Result.extractBits(numBits: 64, bitPosition: 0), DestUnsigned);
3332 APSInt ResultHigh(Result.extractBits(numBits: 64, bitPosition: 64), DestUnsigned);
3333
3334 INT_TYPE_SWITCH_NO_BOOL(DestElemT, {
3335 Dst.elem<T>(Lane + 0) = static_cast<T>(ResultLow);
3336 Dst.elem<T>(Lane + 1) = static_cast<T>(ResultHigh);
3337 });
3338 }
3339
3340 Dst.initializeAllElements();
3341 return true;
3342}
3343
3344static bool interp__builtin_elementwise_triop_fp(
3345 InterpState &S, CodePtr OpPC, const CallExpr *Call,
3346 llvm::function_ref<APFloat(const APFloat &, const APFloat &,
3347 const APFloat &, llvm::RoundingMode)>
3348 Fn) {
3349 assert(Call->getNumArgs() == 3);
3350
3351 FPOptions FPO = Call->getFPFeaturesInEffect(LO: S.Ctx.getLangOpts());
3352 llvm::RoundingMode RM = getRoundingMode(FPO);
3353 QualType Arg1Type = Call->getArg(Arg: 0)->getType();
3354 QualType Arg2Type = Call->getArg(Arg: 1)->getType();
3355 QualType Arg3Type = Call->getArg(Arg: 2)->getType();
3356
3357 // Non-vector floating point types.
3358 if (!Arg1Type->isVectorType()) {
3359 assert(!Arg2Type->isVectorType());
3360 assert(!Arg3Type->isVectorType());
3361 (void)Arg2Type;
3362 (void)Arg3Type;
3363
3364 const Floating &Z = S.Stk.pop<Floating>();
3365 const Floating &Y = S.Stk.pop<Floating>();
3366 const Floating &X = S.Stk.pop<Floating>();
3367 APFloat F = Fn(X.getAPFloat(), Y.getAPFloat(), Z.getAPFloat(), RM);
3368 Floating Result = S.allocFloat(Sem: X.getSemantics());
3369 Result.copy(F);
3370 S.Stk.push<Floating>(Args&: Result);
3371 return true;
3372 }
3373
3374 // Vector type.
3375 assert(Arg1Type->isVectorType() && Arg2Type->isVectorType() &&
3376 Arg3Type->isVectorType());
3377
3378 const VectorType *VecTy = Arg1Type->castAs<VectorType>();
3379 QualType ElemQT = VecTy->getElementType();
3380 unsigned NumElems = VecTy->getNumElements();
3381
3382 assert(ElemQT == Arg2Type->castAs<VectorType>()->getElementType() &&
3383 ElemQT == Arg3Type->castAs<VectorType>()->getElementType());
3384 assert(NumElems == Arg2Type->castAs<VectorType>()->getNumElements() &&
3385 NumElems == Arg3Type->castAs<VectorType>()->getNumElements());
3386 assert(ElemQT->isRealFloatingType());
3387 (void)ElemQT;
3388
3389 const Pointer &VZ = S.Stk.pop<Pointer>();
3390 const Pointer &VY = S.Stk.pop<Pointer>();
3391 const Pointer &VX = S.Stk.pop<Pointer>();
3392 const Pointer &Dst = S.Stk.peek<Pointer>();
3393 for (unsigned I = 0; I != NumElems; ++I) {
3394 using T = PrimConv<PT_Float>::T;
3395 APFloat X = VX.elem<T>(I).getAPFloat();
3396 APFloat Y = VY.elem<T>(I).getAPFloat();
3397 APFloat Z = VZ.elem<T>(I).getAPFloat();
3398 APFloat F = Fn(X, Y, Z, RM);
3399 Dst.elem<Floating>(I) = Floating(F);
3400 }
3401 Dst.initializeAllElements();
3402 return true;
3403}
3404
3405/// AVX512 predicated move: "Result = Mask[] ? LHS[] : RHS[]".
3406static bool interp__builtin_ia32_select(InterpState &S, CodePtr OpPC,
3407 const CallExpr *Call) {
3408 const Pointer &RHS = S.Stk.pop<Pointer>();
3409 const Pointer &LHS = S.Stk.pop<Pointer>();
3410 APSInt Mask;
3411 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: Mask))
3412 return false;
3413 const Pointer &Dst = S.Stk.peek<Pointer>();
3414
3415 assert(LHS.getNumElems() == RHS.getNumElems());
3416 assert(LHS.getNumElems() == Dst.getNumElems());
3417 unsigned NumElems = LHS.getNumElems();
3418 PrimType ElemT = LHS.getFieldDesc()->getPrimType();
3419 PrimType DstElemT = Dst.getFieldDesc()->getPrimType();
3420
3421 for (unsigned I = 0; I != NumElems; ++I) {
3422 if (ElemT == PT_Float) {
3423 assert(DstElemT == PT_Float);
3424 Dst.elem<Floating>(I) =
3425 Mask[I] ? LHS.elem<Floating>(I) : RHS.elem<Floating>(I);
3426 } else {
3427 APSInt Elem;
3428 INT_TYPE_SWITCH(ElemT, {
3429 Elem = Mask[I] ? LHS.elem<T>(I).toAPSInt() : RHS.elem<T>(I).toAPSInt();
3430 });
3431 INT_TYPE_SWITCH_NO_BOOL(DstElemT,
3432 { Dst.elem<T>(I) = static_cast<T>(Elem); });
3433 }
3434 }
3435 Dst.initializeAllElements();
3436
3437 return true;
3438}
3439
3440/// Scalar variant of AVX512 predicated select:
3441/// Result[i] = (Mask bit 0) ? LHS[i] : RHS[i], but only element 0 may change.
3442/// All other elements are taken from RHS.
3443static bool interp__builtin_ia32_select_scalar(InterpState &S,
3444 const CallExpr *Call) {
3445 unsigned N =
3446 Call->getArg(Arg: 1)->getType()->castAs<VectorType>()->getNumElements();
3447
3448 const Pointer &W = S.Stk.pop<Pointer>();
3449 const Pointer &A = S.Stk.pop<Pointer>();
3450 APSInt U;
3451 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: U))
3452 return false;
3453 const Pointer &Dst = S.Stk.peek<Pointer>();
3454
3455 bool TakeA0 = U.getZExtValue() & 1ULL;
3456
3457 for (unsigned I = TakeA0; I != N; ++I)
3458 Dst.elem<Floating>(I) = W.elem<Floating>(I);
3459 if (TakeA0)
3460 Dst.elem<Floating>(I: 0) = A.elem<Floating>(I: 0);
3461
3462 Dst.initializeAllElements();
3463 return true;
3464}
3465
3466static bool interp__builtin_ia32_test_op(
3467 InterpState &S, CodePtr OpPC, const CallExpr *Call,
3468 llvm::function_ref<bool(const APInt &A, const APInt &B)> Fn) {
3469 const Pointer &RHS = S.Stk.pop<Pointer>();
3470 const Pointer &LHS = S.Stk.pop<Pointer>();
3471
3472 assert(LHS.getNumElems() == RHS.getNumElems());
3473
3474 unsigned SourceLen = LHS.getNumElems();
3475 QualType ElemQT = getElemType(P: LHS);
3476 OptPrimType ElemPT = S.getContext().classify(T: ElemQT);
3477 unsigned LaneWidth = S.getASTContext().getTypeSize(T: ElemQT);
3478
3479 APInt AWide(LaneWidth * SourceLen, 0);
3480 APInt BWide(LaneWidth * SourceLen, 0);
3481
3482 for (unsigned I = 0; I != SourceLen; ++I) {
3483 APInt ALane;
3484 APInt BLane;
3485
3486 if (ElemQT->isIntegerType()) { // Get value.
3487 INT_TYPE_SWITCH_NO_BOOL(*ElemPT, {
3488 ALane = LHS.elem<T>(I).toAPSInt();
3489 BLane = RHS.elem<T>(I).toAPSInt();
3490 });
3491 } else if (ElemQT->isFloatingType()) { // Get only sign bit.
3492 using T = PrimConv<PT_Float>::T;
3493 ALane = LHS.elem<T>(I).getAPFloat().bitcastToAPInt().isNegative();
3494 BLane = RHS.elem<T>(I).getAPFloat().bitcastToAPInt().isNegative();
3495 } else { // Must be integer or floating type.
3496 return false;
3497 }
3498 AWide.insertBits(SubBits: ALane, bitPosition: I * LaneWidth);
3499 BWide.insertBits(SubBits: BLane, bitPosition: I * LaneWidth);
3500 }
3501 pushInteger(S, Val: Fn(AWide, BWide), QT: Call->getType());
3502 return true;
3503}
3504
3505static bool interp__builtin_ia32_movmsk_op(InterpState &S, CodePtr OpPC,
3506 const CallExpr *Call) {
3507 assert(Call->getNumArgs() == 1);
3508
3509 const Pointer &Source = S.Stk.pop<Pointer>();
3510
3511 unsigned SourceLen = Source.getNumElems();
3512 QualType ElemQT = getElemType(P: Source);
3513 OptPrimType ElemT = S.getContext().classify(T: ElemQT);
3514 unsigned ResultLen =
3515 S.getASTContext().getTypeSize(T: Call->getType()); // Always 32-bit integer.
3516 APInt Result(ResultLen, 0);
3517
3518 for (unsigned I = 0; I != SourceLen; ++I) {
3519 APInt Elem;
3520 if (ElemQT->isIntegerType()) {
3521 INT_TYPE_SWITCH_NO_BOOL(*ElemT, { Elem = Source.elem<T>(I).toAPSInt(); });
3522 } else if (ElemQT->isRealFloatingType()) {
3523 using T = PrimConv<PT_Float>::T;
3524 Elem = Source.elem<T>(I).getAPFloat().bitcastToAPInt();
3525 } else {
3526 return false;
3527 }
3528 Result.setBitVal(BitPosition: I, BitValue: Elem.isNegative());
3529 }
3530 pushInteger(S, Val: Result, QT: Call->getType());
3531 return true;
3532}
3533
3534static bool interp__builtin_elementwise_triop(
3535 InterpState &S, CodePtr OpPC, const CallExpr *Call,
3536 llvm::function_ref<APInt(const APSInt &, const APSInt &, const APSInt &)>
3537 Fn) {
3538 assert(Call->getNumArgs() == 3);
3539
3540 QualType Arg0Type = Call->getArg(Arg: 0)->getType();
3541 QualType Arg2Type = Call->getArg(Arg: 2)->getType();
3542 // Non-vector integer types.
3543 if (!Arg0Type->isVectorType()) {
3544 APSInt Op2;
3545 if (!popToAPSInt(S, T: Arg2Type, Out&: Op2))
3546 return false;
3547 APSInt Op1;
3548 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: Op1))
3549 return false;
3550 APSInt Op0;
3551 if (!popToAPSInt(S, T: Arg0Type, Out&: Op0))
3552 return false;
3553 APSInt Result = APSInt(Fn(Op0, Op1, Op2), Op0.isUnsigned());
3554 pushInteger(S, Val: Result, QT: Call->getType());
3555 return true;
3556 }
3557
3558 const auto *VecT = Arg0Type->castAs<VectorType>();
3559 PrimType ElemT = *S.getContext().classify(T: VecT->getElementType());
3560 unsigned NumElems = VecT->getNumElements();
3561 bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
3562
3563 // Vector + Vector + Scalar case.
3564 if (!Arg2Type->isVectorType()) {
3565 APSInt Op2;
3566 if (!popToAPSInt(S, T: Arg2Type, Out&: Op2))
3567 return false;
3568
3569 const Pointer &Op1 = S.Stk.pop<Pointer>();
3570 const Pointer &Op0 = S.Stk.pop<Pointer>();
3571 const Pointer &Dst = S.Stk.peek<Pointer>();
3572 for (unsigned I = 0; I != NumElems; ++I) {
3573 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
3574 Dst.elem<T>(I) = static_cast<T>(APSInt(
3575 Fn(Op0.elem<T>(I).toAPSInt(), Op1.elem<T>(I).toAPSInt(), Op2),
3576 DestUnsigned));
3577 });
3578 }
3579 Dst.initializeAllElements();
3580
3581 return true;
3582 }
3583
3584 // Vector type.
3585 const Pointer &Op2 = S.Stk.pop<Pointer>();
3586 const Pointer &Op1 = S.Stk.pop<Pointer>();
3587 const Pointer &Op0 = S.Stk.pop<Pointer>();
3588 const Pointer &Dst = S.Stk.peek<Pointer>();
3589 for (unsigned I = 0; I != NumElems; ++I) {
3590 APSInt Val0, Val1, Val2;
3591 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
3592 Val0 = Op0.elem<T>(I).toAPSInt();
3593 Val1 = Op1.elem<T>(I).toAPSInt();
3594 Val2 = Op2.elem<T>(I).toAPSInt();
3595 });
3596 APSInt Result = APSInt(Fn(Val0, Val1, Val2), Val0.isUnsigned());
3597 INT_TYPE_SWITCH_NO_BOOL(ElemT,
3598 { Dst.elem<T>(I) = static_cast<T>(Result); });
3599 }
3600 Dst.initializeAllElements();
3601
3602 return true;
3603}
3604
3605static bool interp__builtin_ia32_extract_vector(InterpState &S, CodePtr OpPC,
3606 const CallExpr *Call,
3607 unsigned ID) {
3608 assert(Call->getNumArgs() == 2);
3609
3610 APSInt ImmAPS;
3611 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: ImmAPS))
3612 return false;
3613 uint64_t Index = ImmAPS.getZExtValue();
3614
3615 const Pointer &Src = S.Stk.pop<Pointer>();
3616 if (!Src.getFieldDesc()->isPrimitiveArray())
3617 return false;
3618
3619 const Pointer &Dst = S.Stk.peek<Pointer>();
3620 if (!Dst.getFieldDesc()->isPrimitiveArray())
3621 return false;
3622
3623 unsigned SrcElems = Src.getNumElems();
3624 unsigned DstElems = Dst.getNumElems();
3625
3626 unsigned NumLanes = SrcElems / DstElems;
3627 unsigned Lane = static_cast<unsigned>(Index % NumLanes);
3628 unsigned ExtractPos = Lane * DstElems;
3629
3630 PrimType ElemT = Src.getFieldDesc()->getPrimType();
3631
3632 TYPE_SWITCH(ElemT, {
3633 for (unsigned I = 0; I != DstElems; ++I) {
3634 Dst.elem<T>(I) = Src.elem<T>(ExtractPos + I);
3635 }
3636 });
3637
3638 Dst.initializeAllElements();
3639 return true;
3640}
3641
3642static bool interp__builtin_ia32_extract_vector_masked(InterpState &S,
3643 CodePtr OpPC,
3644 const CallExpr *Call,
3645 unsigned ID) {
3646 assert(Call->getNumArgs() == 4);
3647
3648 APSInt MaskAPS;
3649 if (!popToAPSInt(S, E: Call->getArg(Arg: 3), Out&: MaskAPS))
3650 return false;
3651 const Pointer &Merge = S.Stk.pop<Pointer>();
3652 APSInt ImmAPS;
3653 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: ImmAPS))
3654 return false;
3655 const Pointer &Src = S.Stk.pop<Pointer>();
3656
3657 if (!Src.getFieldDesc()->isPrimitiveArray() ||
3658 !Merge.getFieldDesc()->isPrimitiveArray())
3659 return false;
3660
3661 const Pointer &Dst = S.Stk.peek<Pointer>();
3662 if (!Dst.getFieldDesc()->isPrimitiveArray())
3663 return false;
3664
3665 unsigned SrcElems = Src.getNumElems();
3666 unsigned DstElems = Dst.getNumElems();
3667
3668 unsigned NumLanes = SrcElems / DstElems;
3669 unsigned Lane = static_cast<unsigned>(ImmAPS.getZExtValue() % NumLanes);
3670 unsigned Base = Lane * DstElems;
3671
3672 PrimType ElemT = Src.getFieldDesc()->getPrimType();
3673
3674 TYPE_SWITCH(ElemT, {
3675 for (unsigned I = 0; I != DstElems; ++I) {
3676 if (MaskAPS[I])
3677 Dst.elem<T>(I) = Src.elem<T>(Base + I);
3678 else
3679 Dst.elem<T>(I) = Merge.elem<T>(I);
3680 }
3681 });
3682
3683 Dst.initializeAllElements();
3684 return true;
3685}
3686
3687static bool interp__builtin_ia32_insert_subvector(InterpState &S, CodePtr OpPC,
3688 const CallExpr *Call,
3689 unsigned ID) {
3690 assert(Call->getNumArgs() == 3);
3691
3692 APSInt ImmAPS;
3693 if (!popToAPSInt(S, E: Call->getArg(Arg: 2), Out&: ImmAPS))
3694 return false;
3695 uint64_t Index = ImmAPS.getZExtValue();
3696
3697 const Pointer &SubVec = S.Stk.pop<Pointer>();
3698 if (!SubVec.getFieldDesc()->isPrimitiveArray())
3699 return false;
3700
3701 const Pointer &BaseVec = S.Stk.pop<Pointer>();
3702 if (!BaseVec.getFieldDesc()->isPrimitiveArray())
3703 return false;
3704
3705 const Pointer &Dst = S.Stk.peek<Pointer>();
3706
3707 unsigned BaseElements = BaseVec.getNumElems();
3708 unsigned SubElements = SubVec.getNumElems();
3709
3710 assert(SubElements != 0 && BaseElements != 0 &&
3711 (BaseElements % SubElements) == 0);
3712
3713 unsigned NumLanes = BaseElements / SubElements;
3714 unsigned Lane = static_cast<unsigned>(Index % NumLanes);
3715 unsigned InsertPos = Lane * SubElements;
3716
3717 PrimType ElemT = BaseVec.getFieldDesc()->getPrimType();
3718
3719 TYPE_SWITCH(ElemT, {
3720 for (unsigned I = 0; I != BaseElements; ++I)
3721 Dst.elem<T>(I) = BaseVec.elem<T>(I);
3722 for (unsigned I = 0; I != SubElements; ++I)
3723 Dst.elem<T>(InsertPos + I) = SubVec.elem<T>(I);
3724 });
3725
3726 Dst.initializeAllElements();
3727 return true;
3728}
3729
3730static bool interp__builtin_ia32_phminposuw(InterpState &S, CodePtr OpPC,
3731 const CallExpr *Call) {
3732 assert(Call->getNumArgs() == 1);
3733
3734 const Pointer &Source = S.Stk.pop<Pointer>();
3735 const Pointer &Dest = S.Stk.peek<Pointer>();
3736
3737 unsigned SourceLen = Source.getNumElems();
3738 QualType ElemQT = getElemType(P: Source);
3739 OptPrimType ElemT = S.getContext().classify(T: ElemQT);
3740 unsigned ElemBitWidth = S.getASTContext().getTypeSize(T: ElemQT);
3741
3742 bool DestUnsigned = Call->getCallReturnType(Ctx: S.getASTContext())
3743 ->castAs<VectorType>()
3744 ->getElementType()
3745 ->isUnsignedIntegerOrEnumerationType();
3746
3747 INT_TYPE_SWITCH_NO_BOOL(*ElemT, {
3748 APSInt MinIndex(ElemBitWidth, DestUnsigned);
3749 APSInt MinVal = Source.elem<T>(0).toAPSInt();
3750
3751 for (unsigned I = 1; I != SourceLen; ++I) {
3752 APSInt Val = Source.elem<T>(I).toAPSInt();
3753 if (MinVal.ugt(Val)) {
3754 MinVal = Val;
3755 MinIndex = I;
3756 }
3757 }
3758
3759 Dest.elem<T>(0) = static_cast<T>(MinVal);
3760 Dest.elem<T>(1) = static_cast<T>(MinIndex);
3761 for (unsigned I = 2; I != SourceLen; ++I) {
3762 Dest.elem<T>(I) = static_cast<T>(APSInt(ElemBitWidth, DestUnsigned));
3763 }
3764 });
3765 Dest.initializeAllElements();
3766 return true;
3767}
3768
3769static bool interp__builtin_ia32_pternlog(InterpState &S, CodePtr OpPC,
3770 const CallExpr *Call, bool MaskZ) {
3771 assert(Call->getNumArgs() == 5);
3772
3773 APSInt UVal;
3774 if (!popToAPSInt(S, E: Call->getArg(Arg: 4), Out&: UVal))
3775 return false;
3776 APInt U = UVal; // Lane mask
3777 APSInt ImmVal;
3778 if (!popToAPSInt(S, E: Call->getArg(Arg: 3), Out&: ImmVal))
3779 return false;
3780 APInt Imm = ImmVal; // Ternary truth table
3781 const Pointer &C = S.Stk.pop<Pointer>();
3782 const Pointer &B = S.Stk.pop<Pointer>();
3783 const Pointer &A = S.Stk.pop<Pointer>();
3784 const Pointer &Dst = S.Stk.peek<Pointer>();
3785
3786 unsigned DstLen = A.getNumElems();
3787 QualType ElemQT = getElemType(P: A);
3788 OptPrimType ElemT = S.getContext().classify(T: ElemQT);
3789 unsigned LaneWidth = S.getASTContext().getTypeSize(T: ElemQT);
3790 bool DstUnsigned = ElemQT->isUnsignedIntegerOrEnumerationType();
3791
3792 INT_TYPE_SWITCH_NO_BOOL(*ElemT, {
3793 for (unsigned I = 0; I != DstLen; ++I) {
3794 APInt ALane = A.elem<T>(I).toAPSInt();
3795 APInt BLane = B.elem<T>(I).toAPSInt();
3796 APInt CLane = C.elem<T>(I).toAPSInt();
3797 APInt RLane(LaneWidth, 0);
3798 if (U[I]) { // If lane not masked, compute ternary logic.
3799 for (unsigned Bit = 0; Bit != LaneWidth; ++Bit) {
3800 unsigned ABit = ALane[Bit];
3801 unsigned BBit = BLane[Bit];
3802 unsigned CBit = CLane[Bit];
3803 unsigned Idx = (ABit << 2) | (BBit << 1) | (CBit);
3804 RLane.setBitVal(Bit, Imm[Idx]);
3805 }
3806 Dst.elem<T>(I) = static_cast<T>(APSInt(RLane, DstUnsigned));
3807 } else if (MaskZ) { // If zero masked, zero the lane.
3808 Dst.elem<T>(I) = static_cast<T>(APSInt(RLane, DstUnsigned));
3809 } else { // Just masked, put in A lane.
3810 Dst.elem<T>(I) = static_cast<T>(APSInt(ALane, DstUnsigned));
3811 }
3812 }
3813 });
3814 Dst.initializeAllElements();
3815 return true;
3816}
3817
3818static bool interp__builtin_ia32_vec_ext(InterpState &S, CodePtr OpPC,
3819 const CallExpr *Call, unsigned ID) {
3820 assert(Call->getNumArgs() == 2);
3821
3822 APSInt ImmAPS;
3823 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: ImmAPS))
3824 return false;
3825 const Pointer &Vec = S.Stk.pop<Pointer>();
3826 if (!Vec.getFieldDesc()->isPrimitiveArray())
3827 return false;
3828
3829 unsigned NumElems = Vec.getNumElems();
3830 unsigned Index =
3831 static_cast<unsigned>(ImmAPS.getZExtValue() & (NumElems - 1));
3832
3833 PrimType ElemT = Vec.getFieldDesc()->getPrimType();
3834 // FIXME(#161685): Replace float+int split with a numeric-only type switch
3835 if (ElemT == PT_Float) {
3836 S.Stk.push<Floating>(Args&: Vec.elem<Floating>(I: Index));
3837 return true;
3838 }
3839 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
3840 APSInt V = Vec.elem<T>(Index).toAPSInt();
3841 pushInteger(S, V, Call->getType());
3842 });
3843
3844 return true;
3845}
3846
3847static bool interp__builtin_ia32_vec_set(InterpState &S, CodePtr OpPC,
3848 const CallExpr *Call, unsigned ID) {
3849 assert(Call->getNumArgs() == 3);
3850
3851 APSInt ImmAPS;
3852 if (!popToAPSInt(S, E: Call->getArg(Arg: 2), Out&: ImmAPS))
3853 return false;
3854 APSInt ValAPS;
3855 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: ValAPS))
3856 return false;
3857
3858 const Pointer &Base = S.Stk.pop<Pointer>();
3859 if (!Base.getFieldDesc()->isPrimitiveArray())
3860 return false;
3861
3862 const Pointer &Dst = S.Stk.peek<Pointer>();
3863
3864 unsigned NumElems = Base.getNumElems();
3865 unsigned Index =
3866 static_cast<unsigned>(ImmAPS.getZExtValue() & (NumElems - 1));
3867
3868 PrimType ElemT = Base.getFieldDesc()->getPrimType();
3869 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
3870 for (unsigned I = 0; I != NumElems; ++I)
3871 Dst.elem<T>(I) = Base.elem<T>(I);
3872 Dst.elem<T>(Index) = static_cast<T>(ValAPS);
3873 });
3874
3875 Dst.initializeAllElements();
3876 return true;
3877}
3878
3879static bool evalICmpImm(uint8_t Imm, const APSInt &A, const APSInt &B,
3880 bool IsUnsigned) {
3881 switch (Imm & 0x7) {
3882 case 0x00: // _MM_CMPINT_EQ
3883 return (A == B);
3884 case 0x01: // _MM_CMPINT_LT
3885 return IsUnsigned ? A.ult(RHS: B) : A.slt(RHS: B);
3886 case 0x02: // _MM_CMPINT_LE
3887 return IsUnsigned ? A.ule(RHS: B) : A.sle(RHS: B);
3888 case 0x03: // _MM_CMPINT_FALSE
3889 return false;
3890 case 0x04: // _MM_CMPINT_NE
3891 return (A != B);
3892 case 0x05: // _MM_CMPINT_NLT
3893 return IsUnsigned ? A.uge(RHS: B) : A.sge(RHS: B);
3894 case 0x06: // _MM_CMPINT_NLE
3895 return IsUnsigned ? A.ugt(RHS: B) : A.sgt(RHS: B);
3896 case 0x07: // _MM_CMPINT_TRUE
3897 return true;
3898 default:
3899 llvm_unreachable("Invalid Op");
3900 }
3901}
3902
3903static bool interp__builtin_ia32_cmp_mask(InterpState &S, CodePtr OpPC,
3904 const CallExpr *Call, unsigned ID,
3905 bool IsUnsigned) {
3906 assert(Call->getNumArgs() == 4);
3907
3908 APSInt Mask;
3909 if (!popToAPSInt(S, E: Call->getArg(Arg: 3), Out&: Mask))
3910 return false;
3911 APSInt Opcode;
3912 if (!popToAPSInt(S, E: Call->getArg(Arg: 2), Out&: Opcode))
3913 return false;
3914 unsigned CmpOp = static_cast<unsigned>(Opcode.getZExtValue());
3915 const Pointer &RHS = S.Stk.pop<Pointer>();
3916 const Pointer &LHS = S.Stk.pop<Pointer>();
3917
3918 assert(LHS.getNumElems() == RHS.getNumElems());
3919
3920 APInt RetMask = APInt::getZero(numBits: LHS.getNumElems());
3921 unsigned VectorLen = LHS.getNumElems();
3922 PrimType ElemT = LHS.getFieldDesc()->getPrimType();
3923
3924 for (unsigned ElemNum = 0; ElemNum < VectorLen; ++ElemNum) {
3925 APSInt A, B;
3926 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
3927 A = LHS.elem<T>(ElemNum).toAPSInt();
3928 B = RHS.elem<T>(ElemNum).toAPSInt();
3929 });
3930 RetMask.setBitVal(BitPosition: ElemNum,
3931 BitValue: Mask[ElemNum] && evalICmpImm(Imm: CmpOp, A, B, IsUnsigned));
3932 }
3933 pushInteger(S, Val: RetMask, QT: Call->getType());
3934 return true;
3935}
3936
3937static bool interp__builtin_ia32_vpconflict(InterpState &S, CodePtr OpPC,
3938 const CallExpr *Call) {
3939 assert(Call->getNumArgs() == 1);
3940
3941 QualType Arg0Type = Call->getArg(Arg: 0)->getType();
3942 const auto *VecT = Arg0Type->castAs<VectorType>();
3943 PrimType ElemT = *S.getContext().classify(T: VecT->getElementType());
3944 unsigned NumElems = VecT->getNumElements();
3945 bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
3946 const Pointer &Src = S.Stk.pop<Pointer>();
3947 const Pointer &Dst = S.Stk.peek<Pointer>();
3948
3949 for (unsigned I = 0; I != NumElems; ++I) {
3950 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
3951 APSInt ElemI = Src.elem<T>(I).toAPSInt();
3952 APInt ConflictMask(ElemI.getBitWidth(), 0);
3953 for (unsigned J = 0; J != I; ++J) {
3954 APSInt ElemJ = Src.elem<T>(J).toAPSInt();
3955 ConflictMask.setBitVal(J, ElemI == ElemJ);
3956 }
3957 Dst.elem<T>(I) = static_cast<T>(APSInt(ConflictMask, DestUnsigned));
3958 });
3959 }
3960 Dst.initializeAllElements();
3961 return true;
3962}
3963
3964static bool interp__builtin_ia32_cvt_vec2mask(InterpState &S, CodePtr OpPC,
3965 const CallExpr *Call,
3966 unsigned ID) {
3967 assert(Call->getNumArgs() == 1);
3968
3969 const Pointer &Vec = S.Stk.pop<Pointer>();
3970 unsigned RetWidth = S.getASTContext().getIntWidth(T: Call->getType());
3971 APInt RetMask(RetWidth, 0);
3972
3973 unsigned VectorLen = Vec.getNumElems();
3974 PrimType ElemT = Vec.getFieldDesc()->getPrimType();
3975
3976 for (unsigned ElemNum = 0; ElemNum != VectorLen; ++ElemNum) {
3977 APSInt A;
3978 INT_TYPE_SWITCH_NO_BOOL(ElemT, { A = Vec.elem<T>(ElemNum).toAPSInt(); });
3979 unsigned MSB = A[A.getBitWidth() - 1];
3980 RetMask.setBitVal(BitPosition: ElemNum, BitValue: MSB);
3981 }
3982 pushInteger(S, Val: RetMask, QT: Call->getType());
3983 return true;
3984}
3985
3986static bool interp__builtin_ia32_cvt_mask2vec(InterpState &S, CodePtr OpPC,
3987 const CallExpr *Call,
3988 unsigned ID) {
3989 assert(Call->getNumArgs() == 1);
3990
3991 APSInt Mask;
3992 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: Mask))
3993 return false;
3994
3995 const Pointer &Vec = S.Stk.peek<Pointer>();
3996 unsigned NumElems = Vec.getNumElems();
3997 PrimType ElemT = Vec.getFieldDesc()->getPrimType();
3998
3999 for (unsigned I = 0; I != NumElems; ++I) {
4000 bool BitSet = Mask[I];
4001
4002 INT_TYPE_SWITCH_NO_BOOL(
4003 ElemT, { Vec.elem<T>(I) = BitSet ? T::from(-1) : T::from(0); });
4004 }
4005
4006 Vec.initializeAllElements();
4007
4008 return true;
4009}
4010
4011static bool interp__builtin_ia32_cvtsd2ss(InterpState &S, CodePtr OpPC,
4012 const CallExpr *Call,
4013 bool HasRoundingMask) {
4014 APSInt Rounding, MaskInt;
4015 Pointer Src, B, A;
4016
4017 if (HasRoundingMask) {
4018 assert(Call->getNumArgs() == 5);
4019 if (!popToAPSInt(S, E: Call->getArg(Arg: 4), Out&: Rounding))
4020 return false;
4021 if (!popToAPSInt(S, E: Call->getArg(Arg: 3), Out&: MaskInt))
4022 return false;
4023 Src = S.Stk.pop<Pointer>();
4024 B = S.Stk.pop<Pointer>();
4025 A = S.Stk.pop<Pointer>();
4026 if (!CheckLoad(S, OpPC, Ptr: A) || !CheckLoad(S, OpPC, Ptr: B) ||
4027 !CheckLoad(S, OpPC, Ptr: Src))
4028 return false;
4029 } else {
4030 assert(Call->getNumArgs() == 2);
4031 B = S.Stk.pop<Pointer>();
4032 A = S.Stk.pop<Pointer>();
4033 if (!CheckLoad(S, OpPC, Ptr: A) || !CheckLoad(S, OpPC, Ptr: B))
4034 return false;
4035 }
4036
4037 const auto *DstVTy = Call->getType()->castAs<VectorType>();
4038 unsigned NumElems = DstVTy->getNumElements();
4039 const Pointer &Dst = S.Stk.peek<Pointer>();
4040
4041 // Copy all elements except lane 0 (overwritten below) from A to Dst.
4042 for (unsigned I = 1; I != NumElems; ++I)
4043 Dst.elem<Floating>(I) = A.elem<Floating>(I);
4044
4045 // Convert element 0 from double to float, or use Src if masked off.
4046 if (!HasRoundingMask || (MaskInt.getZExtValue() & 0x1)) {
4047 assert(S.getASTContext().FloatTy == DstVTy->getElementType() &&
4048 "cvtsd2ss requires float element type in destination vector");
4049
4050 Floating Conv = S.allocFloat(
4051 Sem: S.getASTContext().getFloatTypeSemantics(T: DstVTy->getElementType()));
4052 APFloat SrcVal = B.elem<Floating>(I: 0).getAPFloat();
4053 if (!convertDoubleToFloatStrict(Src: SrcVal, Dst&: Conv, S, DiagExpr: Call))
4054 return false;
4055 Dst.elem<Floating>(I: 0) = Conv;
4056 } else {
4057 Dst.elem<Floating>(I: 0) = Src.elem<Floating>(I: 0);
4058 }
4059
4060 Dst.initializeAllElements();
4061 return true;
4062}
4063
4064static bool interp__builtin_ia32_cvtpd2ps(InterpState &S, CodePtr OpPC,
4065 const CallExpr *Call, bool IsMasked,
4066 bool HasRounding) {
4067 APSInt MaskVal;
4068 Pointer PassThrough;
4069 Pointer Src;
4070 APSInt Rounding;
4071
4072 if (IsMasked) {
4073 // Pop in reverse order.
4074 if (HasRounding) {
4075 if (!popToAPSInt(S, E: Call->getArg(Arg: 3), Out&: Rounding))
4076 return false;
4077 if (!popToAPSInt(S, E: Call->getArg(Arg: 2), Out&: MaskVal))
4078 return false;
4079 PassThrough = S.Stk.pop<Pointer>();
4080 Src = S.Stk.pop<Pointer>();
4081 } else {
4082 if (!popToAPSInt(S, E: Call->getArg(Arg: 2), Out&: MaskVal))
4083 return false;
4084 PassThrough = S.Stk.pop<Pointer>();
4085 Src = S.Stk.pop<Pointer>();
4086 }
4087
4088 if (!CheckLoad(S, OpPC, Ptr: PassThrough))
4089 return false;
4090 } else {
4091 // Pop source only.
4092 Src = S.Stk.pop<Pointer>();
4093 }
4094
4095 if (!CheckLoad(S, OpPC, Ptr: Src))
4096 return false;
4097
4098 const auto *RetVTy = Call->getType()->castAs<VectorType>();
4099 unsigned RetElems = RetVTy->getNumElements();
4100 unsigned SrcElems = Src.getNumElems();
4101 const Pointer &Dst = S.Stk.peek<Pointer>();
4102
4103 // Initialize destination with passthrough or zeros.
4104 for (unsigned I = 0; I != RetElems; ++I)
4105 if (IsMasked)
4106 Dst.elem<Floating>(I) = PassThrough.elem<Floating>(I);
4107 else
4108 Dst.elem<Floating>(I) = Floating(APFloat(0.0f));
4109
4110 assert(S.getASTContext().FloatTy == RetVTy->getElementType() &&
4111 "cvtpd2ps requires float element type in return vector");
4112
4113 // Convert double to float for enabled elements (only process source elements
4114 // that exist).
4115 for (unsigned I = 0; I != SrcElems; ++I) {
4116 if (IsMasked && !MaskVal[I])
4117 continue;
4118
4119 APFloat SrcVal = Src.elem<Floating>(I).getAPFloat();
4120
4121 Floating Conv = S.allocFloat(
4122 Sem: S.getASTContext().getFloatTypeSemantics(T: RetVTy->getElementType()));
4123 if (!convertDoubleToFloatStrict(Src: SrcVal, Dst&: Conv, S, DiagExpr: Call))
4124 return false;
4125 Dst.elem<Floating>(I) = Conv;
4126 }
4127
4128 Dst.initializeAllElements();
4129 return true;
4130}
4131
4132static bool interp__builtin_ia32_shuffle_generic(
4133 InterpState &S, CodePtr OpPC, const CallExpr *Call,
4134 llvm::function_ref<std::pair<unsigned, int>(unsigned, const APInt &)>
4135 GetSourceIndex) {
4136
4137 assert(Call->getNumArgs() == 2 || Call->getNumArgs() == 3);
4138
4139 APInt ShuffleMask;
4140 Pointer A, MaskVector, B;
4141 bool IsVectorMask = false;
4142 bool IsSingleOperand = (Call->getNumArgs() == 2);
4143
4144 if (IsSingleOperand) {
4145 QualType MaskType = Call->getArg(Arg: 1)->getType();
4146 if (MaskType->isVectorType()) {
4147 IsVectorMask = true;
4148 MaskVector = S.Stk.pop<Pointer>();
4149 A = S.Stk.pop<Pointer>();
4150 B = A;
4151 } else if (MaskType->isIntegerType()) {
4152 APSInt MaskVal;
4153 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: MaskVal))
4154 return false;
4155 ShuffleMask = MaskVal;
4156 A = S.Stk.pop<Pointer>();
4157 B = A;
4158 } else {
4159 return false;
4160 }
4161 } else {
4162 QualType Arg2Type = Call->getArg(Arg: 2)->getType();
4163 if (Arg2Type->isVectorType()) {
4164 IsVectorMask = true;
4165 B = S.Stk.pop<Pointer>();
4166 MaskVector = S.Stk.pop<Pointer>();
4167 A = S.Stk.pop<Pointer>();
4168 } else if (Arg2Type->isIntegerType()) {
4169 APSInt MaskVal;
4170 if (!popToAPSInt(S, E: Call->getArg(Arg: 2), Out&: MaskVal))
4171 return false;
4172 ShuffleMask = MaskVal;
4173 B = S.Stk.pop<Pointer>();
4174 A = S.Stk.pop<Pointer>();
4175 } else {
4176 return false;
4177 }
4178 }
4179
4180 QualType Arg0Type = Call->getArg(Arg: 0)->getType();
4181 const auto *VecT = Arg0Type->castAs<VectorType>();
4182 PrimType ElemT = *S.getContext().classify(T: VecT->getElementType());
4183 unsigned NumElems = VecT->getNumElements();
4184
4185 const Pointer &Dst = S.Stk.peek<Pointer>();
4186
4187 PrimType MaskElemT = PT_Uint32;
4188 if (IsVectorMask) {
4189 QualType Arg1Type = Call->getArg(Arg: 1)->getType();
4190 const auto *MaskVecT = Arg1Type->castAs<VectorType>();
4191 QualType MaskElemType = MaskVecT->getElementType();
4192 MaskElemT = *S.getContext().classify(T: MaskElemType);
4193 }
4194
4195 for (unsigned DstIdx = 0; DstIdx != NumElems; ++DstIdx) {
4196 if (IsVectorMask) {
4197 INT_TYPE_SWITCH(MaskElemT,
4198 { ShuffleMask = MaskVector.elem<T>(DstIdx).toAPSInt(); });
4199 }
4200
4201 auto [SrcVecIdx, SrcIdx] = GetSourceIndex(DstIdx, ShuffleMask);
4202
4203 if (SrcIdx < 0) {
4204 // Zero out this element
4205 if (ElemT == PT_Float) {
4206 Dst.elem<Floating>(I: DstIdx) = Floating(
4207 S.getASTContext().getFloatTypeSemantics(T: VecT->getElementType()));
4208 } else {
4209 INT_TYPE_SWITCH_NO_BOOL(ElemT, { Dst.elem<T>(DstIdx) = T::from(0); });
4210 }
4211 } else {
4212 const Pointer &Src = (SrcVecIdx == 0) ? A : B;
4213 TYPE_SWITCH(ElemT, { Dst.elem<T>(DstIdx) = Src.elem<T>(SrcIdx); });
4214 }
4215 }
4216 Dst.initializeAllElements();
4217
4218 return true;
4219}
4220
4221static bool interp__builtin_ia32_shuffle_generic(
4222 InterpState &S, CodePtr OpPC, const CallExpr *Call,
4223 llvm::function_ref<std::pair<unsigned, int>(unsigned, unsigned)>
4224 GetSourceIndex) {
4225 return interp__builtin_ia32_shuffle_generic(
4226 S, OpPC, Call,
4227 GetSourceIndex: [&GetSourceIndex](unsigned DstIdx,
4228 const APInt &Mask) -> std::pair<unsigned, int> {
4229 return GetSourceIndex(DstIdx, Mask.getZExtValue());
4230 });
4231}
4232
4233static bool interp__builtin_ia32_shift_with_count(
4234 InterpState &S, CodePtr OpPC, const CallExpr *Call,
4235 llvm::function_ref<APInt(const APInt &, uint64_t)> ShiftOp,
4236 llvm::function_ref<APInt(const APInt &, unsigned)> OverflowOp) {
4237
4238 assert(Call->getNumArgs() == 2);
4239
4240 const Pointer &Count = S.Stk.pop<Pointer>();
4241 const Pointer &Source = S.Stk.pop<Pointer>();
4242
4243 QualType SourceType = Call->getArg(Arg: 0)->getType();
4244 QualType CountType = Call->getArg(Arg: 1)->getType();
4245 assert(SourceType->isVectorType() && CountType->isVectorType());
4246
4247 const auto *SourceVecT = SourceType->castAs<VectorType>();
4248 const auto *CountVecT = CountType->castAs<VectorType>();
4249 PrimType SourceElemT = *S.getContext().classify(T: SourceVecT->getElementType());
4250 PrimType CountElemT = *S.getContext().classify(T: CountVecT->getElementType());
4251
4252 const Pointer &Dst = S.Stk.peek<Pointer>();
4253
4254 unsigned DestEltWidth =
4255 S.getASTContext().getTypeSize(T: SourceVecT->getElementType());
4256 bool IsDestUnsigned = SourceVecT->getElementType()->isUnsignedIntegerType();
4257 unsigned DestLen = SourceVecT->getNumElements();
4258 unsigned CountEltWidth =
4259 S.getASTContext().getTypeSize(T: CountVecT->getElementType());
4260 unsigned NumBitsInQWord = 64;
4261 unsigned NumCountElts = NumBitsInQWord / CountEltWidth;
4262
4263 uint64_t CountLQWord = 0;
4264 for (unsigned EltIdx = 0; EltIdx != NumCountElts; ++EltIdx) {
4265 uint64_t Elt = 0;
4266 INT_TYPE_SWITCH(CountElemT,
4267 { Elt = static_cast<uint64_t>(Count.elem<T>(EltIdx)); });
4268 CountLQWord |= (Elt << (EltIdx * CountEltWidth));
4269 }
4270
4271 for (unsigned EltIdx = 0; EltIdx != DestLen; ++EltIdx) {
4272 APSInt Elt;
4273 INT_TYPE_SWITCH(SourceElemT, { Elt = Source.elem<T>(EltIdx).toAPSInt(); });
4274
4275 APInt Result;
4276 if (CountLQWord < DestEltWidth) {
4277 Result = ShiftOp(Elt, CountLQWord);
4278 } else {
4279 Result = OverflowOp(Elt, DestEltWidth);
4280 }
4281 if (IsDestUnsigned) {
4282 INT_TYPE_SWITCH(SourceElemT, {
4283 Dst.elem<T>(EltIdx) = T::from(Result.getZExtValue());
4284 });
4285 } else {
4286 INT_TYPE_SWITCH(SourceElemT, {
4287 Dst.elem<T>(EltIdx) = T::from(Result.getSExtValue());
4288 });
4289 }
4290 }
4291
4292 Dst.initializeAllElements();
4293 return true;
4294}
4295
4296static bool interp__builtin_ia32_shufbitqmb_mask(InterpState &S, CodePtr OpPC,
4297 const CallExpr *Call) {
4298
4299 assert(Call->getNumArgs() == 3);
4300
4301 QualType SourceType = Call->getArg(Arg: 0)->getType();
4302 QualType ShuffleMaskType = Call->getArg(Arg: 1)->getType();
4303 QualType ZeroMaskType = Call->getArg(Arg: 2)->getType();
4304 if (!SourceType->isVectorType() || !ShuffleMaskType->isVectorType() ||
4305 !ZeroMaskType->isIntegerType()) {
4306 return false;
4307 }
4308
4309 Pointer Source, ShuffleMask;
4310 APSInt ZeroMask;
4311 if (!popToAPSInt(S, E: Call->getArg(Arg: 2), Out&: ZeroMask))
4312 return false;
4313 ShuffleMask = S.Stk.pop<Pointer>();
4314 Source = S.Stk.pop<Pointer>();
4315
4316 const auto *SourceVecT = SourceType->castAs<VectorType>();
4317 const auto *ShuffleMaskVecT = ShuffleMaskType->castAs<VectorType>();
4318 assert(SourceVecT->getNumElements() == ShuffleMaskVecT->getNumElements());
4319 assert(ZeroMask.getBitWidth() == SourceVecT->getNumElements());
4320
4321 PrimType SourceElemT = *S.getContext().classify(T: SourceVecT->getElementType());
4322 PrimType ShuffleMaskElemT =
4323 *S.getContext().classify(T: ShuffleMaskVecT->getElementType());
4324
4325 unsigned NumBytesInQWord = 8;
4326 unsigned NumBitsInByte = 8;
4327 unsigned NumBytes = SourceVecT->getNumElements();
4328 unsigned NumQWords = NumBytes / NumBytesInQWord;
4329 unsigned RetWidth = ZeroMask.getBitWidth();
4330 APSInt RetMask(llvm::APInt(RetWidth, 0), /*isUnsigned=*/true);
4331
4332 for (unsigned QWordId = 0; QWordId != NumQWords; ++QWordId) {
4333 APInt SourceQWord(64, 0);
4334 for (unsigned ByteIdx = 0; ByteIdx != NumBytesInQWord; ++ByteIdx) {
4335 uint64_t Byte = 0;
4336 INT_TYPE_SWITCH(SourceElemT, {
4337 Byte = static_cast<uint64_t>(
4338 Source.elem<T>(QWordId * NumBytesInQWord + ByteIdx));
4339 });
4340 SourceQWord.insertBits(SubBits: APInt(8, Byte & 0xFF), bitPosition: ByteIdx * NumBitsInByte);
4341 }
4342
4343 for (unsigned ByteIdx = 0; ByteIdx != NumBytesInQWord; ++ByteIdx) {
4344 unsigned SelIdx = QWordId * NumBytesInQWord + ByteIdx;
4345 unsigned M = 0;
4346 INT_TYPE_SWITCH(ShuffleMaskElemT, {
4347 M = static_cast<unsigned>(ShuffleMask.elem<T>(SelIdx)) & 0x3F;
4348 });
4349
4350 if (ZeroMask[SelIdx]) {
4351 RetMask.setBitVal(BitPosition: SelIdx, BitValue: SourceQWord[M]);
4352 }
4353 }
4354 }
4355
4356 pushInteger(S, Val: RetMask, QT: Call->getType());
4357 return true;
4358}
4359
4360static bool interp__builtin_ia32_vcvtps2ph(InterpState &S, CodePtr OpPC,
4361 const CallExpr *Call) {
4362 // Arguments are: vector of floats, rounding immediate
4363 assert(Call->getNumArgs() == 2);
4364
4365 APSInt Imm;
4366 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: Imm))
4367 return false;
4368 const Pointer &Src = S.Stk.pop<Pointer>();
4369 const Pointer &Dst = S.Stk.peek<Pointer>();
4370
4371 assert(Src.getFieldDesc()->isPrimitiveArray());
4372 assert(Dst.getFieldDesc()->isPrimitiveArray());
4373
4374 const auto *SrcVTy = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
4375 unsigned SrcNumElems = SrcVTy->getNumElements();
4376 const auto *DstVTy = Call->getType()->castAs<VectorType>();
4377 unsigned DstNumElems = DstVTy->getNumElements();
4378
4379 const llvm::fltSemantics &HalfSem =
4380 S.getASTContext().getFloatTypeSemantics(T: S.getASTContext().HalfTy);
4381
4382 // imm[2] == 1 means use MXCSR rounding mode.
4383 // In that case, we can only evaluate if the conversion is exact.
4384 int ImmVal = Imm.getZExtValue();
4385 bool UseMXCSR = (ImmVal & 4) != 0;
4386 bool IsFPConstrained =
4387 Call->getFPFeaturesInEffect(LO: S.getASTContext().getLangOpts())
4388 .isFPConstrained();
4389
4390 llvm::RoundingMode RM;
4391 if (!UseMXCSR) {
4392 switch (ImmVal & 3) {
4393 case 0:
4394 RM = llvm::RoundingMode::NearestTiesToEven;
4395 break;
4396 case 1:
4397 RM = llvm::RoundingMode::TowardNegative;
4398 break;
4399 case 2:
4400 RM = llvm::RoundingMode::TowardPositive;
4401 break;
4402 case 3:
4403 RM = llvm::RoundingMode::TowardZero;
4404 break;
4405 default:
4406 llvm_unreachable("Invalid immediate rounding mode");
4407 }
4408 } else {
4409 // For MXCSR, we must check for exactness. We can use any rounding mode
4410 // for the trial conversion since the result is the same if it's exact.
4411 RM = llvm::RoundingMode::NearestTiesToEven;
4412 }
4413
4414 QualType DstElemQT = Dst.getFieldDesc()->getElemQualType();
4415 PrimType DstElemT = *S.getContext().classify(T: DstElemQT);
4416
4417 for (unsigned I = 0; I != SrcNumElems; ++I) {
4418 Floating SrcVal = Src.elem<Floating>(I);
4419 APFloat DstVal = SrcVal.getAPFloat();
4420
4421 bool LostInfo;
4422 APFloat::opStatus St = DstVal.convert(ToSemantics: HalfSem, RM, losesInfo: &LostInfo);
4423
4424 if (UseMXCSR && IsFPConstrained && St != APFloat::opOK) {
4425 S.FFDiag(SI: S.Current->getSource(PC: OpPC),
4426 DiagId: diag::note_constexpr_dynamic_rounding);
4427 return false;
4428 }
4429
4430 INT_TYPE_SWITCH_NO_BOOL(DstElemT, {
4431 // Convert the destination value's bit pattern to an unsigned integer,
4432 // then reconstruct the element using the target type's 'from' method.
4433 uint64_t RawBits = DstVal.bitcastToAPInt().getZExtValue();
4434 Dst.elem<T>(I) = T::from(RawBits);
4435 });
4436 }
4437
4438 // Zero out remaining elements if the destination has more elements
4439 // (e.g., vcvtps2ph converting 4 floats to 8 shorts).
4440 if (DstNumElems > SrcNumElems) {
4441 for (unsigned I = SrcNumElems; I != DstNumElems; ++I) {
4442 INT_TYPE_SWITCH_NO_BOOL(DstElemT, { Dst.elem<T>(I) = T::from(0); });
4443 }
4444 }
4445
4446 Dst.initializeAllElements();
4447 return true;
4448}
4449
4450static bool interp__builtin_ia32_multishiftqb(InterpState &S, CodePtr OpPC,
4451 const CallExpr *Call) {
4452 assert(Call->getNumArgs() == 2);
4453
4454 QualType ATy = Call->getArg(Arg: 0)->getType();
4455 QualType BTy = Call->getArg(Arg: 1)->getType();
4456 if (!ATy->isVectorType() || !BTy->isVectorType()) {
4457 return false;
4458 }
4459
4460 const Pointer &BPtr = S.Stk.pop<Pointer>();
4461 const Pointer &APtr = S.Stk.pop<Pointer>();
4462 const auto *AVecT = ATy->castAs<VectorType>();
4463 assert(AVecT->getNumElements() ==
4464 BTy->castAs<VectorType>()->getNumElements());
4465
4466 PrimType ElemT = *S.getContext().classify(T: AVecT->getElementType());
4467
4468 unsigned NumBytesInQWord = 8;
4469 unsigned NumBitsInByte = 8;
4470 unsigned NumBytes = AVecT->getNumElements();
4471 unsigned NumQWords = NumBytes / NumBytesInQWord;
4472 const Pointer &Dst = S.Stk.peek<Pointer>();
4473
4474 for (unsigned QWordId = 0; QWordId != NumQWords; ++QWordId) {
4475 APInt BQWord(64, 0);
4476 for (unsigned ByteIdx = 0; ByteIdx != NumBytesInQWord; ++ByteIdx) {
4477 unsigned Idx = QWordId * NumBytesInQWord + ByteIdx;
4478 INT_TYPE_SWITCH(ElemT, {
4479 uint64_t Byte = static_cast<uint64_t>(BPtr.elem<T>(Idx));
4480 BQWord.insertBits(APInt(8, Byte & 0xFF), ByteIdx * NumBitsInByte);
4481 });
4482 }
4483
4484 for (unsigned ByteIdx = 0; ByteIdx != NumBytesInQWord; ++ByteIdx) {
4485 unsigned Idx = QWordId * NumBytesInQWord + ByteIdx;
4486 uint64_t Ctrl = 0;
4487 INT_TYPE_SWITCH(
4488 ElemT, { Ctrl = static_cast<uint64_t>(APtr.elem<T>(Idx)) & 0x3F; });
4489
4490 APInt Byte(8, 0);
4491 for (unsigned BitIdx = 0; BitIdx != NumBitsInByte; ++BitIdx) {
4492 Byte.setBitVal(BitPosition: BitIdx, BitValue: BQWord[(Ctrl + BitIdx) & 0x3F]);
4493 }
4494 INT_TYPE_SWITCH(ElemT,
4495 { Dst.elem<T>(Idx) = T::from(Byte.getZExtValue()); });
4496 }
4497 }
4498
4499 Dst.initializeAllElements();
4500
4501 return true;
4502}
4503
4504static bool interp__builtin_ia32_gfni_affine(InterpState &S, CodePtr OpPC,
4505 const CallExpr *Call,
4506 bool Inverse) {
4507 assert(Call->getNumArgs() == 3);
4508 QualType XType = Call->getArg(Arg: 0)->getType();
4509 QualType AType = Call->getArg(Arg: 1)->getType();
4510 QualType ImmType = Call->getArg(Arg: 2)->getType();
4511 if (!XType->isVectorType() || !AType->isVectorType() ||
4512 !ImmType->isIntegerType()) {
4513 return false;
4514 }
4515
4516 Pointer X, A;
4517 APSInt Imm;
4518 if (!popToAPSInt(S, E: Call->getArg(Arg: 2), Out&: Imm))
4519 return false;
4520 A = S.Stk.pop<Pointer>();
4521 X = S.Stk.pop<Pointer>();
4522
4523 const Pointer &Dst = S.Stk.peek<Pointer>();
4524 const auto *AVecT = AType->castAs<VectorType>();
4525 assert(XType->castAs<VectorType>()->getNumElements() ==
4526 AVecT->getNumElements());
4527 unsigned NumBytesInQWord = 8;
4528 unsigned NumBytes = AVecT->getNumElements();
4529 unsigned NumBitsInQWord = 64;
4530 unsigned NumQWords = NumBytes / NumBytesInQWord;
4531 unsigned NumBitsInByte = 8;
4532 PrimType AElemT = *S.getContext().classify(T: AVecT->getElementType());
4533
4534 // computing A*X + Imm
4535 for (unsigned QWordIdx = 0; QWordIdx != NumQWords; ++QWordIdx) {
4536 // Extract the QWords from X, A
4537 APInt XQWord(NumBitsInQWord, 0);
4538 APInt AQWord(NumBitsInQWord, 0);
4539 for (unsigned ByteIdx = 0; ByteIdx != NumBytesInQWord; ++ByteIdx) {
4540 unsigned Idx = QWordIdx * NumBytesInQWord + ByteIdx;
4541 uint8_t XByte;
4542 uint8_t AByte;
4543 INT_TYPE_SWITCH(AElemT, {
4544 XByte = static_cast<uint8_t>(X.elem<T>(Idx));
4545 AByte = static_cast<uint8_t>(A.elem<T>(Idx));
4546 });
4547
4548 XQWord.insertBits(SubBits: APInt(NumBitsInByte, XByte), bitPosition: ByteIdx * NumBitsInByte);
4549 AQWord.insertBits(SubBits: APInt(NumBitsInByte, AByte), bitPosition: ByteIdx * NumBitsInByte);
4550 }
4551
4552 for (unsigned ByteIdx = 0; ByteIdx != NumBytesInQWord; ++ByteIdx) {
4553 unsigned Idx = QWordIdx * NumBytesInQWord + ByteIdx;
4554 uint8_t XByte =
4555 XQWord.lshr(shiftAmt: ByteIdx * NumBitsInByte).getLoBits(numBits: 8).getZExtValue();
4556 INT_TYPE_SWITCH(AElemT, {
4557 Dst.elem<T>(Idx) = T::from(GFNIAffine(XByte, AQWord, Imm, Inverse));
4558 });
4559 }
4560 }
4561 Dst.initializeAllElements();
4562 return true;
4563}
4564
4565static bool interp__builtin_ia32_gfni_mul(InterpState &S, CodePtr OpPC,
4566 const CallExpr *Call) {
4567 assert(Call->getNumArgs() == 2);
4568
4569 QualType AType = Call->getArg(Arg: 0)->getType();
4570 QualType BType = Call->getArg(Arg: 1)->getType();
4571 if (!AType->isVectorType() || !BType->isVectorType()) {
4572 return false;
4573 }
4574
4575 Pointer A, B;
4576 B = S.Stk.pop<Pointer>();
4577 A = S.Stk.pop<Pointer>();
4578
4579 const Pointer &Dst = S.Stk.peek<Pointer>();
4580 const auto *AVecT = AType->castAs<VectorType>();
4581 assert(AVecT->getNumElements() ==
4582 BType->castAs<VectorType>()->getNumElements());
4583
4584 PrimType AElemT = *S.getContext().classify(T: AVecT->getElementType());
4585 unsigned NumBytes = A.getNumElems();
4586
4587 for (unsigned ByteIdx = 0; ByteIdx != NumBytes; ++ByteIdx) {
4588 uint8_t AByte, BByte;
4589 INT_TYPE_SWITCH(AElemT, {
4590 AByte = static_cast<uint8_t>(A.elem<T>(ByteIdx));
4591 BByte = static_cast<uint8_t>(B.elem<T>(ByteIdx));
4592 Dst.elem<T>(ByteIdx) = T::from(GFNIMul(AByte, BByte));
4593 });
4594 }
4595
4596 Dst.initializeAllElements();
4597 return true;
4598}
4599
4600static bool interp__builtin_ia32_vpdp(InterpState &S, CodePtr OpPC,
4601 const CallExpr *Call, bool IsSaturating) {
4602 assert(Call->getNumArgs() == 3);
4603
4604 QualType SrcT = Call->getArg(Arg: 0)->getType();
4605 QualType OpAT = Call->getArg(Arg: 1)->getType();
4606 QualType OpBT = Call->getArg(Arg: 2)->getType();
4607 QualType DstT = Call->getType();
4608 if (!SrcT->isVectorType() || !OpAT->isVectorType() || !OpBT->isVectorType() ||
4609 !DstT->isVectorType())
4610 return false;
4611
4612 const auto *SrcVecT = SrcT->castAs<VectorType>();
4613 const auto *OpAVecT = OpAT->castAs<VectorType>();
4614 const auto *OpBVecT = OpBT->castAs<VectorType>();
4615 const auto *DstVecT = DstT->castAs<VectorType>();
4616
4617 assert(OpAVecT->getNumElements() == OpBVecT->getNumElements());
4618
4619 unsigned NumSrcElems = SrcVecT->getNumElements();
4620 unsigned NumOperandElems = OpAVecT->getNumElements();
4621 unsigned ElemsPerLane = NumOperandElems / NumSrcElems;
4622
4623 PrimType SrcElemT = *S.getContext().classify(T: SrcVecT->getElementType());
4624 PrimType OpAElemT = *S.getContext().classify(T: OpAVecT->getElementType());
4625 PrimType OpBElemT = *S.getContext().classify(T: OpBVecT->getElementType());
4626 PrimType DstElemT = *S.getContext().classify(T: DstVecT->getElementType());
4627
4628 assert(SrcElemT == DstElemT);
4629
4630 const Pointer &OpBPtr = S.Stk.pop<Pointer>();
4631 const Pointer &OpAPtr = S.Stk.pop<Pointer>();
4632 const Pointer &SrcPtr = S.Stk.pop<Pointer>();
4633 const Pointer &Dst = S.Stk.peek<Pointer>();
4634
4635 for (unsigned I = 0; I != NumSrcElems; ++I) {
4636 APSInt Acc;
4637 INT_TYPE_SWITCH_NO_BOOL(SrcElemT, { Acc = SrcPtr.elem<T>(I).toAPSInt(); });
4638 Acc = Acc.sext(width: 64);
4639 for (unsigned J = 0; J != ElemsPerLane; ++J) {
4640 APSInt OpA, OpB;
4641 INT_TYPE_SWITCH_NO_BOOL(
4642 OpAElemT, { OpA = OpAPtr.elem<T>(ElemsPerLane * I + J).toAPSInt(); });
4643 INT_TYPE_SWITCH_NO_BOOL(
4644 OpBElemT, { OpB = OpBPtr.elem<T>(ElemsPerLane * I + J).toAPSInt(); });
4645 OpA = APSInt(OpA.extend(width: 64), false);
4646 OpB = APSInt(OpB.extend(width: 64), false);
4647 Acc += OpA * OpB;
4648 }
4649 if (IsSaturating)
4650 Acc = APSInt(Acc.truncSSat(width: 32), false);
4651 else
4652 Acc = APSInt(Acc.trunc(width: 32), false);
4653 INT_TYPE_SWITCH_NO_BOOL(DstElemT,
4654 { Dst.elem<T>(I) = static_cast<T>(Acc); });
4655 }
4656 Dst.initializeAllElements();
4657 return true;
4658}
4659
4660// Bit Matrix Multiply and Accumulate (AVX512BMM). Each 256-bit lane holds a
4661// 16x16 bit matrix as 16 x i16 elements; element i is row i and bit j of that
4662// element is entry [i][j]. The accumulator (third argument, src1 in the AMD
4663// ISA) provides the initial value of each result bit, into which the bit-matrix
4664// product of the first two arguments (src2 * src3) is reduced with OR (vbmacor)
4665// or XOR (vbmacxor):
4666// for i in 0..15, j in 0..15:
4667// bit = C[16*i+j]
4668// for k in 0..15: bit OP= A[16*i+k] & B[16*k+j]
4669// dest[16*i+j] = bit
4670static bool interp__builtin_ia32_bmac(InterpState &S, CodePtr OpPC,
4671 const CallExpr *Call, bool IsXor) {
4672 assert(Call->getNumArgs() == 3);
4673
4674 // AST-based type checks before popping the stack.
4675 QualType AType = Call->getArg(Arg: 0)->getType();
4676 QualType BType = Call->getArg(Arg: 1)->getType();
4677 QualType CType = Call->getArg(Arg: 2)->getType();
4678 if (!AType->isVectorType() || !BType->isVectorType() ||
4679 !CType->isVectorType())
4680 return false;
4681
4682 const Pointer &C = S.Stk.pop<Pointer>();
4683 const Pointer &B = S.Stk.pop<Pointer>();
4684 const Pointer &A = S.Stk.pop<Pointer>();
4685 const Pointer &Dst = S.Stk.peek<Pointer>();
4686
4687 // check if all three primitive arrays are with 16-bit elements.
4688 auto isValid16BitArray = [](const Pointer &P) {
4689 const Descriptor *D = P.getFieldDesc();
4690 if (!D->isPrimitiveArray())
4691 return false;
4692 PrimType PT = D->getPrimType();
4693 return ((PT == PT_Sint16) || (PT == PT_Uint16));
4694 };
4695
4696 if (!isValid16BitArray(A) || !isValid16BitArray(B) || !isValid16BitArray(C))
4697 return false;
4698
4699 PrimType ElemT = A.getFieldDesc()->getPrimType();
4700 unsigned NumElems = A.getNumElems();
4701 assert(NumElems % 16 == 0 && "BMM operates on 256-bit lanes of 16 x i16");
4702 bool DstUnsigned = ElemT == PT_Uint16;
4703
4704 // Lanes are always 16-bit; gather them so the reduction below is untyped.
4705 SmallVector<uint16_t> AVals(NumElems), BVals(NumElems), Acc(NumElems);
4706 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
4707 for (unsigned I = 0; I != NumElems; ++I) {
4708 AVals[I] = (uint16_t)A.elem<T>(I).toAPSInt().getZExtValue();
4709 BVals[I] = (uint16_t)B.elem<T>(I).toAPSInt().getZExtValue();
4710 Acc[I] = (uint16_t)C.elem<T>(I).toAPSInt().getZExtValue();
4711 }
4712 });
4713
4714 for (unsigned Lane = 0; Lane != NumElems; Lane += 16) {
4715 for (unsigned I = 0; I != 16; ++I) {
4716 uint16_t AVal = AVals[Lane + I], DVal = Acc[Lane + I];
4717 for (unsigned J = 0; J != 16; ++J) {
4718 // Seed the reduction with the accumulator bit, then fold in each
4719 // product term with the same operator (OR for vbmacor, XOR for
4720 // vbmacxor).
4721 unsigned Bit = (DVal >> J) & 1u;
4722 for (unsigned K = 0; K != 16; ++K) {
4723 unsigned Product = ((AVal >> K) & 1u) & ((BVals[Lane + K] >> J) & 1u);
4724 Bit = IsXor ? (Bit ^ Product) : (Bit | Product);
4725 }
4726 DVal = (DVal & ~(uint16_t(1) << J)) | (uint16_t(Bit) << J);
4727 }
4728 Acc[Lane + I] = DVal;
4729 }
4730 }
4731
4732 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
4733 for (unsigned I = 0; I != NumElems; ++I)
4734 Dst.elem<T>(I) = static_cast<T>(APSInt(APInt(16, Acc[I]), DstUnsigned));
4735 });
4736 Dst.initializeAllElements();
4737 return true;
4738}
4739
4740static bool interp_builtin_ia32_cvt_scalar_to_int(InterpState &S, CodePtr OpPC,
4741 const CallExpr *E) {
4742 Pointer SrcVecPtr = S.Stk.pop<Pointer>();
4743 const Floating &FloatElem = SrcVecPtr.elem<Floating>(I: 0);
4744
4745 unsigned BitWidth = S.getASTContext().getIntWidth(T: E->getType());
4746 bool IsUnsigned = E->getType()->isUnsignedIntegerType();
4747
4748 llvm::APSInt IntResult(BitWidth, IsUnsigned);
4749 bool IsExact = false;
4750 // We only allow exact conversions so rounding mode does not matter for cvt*
4751 // and cvtt* builtins
4752 FloatElem.getAPFloat().convertToInteger(
4753 Result&: IntResult, RM: llvm::APFloat::rmTowardZero, IsExact: &IsExact);
4754 if (!IsExact)
4755 return false;
4756
4757 pushInteger(S, Val: IntResult, QT: E->getType());
4758 return true;
4759}
4760
4761static bool interp_builtin_ia32_cvt_vector_to_int(InterpState &S, CodePtr OpPC,
4762 const CallExpr *E) {
4763 Pointer SrcVecPtr = S.Stk.pop<Pointer>();
4764 const Pointer &Dst = S.Stk.peek<Pointer>();
4765
4766 unsigned NumSrcElems = SrcVecPtr.getNumElems();
4767 unsigned NumDstElems = Dst.getNumElems();
4768
4769 if (NumSrcElems > NumDstElems)
4770 return false;
4771
4772 QualType ElemType = Dst.getFieldDesc()->getElemQualType();
4773 unsigned BitWidth = S.getASTContext().getIntWidth(T: ElemType);
4774 bool IsUnsigned = ElemType->isUnsignedIntegerType();
4775
4776 PrimType ElemT = *S.getContext().classify(T: ElemType);
4777 for (unsigned I = 0; I != NumSrcElems; ++I) {
4778 const Floating &FloatElem = SrcVecPtr.elem<Floating>(I);
4779 llvm::APSInt IntResult(BitWidth, IsUnsigned);
4780
4781 bool IsExact = false;
4782 // We only allow exact conversions so rounding mode does not matter for
4783 // cvt* and cvtt* builtins
4784 FloatElem.getAPFloat().convertToInteger(
4785 Result&: IntResult, RM: llvm::APFloat::rmTowardZero, IsExact: &IsExact);
4786 if (!IsExact)
4787 return false;
4788 INT_TYPE_SWITCH_NO_BOOL(
4789 ElemT, { Dst.elem<T>(I) = T::from(IntResult.getZExtValue()); });
4790 }
4791
4792 // Zero out remaining elements if the destination has more elements
4793 // (e.g., cvtpd2dq converting 2 doubles(_m128d) to 2 ints stored in _m128i).
4794 for (unsigned I = NumSrcElems; I != NumDstElems; ++I)
4795 INT_TYPE_SWITCH_NO_BOOL(ElemT, { Dst.elem<T>(I) = T::from(0); });
4796
4797 Dst.initializeAllElements();
4798 return true;
4799}
4800
4801static bool interp__builtin_meta_is_type(InterpState &S, CodePtr OpPC,
4802 const CallExpr *) {
4803 const Reflect &R = S.Stk.pop<Reflect>();
4804 S.Stk.push<Boolean>(Args: Boolean::from(Value: R.getKind() == ReflectionKind::Type));
4805 return true;
4806}
4807
4808bool InterpretBuiltin(InterpState &S, CodePtr OpPC, const CallExpr *Call,
4809 uint32_t BuiltinID) {
4810 const InterpFrame *Frame = S.Current;
4811 switch (BuiltinID) {
4812 case Builtin::BI__builtin_is_constant_evaluated:
4813 return interp__builtin_is_constant_evaluated(S, OpPC, Frame, Call);
4814
4815 case Builtin::BI__builtin_assume:
4816 case Builtin::BI__assume:
4817 return interp__builtin_assume(S, OpPC, Frame, Call);
4818
4819 case Builtin::BI__builtin_strcmp:
4820 case Builtin::BIstrcmp:
4821 case Builtin::BI__builtin_strncmp:
4822 case Builtin::BIstrncmp:
4823 case Builtin::BI__builtin_wcsncmp:
4824 case Builtin::BIwcsncmp:
4825 case Builtin::BI__builtin_wcscmp:
4826 case Builtin::BIwcscmp:
4827 return interp__builtin_strcmp(S, OpPC, Frame, Call, ID: BuiltinID);
4828
4829 case Builtin::BI__builtin_strlen:
4830 case Builtin::BIstrlen:
4831 case Builtin::BI__builtin_wcslen:
4832 case Builtin::BIwcslen:
4833 return interp__builtin_strlen(S, OpPC, Frame, Call, ID: BuiltinID);
4834
4835 case Builtin::BI__builtin_nan:
4836 case Builtin::BI__builtin_nanf:
4837 case Builtin::BI__builtin_nanl:
4838 case Builtin::BI__builtin_nanf16:
4839 case Builtin::BI__builtin_nanf128:
4840 return interp__builtin_nan(S, OpPC, Frame, Call, /*Signaling=*/false);
4841
4842 case Builtin::BI__builtin_nans:
4843 case Builtin::BI__builtin_nansf:
4844 case Builtin::BI__builtin_nansl:
4845 case Builtin::BI__builtin_nansf16:
4846 case Builtin::BI__builtin_nansf128:
4847 return interp__builtin_nan(S, OpPC, Frame, Call, /*Signaling=*/true);
4848
4849 case Builtin::BI__builtin_huge_val:
4850 case Builtin::BI__builtin_huge_valf:
4851 case Builtin::BI__builtin_huge_vall:
4852 case Builtin::BI__builtin_huge_valf16:
4853 case Builtin::BI__builtin_huge_valf128:
4854 case Builtin::BI__builtin_inf:
4855 case Builtin::BI__builtin_inff:
4856 case Builtin::BI__builtin_infl:
4857 case Builtin::BI__builtin_inff16:
4858 case Builtin::BI__builtin_inff128:
4859 return interp__builtin_inf(S, OpPC, Frame, Call);
4860
4861 case Builtin::BI__builtin_copysign:
4862 case Builtin::BI__builtin_copysignf:
4863 case Builtin::BI__builtin_copysignl:
4864 case Builtin::BI__builtin_copysignf128:
4865 return interp__builtin_copysign(S, OpPC, Frame);
4866
4867 case Builtin::BI__builtin_fmin:
4868 case Builtin::BI__builtin_fminf:
4869 case Builtin::BI__builtin_fminl:
4870 case Builtin::BI__builtin_fminf16:
4871 case Builtin::BI__builtin_fminf128:
4872 return interp__builtin_fmin(S, OpPC, Frame, /*IsNumBuiltin=*/false);
4873
4874 case Builtin::BI__builtin_fminimum_num:
4875 case Builtin::BI__builtin_fminimum_numf:
4876 case Builtin::BI__builtin_fminimum_numl:
4877 case Builtin::BI__builtin_fminimum_numf16:
4878 case Builtin::BI__builtin_fminimum_numf128:
4879 return interp__builtin_fmin(S, OpPC, Frame, /*IsNumBuiltin=*/true);
4880
4881 case Builtin::BI__builtin_fmax:
4882 case Builtin::BI__builtin_fmaxf:
4883 case Builtin::BI__builtin_fmaxl:
4884 case Builtin::BI__builtin_fmaxf16:
4885 case Builtin::BI__builtin_fmaxf128:
4886 return interp__builtin_fmax(S, OpPC, Frame, /*IsNumBuiltin=*/false);
4887
4888 case Builtin::BI__builtin_fmaximum_num:
4889 case Builtin::BI__builtin_fmaximum_numf:
4890 case Builtin::BI__builtin_fmaximum_numl:
4891 case Builtin::BI__builtin_fmaximum_numf16:
4892 case Builtin::BI__builtin_fmaximum_numf128:
4893 return interp__builtin_fmax(S, OpPC, Frame, /*IsNumBuiltin=*/true);
4894
4895 case Builtin::BI__builtin_isnan:
4896 return interp__builtin_isnan(S, OpPC, Frame, Call);
4897
4898 case Builtin::BI__builtin_issignaling:
4899 return interp__builtin_issignaling(S, OpPC, Frame, Call);
4900
4901 case Builtin::BI__builtin_isinf:
4902 return interp__builtin_isinf(S, OpPC, Frame, /*Sign=*/CheckSign: false, Call);
4903
4904 case Builtin::BI__builtin_isinf_sign:
4905 return interp__builtin_isinf(S, OpPC, Frame, /*Sign=*/CheckSign: true, Call);
4906
4907 case Builtin::BI__builtin_isfinite:
4908 return interp__builtin_isfinite(S, OpPC, Frame, Call);
4909
4910 case Builtin::BI__builtin_isnormal:
4911 return interp__builtin_isnormal(S, OpPC, Frame, Call);
4912
4913 case Builtin::BI__builtin_issubnormal:
4914 return interp__builtin_issubnormal(S, OpPC, Frame, Call);
4915
4916 case Builtin::BI__builtin_iszero:
4917 return interp__builtin_iszero(S, OpPC, Frame, Call);
4918
4919 case Builtin::BI__builtin_signbit:
4920 case Builtin::BI__builtin_signbitf:
4921 case Builtin::BI__builtin_signbitl:
4922 return interp__builtin_signbit(S, OpPC, Frame, Call);
4923
4924 case Builtin::BI__builtin_isgreater:
4925 case Builtin::BI__builtin_isgreaterequal:
4926 case Builtin::BI__builtin_isless:
4927 case Builtin::BI__builtin_islessequal:
4928 case Builtin::BI__builtin_islessgreater:
4929 case Builtin::BI__builtin_isunordered:
4930 return interp_floating_comparison(S, OpPC, Call, ID: BuiltinID);
4931
4932 case Builtin::BI__builtin_isfpclass:
4933 return interp__builtin_isfpclass(S, OpPC, Frame, Call);
4934
4935 case Builtin::BI__builtin_fpclassify:
4936 return interp__builtin_fpclassify(S, OpPC, Frame, Call);
4937
4938 case Builtin::BI__builtin_fabs:
4939 case Builtin::BI__builtin_fabsf:
4940 case Builtin::BI__builtin_fabsl:
4941 case Builtin::BI__builtin_fabsf128:
4942 return interp__builtin_fabs(S, OpPC, Frame);
4943
4944 case Builtin::BI__builtin_abs:
4945 case Builtin::BI__builtin_labs:
4946 case Builtin::BI__builtin_llabs:
4947 return interp__builtin_abs(S, OpPC, Frame, Call);
4948
4949 case Builtin::BI__builtin_popcount:
4950 case Builtin::BI__builtin_popcountl:
4951 case Builtin::BI__builtin_popcountll:
4952 case Builtin::BI__builtin_popcountg:
4953 case Builtin::BI__popcnt16: // Microsoft variants of popcount
4954 case Builtin::BI__popcnt:
4955 case Builtin::BI__popcnt64:
4956 return interp__builtin_popcount(S, OpPC, Frame, Call);
4957
4958 case Builtin::BI__builtin_parity:
4959 case Builtin::BI__builtin_parityl:
4960 case Builtin::BI__builtin_parityll:
4961 return interp__builtin_elementwise_int_unaryop(
4962 S, OpPC, Call, Fn: [](const APSInt &Val) {
4963 return APInt(Val.getBitWidth(), Val.popcount() % 2);
4964 });
4965 case Builtin::BI__builtin_clrsb:
4966 case Builtin::BI__builtin_clrsbl:
4967 case Builtin::BI__builtin_clrsbll:
4968 return interp__builtin_elementwise_int_unaryop(
4969 S, OpPC, Call, Fn: [](const APSInt &Val) {
4970 return APInt(Val.getBitWidth(),
4971 Val.getBitWidth() - Val.getSignificantBits());
4972 });
4973 case Builtin::BI__builtin_bitreverseg:
4974 case Builtin::BI__builtin_bitreverse8:
4975 case Builtin::BI__builtin_bitreverse16:
4976 case Builtin::BI__builtin_bitreverse32:
4977 case Builtin::BI__builtin_bitreverse64:
4978 return interp__builtin_elementwise_int_unaryop(
4979 S, OpPC, Call, Fn: [](const APSInt &Val) { return Val.reverseBits(); });
4980
4981 case Builtin::BI__builtin_classify_type:
4982 return interp__builtin_classify_type(S, OpPC, Frame, Call);
4983
4984 case Builtin::BI__builtin_expect:
4985 case Builtin::BI__builtin_expect_with_probability:
4986 return interp__builtin_expect(S, OpPC, Frame, Call);
4987
4988 case Builtin::BI__builtin_rotateleft8:
4989 case Builtin::BI__builtin_rotateleft16:
4990 case Builtin::BI__builtin_rotateleft32:
4991 case Builtin::BI__builtin_rotateleft64:
4992 case Builtin::BI__builtin_stdc_rotate_left:
4993 case Builtin::BIstdc_rotate_left_uc:
4994 case Builtin::BIstdc_rotate_left_us:
4995 case Builtin::BIstdc_rotate_left_ui:
4996 case Builtin::BIstdc_rotate_left_ul:
4997 case Builtin::BIstdc_rotate_left_ull:
4998 case Builtin::BI_rotl8: // Microsoft variants of rotate left
4999 case Builtin::BI_rotl16:
5000 case Builtin::BI_rotl:
5001 case Builtin::BI_lrotl:
5002 case Builtin::BI_rotl64:
5003 case Builtin::BI__builtin_rotateright8:
5004 case Builtin::BI__builtin_rotateright16:
5005 case Builtin::BI__builtin_rotateright32:
5006 case Builtin::BI__builtin_rotateright64:
5007 case Builtin::BI__builtin_stdc_rotate_right:
5008 case Builtin::BIstdc_rotate_right_uc:
5009 case Builtin::BIstdc_rotate_right_us:
5010 case Builtin::BIstdc_rotate_right_ui:
5011 case Builtin::BIstdc_rotate_right_ul:
5012 case Builtin::BIstdc_rotate_right_ull:
5013 case Builtin::BI_rotr8: // Microsoft variants of rotate right
5014 case Builtin::BI_rotr16:
5015 case Builtin::BI_rotr:
5016 case Builtin::BI_lrotr:
5017 case Builtin::BI_rotr64: {
5018 // Determine if this is a rotate right operation
5019 bool IsRotateRight;
5020 switch (BuiltinID) {
5021 case Builtin::BI__builtin_rotateright8:
5022 case Builtin::BI__builtin_rotateright16:
5023 case Builtin::BI__builtin_rotateright32:
5024 case Builtin::BI__builtin_rotateright64:
5025 case Builtin::BI__builtin_stdc_rotate_right:
5026 case Builtin::BIstdc_rotate_right_uc:
5027 case Builtin::BIstdc_rotate_right_us:
5028 case Builtin::BIstdc_rotate_right_ui:
5029 case Builtin::BIstdc_rotate_right_ul:
5030 case Builtin::BIstdc_rotate_right_ull:
5031 case Builtin::BI_rotr8:
5032 case Builtin::BI_rotr16:
5033 case Builtin::BI_rotr:
5034 case Builtin::BI_lrotr:
5035 case Builtin::BI_rotr64:
5036 IsRotateRight = true;
5037 break;
5038 default:
5039 IsRotateRight = false;
5040 break;
5041 }
5042
5043 return interp__builtin_elementwise_int_binop(
5044 S, OpPC, Call, Fn: [IsRotateRight](const APSInt &Value, APSInt Amount) {
5045 Amount = NormalizeRotateAmount(Value, Amount);
5046 return IsRotateRight ? Value.rotr(rotateAmt: Amount.getZExtValue())
5047 : Value.rotl(rotateAmt: Amount.getZExtValue());
5048 });
5049 }
5050
5051 case Builtin::BIstdc_leading_zeros_uc:
5052 case Builtin::BIstdc_leading_zeros_us:
5053 case Builtin::BIstdc_leading_zeros_ui:
5054 case Builtin::BIstdc_leading_zeros_ul:
5055 case Builtin::BIstdc_leading_zeros_ull:
5056 case Builtin::BI__builtin_stdc_leading_zeros: {
5057 unsigned ResWidth = S.getASTContext().getIntWidth(T: Call->getType());
5058 return interp__builtin_elementwise_int_unaryop(
5059 S, OpPC, Call, Fn: [ResWidth](const APSInt &Val) {
5060 return APInt(ResWidth, Val.countl_zero());
5061 });
5062 }
5063
5064 case Builtin::BIstdc_leading_ones_uc:
5065 case Builtin::BIstdc_leading_ones_us:
5066 case Builtin::BIstdc_leading_ones_ui:
5067 case Builtin::BIstdc_leading_ones_ul:
5068 case Builtin::BIstdc_leading_ones_ull:
5069 case Builtin::BI__builtin_stdc_leading_ones: {
5070 unsigned ResWidth = S.getASTContext().getIntWidth(T: Call->getType());
5071 return interp__builtin_elementwise_int_unaryop(
5072 S, OpPC, Call, Fn: [ResWidth](const APSInt &Val) {
5073 return APInt(ResWidth, Val.countl_one());
5074 });
5075 }
5076
5077 case Builtin::BIstdc_trailing_zeros_uc:
5078 case Builtin::BIstdc_trailing_zeros_us:
5079 case Builtin::BIstdc_trailing_zeros_ui:
5080 case Builtin::BIstdc_trailing_zeros_ul:
5081 case Builtin::BIstdc_trailing_zeros_ull:
5082 case Builtin::BI__builtin_stdc_trailing_zeros: {
5083 unsigned ResWidth = S.getASTContext().getIntWidth(T: Call->getType());
5084 return interp__builtin_elementwise_int_unaryop(
5085 S, OpPC, Call, Fn: [ResWidth](const APSInt &Val) {
5086 return APInt(ResWidth, Val.countr_zero());
5087 });
5088 }
5089
5090 case Builtin::BIstdc_trailing_ones_uc:
5091 case Builtin::BIstdc_trailing_ones_us:
5092 case Builtin::BIstdc_trailing_ones_ui:
5093 case Builtin::BIstdc_trailing_ones_ul:
5094 case Builtin::BIstdc_trailing_ones_ull:
5095 case Builtin::BI__builtin_stdc_trailing_ones: {
5096 unsigned ResWidth = S.getASTContext().getIntWidth(T: Call->getType());
5097 return interp__builtin_elementwise_int_unaryop(
5098 S, OpPC, Call, Fn: [ResWidth](const APSInt &Val) {
5099 return APInt(ResWidth, Val.countr_one());
5100 });
5101 }
5102
5103 case Builtin::BIstdc_first_leading_zero_uc:
5104 case Builtin::BIstdc_first_leading_zero_us:
5105 case Builtin::BIstdc_first_leading_zero_ui:
5106 case Builtin::BIstdc_first_leading_zero_ul:
5107 case Builtin::BIstdc_first_leading_zero_ull:
5108 case Builtin::BI__builtin_stdc_first_leading_zero: {
5109 unsigned ResWidth = S.getASTContext().getIntWidth(T: Call->getType());
5110 return interp__builtin_elementwise_int_unaryop(
5111 S, OpPC, Call, Fn: [ResWidth](const APSInt &Val) {
5112 return APInt(ResWidth, Val.isAllOnes() ? 0 : Val.countl_one() + 1);
5113 });
5114 }
5115
5116 case Builtin::BIstdc_first_leading_one_uc:
5117 case Builtin::BIstdc_first_leading_one_us:
5118 case Builtin::BIstdc_first_leading_one_ui:
5119 case Builtin::BIstdc_first_leading_one_ul:
5120 case Builtin::BIstdc_first_leading_one_ull:
5121 case Builtin::BI__builtin_stdc_first_leading_one: {
5122 unsigned ResWidth = S.getASTContext().getIntWidth(T: Call->getType());
5123 return interp__builtin_elementwise_int_unaryop(
5124 S, OpPC, Call, Fn: [ResWidth](const APSInt &Val) {
5125 return APInt(ResWidth, Val.isZero() ? 0 : Val.countl_zero() + 1);
5126 });
5127 }
5128
5129 case Builtin::BIstdc_first_trailing_zero_uc:
5130 case Builtin::BIstdc_first_trailing_zero_us:
5131 case Builtin::BIstdc_first_trailing_zero_ui:
5132 case Builtin::BIstdc_first_trailing_zero_ul:
5133 case Builtin::BIstdc_first_trailing_zero_ull:
5134 case Builtin::BI__builtin_stdc_first_trailing_zero: {
5135 unsigned ResWidth = S.getASTContext().getIntWidth(T: Call->getType());
5136 return interp__builtin_elementwise_int_unaryop(
5137 S, OpPC, Call, Fn: [ResWidth](const APSInt &Val) {
5138 return APInt(ResWidth, Val.isAllOnes() ? 0 : Val.countr_one() + 1);
5139 });
5140 }
5141
5142 case Builtin::BIstdc_first_trailing_one_uc:
5143 case Builtin::BIstdc_first_trailing_one_us:
5144 case Builtin::BIstdc_first_trailing_one_ui:
5145 case Builtin::BIstdc_first_trailing_one_ul:
5146 case Builtin::BIstdc_first_trailing_one_ull:
5147 case Builtin::BI__builtin_stdc_first_trailing_one: {
5148 unsigned ResWidth = S.getASTContext().getIntWidth(T: Call->getType());
5149 return interp__builtin_elementwise_int_unaryop(
5150 S, OpPC, Call, Fn: [ResWidth](const APSInt &Val) {
5151 return APInt(ResWidth, Val.isZero() ? 0 : Val.countr_zero() + 1);
5152 });
5153 }
5154
5155 case Builtin::BIstdc_count_zeros_uc:
5156 case Builtin::BIstdc_count_zeros_us:
5157 case Builtin::BIstdc_count_zeros_ui:
5158 case Builtin::BIstdc_count_zeros_ul:
5159 case Builtin::BIstdc_count_zeros_ull:
5160 case Builtin::BI__builtin_stdc_count_zeros: {
5161 unsigned ResWidth = S.getASTContext().getIntWidth(T: Call->getType());
5162 return interp__builtin_elementwise_int_unaryop(
5163 S, OpPC, Call, Fn: [ResWidth](const APSInt &Val) {
5164 unsigned BitWidth = Val.getBitWidth();
5165 return APInt(ResWidth, BitWidth - Val.popcount());
5166 });
5167 }
5168
5169 case Builtin::BIstdc_count_ones_uc:
5170 case Builtin::BIstdc_count_ones_us:
5171 case Builtin::BIstdc_count_ones_ui:
5172 case Builtin::BIstdc_count_ones_ul:
5173 case Builtin::BIstdc_count_ones_ull:
5174 case Builtin::BI__builtin_stdc_count_ones: {
5175 unsigned ResWidth = S.getASTContext().getIntWidth(T: Call->getType());
5176 return interp__builtin_elementwise_int_unaryop(
5177 S, OpPC, Call, Fn: [ResWidth](const APSInt &Val) {
5178 return APInt(ResWidth, Val.popcount());
5179 });
5180 }
5181
5182 case Builtin::BIstdc_has_single_bit_uc:
5183 case Builtin::BIstdc_has_single_bit_us:
5184 case Builtin::BIstdc_has_single_bit_ui:
5185 case Builtin::BIstdc_has_single_bit_ul:
5186 case Builtin::BIstdc_has_single_bit_ull:
5187 case Builtin::BI__builtin_stdc_has_single_bit: {
5188 unsigned ResWidth = S.getASTContext().getIntWidth(T: Call->getType());
5189 return interp__builtin_elementwise_int_unaryop(
5190 S, OpPC, Call, Fn: [ResWidth](const APSInt &Val) {
5191 return APInt(ResWidth, Val.popcount() == 1 ? 1 : 0);
5192 });
5193 }
5194
5195 case Builtin::BIstdc_bit_width_uc:
5196 case Builtin::BIstdc_bit_width_us:
5197 case Builtin::BIstdc_bit_width_ui:
5198 case Builtin::BIstdc_bit_width_ul:
5199 case Builtin::BIstdc_bit_width_ull:
5200 case Builtin::BI__builtin_stdc_bit_width: {
5201 unsigned ResWidth = S.getASTContext().getIntWidth(T: Call->getType());
5202 return interp__builtin_elementwise_int_unaryop(
5203 S, OpPC, Call, Fn: [ResWidth](const APSInt &Val) {
5204 unsigned BitWidth = Val.getBitWidth();
5205 return APInt(ResWidth, BitWidth - Val.countl_zero());
5206 });
5207 }
5208
5209 case Builtin::BIstdc_bit_floor_uc:
5210 case Builtin::BIstdc_bit_floor_us:
5211 case Builtin::BIstdc_bit_floor_ui:
5212 case Builtin::BIstdc_bit_floor_ul:
5213 case Builtin::BIstdc_bit_floor_ull:
5214 case Builtin::BI__builtin_stdc_bit_floor:
5215 return interp__builtin_elementwise_int_unaryop(
5216 S, OpPC, Call, Fn: [](const APSInt &Val) {
5217 unsigned BitWidth = Val.getBitWidth();
5218 if (Val.isZero())
5219 return APInt::getZero(numBits: BitWidth);
5220 return APInt::getOneBitSet(numBits: BitWidth,
5221 BitNo: BitWidth - Val.countl_zero() - 1);
5222 });
5223
5224 case Builtin::BIstdc_bit_ceil_uc:
5225 case Builtin::BIstdc_bit_ceil_us:
5226 case Builtin::BIstdc_bit_ceil_ui:
5227 case Builtin::BIstdc_bit_ceil_ul:
5228 case Builtin::BIstdc_bit_ceil_ull:
5229 case Builtin::BI__builtin_stdc_bit_ceil:
5230 return interp__builtin_elementwise_int_unaryop(
5231 S, OpPC, Call, Fn: [](const APSInt &Val) {
5232 unsigned BitWidth = Val.getBitWidth();
5233 if (Val.ule(RHS: 1))
5234 return APInt(BitWidth, 1);
5235 APInt V = Val;
5236 APInt ValMinusOne = V - 1;
5237 unsigned LeadingZeros = ValMinusOne.countl_zero();
5238 if (LeadingZeros == 0)
5239 return APInt(BitWidth, 0); // overflows; wrap to 0
5240 return APInt::getOneBitSet(numBits: BitWidth, BitNo: BitWidth - LeadingZeros);
5241 });
5242
5243 case Builtin::BI__builtin_ffs:
5244 case Builtin::BI__builtin_ffsl:
5245 case Builtin::BI__builtin_ffsll:
5246 return interp__builtin_elementwise_int_unaryop(
5247 S, OpPC, Call, Fn: [](const APSInt &Val) {
5248 return APInt(Val.getBitWidth(),
5249 Val.isZero() ? 0u : Val.countTrailingZeros() + 1u);
5250 });
5251
5252 case Builtin::BIaddressof:
5253 case Builtin::BI__addressof:
5254 case Builtin::BI__builtin_addressof:
5255 assert(isNoopBuiltin(BuiltinID));
5256 return interp__builtin_addressof(S, OpPC, Frame, Call);
5257
5258 case Builtin::BIas_const:
5259 case Builtin::BIforward:
5260 case Builtin::BIforward_like:
5261 case Builtin::BImove:
5262 case Builtin::BImove_if_noexcept:
5263 assert(isNoopBuiltin(BuiltinID));
5264 return interp__builtin_move(S, OpPC, Frame, Call);
5265
5266 case Builtin::BI__builtin_eh_return_data_regno:
5267 return interp__builtin_eh_return_data_regno(S, OpPC, Frame, Call);
5268
5269 case Builtin::BI__builtin_launder:
5270 assert(isNoopBuiltin(BuiltinID));
5271 return true;
5272
5273 case Builtin::BI__builtin_add_overflow:
5274 case Builtin::BI__builtin_sub_overflow:
5275 case Builtin::BI__builtin_mul_overflow:
5276 case Builtin::BI__builtin_sadd_overflow:
5277 case Builtin::BI__builtin_uadd_overflow:
5278 case Builtin::BI__builtin_uaddl_overflow:
5279 case Builtin::BI__builtin_uaddll_overflow:
5280 case Builtin::BI__builtin_usub_overflow:
5281 case Builtin::BI__builtin_usubl_overflow:
5282 case Builtin::BI__builtin_usubll_overflow:
5283 case Builtin::BI__builtin_umul_overflow:
5284 case Builtin::BI__builtin_umull_overflow:
5285 case Builtin::BI__builtin_umulll_overflow:
5286 case Builtin::BI__builtin_saddl_overflow:
5287 case Builtin::BI__builtin_saddll_overflow:
5288 case Builtin::BI__builtin_ssub_overflow:
5289 case Builtin::BI__builtin_ssubl_overflow:
5290 case Builtin::BI__builtin_ssubll_overflow:
5291 case Builtin::BI__builtin_smul_overflow:
5292 case Builtin::BI__builtin_smull_overflow:
5293 case Builtin::BI__builtin_smulll_overflow:
5294 return interp__builtin_overflowop(S, OpPC, Call, BuiltinOp: BuiltinID);
5295
5296 case Builtin::BI__builtin_addcb:
5297 case Builtin::BI__builtin_addcs:
5298 case Builtin::BI__builtin_addc:
5299 case Builtin::BI__builtin_addcl:
5300 case Builtin::BI__builtin_addcll:
5301 case Builtin::BI__builtin_subcb:
5302 case Builtin::BI__builtin_subcs:
5303 case Builtin::BI__builtin_subc:
5304 case Builtin::BI__builtin_subcl:
5305 case Builtin::BI__builtin_subcll:
5306 return interp__builtin_carryop(S, OpPC, Frame, Call, BuiltinOp: BuiltinID);
5307
5308 case Builtin::BI__builtin_clz:
5309 case Builtin::BI__builtin_clzl:
5310 case Builtin::BI__builtin_clzll:
5311 case Builtin::BI__builtin_clzs:
5312 case Builtin::BI__builtin_clzg:
5313 case Builtin::BI__lzcnt16: // Microsoft variants of count leading-zeroes
5314 case Builtin::BI__lzcnt:
5315 case Builtin::BI__lzcnt64:
5316 return interp__builtin_clz(S, OpPC, Frame, Call, BuiltinOp: BuiltinID);
5317
5318 case Builtin::BI__builtin_ctz:
5319 case Builtin::BI__builtin_ctzl:
5320 case Builtin::BI__builtin_ctzll:
5321 case Builtin::BI__builtin_ctzs:
5322 case Builtin::BI__builtin_ctzg:
5323 return interp__builtin_ctz(S, OpPC, Frame, Call, BuiltinID);
5324
5325 case Builtin::BI__builtin_elementwise_clzg:
5326 case Builtin::BI__builtin_elementwise_ctzg:
5327 return interp__builtin_elementwise_countzeroes(S, OpPC, Frame, Call,
5328 BuiltinID);
5329 case Builtin::BI__builtin_bswapg:
5330 case Builtin::BI__builtin_bswap16:
5331 case Builtin::BI__builtin_bswap32:
5332 case Builtin::BI__builtin_bswap64:
5333 case Builtin::BIstdc_memreverse8u8:
5334 case Builtin::BIstdc_memreverse8u16:
5335 case Builtin::BIstdc_memreverse8u32:
5336 case Builtin::BIstdc_memreverse8u64:
5337 return interp__builtin_bswap(S, OpPC, Frame, Call);
5338
5339 case Builtin::BIstdc_memreverse8:
5340 case Builtin::BI__builtin_stdc_memreverse8:
5341 return interp__builtin_stdc_memreverse8(S, OpPC, Frame, Call);
5342
5343 case Builtin::BIstdc_load8_leu8:
5344 case Builtin::BIstdc_load8_leu16:
5345 case Builtin::BIstdc_load8_leu32:
5346 case Builtin::BIstdc_load8_leu64:
5347 case Builtin::BIstdc_load8_les8:
5348 case Builtin::BIstdc_load8_les16:
5349 case Builtin::BIstdc_load8_les32:
5350 case Builtin::BIstdc_load8_les64:
5351 return interp__builtin_load8(S, OpPC, Frame, Call, /*IsBigEndian=*/false,
5352 /*IsAligned=*/false);
5353
5354 case Builtin::BIstdc_load8_aligned_leu8:
5355 case Builtin::BIstdc_load8_aligned_leu16:
5356 case Builtin::BIstdc_load8_aligned_leu32:
5357 case Builtin::BIstdc_load8_aligned_leu64:
5358 case Builtin::BIstdc_load8_aligned_les8:
5359 case Builtin::BIstdc_load8_aligned_les16:
5360 case Builtin::BIstdc_load8_aligned_les32:
5361 case Builtin::BIstdc_load8_aligned_les64:
5362 return interp__builtin_load8(S, OpPC, Frame, Call, /*IsBigEndian=*/false,
5363 /*IsAligned=*/true);
5364
5365 case Builtin::BIstdc_load8_beu8:
5366 case Builtin::BIstdc_load8_beu16:
5367 case Builtin::BIstdc_load8_beu32:
5368 case Builtin::BIstdc_load8_beu64:
5369 case Builtin::BIstdc_load8_bes8:
5370 case Builtin::BIstdc_load8_bes16:
5371 case Builtin::BIstdc_load8_bes32:
5372 case Builtin::BIstdc_load8_bes64:
5373 return interp__builtin_load8(S, OpPC, Frame, Call, /*IsBigEndian=*/true,
5374 /*IsAligned=*/false);
5375
5376 case Builtin::BIstdc_load8_aligned_beu8:
5377 case Builtin::BIstdc_load8_aligned_beu16:
5378 case Builtin::BIstdc_load8_aligned_beu32:
5379 case Builtin::BIstdc_load8_aligned_beu64:
5380 case Builtin::BIstdc_load8_aligned_bes8:
5381 case Builtin::BIstdc_load8_aligned_bes16:
5382 case Builtin::BIstdc_load8_aligned_bes32:
5383 case Builtin::BIstdc_load8_aligned_bes64:
5384 return interp__builtin_load8(S, OpPC, Frame, Call, /*IsBigEndian=*/true,
5385 /*IsAligned=*/true);
5386
5387 case Builtin::BI__atomic_always_lock_free:
5388 case Builtin::BI__atomic_is_lock_free:
5389 return interp__builtin_atomic_lock_free(S, OpPC, Frame, Call, BuiltinOp: BuiltinID);
5390
5391 case Builtin::BI__c11_atomic_is_lock_free:
5392 return interp__builtin_c11_atomic_is_lock_free(S, OpPC, Frame, Call);
5393
5394 case Builtin::BI__builtin_complex:
5395 return interp__builtin_complex(S, OpPC, Frame, Call);
5396
5397 case Builtin::BI__builtin_is_aligned:
5398 case Builtin::BI__builtin_align_up:
5399 case Builtin::BI__builtin_align_down:
5400 return interp__builtin_is_aligned_up_down(S, OpPC, Frame, Call, BuiltinOp: BuiltinID);
5401
5402 case Builtin::BI__builtin_assume_aligned:
5403 return interp__builtin_assume_aligned(S, OpPC, Frame, Call);
5404
5405 case clang::X86::BI__builtin_ia32_crc32qi:
5406 return interp__builtin_ia32_crc32(S, OpPC, Frame, Call, DataBytes: 1);
5407 case clang::X86::BI__builtin_ia32_crc32hi:
5408 return interp__builtin_ia32_crc32(S, OpPC, Frame, Call, DataBytes: 2);
5409 case clang::X86::BI__builtin_ia32_crc32si:
5410 return interp__builtin_ia32_crc32(S, OpPC, Frame, Call, DataBytes: 4);
5411 case clang::X86::BI__builtin_ia32_crc32di:
5412 return interp__builtin_ia32_crc32(S, OpPC, Frame, Call, DataBytes: 8);
5413
5414 case clang::X86::BI__builtin_ia32_bextr_u32:
5415 case clang::X86::BI__builtin_ia32_bextr_u64:
5416 case clang::X86::BI__builtin_ia32_bextri_u32:
5417 case clang::X86::BI__builtin_ia32_bextri_u64:
5418 return interp__builtin_elementwise_int_binop(
5419 S, OpPC, Call, Fn: [](const APSInt &Val, const APSInt &Idx) {
5420 unsigned BitWidth = Val.getBitWidth();
5421 uint64_t Shift = Idx.extractBitsAsZExtValue(numBits: 8, bitPosition: 0);
5422 uint64_t Length = Idx.extractBitsAsZExtValue(numBits: 8, bitPosition: 8);
5423 if (Length > BitWidth) {
5424 Length = BitWidth;
5425 }
5426
5427 // Handle out of bounds cases.
5428 if (Length == 0 || Shift >= BitWidth)
5429 return APInt(BitWidth, 0);
5430
5431 uint64_t Result = Val.getZExtValue() >> Shift;
5432 Result &= llvm::maskTrailingOnes<uint64_t>(N: Length);
5433 return APInt(BitWidth, Result);
5434 });
5435
5436 case clang::X86::BI__builtin_ia32_bzhi_si:
5437 case clang::X86::BI__builtin_ia32_bzhi_di:
5438 return interp__builtin_elementwise_int_binop(
5439 S, OpPC, Call, Fn: [](const APSInt &Val, const APSInt &Idx) {
5440 unsigned BitWidth = Val.getBitWidth();
5441 uint64_t Index = Idx.extractBitsAsZExtValue(numBits: 8, bitPosition: 0);
5442 APSInt Result = Val;
5443
5444 if (Index < BitWidth)
5445 Result.clearHighBits(hiBits: BitWidth - Index);
5446
5447 return Result;
5448 });
5449
5450 case clang::X86::BI__builtin_ia32_ktestcqi:
5451 case clang::X86::BI__builtin_ia32_ktestchi:
5452 case clang::X86::BI__builtin_ia32_ktestcsi:
5453 case clang::X86::BI__builtin_ia32_ktestcdi:
5454 return interp__builtin_elementwise_int_binop(
5455 S, OpPC, Call, Fn: [](const APSInt &A, const APSInt &B) {
5456 return APInt(sizeof(unsigned char) * 8, (~A & B) == 0);
5457 });
5458
5459 case clang::X86::BI__builtin_ia32_ktestzqi:
5460 case clang::X86::BI__builtin_ia32_ktestzhi:
5461 case clang::X86::BI__builtin_ia32_ktestzsi:
5462 case clang::X86::BI__builtin_ia32_ktestzdi:
5463 return interp__builtin_elementwise_int_binop(
5464 S, OpPC, Call, Fn: [](const APSInt &A, const APSInt &B) {
5465 return APInt(sizeof(unsigned char) * 8, (A & B) == 0);
5466 });
5467
5468 case clang::X86::BI__builtin_ia32_kortestcqi:
5469 case clang::X86::BI__builtin_ia32_kortestchi:
5470 case clang::X86::BI__builtin_ia32_kortestcsi:
5471 case clang::X86::BI__builtin_ia32_kortestcdi:
5472 return interp__builtin_elementwise_int_binop(
5473 S, OpPC, Call, Fn: [](const APSInt &A, const APSInt &B) {
5474 return APInt(sizeof(unsigned char) * 8, ~(A | B) == 0);
5475 });
5476
5477 case clang::X86::BI__builtin_ia32_kortestzqi:
5478 case clang::X86::BI__builtin_ia32_kortestzhi:
5479 case clang::X86::BI__builtin_ia32_kortestzsi:
5480 case clang::X86::BI__builtin_ia32_kortestzdi:
5481 return interp__builtin_elementwise_int_binop(
5482 S, OpPC, Call, Fn: [](const APSInt &A, const APSInt &B) {
5483 return APInt(sizeof(unsigned char) * 8, (A | B) == 0);
5484 });
5485
5486 case clang::X86::BI__builtin_ia32_kshiftliqi:
5487 case clang::X86::BI__builtin_ia32_kshiftlihi:
5488 case clang::X86::BI__builtin_ia32_kshiftlisi:
5489 case clang::X86::BI__builtin_ia32_kshiftlidi:
5490 return interp__builtin_elementwise_int_binop(
5491 S, OpPC, Call, Fn: [](const APSInt &LHS, const APSInt &RHS) {
5492 unsigned Amt = RHS.getZExtValue() & 0xFF;
5493 if (Amt >= LHS.getBitWidth())
5494 return APInt::getZero(numBits: LHS.getBitWidth());
5495 return LHS.shl(shiftAmt: Amt);
5496 });
5497
5498 case clang::X86::BI__builtin_ia32_kshiftriqi:
5499 case clang::X86::BI__builtin_ia32_kshiftrihi:
5500 case clang::X86::BI__builtin_ia32_kshiftrisi:
5501 case clang::X86::BI__builtin_ia32_kshiftridi:
5502 return interp__builtin_elementwise_int_binop(
5503 S, OpPC, Call, Fn: [](const APSInt &LHS, const APSInt &RHS) {
5504 unsigned Amt = RHS.getZExtValue() & 0xFF;
5505 if (Amt >= LHS.getBitWidth())
5506 return APInt::getZero(numBits: LHS.getBitWidth());
5507 return LHS.lshr(shiftAmt: Amt);
5508 });
5509
5510 case clang::X86::BI__builtin_ia32_lzcnt_u16:
5511 case clang::X86::BI__builtin_ia32_lzcnt_u32:
5512 case clang::X86::BI__builtin_ia32_lzcnt_u64:
5513 return interp__builtin_elementwise_int_unaryop(
5514 S, OpPC, Call, Fn: [](const APSInt &Src) {
5515 return APInt(Src.getBitWidth(), Src.countLeadingZeros());
5516 });
5517
5518 case clang::X86::BI__builtin_ia32_tzcnt_u16:
5519 case clang::X86::BI__builtin_ia32_tzcnt_u32:
5520 case clang::X86::BI__builtin_ia32_tzcnt_u64:
5521 return interp__builtin_elementwise_int_unaryop(
5522 S, OpPC, Call, Fn: [](const APSInt &Src) {
5523 return APInt(Src.getBitWidth(), Src.countTrailingZeros());
5524 });
5525
5526 case clang::X86::BI__builtin_ia32_addcarryx_u32:
5527 case clang::X86::BI__builtin_ia32_addcarryx_u64:
5528 return interp__builtin_ia32_addcarry_subborrow(S, OpPC, Frame, Call,
5529 /*IsAdd=*/true);
5530
5531 case clang::X86::BI__builtin_ia32_subborrow_u32:
5532 case clang::X86::BI__builtin_ia32_subborrow_u64:
5533 return interp__builtin_ia32_addcarry_subborrow(S, OpPC, Frame, Call,
5534 /*IsAdd=*/false);
5535
5536 case Builtin::BI__builtin_os_log_format_buffer_size:
5537 return interp__builtin_os_log_format_buffer_size(S, OpPC, Frame, Call);
5538
5539 case Builtin::BI__builtin_ptrauth_string_discriminator:
5540 return interp__builtin_ptrauth_string_discriminator(S, OpPC, Frame, Call);
5541
5542 case Builtin::BI__builtin_infer_alloc_token:
5543 return interp__builtin_infer_alloc_token(S, OpPC, Frame, Call);
5544
5545 case Builtin::BI__noop:
5546 pushInteger(S, Val: 0, QT: Call->getType());
5547 return true;
5548
5549 case Builtin::BI__builtin_operator_new:
5550 return interp__builtin_operator_new(S, OpPC, Frame, Call);
5551
5552 case Builtin::BI__builtin_operator_delete:
5553 return interp__builtin_operator_delete(S, OpPC, Frame, Call);
5554
5555 case Builtin::BI__arithmetic_fence:
5556 return interp__builtin_arithmetic_fence(S, OpPC, Frame, Call);
5557
5558 case Builtin::BI__builtin_reduce_add:
5559 case Builtin::BI__builtin_reduce_mul:
5560 case Builtin::BI__builtin_reduce_and:
5561 case Builtin::BI__builtin_reduce_or:
5562 case Builtin::BI__builtin_reduce_xor:
5563 case Builtin::BI__builtin_reduce_min:
5564 case Builtin::BI__builtin_reduce_max:
5565 return interp__builtin_vector_reduce(S, OpPC, Call, ID: BuiltinID);
5566
5567 case Builtin::BI__builtin_elementwise_popcount:
5568 return interp__builtin_elementwise_int_unaryop(
5569 S, OpPC, Call, Fn: [](const APSInt &Src) {
5570 return APInt(Src.getBitWidth(), Src.popcount());
5571 });
5572 case Builtin::BI__builtin_elementwise_bitreverse:
5573 return interp__builtin_elementwise_int_unaryop(
5574 S, OpPC, Call, Fn: [](const APSInt &Src) { return Src.reverseBits(); });
5575
5576 case Builtin::BI__builtin_elementwise_abs:
5577 return interp__builtin_elementwise_abs(S, OpPC, Frame, Call, BuiltinID);
5578
5579 case Builtin::BI__builtin_memcpy:
5580 case Builtin::BImemcpy:
5581 case Builtin::BI__builtin_wmemcpy:
5582 case Builtin::BIwmemcpy:
5583 case Builtin::BI__builtin_memmove:
5584 case Builtin::BImemmove:
5585 case Builtin::BI__builtin_wmemmove:
5586 case Builtin::BIwmemmove:
5587 return interp__builtin_memcpy(S, OpPC, Frame, Call, ID: BuiltinID);
5588
5589 case Builtin::BI__builtin_memcmp:
5590 case Builtin::BImemcmp:
5591 case Builtin::BI__builtin_bcmp:
5592 case Builtin::BIbcmp:
5593 case Builtin::BI__builtin_wmemcmp:
5594 case Builtin::BIwmemcmp:
5595 return interp__builtin_memcmp(S, OpPC, Frame, Call, ID: BuiltinID);
5596
5597 case Builtin::BImemchr:
5598 case Builtin::BI__builtin_memchr:
5599 case Builtin::BIstrchr:
5600 case Builtin::BI__builtin_strchr:
5601 case Builtin::BIwmemchr:
5602 case Builtin::BI__builtin_wmemchr:
5603 case Builtin::BIwcschr:
5604 case Builtin::BI__builtin_wcschr:
5605 case Builtin::BI__builtin_char_memchr:
5606 return interp__builtin_memchr(S, OpPC, Call, ID: BuiltinID);
5607
5608 case Builtin::BI__builtin_object_size:
5609 return interp__builtin_object_size(S, OpPC, Frame, Call,
5610 /*IsDynamic=*/false);
5611 case Builtin::BI__builtin_dynamic_object_size:
5612 return interp__builtin_object_size(S, OpPC, Frame, Call,
5613 /*IsDynamic=*/true);
5614
5615 case Builtin::BI__builtin_is_within_lifetime:
5616 return interp__builtin_is_within_lifetime(S, OpPC, Call);
5617
5618 case Builtin::BI__builtin_elementwise_add_sat:
5619 return interp__builtin_elementwise_int_binop(
5620 S, OpPC, Call, Fn: [](const APSInt &LHS, const APSInt &RHS) {
5621 return LHS.isSigned() ? LHS.sadd_sat(RHS) : LHS.uadd_sat(RHS);
5622 });
5623
5624 case Builtin::BI__builtin_elementwise_sub_sat:
5625 return interp__builtin_elementwise_int_binop(
5626 S, OpPC, Call, Fn: [](const APSInt &LHS, const APSInt &RHS) {
5627 return LHS.isSigned() ? LHS.ssub_sat(RHS) : LHS.usub_sat(RHS);
5628 });
5629
5630 case Builtin::BI__builtin_elementwise_pdep:
5631 return interp__builtin_elementwise_int_binop(S, OpPC, Call,
5632 Fn: llvm::APIntOps::pdep);
5633
5634 case Builtin::BI__builtin_elementwise_pext:
5635 return interp__builtin_elementwise_int_binop(S, OpPC, Call,
5636 Fn: llvm::APIntOps::pext);
5637
5638 case X86::BI__builtin_ia32_extract128i256:
5639 case X86::BI__builtin_ia32_vextractf128_pd256:
5640 case X86::BI__builtin_ia32_vextractf128_ps256:
5641 case X86::BI__builtin_ia32_vextractf128_si256:
5642 return interp__builtin_ia32_extract_vector(S, OpPC, Call, ID: BuiltinID);
5643
5644 case X86::BI__builtin_ia32_extractf32x4_256_mask:
5645 case X86::BI__builtin_ia32_extractf32x4_mask:
5646 case X86::BI__builtin_ia32_extractf32x8_mask:
5647 case X86::BI__builtin_ia32_extractf64x2_256_mask:
5648 case X86::BI__builtin_ia32_extractf64x2_512_mask:
5649 case X86::BI__builtin_ia32_extractf64x4_mask:
5650 case X86::BI__builtin_ia32_extracti32x4_256_mask:
5651 case X86::BI__builtin_ia32_extracti32x4_mask:
5652 case X86::BI__builtin_ia32_extracti32x8_mask:
5653 case X86::BI__builtin_ia32_extracti64x2_256_mask:
5654 case X86::BI__builtin_ia32_extracti64x2_512_mask:
5655 case X86::BI__builtin_ia32_extracti64x4_mask:
5656 return interp__builtin_ia32_extract_vector_masked(S, OpPC, Call, ID: BuiltinID);
5657
5658 case clang::X86::BI__builtin_ia32_pmulhrsw128:
5659 case clang::X86::BI__builtin_ia32_pmulhrsw256:
5660 case clang::X86::BI__builtin_ia32_pmulhrsw512:
5661 return interp__builtin_elementwise_int_binop(
5662 S, OpPC, Call, Fn: [](const APSInt &LHS, const APSInt &RHS) {
5663 return (llvm::APIntOps::mulsExtended(C1: LHS, C2: RHS).ashr(ShiftAmt: 14) + 1)
5664 .extractBits(numBits: 16, bitPosition: 1);
5665 });
5666
5667 case clang::X86::BI__builtin_ia32_movmskps:
5668 case clang::X86::BI__builtin_ia32_movmskpd:
5669 case clang::X86::BI__builtin_ia32_pmovmskb128:
5670 case clang::X86::BI__builtin_ia32_pmovmskb256:
5671 case clang::X86::BI__builtin_ia32_movmskps256:
5672 case clang::X86::BI__builtin_ia32_movmskpd256: {
5673 return interp__builtin_ia32_movmsk_op(S, OpPC, Call);
5674 }
5675
5676 case X86::BI__builtin_ia32_psignb128:
5677 case X86::BI__builtin_ia32_psignb256:
5678 case X86::BI__builtin_ia32_psignw128:
5679 case X86::BI__builtin_ia32_psignw256:
5680 case X86::BI__builtin_ia32_psignd128:
5681 case X86::BI__builtin_ia32_psignd256:
5682 return interp__builtin_elementwise_int_binop(
5683 S, OpPC, Call, Fn: [](const APInt &AElem, const APInt &BElem) {
5684 if (BElem.isZero())
5685 return APInt::getZero(numBits: AElem.getBitWidth());
5686 if (BElem.isNegative())
5687 return -AElem;
5688 return AElem;
5689 });
5690
5691 case clang::X86::BI__builtin_ia32_pavgb128:
5692 case clang::X86::BI__builtin_ia32_pavgw128:
5693 case clang::X86::BI__builtin_ia32_pavgb256:
5694 case clang::X86::BI__builtin_ia32_pavgw256:
5695 case clang::X86::BI__builtin_ia32_pavgb512:
5696 case clang::X86::BI__builtin_ia32_pavgw512:
5697 return interp__builtin_elementwise_int_binop(S, OpPC, Call,
5698 Fn: llvm::APIntOps::avgCeilU);
5699
5700 case clang::X86::BI__builtin_ia32_pmaddubsw128:
5701 case clang::X86::BI__builtin_ia32_pmaddubsw256:
5702 case clang::X86::BI__builtin_ia32_pmaddubsw512:
5703 return interp__builtin_ia32_pmul(
5704 S, OpPC, Call,
5705 Fn: [](const APSInt &LoLHS, const APSInt &HiLHS, const APSInt &LoRHS,
5706 const APSInt &HiRHS) {
5707 unsigned BitWidth = 2 * LoLHS.getBitWidth();
5708 return (LoLHS.zext(width: BitWidth) * LoRHS.sext(width: BitWidth))
5709 .sadd_sat(RHS: (HiLHS.zext(width: BitWidth) * HiRHS.sext(width: BitWidth)));
5710 });
5711
5712 case clang::X86::BI__builtin_ia32_pmaddwd128:
5713 case clang::X86::BI__builtin_ia32_pmaddwd256:
5714 case clang::X86::BI__builtin_ia32_pmaddwd512:
5715 return interp__builtin_ia32_pmul(
5716 S, OpPC, Call,
5717 Fn: [](const APSInt &LoLHS, const APSInt &HiLHS, const APSInt &LoRHS,
5718 const APSInt &HiRHS) {
5719 unsigned BitWidth = 2 * LoLHS.getBitWidth();
5720 return (LoLHS.sext(width: BitWidth) * LoRHS.sext(width: BitWidth)) +
5721 (HiLHS.sext(width: BitWidth) * HiRHS.sext(width: BitWidth));
5722 });
5723
5724 case clang::X86::BI__builtin_ia32_psadbw128:
5725 case clang::X86::BI__builtin_ia32_psadbw256:
5726 case clang::X86::BI__builtin_ia32_psadbw512:
5727 return interp__builtin_ia32_psadbw(S, OpPC, Call);
5728
5729 case clang::X86::BI__builtin_ia32_dbpsadbw128:
5730 case clang::X86::BI__builtin_ia32_dbpsadbw256:
5731 case clang::X86::BI__builtin_ia32_dbpsadbw512:
5732 return interp__builtin_ia32_dbpsadbw(S, OpPC, Call);
5733
5734 case clang::X86::BI__builtin_ia32_mpsadbw128:
5735 case clang::X86::BI__builtin_ia32_mpsadbw256:
5736 return interp__builtin_ia32_mpsadbw(S, OpPC, Call);
5737
5738 case clang::X86::BI__builtin_ia32_pmulhuw128:
5739 case clang::X86::BI__builtin_ia32_pmulhuw256:
5740 case clang::X86::BI__builtin_ia32_pmulhuw512:
5741 return interp__builtin_elementwise_int_binop(S, OpPC, Call,
5742 Fn: llvm::APIntOps::mulhu);
5743
5744 case clang::X86::BI__builtin_ia32_pmulhw128:
5745 case clang::X86::BI__builtin_ia32_pmulhw256:
5746 case clang::X86::BI__builtin_ia32_pmulhw512:
5747 return interp__builtin_elementwise_int_binop(S, OpPC, Call,
5748 Fn: llvm::APIntOps::mulhs);
5749
5750 case clang::X86::BI__builtin_ia32_psllv2di:
5751 case clang::X86::BI__builtin_ia32_psllv4di:
5752 case clang::X86::BI__builtin_ia32_psllv4si:
5753 case clang::X86::BI__builtin_ia32_psllv8di:
5754 case clang::X86::BI__builtin_ia32_psllv8hi:
5755 case clang::X86::BI__builtin_ia32_psllv8si:
5756 case clang::X86::BI__builtin_ia32_psllv16hi:
5757 case clang::X86::BI__builtin_ia32_psllv16si:
5758 case clang::X86::BI__builtin_ia32_psllv32hi:
5759 case clang::X86::BI__builtin_ia32_psllwi128:
5760 case clang::X86::BI__builtin_ia32_psllwi256:
5761 case clang::X86::BI__builtin_ia32_psllwi512:
5762 case clang::X86::BI__builtin_ia32_pslldi128:
5763 case clang::X86::BI__builtin_ia32_pslldi256:
5764 case clang::X86::BI__builtin_ia32_pslldi512:
5765 case clang::X86::BI__builtin_ia32_psllqi128:
5766 case clang::X86::BI__builtin_ia32_psllqi256:
5767 case clang::X86::BI__builtin_ia32_psllqi512:
5768 return interp__builtin_elementwise_int_binop(
5769 S, OpPC, Call, Fn: [](const APSInt &LHS, const APSInt &RHS) {
5770 if (RHS.uge(RHS: LHS.getBitWidth())) {
5771 return APInt::getZero(numBits: LHS.getBitWidth());
5772 }
5773 return LHS.shl(shiftAmt: RHS.getZExtValue());
5774 });
5775
5776 case clang::X86::BI__builtin_ia32_psrav4si:
5777 case clang::X86::BI__builtin_ia32_psrav8di:
5778 case clang::X86::BI__builtin_ia32_psrav8hi:
5779 case clang::X86::BI__builtin_ia32_psrav8si:
5780 case clang::X86::BI__builtin_ia32_psrav16hi:
5781 case clang::X86::BI__builtin_ia32_psrav16si:
5782 case clang::X86::BI__builtin_ia32_psrav32hi:
5783 case clang::X86::BI__builtin_ia32_psravq128:
5784 case clang::X86::BI__builtin_ia32_psravq256:
5785 case clang::X86::BI__builtin_ia32_psrawi128:
5786 case clang::X86::BI__builtin_ia32_psrawi256:
5787 case clang::X86::BI__builtin_ia32_psrawi512:
5788 case clang::X86::BI__builtin_ia32_psradi128:
5789 case clang::X86::BI__builtin_ia32_psradi256:
5790 case clang::X86::BI__builtin_ia32_psradi512:
5791 case clang::X86::BI__builtin_ia32_psraqi128:
5792 case clang::X86::BI__builtin_ia32_psraqi256:
5793 case clang::X86::BI__builtin_ia32_psraqi512:
5794 return interp__builtin_elementwise_int_binop(
5795 S, OpPC, Call, Fn: [](const APSInt &LHS, const APSInt &RHS) {
5796 if (RHS.uge(RHS: LHS.getBitWidth())) {
5797 return LHS.ashr(ShiftAmt: LHS.getBitWidth() - 1);
5798 }
5799 return LHS.ashr(ShiftAmt: RHS.getZExtValue());
5800 });
5801
5802 case clang::X86::BI__builtin_ia32_psrlv2di:
5803 case clang::X86::BI__builtin_ia32_psrlv4di:
5804 case clang::X86::BI__builtin_ia32_psrlv4si:
5805 case clang::X86::BI__builtin_ia32_psrlv8di:
5806 case clang::X86::BI__builtin_ia32_psrlv8hi:
5807 case clang::X86::BI__builtin_ia32_psrlv8si:
5808 case clang::X86::BI__builtin_ia32_psrlv16hi:
5809 case clang::X86::BI__builtin_ia32_psrlv16si:
5810 case clang::X86::BI__builtin_ia32_psrlv32hi:
5811 case clang::X86::BI__builtin_ia32_psrlwi128:
5812 case clang::X86::BI__builtin_ia32_psrlwi256:
5813 case clang::X86::BI__builtin_ia32_psrlwi512:
5814 case clang::X86::BI__builtin_ia32_psrldi128:
5815 case clang::X86::BI__builtin_ia32_psrldi256:
5816 case clang::X86::BI__builtin_ia32_psrldi512:
5817 case clang::X86::BI__builtin_ia32_psrlqi128:
5818 case clang::X86::BI__builtin_ia32_psrlqi256:
5819 case clang::X86::BI__builtin_ia32_psrlqi512:
5820 return interp__builtin_elementwise_int_binop(
5821 S, OpPC, Call, Fn: [](const APSInt &LHS, const APSInt &RHS) {
5822 if (RHS.uge(RHS: LHS.getBitWidth())) {
5823 return APInt::getZero(numBits: LHS.getBitWidth());
5824 }
5825 return LHS.lshr(shiftAmt: RHS.getZExtValue());
5826 });
5827 case clang::X86::BI__builtin_ia32_packsswb128:
5828 case clang::X86::BI__builtin_ia32_packsswb256:
5829 case clang::X86::BI__builtin_ia32_packsswb512:
5830 case clang::X86::BI__builtin_ia32_packssdw128:
5831 case clang::X86::BI__builtin_ia32_packssdw256:
5832 case clang::X86::BI__builtin_ia32_packssdw512:
5833 return interp__builtin_ia32_pack(S, OpPC, E: Call, PackFn: [](const APSInt &Src) {
5834 return APInt(Src).truncSSat(width: Src.getBitWidth() / 2);
5835 });
5836 case clang::X86::BI__builtin_ia32_packusdw128:
5837 case clang::X86::BI__builtin_ia32_packusdw256:
5838 case clang::X86::BI__builtin_ia32_packusdw512:
5839 case clang::X86::BI__builtin_ia32_packuswb128:
5840 case clang::X86::BI__builtin_ia32_packuswb256:
5841 case clang::X86::BI__builtin_ia32_packuswb512:
5842 return interp__builtin_ia32_pack(S, OpPC, E: Call, PackFn: [](const APSInt &Src) {
5843 return APInt(Src).truncSSatU(width: Src.getBitWidth() / 2);
5844 });
5845
5846 case clang::X86::BI__builtin_ia32_selectss_128:
5847 case clang::X86::BI__builtin_ia32_selectsd_128:
5848 case clang::X86::BI__builtin_ia32_selectsh_128:
5849 case clang::X86::BI__builtin_ia32_selectsbf_128:
5850 return interp__builtin_ia32_select_scalar(S, Call);
5851 case clang::X86::BI__builtin_ia32_vprotbi:
5852 case clang::X86::BI__builtin_ia32_vprotdi:
5853 case clang::X86::BI__builtin_ia32_vprotqi:
5854 case clang::X86::BI__builtin_ia32_vprotwi:
5855 case clang::X86::BI__builtin_ia32_prold128:
5856 case clang::X86::BI__builtin_ia32_prold256:
5857 case clang::X86::BI__builtin_ia32_prold512:
5858 case clang::X86::BI__builtin_ia32_prolq128:
5859 case clang::X86::BI__builtin_ia32_prolq256:
5860 case clang::X86::BI__builtin_ia32_prolq512:
5861 return interp__builtin_elementwise_int_binop(
5862 S, OpPC, Call,
5863 Fn: [](const APSInt &LHS, const APSInt &RHS) { return LHS.rotl(rotateAmt: RHS); });
5864
5865 case clang::X86::BI__builtin_ia32_prord128:
5866 case clang::X86::BI__builtin_ia32_prord256:
5867 case clang::X86::BI__builtin_ia32_prord512:
5868 case clang::X86::BI__builtin_ia32_prorq128:
5869 case clang::X86::BI__builtin_ia32_prorq256:
5870 case clang::X86::BI__builtin_ia32_prorq512:
5871 return interp__builtin_elementwise_int_binop(
5872 S, OpPC, Call,
5873 Fn: [](const APSInt &LHS, const APSInt &RHS) { return LHS.rotr(rotateAmt: RHS); });
5874
5875 case Builtin::BI__builtin_elementwise_max:
5876 case Builtin::BI__builtin_elementwise_min:
5877 return interp__builtin_elementwise_maxmin(S, OpPC, Call, BuiltinID);
5878
5879 case clang::X86::BI__builtin_ia32_phaddw128:
5880 case clang::X86::BI__builtin_ia32_phaddw256:
5881 case clang::X86::BI__builtin_ia32_phaddd128:
5882 case clang::X86::BI__builtin_ia32_phaddd256:
5883 return interp_builtin_horizontal_int_binop(
5884 S, OpPC, Call,
5885 Fn: [](const APSInt &LHS, const APSInt &RHS) { return LHS + RHS; });
5886 case clang::X86::BI__builtin_ia32_phaddsw128:
5887 case clang::X86::BI__builtin_ia32_phaddsw256:
5888 return interp_builtin_horizontal_int_binop(
5889 S, OpPC, Call,
5890 Fn: [](const APSInt &LHS, const APSInt &RHS) { return LHS.sadd_sat(RHS); });
5891 case clang::X86::BI__builtin_ia32_phsubw128:
5892 case clang::X86::BI__builtin_ia32_phsubw256:
5893 case clang::X86::BI__builtin_ia32_phsubd128:
5894 case clang::X86::BI__builtin_ia32_phsubd256:
5895 return interp_builtin_horizontal_int_binop(
5896 S, OpPC, Call,
5897 Fn: [](const APSInt &LHS, const APSInt &RHS) { return LHS - RHS; });
5898 case clang::X86::BI__builtin_ia32_phsubsw128:
5899 case clang::X86::BI__builtin_ia32_phsubsw256:
5900 return interp_builtin_horizontal_int_binop(
5901 S, OpPC, Call,
5902 Fn: [](const APSInt &LHS, const APSInt &RHS) { return LHS.ssub_sat(RHS); });
5903 case clang::X86::BI__builtin_ia32_haddpd:
5904 case clang::X86::BI__builtin_ia32_haddps:
5905 case clang::X86::BI__builtin_ia32_haddpd256:
5906 case clang::X86::BI__builtin_ia32_haddps256:
5907 return interp_builtin_horizontal_fp_binop(
5908 S, OpPC, Call,
5909 Fn: [](const APFloat &LHS, const APFloat &RHS, llvm::RoundingMode RM) {
5910 APFloat F = LHS;
5911 F.add(RHS, RM);
5912 return F;
5913 });
5914 case clang::X86::BI__builtin_ia32_hsubpd:
5915 case clang::X86::BI__builtin_ia32_hsubps:
5916 case clang::X86::BI__builtin_ia32_hsubpd256:
5917 case clang::X86::BI__builtin_ia32_hsubps256:
5918 return interp_builtin_horizontal_fp_binop(
5919 S, OpPC, Call,
5920 Fn: [](const APFloat &LHS, const APFloat &RHS, llvm::RoundingMode RM) {
5921 APFloat F = LHS;
5922 F.subtract(RHS, RM);
5923 return F;
5924 });
5925 case clang::X86::BI__builtin_ia32_addsubpd:
5926 case clang::X86::BI__builtin_ia32_addsubps:
5927 case clang::X86::BI__builtin_ia32_addsubpd256:
5928 case clang::X86::BI__builtin_ia32_addsubps256:
5929 return interp__builtin_ia32_addsub(S, OpPC, Call);
5930
5931 case clang::X86::BI__builtin_ia32_pmuldq128:
5932 case clang::X86::BI__builtin_ia32_pmuldq256:
5933 case clang::X86::BI__builtin_ia32_pmuldq512:
5934 return interp__builtin_ia32_pmul(
5935 S, OpPC, Call,
5936 Fn: [](const APSInt &LoLHS, const APSInt &HiLHS, const APSInt &LoRHS,
5937 const APSInt &HiRHS) {
5938 return llvm::APIntOps::mulsExtended(C1: LoLHS, C2: LoRHS);
5939 });
5940
5941 case clang::X86::BI__builtin_ia32_pmuludq128:
5942 case clang::X86::BI__builtin_ia32_pmuludq256:
5943 case clang::X86::BI__builtin_ia32_pmuludq512:
5944 return interp__builtin_ia32_pmul(
5945 S, OpPC, Call,
5946 Fn: [](const APSInt &LoLHS, const APSInt &HiLHS, const APSInt &LoRHS,
5947 const APSInt &HiRHS) {
5948 return llvm::APIntOps::muluExtended(C1: LoLHS, C2: LoRHS);
5949 });
5950
5951 case clang::X86::BI__builtin_ia32_pclmulqdq128:
5952 case clang::X86::BI__builtin_ia32_pclmulqdq256:
5953 case clang::X86::BI__builtin_ia32_pclmulqdq512:
5954 return interp__builtin_ia32_pclmulqdq(S, OpPC, Call);
5955 case Builtin::BI__builtin_elementwise_clmul:
5956 return interp__builtin_elementwise_int_binop(S, OpPC, Call,
5957 Fn: llvm::APIntOps::clmul);
5958
5959 case Builtin::BI__builtin_elementwise_fma:
5960 return interp__builtin_elementwise_triop_fp(
5961 S, OpPC, Call,
5962 Fn: [](const APFloat &X, const APFloat &Y, const APFloat &Z,
5963 llvm::RoundingMode RM) {
5964 APFloat F = X;
5965 F.fusedMultiplyAdd(Multiplicand: Y, Addend: Z, RM);
5966 return F;
5967 });
5968
5969 case X86::BI__builtin_ia32_vpmadd52luq128:
5970 case X86::BI__builtin_ia32_vpmadd52luq256:
5971 case X86::BI__builtin_ia32_vpmadd52luq512:
5972 return interp__builtin_elementwise_triop(
5973 S, OpPC, Call, Fn: [](const APSInt &A, const APSInt &B, const APSInt &C) {
5974 return A + (B.trunc(width: 52) * C.trunc(width: 52)).zext(width: 64);
5975 });
5976 case X86::BI__builtin_ia32_vpmadd52huq128:
5977 case X86::BI__builtin_ia32_vpmadd52huq256:
5978 case X86::BI__builtin_ia32_vpmadd52huq512:
5979 return interp__builtin_elementwise_triop(
5980 S, OpPC, Call, Fn: [](const APSInt &A, const APSInt &B, const APSInt &C) {
5981 return A + llvm::APIntOps::mulhu(C1: B.trunc(width: 52), C2: C.trunc(width: 52)).zext(width: 64);
5982 });
5983
5984 case X86::BI__builtin_ia32_vpshldd128:
5985 case X86::BI__builtin_ia32_vpshldd256:
5986 case X86::BI__builtin_ia32_vpshldd512:
5987 case X86::BI__builtin_ia32_vpshldq128:
5988 case X86::BI__builtin_ia32_vpshldq256:
5989 case X86::BI__builtin_ia32_vpshldq512:
5990 case X86::BI__builtin_ia32_vpshldw128:
5991 case X86::BI__builtin_ia32_vpshldw256:
5992 case X86::BI__builtin_ia32_vpshldw512:
5993 return interp__builtin_elementwise_triop(
5994 S, OpPC, Call,
5995 Fn: [](const APSInt &Hi, const APSInt &Lo, const APSInt &Amt) {
5996 return llvm::APIntOps::fshl(Hi, Lo, Shift: Amt);
5997 });
5998
5999 case X86::BI__builtin_ia32_vpshrdd128:
6000 case X86::BI__builtin_ia32_vpshrdd256:
6001 case X86::BI__builtin_ia32_vpshrdd512:
6002 case X86::BI__builtin_ia32_vpshrdq128:
6003 case X86::BI__builtin_ia32_vpshrdq256:
6004 case X86::BI__builtin_ia32_vpshrdq512:
6005 case X86::BI__builtin_ia32_vpshrdw128:
6006 case X86::BI__builtin_ia32_vpshrdw256:
6007 case X86::BI__builtin_ia32_vpshrdw512:
6008 // NOTE: Reversed Hi/Lo operands.
6009 return interp__builtin_elementwise_triop(
6010 S, OpPC, Call,
6011 Fn: [](const APSInt &Lo, const APSInt &Hi, const APSInt &Amt) {
6012 return llvm::APIntOps::fshr(Hi, Lo, Shift: Amt);
6013 });
6014 case X86::BI__builtin_ia32_vpconflictsi_128:
6015 case X86::BI__builtin_ia32_vpconflictsi_256:
6016 case X86::BI__builtin_ia32_vpconflictsi_512:
6017 case X86::BI__builtin_ia32_vpconflictdi_128:
6018 case X86::BI__builtin_ia32_vpconflictdi_256:
6019 case X86::BI__builtin_ia32_vpconflictdi_512:
6020 return interp__builtin_ia32_vpconflict(S, OpPC, Call);
6021 case X86::BI__builtin_ia32_compressdf128_mask:
6022 case X86::BI__builtin_ia32_compressdf256_mask:
6023 case X86::BI__builtin_ia32_compressdf512_mask:
6024 case X86::BI__builtin_ia32_compressdi128_mask:
6025 case X86::BI__builtin_ia32_compressdi256_mask:
6026 case X86::BI__builtin_ia32_compressdi512_mask:
6027 case X86::BI__builtin_ia32_compresshi128_mask:
6028 case X86::BI__builtin_ia32_compresshi256_mask:
6029 case X86::BI__builtin_ia32_compresshi512_mask:
6030 case X86::BI__builtin_ia32_compressqi128_mask:
6031 case X86::BI__builtin_ia32_compressqi256_mask:
6032 case X86::BI__builtin_ia32_compressqi512_mask:
6033 case X86::BI__builtin_ia32_compresssf128_mask:
6034 case X86::BI__builtin_ia32_compresssf256_mask:
6035 case X86::BI__builtin_ia32_compresssf512_mask:
6036 case X86::BI__builtin_ia32_compresssi128_mask:
6037 case X86::BI__builtin_ia32_compresssi256_mask:
6038 case X86::BI__builtin_ia32_compresssi512_mask: {
6039 unsigned NumElems =
6040 Call->getArg(Arg: 0)->getType()->castAs<VectorType>()->getNumElements();
6041 return interp__builtin_ia32_shuffle_generic(
6042 S, OpPC, Call, GetSourceIndex: [NumElems](unsigned DstIdx, const APInt &ShuffleMask) {
6043 APInt CompressMask = ShuffleMask.trunc(width: NumElems);
6044 if (DstIdx < CompressMask.popcount()) {
6045 while (DstIdx != 0) {
6046 CompressMask = CompressMask & (CompressMask - 1);
6047 DstIdx--;
6048 }
6049 return std::pair<unsigned, int>{
6050 0, static_cast<int>(CompressMask.countr_zero())};
6051 }
6052 return std::pair<unsigned, int>{1, static_cast<int>(DstIdx)};
6053 });
6054 }
6055 case X86::BI__builtin_ia32_expanddf128_mask:
6056 case X86::BI__builtin_ia32_expanddf256_mask:
6057 case X86::BI__builtin_ia32_expanddf512_mask:
6058 case X86::BI__builtin_ia32_expanddi128_mask:
6059 case X86::BI__builtin_ia32_expanddi256_mask:
6060 case X86::BI__builtin_ia32_expanddi512_mask:
6061 case X86::BI__builtin_ia32_expandhi128_mask:
6062 case X86::BI__builtin_ia32_expandhi256_mask:
6063 case X86::BI__builtin_ia32_expandhi512_mask:
6064 case X86::BI__builtin_ia32_expandqi128_mask:
6065 case X86::BI__builtin_ia32_expandqi256_mask:
6066 case X86::BI__builtin_ia32_expandqi512_mask:
6067 case X86::BI__builtin_ia32_expandsf128_mask:
6068 case X86::BI__builtin_ia32_expandsf256_mask:
6069 case X86::BI__builtin_ia32_expandsf512_mask:
6070 case X86::BI__builtin_ia32_expandsi128_mask:
6071 case X86::BI__builtin_ia32_expandsi256_mask:
6072 case X86::BI__builtin_ia32_expandsi512_mask: {
6073 return interp__builtin_ia32_shuffle_generic(
6074 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, const APInt &ShuffleMask) {
6075 // Trunc to the sub-mask for the dst index and count the number of
6076 // src elements used prior to that.
6077 APInt ExpandMask = ShuffleMask.trunc(width: DstIdx + 1);
6078 if (ExpandMask[DstIdx]) {
6079 int SrcIdx = ExpandMask.popcount() - 1;
6080 return std::pair<unsigned, int>{0, SrcIdx};
6081 }
6082 return std::pair<unsigned, int>{1, static_cast<int>(DstIdx)};
6083 });
6084 }
6085 case clang::X86::BI__builtin_ia32_blendpd:
6086 case clang::X86::BI__builtin_ia32_blendpd256:
6087 case clang::X86::BI__builtin_ia32_blendps:
6088 case clang::X86::BI__builtin_ia32_blendps256:
6089 case clang::X86::BI__builtin_ia32_pblendw128:
6090 case clang::X86::BI__builtin_ia32_pblendw256:
6091 case clang::X86::BI__builtin_ia32_pblendd128:
6092 case clang::X86::BI__builtin_ia32_pblendd256:
6093 return interp__builtin_ia32_shuffle_generic(
6094 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6095 // Bit index for mask.
6096 unsigned MaskBit = (ShuffleMask >> (DstIdx % 8)) & 0x1;
6097 unsigned SrcVecIdx = MaskBit ? 1 : 0; // 1 = TrueVec, 0 = FalseVec
6098 return std::pair<unsigned, int>{SrcVecIdx, static_cast<int>(DstIdx)};
6099 });
6100
6101
6102
6103 case clang::X86::BI__builtin_ia32_blendvpd:
6104 case clang::X86::BI__builtin_ia32_blendvpd256:
6105 case clang::X86::BI__builtin_ia32_blendvps:
6106 case clang::X86::BI__builtin_ia32_blendvps256:
6107 return interp__builtin_elementwise_triop_fp(
6108 S, OpPC, Call,
6109 Fn: [](const APFloat &F, const APFloat &T, const APFloat &C,
6110 llvm::RoundingMode) { return C.isNegative() ? T : F; });
6111
6112 case clang::X86::BI__builtin_ia32_pblendvb128:
6113 case clang::X86::BI__builtin_ia32_pblendvb256:
6114 return interp__builtin_elementwise_triop(
6115 S, OpPC, Call, Fn: [](const APSInt &F, const APSInt &T, const APSInt &C) {
6116 return ((APInt)C).isNegative() ? T : F;
6117 });
6118 case X86::BI__builtin_ia32_ptestz128:
6119 case X86::BI__builtin_ia32_ptestz256:
6120 case X86::BI__builtin_ia32_vtestzps:
6121 case X86::BI__builtin_ia32_vtestzps256:
6122 case X86::BI__builtin_ia32_vtestzpd:
6123 case X86::BI__builtin_ia32_vtestzpd256:
6124 return interp__builtin_ia32_test_op(
6125 S, OpPC, Call,
6126 Fn: [](const APInt &A, const APInt &B) { return (A & B) == 0; });
6127 case X86::BI__builtin_ia32_ptestc128:
6128 case X86::BI__builtin_ia32_ptestc256:
6129 case X86::BI__builtin_ia32_vtestcps:
6130 case X86::BI__builtin_ia32_vtestcps256:
6131 case X86::BI__builtin_ia32_vtestcpd:
6132 case X86::BI__builtin_ia32_vtestcpd256:
6133 return interp__builtin_ia32_test_op(
6134 S, OpPC, Call,
6135 Fn: [](const APInt &A, const APInt &B) { return (~A & B) == 0; });
6136 case X86::BI__builtin_ia32_ptestnzc128:
6137 case X86::BI__builtin_ia32_ptestnzc256:
6138 case X86::BI__builtin_ia32_vtestnzcps:
6139 case X86::BI__builtin_ia32_vtestnzcps256:
6140 case X86::BI__builtin_ia32_vtestnzcpd:
6141 case X86::BI__builtin_ia32_vtestnzcpd256:
6142 return interp__builtin_ia32_test_op(
6143 S, OpPC, Call, Fn: [](const APInt &A, const APInt &B) {
6144 return ((A & B) != 0) && ((~A & B) != 0);
6145 });
6146 case X86::BI__builtin_ia32_selectb_128:
6147 case X86::BI__builtin_ia32_selectb_256:
6148 case X86::BI__builtin_ia32_selectb_512:
6149 case X86::BI__builtin_ia32_selectw_128:
6150 case X86::BI__builtin_ia32_selectw_256:
6151 case X86::BI__builtin_ia32_selectw_512:
6152 case X86::BI__builtin_ia32_selectd_128:
6153 case X86::BI__builtin_ia32_selectd_256:
6154 case X86::BI__builtin_ia32_selectd_512:
6155 case X86::BI__builtin_ia32_selectq_128:
6156 case X86::BI__builtin_ia32_selectq_256:
6157 case X86::BI__builtin_ia32_selectq_512:
6158 case X86::BI__builtin_ia32_selectph_128:
6159 case X86::BI__builtin_ia32_selectph_256:
6160 case X86::BI__builtin_ia32_selectph_512:
6161 case X86::BI__builtin_ia32_selectpbf_128:
6162 case X86::BI__builtin_ia32_selectpbf_256:
6163 case X86::BI__builtin_ia32_selectpbf_512:
6164 case X86::BI__builtin_ia32_selectps_128:
6165 case X86::BI__builtin_ia32_selectps_256:
6166 case X86::BI__builtin_ia32_selectps_512:
6167 case X86::BI__builtin_ia32_selectpd_128:
6168 case X86::BI__builtin_ia32_selectpd_256:
6169 case X86::BI__builtin_ia32_selectpd_512:
6170 return interp__builtin_ia32_select(S, OpPC, Call);
6171
6172 case X86::BI__builtin_ia32_shufps:
6173 case X86::BI__builtin_ia32_shufps256:
6174 case X86::BI__builtin_ia32_shufps512:
6175 return interp__builtin_ia32_shuffle_generic(
6176 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6177 unsigned NumElemPerLane = 4;
6178 unsigned NumSelectableElems = NumElemPerLane / 2;
6179 unsigned BitsPerElem = 2;
6180 unsigned IndexMask = 0x3;
6181 unsigned MaskBits = 8;
6182 unsigned Lane = DstIdx / NumElemPerLane;
6183 unsigned ElemInLane = DstIdx % NumElemPerLane;
6184 unsigned LaneOffset = Lane * NumElemPerLane;
6185 unsigned SrcIdx = ElemInLane >= NumSelectableElems ? 1 : 0;
6186 unsigned BitIndex = (DstIdx * BitsPerElem) % MaskBits;
6187 unsigned Index = (ShuffleMask >> BitIndex) & IndexMask;
6188 return std::pair<unsigned, int>{SrcIdx,
6189 static_cast<int>(LaneOffset + Index)};
6190 });
6191 case X86::BI__builtin_ia32_shufpd:
6192 case X86::BI__builtin_ia32_shufpd256:
6193 case X86::BI__builtin_ia32_shufpd512:
6194 return interp__builtin_ia32_shuffle_generic(
6195 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6196 unsigned NumElemPerLane = 2;
6197 unsigned NumSelectableElems = NumElemPerLane / 2;
6198 unsigned BitsPerElem = 1;
6199 unsigned IndexMask = 0x1;
6200 unsigned MaskBits = 8;
6201 unsigned Lane = DstIdx / NumElemPerLane;
6202 unsigned ElemInLane = DstIdx % NumElemPerLane;
6203 unsigned LaneOffset = Lane * NumElemPerLane;
6204 unsigned SrcIdx = ElemInLane >= NumSelectableElems ? 1 : 0;
6205 unsigned BitIndex = (DstIdx * BitsPerElem) % MaskBits;
6206 unsigned Index = (ShuffleMask >> BitIndex) & IndexMask;
6207 return std::pair<unsigned, int>{SrcIdx,
6208 static_cast<int>(LaneOffset + Index)};
6209 });
6210
6211 case X86::BI__builtin_ia32_vgf2p8affineinvqb_v16qi:
6212 case X86::BI__builtin_ia32_vgf2p8affineinvqb_v32qi:
6213 case X86::BI__builtin_ia32_vgf2p8affineinvqb_v64qi:
6214 return interp__builtin_ia32_gfni_affine(S, OpPC, Call, Inverse: true);
6215 case X86::BI__builtin_ia32_vgf2p8affineqb_v16qi:
6216 case X86::BI__builtin_ia32_vgf2p8affineqb_v32qi:
6217 case X86::BI__builtin_ia32_vgf2p8affineqb_v64qi:
6218 return interp__builtin_ia32_gfni_affine(S, OpPC, Call, Inverse: false);
6219
6220 case X86::BI__builtin_ia32_vgf2p8mulb_v16qi:
6221 case X86::BI__builtin_ia32_vgf2p8mulb_v32qi:
6222 case X86::BI__builtin_ia32_vgf2p8mulb_v64qi:
6223 return interp__builtin_ia32_gfni_mul(S, OpPC, Call);
6224
6225 case X86::BI__builtin_ia32_bmacor16x16x16_v16hi:
6226 case X86::BI__builtin_ia32_bmacor16x16x16_v32hi:
6227 return interp__builtin_ia32_bmac(S, OpPC, Call, /*IsXor=*/false);
6228 case X86::BI__builtin_ia32_bmacxor16x16x16_v16hi:
6229 case X86::BI__builtin_ia32_bmacxor16x16x16_v32hi:
6230 return interp__builtin_ia32_bmac(S, OpPC, Call, /*IsXor=*/true);
6231
6232 case X86::BI__builtin_ia32_insertps128:
6233 return interp__builtin_ia32_shuffle_generic(
6234 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned Mask) {
6235 // Bits [3:0]: zero mask - if bit is set, zero this element
6236 if ((Mask & (1 << DstIdx)) != 0) {
6237 return std::pair<unsigned, int>{0, -1};
6238 }
6239 // Bits [7:6]: select element from source vector Y (0-3)
6240 // Bits [5:4]: select destination position (0-3)
6241 unsigned SrcElem = (Mask >> 6) & 0x3;
6242 unsigned DstElem = (Mask >> 4) & 0x3;
6243 if (DstIdx == DstElem) {
6244 // Insert element from source vector (B) at this position
6245 return std::pair<unsigned, int>{1, static_cast<int>(SrcElem)};
6246 } else {
6247 // Copy from destination vector (A)
6248 return std::pair<unsigned, int>{0, static_cast<int>(DstIdx)};
6249 }
6250 });
6251 case X86::BI__builtin_ia32_permvarsi256:
6252 case X86::BI__builtin_ia32_permvarsf256:
6253 case X86::BI__builtin_ia32_permvardf512:
6254 case X86::BI__builtin_ia32_permvardi512:
6255 case X86::BI__builtin_ia32_permvarhi128:
6256 return interp__builtin_ia32_shuffle_generic(
6257 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6258 int Offset = ShuffleMask & 0x7;
6259 return std::pair<unsigned, int>{0, Offset};
6260 });
6261 case X86::BI__builtin_ia32_permvarqi128:
6262 case X86::BI__builtin_ia32_permvarhi256:
6263 case X86::BI__builtin_ia32_permvarsi512:
6264 case X86::BI__builtin_ia32_permvarsf512:
6265 return interp__builtin_ia32_shuffle_generic(
6266 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6267 int Offset = ShuffleMask & 0xF;
6268 return std::pair<unsigned, int>{0, Offset};
6269 });
6270 case X86::BI__builtin_ia32_permvardi256:
6271 case X86::BI__builtin_ia32_permvardf256:
6272 return interp__builtin_ia32_shuffle_generic(
6273 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6274 int Offset = ShuffleMask & 0x3;
6275 return std::pair<unsigned, int>{0, Offset};
6276 });
6277 case X86::BI__builtin_ia32_permvarqi256:
6278 case X86::BI__builtin_ia32_permvarhi512:
6279 return interp__builtin_ia32_shuffle_generic(
6280 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6281 int Offset = ShuffleMask & 0x1F;
6282 return std::pair<unsigned, int>{0, Offset};
6283 });
6284 case X86::BI__builtin_ia32_permvarqi512:
6285 return interp__builtin_ia32_shuffle_generic(
6286 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6287 int Offset = ShuffleMask & 0x3F;
6288 return std::pair<unsigned, int>{0, Offset};
6289 });
6290 case X86::BI__builtin_ia32_vpermi2varq128:
6291 case X86::BI__builtin_ia32_vpermi2varpd128:
6292 return interp__builtin_ia32_shuffle_generic(
6293 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6294 int Offset = ShuffleMask & 0x1;
6295 unsigned SrcIdx = (ShuffleMask >> 1) & 0x1;
6296 return std::pair<unsigned, int>{SrcIdx, Offset};
6297 });
6298 case X86::BI__builtin_ia32_vpermi2vard128:
6299 case X86::BI__builtin_ia32_vpermi2varps128:
6300 case X86::BI__builtin_ia32_vpermi2varq256:
6301 case X86::BI__builtin_ia32_vpermi2varpd256:
6302 return interp__builtin_ia32_shuffle_generic(
6303 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6304 int Offset = ShuffleMask & 0x3;
6305 unsigned SrcIdx = (ShuffleMask >> 2) & 0x1;
6306 return std::pair<unsigned, int>{SrcIdx, Offset};
6307 });
6308 case X86::BI__builtin_ia32_vpermi2varhi128:
6309 case X86::BI__builtin_ia32_vpermi2vard256:
6310 case X86::BI__builtin_ia32_vpermi2varps256:
6311 case X86::BI__builtin_ia32_vpermi2varq512:
6312 case X86::BI__builtin_ia32_vpermi2varpd512:
6313 return interp__builtin_ia32_shuffle_generic(
6314 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6315 int Offset = ShuffleMask & 0x7;
6316 unsigned SrcIdx = (ShuffleMask >> 3) & 0x1;
6317 return std::pair<unsigned, int>{SrcIdx, Offset};
6318 });
6319 case X86::BI__builtin_ia32_vpermi2varqi128:
6320 case X86::BI__builtin_ia32_vpermi2varhi256:
6321 case X86::BI__builtin_ia32_vpermi2vard512:
6322 case X86::BI__builtin_ia32_vpermi2varps512:
6323 return interp__builtin_ia32_shuffle_generic(
6324 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6325 int Offset = ShuffleMask & 0xF;
6326 unsigned SrcIdx = (ShuffleMask >> 4) & 0x1;
6327 return std::pair<unsigned, int>{SrcIdx, Offset};
6328 });
6329 case X86::BI__builtin_ia32_vpermi2varqi256:
6330 case X86::BI__builtin_ia32_vpermi2varhi512:
6331 return interp__builtin_ia32_shuffle_generic(
6332 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6333 int Offset = ShuffleMask & 0x1F;
6334 unsigned SrcIdx = (ShuffleMask >> 5) & 0x1;
6335 return std::pair<unsigned, int>{SrcIdx, Offset};
6336 });
6337 case X86::BI__builtin_ia32_vpermi2varqi512:
6338 return interp__builtin_ia32_shuffle_generic(
6339 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6340 int Offset = ShuffleMask & 0x3F;
6341 unsigned SrcIdx = (ShuffleMask >> 6) & 0x1;
6342 return std::pair<unsigned, int>{SrcIdx, Offset};
6343 });
6344 case X86::BI__builtin_ia32_vperm2f128_pd256:
6345 case X86::BI__builtin_ia32_vperm2f128_ps256:
6346 case X86::BI__builtin_ia32_vperm2f128_si256:
6347 case X86::BI__builtin_ia32_permti256: {
6348 unsigned NumElements =
6349 Call->getArg(Arg: 0)->getType()->castAs<VectorType>()->getNumElements();
6350 unsigned PreservedBitsCnt = NumElements >> 2;
6351 return interp__builtin_ia32_shuffle_generic(
6352 S, OpPC, Call,
6353 GetSourceIndex: [PreservedBitsCnt](unsigned DstIdx, unsigned ShuffleMask) {
6354 unsigned ControlBitsCnt = DstIdx >> PreservedBitsCnt << 2;
6355 unsigned ControlBits = ShuffleMask >> ControlBitsCnt;
6356
6357 if (ControlBits & 0b1000)
6358 return std::make_pair(x: 0u, y: -1);
6359
6360 unsigned SrcVecIdx = (ControlBits & 0b10) >> 1;
6361 unsigned PreservedBitsMask = (1 << PreservedBitsCnt) - 1;
6362 int SrcIdx = ((ControlBits & 0b1) << PreservedBitsCnt) |
6363 (DstIdx & PreservedBitsMask);
6364 return std::make_pair(x&: SrcVecIdx, y&: SrcIdx);
6365 });
6366 }
6367 case X86::BI__builtin_ia32_pshufb128:
6368 case X86::BI__builtin_ia32_pshufb256:
6369 case X86::BI__builtin_ia32_pshufb512:
6370 return interp__builtin_ia32_shuffle_generic(
6371 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6372 uint8_t Ctlb = static_cast<uint8_t>(ShuffleMask);
6373 if (Ctlb & 0x80)
6374 return std::make_pair(x: 0, y: -1);
6375
6376 unsigned LaneBase = (DstIdx / 16) * 16;
6377 unsigned SrcOffset = Ctlb & 0x0F;
6378 unsigned SrcIdx = LaneBase + SrcOffset;
6379 return std::make_pair(x: 0, y: static_cast<int>(SrcIdx));
6380 });
6381
6382 case X86::BI__builtin_ia32_pshuflw:
6383 case X86::BI__builtin_ia32_pshuflw256:
6384 case X86::BI__builtin_ia32_pshuflw512:
6385 return interp__builtin_ia32_shuffle_generic(
6386 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6387 unsigned LaneBase = (DstIdx / 8) * 8;
6388 unsigned LaneIdx = DstIdx % 8;
6389 if (LaneIdx < 4) {
6390 unsigned Sel = (ShuffleMask >> (2 * LaneIdx)) & 0x3;
6391 return std::make_pair(x: 0, y: static_cast<int>(LaneBase + Sel));
6392 }
6393
6394 return std::make_pair(x: 0, y: static_cast<int>(DstIdx));
6395 });
6396
6397 case X86::BI__builtin_ia32_pshufhw:
6398 case X86::BI__builtin_ia32_pshufhw256:
6399 case X86::BI__builtin_ia32_pshufhw512:
6400 return interp__builtin_ia32_shuffle_generic(
6401 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6402 unsigned LaneBase = (DstIdx / 8) * 8;
6403 unsigned LaneIdx = DstIdx % 8;
6404 if (LaneIdx >= 4) {
6405 unsigned Sel = (ShuffleMask >> (2 * (LaneIdx - 4))) & 0x3;
6406 return std::make_pair(x: 0, y: static_cast<int>(LaneBase + 4 + Sel));
6407 }
6408
6409 return std::make_pair(x: 0, y: static_cast<int>(DstIdx));
6410 });
6411
6412 case X86::BI__builtin_ia32_pshufd:
6413 case X86::BI__builtin_ia32_pshufd256:
6414 case X86::BI__builtin_ia32_pshufd512:
6415 case X86::BI__builtin_ia32_vpermilps:
6416 case X86::BI__builtin_ia32_vpermilps256:
6417 case X86::BI__builtin_ia32_vpermilps512:
6418 return interp__builtin_ia32_shuffle_generic(
6419 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6420 unsigned LaneBase = (DstIdx / 4) * 4;
6421 unsigned LaneIdx = DstIdx % 4;
6422 unsigned Sel = (ShuffleMask >> (2 * LaneIdx)) & 0x3;
6423 return std::make_pair(x: 0, y: static_cast<int>(LaneBase + Sel));
6424 });
6425
6426 case X86::BI__builtin_ia32_vpermilvarpd:
6427 case X86::BI__builtin_ia32_vpermilvarpd256:
6428 case X86::BI__builtin_ia32_vpermilvarpd512:
6429 return interp__builtin_ia32_shuffle_generic(
6430 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6431 unsigned NumElemPerLane = 2;
6432 unsigned Lane = DstIdx / NumElemPerLane;
6433 unsigned Offset = ShuffleMask & 0b10 ? 1 : 0;
6434 return std::make_pair(
6435 x: 0, y: static_cast<int>(Lane * NumElemPerLane + Offset));
6436 });
6437
6438 case X86::BI__builtin_ia32_vpermilvarps:
6439 case X86::BI__builtin_ia32_vpermilvarps256:
6440 case X86::BI__builtin_ia32_vpermilvarps512:
6441 return interp__builtin_ia32_shuffle_generic(
6442 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6443 unsigned NumElemPerLane = 4;
6444 unsigned Lane = DstIdx / NumElemPerLane;
6445 unsigned Offset = ShuffleMask & 0b11;
6446 return std::make_pair(
6447 x: 0, y: static_cast<int>(Lane * NumElemPerLane + Offset));
6448 });
6449
6450 case X86::BI__builtin_ia32_vpermilpd:
6451 case X86::BI__builtin_ia32_vpermilpd256:
6452 case X86::BI__builtin_ia32_vpermilpd512:
6453 return interp__builtin_ia32_shuffle_generic(
6454 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned Control) {
6455 unsigned NumElemPerLane = 2;
6456 unsigned BitsPerElem = 1;
6457 unsigned MaskBits = 8;
6458 unsigned IndexMask = 0x1;
6459 unsigned Lane = DstIdx / NumElemPerLane;
6460 unsigned LaneOffset = Lane * NumElemPerLane;
6461 unsigned BitIndex = (DstIdx * BitsPerElem) % MaskBits;
6462 unsigned Index = (Control >> BitIndex) & IndexMask;
6463 return std::make_pair(x: 0, y: static_cast<int>(LaneOffset + Index));
6464 });
6465
6466 case X86::BI__builtin_ia32_permdf256:
6467 case X86::BI__builtin_ia32_permdi256:
6468 return interp__builtin_ia32_shuffle_generic(
6469 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned Control) {
6470 // permute4x64 operates on 4 64-bit elements
6471 // For element i (0-3), extract bits [2*i+1:2*i] from Control
6472 unsigned Index = (Control >> (2 * DstIdx)) & 0x3;
6473 return std::make_pair(x: 0, y: static_cast<int>(Index));
6474 });
6475
6476 case X86::BI__builtin_ia32_vpmultishiftqb128:
6477 case X86::BI__builtin_ia32_vpmultishiftqb256:
6478 case X86::BI__builtin_ia32_vpmultishiftqb512:
6479 return interp__builtin_ia32_multishiftqb(S, OpPC, Call);
6480 case X86::BI__builtin_ia32_kandqi:
6481 case X86::BI__builtin_ia32_kandhi:
6482 case X86::BI__builtin_ia32_kandsi:
6483 case X86::BI__builtin_ia32_kanddi:
6484 return interp__builtin_elementwise_int_binop(
6485 S, OpPC, Call,
6486 Fn: [](const APSInt &LHS, const APSInt &RHS) { return LHS & RHS; });
6487
6488 case X86::BI__builtin_ia32_kandnqi:
6489 case X86::BI__builtin_ia32_kandnhi:
6490 case X86::BI__builtin_ia32_kandnsi:
6491 case X86::BI__builtin_ia32_kandndi:
6492 return interp__builtin_elementwise_int_binop(
6493 S, OpPC, Call,
6494 Fn: [](const APSInt &LHS, const APSInt &RHS) { return ~LHS & RHS; });
6495
6496 case X86::BI__builtin_ia32_korqi:
6497 case X86::BI__builtin_ia32_korhi:
6498 case X86::BI__builtin_ia32_korsi:
6499 case X86::BI__builtin_ia32_kordi:
6500 return interp__builtin_elementwise_int_binop(
6501 S, OpPC, Call,
6502 Fn: [](const APSInt &LHS, const APSInt &RHS) { return LHS | RHS; });
6503
6504 case X86::BI__builtin_ia32_kxnorqi:
6505 case X86::BI__builtin_ia32_kxnorhi:
6506 case X86::BI__builtin_ia32_kxnorsi:
6507 case X86::BI__builtin_ia32_kxnordi:
6508 return interp__builtin_elementwise_int_binop(
6509 S, OpPC, Call,
6510 Fn: [](const APSInt &LHS, const APSInt &RHS) { return ~(LHS ^ RHS); });
6511
6512 case X86::BI__builtin_ia32_kxorqi:
6513 case X86::BI__builtin_ia32_kxorhi:
6514 case X86::BI__builtin_ia32_kxorsi:
6515 case X86::BI__builtin_ia32_kxordi:
6516 return interp__builtin_elementwise_int_binop(
6517 S, OpPC, Call,
6518 Fn: [](const APSInt &LHS, const APSInt &RHS) { return LHS ^ RHS; });
6519
6520 case X86::BI__builtin_ia32_knotqi:
6521 case X86::BI__builtin_ia32_knothi:
6522 case X86::BI__builtin_ia32_knotsi:
6523 case X86::BI__builtin_ia32_knotdi:
6524 return interp__builtin_elementwise_int_unaryop(
6525 S, OpPC, Call, Fn: [](const APSInt &Src) { return ~Src; });
6526
6527 case X86::BI__builtin_ia32_kaddqi:
6528 case X86::BI__builtin_ia32_kaddhi:
6529 case X86::BI__builtin_ia32_kaddsi:
6530 case X86::BI__builtin_ia32_kadddi:
6531 return interp__builtin_elementwise_int_binop(
6532 S, OpPC, Call,
6533 Fn: [](const APSInt &LHS, const APSInt &RHS) { return LHS + RHS; });
6534
6535 case X86::BI__builtin_ia32_kmovb:
6536 case X86::BI__builtin_ia32_kmovw:
6537 case X86::BI__builtin_ia32_kmovd:
6538 case X86::BI__builtin_ia32_kmovq:
6539 return interp__builtin_elementwise_int_unaryop(
6540 S, OpPC, Call, Fn: [](const APSInt &Src) { return Src; });
6541
6542 case X86::BI__builtin_ia32_kunpckhi:
6543 case X86::BI__builtin_ia32_kunpckdi:
6544 case X86::BI__builtin_ia32_kunpcksi:
6545 return interp__builtin_elementwise_int_binop(
6546 S, OpPC, Call, Fn: [](const APSInt &A, const APSInt &B) {
6547 // Generic kunpack: extract lower half of each operand and concatenate
6548 // Result = A[HalfWidth-1:0] concat B[HalfWidth-1:0]
6549 unsigned BW = A.getBitWidth();
6550 return APSInt(A.trunc(width: BW / 2).concat(NewLSB: B.trunc(width: BW / 2)),
6551 A.isUnsigned());
6552 });
6553
6554 case X86::BI__builtin_ia32_phminposuw128:
6555 return interp__builtin_ia32_phminposuw(S, OpPC, Call);
6556
6557 case X86::BI__builtin_ia32_psraq128:
6558 case X86::BI__builtin_ia32_psraq256:
6559 case X86::BI__builtin_ia32_psraq512:
6560 case X86::BI__builtin_ia32_psrad128:
6561 case X86::BI__builtin_ia32_psrad256:
6562 case X86::BI__builtin_ia32_psrad512:
6563 case X86::BI__builtin_ia32_psraw128:
6564 case X86::BI__builtin_ia32_psraw256:
6565 case X86::BI__builtin_ia32_psraw512:
6566 return interp__builtin_ia32_shift_with_count(
6567 S, OpPC, Call,
6568 ShiftOp: [](const APInt &Elt, uint64_t Count) { return Elt.ashr(ShiftAmt: Count); },
6569 OverflowOp: [](const APInt &Elt, unsigned Width) { return Elt.ashr(ShiftAmt: Width - 1); });
6570
6571 case X86::BI__builtin_ia32_psllq128:
6572 case X86::BI__builtin_ia32_psllq256:
6573 case X86::BI__builtin_ia32_psllq512:
6574 case X86::BI__builtin_ia32_pslld128:
6575 case X86::BI__builtin_ia32_pslld256:
6576 case X86::BI__builtin_ia32_pslld512:
6577 case X86::BI__builtin_ia32_psllw128:
6578 case X86::BI__builtin_ia32_psllw256:
6579 case X86::BI__builtin_ia32_psllw512:
6580 return interp__builtin_ia32_shift_with_count(
6581 S, OpPC, Call,
6582 ShiftOp: [](const APInt &Elt, uint64_t Count) { return Elt.shl(shiftAmt: Count); },
6583 OverflowOp: [](const APInt &Elt, unsigned Width) { return APInt::getZero(numBits: Width); });
6584
6585 case X86::BI__builtin_ia32_psrlq128:
6586 case X86::BI__builtin_ia32_psrlq256:
6587 case X86::BI__builtin_ia32_psrlq512:
6588 case X86::BI__builtin_ia32_psrld128:
6589 case X86::BI__builtin_ia32_psrld256:
6590 case X86::BI__builtin_ia32_psrld512:
6591 case X86::BI__builtin_ia32_psrlw128:
6592 case X86::BI__builtin_ia32_psrlw256:
6593 case X86::BI__builtin_ia32_psrlw512:
6594 return interp__builtin_ia32_shift_with_count(
6595 S, OpPC, Call,
6596 ShiftOp: [](const APInt &Elt, uint64_t Count) { return Elt.lshr(shiftAmt: Count); },
6597 OverflowOp: [](const APInt &Elt, unsigned Width) { return APInt::getZero(numBits: Width); });
6598
6599 case X86::BI__builtin_ia32_pternlogd128_mask:
6600 case X86::BI__builtin_ia32_pternlogd256_mask:
6601 case X86::BI__builtin_ia32_pternlogd512_mask:
6602 case X86::BI__builtin_ia32_pternlogq128_mask:
6603 case X86::BI__builtin_ia32_pternlogq256_mask:
6604 case X86::BI__builtin_ia32_pternlogq512_mask:
6605 return interp__builtin_ia32_pternlog(S, OpPC, Call, /*MaskZ=*/false);
6606 case X86::BI__builtin_ia32_pternlogd128_maskz:
6607 case X86::BI__builtin_ia32_pternlogd256_maskz:
6608 case X86::BI__builtin_ia32_pternlogd512_maskz:
6609 case X86::BI__builtin_ia32_pternlogq128_maskz:
6610 case X86::BI__builtin_ia32_pternlogq256_maskz:
6611 case X86::BI__builtin_ia32_pternlogq512_maskz:
6612 return interp__builtin_ia32_pternlog(S, OpPC, Call, /*MaskZ=*/true);
6613 case Builtin::BI__builtin_elementwise_fshl:
6614 return interp__builtin_elementwise_triop(S, OpPC, Call,
6615 Fn: llvm::APIntOps::fshl);
6616 case Builtin::BI__builtin_elementwise_fshr:
6617 return interp__builtin_elementwise_triop(S, OpPC, Call,
6618 Fn: llvm::APIntOps::fshr);
6619
6620 case X86::BI__builtin_ia32_shuf_f32x4_256:
6621 case X86::BI__builtin_ia32_shuf_i32x4_256:
6622 case X86::BI__builtin_ia32_shuf_f64x2_256:
6623 case X86::BI__builtin_ia32_shuf_i64x2_256:
6624 case X86::BI__builtin_ia32_shuf_f32x4:
6625 case X86::BI__builtin_ia32_shuf_i32x4:
6626 case X86::BI__builtin_ia32_shuf_f64x2:
6627 case X86::BI__builtin_ia32_shuf_i64x2: {
6628 // Destination and sources A, B all have the same type.
6629 QualType VecQT = Call->getArg(Arg: 0)->getType();
6630 const auto *VecT = VecQT->castAs<VectorType>();
6631 unsigned NumElems = VecT->getNumElements();
6632 unsigned ElemBits = S.getASTContext().getTypeSize(T: VecT->getElementType());
6633 unsigned LaneBits = 128u;
6634 unsigned NumLanes = (NumElems * ElemBits) / LaneBits;
6635 unsigned NumElemsPerLane = LaneBits / ElemBits;
6636
6637 return interp__builtin_ia32_shuffle_generic(
6638 S, OpPC, Call,
6639 GetSourceIndex: [NumLanes, NumElemsPerLane](unsigned DstIdx, unsigned ShuffleMask) {
6640 // DstIdx determines source. ShuffleMask selects lane in source.
6641 unsigned BitsPerElem = NumLanes / 2;
6642 unsigned IndexMask = (1u << BitsPerElem) - 1;
6643 unsigned Lane = DstIdx / NumElemsPerLane;
6644 unsigned SrcIdx = (Lane < NumLanes / 2) ? 0 : 1;
6645 unsigned BitIdx = BitsPerElem * Lane;
6646 unsigned SrcLaneIdx = (ShuffleMask >> BitIdx) & IndexMask;
6647 unsigned ElemInLane = DstIdx % NumElemsPerLane;
6648 unsigned IdxToPick = SrcLaneIdx * NumElemsPerLane + ElemInLane;
6649 return std::pair<unsigned, int>{SrcIdx, IdxToPick};
6650 });
6651 }
6652
6653 case X86::BI__builtin_ia32_insertf32x4_256:
6654 case X86::BI__builtin_ia32_inserti32x4_256:
6655 case X86::BI__builtin_ia32_insertf64x2_256:
6656 case X86::BI__builtin_ia32_inserti64x2_256:
6657 case X86::BI__builtin_ia32_insertf32x4:
6658 case X86::BI__builtin_ia32_inserti32x4:
6659 case X86::BI__builtin_ia32_insertf64x2_512:
6660 case X86::BI__builtin_ia32_inserti64x2_512:
6661 case X86::BI__builtin_ia32_insertf32x8:
6662 case X86::BI__builtin_ia32_inserti32x8:
6663 case X86::BI__builtin_ia32_insertf64x4:
6664 case X86::BI__builtin_ia32_inserti64x4:
6665 case X86::BI__builtin_ia32_vinsertf128_ps256:
6666 case X86::BI__builtin_ia32_vinsertf128_pd256:
6667 case X86::BI__builtin_ia32_vinsertf128_si256:
6668 case X86::BI__builtin_ia32_insert128i256:
6669 return interp__builtin_ia32_insert_subvector(S, OpPC, Call, ID: BuiltinID);
6670
6671 case clang::X86::BI__builtin_ia32_vcvtps2ph:
6672 case clang::X86::BI__builtin_ia32_vcvtps2ph256:
6673 return interp__builtin_ia32_vcvtps2ph(S, OpPC, Call);
6674
6675 case X86::BI__builtin_ia32_vec_ext_v4hi:
6676 case X86::BI__builtin_ia32_vec_ext_v16qi:
6677 case X86::BI__builtin_ia32_vec_ext_v8hi:
6678 case X86::BI__builtin_ia32_vec_ext_v4si:
6679 case X86::BI__builtin_ia32_vec_ext_v2di:
6680 case X86::BI__builtin_ia32_vec_ext_v32qi:
6681 case X86::BI__builtin_ia32_vec_ext_v16hi:
6682 case X86::BI__builtin_ia32_vec_ext_v8si:
6683 case X86::BI__builtin_ia32_vec_ext_v4di:
6684 case X86::BI__builtin_ia32_vec_ext_v4sf:
6685 return interp__builtin_ia32_vec_ext(S, OpPC, Call, ID: BuiltinID);
6686
6687 case X86::BI__builtin_ia32_vec_set_v4hi:
6688 case X86::BI__builtin_ia32_vec_set_v16qi:
6689 case X86::BI__builtin_ia32_vec_set_v8hi:
6690 case X86::BI__builtin_ia32_vec_set_v4si:
6691 case X86::BI__builtin_ia32_vec_set_v2di:
6692 case X86::BI__builtin_ia32_vec_set_v32qi:
6693 case X86::BI__builtin_ia32_vec_set_v16hi:
6694 case X86::BI__builtin_ia32_vec_set_v8si:
6695 case X86::BI__builtin_ia32_vec_set_v4di:
6696 return interp__builtin_ia32_vec_set(S, OpPC, Call, ID: BuiltinID);
6697
6698 case X86::BI__builtin_ia32_cvtb2mask128:
6699 case X86::BI__builtin_ia32_cvtb2mask256:
6700 case X86::BI__builtin_ia32_cvtb2mask512:
6701 case X86::BI__builtin_ia32_cvtw2mask128:
6702 case X86::BI__builtin_ia32_cvtw2mask256:
6703 case X86::BI__builtin_ia32_cvtw2mask512:
6704 case X86::BI__builtin_ia32_cvtd2mask128:
6705 case X86::BI__builtin_ia32_cvtd2mask256:
6706 case X86::BI__builtin_ia32_cvtd2mask512:
6707 case X86::BI__builtin_ia32_cvtq2mask128:
6708 case X86::BI__builtin_ia32_cvtq2mask256:
6709 case X86::BI__builtin_ia32_cvtq2mask512:
6710 return interp__builtin_ia32_cvt_vec2mask(S, OpPC, Call, ID: BuiltinID);
6711
6712 case X86::BI__builtin_ia32_cvtmask2b128:
6713 case X86::BI__builtin_ia32_cvtmask2b256:
6714 case X86::BI__builtin_ia32_cvtmask2b512:
6715 case X86::BI__builtin_ia32_cvtmask2w128:
6716 case X86::BI__builtin_ia32_cvtmask2w256:
6717 case X86::BI__builtin_ia32_cvtmask2w512:
6718 case X86::BI__builtin_ia32_cvtmask2d128:
6719 case X86::BI__builtin_ia32_cvtmask2d256:
6720 case X86::BI__builtin_ia32_cvtmask2d512:
6721 case X86::BI__builtin_ia32_cvtmask2q128:
6722 case X86::BI__builtin_ia32_cvtmask2q256:
6723 case X86::BI__builtin_ia32_cvtmask2q512:
6724 return interp__builtin_ia32_cvt_mask2vec(S, OpPC, Call, ID: BuiltinID);
6725
6726 case X86::BI__builtin_ia32_cvtsd2ss:
6727 return interp__builtin_ia32_cvtsd2ss(S, OpPC, Call, HasRoundingMask: false);
6728
6729 case X86::BI__builtin_ia32_cvtsd2ss_round_mask:
6730 return interp__builtin_ia32_cvtsd2ss(S, OpPC, Call, HasRoundingMask: true);
6731
6732 case X86::BI__builtin_ia32_cvtpd2ps:
6733 case X86::BI__builtin_ia32_cvtpd2ps256:
6734 return interp__builtin_ia32_cvtpd2ps(S, OpPC, Call, IsMasked: false, HasRounding: false);
6735 case X86::BI__builtin_ia32_cvtpd2ps_mask:
6736 return interp__builtin_ia32_cvtpd2ps(S, OpPC, Call, IsMasked: true, HasRounding: false);
6737 case X86::BI__builtin_ia32_cvtpd2ps512_mask:
6738 return interp__builtin_ia32_cvtpd2ps(S, OpPC, Call, IsMasked: true, HasRounding: true);
6739
6740 case X86::BI__builtin_ia32_cmpb128_mask:
6741 case X86::BI__builtin_ia32_cmpw128_mask:
6742 case X86::BI__builtin_ia32_cmpd128_mask:
6743 case X86::BI__builtin_ia32_cmpq128_mask:
6744 case X86::BI__builtin_ia32_cmpb256_mask:
6745 case X86::BI__builtin_ia32_cmpw256_mask:
6746 case X86::BI__builtin_ia32_cmpd256_mask:
6747 case X86::BI__builtin_ia32_cmpq256_mask:
6748 case X86::BI__builtin_ia32_cmpb512_mask:
6749 case X86::BI__builtin_ia32_cmpw512_mask:
6750 case X86::BI__builtin_ia32_cmpd512_mask:
6751 case X86::BI__builtin_ia32_cmpq512_mask:
6752 return interp__builtin_ia32_cmp_mask(S, OpPC, Call, ID: BuiltinID,
6753 /*IsUnsigned=*/false);
6754
6755 case X86::BI__builtin_ia32_ucmpb128_mask:
6756 case X86::BI__builtin_ia32_ucmpw128_mask:
6757 case X86::BI__builtin_ia32_ucmpd128_mask:
6758 case X86::BI__builtin_ia32_ucmpq128_mask:
6759 case X86::BI__builtin_ia32_ucmpb256_mask:
6760 case X86::BI__builtin_ia32_ucmpw256_mask:
6761 case X86::BI__builtin_ia32_ucmpd256_mask:
6762 case X86::BI__builtin_ia32_ucmpq256_mask:
6763 case X86::BI__builtin_ia32_ucmpb512_mask:
6764 case X86::BI__builtin_ia32_ucmpw512_mask:
6765 case X86::BI__builtin_ia32_ucmpd512_mask:
6766 case X86::BI__builtin_ia32_ucmpq512_mask:
6767 return interp__builtin_ia32_cmp_mask(S, OpPC, Call, ID: BuiltinID,
6768 /*IsUnsigned=*/true);
6769
6770 case X86::BI__builtin_ia32_vpshufbitqmb128_mask:
6771 case X86::BI__builtin_ia32_vpshufbitqmb256_mask:
6772 case X86::BI__builtin_ia32_vpshufbitqmb512_mask:
6773 return interp__builtin_ia32_shufbitqmb_mask(S, OpPC, Call);
6774
6775 case X86::BI__builtin_ia32_pslldqi128_byteshift:
6776 case X86::BI__builtin_ia32_pslldqi256_byteshift:
6777 case X86::BI__builtin_ia32_pslldqi512_byteshift:
6778 // These SLLDQ intrinsics always operate on byte elements (8 bits).
6779 // The lane width is hardcoded to 16 to match the SIMD register size,
6780 // but the algorithm processes one byte per iteration,
6781 // so APInt(8, ...) is correct and intentional.
6782 return interp__builtin_ia32_shuffle_generic(
6783 S, OpPC, Call,
6784 GetSourceIndex: [](unsigned DstIdx, unsigned Shift) -> std::pair<unsigned, int> {
6785 unsigned LaneBase = (DstIdx / 16) * 16;
6786 unsigned LaneIdx = DstIdx % 16;
6787 if (LaneIdx < Shift)
6788 return std::make_pair(x: 0, y: -1);
6789
6790 return std::make_pair(x: 0,
6791 y: static_cast<int>(LaneBase + LaneIdx - Shift));
6792 });
6793
6794 case X86::BI__builtin_ia32_psrldqi128_byteshift:
6795 case X86::BI__builtin_ia32_psrldqi256_byteshift:
6796 case X86::BI__builtin_ia32_psrldqi512_byteshift:
6797 // These SRLDQ intrinsics always operate on byte elements (8 bits).
6798 // The lane width is hardcoded to 16 to match the SIMD register size,
6799 // but the algorithm processes one byte per iteration,
6800 // so APInt(8, ...) is correct and intentional.
6801 return interp__builtin_ia32_shuffle_generic(
6802 S, OpPC, Call,
6803 GetSourceIndex: [](unsigned DstIdx, unsigned Shift) -> std::pair<unsigned, int> {
6804 unsigned LaneBase = (DstIdx / 16) * 16;
6805 unsigned LaneIdx = DstIdx % 16;
6806 if (LaneIdx + Shift < 16)
6807 return std::make_pair(x: 0,
6808 y: static_cast<int>(LaneBase + LaneIdx + Shift));
6809
6810 return std::make_pair(x: 0, y: -1);
6811 });
6812
6813 case X86::BI__builtin_ia32_palignr128:
6814 case X86::BI__builtin_ia32_palignr256:
6815 case X86::BI__builtin_ia32_palignr512:
6816 return interp__builtin_ia32_shuffle_generic(
6817 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned Shift) {
6818 // Default to -1 → zero-fill this destination element
6819 unsigned VecIdx = 1;
6820 int ElemIdx = -1;
6821
6822 int Lane = DstIdx / 16;
6823 int Offset = DstIdx % 16;
6824
6825 // Elements come from VecB first, then VecA after the shift boundary
6826 unsigned ShiftedIdx = Offset + (Shift & 0xFF);
6827 if (ShiftedIdx < 16) { // from VecB
6828 ElemIdx = ShiftedIdx + (Lane * 16);
6829 } else if (ShiftedIdx < 32) { // from VecA
6830 VecIdx = 0;
6831 ElemIdx = (ShiftedIdx - 16) + (Lane * 16);
6832 }
6833
6834 return std::pair<unsigned, int>{VecIdx, ElemIdx};
6835 });
6836
6837 case X86::BI__builtin_ia32_alignd128:
6838 case X86::BI__builtin_ia32_alignd256:
6839 case X86::BI__builtin_ia32_alignd512:
6840 case X86::BI__builtin_ia32_alignq128:
6841 case X86::BI__builtin_ia32_alignq256:
6842 case X86::BI__builtin_ia32_alignq512: {
6843 unsigned NumElems = Call->getType()->castAs<VectorType>()->getNumElements();
6844 return interp__builtin_ia32_shuffle_generic(
6845 S, OpPC, Call, GetSourceIndex: [NumElems](unsigned DstIdx, unsigned Shift) {
6846 unsigned Imm = Shift & 0xFF;
6847 unsigned EffectiveShift = Imm & (NumElems - 1);
6848 unsigned SourcePos = DstIdx + EffectiveShift;
6849 unsigned VecIdx = SourcePos < NumElems ? 1u : 0u;
6850 unsigned ElemIdx = SourcePos & (NumElems - 1);
6851 return std::pair<unsigned, int>{VecIdx, static_cast<int>(ElemIdx)};
6852 });
6853 }
6854
6855 case clang::X86::BI__builtin_ia32_minps:
6856 case clang::X86::BI__builtin_ia32_minpd:
6857 case clang::X86::BI__builtin_ia32_minph128:
6858 case clang::X86::BI__builtin_ia32_minph256:
6859 case clang::X86::BI__builtin_ia32_minps256:
6860 case clang::X86::BI__builtin_ia32_minpd256:
6861 case clang::X86::BI__builtin_ia32_minps512:
6862 case clang::X86::BI__builtin_ia32_minpd512:
6863 case clang::X86::BI__builtin_ia32_minph512:
6864 return interp__builtin_elementwise_fp_binop(
6865 S, OpPC, Call,
6866 Fn: [](const APFloat &A, const APFloat &B,
6867 std::optional<APSInt>) -> std::optional<APFloat> {
6868 if (A.isNaN() || A.isInfinity() || A.isDenormal() || B.isNaN() ||
6869 B.isInfinity() || B.isDenormal())
6870 return std::nullopt;
6871 if (A.isZero() && B.isZero())
6872 return B;
6873 return llvm::minimum(A, B);
6874 });
6875
6876 case clang::X86::BI__builtin_ia32_minss:
6877 case clang::X86::BI__builtin_ia32_minsd:
6878 return interp__builtin_elementwise_fp_binop(
6879 S, OpPC, Call,
6880 Fn: [](const APFloat &A, const APFloat &B,
6881 std::optional<APSInt> RoundingMode) -> std::optional<APFloat> {
6882 return EvalScalarMinMaxFp(A, B, RoundingMode, /*IsMin=*/true);
6883 },
6884 /*IsScalar=*/true);
6885
6886 case clang::X86::BI__builtin_ia32_minsd_round_mask:
6887 case clang::X86::BI__builtin_ia32_minss_round_mask:
6888 case clang::X86::BI__builtin_ia32_minsh_round_mask:
6889 case clang::X86::BI__builtin_ia32_maxsd_round_mask:
6890 case clang::X86::BI__builtin_ia32_maxss_round_mask:
6891 case clang::X86::BI__builtin_ia32_maxsh_round_mask: {
6892 bool IsMin = BuiltinID == clang::X86::BI__builtin_ia32_minsd_round_mask ||
6893 BuiltinID == clang::X86::BI__builtin_ia32_minss_round_mask ||
6894 BuiltinID == clang::X86::BI__builtin_ia32_minsh_round_mask;
6895 return interp__builtin_scalar_fp_round_mask_binop(
6896 S, OpPC, Call,
6897 Fn: [IsMin](const APFloat &A, const APFloat &B,
6898 std::optional<APSInt> RoundingMode) -> std::optional<APFloat> {
6899 return EvalScalarMinMaxFp(A, B, RoundingMode, IsMin);
6900 });
6901 }
6902
6903 case clang::X86::BI__builtin_ia32_maxps:
6904 case clang::X86::BI__builtin_ia32_maxpd:
6905 case clang::X86::BI__builtin_ia32_maxph128:
6906 case clang::X86::BI__builtin_ia32_maxph256:
6907 case clang::X86::BI__builtin_ia32_maxps256:
6908 case clang::X86::BI__builtin_ia32_maxpd256:
6909 case clang::X86::BI__builtin_ia32_maxps512:
6910 case clang::X86::BI__builtin_ia32_maxpd512:
6911 case clang::X86::BI__builtin_ia32_maxph512:
6912 return interp__builtin_elementwise_fp_binop(
6913 S, OpPC, Call,
6914 Fn: [](const APFloat &A, const APFloat &B,
6915 std::optional<APSInt>) -> std::optional<APFloat> {
6916 if (A.isNaN() || A.isInfinity() || A.isDenormal() || B.isNaN() ||
6917 B.isInfinity() || B.isDenormal())
6918 return std::nullopt;
6919 if (A.isZero() && B.isZero())
6920 return B;
6921 return llvm::maximum(A, B);
6922 });
6923
6924 case clang::X86::BI__builtin_ia32_maxss:
6925 case clang::X86::BI__builtin_ia32_maxsd:
6926 return interp__builtin_elementwise_fp_binop(
6927 S, OpPC, Call,
6928 Fn: [](const APFloat &A, const APFloat &B,
6929 std::optional<APSInt> RoundingMode) -> std::optional<APFloat> {
6930 return EvalScalarMinMaxFp(A, B, RoundingMode, /*IsMin=*/false);
6931 },
6932 /*IsScalar=*/true);
6933 case X86::BI__builtin_ia32_vpdpwssd128:
6934 case X86::BI__builtin_ia32_vpdpwssd256:
6935 case X86::BI__builtin_ia32_vpdpwssd512:
6936 case X86::BI__builtin_ia32_vpdpbusd128:
6937 case X86::BI__builtin_ia32_vpdpbusd256:
6938 case X86::BI__builtin_ia32_vpdpbusd512:
6939 return interp__builtin_ia32_vpdp(S, OpPC, Call, IsSaturating: false);
6940 case X86::BI__builtin_ia32_vpdpwssds128:
6941 case X86::BI__builtin_ia32_vpdpwssds256:
6942 case X86::BI__builtin_ia32_vpdpwssds512:
6943 case X86::BI__builtin_ia32_vpdpbusds128:
6944 case X86::BI__builtin_ia32_vpdpbusds256:
6945 case X86::BI__builtin_ia32_vpdpbusds512:
6946 return interp__builtin_ia32_vpdp(S, OpPC, Call, IsSaturating: true);
6947 case X86::BI__builtin_ia32_cvtss2si:
6948 case X86::BI__builtin_ia32_cvtsd2si:
6949 case X86::BI__builtin_ia32_cvttss2si:
6950 case X86::BI__builtin_ia32_cvttsd2si:
6951 case X86::BI__builtin_ia32_cvtss2si64:
6952 case X86::BI__builtin_ia32_cvtsd2si64:
6953 case X86::BI__builtin_ia32_cvttss2si64:
6954 case X86::BI__builtin_ia32_cvttsd2si64:
6955 return interp_builtin_ia32_cvt_scalar_to_int(S, OpPC, E: Call);
6956 case X86::BI__builtin_ia32_cvtpd2dq:
6957 case X86::BI__builtin_ia32_cvttpd2dq:
6958 case X86::BI__builtin_ia32_cvtps2dq:
6959 case X86::BI__builtin_ia32_cvtpd2dq256:
6960 case X86::BI__builtin_ia32_cvtps2dq256:
6961 case X86::BI__builtin_ia32_cvttps2dq:
6962 case X86::BI__builtin_ia32_cvttpd2dq256:
6963 case X86::BI__builtin_ia32_cvttps2dq256:
6964 return interp_builtin_ia32_cvt_vector_to_int(S, OpPC, E: Call);
6965
6966 case Builtin::BI__builtin_meta_is_type:
6967 return interp__builtin_meta_is_type(S, OpPC, Call);
6968
6969 default:
6970 return Invalid(S, OpPC);
6971 }
6972
6973 llvm_unreachable("Unhandled builtin ID");
6974}
6975
6976bool InterpretOffsetOf(InterpState &S, CodePtr OpPC, const OffsetOfExpr *E,
6977 ArrayRef<int64_t> ArrayIndices, int64_t &IntResult) {
6978 S.getASTContext().recordOffsetOfEvaluation(E);
6979 CharUnits Result;
6980 unsigned N = E->getNumComponents();
6981 assert(N > 0);
6982
6983 unsigned ArrayIndex = 0;
6984 QualType CurrentType = E->getTypeSourceInfo()->getType();
6985 for (unsigned I = 0; I != N; ++I) {
6986 const OffsetOfNode &Node = E->getComponent(Idx: I);
6987 switch (Node.getKind()) {
6988 case OffsetOfNode::Field: {
6989 const FieldDecl *MemberDecl = Node.getField();
6990 const auto *RD = CurrentType->getAsRecordDecl();
6991 if (!RD || RD->isInvalidDecl())
6992 return false;
6993 const ASTRecordLayout &RL = S.getASTContext().getASTRecordLayout(D: RD);
6994 unsigned FieldIndex = MemberDecl->getFieldIndex();
6995 assert(FieldIndex < RL.getFieldCount() && "offsetof field in wrong type");
6996 Result +=
6997 S.getASTContext().toCharUnitsFromBits(BitSize: RL.getFieldOffset(FieldNo: FieldIndex));
6998 CurrentType = MemberDecl->getType().getNonReferenceType();
6999 break;
7000 }
7001 case OffsetOfNode::Array: {
7002 // When generating bytecode, we put all the index expressions as Sint64 on
7003 // the stack.
7004 int64_t Index = ArrayIndices[ArrayIndex];
7005 if (Index < 0)
7006 return Invalid(S, OpPC);
7007 const ArrayType *AT = S.getASTContext().getAsArrayType(T: CurrentType);
7008 if (!AT)
7009 return false;
7010 CurrentType = AT->getElementType();
7011 CharUnits ElementSize = S.getASTContext().getTypeSizeInChars(T: CurrentType);
7012 int64_t ElemSize = ElementSize.getQuantity();
7013 if (Index != 0 && ElemSize > (llvm::maxIntN(N: 64) / Index)) {
7014 S.FFDiag(Loc: S.Current->getLocation(PC: OpPC),
7015 DiagId: diag::note_constexpr_offsetof_overflow)
7016 << S.Current->getRange(PC: OpPC);
7017 return false;
7018 }
7019 int64_t Offset = Index * ElemSize;
7020 if (Result.getQuantity() > llvm::maxIntN(N: 64) - Offset) {
7021 S.FFDiag(Loc: S.Current->getLocation(PC: OpPC),
7022 DiagId: diag::note_constexpr_offsetof_overflow)
7023 << S.Current->getRange(PC: OpPC);
7024 return false;
7025 }
7026 Result += CharUnits::fromQuantity(Quantity: Offset);
7027 ++ArrayIndex;
7028 break;
7029 }
7030 case OffsetOfNode::Base: {
7031 const CXXBaseSpecifier *BaseSpec = Node.getBase();
7032 if (BaseSpec->isVirtual())
7033 return false;
7034
7035 // Find the layout of the class whose base we are looking into.
7036 const auto *RD = CurrentType->getAsCXXRecordDecl();
7037 if (!RD || RD->isInvalidDecl())
7038 return false;
7039 const ASTRecordLayout &RL = S.getASTContext().getASTRecordLayout(D: RD);
7040
7041 // Find the base class itself.
7042 CurrentType = BaseSpec->getType();
7043 const auto *BaseRD = CurrentType->getAsCXXRecordDecl();
7044 if (!BaseRD)
7045 return false;
7046
7047 // Add the offset to the base.
7048 Result += RL.getBaseClassOffset(Base: BaseRD);
7049 break;
7050 }
7051 case OffsetOfNode::Identifier:
7052 llvm_unreachable("Dependent OffsetOfExpr?");
7053 }
7054 }
7055
7056 IntResult = Result.getQuantity();
7057
7058 return true;
7059}
7060
7061bool SetThreeWayComparisonField(InterpState &S, CodePtr OpPC,
7062 const Pointer &Ptr, const APSInt &IntValue) {
7063
7064 const Record *R = Ptr.getRecord();
7065 assert(R);
7066 assert(R->getNumFields() == 1);
7067
7068 unsigned FieldOffset = R->getField(I: 0u)->Offset;
7069 PtrView FieldPtr = Ptr.view().atField(Offset: FieldOffset);
7070 PrimType FieldT = FieldPtr.getFieldDesc()->getPrimType();
7071
7072 INT_TYPE_SWITCH(FieldT,
7073 FieldPtr.deref<T>() = T::from(IntValue.getSExtValue()));
7074 FieldPtr.initialize();
7075 return true;
7076}
7077
7078static void zeroAll(PtrView Dest) {
7079 const Descriptor *Desc = Dest.getFieldDesc();
7080
7081 if (Desc->isPrimitive()) {
7082 TYPE_SWITCH(Desc->getPrimType(), {
7083 Dest.deref<T>().~T();
7084 new (&Dest.deref<T>()) T();
7085 });
7086 return;
7087 }
7088
7089 if (Desc->isRecord()) {
7090 const Record *R = Desc->ElemRecord;
7091 for (const Record::Field &F : R->fields()) {
7092 PtrView FieldPtr = Dest.atField(Offset: F.Offset);
7093 zeroAll(Dest: FieldPtr);
7094 }
7095 return;
7096 }
7097
7098 if (Desc->isPrimitiveArray()) {
7099 for (unsigned I = 0, N = Desc->getNumElems(); I != N; ++I) {
7100 TYPE_SWITCH(Desc->getPrimType(), {
7101 Dest.elem<T>(I).~T();
7102 new (&Dest.elem<T>(I)) T();
7103 });
7104 }
7105 return;
7106 }
7107
7108 if (Desc->isCompositeArray()) {
7109 for (unsigned I = 0, N = Desc->getNumElems(); I != N; ++I) {
7110 PtrView ElemPtr = Dest.atIndex(Idx: I).narrow();
7111 zeroAll(Dest: ElemPtr);
7112 }
7113 return;
7114 }
7115}
7116
7117static bool copyComposite(InterpState &S, CodePtr OpPC, PtrView Src,
7118 PtrView Dest, bool Activate, bool Diagnose);
7119static bool copyRecord(InterpState &S, CodePtr OpPC, PtrView Src, PtrView Dest,
7120 bool Activate = false, bool Diagnose = true) {
7121 [[maybe_unused]] const Descriptor *SrcDesc = Src.getFieldDesc();
7122 const Descriptor *DestDesc = Dest.getFieldDesc();
7123
7124 auto copyField = [&](const Record::Field &F, bool Activate) -> bool {
7125 PtrView DestField = Dest.atField(Offset: F.Offset);
7126 PtrView SrcField = Src.atField(Offset: F.Offset);
7127
7128 if (OptPrimType FT = F.T) {
7129 if (!SrcField.isInitialized()) {
7130 if (Diagnose)
7131 return diagnoseUninitialized(S, OpPC, Extern: false, B: SrcField.block(),
7132 LT: SrcField.getLifetime(), AK: AK_Read);
7133 // Just skip.
7134 return true;
7135 }
7136
7137 TYPE_SWITCH(*FT, DestField.deref<T>() = SrcField.deref<T>(););
7138 DestField.initialize();
7139 if (Activate)
7140 DestField.activate();
7141 return true;
7142 }
7143
7144 return copyComposite(S, OpPC, Src: SrcField, Dest: DestField, Activate, Diagnose);
7145 };
7146
7147 assert(SrcDesc->isRecord());
7148 assert(SrcDesc->ElemRecord == DestDesc->ElemRecord);
7149 const Record *R = DestDesc->ElemRecord;
7150 for (const Record::Field &F : R->fields()) {
7151 PtrView FP = Src.atField(Offset: F.Offset);
7152
7153 if (!CheckMutable(S, OpPC, Ptr: FP))
7154 return false;
7155
7156 if (R->isUnion()) {
7157 // For unions, only copy the active field. Zero all others.
7158 if (FP.isActive()) {
7159 if (!copyField(F, /*Activate=*/true))
7160 return false;
7161 } else {
7162 PtrView DestField = Dest.atField(Offset: F.Offset);
7163 zeroAll(Dest: DestField);
7164 }
7165 } else {
7166 if (!copyField(F, Activate))
7167 return false;
7168 }
7169 }
7170
7171 for (const Record::Base &B : R->bases()) {
7172 PtrView DestBase = Dest.atField(Offset: B.Offset);
7173 if (!copyRecord(S, OpPC, Src: Src.atField(Offset: B.Offset), Dest: DestBase, Activate,
7174 Diagnose))
7175 return false;
7176 }
7177
7178 Dest.initialize();
7179 if (Activate)
7180 Dest.activate();
7181 return true;
7182}
7183
7184static bool copyComposite(InterpState &S, CodePtr OpPC, PtrView Src,
7185 PtrView Dest, bool Activate = false,
7186 bool Diagnose = false) {
7187 assert(Src.isLive() && Dest.isLive());
7188
7189 [[maybe_unused]] const Descriptor *SrcDesc = Src.getFieldDesc();
7190 const Descriptor *DestDesc = Dest.getFieldDesc();
7191
7192 assert(!DestDesc->isPrimitive() && !SrcDesc->isPrimitive());
7193
7194 if (DestDesc->isPrimitiveArray()) {
7195 if (!SrcDesc->isPrimitiveArray())
7196 return false;
7197 // For floating types, check the actual QualType so we don't accidentally
7198 // mix up semantics.
7199 if (SrcDesc->getPrimType() == PT_Float) {
7200 if (!S.getASTContext().hasSimilarType(T1: SrcDesc->getElemQualType(),
7201 T2: DestDesc->getElemQualType()))
7202 return false;
7203 }
7204
7205 assert(SrcDesc->isPrimitiveArray());
7206 assert(SrcDesc->getNumElems() == DestDesc->getNumElems());
7207 assert(SrcDesc->getPrimType() == DestDesc->getPrimType());
7208 PrimType ET = DestDesc->getPrimType();
7209 for (unsigned I = 0, N = DestDesc->getNumElems(); I != N; ++I) {
7210 PtrView DestElem = Dest.atIndex(Idx: I);
7211 TYPE_SWITCH(ET, { DestElem.deref<T>() = Src.elem<T>(I); });
7212 DestElem.initializeElement(Index: I);
7213 }
7214 return true;
7215 }
7216
7217 if (DestDesc->isCompositeArray()) {
7218 if (!SrcDesc->isCompositeArray())
7219 return false;
7220 assert(SrcDesc->isCompositeArray());
7221 assert(SrcDesc->getNumElems() == DestDesc->getNumElems());
7222 for (unsigned I = 0, N = DestDesc->getNumElems(); I != N; ++I) {
7223 PtrView SrcElem = Src.atIndex(Idx: I).narrow();
7224 PtrView DestElem = Dest.atIndex(Idx: I).narrow();
7225 if (!copyComposite(S, OpPC, Src: SrcElem, Dest: DestElem, Activate))
7226 return false;
7227 }
7228 return true;
7229 }
7230
7231 if (DestDesc->isRecord()) {
7232 if (!SrcDesc->isRecord())
7233 return false;
7234 return copyRecord(S, OpPC, Src, Dest, Activate, Diagnose);
7235 }
7236 return Invalid(S, OpPC);
7237}
7238
7239bool DoMemcpy(InterpState &S, CodePtr OpPC, const Pointer &Src, Pointer &Dest,
7240 bool Activate, bool Diagnose) {
7241 if (!Src.isBlockPointer() || Src.getFieldDesc()->isPrimitive())
7242 return false;
7243 if (!Dest.isBlockPointer() || Dest.getFieldDesc()->isPrimitive())
7244 return false;
7245
7246 return copyComposite(S, OpPC, Src: Src.view(), Dest: Dest.view(), Activate, Diagnose);
7247}
7248
7249} // namespace interp
7250} // namespace clang
7251