1//===--- InterpBuiltin.cpp - Interpreter for the constexpr VM ---*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8#include "../ExprConstShared.h"
9#include "Boolean.h"
10#include "Char.h"
11#include "EvalEmitter.h"
12#include "InterpBuiltinBitCast.h"
13#include "InterpHelpers.h"
14#include "PrimType.h"
15#include "Program.h"
16#include "clang/AST/InferAlloc.h"
17#include "clang/AST/OSLog.h"
18#include "clang/AST/RecordLayout.h"
19#include "clang/Basic/Builtins.h"
20#include "clang/Basic/TargetBuiltins.h"
21#include "clang/Basic/TargetInfo.h"
22#include "llvm/ADT/StringExtras.h"
23#include "llvm/Support/AllocToken.h"
24#include "llvm/Support/ErrorHandling.h"
25#include "llvm/Support/SipHash.h"
26
27namespace clang {
28namespace interp {
29
30[[maybe_unused]] static bool isNoopBuiltin(unsigned ID) {
31 switch (ID) {
32 case Builtin::BIas_const:
33 case Builtin::BIforward:
34 case Builtin::BIforward_like:
35 case Builtin::BImove:
36 case Builtin::BImove_if_noexcept:
37 case Builtin::BIaddressof:
38 case Builtin::BI__addressof:
39 case Builtin::BI__builtin_addressof:
40 case Builtin::BI__builtin_launder:
41 return true;
42 default:
43 return false;
44 }
45 return false;
46}
47
48static void discard(InterpStack &Stk, PrimType T) {
49 TYPE_SWITCH(T, { Stk.discard<T>(); });
50}
51
52static bool popToUInt64(const InterpState &S, const Expr *E, uint64_t &Out) {
53 INT_TYPE_SWITCH(*S.getContext().classify(E->getType()), {
54 const auto &Val = S.Stk.pop<T>();
55 if (!Val.isNumber())
56 return false;
57 Out = static_cast<uint64_t>(Val);
58 return true;
59 });
60}
61
62static bool popToAPSInt(InterpStack &Stk, PrimType T, APSInt &Out) {
63 INT_TYPE_SWITCH(T, {
64 const auto &Val = Stk.pop<T>();
65 if (!Val.isNumber())
66 return false;
67 Out = Val.toAPSInt();
68 return true;
69 });
70}
71
72static bool popToAPSInt(InterpState &S, const Expr *E, APSInt &Out) {
73 return popToAPSInt(Stk&: S.Stk, T: *S.getContext().classify(T: E->getType()), Out);
74}
75static bool popToAPSInt(InterpState &S, QualType T, APSInt &Out) {
76 return popToAPSInt(Stk&: S.Stk, T: *S.getContext().classify(T), Out);
77}
78
79/// Check for common reasons a pointer can't be read from, which
80/// are usually not diagnosed in a builtin function.
81static bool isReadable(const Pointer &P) {
82 if (P.isDummy())
83 return false;
84 if (!P.isBlockPointer())
85 return false;
86 if (!P.isLive())
87 return false;
88 if (P.isOnePastEnd())
89 return false;
90 return true;
91}
92
93/// Pushes \p Val on the stack as the type given by \p QT.
94static void pushInteger(InterpState &S, const APSInt &Val, QualType QT) {
95 assert(QT->isSignedIntegerOrEnumerationType() ||
96 QT->isUnsignedIntegerOrEnumerationType());
97 OptPrimType T = *S.getContext().classify(T: QT);
98 assert(T);
99
100 if (T == PT_IntAPS) {
101 unsigned BitWidth = S.getASTContext().getIntWidth(T: QT);
102 auto Result = S.allocAP<IntegralAP<true>>(BitWidth);
103 Result.copy(V: Val.extOrTrunc(width: BitWidth));
104 S.Stk.push<IntegralAP<true>>(Args&: Result);
105 return;
106 }
107
108 if (T == PT_IntAP) {
109 unsigned BitWidth = S.getASTContext().getIntWidth(T: QT);
110 auto Result = S.allocAP<IntegralAP<false>>(BitWidth);
111 Result.copy(V: Val.extOrTrunc(width: BitWidth));
112 S.Stk.push<IntegralAP<false>>(Args&: Result);
113 return;
114 }
115
116 if (isSignedType(T: *T)) {
117 int64_t V = Val.getSExtValue();
118 INT_TYPE_SWITCH(*T, { S.Stk.push<T>(T::from(V)); });
119 } else {
120 assert(QT->isUnsignedIntegerOrEnumerationType());
121 uint64_t V = Val.getZExtValue();
122 INT_TYPE_SWITCH(*T, { S.Stk.push<T>(T::from(V)); });
123 }
124}
125
126template <typename T>
127static void pushInteger(InterpState &S, T Val, QualType QT) {
128 if constexpr (std::is_same_v<T, APInt>)
129 pushInteger(S, Val: APSInt(Val, !std::is_signed_v<T>), QT);
130 else if constexpr (std::is_same_v<T, APSInt>)
131 pushInteger(S, Val, QT);
132 else
133 pushInteger(S,
134 Val: APSInt(APInt(sizeof(T) * 8, static_cast<uint64_t>(Val),
135 std::is_signed_v<T>),
136 !std::is_signed_v<T>),
137 QT);
138}
139
140static void assignIntegral(InterpState &S, const Pointer &Dest, PrimType ValueT,
141 const APSInt &Value) {
142
143 if (ValueT == PT_IntAPS) {
144 Dest.deref<IntegralAP<true>>() =
145 S.allocAP<IntegralAP<true>>(BitWidth: Value.getBitWidth());
146 Dest.deref<IntegralAP<true>>().copy(V: Value);
147 } else if (ValueT == PT_IntAP) {
148 Dest.deref<IntegralAP<false>>() =
149 S.allocAP<IntegralAP<false>>(BitWidth: Value.getBitWidth());
150 Dest.deref<IntegralAP<false>>().copy(V: Value);
151 } else if (ValueT == PT_Bool) {
152 Dest.deref<Boolean>() = Boolean::from(Value: !Value.isZero());
153 } else {
154 INT_TYPE_SWITCH_NO_BOOL(
155 ValueT, { Dest.deref<T>() = T::from(static_cast<T>(Value)); });
156 }
157}
158
159static QualType getElemType(const Pointer &P) {
160 const Descriptor *Desc = P.getFieldDesc();
161 QualType T = Desc->getType();
162 if (Desc->isPrimitive())
163 return T;
164 if (T->isPointerType())
165 return T->castAs<PointerType>()->getPointeeType();
166 if (Desc->isArray())
167 return Desc->getElemQualType();
168 if (const auto *AT = T->getAsArrayTypeUnsafe())
169 return AT->getElementType();
170 return T;
171}
172
173static void diagnoseNonConstexprBuiltin(InterpState &S, CodePtr OpPC,
174 unsigned ID) {
175 if (!S.diagnosing())
176 return;
177
178 auto Loc = S.Current->getSource(PC: OpPC);
179 if (S.getLangOpts().CPlusPlus11)
180 S.CCEDiag(SI: Loc, DiagId: diag::note_constexpr_invalid_function)
181 << /*isConstexpr=*/0 << /*isConstructor=*/0
182 << S.getASTContext().BuiltinInfo.getQuotedName(ID);
183 else
184 S.CCEDiag(SI: Loc, DiagId: diag::note_invalid_subexpr_in_const_expr);
185}
186
187static llvm::APSInt convertBoolVectorToInt(const Pointer &Val) {
188 assert(Val.getFieldDesc()->isPrimitiveArray() &&
189 Val.getFieldDesc()->getElemQualType()->isBooleanType() &&
190 "Not a boolean vector");
191 unsigned NumElems = Val.getNumElems();
192
193 // Each element is one bit, so create an integer with NumElts bits.
194 llvm::APSInt Result(NumElems, 0);
195 for (unsigned I = 0; I != NumElems; ++I) {
196 if (Val.elem<bool>(I))
197 Result.setBit(I);
198 }
199
200 return Result;
201}
202
203// Strict double -> float conversion used for X86 PD2PS/cvtsd2ss intrinsics.
204// Reject NaN/Inf/Subnormal inputs and any lossy/inexact conversions.
205static bool convertDoubleToFloatStrict(APFloat Src, Floating &Dst,
206 InterpState &S, const Expr *DiagExpr) {
207 if (Src.isInfinity()) {
208 if (S.diagnosing())
209 S.CCEDiag(E: DiagExpr, DiagId: diag::note_constexpr_float_arithmetic) << 0;
210 return false;
211 }
212 if (Src.isNaN()) {
213 if (S.diagnosing())
214 S.CCEDiag(E: DiagExpr, DiagId: diag::note_constexpr_float_arithmetic) << 1;
215 return false;
216 }
217 APFloat Val = Src;
218 bool LosesInfo = false;
219 APFloat::opStatus Status = Val.convert(
220 ToSemantics: APFloat::IEEEsingle(), RM: APFloat::rmNearestTiesToEven, losesInfo: &LosesInfo);
221 if (LosesInfo || Val.isDenormal()) {
222 if (S.diagnosing())
223 S.CCEDiag(E: DiagExpr, DiagId: diag::note_constexpr_float_arithmetic_strict);
224 return false;
225 }
226 if (Status != APFloat::opOK) {
227 if (S.diagnosing())
228 S.CCEDiag(E: DiagExpr, DiagId: diag::note_invalid_subexpr_in_const_expr);
229 return false;
230 }
231 Dst.copy(F: Val);
232 return true;
233}
234
235static bool interp__builtin_is_constant_evaluated(InterpState &S, CodePtr OpPC,
236 const InterpFrame *Frame,
237 const CallExpr *Call) {
238 unsigned Depth = S.Current->getDepth();
239 auto isStdCall = [](const FunctionDecl *F) -> bool {
240 return F && F->isInStdNamespace() && F->getIdentifier() &&
241 F->getIdentifier()->isStr(Str: "is_constant_evaluated");
242 };
243 const InterpFrame *Caller = Frame->Caller;
244 // The current frame is the one for __builtin_is_constant_evaluated.
245 // The one above that, potentially the one for std::is_constant_evaluated().
246 if (S.inConstantContext() && !S.checkingPotentialConstantExpression() &&
247 S.getEvalStatus().Diag &&
248 (Depth == 0 || (Depth == 1 && isStdCall(Frame->getCallee())))) {
249 if (Caller && isStdCall(Frame->getCallee())) {
250 const Expr *E = Caller->getExpr(PC: Caller->getRetPC());
251 S.report(Loc: E->getExprLoc(),
252 DiagId: diag::warn_is_constant_evaluated_always_true_constexpr)
253 << "std::is_constant_evaluated" << E->getSourceRange();
254 } else {
255 S.report(Loc: Call->getExprLoc(),
256 DiagId: diag::warn_is_constant_evaluated_always_true_constexpr)
257 << "__builtin_is_constant_evaluated" << Call->getSourceRange();
258 }
259 }
260
261 S.Stk.push<Boolean>(Args: Boolean::from(Value: S.inConstantContext()));
262 return true;
263}
264
265// __builtin_assume
266// __assume (MS extension)
267static bool interp__builtin_assume(InterpState &S, CodePtr OpPC,
268 const InterpFrame *Frame,
269 const CallExpr *Call) {
270 // Nothing to be done here since the argument is NOT evaluated.
271 assert(Call->getNumArgs() == 1);
272 return true;
273}
274
275static bool interp__builtin_strcmp(InterpState &S, CodePtr OpPC,
276 const InterpFrame *Frame,
277 const CallExpr *Call, unsigned ID) {
278 uint64_t Limit = ~static_cast<uint64_t>(0);
279 if (ID == Builtin::BIstrncmp || ID == Builtin::BI__builtin_strncmp ||
280 ID == Builtin::BIwcsncmp || ID == Builtin::BI__builtin_wcsncmp) {
281 if (!popToUInt64(S, E: Call->getArg(Arg: 2), Out&: Limit))
282 return false;
283 }
284
285 const Pointer &B = S.Stk.pop<Pointer>();
286 const Pointer &A = S.Stk.pop<Pointer>();
287 if (ID == Builtin::BIstrcmp || ID == Builtin::BIstrncmp ||
288 ID == Builtin::BIwcscmp || ID == Builtin::BIwcsncmp)
289 diagnoseNonConstexprBuiltin(S, OpPC, ID);
290
291 if (Limit == 0) {
292 pushInteger(S, Val: 0, QT: Call->getType());
293 return true;
294 }
295
296 if (!CheckLive(S, OpPC, Ptr: A, AK: AK_Read) || !CheckLive(S, OpPC, Ptr: B, AK: AK_Read))
297 return false;
298
299 if (A.isDummy() || B.isDummy())
300 return false;
301 if (!A.isBlockPointer() || !B.isBlockPointer())
302 return false;
303 if (!A.getFieldDesc()->isPrimitiveArray() ||
304 !B.getFieldDesc()->isPrimitiveArray())
305 return false;
306
307 bool IsWide = ID == Builtin::BIwcscmp || ID == Builtin::BIwcsncmp ||
308 ID == Builtin::BI__builtin_wcscmp ||
309 ID == Builtin::BI__builtin_wcsncmp;
310 assert(A.getFieldDesc()->isPrimitiveArray());
311 assert(B.getFieldDesc()->isPrimitiveArray());
312
313 // Different element types shouldn't happen, but with casts they can.
314 if (!S.getASTContext().hasSameUnqualifiedType(T1: getElemType(P: A), T2: getElemType(P: B)))
315 return false;
316
317 PrimType ElemT = *S.getContext().classify(T: getElemType(P: A));
318
319 auto returnResult = [&](int V) -> bool {
320 pushInteger(S, Val: V, QT: Call->getType());
321 return true;
322 };
323
324 unsigned IndexA = A.getIndex();
325 unsigned IndexB = B.getIndex();
326 uint64_t Steps = 0;
327 for (;; ++IndexA, ++IndexB, ++Steps) {
328
329 if (Steps >= Limit)
330 break;
331 PtrView PA = A.view().atIndex(Idx: IndexA);
332 PtrView PB = B.view().atIndex(Idx: IndexB);
333 if (!CheckRange(S, OpPC, Ptr: PA, AK: AK_Read) ||
334 !CheckRange(S, OpPC, Ptr: PB, AK: AK_Read)) {
335 return false;
336 }
337
338 if (IsWide) {
339 INT_TYPE_SWITCH(ElemT, {
340 T CA = PA.deref<T>();
341 T CB = PB.deref<T>();
342 if (CA > CB)
343 return returnResult(1);
344 if (CA < CB)
345 return returnResult(-1);
346 if (CA.isZero() || CB.isZero())
347 return returnResult(0);
348 });
349 continue;
350 }
351
352 uint8_t CA = PA.deref<uint8_t>();
353 uint8_t CB = PB.deref<uint8_t>();
354
355 if (CA > CB)
356 return returnResult(1);
357 if (CA < CB)
358 return returnResult(-1);
359 if (CA == 0 || CB == 0)
360 return returnResult(0);
361 }
362
363 return returnResult(0);
364}
365
366static bool interp__builtin_strlen(InterpState &S, CodePtr OpPC,
367 const InterpFrame *Frame,
368 const CallExpr *Call, unsigned ID) {
369 const Pointer &StrPtr = S.Stk.pop<Pointer>().expand();
370
371 if (ID == Builtin::BIstrlen || ID == Builtin::BIwcslen)
372 diagnoseNonConstexprBuiltin(S, OpPC, ID);
373
374 if (StrPtr.isConstexprUnknown())
375 return false;
376
377 if (!CheckArray(S, OpPC, Ptr: StrPtr))
378 return false;
379
380 if (!CheckLive(S, OpPC, Ptr: StrPtr, AK: AK_Read))
381 return false;
382
383 if (!StrPtr.isBlockPointer())
384 return false;
385
386 if (!CheckDummy(S, OpPC, B: StrPtr.block(), AK: AK_Read))
387 return false;
388
389 if (!StrPtr.getFieldDesc()->isPrimitiveArray())
390 return false;
391
392 assert(StrPtr.getFieldDesc()->isPrimitiveArray());
393 PrimType ElemT = StrPtr.getFieldDesc()->getPrimType();
394 unsigned ElemSize = StrPtr.getFieldDesc()->getElemDataSize();
395 if (ElemSize != 1 && ElemSize != 2 && ElemSize != 4)
396 return Invalid(S, OpPC);
397
398 if (ID == Builtin::BI__builtin_wcslen || ID == Builtin::BIwcslen) {
399 const ASTContext &AC = S.getASTContext();
400 unsigned WCharSize = AC.getTypeSizeInChars(T: AC.getWCharType()).getQuantity();
401 if (StrPtr.getFieldDesc()->getElemDataSize() != WCharSize)
402 return false;
403 }
404
405 size_t Len = 0;
406 for (size_t I = StrPtr.getIndex();; ++I, ++Len) {
407 PtrView ElemPtr = StrPtr.view().atIndex(Idx: I);
408
409 if (!CheckRange(S, OpPC, Ptr: ElemPtr, AK: AK_Read))
410 return false;
411
412 uint32_t Val;
413 FIXED_SIZE_INT_TYPE_SWITCH(
414 ElemT, { Val = static_cast<uint32_t>(ElemPtr.deref<T>()); });
415 if (Val == 0)
416 break;
417 }
418
419 pushInteger(S, Val: Len, QT: Call->getType());
420
421 return true;
422}
423
424static bool interp__builtin_nan(InterpState &S, CodePtr OpPC,
425 const InterpFrame *Frame, const CallExpr *Call,
426 bool Signaling) {
427 const Pointer &Arg = S.Stk.pop<Pointer>();
428
429 if (!CheckLoad(S, OpPC, Ptr: Arg))
430 return false;
431
432 if (!Arg.getFieldDesc()->isPrimitiveArray())
433 return Invalid(S, OpPC);
434
435 // Convert the given string to an integer using StringRef's API.
436 llvm::APInt Fill;
437 std::string Str;
438 unsigned ArgLength = Arg.getNumElems();
439 bool FoundZero = false;
440 for (unsigned I = 0; I != ArgLength; ++I) {
441 if (!Arg.isElementInitialized(Index: I))
442 return false;
443
444 if (Arg.elem<int8_t>(I) == 0) {
445 FoundZero = true;
446 break;
447 }
448 Str += Arg.elem<char>(I);
449 }
450
451 // If we didn't find a NUL byte, diagnose as a one-past-the-end read.
452 if (!FoundZero)
453 return CheckRange(S, OpPC, Ptr: Arg.atIndex(Idx: ArgLength), AK: AK_Read);
454
455 // Treat empty strings as if they were zero.
456 if (Str.empty())
457 Fill = llvm::APInt(32, 0);
458 else if (StringRef(Str).getAsInteger(Radix: 0, Result&: Fill))
459 return false;
460
461 const llvm::fltSemantics &TargetSemantics =
462 S.getASTContext().getFloatTypeSemantics(
463 T: Call->getDirectCallee()->getReturnType());
464
465 Floating Result = S.allocFloat(Sem: TargetSemantics);
466 if (S.getASTContext().getTargetInfo().isNan2008()) {
467 if (Signaling)
468 Result.copy(
469 F: llvm::APFloat::getSNaN(Sem: TargetSemantics, /*Negative=*/false, payload: &Fill));
470 else
471 Result.copy(
472 F: llvm::APFloat::getQNaN(Sem: TargetSemantics, /*Negative=*/false, payload: &Fill));
473 } else {
474 // Prior to IEEE 754-2008, architectures were allowed to choose whether
475 // the first bit of their significand was set for qNaN or sNaN. MIPS chose
476 // a different encoding to what became a standard in 2008, and for pre-
477 // 2008 revisions, MIPS interpreted sNaN-2008 as qNan and qNaN-2008 as
478 // sNaN. This is now known as "legacy NaN" encoding.
479 if (Signaling)
480 Result.copy(
481 F: llvm::APFloat::getQNaN(Sem: TargetSemantics, /*Negative=*/false, payload: &Fill));
482 else
483 Result.copy(
484 F: llvm::APFloat::getSNaN(Sem: TargetSemantics, /*Negative=*/false, payload: &Fill));
485 }
486
487 S.Stk.push<Floating>(Args&: Result);
488 return true;
489}
490
491static bool interp__builtin_inf(InterpState &S, CodePtr OpPC,
492 const InterpFrame *Frame,
493 const CallExpr *Call) {
494 const llvm::fltSemantics &TargetSemantics =
495 S.getASTContext().getFloatTypeSemantics(
496 T: Call->getDirectCallee()->getReturnType());
497
498 Floating Result = S.allocFloat(Sem: TargetSemantics);
499 Result.copy(F: APFloat::getInf(Sem: TargetSemantics));
500 S.Stk.push<Floating>(Args&: Result);
501 return true;
502}
503
504static bool interp__builtin_copysign(InterpState &S, CodePtr OpPC,
505 const InterpFrame *Frame) {
506 const Floating &Arg2 = S.Stk.pop<Floating>();
507 const Floating &Arg1 = S.Stk.pop<Floating>();
508 Floating Result = S.allocFloat(Sem: Arg1.getSemantics());
509
510 APFloat Copy = Arg1.getAPFloat();
511 Copy.copySign(RHS: Arg2.getAPFloat());
512 Result.copy(F: Copy);
513 S.Stk.push<Floating>(Args&: Result);
514
515 return true;
516}
517
518static bool interp__builtin_fmin(InterpState &S, CodePtr OpPC,
519 const InterpFrame *Frame, bool IsNumBuiltin) {
520 const Floating &RHS = S.Stk.pop<Floating>();
521 const Floating &LHS = S.Stk.pop<Floating>();
522 Floating Result = S.allocFloat(Sem: LHS.getSemantics());
523
524 if (IsNumBuiltin)
525 Result.copy(F: llvm::minimumnum(A: LHS.getAPFloat(), B: RHS.getAPFloat()));
526 else
527 Result.copy(F: minnum(A: LHS.getAPFloat(), B: RHS.getAPFloat()));
528 S.Stk.push<Floating>(Args&: Result);
529 return true;
530}
531
532static bool interp__builtin_fmax(InterpState &S, CodePtr OpPC,
533 const InterpFrame *Frame, bool IsNumBuiltin) {
534 const Floating &RHS = S.Stk.pop<Floating>();
535 const Floating &LHS = S.Stk.pop<Floating>();
536 Floating Result = S.allocFloat(Sem: LHS.getSemantics());
537
538 if (IsNumBuiltin)
539 Result.copy(F: llvm::maximumnum(A: LHS.getAPFloat(), B: RHS.getAPFloat()));
540 else
541 Result.copy(F: maxnum(A: LHS.getAPFloat(), B: RHS.getAPFloat()));
542 S.Stk.push<Floating>(Args&: Result);
543 return true;
544}
545
546/// Defined as __builtin_isnan(...), to accommodate the fact that it can
547/// take a float, double, long double, etc.
548/// But for us, that's all a Floating anyway.
549static bool interp__builtin_isnan(InterpState &S, CodePtr OpPC,
550 const InterpFrame *Frame,
551 const CallExpr *Call) {
552 const Floating &Arg = S.Stk.pop<Floating>();
553
554 pushInteger(S, Val: Arg.isNan(), QT: Call->getType());
555 return true;
556}
557
558static bool interp__builtin_issignaling(InterpState &S, CodePtr OpPC,
559 const InterpFrame *Frame,
560 const CallExpr *Call) {
561 const Floating &Arg = S.Stk.pop<Floating>();
562
563 pushInteger(S, Val: Arg.isSignaling(), QT: Call->getType());
564 return true;
565}
566
567static bool interp__builtin_isinf(InterpState &S, CodePtr OpPC,
568 const InterpFrame *Frame, bool CheckSign,
569 const CallExpr *Call) {
570 const Floating &Arg = S.Stk.pop<Floating>();
571 APFloat F = Arg.getAPFloat();
572 bool IsInf = F.isInfinity();
573
574 if (CheckSign)
575 pushInteger(S, Val: IsInf ? (F.isNegative() ? -1 : 1) : 0, QT: Call->getType());
576 else
577 pushInteger(S, Val: IsInf, QT: Call->getType());
578 return true;
579}
580
581static bool interp__builtin_isfinite(InterpState &S, CodePtr OpPC,
582 const InterpFrame *Frame,
583 const CallExpr *Call) {
584 const Floating &Arg = S.Stk.pop<Floating>();
585
586 pushInteger(S, Val: Arg.isFinite(), QT: Call->getType());
587 return true;
588}
589
590static bool interp__builtin_isnormal(InterpState &S, CodePtr OpPC,
591 const InterpFrame *Frame,
592 const CallExpr *Call) {
593 const Floating &Arg = S.Stk.pop<Floating>();
594
595 pushInteger(S, Val: Arg.isNormal(), QT: Call->getType());
596 return true;
597}
598
599static bool interp__builtin_issubnormal(InterpState &S, CodePtr OpPC,
600 const InterpFrame *Frame,
601 const CallExpr *Call) {
602 const Floating &Arg = S.Stk.pop<Floating>();
603
604 pushInteger(S, Val: Arg.isDenormal(), QT: Call->getType());
605 return true;
606}
607
608static bool interp__builtin_iszero(InterpState &S, CodePtr OpPC,
609 const InterpFrame *Frame,
610 const CallExpr *Call) {
611 const Floating &Arg = S.Stk.pop<Floating>();
612
613 pushInteger(S, Val: Arg.isZero(), QT: Call->getType());
614 return true;
615}
616
617static bool interp__builtin_signbit(InterpState &S, CodePtr OpPC,
618 const InterpFrame *Frame,
619 const CallExpr *Call) {
620 const Floating &Arg = S.Stk.pop<Floating>();
621
622 pushInteger(S, Val: Arg.isNegative(), QT: Call->getType());
623 return true;
624}
625
626static bool interp_floating_comparison(InterpState &S, CodePtr OpPC,
627 const CallExpr *Call, unsigned ID) {
628 const Floating &RHS = S.Stk.pop<Floating>();
629 const Floating &LHS = S.Stk.pop<Floating>();
630
631 pushInteger(
632 S,
633 Val: [&] {
634 switch (ID) {
635 case Builtin::BI__builtin_isgreater:
636 return LHS > RHS;
637 case Builtin::BI__builtin_isgreaterequal:
638 return LHS >= RHS;
639 case Builtin::BI__builtin_isless:
640 return LHS < RHS;
641 case Builtin::BI__builtin_islessequal:
642 return LHS <= RHS;
643 case Builtin::BI__builtin_islessgreater: {
644 ComparisonCategoryResult Cmp = LHS.compare(RHS);
645 return Cmp == ComparisonCategoryResult::Less ||
646 Cmp == ComparisonCategoryResult::Greater;
647 }
648 case Builtin::BI__builtin_isunordered:
649 return LHS.compare(RHS) == ComparisonCategoryResult::Unordered;
650 default:
651 llvm_unreachable("Unexpected builtin ID: Should be a floating point "
652 "comparison function");
653 }
654 }(),
655 QT: Call->getType());
656 return true;
657}
658
659/// First parameter to __builtin_isfpclass is the floating value, the
660/// second one is an integral value.
661static bool interp__builtin_isfpclass(InterpState &S, CodePtr OpPC,
662 const InterpFrame *Frame,
663 const CallExpr *Call) {
664 APSInt FPClassArg;
665 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: FPClassArg))
666 return false;
667 const Floating &F = S.Stk.pop<Floating>();
668
669 int32_t Result = static_cast<int32_t>(
670 (F.classify() & std::move(FPClassArg)).getZExtValue());
671 pushInteger(S, Val: Result, QT: Call->getType());
672
673 return true;
674}
675
676/// Five int values followed by one floating value.
677/// __builtin_fpclassify(int, int, int, int, int, float)
678static bool interp__builtin_fpclassify(InterpState &S, CodePtr OpPC,
679 const InterpFrame *Frame,
680 const CallExpr *Call) {
681 const Floating &Val = S.Stk.pop<Floating>();
682
683 PrimType IntT = *S.getContext().classify(E: Call->getArg(Arg: 0));
684 APSInt Values[5];
685 for (unsigned I = 0; I != 5; ++I) {
686 if (!popToAPSInt(Stk&: S.Stk, T: IntT, Out&: Values[4 - I]))
687 return false;
688 }
689
690 unsigned Index;
691 switch (Val.getCategory()) {
692 case APFloat::fcNaN:
693 Index = 0;
694 break;
695 case APFloat::fcInfinity:
696 Index = 1;
697 break;
698 case APFloat::fcNormal:
699 Index = Val.isDenormal() ? 3 : 2;
700 break;
701 case APFloat::fcZero:
702 Index = 4;
703 break;
704 }
705
706 // The last argument is first on the stack.
707 assert(Index <= 4);
708
709 pushInteger(S, Val: Values[Index], QT: Call->getType());
710 return true;
711}
712
713static inline Floating abs(InterpState &S, const Floating &In) {
714 if (!In.isNegative())
715 return In;
716
717 Floating Output = S.allocFloat(Sem: In.getSemantics());
718 APFloat New = In.getAPFloat();
719 New.changeSign();
720 Output.copy(F: New);
721 return Output;
722}
723
724// The C standard says "fabs raises no floating-point exceptions,
725// even if x is a signaling NaN. The returned value is independent of
726// the current rounding direction mode." Therefore constant folding can
727// proceed without regard to the floating point settings.
728// Reference, WG14 N2478 F.10.4.3
729static bool interp__builtin_fabs(InterpState &S, CodePtr OpPC,
730 const InterpFrame *Frame) {
731 const Floating &Val = S.Stk.pop<Floating>();
732 S.Stk.push<Floating>(Args: abs(S, In: Val));
733 return true;
734}
735
736static bool interp__builtin_abs(InterpState &S, CodePtr OpPC,
737 const InterpFrame *Frame,
738 const CallExpr *Call) {
739 APSInt Val;
740 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: Val))
741 return false;
742 if (Val ==
743 APSInt(APInt::getSignedMinValue(numBits: Val.getBitWidth()), /*IsUnsigned=*/false))
744 return false;
745 if (Val.isNegative())
746 Val.negate();
747 pushInteger(S, Val, QT: Call->getType());
748 return true;
749}
750
751static bool interp__builtin_popcount(InterpState &S, CodePtr OpPC,
752 const InterpFrame *Frame,
753 const CallExpr *Call) {
754 APSInt Val;
755 if (Call->getArg(Arg: 0)->getType()->isExtVectorBoolType()) {
756 const Pointer &Arg = S.Stk.pop<Pointer>();
757 Val = convertBoolVectorToInt(Val: Arg);
758 } else {
759 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: Val))
760 return false;
761 }
762 pushInteger(S, Val: Val.popcount(), QT: Call->getType());
763 return true;
764}
765
766static bool interp__builtin_ia32_crc32(InterpState &S, CodePtr OpPC,
767 const InterpFrame *Frame,
768 const CallExpr *Call,
769 unsigned DataBytes) {
770 uint64_t DataVal;
771 if (!popToUInt64(S, E: Call->getArg(Arg: 1), Out&: DataVal))
772 return false;
773 uint64_t CRCVal;
774 if (!popToUInt64(S, E: Call->getArg(Arg: 0), Out&: CRCVal))
775 return false;
776
777 // CRC32C polynomial (iSCSI polynomial, bit-reversed)
778 static const uint32_t CRC32C_POLY = 0x82F63B78;
779
780 // Process each byte
781 uint32_t Result = static_cast<uint32_t>(CRCVal);
782 for (unsigned I = 0; I != DataBytes; ++I) {
783 uint8_t Byte = static_cast<uint8_t>((DataVal >> (I * 8)) & 0xFF);
784 Result ^= Byte;
785 for (int J = 0; J != 8; ++J) {
786 Result = (Result >> 1) ^ ((Result & 1) ? CRC32C_POLY : 0);
787 }
788 }
789
790 pushInteger(S, Val: Result, QT: Call->getType());
791 return true;
792}
793
794static bool interp__builtin_classify_type(InterpState &S, CodePtr OpPC,
795 const InterpFrame *Frame,
796 const CallExpr *Call) {
797 // This is an unevaluated call, so there are no arguments on the stack.
798 assert(Call->getNumArgs() == 1);
799 const Expr *Arg = Call->getArg(Arg: 0);
800
801 GCCTypeClass ResultClass =
802 EvaluateBuiltinClassifyType(T: Arg->getType(), LangOpts: S.getLangOpts());
803 int32_t ReturnVal = static_cast<int32_t>(ResultClass);
804 pushInteger(S, Val: ReturnVal, QT: Call->getType());
805 return true;
806}
807
808// __builtin_expect(long, long)
809// __builtin_expect_with_probability(long, long, double)
810static bool interp__builtin_expect(InterpState &S, CodePtr OpPC,
811 const InterpFrame *Frame,
812 const CallExpr *Call) {
813 // The return value is simply the value of the first parameter.
814 // We ignore the probability.
815 unsigned NumArgs = Call->getNumArgs();
816 assert(NumArgs == 2 || NumArgs == 3);
817
818 PrimType ArgT = *S.getContext().classify(T: Call->getArg(Arg: 0)->getType());
819 if (NumArgs == 3)
820 S.Stk.discard<Floating>();
821 discard(Stk&: S.Stk, T: ArgT);
822
823 APSInt Val;
824 if (!popToAPSInt(Stk&: S.Stk, T: ArgT, Out&: Val))
825 return false;
826 pushInteger(S, Val, QT: Call->getType());
827 return true;
828}
829
830static bool interp__builtin_addressof(InterpState &S, CodePtr OpPC,
831 const InterpFrame *Frame,
832 const CallExpr *Call) {
833#ifndef NDEBUG
834 assert(Call->getArg(0)->isLValue());
835 PrimType PtrT = S.getContext().classify(Call->getArg(0)).value_or(PT_Ptr);
836 assert(PtrT == PT_Ptr &&
837 "Unsupported pointer type passed to __builtin_addressof()");
838#endif
839 return true;
840}
841
842static bool interp__builtin_move(InterpState &S, CodePtr OpPC,
843 const InterpFrame *Frame,
844 const CallExpr *Call) {
845 return Call->getDirectCallee()->isConstexpr();
846}
847
848static bool interp__builtin_eh_return_data_regno(InterpState &S, CodePtr OpPC,
849 const InterpFrame *Frame,
850 const CallExpr *Call) {
851 APSInt Arg;
852 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: Arg))
853 return false;
854
855 int Result = S.getASTContext().getTargetInfo().getEHDataRegisterNumber(
856 RegNo: Arg.getZExtValue());
857 pushInteger(S, Val: Result, QT: Call->getType());
858 return true;
859}
860
861// Two integral values followed by a pointer (lhs, rhs, resultOut)
862static bool interp__builtin_overflowop(InterpState &S, CodePtr OpPC,
863 const CallExpr *Call,
864 unsigned BuiltinOp) {
865 const Pointer &ResultPtr = S.Stk.pop<Pointer>();
866 if (ResultPtr.isDummy() || !ResultPtr.isBlockPointer())
867 return false;
868
869 PrimType RHST = *S.getContext().classify(T: Call->getArg(Arg: 1)->getType());
870 PrimType LHST = *S.getContext().classify(T: Call->getArg(Arg: 0)->getType());
871 APSInt RHS;
872 if (!popToAPSInt(Stk&: S.Stk, T: RHST, Out&: RHS))
873 return false;
874 APSInt LHS;
875 if (!popToAPSInt(Stk&: S.Stk, T: LHST, Out&: LHS))
876 return false;
877 QualType ResultType = Call->getArg(Arg: 2)->getType()->getPointeeType();
878 PrimType ResultT = *S.getContext().classify(T: ResultType);
879 bool Overflow;
880
881 APSInt Result;
882 if (BuiltinOp == Builtin::BI__builtin_add_overflow ||
883 BuiltinOp == Builtin::BI__builtin_sub_overflow ||
884 BuiltinOp == Builtin::BI__builtin_mul_overflow) {
885 bool IsSigned = LHS.isSigned() || RHS.isSigned() ||
886 ResultType->isSignedIntegerOrEnumerationType();
887 bool AllSigned = LHS.isSigned() && RHS.isSigned() &&
888 ResultType->isSignedIntegerOrEnumerationType();
889 uint64_t LHSSize = LHS.getBitWidth();
890 uint64_t RHSSize = RHS.getBitWidth();
891 uint64_t ResultSize = S.getASTContext().getIntWidth(T: ResultType);
892 uint64_t MaxBits = std::max(a: std::max(a: LHSSize, b: RHSSize), b: ResultSize);
893
894 // Add an additional bit if the signedness isn't uniformly agreed to. We
895 // could do this ONLY if there is a signed and an unsigned that both have
896 // MaxBits, but the code to check that is pretty nasty. The issue will be
897 // caught in the shrink-to-result later anyway.
898 if (IsSigned && !AllSigned)
899 ++MaxBits;
900
901 LHS = APSInt(LHS.extOrTrunc(width: MaxBits), !IsSigned);
902 RHS = APSInt(RHS.extOrTrunc(width: MaxBits), !IsSigned);
903 Result = APSInt(MaxBits, !IsSigned);
904 }
905
906 // Find largest int.
907 switch (BuiltinOp) {
908 default:
909 llvm_unreachable("Invalid value for BuiltinOp");
910 case Builtin::BI__builtin_add_overflow:
911 case Builtin::BI__builtin_sadd_overflow:
912 case Builtin::BI__builtin_saddl_overflow:
913 case Builtin::BI__builtin_saddll_overflow:
914 case Builtin::BI__builtin_uadd_overflow:
915 case Builtin::BI__builtin_uaddl_overflow:
916 case Builtin::BI__builtin_uaddll_overflow:
917 Result = LHS.isSigned() ? LHS.sadd_ov(RHS, Overflow)
918 : LHS.uadd_ov(RHS, Overflow);
919 break;
920 case Builtin::BI__builtin_sub_overflow:
921 case Builtin::BI__builtin_ssub_overflow:
922 case Builtin::BI__builtin_ssubl_overflow:
923 case Builtin::BI__builtin_ssubll_overflow:
924 case Builtin::BI__builtin_usub_overflow:
925 case Builtin::BI__builtin_usubl_overflow:
926 case Builtin::BI__builtin_usubll_overflow:
927 Result = LHS.isSigned() ? LHS.ssub_ov(RHS, Overflow)
928 : LHS.usub_ov(RHS, Overflow);
929 break;
930 case Builtin::BI__builtin_mul_overflow:
931 case Builtin::BI__builtin_smul_overflow:
932 case Builtin::BI__builtin_smull_overflow:
933 case Builtin::BI__builtin_smulll_overflow:
934 case Builtin::BI__builtin_umul_overflow:
935 case Builtin::BI__builtin_umull_overflow:
936 case Builtin::BI__builtin_umulll_overflow:
937 Result = LHS.isSigned() ? LHS.smul_ov(RHS, Overflow)
938 : LHS.umul_ov(RHS, Overflow);
939 break;
940 }
941
942 // In the case where multiple sizes are allowed, truncate and see if
943 // the values are the same.
944 if (BuiltinOp == Builtin::BI__builtin_add_overflow ||
945 BuiltinOp == Builtin::BI__builtin_sub_overflow ||
946 BuiltinOp == Builtin::BI__builtin_mul_overflow) {
947 // APSInt doesn't have a TruncOrSelf, so we use extOrTrunc instead,
948 // since it will give us the behavior of a TruncOrSelf in the case where
949 // its parameter <= its size. We previously set Result to be at least the
950 // integer width of the result, so getIntWidth(ResultType) <=
951 // Result.BitWidth
952 APSInt Temp = Result.extOrTrunc(width: S.getASTContext().getIntWidth(T: ResultType));
953 Temp.setIsSigned(ResultType->isSignedIntegerOrEnumerationType());
954
955 if (!APSInt::isSameValue(I1: Temp, I2: Result))
956 Overflow = true;
957 Result = std::move(Temp);
958 }
959
960 // Write Result to ResultPtr and put Overflow on the stack.
961 assignIntegral(S, Dest: ResultPtr, ValueT: ResultT, Value: Result);
962 if (ResultPtr.canBeInitialized())
963 ResultPtr.initialize();
964
965 assert(Call->getDirectCallee()->getReturnType()->isBooleanType());
966 S.Stk.push<Boolean>(Args&: Overflow);
967 return true;
968}
969
970/// Three integral values followed by a pointer (lhs, rhs, carry, carryOut).
971static bool interp__builtin_carryop(InterpState &S, CodePtr OpPC,
972 const InterpFrame *Frame,
973 const CallExpr *Call, unsigned BuiltinOp) {
974 const Pointer &CarryOutPtr = S.Stk.pop<Pointer>();
975 PrimType LHST = *S.getContext().classify(T: Call->getArg(Arg: 0)->getType());
976 PrimType RHST = *S.getContext().classify(T: Call->getArg(Arg: 1)->getType());
977 APSInt CarryIn;
978 if (!popToAPSInt(Stk&: S.Stk, T: LHST, Out&: CarryIn))
979 return false;
980 APSInt RHS;
981 if (!popToAPSInt(Stk&: S.Stk, T: RHST, Out&: RHS))
982 return false;
983 APSInt LHS;
984 if (!popToAPSInt(Stk&: S.Stk, T: LHST, Out&: LHS))
985 return false;
986
987 if (!isReadable(P: CarryOutPtr))
988 return false;
989
990 APSInt CarryOut;
991
992 APSInt Result;
993 // Copy the number of bits and sign.
994 Result = LHS;
995 CarryOut = LHS;
996
997 bool FirstOverflowed = false;
998 bool SecondOverflowed = false;
999 switch (BuiltinOp) {
1000 default:
1001 llvm_unreachable("Invalid value for BuiltinOp");
1002 case Builtin::BI__builtin_addcb:
1003 case Builtin::BI__builtin_addcs:
1004 case Builtin::BI__builtin_addc:
1005 case Builtin::BI__builtin_addcl:
1006 case Builtin::BI__builtin_addcll:
1007 Result =
1008 LHS.uadd_ov(RHS, Overflow&: FirstOverflowed).uadd_ov(RHS: CarryIn, Overflow&: SecondOverflowed);
1009 break;
1010 case Builtin::BI__builtin_subcb:
1011 case Builtin::BI__builtin_subcs:
1012 case Builtin::BI__builtin_subc:
1013 case Builtin::BI__builtin_subcl:
1014 case Builtin::BI__builtin_subcll:
1015 Result =
1016 LHS.usub_ov(RHS, Overflow&: FirstOverflowed).usub_ov(RHS: CarryIn, Overflow&: SecondOverflowed);
1017 break;
1018 }
1019 // It is possible for both overflows to happen but CGBuiltin uses an OR so
1020 // this is consistent.
1021 CarryOut = (uint64_t)(FirstOverflowed | SecondOverflowed);
1022
1023 QualType CarryOutType = Call->getArg(Arg: 3)->getType()->getPointeeType();
1024 PrimType CarryOutT = *S.getContext().classify(T: CarryOutType);
1025 assignIntegral(S, Dest: CarryOutPtr, ValueT: CarryOutT, Value: CarryOut);
1026 if (CarryOutPtr.canBeInitialized())
1027 CarryOutPtr.initialize();
1028
1029 assert(S.getASTContext().hasSimilarType(Call->getType(),
1030 Call->getArg(0)->getType()));
1031 pushInteger(S, Val: Result, QT: Call->getType());
1032 return true;
1033}
1034
1035static bool interp__builtin_clz(InterpState &S, CodePtr OpPC,
1036 const InterpFrame *Frame, const CallExpr *Call,
1037 unsigned BuiltinOp) {
1038
1039 std::optional<APSInt> Fallback;
1040 if (BuiltinOp == Builtin::BI__builtin_clzg && Call->getNumArgs() == 2) {
1041 APSInt FallbackVal;
1042 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: FallbackVal))
1043 return false;
1044 Fallback = FallbackVal;
1045 }
1046
1047 APSInt Val;
1048 if (Call->getArg(Arg: 0)->getType()->isExtVectorBoolType()) {
1049 const Pointer &Arg = S.Stk.pop<Pointer>();
1050 Val = convertBoolVectorToInt(Val: Arg);
1051 } else {
1052 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: Val))
1053 return false;
1054 }
1055
1056 // When the argument is 0, the result of GCC builtins is undefined, whereas
1057 // for Microsoft intrinsics, the result is the bit-width of the argument.
1058 bool ZeroIsUndefined = BuiltinOp != Builtin::BI__lzcnt16 &&
1059 BuiltinOp != Builtin::BI__lzcnt &&
1060 BuiltinOp != Builtin::BI__lzcnt64;
1061
1062 if (Val == 0) {
1063 if (Fallback) {
1064 pushInteger(S, Val: *Fallback, QT: Call->getType());
1065 return true;
1066 }
1067
1068 if (ZeroIsUndefined)
1069 return false;
1070 }
1071
1072 pushInteger(S, Val: Val.countl_zero(), QT: Call->getType());
1073 return true;
1074}
1075
1076static bool interp__builtin_ctz(InterpState &S, CodePtr OpPC,
1077 const InterpFrame *Frame, const CallExpr *Call,
1078 unsigned BuiltinID) {
1079 std::optional<APSInt> Fallback;
1080 if (BuiltinID == Builtin::BI__builtin_ctzg && Call->getNumArgs() == 2) {
1081 APSInt FallbackVal;
1082 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: FallbackVal))
1083 return false;
1084 Fallback = FallbackVal;
1085 }
1086
1087 APSInt Val;
1088 if (Call->getArg(Arg: 0)->getType()->isExtVectorBoolType()) {
1089 const Pointer &Arg = S.Stk.pop<Pointer>();
1090 Val = convertBoolVectorToInt(Val: Arg);
1091 } else {
1092 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: Val))
1093 return false;
1094 }
1095
1096 if (Val == 0) {
1097 if (Fallback) {
1098 pushInteger(S, Val: *Fallback, QT: Call->getType());
1099 return true;
1100 }
1101 return false;
1102 }
1103
1104 pushInteger(S, Val: Val.countr_zero(), QT: Call->getType());
1105 return true;
1106}
1107
1108static bool interp__builtin_bswap(InterpState &S, CodePtr OpPC,
1109 const InterpFrame *Frame,
1110 const CallExpr *Call) {
1111 APSInt Val;
1112 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: Val))
1113 return false;
1114 if (Val.getBitWidth() == 8 || Val.getBitWidth() == 1)
1115 pushInteger(S, Val, QT: Call->getType());
1116 else
1117 pushInteger(S, Val: Val.byteSwap(), QT: Call->getType());
1118 return true;
1119}
1120
1121/// bool __atomic_always_lock_free(size_t, void const volatile*)
1122/// bool __atomic_is_lock_free(size_t, void const volatile*)
1123static bool interp__builtin_atomic_lock_free(InterpState &S, CodePtr OpPC,
1124 const InterpFrame *Frame,
1125 const CallExpr *Call,
1126 unsigned BuiltinOp) {
1127 auto returnBool = [&S](bool Value) -> bool {
1128 S.Stk.push<Boolean>(Args&: Value);
1129 return true;
1130 };
1131
1132 const Pointer &Ptr = S.Stk.pop<Pointer>();
1133 uint64_t SizeVal;
1134 if (!popToUInt64(S, E: Call->getArg(Arg: 0), Out&: SizeVal))
1135 return false;
1136
1137 // For __atomic_is_lock_free(sizeof(_Atomic(T))), if the size is a power
1138 // of two less than or equal to the maximum inline atomic width, we know it
1139 // is lock-free. If the size isn't a power of two, or greater than the
1140 // maximum alignment where we promote atomics, we know it is not lock-free
1141 // (at least not in the sense of atomic_is_lock_free). Otherwise,
1142 // the answer can only be determined at runtime; for example, 16-byte
1143 // atomics have lock-free implementations on some, but not all,
1144 // x86-64 processors.
1145
1146 // Check power-of-two.
1147 CharUnits Size = CharUnits::fromQuantity(Quantity: SizeVal);
1148 if (Size.isPowerOfTwo()) {
1149 // Check against inlining width.
1150 unsigned InlineWidthBits =
1151 S.getASTContext().getTargetInfo().getMaxAtomicInlineWidth();
1152 if (Size <= S.getASTContext().toCharUnitsFromBits(BitSize: InlineWidthBits)) {
1153
1154 // OK, we will inline appropriately-aligned operations of this size,
1155 // and _Atomic(T) is appropriately-aligned.
1156 if (Size == CharUnits::One())
1157 return returnBool(true);
1158
1159 // Same for null pointers.
1160 assert(BuiltinOp != Builtin::BI__c11_atomic_is_lock_free);
1161 if (Ptr.isZero())
1162 return returnBool(true);
1163
1164 if (Ptr.isIntegralPointer()) {
1165 uint64_t IntVal = Ptr.getIntegerRepresentation();
1166 if (APSInt(APInt(64, IntVal, false), true).isAligned(A: Size.getAsAlign()))
1167 return returnBool(true);
1168 }
1169
1170 const Expr *PtrArg = Call->getArg(Arg: 1);
1171 // Otherwise, check if the type's alignment against Size.
1172 if (const auto *ICE = dyn_cast<ImplicitCastExpr>(Val: PtrArg)) {
1173 // Drop the potential implicit-cast to 'const volatile void*', getting
1174 // the underlying type.
1175 if (ICE->getCastKind() == CK_BitCast)
1176 PtrArg = ICE->getSubExpr();
1177 }
1178
1179 if (const auto *PtrTy = PtrArg->getType()->getAs<PointerType>()) {
1180 QualType PointeeType = PtrTy->getPointeeType();
1181 if (!PointeeType->isIncompleteType() &&
1182 S.getASTContext().getTypeAlignInChars(T: PointeeType) >= Size) {
1183 // OK, we will inline operations on this object.
1184 return returnBool(true);
1185 }
1186 }
1187 }
1188 }
1189
1190 if (BuiltinOp == Builtin::BI__atomic_always_lock_free)
1191 return returnBool(false);
1192
1193 return Invalid(S, OpPC);
1194}
1195
1196/// bool __c11_atomic_is_lock_free(size_t)
1197static bool interp__builtin_c11_atomic_is_lock_free(InterpState &S,
1198 CodePtr OpPC,
1199 const InterpFrame *Frame,
1200 const CallExpr *Call) {
1201 uint64_t SizeVal;
1202 if (!popToUInt64(S, E: Call->getArg(Arg: 0), Out&: SizeVal))
1203 return false;
1204
1205 CharUnits Size = CharUnits::fromQuantity(Quantity: SizeVal);
1206 if (Size.isPowerOfTwo()) {
1207 // Check against inlining width.
1208 unsigned InlineWidthBits =
1209 S.getASTContext().getTargetInfo().getMaxAtomicInlineWidth();
1210 if (Size <= S.getASTContext().toCharUnitsFromBits(BitSize: InlineWidthBits)) {
1211 S.Stk.push<Boolean>(Args: true);
1212 return true;
1213 }
1214 }
1215
1216 return false; // returnBool(false);
1217}
1218
1219/// __builtin_complex(Float A, float B);
1220static bool interp__builtin_complex(InterpState &S, CodePtr OpPC,
1221 const InterpFrame *Frame,
1222 const CallExpr *Call) {
1223 const Floating &Arg2 = S.Stk.pop<Floating>();
1224 const Floating &Arg1 = S.Stk.pop<Floating>();
1225 Pointer &Result = S.Stk.peek<Pointer>();
1226
1227 Result.elem<Floating>(I: 0) = Arg1;
1228 Result.elem<Floating>(I: 1) = Arg2;
1229 Result.initializeAllElements();
1230
1231 return true;
1232}
1233
1234/// __builtin_is_aligned()
1235/// __builtin_align_up()
1236/// __builtin_align_down()
1237/// The first parameter is either an integer or a pointer.
1238/// The second parameter is the requested alignment as an integer.
1239static bool interp__builtin_is_aligned_up_down(InterpState &S, CodePtr OpPC,
1240 const InterpFrame *Frame,
1241 const CallExpr *Call,
1242 unsigned BuiltinOp) {
1243 APSInt Alignment;
1244 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: Alignment))
1245 return false;
1246
1247 if (Alignment < 0 || !Alignment.isPowerOf2()) {
1248 S.FFDiag(E: Call, DiagId: diag::note_constexpr_invalid_alignment) << Alignment;
1249 return false;
1250 }
1251 unsigned SrcWidth = S.getASTContext().getIntWidth(T: Call->getArg(Arg: 0)->getType());
1252 APSInt MaxValue(APInt::getOneBitSet(numBits: SrcWidth, BitNo: SrcWidth - 1));
1253 if (APSInt::compareValues(I1: Alignment, I2: MaxValue) > 0) {
1254 S.FFDiag(E: Call, DiagId: diag::note_constexpr_alignment_too_big)
1255 << MaxValue << Call->getArg(Arg: 0)->getType() << Alignment;
1256 return false;
1257 }
1258
1259 // The first parameter is either an integer or a pointer.
1260 PrimType FirstArgT = *S.Ctx.classify(E: Call->getArg(Arg: 0));
1261
1262 if (isIntegerType(T: FirstArgT)) {
1263 APSInt Src;
1264 if (!popToAPSInt(Stk&: S.Stk, T: FirstArgT, Out&: Src))
1265 return false;
1266 APInt AlignMinusOne = Alignment.extOrTrunc(width: Src.getBitWidth()) - 1;
1267 if (BuiltinOp == Builtin::BI__builtin_align_up) {
1268 APSInt AlignedVal =
1269 APSInt((Src + AlignMinusOne) & ~AlignMinusOne, Src.isUnsigned());
1270 pushInteger(S, Val: AlignedVal, QT: Call->getType());
1271 } else if (BuiltinOp == Builtin::BI__builtin_align_down) {
1272 APSInt AlignedVal = APSInt(Src & ~AlignMinusOne, Src.isUnsigned());
1273 pushInteger(S, Val: AlignedVal, QT: Call->getType());
1274 } else {
1275 assert(*S.Ctx.classify(Call->getType()) == PT_Bool);
1276 S.Stk.push<Boolean>(Args: (Src & AlignMinusOne) == 0);
1277 }
1278 return true;
1279 }
1280 assert(FirstArgT == PT_Ptr);
1281 const Pointer &Ptr = S.Stk.pop<Pointer>();
1282 if (!Ptr.isBlockPointer())
1283 return false;
1284
1285 const ValueDecl *PtrDecl = Ptr.getDeclDesc()->asValueDecl();
1286 // We need a pointer for a declaration here.
1287 if (!PtrDecl) {
1288 if (BuiltinOp == Builtin::BI__builtin_is_aligned)
1289 S.FFDiag(E: Call->getArg(Arg: 0), DiagId: diag::note_constexpr_alignment_compute)
1290 << Alignment;
1291 else
1292 S.FFDiag(E: Call->getArg(Arg: 0), DiagId: diag::note_constexpr_alignment_adjust)
1293 << Alignment;
1294 return false;
1295 }
1296
1297 // For one-past-end pointers, we can't call getIndex() since it asserts.
1298 // Use getNumElems() instead which gives the correct index for past-end.
1299 unsigned PtrOffset =
1300 Ptr.isElementPastEnd() ? Ptr.getNumElems() : Ptr.getIndex();
1301 CharUnits BaseAlignment = S.getASTContext().getDeclAlign(D: PtrDecl);
1302 CharUnits PtrAlign =
1303 BaseAlignment.alignmentAtOffset(offset: CharUnits::fromQuantity(Quantity: PtrOffset));
1304
1305 if (BuiltinOp == Builtin::BI__builtin_is_aligned) {
1306 if (PtrAlign.getQuantity() >= Alignment) {
1307 S.Stk.push<Boolean>(Args: true);
1308 return true;
1309 }
1310 // If the alignment is not known to be sufficient, some cases could still
1311 // be aligned at run time. However, if the requested alignment is less or
1312 // equal to the base alignment and the offset is not aligned, we know that
1313 // the run-time value can never be aligned.
1314 if (BaseAlignment.getQuantity() >= Alignment &&
1315 PtrAlign.getQuantity() < Alignment) {
1316 S.Stk.push<Boolean>(Args: false);
1317 return true;
1318 }
1319
1320 S.FFDiag(E: Call->getArg(Arg: 0), DiagId: diag::note_constexpr_alignment_compute)
1321 << Alignment;
1322 return false;
1323 }
1324
1325 assert(BuiltinOp == Builtin::BI__builtin_align_down ||
1326 BuiltinOp == Builtin::BI__builtin_align_up);
1327
1328 // For align_up/align_down, we can return the same value if the alignment
1329 // is known to be greater or equal to the requested value.
1330 if (PtrAlign.getQuantity() >= Alignment) {
1331 S.Stk.push<Pointer>(Args: Ptr);
1332 return true;
1333 }
1334
1335 // The alignment could be greater than the minimum at run-time, so we cannot
1336 // infer much about the resulting pointer value. One case is possible:
1337 // For `_Alignas(32) char buf[N]; __builtin_align_down(&buf[idx], 32)` we
1338 // can infer the correct index if the requested alignment is smaller than
1339 // the base alignment so we can perform the computation on the offset.
1340 if (BaseAlignment.getQuantity() >= Alignment) {
1341 assert(Alignment.getBitWidth() <= 64 &&
1342 "Cannot handle > 64-bit address-space");
1343 uint64_t Alignment64 = Alignment.getZExtValue();
1344 CharUnits NewOffset =
1345 CharUnits::fromQuantity(Quantity: BuiltinOp == Builtin::BI__builtin_align_down
1346 ? llvm::alignDown(Value: PtrOffset, Align: Alignment64)
1347 : llvm::alignTo(Value: PtrOffset, Align: Alignment64));
1348
1349 S.Stk.push<Pointer>(Args: Ptr.atIndex(Idx: NewOffset.getQuantity()));
1350 return true;
1351 }
1352
1353 // Otherwise, we cannot constant-evaluate the result.
1354 S.FFDiag(E: Call->getArg(Arg: 0), DiagId: diag::note_constexpr_alignment_adjust) << Alignment;
1355 return false;
1356}
1357
1358/// __builtin_assume_aligned(Ptr, Alignment[, ExtraOffset])
1359static bool interp__builtin_assume_aligned(InterpState &S, CodePtr OpPC,
1360 const InterpFrame *Frame,
1361 const CallExpr *Call) {
1362 assert(Call->getNumArgs() == 2 || Call->getNumArgs() == 3);
1363
1364 std::optional<APSInt> ExtraOffset;
1365 if (Call->getNumArgs() == 3) {
1366 APSInt ExtraOffsetVal;
1367 if (!popToAPSInt(Stk&: S.Stk, T: *S.Ctx.classify(E: Call->getArg(Arg: 2)), Out&: ExtraOffsetVal))
1368 return false;
1369 ExtraOffset = ExtraOffsetVal;
1370 }
1371
1372 APSInt Alignment;
1373 if (!popToAPSInt(Stk&: S.Stk, T: *S.Ctx.classify(E: Call->getArg(Arg: 1)), Out&: Alignment))
1374 return false;
1375 const Pointer &Ptr = S.Stk.pop<Pointer>();
1376
1377 CharUnits Align = CharUnits::fromQuantity(Quantity: Alignment.getZExtValue());
1378
1379 // If there is a base object, then it must have the correct alignment.
1380 if (Ptr.isBlockPointer()) {
1381 CharUnits BaseAlignment;
1382 if (const auto *VD = Ptr.getDeclDesc()->asValueDecl())
1383 BaseAlignment = S.getASTContext().getDeclAlign(D: VD);
1384 else if (const auto *E = Ptr.getDeclDesc()->asExpr())
1385 BaseAlignment = GetAlignOfExpr(Ctx: S.getASTContext(), E, ExprKind: UETT_AlignOf);
1386
1387 if (BaseAlignment < Align) {
1388 S.CCEDiag(E: Call->getArg(Arg: 0),
1389 DiagId: diag::note_constexpr_baa_insufficient_alignment)
1390 << 0 << BaseAlignment.getQuantity() << Align.getQuantity();
1391 return false;
1392 }
1393 }
1394
1395 APValue AV = Ptr.toAPValue(ASTCtx: S.getASTContext());
1396 CharUnits AVOffset = AV.getLValueOffset();
1397 if (ExtraOffset)
1398 AVOffset -= CharUnits::fromQuantity(Quantity: ExtraOffset->getZExtValue());
1399 if (AVOffset.alignTo(Align) != AVOffset) {
1400 if (Ptr.isBlockPointer())
1401 S.CCEDiag(E: Call->getArg(Arg: 0),
1402 DiagId: diag::note_constexpr_baa_insufficient_alignment)
1403 << 1 << AVOffset.getQuantity() << Align.getQuantity();
1404 else
1405 S.CCEDiag(E: Call->getArg(Arg: 0),
1406 DiagId: diag::note_constexpr_baa_value_insufficient_alignment)
1407 << AVOffset.getQuantity() << Align.getQuantity();
1408 return false;
1409 }
1410
1411 S.Stk.push<Pointer>(Args: Ptr);
1412 return true;
1413}
1414
1415/// (CarryIn, LHS, RHS, Result)
1416static bool interp__builtin_ia32_addcarry_subborrow(InterpState &S,
1417 CodePtr OpPC,
1418 const InterpFrame *Frame,
1419 const CallExpr *Call,
1420 bool IsAdd) {
1421 if (Call->getNumArgs() != 4 || !Call->getArg(Arg: 0)->getType()->isIntegerType() ||
1422 !Call->getArg(Arg: 1)->getType()->isIntegerType() ||
1423 !Call->getArg(Arg: 2)->getType()->isIntegerType())
1424 return false;
1425
1426 const Pointer &CarryOutPtr = S.Stk.pop<Pointer>();
1427
1428 APSInt RHS;
1429 if (!popToAPSInt(S, E: Call->getArg(Arg: 2), Out&: RHS))
1430 return false;
1431 APSInt LHS;
1432 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: LHS))
1433 return false;
1434 APSInt CarryIn;
1435 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: CarryIn))
1436 return false;
1437
1438 unsigned BitWidth = LHS.getBitWidth();
1439 unsigned CarryInBit = CarryIn.ugt(RHS: 0) ? 1 : 0;
1440 APInt ExResult =
1441 IsAdd ? (LHS.zext(width: BitWidth + 1) + (RHS.zext(width: BitWidth + 1) + CarryInBit))
1442 : (LHS.zext(width: BitWidth + 1) - (RHS.zext(width: BitWidth + 1) + CarryInBit));
1443
1444 APInt Result = ExResult.extractBits(numBits: BitWidth, bitPosition: 0);
1445 APSInt CarryOut =
1446 APSInt(ExResult.extractBits(numBits: 1, bitPosition: BitWidth), /*IsUnsigned=*/true);
1447
1448 QualType CarryOutType = Call->getArg(Arg: 3)->getType()->getPointeeType();
1449 PrimType CarryOutT = *S.getContext().classify(T: CarryOutType);
1450 assignIntegral(S, Dest: CarryOutPtr, ValueT: CarryOutT, Value: APSInt(std::move(Result), true));
1451
1452 pushInteger(S, Val: CarryOut, QT: Call->getType());
1453
1454 return true;
1455}
1456
1457static bool interp__builtin_os_log_format_buffer_size(InterpState &S,
1458 CodePtr OpPC,
1459 const InterpFrame *Frame,
1460 const CallExpr *Call) {
1461 analyze_os_log::OSLogBufferLayout Layout;
1462 analyze_os_log::computeOSLogBufferLayout(Ctx&: S.getASTContext(), E: Call, layout&: Layout);
1463 pushInteger(S, Val: Layout.size().getQuantity(), QT: Call->getType());
1464 return true;
1465}
1466
1467static bool
1468interp__builtin_ptrauth_string_discriminator(InterpState &S, CodePtr OpPC,
1469 const InterpFrame *Frame,
1470 const CallExpr *Call) {
1471 const auto &Ptr = S.Stk.pop<Pointer>();
1472 assert(Ptr.getFieldDesc()->isPrimitiveArray());
1473
1474 // This should be created for a StringLiteral, so always holds at least
1475 // one array element.
1476 assert(Ptr.getFieldDesc()->getNumElems() >= 1);
1477 uint64_t Result = getPointerAuthStableSipHash(
1478 S: cast<StringLiteral>(Val: Ptr.getFieldDesc()->asExpr())->getString());
1479 pushInteger(S, Val: Result, QT: Call->getType());
1480 return true;
1481}
1482
1483static bool interp__builtin_infer_alloc_token(InterpState &S, CodePtr OpPC,
1484 const InterpFrame *Frame,
1485 const CallExpr *Call) {
1486 const ASTContext &ASTCtx = S.getASTContext();
1487 uint64_t BitWidth = ASTCtx.getTypeSize(T: ASTCtx.getSizeType());
1488 auto Mode =
1489 ASTCtx.getLangOpts().AllocTokenMode.value_or(u: llvm::DefaultAllocTokenMode);
1490 auto MaxTokensOpt = ASTCtx.getLangOpts().AllocTokenMax;
1491 uint64_t MaxTokens =
1492 MaxTokensOpt.value_or(u: 0) ? *MaxTokensOpt : (~0ULL >> (64 - BitWidth));
1493
1494 // We do not read any of the arguments; discard them.
1495 for (int I = Call->getNumArgs() - 1; I >= 0; --I)
1496 discard(Stk&: S.Stk, T: S.getContext().classify(E: Call->getArg(Arg: I)).value_or(PT: PT_Ptr));
1497
1498 // Note: Type inference from a surrounding cast is not supported in
1499 // constexpr evaluation.
1500 QualType AllocType = infer_alloc::inferPossibleType(E: Call, Ctx: ASTCtx, CastE: nullptr);
1501 if (AllocType.isNull()) {
1502 S.CCEDiag(E: Call,
1503 DiagId: diag::note_constexpr_infer_alloc_token_type_inference_failed);
1504 return false;
1505 }
1506
1507 auto ATMD = infer_alloc::getAllocTokenMetadata(T: AllocType, Ctx: ASTCtx);
1508 if (!ATMD) {
1509 S.CCEDiag(E: Call, DiagId: diag::note_constexpr_infer_alloc_token_no_metadata);
1510 return false;
1511 }
1512
1513 auto MaybeToken = llvm::getAllocToken(Mode, Metadata: *ATMD, MaxTokens);
1514 if (!MaybeToken) {
1515 S.CCEDiag(E: Call, DiagId: diag::note_constexpr_infer_alloc_token_stateful_mode);
1516 return false;
1517 }
1518
1519 pushInteger(S, Val: llvm::APInt(BitWidth, *MaybeToken), QT: ASTCtx.getSizeType());
1520 return true;
1521}
1522
1523static bool interp__builtin_operator_new(InterpState &S, CodePtr OpPC,
1524 const InterpFrame *Frame,
1525 const CallExpr *Call) {
1526 // A call to __operator_new is only valid within std::allocate<>::allocate.
1527 // Walk up the call stack to find the appropriate caller and get the
1528 // element type from it.
1529 auto [NewCall, ElemType] = S.getStdAllocatorCaller(Name: "allocate");
1530
1531 if (ElemType.isNull()) {
1532 S.FFDiag(E: Call, DiagId: S.getLangOpts().CPlusPlus20
1533 ? diag::note_constexpr_new_untyped
1534 : diag::note_constexpr_new);
1535 return false;
1536 }
1537 assert(NewCall);
1538
1539 if (ElemType->isIncompleteType() || ElemType->isFunctionType()) {
1540 S.FFDiag(E: Call, DiagId: diag::note_constexpr_new_not_complete_object_type)
1541 << (ElemType->isIncompleteType() ? 0 : 1) << ElemType;
1542 return false;
1543 }
1544
1545 // We only care about the first parameter (the size), so discard all the
1546 // others.
1547 {
1548 unsigned NumArgs = Call->getNumArgs();
1549 assert(NumArgs >= 1);
1550
1551 // The std::nothrow_t arg never gets put on the stack.
1552 if (Call->getArg(Arg: NumArgs - 1)->getType()->isNothrowT())
1553 --NumArgs;
1554 auto Args = ArrayRef(Call->getArgs(), Call->getNumArgs());
1555 // First arg is needed.
1556 Args = Args.drop_front();
1557
1558 // Discard the rest.
1559 for (const Expr *Arg : Args)
1560 discard(Stk&: S.Stk, T: *S.getContext().classify(E: Arg));
1561 }
1562
1563 APSInt Bytes;
1564 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: Bytes))
1565 return false;
1566 CharUnits ElemSize = S.getASTContext().getTypeSizeInChars(T: ElemType);
1567 assert(!ElemSize.isZero());
1568 // Divide the number of bytes by sizeof(ElemType), so we get the number of
1569 // elements we should allocate.
1570 APInt NumElems, Remainder;
1571 APInt ElemSizeAP(Bytes.getBitWidth(), ElemSize.getQuantity());
1572 APInt::udivrem(LHS: Bytes, RHS: ElemSizeAP, Quotient&: NumElems, Remainder);
1573 if (Remainder != 0) {
1574 // This likely indicates a bug in the implementation of 'std::allocator'.
1575 S.FFDiag(E: Call, DiagId: diag::note_constexpr_operator_new_bad_size)
1576 << Bytes << APSInt(ElemSizeAP, true) << ElemType;
1577 return false;
1578 }
1579
1580 // NB: The same check we're using in CheckArraySize()
1581 if (NumElems.getActiveBits() >
1582 ConstantArrayType::getMaxSizeBits(Context: S.getASTContext()) ||
1583 NumElems.ugt(RHS: Descriptor::MaxArrayElemBytes / ElemSize.getQuantity())) {
1584 // FIXME: NoThrow check?
1585 const SourceInfo &Loc = S.Current->getSource(PC: OpPC);
1586 S.FFDiag(SI: Loc, DiagId: diag::note_constexpr_new_too_large)
1587 << NumElems.getZExtValue();
1588 return false;
1589 }
1590
1591 if (!CheckArraySize(S, OpPC, NumElems: NumElems.getZExtValue()))
1592 return false;
1593
1594 bool IsArray = NumElems.ugt(RHS: 1);
1595 OptPrimType ElemT = S.getContext().classify(T: ElemType);
1596 DynamicAllocator &Allocator = S.getAllocator();
1597 if (ElemT) {
1598 Block *B =
1599 Allocator.allocate(Source: NewCall, T: *ElemT, NumElements: NumElems.getZExtValue(),
1600 EvalID: S.Ctx.getEvalID(), AllocForm: DynamicAllocator::Form::Operator);
1601 assert(B);
1602 S.Stk.push<Pointer>(Args: Pointer(B).atIndex(Idx: 0));
1603 return true;
1604 }
1605
1606 assert(!ElemT);
1607
1608 // Composite arrays
1609 if (IsArray) {
1610 const Descriptor *Desc =
1611 S.P.createDescriptor(D: NewCall, Ty: ElemType.getTypePtr(), MDSize: std::nullopt);
1612 Block *B =
1613 Allocator.allocate(D: Desc, NumElements: NumElems.getZExtValue(), EvalID: S.Ctx.getEvalID(),
1614 AllocForm: DynamicAllocator::Form::Operator);
1615 assert(B);
1616 S.Stk.push<Pointer>(Args: Pointer(B).atIndex(Idx: 0).narrow());
1617 return true;
1618 }
1619
1620 // Records. Still allocate them as single-element arrays.
1621 QualType AllocType = S.getASTContext().getConstantArrayType(
1622 EltTy: ElemType, ArySize: NumElems, SizeExpr: nullptr, ASM: ArraySizeModifier::Normal, IndexTypeQuals: 0);
1623
1624 const Descriptor *Desc = S.P.createDescriptor(D: NewCall, Ty: AllocType.getTypePtr(),
1625 MDSize: Descriptor::InlineDescMD);
1626 Block *B = Allocator.allocate(D: Desc, EvalID: S.getContext().getEvalID(),
1627 AllocForm: DynamicAllocator::Form::Operator);
1628 assert(B);
1629 S.Stk.push<Pointer>(Args: Pointer(B).atIndex(Idx: 0).narrow());
1630 return true;
1631}
1632
1633static bool interp__builtin_operator_delete(InterpState &S, CodePtr OpPC,
1634 const InterpFrame *Frame,
1635 const CallExpr *Call) {
1636 const Expr *Source = nullptr;
1637 const Block *BlockToDelete = nullptr;
1638
1639 unsigned NumArgs = Call->getNumArgs();
1640 assert(NumArgs >= 1);
1641
1642 // Args are pushed in source order. The trailing sized/aligned delete
1643 // operands are above the pointer on the stack.
1644 for (unsigned I = NumArgs - 1; I != 0; --I)
1645 discard(Stk&: S.Stk, T: *S.getContext().classify(E: Call->getArg(Arg: I)));
1646
1647 if (S.checkingPotentialConstantExpression()) {
1648 S.Stk.discard<Pointer>();
1649 return false;
1650 }
1651
1652 // This is permitted only within a call to std::allocator<T>::deallocate.
1653 if (!S.getStdAllocatorCaller(Name: "deallocate")) {
1654 S.FFDiag(E: Call);
1655 S.Stk.discard<Pointer>();
1656 return true;
1657 }
1658
1659 {
1660 const Pointer &Ptr = S.Stk.pop<Pointer>();
1661
1662 if (Ptr.isZero()) {
1663 S.CCEDiag(E: Call, DiagId: diag::note_constexpr_deallocate_null);
1664 return true;
1665 }
1666
1667 Source = Ptr.getDeclDesc()->asExpr();
1668 BlockToDelete = Ptr.block();
1669
1670 if (!BlockToDelete->isDynamic()) {
1671 S.FFDiag(E: Call, DiagId: diag::note_constexpr_delete_not_heap_alloc)
1672 << Ptr.toDiagnosticString(Ctx: S.getASTContext());
1673 if (const auto *D = Ptr.getFieldDesc()->asDecl())
1674 S.Note(Loc: D->getLocation(), DiagId: diag::note_declared_at);
1675 }
1676 }
1677 assert(BlockToDelete);
1678
1679 DynamicAllocator &Allocator = S.getAllocator();
1680 const Descriptor *BlockDesc = BlockToDelete->getDescriptor();
1681 std::optional<DynamicAllocator::Form> AllocForm =
1682 Allocator.getAllocationForm(Source);
1683
1684 if (!Allocator.deallocate(Source, BlockToDelete)) {
1685 // Nothing has been deallocated, this must be a double-delete.
1686 const SourceInfo &Loc = S.Current->getSource(PC: OpPC);
1687 S.FFDiag(SI: Loc, DiagId: diag::note_constexpr_double_delete);
1688 return false;
1689 }
1690 assert(AllocForm);
1691
1692 return CheckNewDeleteForms(
1693 S, OpPC, AllocForm: *AllocForm, DeleteForm: DynamicAllocator::Form::Operator, D: BlockDesc, NewExpr: Source);
1694}
1695
1696static bool interp__builtin_arithmetic_fence(InterpState &S, CodePtr OpPC,
1697 const InterpFrame *Frame,
1698 const CallExpr *Call) {
1699 const Floating &Arg0 = S.Stk.pop<Floating>();
1700 S.Stk.push<Floating>(Args: Arg0);
1701 return true;
1702}
1703
1704static bool interp__builtin_vector_reduce(InterpState &S, CodePtr OpPC,
1705 const CallExpr *Call, unsigned ID) {
1706 const Pointer &Arg = S.Stk.pop<Pointer>();
1707 assert(Arg.getFieldDesc()->isPrimitiveArray());
1708
1709 QualType ElemType = Arg.getFieldDesc()->getElemQualType();
1710 assert(Call->getType() == ElemType);
1711 PrimType ElemT = *S.getContext().classify(T: ElemType);
1712 unsigned NumElems = Arg.getNumElems();
1713
1714 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
1715 T Result = Arg.elem<T>(0);
1716 unsigned BitWidth = Result.bitWidth();
1717 for (unsigned I = 1; I != NumElems; ++I) {
1718 T Elem = Arg.elem<T>(I);
1719 T PrevResult = Result;
1720
1721 if (ID == Builtin::BI__builtin_reduce_add) {
1722 if (T::add(Result, Elem, BitWidth, &Result)) {
1723 unsigned OverflowBits = BitWidth + 1;
1724 (void)handleOverflow(S, OpPC,
1725 (PrevResult.toAPSInt(OverflowBits) +
1726 Elem.toAPSInt(OverflowBits)));
1727 return false;
1728 }
1729 } else if (ID == Builtin::BI__builtin_reduce_mul) {
1730 if (T::mul(Result, Elem, BitWidth, &Result)) {
1731 unsigned OverflowBits = BitWidth * 2;
1732 (void)handleOverflow(S, OpPC,
1733 (PrevResult.toAPSInt(OverflowBits) *
1734 Elem.toAPSInt(OverflowBits)));
1735 return false;
1736 }
1737
1738 } else if (ID == Builtin::BI__builtin_reduce_and) {
1739 (void)T::bitAnd(Result, Elem, BitWidth, &Result);
1740 } else if (ID == Builtin::BI__builtin_reduce_or) {
1741 (void)T::bitOr(Result, Elem, BitWidth, &Result);
1742 } else if (ID == Builtin::BI__builtin_reduce_xor) {
1743 (void)T::bitXor(Result, Elem, BitWidth, &Result);
1744 } else if (ID == Builtin::BI__builtin_reduce_min) {
1745 if (Elem < Result)
1746 Result = Elem;
1747 } else if (ID == Builtin::BI__builtin_reduce_max) {
1748 if (Elem > Result)
1749 Result = Elem;
1750 } else {
1751 llvm_unreachable("Unhandled vector reduce builtin");
1752 }
1753 }
1754 pushInteger(S, Result.toAPSInt(), Call->getType());
1755 });
1756
1757 return true;
1758}
1759
1760static bool interp__builtin_elementwise_abs(InterpState &S, CodePtr OpPC,
1761 const InterpFrame *Frame,
1762 const CallExpr *Call,
1763 unsigned BuiltinID) {
1764 assert(Call->getNumArgs() == 1);
1765 QualType Ty = Call->getArg(Arg: 0)->getType();
1766 if (Ty->isIntegerType()) {
1767 APSInt Val;
1768 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: Val))
1769 return false;
1770 pushInteger(S, Val: Val.abs(), QT: Call->getType());
1771 return true;
1772 }
1773
1774 if (Ty->isFloatingType()) {
1775 Floating Val = S.Stk.pop<Floating>();
1776 Floating Result = abs(S, In: Val);
1777 S.Stk.push<Floating>(Args&: Result);
1778 return true;
1779 }
1780
1781 // Otherwise, the argument must be a vector.
1782 assert(Call->getArg(0)->getType()->isVectorType());
1783 const Pointer &Arg = S.Stk.pop<Pointer>();
1784 assert(Arg.getFieldDesc()->isPrimitiveArray());
1785 const Pointer &Dst = S.Stk.peek<Pointer>();
1786 assert(Dst.getFieldDesc()->isPrimitiveArray());
1787 assert(Arg.getFieldDesc()->getNumElems() ==
1788 Dst.getFieldDesc()->getNumElems());
1789
1790 QualType ElemType = Arg.getFieldDesc()->getElemQualType();
1791 PrimType ElemT = *S.getContext().classify(T: ElemType);
1792 unsigned NumElems = Arg.getNumElems();
1793 // we can either have a vector of integer or a vector of floating point
1794 for (unsigned I = 0; I != NumElems; ++I) {
1795 if (ElemType->isIntegerType()) {
1796 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
1797 Dst.elem<T>(I) = T::from(static_cast<T>(
1798 APSInt(Arg.elem<T>(I).toAPSInt().abs(),
1799 ElemType->isUnsignedIntegerOrEnumerationType())));
1800 });
1801 } else {
1802 Floating Val = Arg.elem<Floating>(I);
1803 Dst.elem<Floating>(I) = abs(S, In: Val);
1804 }
1805 }
1806 Dst.initializeAllElements();
1807
1808 return true;
1809}
1810
1811/// Can be called with an integer or vector as the first and only parameter.
1812static bool interp__builtin_elementwise_countzeroes(InterpState &S,
1813 CodePtr OpPC,
1814 const InterpFrame *Frame,
1815 const CallExpr *Call,
1816 unsigned BuiltinID) {
1817 bool HasZeroArg = Call->getNumArgs() == 2;
1818 bool IsCTTZ = BuiltinID == Builtin::BI__builtin_elementwise_ctzg;
1819 assert(Call->getNumArgs() == 1 || HasZeroArg);
1820 if (Call->getArg(Arg: 0)->getType()->isIntegerType()) {
1821 PrimType ArgT = *S.getContext().classify(T: Call->getArg(Arg: 0)->getType());
1822 APSInt Val;
1823 if (!popToAPSInt(Stk&: S.Stk, T: ArgT, Out&: Val))
1824 return false;
1825 std::optional<APSInt> ZeroVal;
1826 if (HasZeroArg) {
1827 ZeroVal = Val;
1828 if (!popToAPSInt(Stk&: S.Stk, T: ArgT, Out&: Val))
1829 return false;
1830 }
1831
1832 if (Val.isZero()) {
1833 if (ZeroVal) {
1834 pushInteger(S, Val: *ZeroVal, QT: Call->getType());
1835 return true;
1836 }
1837 // If we haven't been provided the second argument, the result is
1838 // undefined
1839 S.FFDiag(SI: S.Current->getSource(PC: OpPC),
1840 DiagId: diag::note_constexpr_countzeroes_zero)
1841 << /*IsTrailing=*/IsCTTZ;
1842 return false;
1843 }
1844
1845 if (BuiltinID == Builtin::BI__builtin_elementwise_clzg) {
1846 pushInteger(S, Val: Val.countLeadingZeros(), QT: Call->getType());
1847 } else {
1848 pushInteger(S, Val: Val.countTrailingZeros(), QT: Call->getType());
1849 }
1850 return true;
1851 }
1852 // Otherwise, the argument must be a vector.
1853 const ASTContext &ASTCtx = S.getASTContext();
1854 Pointer ZeroArg;
1855 if (HasZeroArg) {
1856 assert(Call->getArg(1)->getType()->isVectorType() &&
1857 ASTCtx.hasSameUnqualifiedType(Call->getArg(0)->getType(),
1858 Call->getArg(1)->getType()));
1859 (void)ASTCtx;
1860 ZeroArg = S.Stk.pop<Pointer>();
1861 assert(ZeroArg.getFieldDesc()->isPrimitiveArray());
1862 }
1863 assert(Call->getArg(0)->getType()->isVectorType());
1864 const Pointer &Arg = S.Stk.pop<Pointer>();
1865 assert(Arg.getFieldDesc()->isPrimitiveArray());
1866 const Pointer &Dst = S.Stk.peek<Pointer>();
1867 assert(Dst.getFieldDesc()->isPrimitiveArray());
1868 assert(Arg.getFieldDesc()->getNumElems() ==
1869 Dst.getFieldDesc()->getNumElems());
1870
1871 QualType ElemType = Arg.getFieldDesc()->getElemQualType();
1872 PrimType ElemT = *S.getContext().classify(T: ElemType);
1873 unsigned NumElems = Arg.getNumElems();
1874
1875 // FIXME: Reading from uninitialized vector elements?
1876 for (unsigned I = 0; I != NumElems; ++I) {
1877 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
1878 APInt EltVal = Arg.atIndex(I).deref<T>().toAPSInt();
1879 if (EltVal.isZero()) {
1880 if (HasZeroArg) {
1881 Dst.atIndex(I).deref<T>() = ZeroArg.atIndex(I).deref<T>();
1882 } else {
1883 // If we haven't been provided the second argument, the result is
1884 // undefined
1885 S.FFDiag(S.Current->getSource(OpPC),
1886 diag::note_constexpr_countzeroes_zero)
1887 << /*IsTrailing=*/IsCTTZ;
1888 return false;
1889 }
1890 } else if (IsCTTZ) {
1891 Dst.atIndex(I).deref<T>() = T::from(EltVal.countTrailingZeros());
1892 } else {
1893 Dst.atIndex(I).deref<T>() = T::from(EltVal.countLeadingZeros());
1894 }
1895 Dst.atIndex(I).initialize();
1896 });
1897 }
1898
1899 return true;
1900}
1901
1902static bool interp__builtin_memcpy(InterpState &S, CodePtr OpPC,
1903 const InterpFrame *Frame,
1904 const CallExpr *Call, unsigned ID) {
1905 assert(Call->getNumArgs() == 3);
1906 const ASTContext &ASTCtx = S.getASTContext();
1907 uint64_t Size;
1908 if (!popToUInt64(S, E: Call->getArg(Arg: 2), Out&: Size))
1909 return false;
1910 Pointer SrcPtr = S.Stk.pop<Pointer>().expand();
1911 Pointer DestPtr = S.Stk.pop<Pointer>().expand();
1912
1913 if (ID == Builtin::BImemcpy || ID == Builtin::BImemmove)
1914 diagnoseNonConstexprBuiltin(S, OpPC, ID);
1915
1916 bool Move =
1917 (ID == Builtin::BI__builtin_memmove || ID == Builtin::BImemmove ||
1918 ID == Builtin::BI__builtin_wmemmove || ID == Builtin::BIwmemmove);
1919 bool WChar = ID == Builtin::BIwmemcpy || ID == Builtin::BIwmemmove ||
1920 ID == Builtin::BI__builtin_wmemcpy ||
1921 ID == Builtin::BI__builtin_wmemmove;
1922
1923 // If the size is zero, we treat this as always being a valid no-op.
1924 if (Size == 0) {
1925 S.Stk.push<Pointer>(Args&: DestPtr);
1926 return true;
1927 }
1928
1929 if (SrcPtr.isZero() || DestPtr.isZero()) {
1930 Pointer DiagPtr = (SrcPtr.isZero() ? SrcPtr : DestPtr);
1931 S.FFDiag(SI: S.Current->getSource(PC: OpPC), DiagId: diag::note_constexpr_memcpy_null)
1932 << /*IsMove=*/Move << /*IsWchar=*/WChar << !SrcPtr.isZero()
1933 << DiagPtr.toDiagnosticString(Ctx: ASTCtx);
1934 return false;
1935 }
1936
1937 // Diagnose integral src/dest pointers specially.
1938 if (SrcPtr.isIntegralPointer() || DestPtr.isIntegralPointer()) {
1939 std::string DiagVal = "(void *)";
1940 DiagVal += SrcPtr.isIntegralPointer()
1941 ? std::to_string(val: SrcPtr.getIntegerRepresentation())
1942 : std::to_string(val: DestPtr.getIntegerRepresentation());
1943 S.FFDiag(SI: S.Current->getSource(PC: OpPC), DiagId: diag::note_constexpr_memcpy_null)
1944 << Move << WChar << DestPtr.isIntegralPointer() << DiagVal;
1945 return false;
1946 }
1947
1948 if (!isReadable(P: DestPtr) || !isReadable(P: SrcPtr))
1949 return false;
1950
1951 if (DestPtr.getType()->isIncompleteType()) {
1952 S.FFDiag(SI: S.Current->getSource(PC: OpPC),
1953 DiagId: diag::note_constexpr_memcpy_incomplete_type)
1954 << Move << DestPtr.getType();
1955 return false;
1956 }
1957 if (SrcPtr.getType()->isIncompleteType()) {
1958 S.FFDiag(SI: S.Current->getSource(PC: OpPC),
1959 DiagId: diag::note_constexpr_memcpy_incomplete_type)
1960 << Move << SrcPtr.getType();
1961 return false;
1962 }
1963
1964 QualType DestElemType = getElemType(P: DestPtr);
1965 if (DestElemType->isIncompleteType()) {
1966 S.FFDiag(SI: S.Current->getSource(PC: OpPC),
1967 DiagId: diag::note_constexpr_memcpy_incomplete_type)
1968 << Move << DestElemType;
1969 return false;
1970 }
1971
1972 size_t RemainingDestElems;
1973 if (DestPtr.getFieldDesc()->isArray()) {
1974 RemainingDestElems = DestPtr.isUnknownSizeArray()
1975 ? 0
1976 : (DestPtr.getNumElems() - DestPtr.getIndex());
1977 } else {
1978 RemainingDestElems = 1;
1979 }
1980 unsigned DestElemSize = ASTCtx.getTypeSizeInChars(T: DestElemType).getQuantity();
1981
1982 if (WChar) {
1983 uint64_t WCharSize =
1984 ASTCtx.getTypeSizeInChars(T: ASTCtx.getWCharType()).getQuantity();
1985 Size *= WCharSize;
1986 }
1987
1988 if (Size % DestElemSize != 0) {
1989 S.FFDiag(SI: S.Current->getSource(PC: OpPC),
1990 DiagId: diag::note_constexpr_memcpy_unsupported)
1991 << Move << WChar << 0 << DestElemType << Size << DestElemSize;
1992 return false;
1993 }
1994
1995 QualType SrcElemType = getElemType(P: SrcPtr);
1996 size_t RemainingSrcElems;
1997 if (SrcPtr.getFieldDesc()->isArray()) {
1998 RemainingSrcElems = SrcPtr.isUnknownSizeArray()
1999 ? 0
2000 : (SrcPtr.getNumElems() - SrcPtr.getIndex());
2001 } else {
2002 RemainingSrcElems = 1;
2003 }
2004 unsigned SrcElemSize = ASTCtx.getTypeSizeInChars(T: SrcElemType).getQuantity();
2005
2006 if (!ASTCtx.hasSameUnqualifiedType(T1: DestElemType, T2: SrcElemType)) {
2007 S.FFDiag(SI: S.Current->getSource(PC: OpPC), DiagId: diag::note_constexpr_memcpy_type_pun)
2008 << Move << SrcElemType << DestElemType;
2009 return false;
2010 }
2011
2012 if (!DestElemType.isTriviallyCopyableType(Context: ASTCtx)) {
2013 S.FFDiag(SI: S.Current->getSource(PC: OpPC), DiagId: diag::note_constexpr_memcpy_nontrivial)
2014 << Move << DestElemType;
2015 return false;
2016 }
2017
2018 // Check if we have enough elements to read from and write to.
2019 size_t RemainingDestBytes = RemainingDestElems * DestElemSize;
2020 size_t RemainingSrcBytes = RemainingSrcElems * SrcElemSize;
2021 if (Size > RemainingDestBytes || Size > RemainingSrcBytes) {
2022 APInt N = APInt(64, Size / DestElemSize);
2023 S.FFDiag(SI: S.Current->getSource(PC: OpPC),
2024 DiagId: diag::note_constexpr_memcpy_unsupported)
2025 << Move << WChar << (Size > RemainingSrcBytes ? 1 : 2) << DestElemType
2026 << toString(I: N, Radix: 10, /*Signed=*/false);
2027 return false;
2028 }
2029
2030 // Check for overlapping memory regions.
2031 if (!Move && Pointer::pointToSameBlock(A: SrcPtr, B: DestPtr)) {
2032 // Remove base casts.
2033 Pointer SrcP = SrcPtr.stripBaseCasts();
2034 Pointer DestP = DestPtr.stripBaseCasts();
2035
2036 unsigned SrcIndex = SrcP.expand().getIndex() * SrcElemSize;
2037 unsigned DstIndex = DestP.expand().getIndex() * DestElemSize;
2038
2039 if ((SrcIndex <= DstIndex && (SrcIndex + Size) > DstIndex) ||
2040 (DstIndex <= SrcIndex && (DstIndex + Size) > SrcIndex)) {
2041 S.FFDiag(SI: S.Current->getSource(PC: OpPC), DiagId: diag::note_constexpr_memcpy_overlap)
2042 << /*IsWChar=*/false;
2043 return false;
2044 }
2045 }
2046
2047 assert(Size % DestElemSize == 0);
2048 if (!DoMemcpy(S, OpPC, SrcPtr, DestPtr, Size: Bytes(Size).toBits()))
2049 return false;
2050
2051 S.Stk.push<Pointer>(Args&: DestPtr);
2052 return true;
2053}
2054
2055/// Determine if T is a character type for which we guarantee that
2056/// sizeof(T) == 1.
2057static bool isOneByteCharacterType(QualType T) {
2058 return T->isCharType() || T->isChar8Type();
2059}
2060
2061static bool interp__builtin_memcmp(InterpState &S, CodePtr OpPC,
2062 const InterpFrame *Frame,
2063 const CallExpr *Call, unsigned ID) {
2064 assert(Call->getNumArgs() == 3);
2065 uint64_t Size;
2066 if (!popToUInt64(S, E: Call->getArg(Arg: 2), Out&: Size))
2067 return false;
2068 const Pointer &PtrB = S.Stk.pop<Pointer>();
2069 const Pointer &PtrA = S.Stk.pop<Pointer>();
2070
2071 if (ID == Builtin::BImemcmp || ID == Builtin::BIbcmp ||
2072 ID == Builtin::BIwmemcmp)
2073 diagnoseNonConstexprBuiltin(S, OpPC, ID);
2074
2075 if (Size == 0) {
2076 pushInteger(S, Val: 0, QT: Call->getType());
2077 return true;
2078 }
2079
2080 if (!PtrA.isBlockPointer() || !PtrB.isBlockPointer())
2081 return false;
2082
2083 bool IsWide =
2084 (ID == Builtin::BIwmemcmp || ID == Builtin::BI__builtin_wmemcmp);
2085
2086 const ASTContext &ASTCtx = S.getASTContext();
2087 QualType ElemTypeA = getElemType(P: PtrA);
2088 QualType ElemTypeB = getElemType(P: PtrB);
2089 // FIXME: This is an arbitrary limitation the current constant interpreter
2090 // had. We could remove this.
2091 if (!IsWide && (!isOneByteCharacterType(T: ElemTypeA) ||
2092 !isOneByteCharacterType(T: ElemTypeB))) {
2093 S.FFDiag(SI: S.Current->getSource(PC: OpPC),
2094 DiagId: diag::note_constexpr_memcmp_unsupported)
2095 << ASTCtx.BuiltinInfo.getQuotedName(ID) << PtrA.getType()
2096 << PtrB.getType();
2097 return false;
2098 }
2099
2100 if (!CheckLoad(S, OpPC, Ptr: PtrA, AK: AK_Read) || !CheckLoad(S, OpPC, Ptr: PtrB, AK: AK_Read))
2101 return false;
2102
2103 // Now, read both pointers to a buffer and compare those.
2104 BitcastBuffer BufferA(
2105 Bits(ASTCtx.getTypeSize(T: ElemTypeA) * PtrA.getNumElems()));
2106 readPointerToBuffer(Ctx: S.getContext(), FromPtr: PtrA, Buffer&: BufferA, ReturnOnUninit: false);
2107
2108 // FIXME: The swapping here is UNDOING something we do when reading the
2109 // data into the buffer.
2110 if (ASTCtx.getTargetInfo().isBigEndian())
2111 swapBytes(M: BufferA.Data.get(), N: BufferA.byteSize().getQuantity());
2112
2113 BitcastBuffer BufferB(
2114 Bits(ASTCtx.getTypeSize(T: ElemTypeB) * PtrB.getNumElems()));
2115 readPointerToBuffer(Ctx: S.getContext(), FromPtr: PtrB, Buffer&: BufferB, ReturnOnUninit: false);
2116 // FIXME: The swapping here is UNDOING something we do when reading the
2117 // data into the buffer.
2118 if (ASTCtx.getTargetInfo().isBigEndian())
2119 swapBytes(M: BufferB.Data.get(), N: BufferB.byteSize().getQuantity());
2120
2121 size_t MinBufferSize = std::min(a: BufferA.byteSize().getQuantity(),
2122 b: BufferB.byteSize().getQuantity());
2123
2124 unsigned ElemSize = 1;
2125 if (IsWide)
2126 ElemSize = ASTCtx.getTypeSizeInChars(T: ASTCtx.getWCharType()).getQuantity();
2127 // The Size given for the wide variants is in wide-char units. Convert it
2128 // to bytes.
2129 size_t ByteSize = Size * ElemSize;
2130 size_t CmpSize = std::min(a: MinBufferSize, b: ByteSize);
2131
2132 for (size_t I = 0; I != CmpSize; I += ElemSize) {
2133 if (IsWide) {
2134 FIXED_SIZE_INT_TYPE_SWITCH(
2135 *S.getContext().classify(ASTCtx.getWCharType()), {
2136 T A = T::bitcastFromMemory(BufferA.atByte(I), T::bitWidth());
2137 T B = T::bitcastFromMemory(BufferB.atByte(I), T::bitWidth());
2138 if (A < B) {
2139 pushInteger(S, -1, Call->getType());
2140 return true;
2141 }
2142 if (A > B) {
2143 pushInteger(S, 1, Call->getType());
2144 return true;
2145 }
2146 });
2147 } else {
2148 auto A = BufferA.deref<std::byte>(Offset: Bytes(I));
2149 auto B = BufferB.deref<std::byte>(Offset: Bytes(I));
2150
2151 if (A < B) {
2152 pushInteger(S, Val: -1, QT: Call->getType());
2153 return true;
2154 }
2155 if (A > B) {
2156 pushInteger(S, Val: 1, QT: Call->getType());
2157 return true;
2158 }
2159 }
2160 }
2161
2162 // We compared CmpSize bytes above. If the limiting factor was the Size
2163 // passed, we're done and the result is equality (0).
2164 if (ByteSize <= CmpSize) {
2165 pushInteger(S, Val: 0, QT: Call->getType());
2166 return true;
2167 }
2168
2169 // However, if we read all the available bytes but were instructed to read
2170 // even more, diagnose this as a "read of dereferenced one-past-the-end
2171 // pointer". This is what would happen if we called CheckLoad() on every array
2172 // element.
2173 S.FFDiag(SI: S.Current->getSource(PC: OpPC), DiagId: diag::note_constexpr_access_past_end)
2174 << AK_Read << S.Current->getRange(PC: OpPC);
2175 return false;
2176}
2177
2178// __builtin_memchr(ptr, int, int)
2179// __builtin_strchr(ptr, int)
2180static bool interp__builtin_memchr(InterpState &S, CodePtr OpPC,
2181 const CallExpr *Call, unsigned ID) {
2182 if (ID == Builtin::BImemchr || ID == Builtin::BIwcschr ||
2183 ID == Builtin::BIstrchr || ID == Builtin::BIwmemchr)
2184 diagnoseNonConstexprBuiltin(S, OpPC, ID);
2185
2186 std::optional<APSInt> MaxLength;
2187 if (Call->getNumArgs() == 3) {
2188 APSInt MaxLengthVal;
2189 if (!popToAPSInt(S, E: Call->getArg(Arg: 2), Out&: MaxLengthVal))
2190 return false;
2191 MaxLength = MaxLengthVal;
2192 }
2193
2194 APSInt Desired;
2195 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: Desired))
2196 return false;
2197 const Pointer &Ptr = S.Stk.pop<Pointer>();
2198
2199 if (MaxLength && MaxLength->isZero()) {
2200 S.Stk.push<Pointer>();
2201 return true;
2202 }
2203
2204 if (Ptr.isDummy()) {
2205 if (Ptr.getType()->isIncompleteType())
2206 S.FFDiag(SI: S.Current->getSource(PC: OpPC),
2207 DiagId: diag::note_constexpr_ltor_incomplete_type)
2208 << Ptr.getType();
2209 return false;
2210 }
2211
2212 // Null is only okay if the given size is 0.
2213 if (Ptr.isZero()) {
2214 S.FFDiag(SI: S.Current->getSource(PC: OpPC), DiagId: diag::note_constexpr_access_null)
2215 << AK_Read;
2216 return false;
2217 }
2218
2219 if (!Ptr.isBlockPointer())
2220 return false;
2221
2222 QualType ElemTy = Ptr.getFieldDesc()->isArray()
2223 ? Ptr.getFieldDesc()->getElemQualType()
2224 : Ptr.getFieldDesc()->getType();
2225 bool IsRawByte = ID == Builtin::BImemchr || ID == Builtin::BI__builtin_memchr;
2226
2227 // Give up on byte-oriented matching against multibyte elements.
2228 if (IsRawByte && !isOneByteCharacterType(T: ElemTy)) {
2229 S.FFDiag(SI: S.Current->getSource(PC: OpPC),
2230 DiagId: diag::note_constexpr_memchr_unsupported)
2231 << S.getASTContext().BuiltinInfo.getQuotedName(ID) << ElemTy;
2232 return false;
2233 }
2234
2235 if (!isReadable(P: Ptr))
2236 return false;
2237
2238 if (ID == Builtin::BIstrchr || ID == Builtin::BI__builtin_strchr) {
2239 int64_t DesiredTrunc;
2240 if (S.getASTContext().CharTy->isSignedIntegerType())
2241 DesiredTrunc =
2242 Desired.trunc(width: S.getASTContext().getCharWidth()).getSExtValue();
2243 else
2244 DesiredTrunc =
2245 Desired.trunc(width: S.getASTContext().getCharWidth()).getZExtValue();
2246 // strchr compares directly to the passed integer, and therefore
2247 // always fails if given an int that is not a char.
2248 if (Desired != DesiredTrunc) {
2249 S.Stk.push<Pointer>();
2250 return true;
2251 }
2252 }
2253
2254 uint64_t DesiredVal;
2255 if (ID == Builtin::BIwmemchr || ID == Builtin::BI__builtin_wmemchr ||
2256 ID == Builtin::BIwcschr || ID == Builtin::BI__builtin_wcschr) {
2257 // wcschr and wmemchr are given a wchar_t to look for. Just use it.
2258 DesiredVal = Desired.getZExtValue();
2259 } else {
2260 DesiredVal = Desired.trunc(width: S.getASTContext().getCharWidth()).getZExtValue();
2261 }
2262
2263 bool StopAtZero =
2264 (ID == Builtin::BIstrchr || ID == Builtin::BI__builtin_strchr ||
2265 ID == Builtin::BIwcschr || ID == Builtin::BI__builtin_wcschr);
2266
2267 PrimType ElemT =
2268 IsRawByte ? PT_Sint8 : *S.getContext().classify(T: getElemType(P: Ptr));
2269
2270 size_t Index = Ptr.getIndex();
2271 size_t Step = 0;
2272 for (;;) {
2273 const Pointer &ElemPtr =
2274 (Index + Step) > 0 ? Ptr.atIndex(Idx: Index + Step) : Ptr;
2275
2276 if (!CheckLoad(S, OpPC, Ptr: ElemPtr))
2277 return false;
2278
2279 uint64_t V;
2280 INT_TYPE_SWITCH_NO_BOOL(
2281 ElemT, { V = static_cast<uint64_t>(ElemPtr.deref<T>().toUnsigned()); });
2282
2283 if (V == DesiredVal) {
2284 S.Stk.push<Pointer>(Args: ElemPtr);
2285 return true;
2286 }
2287
2288 if (StopAtZero && V == 0)
2289 break;
2290
2291 ++Step;
2292 if (MaxLength && Step == MaxLength->getZExtValue())
2293 break;
2294 }
2295
2296 S.Stk.push<Pointer>();
2297 return true;
2298}
2299
2300static std::optional<unsigned> computeFullDescSize(const ASTContext &ASTCtx,
2301 const Descriptor *Desc) {
2302 if (Desc->isPrimitive() || Desc->isArray())
2303 return ASTCtx.getTypeSizeInChars(T: Desc->getType()).getQuantity();
2304
2305 if (Desc->isRecord()) {
2306 // Can't use Descriptor::getType() as that may return a pointer type. Look
2307 // at the decl directly.
2308 return ASTCtx
2309 .getTypeSizeInChars(
2310 T: ASTCtx.getCanonicalTagType(TD: Desc->ElemRecord->getDecl()))
2311 .getQuantity();
2312 }
2313
2314 return std::nullopt;
2315}
2316
2317/// Compute the byte offset of \p Ptr in the full declaration.
2318static unsigned computePointerOffset(const ASTContext &ASTCtx,
2319 const Pointer &Ptr) {
2320 unsigned Result = 0;
2321
2322 Pointer P = Ptr;
2323 while (P.isField() || P.isArrayElement()) {
2324 P = P.expand();
2325 const Descriptor *D = P.getFieldDesc();
2326
2327 if (P.isArrayElement()) {
2328 unsigned ElemSize =
2329 ASTCtx.getTypeSizeInChars(T: D->getElemQualType()).getQuantity();
2330 if (P.isOnePastEnd())
2331 Result += ElemSize * P.getNumElems();
2332 else
2333 Result += ElemSize * P.getIndex();
2334 P = P.expand().getArray();
2335 } else if (P.isBaseClass()) {
2336 const auto *RD = cast<CXXRecordDecl>(Val: D->asDecl());
2337 bool IsVirtual = Ptr.isVirtualBaseClass();
2338 P = P.getBase();
2339 const Record *BaseRecord = P.getRecord();
2340
2341 const ASTRecordLayout &Layout =
2342 ASTCtx.getASTRecordLayout(D: cast<CXXRecordDecl>(Val: BaseRecord->getDecl()));
2343 if (IsVirtual)
2344 Result += Layout.getVBaseClassOffset(VBase: RD).getQuantity();
2345 else
2346 Result += Layout.getBaseClassOffset(Base: RD).getQuantity();
2347 } else if (P.isField()) {
2348 const FieldDecl *FD = P.getField();
2349 const ASTRecordLayout &Layout =
2350 ASTCtx.getASTRecordLayout(D: FD->getParent());
2351 unsigned FieldIndex = FD->getFieldIndex();
2352 uint64_t FieldOffset =
2353 ASTCtx.toCharUnitsFromBits(BitSize: Layout.getFieldOffset(FieldNo: FieldIndex))
2354 .getQuantity();
2355 Result += FieldOffset;
2356 P = P.getBase();
2357 } else
2358 llvm_unreachable("Unhandled descriptor type");
2359 }
2360
2361 return Result;
2362}
2363
2364/// Does Ptr point to the last subobject?
2365static bool pointsToLastObject(const Pointer &Ptr) {
2366 Pointer P = Ptr;
2367 while (!P.isRoot()) {
2368
2369 if (P.isArrayElement()) {
2370 P = P.expand().getArray();
2371 continue;
2372 }
2373 if (P.isBaseClass()) {
2374 if (P.getRecord()->getNumFields() > 0)
2375 return false;
2376 P = P.getBase();
2377 continue;
2378 }
2379
2380 Pointer Base = P.getBase();
2381 if (const Record *R = Base.getRecord()) {
2382 assert(P.getField());
2383 if (P.getField()->getFieldIndex() != R->getNumFields() - 1)
2384 return false;
2385 }
2386 P = Base;
2387 }
2388
2389 return true;
2390}
2391
2392/// Does Ptr point to the last object AND to a flexible array member?
2393static bool isUserWritingOffTheEnd(const ASTContext &Ctx, const Pointer &Ptr,
2394 bool InvalidBase) {
2395 auto isFlexibleArrayMember = [&](const Descriptor *FieldDesc) {
2396 using FAMKind = LangOptions::StrictFlexArraysLevelKind;
2397 FAMKind StrictFlexArraysLevel =
2398 Ctx.getLangOpts().getStrictFlexArraysLevel();
2399
2400 if (StrictFlexArraysLevel == FAMKind::Default)
2401 return true;
2402
2403 unsigned NumElems = FieldDesc->getNumElems();
2404 if (NumElems == 0 && StrictFlexArraysLevel != FAMKind::IncompleteOnly)
2405 return true;
2406
2407 if (NumElems == 1 && StrictFlexArraysLevel == FAMKind::OneZeroOrIncomplete)
2408 return true;
2409 return false;
2410 };
2411
2412 const Descriptor *FieldDesc = Ptr.getFieldDesc();
2413 if (!FieldDesc->isArray())
2414 return false;
2415
2416 return InvalidBase && pointsToLastObject(Ptr) &&
2417 isFlexibleArrayMember(FieldDesc);
2418}
2419
2420UnsignedOrNone evaluateBuiltinObjectSize(const ASTContext &ASTCtx,
2421 unsigned Kind, Pointer &Ptr) {
2422 if (Ptr.isZero() || !Ptr.isBlockPointer())
2423 return std::nullopt;
2424
2425 if (Ptr.isDummy() && Ptr.getType()->isPointerType())
2426 return std::nullopt;
2427
2428 bool InvalidBase = false;
2429
2430 if (Ptr.isDummy()) {
2431 if (const VarDecl *VD = Ptr.getDeclDesc()->asVarDecl();
2432 VD && VD->getType()->isPointerType())
2433 InvalidBase = true;
2434 }
2435
2436 // According to the GCC documentation, we want the size of the subobject
2437 // denoted by the pointer. But that's not quite right -- what we actually
2438 // want is the size of the immediately-enclosing array, if there is one.
2439 if (Ptr.isArrayElement())
2440 Ptr = Ptr.expand();
2441
2442 bool DetermineForCompleteObject = Ptr.getFieldDesc() == Ptr.getDeclDesc();
2443 const Descriptor *DeclDesc = Ptr.getDeclDesc();
2444 assert(DeclDesc);
2445
2446 bool UseFieldDesc = (Kind & 1u);
2447 bool ReportMinimum = (Kind & 2u);
2448 if (!UseFieldDesc || DetermineForCompleteObject) {
2449 // Can't read beyond the pointer decl desc.
2450 if (!ReportMinimum && DeclDesc->getType()->isPointerType())
2451 return std::nullopt;
2452
2453 if (InvalidBase)
2454 return std::nullopt;
2455 } else {
2456 if (isUserWritingOffTheEnd(Ctx: ASTCtx, Ptr, InvalidBase)) {
2457 // If we cannot determine the size of the initial allocation, then we
2458 // can't given an accurate upper-bound. However, we are still able to give
2459 // conservative lower-bounds for Type=3.
2460 if (Kind == 1)
2461 return std::nullopt;
2462 }
2463 // For Type=1, defer to the runtime path on a true incomplete-array
2464 // flexible array member (e.g. 'char fam[]') even when the base is a
2465 // concrete local/global. Without this, the bytecode interpreter would
2466 // happily fold &af.fam to 'NumElems * elemSize = 0' below; the default
2467 // const-evaluator avoids the same trap, and CGBuiltin emits
2468 // @llvm.objectsize for the correct layout-derived answer (matching
2469 // GCC's __bos/__bdos on '&af.fam').
2470 if (Kind == 1 && pointsToLastObject(Ptr) && Ptr.getFieldDesc()->isArray() &&
2471 Ptr.getFieldDesc()->getType()->isIncompleteArrayType())
2472 return std::nullopt;
2473 }
2474
2475 // The "closest surrounding subobject" is NOT a base class,
2476 // so strip the base class casts.
2477 if (UseFieldDesc && Ptr.isBaseClass())
2478 Ptr = Ptr.stripBaseCasts();
2479
2480 const Descriptor *Desc = UseFieldDesc ? Ptr.getFieldDesc() : DeclDesc;
2481 assert(Desc);
2482
2483 std::optional<unsigned> FullSize = computeFullDescSize(ASTCtx, Desc);
2484 if (!FullSize)
2485 return std::nullopt;
2486
2487 unsigned ByteOffset;
2488 if (UseFieldDesc) {
2489 if (Ptr.isBaseClass()) {
2490 assert(computePointerOffset(ASTCtx, Ptr.getBase()) <=
2491 computePointerOffset(ASTCtx, Ptr));
2492 ByteOffset = computePointerOffset(ASTCtx, Ptr: Ptr.getBase()) -
2493 computePointerOffset(ASTCtx, Ptr);
2494 } else {
2495 if (Ptr.inArray())
2496 ByteOffset =
2497 computePointerOffset(ASTCtx, Ptr) -
2498 computePointerOffset(ASTCtx, Ptr: Ptr.expand().atIndex(Idx: 0).narrow());
2499 else
2500 ByteOffset = 0;
2501 }
2502 } else
2503 ByteOffset = computePointerOffset(ASTCtx, Ptr);
2504
2505 assert(ByteOffset <= *FullSize);
2506 return *FullSize - ByteOffset;
2507}
2508
2509static bool interp__builtin_object_size(InterpState &S, CodePtr OpPC,
2510 const InterpFrame *Frame,
2511 const CallExpr *Call) {
2512 const ASTContext &ASTCtx = S.getASTContext();
2513 // From the GCC docs:
2514 // Kind is an integer constant from 0 to 3. If the least significant bit is
2515 // clear, objects are whole variables. If it is set, a closest surrounding
2516 // subobject is considered the object a pointer points to. The second bit
2517 // determines if maximum or minimum of remaining bytes is computed.
2518 uint64_t Kind;
2519 if (!popToUInt64(S, E: Call->getArg(Arg: 1), Out&: Kind))
2520 return false;
2521 assert(Kind <= 3 && "unexpected kind");
2522 Pointer Ptr = S.Stk.pop<Pointer>();
2523
2524 if (Call->getArg(Arg: 0)->HasSideEffects(Ctx: ASTCtx)) {
2525 // "If there are any side effects in them, it returns (size_t) -1
2526 // for type 0 or 1 and (size_t) 0 for type 2 or 3."
2527 pushInteger(S, Val: Kind <= 1 ? -1 : 0, QT: Call->getType());
2528 return true;
2529 }
2530
2531 if (auto Result = evaluateBuiltinObjectSize(ASTCtx, Kind, Ptr)) {
2532 pushInteger(S, Val: *Result, QT: Call->getType());
2533 return true;
2534 }
2535 return false;
2536}
2537
2538static bool interp__builtin_is_within_lifetime(InterpState &S, CodePtr OpPC,
2539 const CallExpr *Call) {
2540
2541 if (!S.inConstantContext())
2542 return false;
2543
2544 const Pointer &Ptr = S.Stk.pop<Pointer>();
2545
2546 auto Error = [&](int Diag) {
2547 bool CalledFromStd = false;
2548 const auto *Callee = S.Current->getCallee();
2549 if (Callee && Callee->isInStdNamespace()) {
2550 const IdentifierInfo *Identifier = Callee->getIdentifier();
2551 CalledFromStd = Identifier && Identifier->isStr(Str: "is_within_lifetime");
2552 }
2553 S.CCEDiag(SI: CalledFromStd
2554 ? S.Current->Caller->getSource(PC: S.Current->getRetPC())
2555 : S.Current->getSource(PC: OpPC),
2556 DiagId: diag::err_invalid_is_within_lifetime)
2557 << (CalledFromStd ? "std::is_within_lifetime"
2558 : "__builtin_is_within_lifetime")
2559 << Diag;
2560 return false;
2561 };
2562
2563 if (Ptr.isZero())
2564 return Error(0);
2565 if (Ptr.isOnePastEnd())
2566 return Error(1);
2567
2568 bool Result = Ptr.getLifetime() != Lifetime::Ended;
2569 if (!Ptr.isActive()) {
2570 Result = false;
2571 } else {
2572 if (!CheckLive(S, OpPC, Ptr, AK: AK_Read))
2573 return false;
2574 if (!CheckMutable(S, OpPC, Ptr))
2575 return false;
2576 if (!CheckDummy(S, OpPC, B: Ptr.block(), AK: AK_Read))
2577 return false;
2578 }
2579
2580 // Check if we're currently running an initializer.
2581 if (S.initializingBlock(B: Ptr.block()))
2582 return Error(2);
2583 if (S.EvaluatingDecl && Ptr.getDeclDesc()->asVarDecl() == S.EvaluatingDecl)
2584 return Error(2);
2585
2586 pushInteger(S, Val: Result, QT: Call->getType());
2587 return true;
2588}
2589
2590static bool interp__builtin_elementwise_int_unaryop(
2591 InterpState &S, CodePtr OpPC, const CallExpr *Call,
2592 llvm::function_ref<APInt(const APSInt &)> Fn) {
2593 assert(Call->getNumArgs() == 1);
2594
2595 // Single integer case.
2596 if (!Call->getArg(Arg: 0)->getType()->isVectorType()) {
2597 assert(Call->getType()->isIntegerType());
2598 APSInt Src;
2599 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: Src))
2600 return false;
2601 APInt Result = Fn(Src);
2602 pushInteger(S, Val: APSInt(std::move(Result), !Src.isSigned()), QT: Call->getType());
2603 return true;
2604 }
2605
2606 // Vector case.
2607 const Pointer &Arg = S.Stk.pop<Pointer>();
2608 assert(Arg.getFieldDesc()->isPrimitiveArray());
2609 const Pointer &Dst = S.Stk.peek<Pointer>();
2610 assert(Dst.getFieldDesc()->isPrimitiveArray());
2611 assert(Arg.getFieldDesc()->getNumElems() ==
2612 Dst.getFieldDesc()->getNumElems());
2613
2614 QualType ElemType = Arg.getFieldDesc()->getElemQualType();
2615 PrimType ElemT = *S.getContext().classify(T: ElemType);
2616 unsigned NumElems = Arg.getNumElems();
2617 bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
2618
2619 for (unsigned I = 0; I != NumElems; ++I) {
2620 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
2621 APSInt Src = Arg.elem<T>(I).toAPSInt();
2622 APInt Result = Fn(Src);
2623 Dst.elem<T>(I) = static_cast<T>(APSInt(std::move(Result), DestUnsigned));
2624 });
2625 }
2626 Dst.initializeAllElements();
2627
2628 return true;
2629}
2630
2631static bool interp__builtin_elementwise_fp_binop(
2632 InterpState &S, CodePtr OpPC, const CallExpr *Call,
2633 llvm::function_ref<std::optional<APFloat>(
2634 const APFloat &, const APFloat &, std::optional<APSInt> RoundingMode)>
2635 Fn,
2636 bool IsScalar = false) {
2637 assert((Call->getNumArgs() == 2) || (Call->getNumArgs() == 3));
2638 const auto *VT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
2639 assert(VT->getElementType()->isFloatingType());
2640 unsigned NumElems = VT->getNumElements();
2641
2642 // Vector case.
2643 assert(Call->getArg(0)->getType()->isVectorType() &&
2644 Call->getArg(1)->getType()->isVectorType());
2645 assert(VT->getElementType() ==
2646 Call->getArg(1)->getType()->castAs<VectorType>()->getElementType());
2647 assert(VT->getNumElements() ==
2648 Call->getArg(1)->getType()->castAs<VectorType>()->getNumElements());
2649
2650 std::optional<APSInt> RoundingMode = std::nullopt;
2651 if (Call->getNumArgs() == 3) {
2652 APSInt RoundingModeVal;
2653 if (!popToAPSInt(S, E: Call->getArg(Arg: 2), Out&: RoundingModeVal))
2654 return false;
2655 RoundingMode = RoundingModeVal;
2656 }
2657
2658 const Pointer &BPtr = S.Stk.pop<Pointer>();
2659 const Pointer &APtr = S.Stk.pop<Pointer>();
2660 const Pointer &Dst = S.Stk.peek<Pointer>();
2661 for (unsigned ElemIdx = 0; ElemIdx != NumElems; ++ElemIdx) {
2662 using T = PrimConv<PT_Float>::T;
2663 if (IsScalar && ElemIdx > 0) {
2664 Dst.elem<T>(I: ElemIdx) = APtr.elem<T>(I: ElemIdx);
2665 continue;
2666 }
2667 APFloat ElemA = APtr.elem<T>(I: ElemIdx).getAPFloat();
2668 APFloat ElemB = BPtr.elem<T>(I: ElemIdx).getAPFloat();
2669 std::optional<APFloat> Result = Fn(ElemA, ElemB, RoundingMode);
2670 if (!Result)
2671 return false;
2672 Dst.elem<T>(I: ElemIdx) = static_cast<T>(*Result);
2673 }
2674
2675 Dst.initializeAllElements();
2676
2677 return true;
2678}
2679
2680static bool interp__builtin_scalar_fp_round_mask_binop(
2681 InterpState &S, CodePtr OpPC, const CallExpr *Call,
2682 llvm::function_ref<std::optional<APFloat>(const APFloat &, const APFloat &,
2683 std::optional<APSInt>)>
2684 Fn) {
2685 assert(Call->getNumArgs() == 5);
2686 const auto *VT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
2687 unsigned NumElems = VT->getNumElements();
2688
2689 APSInt RoundingMode;
2690 if (!popToAPSInt(S, E: Call->getArg(Arg: 4), Out&: RoundingMode))
2691 return false;
2692 uint64_t MaskVal;
2693 if (!popToUInt64(S, E: Call->getArg(Arg: 3), Out&: MaskVal))
2694 return false;
2695 const Pointer &SrcPtr = S.Stk.pop<Pointer>();
2696 const Pointer &BPtr = S.Stk.pop<Pointer>();
2697 const Pointer &APtr = S.Stk.pop<Pointer>();
2698 const Pointer &Dst = S.Stk.peek<Pointer>();
2699
2700 using T = PrimConv<PT_Float>::T;
2701
2702 if (MaskVal & 1) {
2703 APFloat ElemA = APtr.elem<T>(I: 0).getAPFloat();
2704 APFloat ElemB = BPtr.elem<T>(I: 0).getAPFloat();
2705 std::optional<APFloat> Result = Fn(ElemA, ElemB, RoundingMode);
2706 if (!Result)
2707 return false;
2708 Dst.elem<T>(I: 0) = static_cast<T>(*Result);
2709 } else {
2710 Dst.elem<T>(I: 0) = SrcPtr.elem<T>(I: 0);
2711 }
2712
2713 for (unsigned I = 1; I < NumElems; ++I)
2714 Dst.elem<T>(I) = APtr.elem<T>(I);
2715
2716 Dst.initializeAllElements();
2717
2718 return true;
2719}
2720
2721static bool interp__builtin_elementwise_int_binop(
2722 InterpState &S, CodePtr OpPC, const CallExpr *Call,
2723 llvm::function_ref<APInt(const APSInt &, const APSInt &)> Fn) {
2724 assert(Call->getNumArgs() == 2);
2725
2726 // Single integer case.
2727 if (!Call->getArg(Arg: 0)->getType()->isVectorType()) {
2728 assert(!Call->getArg(1)->getType()->isVectorType());
2729 APSInt RHS;
2730 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: RHS))
2731 return false;
2732 APSInt LHS;
2733 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: LHS))
2734 return false;
2735 APInt Result = Fn(LHS, RHS);
2736 pushInteger(S, Val: APSInt(std::move(Result), !LHS.isSigned()), QT: Call->getType());
2737 return true;
2738 }
2739
2740 const auto *VT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
2741 assert(VT->getElementType()->isIntegralOrEnumerationType());
2742 PrimType ElemT = *S.getContext().classify(T: VT->getElementType());
2743 unsigned NumElems = VT->getNumElements();
2744 bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
2745
2746 // Vector + Scalar case.
2747 if (!Call->getArg(Arg: 1)->getType()->isVectorType()) {
2748 assert(Call->getArg(1)->getType()->isIntegralOrEnumerationType());
2749
2750 APSInt RHS;
2751 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: RHS))
2752 return false;
2753 const Pointer &LHS = S.Stk.pop<Pointer>();
2754 const Pointer &Dst = S.Stk.peek<Pointer>();
2755
2756 for (unsigned I = 0; I != NumElems; ++I) {
2757 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
2758 Dst.elem<T>(I) = static_cast<T>(
2759 APSInt(Fn(LHS.elem<T>(I).toAPSInt(), RHS), DestUnsigned));
2760 });
2761 }
2762 Dst.initializeAllElements();
2763 return true;
2764 }
2765
2766 // Vector case.
2767 assert(Call->getArg(0)->getType()->isVectorType() &&
2768 Call->getArg(1)->getType()->isVectorType());
2769 assert(VT->getElementType() ==
2770 Call->getArg(1)->getType()->castAs<VectorType>()->getElementType());
2771 assert(VT->getNumElements() ==
2772 Call->getArg(1)->getType()->castAs<VectorType>()->getNumElements());
2773 assert(VT->getElementType()->isIntegralOrEnumerationType());
2774
2775 const Pointer &RHS = S.Stk.pop<Pointer>();
2776 const Pointer &LHS = S.Stk.pop<Pointer>();
2777 const Pointer &Dst = S.Stk.peek<Pointer>();
2778 for (unsigned I = 0; I != NumElems; ++I) {
2779 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
2780 APSInt Elem1 = LHS.elem<T>(I).toAPSInt();
2781 APSInt Elem2 = RHS.elem<T>(I).toAPSInt();
2782 Dst.elem<T>(I) = static_cast<T>(APSInt(Fn(Elem1, Elem2), DestUnsigned));
2783 });
2784 }
2785 Dst.initializeAllElements();
2786
2787 return true;
2788}
2789
2790static bool
2791interp__builtin_ia32_pack(InterpState &S, CodePtr, const CallExpr *E,
2792 llvm::function_ref<APInt(const APSInt &)> PackFn) {
2793 const auto *VT0 = E->getArg(Arg: 0)->getType()->castAs<VectorType>();
2794 [[maybe_unused]] const auto *VT1 =
2795 E->getArg(Arg: 1)->getType()->castAs<VectorType>();
2796 assert(VT0 && VT1 && "pack builtin VT0 and VT1 must be VectorType");
2797 assert(VT0->getElementType() == VT1->getElementType() &&
2798 VT0->getNumElements() == VT1->getNumElements() &&
2799 "pack builtin VT0 and VT1 ElementType must be same");
2800
2801 const Pointer &RHS = S.Stk.pop<Pointer>();
2802 const Pointer &LHS = S.Stk.pop<Pointer>();
2803 const Pointer &Dst = S.Stk.peek<Pointer>();
2804
2805 const ASTContext &ASTCtx = S.getASTContext();
2806 unsigned SrcBits = ASTCtx.getIntWidth(T: VT0->getElementType());
2807 unsigned LHSVecLen = VT0->getNumElements();
2808 unsigned SrcPerLane = 128 / SrcBits;
2809 unsigned Lanes = LHSVecLen * SrcBits / 128;
2810
2811 PrimType SrcT = *S.getContext().classify(T: VT0->getElementType());
2812 PrimType DstT = *S.getContext().classify(T: getElemType(P: Dst));
2813 bool IsUnsigend = getElemType(P: Dst)->isUnsignedIntegerType();
2814
2815 for (unsigned Lane = 0; Lane != Lanes; ++Lane) {
2816 unsigned BaseSrc = Lane * SrcPerLane;
2817 unsigned BaseDst = Lane * (2 * SrcPerLane);
2818
2819 for (unsigned I = 0; I != SrcPerLane; ++I) {
2820 INT_TYPE_SWITCH_NO_BOOL(SrcT, {
2821 APSInt A = LHS.elem<T>(BaseSrc + I).toAPSInt();
2822 APSInt B = RHS.elem<T>(BaseSrc + I).toAPSInt();
2823
2824 assignIntegral(S, Dst.atIndex(BaseDst + I), DstT,
2825 APSInt(PackFn(A), IsUnsigend));
2826 assignIntegral(S, Dst.atIndex(BaseDst + SrcPerLane + I), DstT,
2827 APSInt(PackFn(B), IsUnsigend));
2828 });
2829 }
2830 }
2831
2832 Dst.initializeAllElements();
2833 return true;
2834}
2835
2836static bool interp__builtin_elementwise_maxmin(InterpState &S, CodePtr OpPC,
2837 const CallExpr *Call,
2838 unsigned BuiltinID) {
2839 assert(Call->getNumArgs() == 2);
2840
2841 QualType Arg0Type = Call->getArg(Arg: 0)->getType();
2842
2843 // TODO: Support floating-point types.
2844 if (!(Arg0Type->isIntegerType() ||
2845 (Arg0Type->isVectorType() &&
2846 Arg0Type->castAs<VectorType>()->getElementType()->isIntegerType())))
2847 return false;
2848
2849 if (!Arg0Type->isVectorType()) {
2850 assert(!Call->getArg(1)->getType()->isVectorType());
2851 APSInt RHS;
2852 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: RHS))
2853 return false;
2854 APSInt LHS;
2855 if (!popToAPSInt(S, T: Arg0Type, Out&: LHS))
2856 return false;
2857 APInt Result;
2858 if (BuiltinID == Builtin::BI__builtin_elementwise_max) {
2859 Result = std::max(a: LHS, b: RHS);
2860 } else if (BuiltinID == Builtin::BI__builtin_elementwise_min) {
2861 Result = std::min(a: LHS, b: RHS);
2862 } else {
2863 llvm_unreachable("Wrong builtin ID");
2864 }
2865
2866 pushInteger(S, Val: APSInt(Result, !LHS.isSigned()), QT: Call->getType());
2867 return true;
2868 }
2869
2870 // Vector case.
2871 assert(Call->getArg(0)->getType()->isVectorType() &&
2872 Call->getArg(1)->getType()->isVectorType());
2873 const auto *VT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
2874 assert(VT->getElementType() ==
2875 Call->getArg(1)->getType()->castAs<VectorType>()->getElementType());
2876 assert(VT->getNumElements() ==
2877 Call->getArg(1)->getType()->castAs<VectorType>()->getNumElements());
2878 assert(VT->getElementType()->isIntegralOrEnumerationType());
2879
2880 const Pointer &RHS = S.Stk.pop<Pointer>();
2881 const Pointer &LHS = S.Stk.pop<Pointer>();
2882 const Pointer &Dst = S.Stk.peek<Pointer>();
2883 PrimType ElemT = *S.getContext().classify(T: VT->getElementType());
2884 unsigned NumElems = VT->getNumElements();
2885 for (unsigned I = 0; I != NumElems; ++I) {
2886 APSInt Elem1;
2887 APSInt Elem2;
2888 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
2889 Elem1 = LHS.elem<T>(I).toAPSInt();
2890 Elem2 = RHS.elem<T>(I).toAPSInt();
2891 });
2892
2893 APSInt Result;
2894 if (BuiltinID == Builtin::BI__builtin_elementwise_max) {
2895 Result = APSInt(std::max(a: Elem1, b: Elem2),
2896 Call->getType()->isUnsignedIntegerOrEnumerationType());
2897 } else if (BuiltinID == Builtin::BI__builtin_elementwise_min) {
2898 Result = APSInt(std::min(a: Elem1, b: Elem2),
2899 Call->getType()->isUnsignedIntegerOrEnumerationType());
2900 } else {
2901 llvm_unreachable("Wrong builtin ID");
2902 }
2903
2904 INT_TYPE_SWITCH_NO_BOOL(ElemT,
2905 { Dst.elem<T>(I) = static_cast<T>(Result); });
2906 }
2907 Dst.initializeAllElements();
2908
2909 return true;
2910}
2911
2912static bool interp__builtin_ia32_pmul(
2913 InterpState &S, CodePtr OpPC, const CallExpr *Call,
2914 llvm::function_ref<APInt(const APSInt &, const APSInt &, const APSInt &,
2915 const APSInt &)>
2916 Fn) {
2917 assert(Call->getArg(0)->getType()->isVectorType() &&
2918 Call->getArg(1)->getType()->isVectorType());
2919 const Pointer &RHS = S.Stk.pop<Pointer>();
2920 const Pointer &LHS = S.Stk.pop<Pointer>();
2921 const Pointer &Dst = S.Stk.peek<Pointer>();
2922
2923 const auto *VT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
2924 PrimType ElemT = *S.getContext().classify(T: VT->getElementType());
2925 unsigned NumElems = VT->getNumElements();
2926 const auto *DestVT = Call->getType()->castAs<VectorType>();
2927 PrimType DestElemT = *S.getContext().classify(T: DestVT->getElementType());
2928 bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
2929
2930 unsigned DstElem = 0;
2931 for (unsigned I = 0; I != NumElems; I += 2) {
2932 APSInt Result;
2933 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
2934 APSInt LoLHS = LHS.elem<T>(I).toAPSInt();
2935 APSInt HiLHS = LHS.elem<T>(I + 1).toAPSInt();
2936 APSInt LoRHS = RHS.elem<T>(I).toAPSInt();
2937 APSInt HiRHS = RHS.elem<T>(I + 1).toAPSInt();
2938 Result = APSInt(Fn(LoLHS, HiLHS, LoRHS, HiRHS), DestUnsigned);
2939 });
2940
2941 INT_TYPE_SWITCH_NO_BOOL(DestElemT,
2942 { Dst.elem<T>(DstElem) = static_cast<T>(Result); });
2943 ++DstElem;
2944 }
2945
2946 Dst.initializeAllElements();
2947 return true;
2948}
2949
2950static bool interp__builtin_ia32_psadbw(InterpState &S, CodePtr OpPC,
2951 const CallExpr *Call) {
2952 assert(Call->getNumArgs() == 2);
2953
2954 const Pointer &RHS = S.Stk.pop<Pointer>();
2955 const Pointer &LHS = S.Stk.pop<Pointer>();
2956 const Pointer &Dst = S.Stk.peek<Pointer>();
2957
2958 const auto *SrcVT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
2959 PrimType SrcElemT = *S.getContext().classify(T: SrcVT->getElementType());
2960 unsigned SourceLen = SrcVT->getNumElements();
2961 assert((SourceLen % 8) == 0);
2962
2963 const auto *DestVT = Call->getType()->castAs<VectorType>();
2964 PrimType DestElemT = *S.getContext().classify(T: DestVT->getElementType());
2965 bool DestUnsigned =
2966 DestVT->getElementType()->isUnsignedIntegerOrEnumerationType();
2967
2968 unsigned DstElem = 0;
2969 for (unsigned Lane = 0; Lane != SourceLen; Lane += 8) {
2970 APInt Sum(64, 0);
2971 for (unsigned I = 0; I != 8; ++I) {
2972 INT_TYPE_SWITCH_NO_BOOL(SrcElemT, {
2973 APSInt L = LHS.elem<T>(Lane + I).toAPSInt();
2974 APSInt R = RHS.elem<T>(Lane + I).toAPSInt();
2975 Sum += llvm::APIntOps::abdu(L.extOrTrunc(8), R.extOrTrunc(8)).zext(64);
2976 });
2977 }
2978
2979 INT_TYPE_SWITCH_NO_BOOL(DestElemT, {
2980 Dst.elem<T>(DstElem) = static_cast<T>(APSInt(Sum, DestUnsigned));
2981 });
2982 ++DstElem;
2983 }
2984
2985 Dst.initializeAllElements();
2986 return true;
2987}
2988
2989static bool interp__builtin_ia32_dbpsadbw(InterpState &S, CodePtr OpPC,
2990 const CallExpr *Call) {
2991 assert(Call->getNumArgs() == 3);
2992 uint64_t Imm;
2993 if (!popToUInt64(S, E: Call->getArg(Arg: 2), Out&: Imm))
2994 return false;
2995
2996 const Pointer &Src2 = S.Stk.pop<Pointer>();
2997 const Pointer &Src1 = S.Stk.pop<Pointer>();
2998 const Pointer &Dst = S.Stk.peek<Pointer>();
2999
3000 const auto *SrcVT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
3001 PrimType SrcElemT = *S.getContext().classify(T: SrcVT->getElementType());
3002 unsigned SourceLen = SrcVT->getNumElements();
3003
3004 const auto *DestVT = Call->getType()->castAs<VectorType>();
3005 PrimType DestElemT = *S.getContext().classify(T: DestVT->getElementType());
3006 bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
3007
3008 constexpr unsigned LaneSize = 16; // 128-bit lane = 16 bytes
3009
3010 // Phase 1: Shuffle Src2 using all four 2-bit fields of imm8.
3011 // Within each 128-bit lane, for group j (0..3), select a 4-byte block
3012 // from Src2 based on bits [2*j+1:2*j] of imm8.
3013 SmallVector<uint8_t, 64> Shuffled(SourceLen);
3014 for (unsigned I = 0; I < SourceLen; I += LaneSize) {
3015 for (unsigned J = 0; J < 4; ++J) {
3016 unsigned Part = (Imm >> (2 * J)) & 3;
3017 for (unsigned K = 0; K < 4; ++K) {
3018 INT_TYPE_SWITCH_NO_BOOL(SrcElemT, {
3019 Shuffled[I + 4 * J + K] =
3020 static_cast<uint8_t>(Src2.elem<T>(I + 4 * Part + K));
3021 });
3022 }
3023 }
3024 }
3025
3026 // Phase 2: Sliding SAD computation.
3027 // For every group of 4 output u16 values, compute absolute differences
3028 // using overlapping windows into Src1 and the shuffled array.
3029 unsigned Size = SourceLen / 2; // number of output u16 elements
3030 for (unsigned I = 0; I < Size; I += 4) {
3031 unsigned Sad[4] = {0, 0, 0, 0};
3032 for (unsigned J = 0; J < 4; ++J) {
3033 uint8_t A1, A2;
3034 INT_TYPE_SWITCH_NO_BOOL(SrcElemT, {
3035 A1 = static_cast<uint8_t>(Src1.elem<T>(2 * I + J));
3036 A2 = static_cast<uint8_t>(Src1.elem<T>(2 * I + J + 4));
3037 });
3038 uint8_t B0 = Shuffled[2 * I + J];
3039 uint8_t B1 = Shuffled[2 * I + J + 1];
3040 uint8_t B2 = Shuffled[2 * I + J + 2];
3041 uint8_t B3 = Shuffled[2 * I + J + 3];
3042 Sad[0] += (A1 > B0) ? (A1 - B0) : (B0 - A1);
3043 Sad[1] += (A1 > B1) ? (A1 - B1) : (B1 - A1);
3044 Sad[2] += (A2 > B2) ? (A2 - B2) : (B2 - A2);
3045 Sad[3] += (A2 > B3) ? (A2 - B3) : (B3 - A2);
3046 }
3047 for (unsigned R = 0; R < 4; ++R) {
3048 INT_TYPE_SWITCH_NO_BOOL(DestElemT, {
3049 Dst.elem<T>(I + R) =
3050 static_cast<T>(APSInt(APInt(16, Sad[R]), DestUnsigned));
3051 });
3052 }
3053 }
3054
3055 Dst.initializeAllElements();
3056 return true;
3057}
3058
3059static bool interp__builtin_ia32_mpsadbw(InterpState &S, CodePtr OpPC,
3060 const CallExpr *Call) {
3061 assert(Call->getNumArgs() == 3);
3062 uint64_t Imm;
3063 if (!popToUInt64(S, E: Call->getArg(Arg: 2), Out&: Imm))
3064 return false;
3065
3066 const Pointer &Src2 = S.Stk.pop<Pointer>();
3067 const Pointer &Src1 = S.Stk.pop<Pointer>();
3068 const Pointer &Dst = S.Stk.peek<Pointer>();
3069
3070 const auto *SrcVT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
3071 PrimType SrcElemT = *S.getContext().classify(T: SrcVT->getElementType());
3072 unsigned SourceLen = SrcVT->getNumElements();
3073 assert((SourceLen == 16 || SourceLen == 32) &&
3074 "MPSADBW operates on 128-bit or 256-bit vectors");
3075
3076 const auto *DestVT = Call->getType()->castAs<VectorType>();
3077 PrimType DestElemT = *S.getContext().classify(T: DestVT->getElementType());
3078 bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
3079
3080 constexpr unsigned LaneSize = 16; // 128-bit lane = 16 bytes
3081 unsigned NumLanes = SourceLen / LaneSize;
3082
3083 for (unsigned Lane = 0; Lane != NumLanes; ++Lane) {
3084 unsigned Ctrl = (Imm >> (3 * Lane)) & 0x7;
3085 unsigned AOff = ((Ctrl >> 2) & 1) * 4;
3086 unsigned BOff = (Ctrl & 3) * 4;
3087 for (unsigned J = 0; J != 8; ++J) {
3088 uint16_t Sad = 0;
3089 for (unsigned K = 0; K != 4; ++K) {
3090 uint8_t A, B;
3091 INT_TYPE_SWITCH_NO_BOOL(SrcElemT, {
3092 A = static_cast<uint8_t>(
3093 Src1.elem<T>(Lane * LaneSize + AOff + J + K));
3094 B = static_cast<uint8_t>(Src2.elem<T>(Lane * LaneSize + BOff + K));
3095 });
3096 Sad += (A > B) ? (A - B) : (B - A);
3097 }
3098 INT_TYPE_SWITCH_NO_BOOL(DestElemT, {
3099 Dst.elem<T>(Lane * 8 + J) =
3100 static_cast<T>(APSInt(APInt(16, Sad), DestUnsigned));
3101 });
3102 }
3103 }
3104
3105 Dst.initializeAllElements();
3106 return true;
3107}
3108
3109static bool interp_builtin_horizontal_int_binop(
3110 InterpState &S, CodePtr OpPC, const CallExpr *Call,
3111 llvm::function_ref<APInt(const APSInt &, const APSInt &)> Fn) {
3112 const auto *VT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
3113 PrimType ElemT = *S.getContext().classify(T: VT->getElementType());
3114 bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
3115
3116 const Pointer &RHS = S.Stk.pop<Pointer>();
3117 const Pointer &LHS = S.Stk.pop<Pointer>();
3118 const Pointer &Dst = S.Stk.peek<Pointer>();
3119 unsigned NumElts = VT->getNumElements();
3120 unsigned EltBits = S.getASTContext().getIntWidth(T: VT->getElementType());
3121 unsigned EltsPerLane = 128 / EltBits;
3122 unsigned Lanes = NumElts * EltBits / 128;
3123 unsigned DestIndex = 0;
3124
3125 for (unsigned Lane = 0; Lane < Lanes; ++Lane) {
3126 unsigned LaneStart = Lane * EltsPerLane;
3127 for (unsigned I = 0; I < EltsPerLane; I += 2) {
3128 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
3129 APSInt Elem1 = LHS.elem<T>(LaneStart + I).toAPSInt();
3130 APSInt Elem2 = LHS.elem<T>(LaneStart + I + 1).toAPSInt();
3131 APSInt ResL = APSInt(Fn(Elem1, Elem2), DestUnsigned);
3132 Dst.elem<T>(DestIndex++) = static_cast<T>(ResL);
3133 });
3134 }
3135
3136 for (unsigned I = 0; I < EltsPerLane; I += 2) {
3137 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
3138 APSInt Elem1 = RHS.elem<T>(LaneStart + I).toAPSInt();
3139 APSInt Elem2 = RHS.elem<T>(LaneStart + I + 1).toAPSInt();
3140 APSInt ResR = APSInt(Fn(Elem1, Elem2), DestUnsigned);
3141 Dst.elem<T>(DestIndex++) = static_cast<T>(ResR);
3142 });
3143 }
3144 }
3145 Dst.initializeAllElements();
3146 return true;
3147}
3148
3149static bool interp_builtin_horizontal_fp_binop(
3150 InterpState &S, CodePtr OpPC, const CallExpr *Call,
3151 llvm::function_ref<APFloat(const APFloat &, const APFloat &,
3152 llvm::RoundingMode)>
3153 Fn) {
3154 const Pointer &RHS = S.Stk.pop<Pointer>();
3155 const Pointer &LHS = S.Stk.pop<Pointer>();
3156 const Pointer &Dst = S.Stk.peek<Pointer>();
3157 FPOptions FPO = Call->getFPFeaturesInEffect(LO: S.Ctx.getLangOpts());
3158 llvm::RoundingMode RM = getRoundingMode(FPO);
3159 const auto *VT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
3160
3161 unsigned NumElts = VT->getNumElements();
3162 unsigned EltBits = S.getASTContext().getTypeSize(T: VT->getElementType());
3163 unsigned NumLanes = NumElts * EltBits / 128;
3164 unsigned NumElemsPerLane = NumElts / NumLanes;
3165 unsigned HalfElemsPerLane = NumElemsPerLane / 2;
3166
3167 for (unsigned L = 0; L != NumElts; L += NumElemsPerLane) {
3168 using T = PrimConv<PT_Float>::T;
3169 for (unsigned E = 0; E != HalfElemsPerLane; ++E) {
3170 APFloat Elem1 = LHS.elem<T>(I: L + (2 * E) + 0).getAPFloat();
3171 APFloat Elem2 = LHS.elem<T>(I: L + (2 * E) + 1).getAPFloat();
3172 Dst.elem<T>(I: L + E) = static_cast<T>(Fn(Elem1, Elem2, RM));
3173 }
3174 for (unsigned E = 0; E != HalfElemsPerLane; ++E) {
3175 APFloat Elem1 = RHS.elem<T>(I: L + (2 * E) + 0).getAPFloat();
3176 APFloat Elem2 = RHS.elem<T>(I: L + (2 * E) + 1).getAPFloat();
3177 Dst.elem<T>(I: L + E + HalfElemsPerLane) =
3178 static_cast<T>(Fn(Elem1, Elem2, RM));
3179 }
3180 }
3181 Dst.initializeAllElements();
3182 return true;
3183}
3184
3185static bool interp__builtin_ia32_addsub(InterpState &S, CodePtr OpPC,
3186 const CallExpr *Call) {
3187 // Addsub: alternates between subtraction and addition
3188 // Result[i] = (i % 2 == 0) ? (a[i] - b[i]) : (a[i] + b[i])
3189 const Pointer &RHS = S.Stk.pop<Pointer>();
3190 const Pointer &LHS = S.Stk.pop<Pointer>();
3191 const Pointer &Dst = S.Stk.peek<Pointer>();
3192 FPOptions FPO = Call->getFPFeaturesInEffect(LO: S.Ctx.getLangOpts());
3193 llvm::RoundingMode RM = getRoundingMode(FPO);
3194 const auto *VT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
3195 unsigned NumElems = VT->getNumElements();
3196
3197 using T = PrimConv<PT_Float>::T;
3198 for (unsigned I = 0; I != NumElems; ++I) {
3199 APFloat LElem = LHS.elem<T>(I).getAPFloat();
3200 APFloat RElem = RHS.elem<T>(I).getAPFloat();
3201 if (I % 2 == 0) {
3202 // Even indices: subtract
3203 LElem.subtract(RHS: RElem, RM);
3204 } else {
3205 // Odd indices: add
3206 LElem.add(RHS: RElem, RM);
3207 }
3208 Dst.elem<T>(I) = static_cast<T>(LElem);
3209 }
3210 Dst.initializeAllElements();
3211 return true;
3212}
3213
3214static bool interp__builtin_ia32_pclmulqdq(InterpState &S, CodePtr OpPC,
3215 const CallExpr *Call) {
3216 // PCLMULQDQ: carry-less multiplication of selected 64-bit halves
3217 // imm8 bit 0: selects lower (0) or upper (1) 64 bits of first operand
3218 // imm8 bit 4: selects lower (0) or upper (1) 64 bits of second operand
3219 assert(Call->getArg(0)->getType()->isVectorType() &&
3220 Call->getArg(1)->getType()->isVectorType());
3221
3222 // Extract imm8 argument
3223 APSInt Imm8;
3224 if (!popToAPSInt(S, E: Call->getArg(Arg: 2), Out&: Imm8))
3225 return false;
3226 bool SelectUpperA = (Imm8 & 0x01) != 0;
3227 bool SelectUpperB = (Imm8 & 0x10) != 0;
3228
3229 const Pointer &RHS = S.Stk.pop<Pointer>();
3230 const Pointer &LHS = S.Stk.pop<Pointer>();
3231 const Pointer &Dst = S.Stk.peek<Pointer>();
3232
3233 const auto *VT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
3234 PrimType ElemT = *S.getContext().classify(T: VT->getElementType());
3235 unsigned NumElems = VT->getNumElements();
3236 const auto *DestVT = Call->getType()->castAs<VectorType>();
3237 PrimType DestElemT = *S.getContext().classify(T: DestVT->getElementType());
3238 bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
3239
3240 // Process each 128-bit lane (2 elements at a time)
3241 for (unsigned Lane = 0; Lane < NumElems; Lane += 2) {
3242 APSInt A0, A1, B0, B1;
3243 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
3244 A0 = LHS.elem<T>(Lane + 0).toAPSInt();
3245 A1 = LHS.elem<T>(Lane + 1).toAPSInt();
3246 B0 = RHS.elem<T>(Lane + 0).toAPSInt();
3247 B1 = RHS.elem<T>(Lane + 1).toAPSInt();
3248 });
3249
3250 // Select the appropriate 64-bit values based on imm8
3251 APInt A = SelectUpperA ? A1 : A0;
3252 APInt B = SelectUpperB ? B1 : B0;
3253
3254 // Extend both operands to 128 bits for carry-less multiplication
3255 APInt A128 = A.zext(width: 128);
3256 APInt B128 = B.zext(width: 128);
3257
3258 // Use APIntOps::clmul for carry-less multiplication
3259 APInt Result = llvm::APIntOps::clmul(LHS: A128, RHS: B128);
3260
3261 // Split the 128-bit result into two 64-bit halves
3262 APSInt ResultLow(Result.extractBits(numBits: 64, bitPosition: 0), DestUnsigned);
3263 APSInt ResultHigh(Result.extractBits(numBits: 64, bitPosition: 64), DestUnsigned);
3264
3265 INT_TYPE_SWITCH_NO_BOOL(DestElemT, {
3266 Dst.elem<T>(Lane + 0) = static_cast<T>(ResultLow);
3267 Dst.elem<T>(Lane + 1) = static_cast<T>(ResultHigh);
3268 });
3269 }
3270
3271 Dst.initializeAllElements();
3272 return true;
3273}
3274
3275static bool interp__builtin_elementwise_triop_fp(
3276 InterpState &S, CodePtr OpPC, const CallExpr *Call,
3277 llvm::function_ref<APFloat(const APFloat &, const APFloat &,
3278 const APFloat &, llvm::RoundingMode)>
3279 Fn) {
3280 assert(Call->getNumArgs() == 3);
3281
3282 FPOptions FPO = Call->getFPFeaturesInEffect(LO: S.Ctx.getLangOpts());
3283 llvm::RoundingMode RM = getRoundingMode(FPO);
3284 QualType Arg1Type = Call->getArg(Arg: 0)->getType();
3285 QualType Arg2Type = Call->getArg(Arg: 1)->getType();
3286 QualType Arg3Type = Call->getArg(Arg: 2)->getType();
3287
3288 // Non-vector floating point types.
3289 if (!Arg1Type->isVectorType()) {
3290 assert(!Arg2Type->isVectorType());
3291 assert(!Arg3Type->isVectorType());
3292 (void)Arg2Type;
3293 (void)Arg3Type;
3294
3295 const Floating &Z = S.Stk.pop<Floating>();
3296 const Floating &Y = S.Stk.pop<Floating>();
3297 const Floating &X = S.Stk.pop<Floating>();
3298 APFloat F = Fn(X.getAPFloat(), Y.getAPFloat(), Z.getAPFloat(), RM);
3299 Floating Result = S.allocFloat(Sem: X.getSemantics());
3300 Result.copy(F);
3301 S.Stk.push<Floating>(Args&: Result);
3302 return true;
3303 }
3304
3305 // Vector type.
3306 assert(Arg1Type->isVectorType() && Arg2Type->isVectorType() &&
3307 Arg3Type->isVectorType());
3308
3309 const VectorType *VecTy = Arg1Type->castAs<VectorType>();
3310 QualType ElemQT = VecTy->getElementType();
3311 unsigned NumElems = VecTy->getNumElements();
3312
3313 assert(ElemQT == Arg2Type->castAs<VectorType>()->getElementType() &&
3314 ElemQT == Arg3Type->castAs<VectorType>()->getElementType());
3315 assert(NumElems == Arg2Type->castAs<VectorType>()->getNumElements() &&
3316 NumElems == Arg3Type->castAs<VectorType>()->getNumElements());
3317 assert(ElemQT->isRealFloatingType());
3318 (void)ElemQT;
3319
3320 const Pointer &VZ = S.Stk.pop<Pointer>();
3321 const Pointer &VY = S.Stk.pop<Pointer>();
3322 const Pointer &VX = S.Stk.pop<Pointer>();
3323 const Pointer &Dst = S.Stk.peek<Pointer>();
3324 for (unsigned I = 0; I != NumElems; ++I) {
3325 using T = PrimConv<PT_Float>::T;
3326 APFloat X = VX.elem<T>(I).getAPFloat();
3327 APFloat Y = VY.elem<T>(I).getAPFloat();
3328 APFloat Z = VZ.elem<T>(I).getAPFloat();
3329 APFloat F = Fn(X, Y, Z, RM);
3330 Dst.elem<Floating>(I) = Floating(F);
3331 }
3332 Dst.initializeAllElements();
3333 return true;
3334}
3335
3336/// AVX512 predicated move: "Result = Mask[] ? LHS[] : RHS[]".
3337static bool interp__builtin_ia32_select(InterpState &S, CodePtr OpPC,
3338 const CallExpr *Call) {
3339 const Pointer &RHS = S.Stk.pop<Pointer>();
3340 const Pointer &LHS = S.Stk.pop<Pointer>();
3341 APSInt Mask;
3342 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: Mask))
3343 return false;
3344 const Pointer &Dst = S.Stk.peek<Pointer>();
3345
3346 assert(LHS.getNumElems() == RHS.getNumElems());
3347 assert(LHS.getNumElems() == Dst.getNumElems());
3348 unsigned NumElems = LHS.getNumElems();
3349 PrimType ElemT = LHS.getFieldDesc()->getPrimType();
3350 PrimType DstElemT = Dst.getFieldDesc()->getPrimType();
3351
3352 for (unsigned I = 0; I != NumElems; ++I) {
3353 if (ElemT == PT_Float) {
3354 assert(DstElemT == PT_Float);
3355 Dst.elem<Floating>(I) =
3356 Mask[I] ? LHS.elem<Floating>(I) : RHS.elem<Floating>(I);
3357 } else {
3358 APSInt Elem;
3359 INT_TYPE_SWITCH(ElemT, {
3360 Elem = Mask[I] ? LHS.elem<T>(I).toAPSInt() : RHS.elem<T>(I).toAPSInt();
3361 });
3362 INT_TYPE_SWITCH_NO_BOOL(DstElemT,
3363 { Dst.elem<T>(I) = static_cast<T>(Elem); });
3364 }
3365 }
3366 Dst.initializeAllElements();
3367
3368 return true;
3369}
3370
3371/// Scalar variant of AVX512 predicated select:
3372/// Result[i] = (Mask bit 0) ? LHS[i] : RHS[i], but only element 0 may change.
3373/// All other elements are taken from RHS.
3374static bool interp__builtin_ia32_select_scalar(InterpState &S,
3375 const CallExpr *Call) {
3376 unsigned N =
3377 Call->getArg(Arg: 1)->getType()->castAs<VectorType>()->getNumElements();
3378
3379 const Pointer &W = S.Stk.pop<Pointer>();
3380 const Pointer &A = S.Stk.pop<Pointer>();
3381 APSInt U;
3382 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: U))
3383 return false;
3384 const Pointer &Dst = S.Stk.peek<Pointer>();
3385
3386 bool TakeA0 = U.getZExtValue() & 1ULL;
3387
3388 for (unsigned I = TakeA0; I != N; ++I)
3389 Dst.elem<Floating>(I) = W.elem<Floating>(I);
3390 if (TakeA0)
3391 Dst.elem<Floating>(I: 0) = A.elem<Floating>(I: 0);
3392
3393 Dst.initializeAllElements();
3394 return true;
3395}
3396
3397static bool interp__builtin_ia32_test_op(
3398 InterpState &S, CodePtr OpPC, const CallExpr *Call,
3399 llvm::function_ref<bool(const APInt &A, const APInt &B)> Fn) {
3400 const Pointer &RHS = S.Stk.pop<Pointer>();
3401 const Pointer &LHS = S.Stk.pop<Pointer>();
3402
3403 assert(LHS.getNumElems() == RHS.getNumElems());
3404
3405 unsigned SourceLen = LHS.getNumElems();
3406 QualType ElemQT = getElemType(P: LHS);
3407 OptPrimType ElemPT = S.getContext().classify(T: ElemQT);
3408 unsigned LaneWidth = S.getASTContext().getTypeSize(T: ElemQT);
3409
3410 APInt AWide(LaneWidth * SourceLen, 0);
3411 APInt BWide(LaneWidth * SourceLen, 0);
3412
3413 for (unsigned I = 0; I != SourceLen; ++I) {
3414 APInt ALane;
3415 APInt BLane;
3416
3417 if (ElemQT->isIntegerType()) { // Get value.
3418 INT_TYPE_SWITCH_NO_BOOL(*ElemPT, {
3419 ALane = LHS.elem<T>(I).toAPSInt();
3420 BLane = RHS.elem<T>(I).toAPSInt();
3421 });
3422 } else if (ElemQT->isFloatingType()) { // Get only sign bit.
3423 using T = PrimConv<PT_Float>::T;
3424 ALane = LHS.elem<T>(I).getAPFloat().bitcastToAPInt().isNegative();
3425 BLane = RHS.elem<T>(I).getAPFloat().bitcastToAPInt().isNegative();
3426 } else { // Must be integer or floating type.
3427 return false;
3428 }
3429 AWide.insertBits(SubBits: ALane, bitPosition: I * LaneWidth);
3430 BWide.insertBits(SubBits: BLane, bitPosition: I * LaneWidth);
3431 }
3432 pushInteger(S, Val: Fn(AWide, BWide), QT: Call->getType());
3433 return true;
3434}
3435
3436static bool interp__builtin_ia32_movmsk_op(InterpState &S, CodePtr OpPC,
3437 const CallExpr *Call) {
3438 assert(Call->getNumArgs() == 1);
3439
3440 const Pointer &Source = S.Stk.pop<Pointer>();
3441
3442 unsigned SourceLen = Source.getNumElems();
3443 QualType ElemQT = getElemType(P: Source);
3444 OptPrimType ElemT = S.getContext().classify(T: ElemQT);
3445 unsigned ResultLen =
3446 S.getASTContext().getTypeSize(T: Call->getType()); // Always 32-bit integer.
3447 APInt Result(ResultLen, 0);
3448
3449 for (unsigned I = 0; I != SourceLen; ++I) {
3450 APInt Elem;
3451 if (ElemQT->isIntegerType()) {
3452 INT_TYPE_SWITCH_NO_BOOL(*ElemT, { Elem = Source.elem<T>(I).toAPSInt(); });
3453 } else if (ElemQT->isRealFloatingType()) {
3454 using T = PrimConv<PT_Float>::T;
3455 Elem = Source.elem<T>(I).getAPFloat().bitcastToAPInt();
3456 } else {
3457 return false;
3458 }
3459 Result.setBitVal(BitPosition: I, BitValue: Elem.isNegative());
3460 }
3461 pushInteger(S, Val: Result, QT: Call->getType());
3462 return true;
3463}
3464
3465static bool interp__builtin_elementwise_triop(
3466 InterpState &S, CodePtr OpPC, const CallExpr *Call,
3467 llvm::function_ref<APInt(const APSInt &, const APSInt &, const APSInt &)>
3468 Fn) {
3469 assert(Call->getNumArgs() == 3);
3470
3471 QualType Arg0Type = Call->getArg(Arg: 0)->getType();
3472 QualType Arg2Type = Call->getArg(Arg: 2)->getType();
3473 // Non-vector integer types.
3474 if (!Arg0Type->isVectorType()) {
3475 APSInt Op2;
3476 if (!popToAPSInt(S, T: Arg2Type, Out&: Op2))
3477 return false;
3478 APSInt Op1;
3479 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: Op1))
3480 return false;
3481 APSInt Op0;
3482 if (!popToAPSInt(S, T: Arg0Type, Out&: Op0))
3483 return false;
3484 APSInt Result = APSInt(Fn(Op0, Op1, Op2), Op0.isUnsigned());
3485 pushInteger(S, Val: Result, QT: Call->getType());
3486 return true;
3487 }
3488
3489 const auto *VecT = Arg0Type->castAs<VectorType>();
3490 PrimType ElemT = *S.getContext().classify(T: VecT->getElementType());
3491 unsigned NumElems = VecT->getNumElements();
3492 bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
3493
3494 // Vector + Vector + Scalar case.
3495 if (!Arg2Type->isVectorType()) {
3496 APSInt Op2;
3497 if (!popToAPSInt(S, T: Arg2Type, Out&: Op2))
3498 return false;
3499
3500 const Pointer &Op1 = S.Stk.pop<Pointer>();
3501 const Pointer &Op0 = S.Stk.pop<Pointer>();
3502 const Pointer &Dst = S.Stk.peek<Pointer>();
3503 for (unsigned I = 0; I != NumElems; ++I) {
3504 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
3505 Dst.elem<T>(I) = static_cast<T>(APSInt(
3506 Fn(Op0.elem<T>(I).toAPSInt(), Op1.elem<T>(I).toAPSInt(), Op2),
3507 DestUnsigned));
3508 });
3509 }
3510 Dst.initializeAllElements();
3511
3512 return true;
3513 }
3514
3515 // Vector type.
3516 const Pointer &Op2 = S.Stk.pop<Pointer>();
3517 const Pointer &Op1 = S.Stk.pop<Pointer>();
3518 const Pointer &Op0 = S.Stk.pop<Pointer>();
3519 const Pointer &Dst = S.Stk.peek<Pointer>();
3520 for (unsigned I = 0; I != NumElems; ++I) {
3521 APSInt Val0, Val1, Val2;
3522 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
3523 Val0 = Op0.elem<T>(I).toAPSInt();
3524 Val1 = Op1.elem<T>(I).toAPSInt();
3525 Val2 = Op2.elem<T>(I).toAPSInt();
3526 });
3527 APSInt Result = APSInt(Fn(Val0, Val1, Val2), Val0.isUnsigned());
3528 INT_TYPE_SWITCH_NO_BOOL(ElemT,
3529 { Dst.elem<T>(I) = static_cast<T>(Result); });
3530 }
3531 Dst.initializeAllElements();
3532
3533 return true;
3534}
3535
3536static bool interp__builtin_ia32_extract_vector(InterpState &S, CodePtr OpPC,
3537 const CallExpr *Call,
3538 unsigned ID) {
3539 assert(Call->getNumArgs() == 2);
3540
3541 APSInt ImmAPS;
3542 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: ImmAPS))
3543 return false;
3544 uint64_t Index = ImmAPS.getZExtValue();
3545
3546 const Pointer &Src = S.Stk.pop<Pointer>();
3547 if (!Src.getFieldDesc()->isPrimitiveArray())
3548 return false;
3549
3550 const Pointer &Dst = S.Stk.peek<Pointer>();
3551 if (!Dst.getFieldDesc()->isPrimitiveArray())
3552 return false;
3553
3554 unsigned SrcElems = Src.getNumElems();
3555 unsigned DstElems = Dst.getNumElems();
3556
3557 unsigned NumLanes = SrcElems / DstElems;
3558 unsigned Lane = static_cast<unsigned>(Index % NumLanes);
3559 unsigned ExtractPos = Lane * DstElems;
3560
3561 PrimType ElemT = Src.getFieldDesc()->getPrimType();
3562
3563 TYPE_SWITCH(ElemT, {
3564 for (unsigned I = 0; I != DstElems; ++I) {
3565 Dst.elem<T>(I) = Src.elem<T>(ExtractPos + I);
3566 }
3567 });
3568
3569 Dst.initializeAllElements();
3570 return true;
3571}
3572
3573static bool interp__builtin_ia32_extract_vector_masked(InterpState &S,
3574 CodePtr OpPC,
3575 const CallExpr *Call,
3576 unsigned ID) {
3577 assert(Call->getNumArgs() == 4);
3578
3579 APSInt MaskAPS;
3580 if (!popToAPSInt(S, E: Call->getArg(Arg: 3), Out&: MaskAPS))
3581 return false;
3582 const Pointer &Merge = S.Stk.pop<Pointer>();
3583 APSInt ImmAPS;
3584 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: ImmAPS))
3585 return false;
3586 const Pointer &Src = S.Stk.pop<Pointer>();
3587
3588 if (!Src.getFieldDesc()->isPrimitiveArray() ||
3589 !Merge.getFieldDesc()->isPrimitiveArray())
3590 return false;
3591
3592 const Pointer &Dst = S.Stk.peek<Pointer>();
3593 if (!Dst.getFieldDesc()->isPrimitiveArray())
3594 return false;
3595
3596 unsigned SrcElems = Src.getNumElems();
3597 unsigned DstElems = Dst.getNumElems();
3598
3599 unsigned NumLanes = SrcElems / DstElems;
3600 unsigned Lane = static_cast<unsigned>(ImmAPS.getZExtValue() % NumLanes);
3601 unsigned Base = Lane * DstElems;
3602
3603 PrimType ElemT = Src.getFieldDesc()->getPrimType();
3604
3605 TYPE_SWITCH(ElemT, {
3606 for (unsigned I = 0; I != DstElems; ++I) {
3607 if (MaskAPS[I])
3608 Dst.elem<T>(I) = Src.elem<T>(Base + I);
3609 else
3610 Dst.elem<T>(I) = Merge.elem<T>(I);
3611 }
3612 });
3613
3614 Dst.initializeAllElements();
3615 return true;
3616}
3617
3618static bool interp__builtin_ia32_insert_subvector(InterpState &S, CodePtr OpPC,
3619 const CallExpr *Call,
3620 unsigned ID) {
3621 assert(Call->getNumArgs() == 3);
3622
3623 APSInt ImmAPS;
3624 if (!popToAPSInt(S, E: Call->getArg(Arg: 2), Out&: ImmAPS))
3625 return false;
3626 uint64_t Index = ImmAPS.getZExtValue();
3627
3628 const Pointer &SubVec = S.Stk.pop<Pointer>();
3629 if (!SubVec.getFieldDesc()->isPrimitiveArray())
3630 return false;
3631
3632 const Pointer &BaseVec = S.Stk.pop<Pointer>();
3633 if (!BaseVec.getFieldDesc()->isPrimitiveArray())
3634 return false;
3635
3636 const Pointer &Dst = S.Stk.peek<Pointer>();
3637
3638 unsigned BaseElements = BaseVec.getNumElems();
3639 unsigned SubElements = SubVec.getNumElems();
3640
3641 assert(SubElements != 0 && BaseElements != 0 &&
3642 (BaseElements % SubElements) == 0);
3643
3644 unsigned NumLanes = BaseElements / SubElements;
3645 unsigned Lane = static_cast<unsigned>(Index % NumLanes);
3646 unsigned InsertPos = Lane * SubElements;
3647
3648 PrimType ElemT = BaseVec.getFieldDesc()->getPrimType();
3649
3650 TYPE_SWITCH(ElemT, {
3651 for (unsigned I = 0; I != BaseElements; ++I)
3652 Dst.elem<T>(I) = BaseVec.elem<T>(I);
3653 for (unsigned I = 0; I != SubElements; ++I)
3654 Dst.elem<T>(InsertPos + I) = SubVec.elem<T>(I);
3655 });
3656
3657 Dst.initializeAllElements();
3658 return true;
3659}
3660
3661static bool interp__builtin_ia32_phminposuw(InterpState &S, CodePtr OpPC,
3662 const CallExpr *Call) {
3663 assert(Call->getNumArgs() == 1);
3664
3665 const Pointer &Source = S.Stk.pop<Pointer>();
3666 const Pointer &Dest = S.Stk.peek<Pointer>();
3667
3668 unsigned SourceLen = Source.getNumElems();
3669 QualType ElemQT = getElemType(P: Source);
3670 OptPrimType ElemT = S.getContext().classify(T: ElemQT);
3671 unsigned ElemBitWidth = S.getASTContext().getTypeSize(T: ElemQT);
3672
3673 bool DestUnsigned = Call->getCallReturnType(Ctx: S.getASTContext())
3674 ->castAs<VectorType>()
3675 ->getElementType()
3676 ->isUnsignedIntegerOrEnumerationType();
3677
3678 INT_TYPE_SWITCH_NO_BOOL(*ElemT, {
3679 APSInt MinIndex(ElemBitWidth, DestUnsigned);
3680 APSInt MinVal = Source.elem<T>(0).toAPSInt();
3681
3682 for (unsigned I = 1; I != SourceLen; ++I) {
3683 APSInt Val = Source.elem<T>(I).toAPSInt();
3684 if (MinVal.ugt(Val)) {
3685 MinVal = Val;
3686 MinIndex = I;
3687 }
3688 }
3689
3690 Dest.elem<T>(0) = static_cast<T>(MinVal);
3691 Dest.elem<T>(1) = static_cast<T>(MinIndex);
3692 for (unsigned I = 2; I != SourceLen; ++I) {
3693 Dest.elem<T>(I) = static_cast<T>(APSInt(ElemBitWidth, DestUnsigned));
3694 }
3695 });
3696 Dest.initializeAllElements();
3697 return true;
3698}
3699
3700static bool interp__builtin_ia32_pternlog(InterpState &S, CodePtr OpPC,
3701 const CallExpr *Call, bool MaskZ) {
3702 assert(Call->getNumArgs() == 5);
3703
3704 APSInt UVal;
3705 if (!popToAPSInt(S, E: Call->getArg(Arg: 4), Out&: UVal))
3706 return false;
3707 APInt U = UVal; // Lane mask
3708 APSInt ImmVal;
3709 if (!popToAPSInt(S, E: Call->getArg(Arg: 3), Out&: ImmVal))
3710 return false;
3711 APInt Imm = ImmVal; // Ternary truth table
3712 const Pointer &C = S.Stk.pop<Pointer>();
3713 const Pointer &B = S.Stk.pop<Pointer>();
3714 const Pointer &A = S.Stk.pop<Pointer>();
3715 const Pointer &Dst = S.Stk.peek<Pointer>();
3716
3717 unsigned DstLen = A.getNumElems();
3718 QualType ElemQT = getElemType(P: A);
3719 OptPrimType ElemT = S.getContext().classify(T: ElemQT);
3720 unsigned LaneWidth = S.getASTContext().getTypeSize(T: ElemQT);
3721 bool DstUnsigned = ElemQT->isUnsignedIntegerOrEnumerationType();
3722
3723 INT_TYPE_SWITCH_NO_BOOL(*ElemT, {
3724 for (unsigned I = 0; I != DstLen; ++I) {
3725 APInt ALane = A.elem<T>(I).toAPSInt();
3726 APInt BLane = B.elem<T>(I).toAPSInt();
3727 APInt CLane = C.elem<T>(I).toAPSInt();
3728 APInt RLane(LaneWidth, 0);
3729 if (U[I]) { // If lane not masked, compute ternary logic.
3730 for (unsigned Bit = 0; Bit != LaneWidth; ++Bit) {
3731 unsigned ABit = ALane[Bit];
3732 unsigned BBit = BLane[Bit];
3733 unsigned CBit = CLane[Bit];
3734 unsigned Idx = (ABit << 2) | (BBit << 1) | (CBit);
3735 RLane.setBitVal(Bit, Imm[Idx]);
3736 }
3737 Dst.elem<T>(I) = static_cast<T>(APSInt(RLane, DstUnsigned));
3738 } else if (MaskZ) { // If zero masked, zero the lane.
3739 Dst.elem<T>(I) = static_cast<T>(APSInt(RLane, DstUnsigned));
3740 } else { // Just masked, put in A lane.
3741 Dst.elem<T>(I) = static_cast<T>(APSInt(ALane, DstUnsigned));
3742 }
3743 }
3744 });
3745 Dst.initializeAllElements();
3746 return true;
3747}
3748
3749static bool interp__builtin_ia32_vec_ext(InterpState &S, CodePtr OpPC,
3750 const CallExpr *Call, unsigned ID) {
3751 assert(Call->getNumArgs() == 2);
3752
3753 APSInt ImmAPS;
3754 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: ImmAPS))
3755 return false;
3756 const Pointer &Vec = S.Stk.pop<Pointer>();
3757 if (!Vec.getFieldDesc()->isPrimitiveArray())
3758 return false;
3759
3760 unsigned NumElems = Vec.getNumElems();
3761 unsigned Index =
3762 static_cast<unsigned>(ImmAPS.getZExtValue() & (NumElems - 1));
3763
3764 PrimType ElemT = Vec.getFieldDesc()->getPrimType();
3765 // FIXME(#161685): Replace float+int split with a numeric-only type switch
3766 if (ElemT == PT_Float) {
3767 S.Stk.push<Floating>(Args&: Vec.elem<Floating>(I: Index));
3768 return true;
3769 }
3770 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
3771 APSInt V = Vec.elem<T>(Index).toAPSInt();
3772 pushInteger(S, V, Call->getType());
3773 });
3774
3775 return true;
3776}
3777
3778static bool interp__builtin_ia32_vec_set(InterpState &S, CodePtr OpPC,
3779 const CallExpr *Call, unsigned ID) {
3780 assert(Call->getNumArgs() == 3);
3781
3782 APSInt ImmAPS;
3783 if (!popToAPSInt(S, E: Call->getArg(Arg: 2), Out&: ImmAPS))
3784 return false;
3785 APSInt ValAPS;
3786 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: ValAPS))
3787 return false;
3788
3789 const Pointer &Base = S.Stk.pop<Pointer>();
3790 if (!Base.getFieldDesc()->isPrimitiveArray())
3791 return false;
3792
3793 const Pointer &Dst = S.Stk.peek<Pointer>();
3794
3795 unsigned NumElems = Base.getNumElems();
3796 unsigned Index =
3797 static_cast<unsigned>(ImmAPS.getZExtValue() & (NumElems - 1));
3798
3799 PrimType ElemT = Base.getFieldDesc()->getPrimType();
3800 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
3801 for (unsigned I = 0; I != NumElems; ++I)
3802 Dst.elem<T>(I) = Base.elem<T>(I);
3803 Dst.elem<T>(Index) = static_cast<T>(ValAPS);
3804 });
3805
3806 Dst.initializeAllElements();
3807 return true;
3808}
3809
3810static bool evalICmpImm(uint8_t Imm, const APSInt &A, const APSInt &B,
3811 bool IsUnsigned) {
3812 switch (Imm & 0x7) {
3813 case 0x00: // _MM_CMPINT_EQ
3814 return (A == B);
3815 case 0x01: // _MM_CMPINT_LT
3816 return IsUnsigned ? A.ult(RHS: B) : A.slt(RHS: B);
3817 case 0x02: // _MM_CMPINT_LE
3818 return IsUnsigned ? A.ule(RHS: B) : A.sle(RHS: B);
3819 case 0x03: // _MM_CMPINT_FALSE
3820 return false;
3821 case 0x04: // _MM_CMPINT_NE
3822 return (A != B);
3823 case 0x05: // _MM_CMPINT_NLT
3824 return IsUnsigned ? A.ugt(RHS: B) : A.sgt(RHS: B);
3825 case 0x06: // _MM_CMPINT_NLE
3826 return IsUnsigned ? A.uge(RHS: B) : A.sge(RHS: B);
3827 case 0x07: // _MM_CMPINT_TRUE
3828 return true;
3829 default:
3830 llvm_unreachable("Invalid Op");
3831 }
3832}
3833
3834static bool interp__builtin_ia32_cmp_mask(InterpState &S, CodePtr OpPC,
3835 const CallExpr *Call, unsigned ID,
3836 bool IsUnsigned) {
3837 assert(Call->getNumArgs() == 4);
3838
3839 APSInt Mask;
3840 if (!popToAPSInt(S, E: Call->getArg(Arg: 3), Out&: Mask))
3841 return false;
3842 APSInt Opcode;
3843 if (!popToAPSInt(S, E: Call->getArg(Arg: 2), Out&: Opcode))
3844 return false;
3845 unsigned CmpOp = static_cast<unsigned>(Opcode.getZExtValue());
3846 const Pointer &RHS = S.Stk.pop<Pointer>();
3847 const Pointer &LHS = S.Stk.pop<Pointer>();
3848
3849 assert(LHS.getNumElems() == RHS.getNumElems());
3850
3851 APInt RetMask = APInt::getZero(numBits: LHS.getNumElems());
3852 unsigned VectorLen = LHS.getNumElems();
3853 PrimType ElemT = LHS.getFieldDesc()->getPrimType();
3854
3855 for (unsigned ElemNum = 0; ElemNum < VectorLen; ++ElemNum) {
3856 APSInt A, B;
3857 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
3858 A = LHS.elem<T>(ElemNum).toAPSInt();
3859 B = RHS.elem<T>(ElemNum).toAPSInt();
3860 });
3861 RetMask.setBitVal(BitPosition: ElemNum,
3862 BitValue: Mask[ElemNum] && evalICmpImm(Imm: CmpOp, A, B, IsUnsigned));
3863 }
3864 pushInteger(S, Val: RetMask, QT: Call->getType());
3865 return true;
3866}
3867
3868static bool interp__builtin_ia32_vpconflict(InterpState &S, CodePtr OpPC,
3869 const CallExpr *Call) {
3870 assert(Call->getNumArgs() == 1);
3871
3872 QualType Arg0Type = Call->getArg(Arg: 0)->getType();
3873 const auto *VecT = Arg0Type->castAs<VectorType>();
3874 PrimType ElemT = *S.getContext().classify(T: VecT->getElementType());
3875 unsigned NumElems = VecT->getNumElements();
3876 bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
3877 const Pointer &Src = S.Stk.pop<Pointer>();
3878 const Pointer &Dst = S.Stk.peek<Pointer>();
3879
3880 for (unsigned I = 0; I != NumElems; ++I) {
3881 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
3882 APSInt ElemI = Src.elem<T>(I).toAPSInt();
3883 APInt ConflictMask(ElemI.getBitWidth(), 0);
3884 for (unsigned J = 0; J != I; ++J) {
3885 APSInt ElemJ = Src.elem<T>(J).toAPSInt();
3886 ConflictMask.setBitVal(J, ElemI == ElemJ);
3887 }
3888 Dst.elem<T>(I) = static_cast<T>(APSInt(ConflictMask, DestUnsigned));
3889 });
3890 }
3891 Dst.initializeAllElements();
3892 return true;
3893}
3894
3895static bool interp__builtin_ia32_cvt_vec2mask(InterpState &S, CodePtr OpPC,
3896 const CallExpr *Call,
3897 unsigned ID) {
3898 assert(Call->getNumArgs() == 1);
3899
3900 const Pointer &Vec = S.Stk.pop<Pointer>();
3901 unsigned RetWidth = S.getASTContext().getIntWidth(T: Call->getType());
3902 APInt RetMask(RetWidth, 0);
3903
3904 unsigned VectorLen = Vec.getNumElems();
3905 PrimType ElemT = Vec.getFieldDesc()->getPrimType();
3906
3907 for (unsigned ElemNum = 0; ElemNum != VectorLen; ++ElemNum) {
3908 APSInt A;
3909 INT_TYPE_SWITCH_NO_BOOL(ElemT, { A = Vec.elem<T>(ElemNum).toAPSInt(); });
3910 unsigned MSB = A[A.getBitWidth() - 1];
3911 RetMask.setBitVal(BitPosition: ElemNum, BitValue: MSB);
3912 }
3913 pushInteger(S, Val: RetMask, QT: Call->getType());
3914 return true;
3915}
3916
3917static bool interp__builtin_ia32_cvt_mask2vec(InterpState &S, CodePtr OpPC,
3918 const CallExpr *Call,
3919 unsigned ID) {
3920 assert(Call->getNumArgs() == 1);
3921
3922 APSInt Mask;
3923 if (!popToAPSInt(S, E: Call->getArg(Arg: 0), Out&: Mask))
3924 return false;
3925
3926 const Pointer &Vec = S.Stk.peek<Pointer>();
3927 unsigned NumElems = Vec.getNumElems();
3928 PrimType ElemT = Vec.getFieldDesc()->getPrimType();
3929
3930 for (unsigned I = 0; I != NumElems; ++I) {
3931 bool BitSet = Mask[I];
3932
3933 INT_TYPE_SWITCH_NO_BOOL(
3934 ElemT, { Vec.elem<T>(I) = BitSet ? T::from(-1) : T::from(0); });
3935 }
3936
3937 Vec.initializeAllElements();
3938
3939 return true;
3940}
3941
3942static bool interp__builtin_ia32_cvtsd2ss(InterpState &S, CodePtr OpPC,
3943 const CallExpr *Call,
3944 bool HasRoundingMask) {
3945 APSInt Rounding, MaskInt;
3946 Pointer Src, B, A;
3947
3948 if (HasRoundingMask) {
3949 assert(Call->getNumArgs() == 5);
3950 if (!popToAPSInt(S, E: Call->getArg(Arg: 4), Out&: Rounding))
3951 return false;
3952 if (!popToAPSInt(S, E: Call->getArg(Arg: 3), Out&: MaskInt))
3953 return false;
3954 Src = S.Stk.pop<Pointer>();
3955 B = S.Stk.pop<Pointer>();
3956 A = S.Stk.pop<Pointer>();
3957 if (!CheckLoad(S, OpPC, Ptr: A) || !CheckLoad(S, OpPC, Ptr: B) ||
3958 !CheckLoad(S, OpPC, Ptr: Src))
3959 return false;
3960 } else {
3961 assert(Call->getNumArgs() == 2);
3962 B = S.Stk.pop<Pointer>();
3963 A = S.Stk.pop<Pointer>();
3964 if (!CheckLoad(S, OpPC, Ptr: A) || !CheckLoad(S, OpPC, Ptr: B))
3965 return false;
3966 }
3967
3968 const auto *DstVTy = Call->getType()->castAs<VectorType>();
3969 unsigned NumElems = DstVTy->getNumElements();
3970 const Pointer &Dst = S.Stk.peek<Pointer>();
3971
3972 // Copy all elements except lane 0 (overwritten below) from A to Dst.
3973 for (unsigned I = 1; I != NumElems; ++I)
3974 Dst.elem<Floating>(I) = A.elem<Floating>(I);
3975
3976 // Convert element 0 from double to float, or use Src if masked off.
3977 if (!HasRoundingMask || (MaskInt.getZExtValue() & 0x1)) {
3978 assert(S.getASTContext().FloatTy == DstVTy->getElementType() &&
3979 "cvtsd2ss requires float element type in destination vector");
3980
3981 Floating Conv = S.allocFloat(
3982 Sem: S.getASTContext().getFloatTypeSemantics(T: DstVTy->getElementType()));
3983 APFloat SrcVal = B.elem<Floating>(I: 0).getAPFloat();
3984 if (!convertDoubleToFloatStrict(Src: SrcVal, Dst&: Conv, S, DiagExpr: Call))
3985 return false;
3986 Dst.elem<Floating>(I: 0) = Conv;
3987 } else {
3988 Dst.elem<Floating>(I: 0) = Src.elem<Floating>(I: 0);
3989 }
3990
3991 Dst.initializeAllElements();
3992 return true;
3993}
3994
3995static bool interp__builtin_ia32_cvtpd2ps(InterpState &S, CodePtr OpPC,
3996 const CallExpr *Call, bool IsMasked,
3997 bool HasRounding) {
3998 APSInt MaskVal;
3999 Pointer PassThrough;
4000 Pointer Src;
4001 APSInt Rounding;
4002
4003 if (IsMasked) {
4004 // Pop in reverse order.
4005 if (HasRounding) {
4006 if (!popToAPSInt(S, E: Call->getArg(Arg: 3), Out&: Rounding))
4007 return false;
4008 if (!popToAPSInt(S, E: Call->getArg(Arg: 2), Out&: MaskVal))
4009 return false;
4010 PassThrough = S.Stk.pop<Pointer>();
4011 Src = S.Stk.pop<Pointer>();
4012 } else {
4013 if (!popToAPSInt(S, E: Call->getArg(Arg: 2), Out&: MaskVal))
4014 return false;
4015 PassThrough = S.Stk.pop<Pointer>();
4016 Src = S.Stk.pop<Pointer>();
4017 }
4018
4019 if (!CheckLoad(S, OpPC, Ptr: PassThrough))
4020 return false;
4021 } else {
4022 // Pop source only.
4023 Src = S.Stk.pop<Pointer>();
4024 }
4025
4026 if (!CheckLoad(S, OpPC, Ptr: Src))
4027 return false;
4028
4029 const auto *RetVTy = Call->getType()->castAs<VectorType>();
4030 unsigned RetElems = RetVTy->getNumElements();
4031 unsigned SrcElems = Src.getNumElems();
4032 const Pointer &Dst = S.Stk.peek<Pointer>();
4033
4034 // Initialize destination with passthrough or zeros.
4035 for (unsigned I = 0; I != RetElems; ++I)
4036 if (IsMasked)
4037 Dst.elem<Floating>(I) = PassThrough.elem<Floating>(I);
4038 else
4039 Dst.elem<Floating>(I) = Floating(APFloat(0.0f));
4040
4041 assert(S.getASTContext().FloatTy == RetVTy->getElementType() &&
4042 "cvtpd2ps requires float element type in return vector");
4043
4044 // Convert double to float for enabled elements (only process source elements
4045 // that exist).
4046 for (unsigned I = 0; I != SrcElems; ++I) {
4047 if (IsMasked && !MaskVal[I])
4048 continue;
4049
4050 APFloat SrcVal = Src.elem<Floating>(I).getAPFloat();
4051
4052 Floating Conv = S.allocFloat(
4053 Sem: S.getASTContext().getFloatTypeSemantics(T: RetVTy->getElementType()));
4054 if (!convertDoubleToFloatStrict(Src: SrcVal, Dst&: Conv, S, DiagExpr: Call))
4055 return false;
4056 Dst.elem<Floating>(I) = Conv;
4057 }
4058
4059 Dst.initializeAllElements();
4060 return true;
4061}
4062
4063static bool interp__builtin_ia32_shuffle_generic(
4064 InterpState &S, CodePtr OpPC, const CallExpr *Call,
4065 llvm::function_ref<std::pair<unsigned, int>(unsigned, const APInt &)>
4066 GetSourceIndex) {
4067
4068 assert(Call->getNumArgs() == 2 || Call->getNumArgs() == 3);
4069
4070 APInt ShuffleMask;
4071 Pointer A, MaskVector, B;
4072 bool IsVectorMask = false;
4073 bool IsSingleOperand = (Call->getNumArgs() == 2);
4074
4075 if (IsSingleOperand) {
4076 QualType MaskType = Call->getArg(Arg: 1)->getType();
4077 if (MaskType->isVectorType()) {
4078 IsVectorMask = true;
4079 MaskVector = S.Stk.pop<Pointer>();
4080 A = S.Stk.pop<Pointer>();
4081 B = A;
4082 } else if (MaskType->isIntegerType()) {
4083 APSInt MaskVal;
4084 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: MaskVal))
4085 return false;
4086 ShuffleMask = MaskVal;
4087 A = S.Stk.pop<Pointer>();
4088 B = A;
4089 } else {
4090 return false;
4091 }
4092 } else {
4093 QualType Arg2Type = Call->getArg(Arg: 2)->getType();
4094 if (Arg2Type->isVectorType()) {
4095 IsVectorMask = true;
4096 B = S.Stk.pop<Pointer>();
4097 MaskVector = S.Stk.pop<Pointer>();
4098 A = S.Stk.pop<Pointer>();
4099 } else if (Arg2Type->isIntegerType()) {
4100 APSInt MaskVal;
4101 if (!popToAPSInt(S, E: Call->getArg(Arg: 2), Out&: MaskVal))
4102 return false;
4103 ShuffleMask = MaskVal;
4104 B = S.Stk.pop<Pointer>();
4105 A = S.Stk.pop<Pointer>();
4106 } else {
4107 return false;
4108 }
4109 }
4110
4111 QualType Arg0Type = Call->getArg(Arg: 0)->getType();
4112 const auto *VecT = Arg0Type->castAs<VectorType>();
4113 PrimType ElemT = *S.getContext().classify(T: VecT->getElementType());
4114 unsigned NumElems = VecT->getNumElements();
4115
4116 const Pointer &Dst = S.Stk.peek<Pointer>();
4117
4118 PrimType MaskElemT = PT_Uint32;
4119 if (IsVectorMask) {
4120 QualType Arg1Type = Call->getArg(Arg: 1)->getType();
4121 const auto *MaskVecT = Arg1Type->castAs<VectorType>();
4122 QualType MaskElemType = MaskVecT->getElementType();
4123 MaskElemT = *S.getContext().classify(T: MaskElemType);
4124 }
4125
4126 for (unsigned DstIdx = 0; DstIdx != NumElems; ++DstIdx) {
4127 if (IsVectorMask) {
4128 INT_TYPE_SWITCH(MaskElemT,
4129 { ShuffleMask = MaskVector.elem<T>(DstIdx).toAPSInt(); });
4130 }
4131
4132 auto [SrcVecIdx, SrcIdx] = GetSourceIndex(DstIdx, ShuffleMask);
4133
4134 if (SrcIdx < 0) {
4135 // Zero out this element
4136 if (ElemT == PT_Float) {
4137 Dst.elem<Floating>(I: DstIdx) = Floating(
4138 S.getASTContext().getFloatTypeSemantics(T: VecT->getElementType()));
4139 } else {
4140 INT_TYPE_SWITCH_NO_BOOL(ElemT, { Dst.elem<T>(DstIdx) = T::from(0); });
4141 }
4142 } else {
4143 const Pointer &Src = (SrcVecIdx == 0) ? A : B;
4144 TYPE_SWITCH(ElemT, { Dst.elem<T>(DstIdx) = Src.elem<T>(SrcIdx); });
4145 }
4146 }
4147 Dst.initializeAllElements();
4148
4149 return true;
4150}
4151
4152static bool interp__builtin_ia32_shuffle_generic(
4153 InterpState &S, CodePtr OpPC, const CallExpr *Call,
4154 llvm::function_ref<std::pair<unsigned, int>(unsigned, unsigned)>
4155 GetSourceIndex) {
4156 return interp__builtin_ia32_shuffle_generic(
4157 S, OpPC, Call,
4158 GetSourceIndex: [&GetSourceIndex](unsigned DstIdx,
4159 const APInt &Mask) -> std::pair<unsigned, int> {
4160 return GetSourceIndex(DstIdx, Mask.getZExtValue());
4161 });
4162}
4163
4164static bool interp__builtin_ia32_shift_with_count(
4165 InterpState &S, CodePtr OpPC, const CallExpr *Call,
4166 llvm::function_ref<APInt(const APInt &, uint64_t)> ShiftOp,
4167 llvm::function_ref<APInt(const APInt &, unsigned)> OverflowOp) {
4168
4169 assert(Call->getNumArgs() == 2);
4170
4171 const Pointer &Count = S.Stk.pop<Pointer>();
4172 const Pointer &Source = S.Stk.pop<Pointer>();
4173
4174 QualType SourceType = Call->getArg(Arg: 0)->getType();
4175 QualType CountType = Call->getArg(Arg: 1)->getType();
4176 assert(SourceType->isVectorType() && CountType->isVectorType());
4177
4178 const auto *SourceVecT = SourceType->castAs<VectorType>();
4179 const auto *CountVecT = CountType->castAs<VectorType>();
4180 PrimType SourceElemT = *S.getContext().classify(T: SourceVecT->getElementType());
4181 PrimType CountElemT = *S.getContext().classify(T: CountVecT->getElementType());
4182
4183 const Pointer &Dst = S.Stk.peek<Pointer>();
4184
4185 unsigned DestEltWidth =
4186 S.getASTContext().getTypeSize(T: SourceVecT->getElementType());
4187 bool IsDestUnsigned = SourceVecT->getElementType()->isUnsignedIntegerType();
4188 unsigned DestLen = SourceVecT->getNumElements();
4189 unsigned CountEltWidth =
4190 S.getASTContext().getTypeSize(T: CountVecT->getElementType());
4191 unsigned NumBitsInQWord = 64;
4192 unsigned NumCountElts = NumBitsInQWord / CountEltWidth;
4193
4194 uint64_t CountLQWord = 0;
4195 for (unsigned EltIdx = 0; EltIdx != NumCountElts; ++EltIdx) {
4196 uint64_t Elt = 0;
4197 INT_TYPE_SWITCH(CountElemT,
4198 { Elt = static_cast<uint64_t>(Count.elem<T>(EltIdx)); });
4199 CountLQWord |= (Elt << (EltIdx * CountEltWidth));
4200 }
4201
4202 for (unsigned EltIdx = 0; EltIdx != DestLen; ++EltIdx) {
4203 APSInt Elt;
4204 INT_TYPE_SWITCH(SourceElemT, { Elt = Source.elem<T>(EltIdx).toAPSInt(); });
4205
4206 APInt Result;
4207 if (CountLQWord < DestEltWidth) {
4208 Result = ShiftOp(Elt, CountLQWord);
4209 } else {
4210 Result = OverflowOp(Elt, DestEltWidth);
4211 }
4212 if (IsDestUnsigned) {
4213 INT_TYPE_SWITCH(SourceElemT, {
4214 Dst.elem<T>(EltIdx) = T::from(Result.getZExtValue());
4215 });
4216 } else {
4217 INT_TYPE_SWITCH(SourceElemT, {
4218 Dst.elem<T>(EltIdx) = T::from(Result.getSExtValue());
4219 });
4220 }
4221 }
4222
4223 Dst.initializeAllElements();
4224 return true;
4225}
4226
4227static bool interp__builtin_ia32_shufbitqmb_mask(InterpState &S, CodePtr OpPC,
4228 const CallExpr *Call) {
4229
4230 assert(Call->getNumArgs() == 3);
4231
4232 QualType SourceType = Call->getArg(Arg: 0)->getType();
4233 QualType ShuffleMaskType = Call->getArg(Arg: 1)->getType();
4234 QualType ZeroMaskType = Call->getArg(Arg: 2)->getType();
4235 if (!SourceType->isVectorType() || !ShuffleMaskType->isVectorType() ||
4236 !ZeroMaskType->isIntegerType()) {
4237 return false;
4238 }
4239
4240 Pointer Source, ShuffleMask;
4241 APSInt ZeroMask;
4242 if (!popToAPSInt(S, E: Call->getArg(Arg: 2), Out&: ZeroMask))
4243 return false;
4244 ShuffleMask = S.Stk.pop<Pointer>();
4245 Source = S.Stk.pop<Pointer>();
4246
4247 const auto *SourceVecT = SourceType->castAs<VectorType>();
4248 const auto *ShuffleMaskVecT = ShuffleMaskType->castAs<VectorType>();
4249 assert(SourceVecT->getNumElements() == ShuffleMaskVecT->getNumElements());
4250 assert(ZeroMask.getBitWidth() == SourceVecT->getNumElements());
4251
4252 PrimType SourceElemT = *S.getContext().classify(T: SourceVecT->getElementType());
4253 PrimType ShuffleMaskElemT =
4254 *S.getContext().classify(T: ShuffleMaskVecT->getElementType());
4255
4256 unsigned NumBytesInQWord = 8;
4257 unsigned NumBitsInByte = 8;
4258 unsigned NumBytes = SourceVecT->getNumElements();
4259 unsigned NumQWords = NumBytes / NumBytesInQWord;
4260 unsigned RetWidth = ZeroMask.getBitWidth();
4261 APSInt RetMask(llvm::APInt(RetWidth, 0), /*isUnsigned=*/true);
4262
4263 for (unsigned QWordId = 0; QWordId != NumQWords; ++QWordId) {
4264 APInt SourceQWord(64, 0);
4265 for (unsigned ByteIdx = 0; ByteIdx != NumBytesInQWord; ++ByteIdx) {
4266 uint64_t Byte = 0;
4267 INT_TYPE_SWITCH(SourceElemT, {
4268 Byte = static_cast<uint64_t>(
4269 Source.elem<T>(QWordId * NumBytesInQWord + ByteIdx));
4270 });
4271 SourceQWord.insertBits(SubBits: APInt(8, Byte & 0xFF), bitPosition: ByteIdx * NumBitsInByte);
4272 }
4273
4274 for (unsigned ByteIdx = 0; ByteIdx != NumBytesInQWord; ++ByteIdx) {
4275 unsigned SelIdx = QWordId * NumBytesInQWord + ByteIdx;
4276 unsigned M = 0;
4277 INT_TYPE_SWITCH(ShuffleMaskElemT, {
4278 M = static_cast<unsigned>(ShuffleMask.elem<T>(SelIdx)) & 0x3F;
4279 });
4280
4281 if (ZeroMask[SelIdx]) {
4282 RetMask.setBitVal(BitPosition: SelIdx, BitValue: SourceQWord[M]);
4283 }
4284 }
4285 }
4286
4287 pushInteger(S, Val: RetMask, QT: Call->getType());
4288 return true;
4289}
4290
4291static bool interp__builtin_ia32_vcvtps2ph(InterpState &S, CodePtr OpPC,
4292 const CallExpr *Call) {
4293 // Arguments are: vector of floats, rounding immediate
4294 assert(Call->getNumArgs() == 2);
4295
4296 APSInt Imm;
4297 if (!popToAPSInt(S, E: Call->getArg(Arg: 1), Out&: Imm))
4298 return false;
4299 const Pointer &Src = S.Stk.pop<Pointer>();
4300 const Pointer &Dst = S.Stk.peek<Pointer>();
4301
4302 assert(Src.getFieldDesc()->isPrimitiveArray());
4303 assert(Dst.getFieldDesc()->isPrimitiveArray());
4304
4305 const auto *SrcVTy = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
4306 unsigned SrcNumElems = SrcVTy->getNumElements();
4307 const auto *DstVTy = Call->getType()->castAs<VectorType>();
4308 unsigned DstNumElems = DstVTy->getNumElements();
4309
4310 const llvm::fltSemantics &HalfSem =
4311 S.getASTContext().getFloatTypeSemantics(T: S.getASTContext().HalfTy);
4312
4313 // imm[2] == 1 means use MXCSR rounding mode.
4314 // In that case, we can only evaluate if the conversion is exact.
4315 int ImmVal = Imm.getZExtValue();
4316 bool UseMXCSR = (ImmVal & 4) != 0;
4317 bool IsFPConstrained =
4318 Call->getFPFeaturesInEffect(LO: S.getASTContext().getLangOpts())
4319 .isFPConstrained();
4320
4321 llvm::RoundingMode RM;
4322 if (!UseMXCSR) {
4323 switch (ImmVal & 3) {
4324 case 0:
4325 RM = llvm::RoundingMode::NearestTiesToEven;
4326 break;
4327 case 1:
4328 RM = llvm::RoundingMode::TowardNegative;
4329 break;
4330 case 2:
4331 RM = llvm::RoundingMode::TowardPositive;
4332 break;
4333 case 3:
4334 RM = llvm::RoundingMode::TowardZero;
4335 break;
4336 default:
4337 llvm_unreachable("Invalid immediate rounding mode");
4338 }
4339 } else {
4340 // For MXCSR, we must check for exactness. We can use any rounding mode
4341 // for the trial conversion since the result is the same if it's exact.
4342 RM = llvm::RoundingMode::NearestTiesToEven;
4343 }
4344
4345 QualType DstElemQT = Dst.getFieldDesc()->getElemQualType();
4346 PrimType DstElemT = *S.getContext().classify(T: DstElemQT);
4347
4348 for (unsigned I = 0; I != SrcNumElems; ++I) {
4349 Floating SrcVal = Src.elem<Floating>(I);
4350 APFloat DstVal = SrcVal.getAPFloat();
4351
4352 bool LostInfo;
4353 APFloat::opStatus St = DstVal.convert(ToSemantics: HalfSem, RM, losesInfo: &LostInfo);
4354
4355 if (UseMXCSR && IsFPConstrained && St != APFloat::opOK) {
4356 S.FFDiag(SI: S.Current->getSource(PC: OpPC),
4357 DiagId: diag::note_constexpr_dynamic_rounding);
4358 return false;
4359 }
4360
4361 INT_TYPE_SWITCH_NO_BOOL(DstElemT, {
4362 // Convert the destination value's bit pattern to an unsigned integer,
4363 // then reconstruct the element using the target type's 'from' method.
4364 uint64_t RawBits = DstVal.bitcastToAPInt().getZExtValue();
4365 Dst.elem<T>(I) = T::from(RawBits);
4366 });
4367 }
4368
4369 // Zero out remaining elements if the destination has more elements
4370 // (e.g., vcvtps2ph converting 4 floats to 8 shorts).
4371 if (DstNumElems > SrcNumElems) {
4372 for (unsigned I = SrcNumElems; I != DstNumElems; ++I) {
4373 INT_TYPE_SWITCH_NO_BOOL(DstElemT, { Dst.elem<T>(I) = T::from(0); });
4374 }
4375 }
4376
4377 Dst.initializeAllElements();
4378 return true;
4379}
4380
4381static bool interp__builtin_ia32_multishiftqb(InterpState &S, CodePtr OpPC,
4382 const CallExpr *Call) {
4383 assert(Call->getNumArgs() == 2);
4384
4385 QualType ATy = Call->getArg(Arg: 0)->getType();
4386 QualType BTy = Call->getArg(Arg: 1)->getType();
4387 if (!ATy->isVectorType() || !BTy->isVectorType()) {
4388 return false;
4389 }
4390
4391 const Pointer &BPtr = S.Stk.pop<Pointer>();
4392 const Pointer &APtr = S.Stk.pop<Pointer>();
4393 const auto *AVecT = ATy->castAs<VectorType>();
4394 assert(AVecT->getNumElements() ==
4395 BTy->castAs<VectorType>()->getNumElements());
4396
4397 PrimType ElemT = *S.getContext().classify(T: AVecT->getElementType());
4398
4399 unsigned NumBytesInQWord = 8;
4400 unsigned NumBitsInByte = 8;
4401 unsigned NumBytes = AVecT->getNumElements();
4402 unsigned NumQWords = NumBytes / NumBytesInQWord;
4403 const Pointer &Dst = S.Stk.peek<Pointer>();
4404
4405 for (unsigned QWordId = 0; QWordId != NumQWords; ++QWordId) {
4406 APInt BQWord(64, 0);
4407 for (unsigned ByteIdx = 0; ByteIdx != NumBytesInQWord; ++ByteIdx) {
4408 unsigned Idx = QWordId * NumBytesInQWord + ByteIdx;
4409 INT_TYPE_SWITCH(ElemT, {
4410 uint64_t Byte = static_cast<uint64_t>(BPtr.elem<T>(Idx));
4411 BQWord.insertBits(APInt(8, Byte & 0xFF), ByteIdx * NumBitsInByte);
4412 });
4413 }
4414
4415 for (unsigned ByteIdx = 0; ByteIdx != NumBytesInQWord; ++ByteIdx) {
4416 unsigned Idx = QWordId * NumBytesInQWord + ByteIdx;
4417 uint64_t Ctrl = 0;
4418 INT_TYPE_SWITCH(
4419 ElemT, { Ctrl = static_cast<uint64_t>(APtr.elem<T>(Idx)) & 0x3F; });
4420
4421 APInt Byte(8, 0);
4422 for (unsigned BitIdx = 0; BitIdx != NumBitsInByte; ++BitIdx) {
4423 Byte.setBitVal(BitPosition: BitIdx, BitValue: BQWord[(Ctrl + BitIdx) & 0x3F]);
4424 }
4425 INT_TYPE_SWITCH(ElemT,
4426 { Dst.elem<T>(Idx) = T::from(Byte.getZExtValue()); });
4427 }
4428 }
4429
4430 Dst.initializeAllElements();
4431
4432 return true;
4433}
4434
4435static bool interp__builtin_ia32_gfni_affine(InterpState &S, CodePtr OpPC,
4436 const CallExpr *Call,
4437 bool Inverse) {
4438 assert(Call->getNumArgs() == 3);
4439 QualType XType = Call->getArg(Arg: 0)->getType();
4440 QualType AType = Call->getArg(Arg: 1)->getType();
4441 QualType ImmType = Call->getArg(Arg: 2)->getType();
4442 if (!XType->isVectorType() || !AType->isVectorType() ||
4443 !ImmType->isIntegerType()) {
4444 return false;
4445 }
4446
4447 Pointer X, A;
4448 APSInt Imm;
4449 if (!popToAPSInt(S, E: Call->getArg(Arg: 2), Out&: Imm))
4450 return false;
4451 A = S.Stk.pop<Pointer>();
4452 X = S.Stk.pop<Pointer>();
4453
4454 const Pointer &Dst = S.Stk.peek<Pointer>();
4455 const auto *AVecT = AType->castAs<VectorType>();
4456 assert(XType->castAs<VectorType>()->getNumElements() ==
4457 AVecT->getNumElements());
4458 unsigned NumBytesInQWord = 8;
4459 unsigned NumBytes = AVecT->getNumElements();
4460 unsigned NumBitsInQWord = 64;
4461 unsigned NumQWords = NumBytes / NumBytesInQWord;
4462 unsigned NumBitsInByte = 8;
4463 PrimType AElemT = *S.getContext().classify(T: AVecT->getElementType());
4464
4465 // computing A*X + Imm
4466 for (unsigned QWordIdx = 0; QWordIdx != NumQWords; ++QWordIdx) {
4467 // Extract the QWords from X, A
4468 APInt XQWord(NumBitsInQWord, 0);
4469 APInt AQWord(NumBitsInQWord, 0);
4470 for (unsigned ByteIdx = 0; ByteIdx != NumBytesInQWord; ++ByteIdx) {
4471 unsigned Idx = QWordIdx * NumBytesInQWord + ByteIdx;
4472 uint8_t XByte;
4473 uint8_t AByte;
4474 INT_TYPE_SWITCH(AElemT, {
4475 XByte = static_cast<uint8_t>(X.elem<T>(Idx));
4476 AByte = static_cast<uint8_t>(A.elem<T>(Idx));
4477 });
4478
4479 XQWord.insertBits(SubBits: APInt(NumBitsInByte, XByte), bitPosition: ByteIdx * NumBitsInByte);
4480 AQWord.insertBits(SubBits: APInt(NumBitsInByte, AByte), bitPosition: ByteIdx * NumBitsInByte);
4481 }
4482
4483 for (unsigned ByteIdx = 0; ByteIdx != NumBytesInQWord; ++ByteIdx) {
4484 unsigned Idx = QWordIdx * NumBytesInQWord + ByteIdx;
4485 uint8_t XByte =
4486 XQWord.lshr(shiftAmt: ByteIdx * NumBitsInByte).getLoBits(numBits: 8).getZExtValue();
4487 INT_TYPE_SWITCH(AElemT, {
4488 Dst.elem<T>(Idx) = T::from(GFNIAffine(XByte, AQWord, Imm, Inverse));
4489 });
4490 }
4491 }
4492 Dst.initializeAllElements();
4493 return true;
4494}
4495
4496static bool interp__builtin_ia32_gfni_mul(InterpState &S, CodePtr OpPC,
4497 const CallExpr *Call) {
4498 assert(Call->getNumArgs() == 2);
4499
4500 QualType AType = Call->getArg(Arg: 0)->getType();
4501 QualType BType = Call->getArg(Arg: 1)->getType();
4502 if (!AType->isVectorType() || !BType->isVectorType()) {
4503 return false;
4504 }
4505
4506 Pointer A, B;
4507 B = S.Stk.pop<Pointer>();
4508 A = S.Stk.pop<Pointer>();
4509
4510 const Pointer &Dst = S.Stk.peek<Pointer>();
4511 const auto *AVecT = AType->castAs<VectorType>();
4512 assert(AVecT->getNumElements() ==
4513 BType->castAs<VectorType>()->getNumElements());
4514
4515 PrimType AElemT = *S.getContext().classify(T: AVecT->getElementType());
4516 unsigned NumBytes = A.getNumElems();
4517
4518 for (unsigned ByteIdx = 0; ByteIdx != NumBytes; ++ByteIdx) {
4519 uint8_t AByte, BByte;
4520 INT_TYPE_SWITCH(AElemT, {
4521 AByte = static_cast<uint8_t>(A.elem<T>(ByteIdx));
4522 BByte = static_cast<uint8_t>(B.elem<T>(ByteIdx));
4523 Dst.elem<T>(ByteIdx) = T::from(GFNIMul(AByte, BByte));
4524 });
4525 }
4526
4527 Dst.initializeAllElements();
4528 return true;
4529}
4530
4531static bool interp__builtin_ia32_vpdp(InterpState &S, CodePtr OpPC,
4532 const CallExpr *Call, bool IsSaturating) {
4533 assert(Call->getNumArgs() == 3);
4534
4535 QualType SrcT = Call->getArg(Arg: 0)->getType();
4536 QualType OpAT = Call->getArg(Arg: 1)->getType();
4537 QualType OpBT = Call->getArg(Arg: 2)->getType();
4538 QualType DstT = Call->getType();
4539 if (!SrcT->isVectorType() || !OpAT->isVectorType() || !OpBT->isVectorType() ||
4540 !DstT->isVectorType())
4541 return false;
4542
4543 const auto *SrcVecT = SrcT->castAs<VectorType>();
4544 const auto *OpAVecT = OpAT->castAs<VectorType>();
4545 const auto *OpBVecT = OpBT->castAs<VectorType>();
4546 const auto *DstVecT = DstT->castAs<VectorType>();
4547
4548 assert(OpAVecT->getNumElements() == OpBVecT->getNumElements());
4549
4550 unsigned NumSrcElems = SrcVecT->getNumElements();
4551 unsigned NumOperandElems = OpAVecT->getNumElements();
4552 unsigned ElemsPerLane = NumOperandElems / NumSrcElems;
4553
4554 PrimType SrcElemT = *S.getContext().classify(T: SrcVecT->getElementType());
4555 PrimType OpAElemT = *S.getContext().classify(T: OpAVecT->getElementType());
4556 PrimType OpBElemT = *S.getContext().classify(T: OpBVecT->getElementType());
4557 PrimType DstElemT = *S.getContext().classify(T: DstVecT->getElementType());
4558
4559 assert(SrcElemT == DstElemT);
4560
4561 const Pointer &OpBPtr = S.Stk.pop<Pointer>();
4562 const Pointer &OpAPtr = S.Stk.pop<Pointer>();
4563 const Pointer &SrcPtr = S.Stk.pop<Pointer>();
4564 const Pointer &Dst = S.Stk.peek<Pointer>();
4565
4566 for (unsigned I = 0; I != NumSrcElems; ++I) {
4567 APSInt Acc;
4568 INT_TYPE_SWITCH_NO_BOOL(SrcElemT, { Acc = SrcPtr.elem<T>(I).toAPSInt(); });
4569 Acc = Acc.sext(width: 64);
4570 for (unsigned J = 0; J != ElemsPerLane; ++J) {
4571 APSInt OpA, OpB;
4572 INT_TYPE_SWITCH_NO_BOOL(
4573 OpAElemT, { OpA = OpAPtr.elem<T>(ElemsPerLane * I + J).toAPSInt(); });
4574 INT_TYPE_SWITCH_NO_BOOL(
4575 OpBElemT, { OpB = OpBPtr.elem<T>(ElemsPerLane * I + J).toAPSInt(); });
4576 OpA = APSInt(OpA.extend(width: 64), false);
4577 OpB = APSInt(OpB.extend(width: 64), false);
4578 Acc += OpA * OpB;
4579 }
4580 if (IsSaturating)
4581 Acc = APSInt(Acc.truncSSat(width: 32), false);
4582 else
4583 Acc = APSInt(Acc.trunc(width: 32), false);
4584 INT_TYPE_SWITCH_NO_BOOL(DstElemT,
4585 { Dst.elem<T>(I) = static_cast<T>(Acc); });
4586 }
4587 Dst.initializeAllElements();
4588 return true;
4589}
4590
4591// Bit Matrix Multiply and Accumulate (AVX512BMM). Each 256-bit lane holds a
4592// 16x16 bit matrix as 16 x i16 elements; element i is row i and bit j of that
4593// element is entry [i][j]. The accumulator (third argument, src1 in the AMD
4594// ISA) provides the initial value of each result bit, into which the bit-matrix
4595// product of the first two arguments (src2 * src3) is reduced with OR (vbmacor)
4596// or XOR (vbmacxor):
4597// for i in 0..15, j in 0..15:
4598// bit = C[16*i+j]
4599// for k in 0..15: bit OP= A[16*i+k] & B[16*k+j]
4600// dest[16*i+j] = bit
4601static bool interp__builtin_ia32_bmac(InterpState &S, CodePtr OpPC,
4602 const CallExpr *Call, bool IsXor) {
4603 assert(Call->getNumArgs() == 3);
4604
4605 // AST-based type checks before popping the stack.
4606 QualType AType = Call->getArg(Arg: 0)->getType();
4607 QualType BType = Call->getArg(Arg: 1)->getType();
4608 QualType CType = Call->getArg(Arg: 2)->getType();
4609 if (!AType->isVectorType() || !BType->isVectorType() ||
4610 !CType->isVectorType())
4611 return false;
4612
4613 const Pointer &C = S.Stk.pop<Pointer>();
4614 const Pointer &B = S.Stk.pop<Pointer>();
4615 const Pointer &A = S.Stk.pop<Pointer>();
4616 const Pointer &Dst = S.Stk.peek<Pointer>();
4617
4618 // check if all three primitive arrays are with 16-bit elements.
4619 auto isValid16BitArray = [](const Pointer &P) {
4620 const Descriptor *D = P.getFieldDesc();
4621 if (!D->isPrimitiveArray())
4622 return false;
4623 PrimType PT = D->getPrimType();
4624 return ((PT == PT_Sint16) || (PT == PT_Uint16));
4625 };
4626
4627 if (!isValid16BitArray(A) || !isValid16BitArray(B) || !isValid16BitArray(C))
4628 return false;
4629
4630 PrimType ElemT = A.getFieldDesc()->getPrimType();
4631 unsigned NumElems = A.getNumElems();
4632 assert(NumElems % 16 == 0 && "BMM operates on 256-bit lanes of 16 x i16");
4633 bool DstUnsigned = ElemT == PT_Uint16;
4634
4635 // Lanes are always 16-bit; gather them so the reduction below is untyped.
4636 SmallVector<uint16_t> AVals(NumElems), BVals(NumElems), Acc(NumElems);
4637 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
4638 for (unsigned I = 0; I != NumElems; ++I) {
4639 AVals[I] = (uint16_t)A.elem<T>(I).toAPSInt().getZExtValue();
4640 BVals[I] = (uint16_t)B.elem<T>(I).toAPSInt().getZExtValue();
4641 Acc[I] = (uint16_t)C.elem<T>(I).toAPSInt().getZExtValue();
4642 }
4643 });
4644
4645 for (unsigned Lane = 0; Lane != NumElems; Lane += 16) {
4646 for (unsigned I = 0; I != 16; ++I) {
4647 uint16_t AVal = AVals[Lane + I], DVal = Acc[Lane + I];
4648 for (unsigned J = 0; J != 16; ++J) {
4649 // Seed the reduction with the accumulator bit, then fold in each
4650 // product term with the same operator (OR for vbmacor, XOR for
4651 // vbmacxor).
4652 unsigned Bit = (DVal >> J) & 1u;
4653 for (unsigned K = 0; K != 16; ++K) {
4654 unsigned Product = ((AVal >> K) & 1u) & ((BVals[Lane + K] >> J) & 1u);
4655 Bit = IsXor ? (Bit ^ Product) : (Bit | Product);
4656 }
4657 DVal = (DVal & ~(uint16_t(1) << J)) | (uint16_t(Bit) << J);
4658 }
4659 Acc[Lane + I] = DVal;
4660 }
4661 }
4662
4663 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
4664 for (unsigned I = 0; I != NumElems; ++I)
4665 Dst.elem<T>(I) = static_cast<T>(APSInt(APInt(16, Acc[I]), DstUnsigned));
4666 });
4667 Dst.initializeAllElements();
4668 return true;
4669}
4670
4671bool InterpretBuiltin(InterpState &S, CodePtr OpPC, const CallExpr *Call,
4672 uint32_t BuiltinID) {
4673 const ASTContext &ASTCtx = S.getASTContext();
4674
4675 // BuiltinID is the raw ID baked into the bytecode. The "is constant
4676 // evaluated" gate needs the raw ID so that auxiliary-target IDs resolve into
4677 // the correct (aux-target) builtin records.
4678 if (!ASTCtx.BuiltinInfo.isConstantEvaluated(ID: BuiltinID))
4679 return Invalid(S, OpPC);
4680
4681 // Convert an auxiliary x86 target builtin ID to its canonical X86::BI* value
4682 // so the target-specific cases below (and the handlers they call) match. This
4683 // is a cheap integer operation (a single comparison for the common,
4684 // target-independent case); we deliberately avoid re-deriving the ID from the
4685 // call expression, which is comparatively slow.
4686 BuiltinID = ConvertBuiltinIDToX86BuiltinID(Ctx: ASTCtx, BuiltinID);
4687
4688 const InterpFrame *Frame = S.Current;
4689 switch (BuiltinID) {
4690 case Builtin::BI__builtin_is_constant_evaluated:
4691 return interp__builtin_is_constant_evaluated(S, OpPC, Frame, Call);
4692
4693 case Builtin::BI__builtin_assume:
4694 case Builtin::BI__assume:
4695 return interp__builtin_assume(S, OpPC, Frame, Call);
4696
4697 case Builtin::BI__builtin_strcmp:
4698 case Builtin::BIstrcmp:
4699 case Builtin::BI__builtin_strncmp:
4700 case Builtin::BIstrncmp:
4701 case Builtin::BI__builtin_wcsncmp:
4702 case Builtin::BIwcsncmp:
4703 case Builtin::BI__builtin_wcscmp:
4704 case Builtin::BIwcscmp:
4705 return interp__builtin_strcmp(S, OpPC, Frame, Call, ID: BuiltinID);
4706
4707 case Builtin::BI__builtin_strlen:
4708 case Builtin::BIstrlen:
4709 case Builtin::BI__builtin_wcslen:
4710 case Builtin::BIwcslen:
4711 return interp__builtin_strlen(S, OpPC, Frame, Call, ID: BuiltinID);
4712
4713 case Builtin::BI__builtin_nan:
4714 case Builtin::BI__builtin_nanf:
4715 case Builtin::BI__builtin_nanl:
4716 case Builtin::BI__builtin_nanf16:
4717 case Builtin::BI__builtin_nanf128:
4718 return interp__builtin_nan(S, OpPC, Frame, Call, /*Signaling=*/false);
4719
4720 case Builtin::BI__builtin_nans:
4721 case Builtin::BI__builtin_nansf:
4722 case Builtin::BI__builtin_nansl:
4723 case Builtin::BI__builtin_nansf16:
4724 case Builtin::BI__builtin_nansf128:
4725 return interp__builtin_nan(S, OpPC, Frame, Call, /*Signaling=*/true);
4726
4727 case Builtin::BI__builtin_huge_val:
4728 case Builtin::BI__builtin_huge_valf:
4729 case Builtin::BI__builtin_huge_vall:
4730 case Builtin::BI__builtin_huge_valf16:
4731 case Builtin::BI__builtin_huge_valf128:
4732 case Builtin::BI__builtin_inf:
4733 case Builtin::BI__builtin_inff:
4734 case Builtin::BI__builtin_infl:
4735 case Builtin::BI__builtin_inff16:
4736 case Builtin::BI__builtin_inff128:
4737 return interp__builtin_inf(S, OpPC, Frame, Call);
4738
4739 case Builtin::BI__builtin_copysign:
4740 case Builtin::BI__builtin_copysignf:
4741 case Builtin::BI__builtin_copysignl:
4742 case Builtin::BI__builtin_copysignf128:
4743 return interp__builtin_copysign(S, OpPC, Frame);
4744
4745 case Builtin::BI__builtin_fmin:
4746 case Builtin::BI__builtin_fminf:
4747 case Builtin::BI__builtin_fminl:
4748 case Builtin::BI__builtin_fminf16:
4749 case Builtin::BI__builtin_fminf128:
4750 return interp__builtin_fmin(S, OpPC, Frame, /*IsNumBuiltin=*/false);
4751
4752 case Builtin::BI__builtin_fminimum_num:
4753 case Builtin::BI__builtin_fminimum_numf:
4754 case Builtin::BI__builtin_fminimum_numl:
4755 case Builtin::BI__builtin_fminimum_numf16:
4756 case Builtin::BI__builtin_fminimum_numf128:
4757 return interp__builtin_fmin(S, OpPC, Frame, /*IsNumBuiltin=*/true);
4758
4759 case Builtin::BI__builtin_fmax:
4760 case Builtin::BI__builtin_fmaxf:
4761 case Builtin::BI__builtin_fmaxl:
4762 case Builtin::BI__builtin_fmaxf16:
4763 case Builtin::BI__builtin_fmaxf128:
4764 return interp__builtin_fmax(S, OpPC, Frame, /*IsNumBuiltin=*/false);
4765
4766 case Builtin::BI__builtin_fmaximum_num:
4767 case Builtin::BI__builtin_fmaximum_numf:
4768 case Builtin::BI__builtin_fmaximum_numl:
4769 case Builtin::BI__builtin_fmaximum_numf16:
4770 case Builtin::BI__builtin_fmaximum_numf128:
4771 return interp__builtin_fmax(S, OpPC, Frame, /*IsNumBuiltin=*/true);
4772
4773 case Builtin::BI__builtin_isnan:
4774 return interp__builtin_isnan(S, OpPC, Frame, Call);
4775
4776 case Builtin::BI__builtin_issignaling:
4777 return interp__builtin_issignaling(S, OpPC, Frame, Call);
4778
4779 case Builtin::BI__builtin_isinf:
4780 return interp__builtin_isinf(S, OpPC, Frame, /*Sign=*/CheckSign: false, Call);
4781
4782 case Builtin::BI__builtin_isinf_sign:
4783 return interp__builtin_isinf(S, OpPC, Frame, /*Sign=*/CheckSign: true, Call);
4784
4785 case Builtin::BI__builtin_isfinite:
4786 return interp__builtin_isfinite(S, OpPC, Frame, Call);
4787
4788 case Builtin::BI__builtin_isnormal:
4789 return interp__builtin_isnormal(S, OpPC, Frame, Call);
4790
4791 case Builtin::BI__builtin_issubnormal:
4792 return interp__builtin_issubnormal(S, OpPC, Frame, Call);
4793
4794 case Builtin::BI__builtin_iszero:
4795 return interp__builtin_iszero(S, OpPC, Frame, Call);
4796
4797 case Builtin::BI__builtin_signbit:
4798 case Builtin::BI__builtin_signbitf:
4799 case Builtin::BI__builtin_signbitl:
4800 return interp__builtin_signbit(S, OpPC, Frame, Call);
4801
4802 case Builtin::BI__builtin_isgreater:
4803 case Builtin::BI__builtin_isgreaterequal:
4804 case Builtin::BI__builtin_isless:
4805 case Builtin::BI__builtin_islessequal:
4806 case Builtin::BI__builtin_islessgreater:
4807 case Builtin::BI__builtin_isunordered:
4808 return interp_floating_comparison(S, OpPC, Call, ID: BuiltinID);
4809
4810 case Builtin::BI__builtin_isfpclass:
4811 return interp__builtin_isfpclass(S, OpPC, Frame, Call);
4812
4813 case Builtin::BI__builtin_fpclassify:
4814 return interp__builtin_fpclassify(S, OpPC, Frame, Call);
4815
4816 case Builtin::BI__builtin_fabs:
4817 case Builtin::BI__builtin_fabsf:
4818 case Builtin::BI__builtin_fabsl:
4819 case Builtin::BI__builtin_fabsf128:
4820 return interp__builtin_fabs(S, OpPC, Frame);
4821
4822 case Builtin::BI__builtin_abs:
4823 case Builtin::BI__builtin_labs:
4824 case Builtin::BI__builtin_llabs:
4825 return interp__builtin_abs(S, OpPC, Frame, Call);
4826
4827 case Builtin::BI__builtin_popcount:
4828 case Builtin::BI__builtin_popcountl:
4829 case Builtin::BI__builtin_popcountll:
4830 case Builtin::BI__builtin_popcountg:
4831 case Builtin::BI__popcnt16: // Microsoft variants of popcount
4832 case Builtin::BI__popcnt:
4833 case Builtin::BI__popcnt64:
4834 return interp__builtin_popcount(S, OpPC, Frame, Call);
4835
4836 case Builtin::BI__builtin_parity:
4837 case Builtin::BI__builtin_parityl:
4838 case Builtin::BI__builtin_parityll:
4839 return interp__builtin_elementwise_int_unaryop(
4840 S, OpPC, Call, Fn: [](const APSInt &Val) {
4841 return APInt(Val.getBitWidth(), Val.popcount() % 2);
4842 });
4843 case Builtin::BI__builtin_clrsb:
4844 case Builtin::BI__builtin_clrsbl:
4845 case Builtin::BI__builtin_clrsbll:
4846 return interp__builtin_elementwise_int_unaryop(
4847 S, OpPC, Call, Fn: [](const APSInt &Val) {
4848 return APInt(Val.getBitWidth(),
4849 Val.getBitWidth() - Val.getSignificantBits());
4850 });
4851 case Builtin::BI__builtin_bitreverseg:
4852 case Builtin::BI__builtin_bitreverse8:
4853 case Builtin::BI__builtin_bitreverse16:
4854 case Builtin::BI__builtin_bitreverse32:
4855 case Builtin::BI__builtin_bitreverse64:
4856 return interp__builtin_elementwise_int_unaryop(
4857 S, OpPC, Call, Fn: [](const APSInt &Val) { return Val.reverseBits(); });
4858
4859 case Builtin::BI__builtin_classify_type:
4860 return interp__builtin_classify_type(S, OpPC, Frame, Call);
4861
4862 case Builtin::BI__builtin_expect:
4863 case Builtin::BI__builtin_expect_with_probability:
4864 return interp__builtin_expect(S, OpPC, Frame, Call);
4865
4866 case Builtin::BI__builtin_rotateleft8:
4867 case Builtin::BI__builtin_rotateleft16:
4868 case Builtin::BI__builtin_rotateleft32:
4869 case Builtin::BI__builtin_rotateleft64:
4870 case Builtin::BI__builtin_stdc_rotate_left:
4871 case Builtin::BIstdc_rotate_left_uc:
4872 case Builtin::BIstdc_rotate_left_us:
4873 case Builtin::BIstdc_rotate_left_ui:
4874 case Builtin::BIstdc_rotate_left_ul:
4875 case Builtin::BIstdc_rotate_left_ull:
4876 case Builtin::BI_rotl8: // Microsoft variants of rotate left
4877 case Builtin::BI_rotl16:
4878 case Builtin::BI_rotl:
4879 case Builtin::BI_lrotl:
4880 case Builtin::BI_rotl64:
4881 case Builtin::BI__builtin_rotateright8:
4882 case Builtin::BI__builtin_rotateright16:
4883 case Builtin::BI__builtin_rotateright32:
4884 case Builtin::BI__builtin_rotateright64:
4885 case Builtin::BI__builtin_stdc_rotate_right:
4886 case Builtin::BIstdc_rotate_right_uc:
4887 case Builtin::BIstdc_rotate_right_us:
4888 case Builtin::BIstdc_rotate_right_ui:
4889 case Builtin::BIstdc_rotate_right_ul:
4890 case Builtin::BIstdc_rotate_right_ull:
4891 case Builtin::BI_rotr8: // Microsoft variants of rotate right
4892 case Builtin::BI_rotr16:
4893 case Builtin::BI_rotr:
4894 case Builtin::BI_lrotr:
4895 case Builtin::BI_rotr64: {
4896 // Determine if this is a rotate right operation
4897 bool IsRotateRight;
4898 switch (BuiltinID) {
4899 case Builtin::BI__builtin_rotateright8:
4900 case Builtin::BI__builtin_rotateright16:
4901 case Builtin::BI__builtin_rotateright32:
4902 case Builtin::BI__builtin_rotateright64:
4903 case Builtin::BI__builtin_stdc_rotate_right:
4904 case Builtin::BIstdc_rotate_right_uc:
4905 case Builtin::BIstdc_rotate_right_us:
4906 case Builtin::BIstdc_rotate_right_ui:
4907 case Builtin::BIstdc_rotate_right_ul:
4908 case Builtin::BIstdc_rotate_right_ull:
4909 case Builtin::BI_rotr8:
4910 case Builtin::BI_rotr16:
4911 case Builtin::BI_rotr:
4912 case Builtin::BI_lrotr:
4913 case Builtin::BI_rotr64:
4914 IsRotateRight = true;
4915 break;
4916 default:
4917 IsRotateRight = false;
4918 break;
4919 }
4920
4921 return interp__builtin_elementwise_int_binop(
4922 S, OpPC, Call, Fn: [IsRotateRight](const APSInt &Value, APSInt Amount) {
4923 Amount = NormalizeRotateAmount(Value, Amount);
4924 return IsRotateRight ? Value.rotr(rotateAmt: Amount.getZExtValue())
4925 : Value.rotl(rotateAmt: Amount.getZExtValue());
4926 });
4927 }
4928
4929 case Builtin::BIstdc_leading_zeros_uc:
4930 case Builtin::BIstdc_leading_zeros_us:
4931 case Builtin::BIstdc_leading_zeros_ui:
4932 case Builtin::BIstdc_leading_zeros_ul:
4933 case Builtin::BIstdc_leading_zeros_ull:
4934 case Builtin::BI__builtin_stdc_leading_zeros: {
4935 unsigned ResWidth = S.getASTContext().getIntWidth(T: Call->getType());
4936 return interp__builtin_elementwise_int_unaryop(
4937 S, OpPC, Call, Fn: [ResWidth](const APSInt &Val) {
4938 return APInt(ResWidth, Val.countl_zero());
4939 });
4940 }
4941
4942 case Builtin::BIstdc_leading_ones_uc:
4943 case Builtin::BIstdc_leading_ones_us:
4944 case Builtin::BIstdc_leading_ones_ui:
4945 case Builtin::BIstdc_leading_ones_ul:
4946 case Builtin::BIstdc_leading_ones_ull:
4947 case Builtin::BI__builtin_stdc_leading_ones: {
4948 unsigned ResWidth = S.getASTContext().getIntWidth(T: Call->getType());
4949 return interp__builtin_elementwise_int_unaryop(
4950 S, OpPC, Call, Fn: [ResWidth](const APSInt &Val) {
4951 return APInt(ResWidth, Val.countl_one());
4952 });
4953 }
4954
4955 case Builtin::BIstdc_trailing_zeros_uc:
4956 case Builtin::BIstdc_trailing_zeros_us:
4957 case Builtin::BIstdc_trailing_zeros_ui:
4958 case Builtin::BIstdc_trailing_zeros_ul:
4959 case Builtin::BIstdc_trailing_zeros_ull:
4960 case Builtin::BI__builtin_stdc_trailing_zeros: {
4961 unsigned ResWidth = S.getASTContext().getIntWidth(T: Call->getType());
4962 return interp__builtin_elementwise_int_unaryop(
4963 S, OpPC, Call, Fn: [ResWidth](const APSInt &Val) {
4964 return APInt(ResWidth, Val.countr_zero());
4965 });
4966 }
4967
4968 case Builtin::BIstdc_trailing_ones_uc:
4969 case Builtin::BIstdc_trailing_ones_us:
4970 case Builtin::BIstdc_trailing_ones_ui:
4971 case Builtin::BIstdc_trailing_ones_ul:
4972 case Builtin::BIstdc_trailing_ones_ull:
4973 case Builtin::BI__builtin_stdc_trailing_ones: {
4974 unsigned ResWidth = S.getASTContext().getIntWidth(T: Call->getType());
4975 return interp__builtin_elementwise_int_unaryop(
4976 S, OpPC, Call, Fn: [ResWidth](const APSInt &Val) {
4977 return APInt(ResWidth, Val.countr_one());
4978 });
4979 }
4980
4981 case Builtin::BIstdc_first_leading_zero_uc:
4982 case Builtin::BIstdc_first_leading_zero_us:
4983 case Builtin::BIstdc_first_leading_zero_ui:
4984 case Builtin::BIstdc_first_leading_zero_ul:
4985 case Builtin::BIstdc_first_leading_zero_ull:
4986 case Builtin::BI__builtin_stdc_first_leading_zero: {
4987 unsigned ResWidth = S.getASTContext().getIntWidth(T: Call->getType());
4988 return interp__builtin_elementwise_int_unaryop(
4989 S, OpPC, Call, Fn: [ResWidth](const APSInt &Val) {
4990 return APInt(ResWidth, Val.isAllOnes() ? 0 : Val.countl_one() + 1);
4991 });
4992 }
4993
4994 case Builtin::BIstdc_first_leading_one_uc:
4995 case Builtin::BIstdc_first_leading_one_us:
4996 case Builtin::BIstdc_first_leading_one_ui:
4997 case Builtin::BIstdc_first_leading_one_ul:
4998 case Builtin::BIstdc_first_leading_one_ull:
4999 case Builtin::BI__builtin_stdc_first_leading_one: {
5000 unsigned ResWidth = S.getASTContext().getIntWidth(T: Call->getType());
5001 return interp__builtin_elementwise_int_unaryop(
5002 S, OpPC, Call, Fn: [ResWidth](const APSInt &Val) {
5003 return APInt(ResWidth, Val.isZero() ? 0 : Val.countl_zero() + 1);
5004 });
5005 }
5006
5007 case Builtin::BIstdc_first_trailing_zero_uc:
5008 case Builtin::BIstdc_first_trailing_zero_us:
5009 case Builtin::BIstdc_first_trailing_zero_ui:
5010 case Builtin::BIstdc_first_trailing_zero_ul:
5011 case Builtin::BIstdc_first_trailing_zero_ull:
5012 case Builtin::BI__builtin_stdc_first_trailing_zero: {
5013 unsigned ResWidth = S.getASTContext().getIntWidth(T: Call->getType());
5014 return interp__builtin_elementwise_int_unaryop(
5015 S, OpPC, Call, Fn: [ResWidth](const APSInt &Val) {
5016 return APInt(ResWidth, Val.isAllOnes() ? 0 : Val.countr_one() + 1);
5017 });
5018 }
5019
5020 case Builtin::BIstdc_first_trailing_one_uc:
5021 case Builtin::BIstdc_first_trailing_one_us:
5022 case Builtin::BIstdc_first_trailing_one_ui:
5023 case Builtin::BIstdc_first_trailing_one_ul:
5024 case Builtin::BIstdc_first_trailing_one_ull:
5025 case Builtin::BI__builtin_stdc_first_trailing_one: {
5026 unsigned ResWidth = S.getASTContext().getIntWidth(T: Call->getType());
5027 return interp__builtin_elementwise_int_unaryop(
5028 S, OpPC, Call, Fn: [ResWidth](const APSInt &Val) {
5029 return APInt(ResWidth, Val.isZero() ? 0 : Val.countr_zero() + 1);
5030 });
5031 }
5032
5033 case Builtin::BIstdc_count_zeros_uc:
5034 case Builtin::BIstdc_count_zeros_us:
5035 case Builtin::BIstdc_count_zeros_ui:
5036 case Builtin::BIstdc_count_zeros_ul:
5037 case Builtin::BIstdc_count_zeros_ull:
5038 case Builtin::BI__builtin_stdc_count_zeros: {
5039 unsigned ResWidth = S.getASTContext().getIntWidth(T: Call->getType());
5040 return interp__builtin_elementwise_int_unaryop(
5041 S, OpPC, Call, Fn: [ResWidth](const APSInt &Val) {
5042 unsigned BitWidth = Val.getBitWidth();
5043 return APInt(ResWidth, BitWidth - Val.popcount());
5044 });
5045 }
5046
5047 case Builtin::BIstdc_count_ones_uc:
5048 case Builtin::BIstdc_count_ones_us:
5049 case Builtin::BIstdc_count_ones_ui:
5050 case Builtin::BIstdc_count_ones_ul:
5051 case Builtin::BIstdc_count_ones_ull:
5052 case Builtin::BI__builtin_stdc_count_ones: {
5053 unsigned ResWidth = S.getASTContext().getIntWidth(T: Call->getType());
5054 return interp__builtin_elementwise_int_unaryop(
5055 S, OpPC, Call, Fn: [ResWidth](const APSInt &Val) {
5056 return APInt(ResWidth, Val.popcount());
5057 });
5058 }
5059
5060 case Builtin::BIstdc_has_single_bit_uc:
5061 case Builtin::BIstdc_has_single_bit_us:
5062 case Builtin::BIstdc_has_single_bit_ui:
5063 case Builtin::BIstdc_has_single_bit_ul:
5064 case Builtin::BIstdc_has_single_bit_ull:
5065 case Builtin::BI__builtin_stdc_has_single_bit: {
5066 unsigned ResWidth = S.getASTContext().getIntWidth(T: Call->getType());
5067 return interp__builtin_elementwise_int_unaryop(
5068 S, OpPC, Call, Fn: [ResWidth](const APSInt &Val) {
5069 return APInt(ResWidth, Val.popcount() == 1 ? 1 : 0);
5070 });
5071 }
5072
5073 case Builtin::BIstdc_bit_width_uc:
5074 case Builtin::BIstdc_bit_width_us:
5075 case Builtin::BIstdc_bit_width_ui:
5076 case Builtin::BIstdc_bit_width_ul:
5077 case Builtin::BIstdc_bit_width_ull:
5078 case Builtin::BI__builtin_stdc_bit_width: {
5079 unsigned ResWidth = S.getASTContext().getIntWidth(T: Call->getType());
5080 return interp__builtin_elementwise_int_unaryop(
5081 S, OpPC, Call, Fn: [ResWidth](const APSInt &Val) {
5082 unsigned BitWidth = Val.getBitWidth();
5083 return APInt(ResWidth, BitWidth - Val.countl_zero());
5084 });
5085 }
5086
5087 case Builtin::BIstdc_bit_floor_uc:
5088 case Builtin::BIstdc_bit_floor_us:
5089 case Builtin::BIstdc_bit_floor_ui:
5090 case Builtin::BIstdc_bit_floor_ul:
5091 case Builtin::BIstdc_bit_floor_ull:
5092 case Builtin::BI__builtin_stdc_bit_floor:
5093 return interp__builtin_elementwise_int_unaryop(
5094 S, OpPC, Call, Fn: [](const APSInt &Val) {
5095 unsigned BitWidth = Val.getBitWidth();
5096 if (Val.isZero())
5097 return APInt::getZero(numBits: BitWidth);
5098 return APInt::getOneBitSet(numBits: BitWidth,
5099 BitNo: BitWidth - Val.countl_zero() - 1);
5100 });
5101
5102 case Builtin::BIstdc_bit_ceil_uc:
5103 case Builtin::BIstdc_bit_ceil_us:
5104 case Builtin::BIstdc_bit_ceil_ui:
5105 case Builtin::BIstdc_bit_ceil_ul:
5106 case Builtin::BIstdc_bit_ceil_ull:
5107 case Builtin::BI__builtin_stdc_bit_ceil:
5108 return interp__builtin_elementwise_int_unaryop(
5109 S, OpPC, Call, Fn: [](const APSInt &Val) {
5110 unsigned BitWidth = Val.getBitWidth();
5111 if (Val.ule(RHS: 1))
5112 return APInt(BitWidth, 1);
5113 APInt V = Val;
5114 APInt ValMinusOne = V - 1;
5115 unsigned LeadingZeros = ValMinusOne.countl_zero();
5116 if (LeadingZeros == 0)
5117 return APInt(BitWidth, 0); // overflows; wrap to 0
5118 return APInt::getOneBitSet(numBits: BitWidth, BitNo: BitWidth - LeadingZeros);
5119 });
5120
5121 case Builtin::BI__builtin_ffs:
5122 case Builtin::BI__builtin_ffsl:
5123 case Builtin::BI__builtin_ffsll:
5124 return interp__builtin_elementwise_int_unaryop(
5125 S, OpPC, Call, Fn: [](const APSInt &Val) {
5126 return APInt(Val.getBitWidth(),
5127 Val.isZero() ? 0u : Val.countTrailingZeros() + 1u);
5128 });
5129
5130 case Builtin::BIaddressof:
5131 case Builtin::BI__addressof:
5132 case Builtin::BI__builtin_addressof:
5133 assert(isNoopBuiltin(BuiltinID));
5134 return interp__builtin_addressof(S, OpPC, Frame, Call);
5135
5136 case Builtin::BIas_const:
5137 case Builtin::BIforward:
5138 case Builtin::BIforward_like:
5139 case Builtin::BImove:
5140 case Builtin::BImove_if_noexcept:
5141 assert(isNoopBuiltin(BuiltinID));
5142 return interp__builtin_move(S, OpPC, Frame, Call);
5143
5144 case Builtin::BI__builtin_eh_return_data_regno:
5145 return interp__builtin_eh_return_data_regno(S, OpPC, Frame, Call);
5146
5147 case Builtin::BI__builtin_launder:
5148 assert(isNoopBuiltin(BuiltinID));
5149 return true;
5150
5151 case Builtin::BI__builtin_add_overflow:
5152 case Builtin::BI__builtin_sub_overflow:
5153 case Builtin::BI__builtin_mul_overflow:
5154 case Builtin::BI__builtin_sadd_overflow:
5155 case Builtin::BI__builtin_uadd_overflow:
5156 case Builtin::BI__builtin_uaddl_overflow:
5157 case Builtin::BI__builtin_uaddll_overflow:
5158 case Builtin::BI__builtin_usub_overflow:
5159 case Builtin::BI__builtin_usubl_overflow:
5160 case Builtin::BI__builtin_usubll_overflow:
5161 case Builtin::BI__builtin_umul_overflow:
5162 case Builtin::BI__builtin_umull_overflow:
5163 case Builtin::BI__builtin_umulll_overflow:
5164 case Builtin::BI__builtin_saddl_overflow:
5165 case Builtin::BI__builtin_saddll_overflow:
5166 case Builtin::BI__builtin_ssub_overflow:
5167 case Builtin::BI__builtin_ssubl_overflow:
5168 case Builtin::BI__builtin_ssubll_overflow:
5169 case Builtin::BI__builtin_smul_overflow:
5170 case Builtin::BI__builtin_smull_overflow:
5171 case Builtin::BI__builtin_smulll_overflow:
5172 return interp__builtin_overflowop(S, OpPC, Call, BuiltinOp: BuiltinID);
5173
5174 case Builtin::BI__builtin_addcb:
5175 case Builtin::BI__builtin_addcs:
5176 case Builtin::BI__builtin_addc:
5177 case Builtin::BI__builtin_addcl:
5178 case Builtin::BI__builtin_addcll:
5179 case Builtin::BI__builtin_subcb:
5180 case Builtin::BI__builtin_subcs:
5181 case Builtin::BI__builtin_subc:
5182 case Builtin::BI__builtin_subcl:
5183 case Builtin::BI__builtin_subcll:
5184 return interp__builtin_carryop(S, OpPC, Frame, Call, BuiltinOp: BuiltinID);
5185
5186 case Builtin::BI__builtin_clz:
5187 case Builtin::BI__builtin_clzl:
5188 case Builtin::BI__builtin_clzll:
5189 case Builtin::BI__builtin_clzs:
5190 case Builtin::BI__builtin_clzg:
5191 case Builtin::BI__lzcnt16: // Microsoft variants of count leading-zeroes
5192 case Builtin::BI__lzcnt:
5193 case Builtin::BI__lzcnt64:
5194 return interp__builtin_clz(S, OpPC, Frame, Call, BuiltinOp: BuiltinID);
5195
5196 case Builtin::BI__builtin_ctz:
5197 case Builtin::BI__builtin_ctzl:
5198 case Builtin::BI__builtin_ctzll:
5199 case Builtin::BI__builtin_ctzs:
5200 case Builtin::BI__builtin_ctzg:
5201 return interp__builtin_ctz(S, OpPC, Frame, Call, BuiltinID);
5202
5203 case Builtin::BI__builtin_elementwise_clzg:
5204 case Builtin::BI__builtin_elementwise_ctzg:
5205 return interp__builtin_elementwise_countzeroes(S, OpPC, Frame, Call,
5206 BuiltinID);
5207 case Builtin::BI__builtin_bswapg:
5208 case Builtin::BI__builtin_bswap16:
5209 case Builtin::BI__builtin_bswap32:
5210 case Builtin::BI__builtin_bswap64:
5211 case Builtin::BIstdc_memreverse8u8:
5212 case Builtin::BIstdc_memreverse8u16:
5213 case Builtin::BIstdc_memreverse8u32:
5214 case Builtin::BIstdc_memreverse8u64:
5215 return interp__builtin_bswap(S, OpPC, Frame, Call);
5216
5217 case Builtin::BI__atomic_always_lock_free:
5218 case Builtin::BI__atomic_is_lock_free:
5219 return interp__builtin_atomic_lock_free(S, OpPC, Frame, Call, BuiltinOp: BuiltinID);
5220
5221 case Builtin::BI__c11_atomic_is_lock_free:
5222 return interp__builtin_c11_atomic_is_lock_free(S, OpPC, Frame, Call);
5223
5224 case Builtin::BI__builtin_complex:
5225 return interp__builtin_complex(S, OpPC, Frame, Call);
5226
5227 case Builtin::BI__builtin_is_aligned:
5228 case Builtin::BI__builtin_align_up:
5229 case Builtin::BI__builtin_align_down:
5230 return interp__builtin_is_aligned_up_down(S, OpPC, Frame, Call, BuiltinOp: BuiltinID);
5231
5232 case Builtin::BI__builtin_assume_aligned:
5233 return interp__builtin_assume_aligned(S, OpPC, Frame, Call);
5234
5235 case clang::X86::BI__builtin_ia32_crc32qi:
5236 return interp__builtin_ia32_crc32(S, OpPC, Frame, Call, DataBytes: 1);
5237 case clang::X86::BI__builtin_ia32_crc32hi:
5238 return interp__builtin_ia32_crc32(S, OpPC, Frame, Call, DataBytes: 2);
5239 case clang::X86::BI__builtin_ia32_crc32si:
5240 return interp__builtin_ia32_crc32(S, OpPC, Frame, Call, DataBytes: 4);
5241 case clang::X86::BI__builtin_ia32_crc32di:
5242 return interp__builtin_ia32_crc32(S, OpPC, Frame, Call, DataBytes: 8);
5243
5244 case clang::X86::BI__builtin_ia32_bextr_u32:
5245 case clang::X86::BI__builtin_ia32_bextr_u64:
5246 case clang::X86::BI__builtin_ia32_bextri_u32:
5247 case clang::X86::BI__builtin_ia32_bextri_u64:
5248 return interp__builtin_elementwise_int_binop(
5249 S, OpPC, Call, Fn: [](const APSInt &Val, const APSInt &Idx) {
5250 unsigned BitWidth = Val.getBitWidth();
5251 uint64_t Shift = Idx.extractBitsAsZExtValue(numBits: 8, bitPosition: 0);
5252 uint64_t Length = Idx.extractBitsAsZExtValue(numBits: 8, bitPosition: 8);
5253 if (Length > BitWidth) {
5254 Length = BitWidth;
5255 }
5256
5257 // Handle out of bounds cases.
5258 if (Length == 0 || Shift >= BitWidth)
5259 return APInt(BitWidth, 0);
5260
5261 uint64_t Result = Val.getZExtValue() >> Shift;
5262 Result &= llvm::maskTrailingOnes<uint64_t>(N: Length);
5263 return APInt(BitWidth, Result);
5264 });
5265
5266 case clang::X86::BI__builtin_ia32_bzhi_si:
5267 case clang::X86::BI__builtin_ia32_bzhi_di:
5268 return interp__builtin_elementwise_int_binop(
5269 S, OpPC, Call, Fn: [](const APSInt &Val, const APSInt &Idx) {
5270 unsigned BitWidth = Val.getBitWidth();
5271 uint64_t Index = Idx.extractBitsAsZExtValue(numBits: 8, bitPosition: 0);
5272 APSInt Result = Val;
5273
5274 if (Index < BitWidth)
5275 Result.clearHighBits(hiBits: BitWidth - Index);
5276
5277 return Result;
5278 });
5279
5280 case clang::X86::BI__builtin_ia32_ktestcqi:
5281 case clang::X86::BI__builtin_ia32_ktestchi:
5282 case clang::X86::BI__builtin_ia32_ktestcsi:
5283 case clang::X86::BI__builtin_ia32_ktestcdi:
5284 return interp__builtin_elementwise_int_binop(
5285 S, OpPC, Call, Fn: [](const APSInt &A, const APSInt &B) {
5286 return APInt(sizeof(unsigned char) * 8, (~A & B) == 0);
5287 });
5288
5289 case clang::X86::BI__builtin_ia32_ktestzqi:
5290 case clang::X86::BI__builtin_ia32_ktestzhi:
5291 case clang::X86::BI__builtin_ia32_ktestzsi:
5292 case clang::X86::BI__builtin_ia32_ktestzdi:
5293 return interp__builtin_elementwise_int_binop(
5294 S, OpPC, Call, Fn: [](const APSInt &A, const APSInt &B) {
5295 return APInt(sizeof(unsigned char) * 8, (A & B) == 0);
5296 });
5297
5298 case clang::X86::BI__builtin_ia32_kortestcqi:
5299 case clang::X86::BI__builtin_ia32_kortestchi:
5300 case clang::X86::BI__builtin_ia32_kortestcsi:
5301 case clang::X86::BI__builtin_ia32_kortestcdi:
5302 return interp__builtin_elementwise_int_binop(
5303 S, OpPC, Call, Fn: [](const APSInt &A, const APSInt &B) {
5304 return APInt(sizeof(unsigned char) * 8, ~(A | B) == 0);
5305 });
5306
5307 case clang::X86::BI__builtin_ia32_kortestzqi:
5308 case clang::X86::BI__builtin_ia32_kortestzhi:
5309 case clang::X86::BI__builtin_ia32_kortestzsi:
5310 case clang::X86::BI__builtin_ia32_kortestzdi:
5311 return interp__builtin_elementwise_int_binop(
5312 S, OpPC, Call, Fn: [](const APSInt &A, const APSInt &B) {
5313 return APInt(sizeof(unsigned char) * 8, (A | B) == 0);
5314 });
5315
5316 case clang::X86::BI__builtin_ia32_kshiftliqi:
5317 case clang::X86::BI__builtin_ia32_kshiftlihi:
5318 case clang::X86::BI__builtin_ia32_kshiftlisi:
5319 case clang::X86::BI__builtin_ia32_kshiftlidi:
5320 return interp__builtin_elementwise_int_binop(
5321 S, OpPC, Call, Fn: [](const APSInt &LHS, const APSInt &RHS) {
5322 unsigned Amt = RHS.getZExtValue() & 0xFF;
5323 if (Amt >= LHS.getBitWidth())
5324 return APInt::getZero(numBits: LHS.getBitWidth());
5325 return LHS.shl(shiftAmt: Amt);
5326 });
5327
5328 case clang::X86::BI__builtin_ia32_kshiftriqi:
5329 case clang::X86::BI__builtin_ia32_kshiftrihi:
5330 case clang::X86::BI__builtin_ia32_kshiftrisi:
5331 case clang::X86::BI__builtin_ia32_kshiftridi:
5332 return interp__builtin_elementwise_int_binop(
5333 S, OpPC, Call, Fn: [](const APSInt &LHS, const APSInt &RHS) {
5334 unsigned Amt = RHS.getZExtValue() & 0xFF;
5335 if (Amt >= LHS.getBitWidth())
5336 return APInt::getZero(numBits: LHS.getBitWidth());
5337 return LHS.lshr(shiftAmt: Amt);
5338 });
5339
5340 case clang::X86::BI__builtin_ia32_lzcnt_u16:
5341 case clang::X86::BI__builtin_ia32_lzcnt_u32:
5342 case clang::X86::BI__builtin_ia32_lzcnt_u64:
5343 return interp__builtin_elementwise_int_unaryop(
5344 S, OpPC, Call, Fn: [](const APSInt &Src) {
5345 return APInt(Src.getBitWidth(), Src.countLeadingZeros());
5346 });
5347
5348 case clang::X86::BI__builtin_ia32_tzcnt_u16:
5349 case clang::X86::BI__builtin_ia32_tzcnt_u32:
5350 case clang::X86::BI__builtin_ia32_tzcnt_u64:
5351 return interp__builtin_elementwise_int_unaryop(
5352 S, OpPC, Call, Fn: [](const APSInt &Src) {
5353 return APInt(Src.getBitWidth(), Src.countTrailingZeros());
5354 });
5355
5356 case clang::X86::BI__builtin_ia32_pdep_si:
5357 case clang::X86::BI__builtin_ia32_pdep_di:
5358 case Builtin::BI__builtin_elementwise_pdep:
5359 return interp__builtin_elementwise_int_binop(S, OpPC, Call,
5360 Fn: llvm::APIntOps::pdep);
5361
5362 case clang::X86::BI__builtin_ia32_pext_si:
5363 case clang::X86::BI__builtin_ia32_pext_di:
5364 case Builtin::BI__builtin_elementwise_pext:
5365 return interp__builtin_elementwise_int_binop(S, OpPC, Call,
5366 Fn: llvm::APIntOps::pext);
5367
5368 case clang::X86::BI__builtin_ia32_addcarryx_u32:
5369 case clang::X86::BI__builtin_ia32_addcarryx_u64:
5370 return interp__builtin_ia32_addcarry_subborrow(S, OpPC, Frame, Call,
5371 /*IsAdd=*/true);
5372
5373 case clang::X86::BI__builtin_ia32_subborrow_u32:
5374 case clang::X86::BI__builtin_ia32_subborrow_u64:
5375 return interp__builtin_ia32_addcarry_subborrow(S, OpPC, Frame, Call,
5376 /*IsAdd=*/false);
5377
5378 case Builtin::BI__builtin_os_log_format_buffer_size:
5379 return interp__builtin_os_log_format_buffer_size(S, OpPC, Frame, Call);
5380
5381 case Builtin::BI__builtin_ptrauth_string_discriminator:
5382 return interp__builtin_ptrauth_string_discriminator(S, OpPC, Frame, Call);
5383
5384 case Builtin::BI__builtin_infer_alloc_token:
5385 return interp__builtin_infer_alloc_token(S, OpPC, Frame, Call);
5386
5387 case Builtin::BI__noop:
5388 pushInteger(S, Val: 0, QT: Call->getType());
5389 return true;
5390
5391 case Builtin::BI__builtin_operator_new:
5392 return interp__builtin_operator_new(S, OpPC, Frame, Call);
5393
5394 case Builtin::BI__builtin_operator_delete:
5395 return interp__builtin_operator_delete(S, OpPC, Frame, Call);
5396
5397 case Builtin::BI__arithmetic_fence:
5398 return interp__builtin_arithmetic_fence(S, OpPC, Frame, Call);
5399
5400 case Builtin::BI__builtin_reduce_add:
5401 case Builtin::BI__builtin_reduce_mul:
5402 case Builtin::BI__builtin_reduce_and:
5403 case Builtin::BI__builtin_reduce_or:
5404 case Builtin::BI__builtin_reduce_xor:
5405 case Builtin::BI__builtin_reduce_min:
5406 case Builtin::BI__builtin_reduce_max:
5407 return interp__builtin_vector_reduce(S, OpPC, Call, ID: BuiltinID);
5408
5409 case Builtin::BI__builtin_elementwise_popcount:
5410 return interp__builtin_elementwise_int_unaryop(
5411 S, OpPC, Call, Fn: [](const APSInt &Src) {
5412 return APInt(Src.getBitWidth(), Src.popcount());
5413 });
5414 case Builtin::BI__builtin_elementwise_bitreverse:
5415 return interp__builtin_elementwise_int_unaryop(
5416 S, OpPC, Call, Fn: [](const APSInt &Src) { return Src.reverseBits(); });
5417
5418 case Builtin::BI__builtin_elementwise_abs:
5419 return interp__builtin_elementwise_abs(S, OpPC, Frame, Call, BuiltinID);
5420
5421 case Builtin::BI__builtin_memcpy:
5422 case Builtin::BImemcpy:
5423 case Builtin::BI__builtin_wmemcpy:
5424 case Builtin::BIwmemcpy:
5425 case Builtin::BI__builtin_memmove:
5426 case Builtin::BImemmove:
5427 case Builtin::BI__builtin_wmemmove:
5428 case Builtin::BIwmemmove:
5429 return interp__builtin_memcpy(S, OpPC, Frame, Call, ID: BuiltinID);
5430
5431 case Builtin::BI__builtin_memcmp:
5432 case Builtin::BImemcmp:
5433 case Builtin::BI__builtin_bcmp:
5434 case Builtin::BIbcmp:
5435 case Builtin::BI__builtin_wmemcmp:
5436 case Builtin::BIwmemcmp:
5437 return interp__builtin_memcmp(S, OpPC, Frame, Call, ID: BuiltinID);
5438
5439 case Builtin::BImemchr:
5440 case Builtin::BI__builtin_memchr:
5441 case Builtin::BIstrchr:
5442 case Builtin::BI__builtin_strchr:
5443 case Builtin::BIwmemchr:
5444 case Builtin::BI__builtin_wmemchr:
5445 case Builtin::BIwcschr:
5446 case Builtin::BI__builtin_wcschr:
5447 case Builtin::BI__builtin_char_memchr:
5448 return interp__builtin_memchr(S, OpPC, Call, ID: BuiltinID);
5449
5450 case Builtin::BI__builtin_object_size:
5451 case Builtin::BI__builtin_dynamic_object_size:
5452 return interp__builtin_object_size(S, OpPC, Frame, Call);
5453
5454 case Builtin::BI__builtin_is_within_lifetime:
5455 return interp__builtin_is_within_lifetime(S, OpPC, Call);
5456
5457 case Builtin::BI__builtin_elementwise_add_sat:
5458 return interp__builtin_elementwise_int_binop(
5459 S, OpPC, Call, Fn: [](const APSInt &LHS, const APSInt &RHS) {
5460 return LHS.isSigned() ? LHS.sadd_sat(RHS) : LHS.uadd_sat(RHS);
5461 });
5462
5463 case Builtin::BI__builtin_elementwise_sub_sat:
5464 return interp__builtin_elementwise_int_binop(
5465 S, OpPC, Call, Fn: [](const APSInt &LHS, const APSInt &RHS) {
5466 return LHS.isSigned() ? LHS.ssub_sat(RHS) : LHS.usub_sat(RHS);
5467 });
5468 case X86::BI__builtin_ia32_extract128i256:
5469 case X86::BI__builtin_ia32_vextractf128_pd256:
5470 case X86::BI__builtin_ia32_vextractf128_ps256:
5471 case X86::BI__builtin_ia32_vextractf128_si256:
5472 return interp__builtin_ia32_extract_vector(S, OpPC, Call, ID: BuiltinID);
5473
5474 case X86::BI__builtin_ia32_extractf32x4_256_mask:
5475 case X86::BI__builtin_ia32_extractf32x4_mask:
5476 case X86::BI__builtin_ia32_extractf32x8_mask:
5477 case X86::BI__builtin_ia32_extractf64x2_256_mask:
5478 case X86::BI__builtin_ia32_extractf64x2_512_mask:
5479 case X86::BI__builtin_ia32_extractf64x4_mask:
5480 case X86::BI__builtin_ia32_extracti32x4_256_mask:
5481 case X86::BI__builtin_ia32_extracti32x4_mask:
5482 case X86::BI__builtin_ia32_extracti32x8_mask:
5483 case X86::BI__builtin_ia32_extracti64x2_256_mask:
5484 case X86::BI__builtin_ia32_extracti64x2_512_mask:
5485 case X86::BI__builtin_ia32_extracti64x4_mask:
5486 return interp__builtin_ia32_extract_vector_masked(S, OpPC, Call, ID: BuiltinID);
5487
5488 case clang::X86::BI__builtin_ia32_pmulhrsw128:
5489 case clang::X86::BI__builtin_ia32_pmulhrsw256:
5490 case clang::X86::BI__builtin_ia32_pmulhrsw512:
5491 return interp__builtin_elementwise_int_binop(
5492 S, OpPC, Call, Fn: [](const APSInt &LHS, const APSInt &RHS) {
5493 return (llvm::APIntOps::mulsExtended(C1: LHS, C2: RHS).ashr(ShiftAmt: 14) + 1)
5494 .extractBits(numBits: 16, bitPosition: 1);
5495 });
5496
5497 case clang::X86::BI__builtin_ia32_movmskps:
5498 case clang::X86::BI__builtin_ia32_movmskpd:
5499 case clang::X86::BI__builtin_ia32_pmovmskb128:
5500 case clang::X86::BI__builtin_ia32_pmovmskb256:
5501 case clang::X86::BI__builtin_ia32_movmskps256:
5502 case clang::X86::BI__builtin_ia32_movmskpd256: {
5503 return interp__builtin_ia32_movmsk_op(S, OpPC, Call);
5504 }
5505
5506 case X86::BI__builtin_ia32_psignb128:
5507 case X86::BI__builtin_ia32_psignb256:
5508 case X86::BI__builtin_ia32_psignw128:
5509 case X86::BI__builtin_ia32_psignw256:
5510 case X86::BI__builtin_ia32_psignd128:
5511 case X86::BI__builtin_ia32_psignd256:
5512 return interp__builtin_elementwise_int_binop(
5513 S, OpPC, Call, Fn: [](const APInt &AElem, const APInt &BElem) {
5514 if (BElem.isZero())
5515 return APInt::getZero(numBits: AElem.getBitWidth());
5516 if (BElem.isNegative())
5517 return -AElem;
5518 return AElem;
5519 });
5520
5521 case clang::X86::BI__builtin_ia32_pavgb128:
5522 case clang::X86::BI__builtin_ia32_pavgw128:
5523 case clang::X86::BI__builtin_ia32_pavgb256:
5524 case clang::X86::BI__builtin_ia32_pavgw256:
5525 case clang::X86::BI__builtin_ia32_pavgb512:
5526 case clang::X86::BI__builtin_ia32_pavgw512:
5527 return interp__builtin_elementwise_int_binop(S, OpPC, Call,
5528 Fn: llvm::APIntOps::avgCeilU);
5529
5530 case clang::X86::BI__builtin_ia32_pmaddubsw128:
5531 case clang::X86::BI__builtin_ia32_pmaddubsw256:
5532 case clang::X86::BI__builtin_ia32_pmaddubsw512:
5533 return interp__builtin_ia32_pmul(
5534 S, OpPC, Call,
5535 Fn: [](const APSInt &LoLHS, const APSInt &HiLHS, const APSInt &LoRHS,
5536 const APSInt &HiRHS) {
5537 unsigned BitWidth = 2 * LoLHS.getBitWidth();
5538 return (LoLHS.zext(width: BitWidth) * LoRHS.sext(width: BitWidth))
5539 .sadd_sat(RHS: (HiLHS.zext(width: BitWidth) * HiRHS.sext(width: BitWidth)));
5540 });
5541
5542 case clang::X86::BI__builtin_ia32_pmaddwd128:
5543 case clang::X86::BI__builtin_ia32_pmaddwd256:
5544 case clang::X86::BI__builtin_ia32_pmaddwd512:
5545 return interp__builtin_ia32_pmul(
5546 S, OpPC, Call,
5547 Fn: [](const APSInt &LoLHS, const APSInt &HiLHS, const APSInt &LoRHS,
5548 const APSInt &HiRHS) {
5549 unsigned BitWidth = 2 * LoLHS.getBitWidth();
5550 return (LoLHS.sext(width: BitWidth) * LoRHS.sext(width: BitWidth)) +
5551 (HiLHS.sext(width: BitWidth) * HiRHS.sext(width: BitWidth));
5552 });
5553
5554 case clang::X86::BI__builtin_ia32_psadbw128:
5555 case clang::X86::BI__builtin_ia32_psadbw256:
5556 case clang::X86::BI__builtin_ia32_psadbw512:
5557 return interp__builtin_ia32_psadbw(S, OpPC, Call);
5558
5559 case clang::X86::BI__builtin_ia32_dbpsadbw128:
5560 case clang::X86::BI__builtin_ia32_dbpsadbw256:
5561 case clang::X86::BI__builtin_ia32_dbpsadbw512:
5562 return interp__builtin_ia32_dbpsadbw(S, OpPC, Call);
5563
5564 case clang::X86::BI__builtin_ia32_mpsadbw128:
5565 case clang::X86::BI__builtin_ia32_mpsadbw256:
5566 return interp__builtin_ia32_mpsadbw(S, OpPC, Call);
5567
5568 case clang::X86::BI__builtin_ia32_pmulhuw128:
5569 case clang::X86::BI__builtin_ia32_pmulhuw256:
5570 case clang::X86::BI__builtin_ia32_pmulhuw512:
5571 return interp__builtin_elementwise_int_binop(S, OpPC, Call,
5572 Fn: llvm::APIntOps::mulhu);
5573
5574 case clang::X86::BI__builtin_ia32_pmulhw128:
5575 case clang::X86::BI__builtin_ia32_pmulhw256:
5576 case clang::X86::BI__builtin_ia32_pmulhw512:
5577 return interp__builtin_elementwise_int_binop(S, OpPC, Call,
5578 Fn: llvm::APIntOps::mulhs);
5579
5580 case clang::X86::BI__builtin_ia32_psllv2di:
5581 case clang::X86::BI__builtin_ia32_psllv4di:
5582 case clang::X86::BI__builtin_ia32_psllv4si:
5583 case clang::X86::BI__builtin_ia32_psllv8di:
5584 case clang::X86::BI__builtin_ia32_psllv8hi:
5585 case clang::X86::BI__builtin_ia32_psllv8si:
5586 case clang::X86::BI__builtin_ia32_psllv16hi:
5587 case clang::X86::BI__builtin_ia32_psllv16si:
5588 case clang::X86::BI__builtin_ia32_psllv32hi:
5589 case clang::X86::BI__builtin_ia32_psllwi128:
5590 case clang::X86::BI__builtin_ia32_psllwi256:
5591 case clang::X86::BI__builtin_ia32_psllwi512:
5592 case clang::X86::BI__builtin_ia32_pslldi128:
5593 case clang::X86::BI__builtin_ia32_pslldi256:
5594 case clang::X86::BI__builtin_ia32_pslldi512:
5595 case clang::X86::BI__builtin_ia32_psllqi128:
5596 case clang::X86::BI__builtin_ia32_psllqi256:
5597 case clang::X86::BI__builtin_ia32_psllqi512:
5598 return interp__builtin_elementwise_int_binop(
5599 S, OpPC, Call, Fn: [](const APSInt &LHS, const APSInt &RHS) {
5600 if (RHS.uge(RHS: LHS.getBitWidth())) {
5601 return APInt::getZero(numBits: LHS.getBitWidth());
5602 }
5603 return LHS.shl(shiftAmt: RHS.getZExtValue());
5604 });
5605
5606 case clang::X86::BI__builtin_ia32_psrav4si:
5607 case clang::X86::BI__builtin_ia32_psrav8di:
5608 case clang::X86::BI__builtin_ia32_psrav8hi:
5609 case clang::X86::BI__builtin_ia32_psrav8si:
5610 case clang::X86::BI__builtin_ia32_psrav16hi:
5611 case clang::X86::BI__builtin_ia32_psrav16si:
5612 case clang::X86::BI__builtin_ia32_psrav32hi:
5613 case clang::X86::BI__builtin_ia32_psravq128:
5614 case clang::X86::BI__builtin_ia32_psravq256:
5615 case clang::X86::BI__builtin_ia32_psrawi128:
5616 case clang::X86::BI__builtin_ia32_psrawi256:
5617 case clang::X86::BI__builtin_ia32_psrawi512:
5618 case clang::X86::BI__builtin_ia32_psradi128:
5619 case clang::X86::BI__builtin_ia32_psradi256:
5620 case clang::X86::BI__builtin_ia32_psradi512:
5621 case clang::X86::BI__builtin_ia32_psraqi128:
5622 case clang::X86::BI__builtin_ia32_psraqi256:
5623 case clang::X86::BI__builtin_ia32_psraqi512:
5624 return interp__builtin_elementwise_int_binop(
5625 S, OpPC, Call, Fn: [](const APSInt &LHS, const APSInt &RHS) {
5626 if (RHS.uge(RHS: LHS.getBitWidth())) {
5627 return LHS.ashr(ShiftAmt: LHS.getBitWidth() - 1);
5628 }
5629 return LHS.ashr(ShiftAmt: RHS.getZExtValue());
5630 });
5631
5632 case clang::X86::BI__builtin_ia32_psrlv2di:
5633 case clang::X86::BI__builtin_ia32_psrlv4di:
5634 case clang::X86::BI__builtin_ia32_psrlv4si:
5635 case clang::X86::BI__builtin_ia32_psrlv8di:
5636 case clang::X86::BI__builtin_ia32_psrlv8hi:
5637 case clang::X86::BI__builtin_ia32_psrlv8si:
5638 case clang::X86::BI__builtin_ia32_psrlv16hi:
5639 case clang::X86::BI__builtin_ia32_psrlv16si:
5640 case clang::X86::BI__builtin_ia32_psrlv32hi:
5641 case clang::X86::BI__builtin_ia32_psrlwi128:
5642 case clang::X86::BI__builtin_ia32_psrlwi256:
5643 case clang::X86::BI__builtin_ia32_psrlwi512:
5644 case clang::X86::BI__builtin_ia32_psrldi128:
5645 case clang::X86::BI__builtin_ia32_psrldi256:
5646 case clang::X86::BI__builtin_ia32_psrldi512:
5647 case clang::X86::BI__builtin_ia32_psrlqi128:
5648 case clang::X86::BI__builtin_ia32_psrlqi256:
5649 case clang::X86::BI__builtin_ia32_psrlqi512:
5650 return interp__builtin_elementwise_int_binop(
5651 S, OpPC, Call, Fn: [](const APSInt &LHS, const APSInt &RHS) {
5652 if (RHS.uge(RHS: LHS.getBitWidth())) {
5653 return APInt::getZero(numBits: LHS.getBitWidth());
5654 }
5655 return LHS.lshr(shiftAmt: RHS.getZExtValue());
5656 });
5657 case clang::X86::BI__builtin_ia32_packsswb128:
5658 case clang::X86::BI__builtin_ia32_packsswb256:
5659 case clang::X86::BI__builtin_ia32_packsswb512:
5660 case clang::X86::BI__builtin_ia32_packssdw128:
5661 case clang::X86::BI__builtin_ia32_packssdw256:
5662 case clang::X86::BI__builtin_ia32_packssdw512:
5663 return interp__builtin_ia32_pack(S, OpPC, E: Call, PackFn: [](const APSInt &Src) {
5664 return APInt(Src).truncSSat(width: Src.getBitWidth() / 2);
5665 });
5666 case clang::X86::BI__builtin_ia32_packusdw128:
5667 case clang::X86::BI__builtin_ia32_packusdw256:
5668 case clang::X86::BI__builtin_ia32_packusdw512:
5669 case clang::X86::BI__builtin_ia32_packuswb128:
5670 case clang::X86::BI__builtin_ia32_packuswb256:
5671 case clang::X86::BI__builtin_ia32_packuswb512:
5672 return interp__builtin_ia32_pack(S, OpPC, E: Call, PackFn: [](const APSInt &Src) {
5673 return APInt(Src).truncSSatU(width: Src.getBitWidth() / 2);
5674 });
5675
5676 case clang::X86::BI__builtin_ia32_selectss_128:
5677 case clang::X86::BI__builtin_ia32_selectsd_128:
5678 case clang::X86::BI__builtin_ia32_selectsh_128:
5679 case clang::X86::BI__builtin_ia32_selectsbf_128:
5680 return interp__builtin_ia32_select_scalar(S, Call);
5681 case clang::X86::BI__builtin_ia32_vprotbi:
5682 case clang::X86::BI__builtin_ia32_vprotdi:
5683 case clang::X86::BI__builtin_ia32_vprotqi:
5684 case clang::X86::BI__builtin_ia32_vprotwi:
5685 case clang::X86::BI__builtin_ia32_prold128:
5686 case clang::X86::BI__builtin_ia32_prold256:
5687 case clang::X86::BI__builtin_ia32_prold512:
5688 case clang::X86::BI__builtin_ia32_prolq128:
5689 case clang::X86::BI__builtin_ia32_prolq256:
5690 case clang::X86::BI__builtin_ia32_prolq512:
5691 return interp__builtin_elementwise_int_binop(
5692 S, OpPC, Call,
5693 Fn: [](const APSInt &LHS, const APSInt &RHS) { return LHS.rotl(rotateAmt: RHS); });
5694
5695 case clang::X86::BI__builtin_ia32_prord128:
5696 case clang::X86::BI__builtin_ia32_prord256:
5697 case clang::X86::BI__builtin_ia32_prord512:
5698 case clang::X86::BI__builtin_ia32_prorq128:
5699 case clang::X86::BI__builtin_ia32_prorq256:
5700 case clang::X86::BI__builtin_ia32_prorq512:
5701 return interp__builtin_elementwise_int_binop(
5702 S, OpPC, Call,
5703 Fn: [](const APSInt &LHS, const APSInt &RHS) { return LHS.rotr(rotateAmt: RHS); });
5704
5705 case Builtin::BI__builtin_elementwise_max:
5706 case Builtin::BI__builtin_elementwise_min:
5707 return interp__builtin_elementwise_maxmin(S, OpPC, Call, BuiltinID);
5708
5709 case clang::X86::BI__builtin_ia32_phaddw128:
5710 case clang::X86::BI__builtin_ia32_phaddw256:
5711 case clang::X86::BI__builtin_ia32_phaddd128:
5712 case clang::X86::BI__builtin_ia32_phaddd256:
5713 return interp_builtin_horizontal_int_binop(
5714 S, OpPC, Call,
5715 Fn: [](const APSInt &LHS, const APSInt &RHS) { return LHS + RHS; });
5716 case clang::X86::BI__builtin_ia32_phaddsw128:
5717 case clang::X86::BI__builtin_ia32_phaddsw256:
5718 return interp_builtin_horizontal_int_binop(
5719 S, OpPC, Call,
5720 Fn: [](const APSInt &LHS, const APSInt &RHS) { return LHS.sadd_sat(RHS); });
5721 case clang::X86::BI__builtin_ia32_phsubw128:
5722 case clang::X86::BI__builtin_ia32_phsubw256:
5723 case clang::X86::BI__builtin_ia32_phsubd128:
5724 case clang::X86::BI__builtin_ia32_phsubd256:
5725 return interp_builtin_horizontal_int_binop(
5726 S, OpPC, Call,
5727 Fn: [](const APSInt &LHS, const APSInt &RHS) { return LHS - RHS; });
5728 case clang::X86::BI__builtin_ia32_phsubsw128:
5729 case clang::X86::BI__builtin_ia32_phsubsw256:
5730 return interp_builtin_horizontal_int_binop(
5731 S, OpPC, Call,
5732 Fn: [](const APSInt &LHS, const APSInt &RHS) { return LHS.ssub_sat(RHS); });
5733 case clang::X86::BI__builtin_ia32_haddpd:
5734 case clang::X86::BI__builtin_ia32_haddps:
5735 case clang::X86::BI__builtin_ia32_haddpd256:
5736 case clang::X86::BI__builtin_ia32_haddps256:
5737 return interp_builtin_horizontal_fp_binop(
5738 S, OpPC, Call,
5739 Fn: [](const APFloat &LHS, const APFloat &RHS, llvm::RoundingMode RM) {
5740 APFloat F = LHS;
5741 F.add(RHS, RM);
5742 return F;
5743 });
5744 case clang::X86::BI__builtin_ia32_hsubpd:
5745 case clang::X86::BI__builtin_ia32_hsubps:
5746 case clang::X86::BI__builtin_ia32_hsubpd256:
5747 case clang::X86::BI__builtin_ia32_hsubps256:
5748 return interp_builtin_horizontal_fp_binop(
5749 S, OpPC, Call,
5750 Fn: [](const APFloat &LHS, const APFloat &RHS, llvm::RoundingMode RM) {
5751 APFloat F = LHS;
5752 F.subtract(RHS, RM);
5753 return F;
5754 });
5755 case clang::X86::BI__builtin_ia32_addsubpd:
5756 case clang::X86::BI__builtin_ia32_addsubps:
5757 case clang::X86::BI__builtin_ia32_addsubpd256:
5758 case clang::X86::BI__builtin_ia32_addsubps256:
5759 return interp__builtin_ia32_addsub(S, OpPC, Call);
5760
5761 case clang::X86::BI__builtin_ia32_pmuldq128:
5762 case clang::X86::BI__builtin_ia32_pmuldq256:
5763 case clang::X86::BI__builtin_ia32_pmuldq512:
5764 return interp__builtin_ia32_pmul(
5765 S, OpPC, Call,
5766 Fn: [](const APSInt &LoLHS, const APSInt &HiLHS, const APSInt &LoRHS,
5767 const APSInt &HiRHS) {
5768 return llvm::APIntOps::mulsExtended(C1: LoLHS, C2: LoRHS);
5769 });
5770
5771 case clang::X86::BI__builtin_ia32_pmuludq128:
5772 case clang::X86::BI__builtin_ia32_pmuludq256:
5773 case clang::X86::BI__builtin_ia32_pmuludq512:
5774 return interp__builtin_ia32_pmul(
5775 S, OpPC, Call,
5776 Fn: [](const APSInt &LoLHS, const APSInt &HiLHS, const APSInt &LoRHS,
5777 const APSInt &HiRHS) {
5778 return llvm::APIntOps::muluExtended(C1: LoLHS, C2: LoRHS);
5779 });
5780
5781 case clang::X86::BI__builtin_ia32_pclmulqdq128:
5782 case clang::X86::BI__builtin_ia32_pclmulqdq256:
5783 case clang::X86::BI__builtin_ia32_pclmulqdq512:
5784 return interp__builtin_ia32_pclmulqdq(S, OpPC, Call);
5785 case Builtin::BI__builtin_elementwise_clmul:
5786 return interp__builtin_elementwise_int_binop(S, OpPC, Call,
5787 Fn: llvm::APIntOps::clmul);
5788
5789 case Builtin::BI__builtin_elementwise_fma:
5790 return interp__builtin_elementwise_triop_fp(
5791 S, OpPC, Call,
5792 Fn: [](const APFloat &X, const APFloat &Y, const APFloat &Z,
5793 llvm::RoundingMode RM) {
5794 APFloat F = X;
5795 F.fusedMultiplyAdd(Multiplicand: Y, Addend: Z, RM);
5796 return F;
5797 });
5798
5799 case X86::BI__builtin_ia32_vpmadd52luq128:
5800 case X86::BI__builtin_ia32_vpmadd52luq256:
5801 case X86::BI__builtin_ia32_vpmadd52luq512:
5802 return interp__builtin_elementwise_triop(
5803 S, OpPC, Call, Fn: [](const APSInt &A, const APSInt &B, const APSInt &C) {
5804 return A + (B.trunc(width: 52) * C.trunc(width: 52)).zext(width: 64);
5805 });
5806 case X86::BI__builtin_ia32_vpmadd52huq128:
5807 case X86::BI__builtin_ia32_vpmadd52huq256:
5808 case X86::BI__builtin_ia32_vpmadd52huq512:
5809 return interp__builtin_elementwise_triop(
5810 S, OpPC, Call, Fn: [](const APSInt &A, const APSInt &B, const APSInt &C) {
5811 return A + llvm::APIntOps::mulhu(C1: B.trunc(width: 52), C2: C.trunc(width: 52)).zext(width: 64);
5812 });
5813
5814 case X86::BI__builtin_ia32_vpshldd128:
5815 case X86::BI__builtin_ia32_vpshldd256:
5816 case X86::BI__builtin_ia32_vpshldd512:
5817 case X86::BI__builtin_ia32_vpshldq128:
5818 case X86::BI__builtin_ia32_vpshldq256:
5819 case X86::BI__builtin_ia32_vpshldq512:
5820 case X86::BI__builtin_ia32_vpshldw128:
5821 case X86::BI__builtin_ia32_vpshldw256:
5822 case X86::BI__builtin_ia32_vpshldw512:
5823 return interp__builtin_elementwise_triop(
5824 S, OpPC, Call,
5825 Fn: [](const APSInt &Hi, const APSInt &Lo, const APSInt &Amt) {
5826 return llvm::APIntOps::fshl(Hi, Lo, Shift: Amt);
5827 });
5828
5829 case X86::BI__builtin_ia32_vpshrdd128:
5830 case X86::BI__builtin_ia32_vpshrdd256:
5831 case X86::BI__builtin_ia32_vpshrdd512:
5832 case X86::BI__builtin_ia32_vpshrdq128:
5833 case X86::BI__builtin_ia32_vpshrdq256:
5834 case X86::BI__builtin_ia32_vpshrdq512:
5835 case X86::BI__builtin_ia32_vpshrdw128:
5836 case X86::BI__builtin_ia32_vpshrdw256:
5837 case X86::BI__builtin_ia32_vpshrdw512:
5838 // NOTE: Reversed Hi/Lo operands.
5839 return interp__builtin_elementwise_triop(
5840 S, OpPC, Call,
5841 Fn: [](const APSInt &Lo, const APSInt &Hi, const APSInt &Amt) {
5842 return llvm::APIntOps::fshr(Hi, Lo, Shift: Amt);
5843 });
5844 case X86::BI__builtin_ia32_vpconflictsi_128:
5845 case X86::BI__builtin_ia32_vpconflictsi_256:
5846 case X86::BI__builtin_ia32_vpconflictsi_512:
5847 case X86::BI__builtin_ia32_vpconflictdi_128:
5848 case X86::BI__builtin_ia32_vpconflictdi_256:
5849 case X86::BI__builtin_ia32_vpconflictdi_512:
5850 return interp__builtin_ia32_vpconflict(S, OpPC, Call);
5851 case X86::BI__builtin_ia32_compressdf128_mask:
5852 case X86::BI__builtin_ia32_compressdf256_mask:
5853 case X86::BI__builtin_ia32_compressdf512_mask:
5854 case X86::BI__builtin_ia32_compressdi128_mask:
5855 case X86::BI__builtin_ia32_compressdi256_mask:
5856 case X86::BI__builtin_ia32_compressdi512_mask:
5857 case X86::BI__builtin_ia32_compresshi128_mask:
5858 case X86::BI__builtin_ia32_compresshi256_mask:
5859 case X86::BI__builtin_ia32_compresshi512_mask:
5860 case X86::BI__builtin_ia32_compressqi128_mask:
5861 case X86::BI__builtin_ia32_compressqi256_mask:
5862 case X86::BI__builtin_ia32_compressqi512_mask:
5863 case X86::BI__builtin_ia32_compresssf128_mask:
5864 case X86::BI__builtin_ia32_compresssf256_mask:
5865 case X86::BI__builtin_ia32_compresssf512_mask:
5866 case X86::BI__builtin_ia32_compresssi128_mask:
5867 case X86::BI__builtin_ia32_compresssi256_mask:
5868 case X86::BI__builtin_ia32_compresssi512_mask: {
5869 unsigned NumElems =
5870 Call->getArg(Arg: 0)->getType()->castAs<VectorType>()->getNumElements();
5871 return interp__builtin_ia32_shuffle_generic(
5872 S, OpPC, Call, GetSourceIndex: [NumElems](unsigned DstIdx, const APInt &ShuffleMask) {
5873 APInt CompressMask = ShuffleMask.trunc(width: NumElems);
5874 if (DstIdx < CompressMask.popcount()) {
5875 while (DstIdx != 0) {
5876 CompressMask = CompressMask & (CompressMask - 1);
5877 DstIdx--;
5878 }
5879 return std::pair<unsigned, int>{
5880 0, static_cast<int>(CompressMask.countr_zero())};
5881 }
5882 return std::pair<unsigned, int>{1, static_cast<int>(DstIdx)};
5883 });
5884 }
5885 case X86::BI__builtin_ia32_expanddf128_mask:
5886 case X86::BI__builtin_ia32_expanddf256_mask:
5887 case X86::BI__builtin_ia32_expanddf512_mask:
5888 case X86::BI__builtin_ia32_expanddi128_mask:
5889 case X86::BI__builtin_ia32_expanddi256_mask:
5890 case X86::BI__builtin_ia32_expanddi512_mask:
5891 case X86::BI__builtin_ia32_expandhi128_mask:
5892 case X86::BI__builtin_ia32_expandhi256_mask:
5893 case X86::BI__builtin_ia32_expandhi512_mask:
5894 case X86::BI__builtin_ia32_expandqi128_mask:
5895 case X86::BI__builtin_ia32_expandqi256_mask:
5896 case X86::BI__builtin_ia32_expandqi512_mask:
5897 case X86::BI__builtin_ia32_expandsf128_mask:
5898 case X86::BI__builtin_ia32_expandsf256_mask:
5899 case X86::BI__builtin_ia32_expandsf512_mask:
5900 case X86::BI__builtin_ia32_expandsi128_mask:
5901 case X86::BI__builtin_ia32_expandsi256_mask:
5902 case X86::BI__builtin_ia32_expandsi512_mask: {
5903 return interp__builtin_ia32_shuffle_generic(
5904 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, const APInt &ShuffleMask) {
5905 // Trunc to the sub-mask for the dst index and count the number of
5906 // src elements used prior to that.
5907 APInt ExpandMask = ShuffleMask.trunc(width: DstIdx + 1);
5908 if (ExpandMask[DstIdx]) {
5909 int SrcIdx = ExpandMask.popcount() - 1;
5910 return std::pair<unsigned, int>{0, SrcIdx};
5911 }
5912 return std::pair<unsigned, int>{1, static_cast<int>(DstIdx)};
5913 });
5914 }
5915 case clang::X86::BI__builtin_ia32_blendpd:
5916 case clang::X86::BI__builtin_ia32_blendpd256:
5917 case clang::X86::BI__builtin_ia32_blendps:
5918 case clang::X86::BI__builtin_ia32_blendps256:
5919 case clang::X86::BI__builtin_ia32_pblendw128:
5920 case clang::X86::BI__builtin_ia32_pblendw256:
5921 case clang::X86::BI__builtin_ia32_pblendd128:
5922 case clang::X86::BI__builtin_ia32_pblendd256:
5923 return interp__builtin_ia32_shuffle_generic(
5924 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
5925 // Bit index for mask.
5926 unsigned MaskBit = (ShuffleMask >> (DstIdx % 8)) & 0x1;
5927 unsigned SrcVecIdx = MaskBit ? 1 : 0; // 1 = TrueVec, 0 = FalseVec
5928 return std::pair<unsigned, int>{SrcVecIdx, static_cast<int>(DstIdx)};
5929 });
5930
5931
5932
5933 case clang::X86::BI__builtin_ia32_blendvpd:
5934 case clang::X86::BI__builtin_ia32_blendvpd256:
5935 case clang::X86::BI__builtin_ia32_blendvps:
5936 case clang::X86::BI__builtin_ia32_blendvps256:
5937 return interp__builtin_elementwise_triop_fp(
5938 S, OpPC, Call,
5939 Fn: [](const APFloat &F, const APFloat &T, const APFloat &C,
5940 llvm::RoundingMode) { return C.isNegative() ? T : F; });
5941
5942 case clang::X86::BI__builtin_ia32_pblendvb128:
5943 case clang::X86::BI__builtin_ia32_pblendvb256:
5944 return interp__builtin_elementwise_triop(
5945 S, OpPC, Call, Fn: [](const APSInt &F, const APSInt &T, const APSInt &C) {
5946 return ((APInt)C).isNegative() ? T : F;
5947 });
5948 case X86::BI__builtin_ia32_ptestz128:
5949 case X86::BI__builtin_ia32_ptestz256:
5950 case X86::BI__builtin_ia32_vtestzps:
5951 case X86::BI__builtin_ia32_vtestzps256:
5952 case X86::BI__builtin_ia32_vtestzpd:
5953 case X86::BI__builtin_ia32_vtestzpd256:
5954 return interp__builtin_ia32_test_op(
5955 S, OpPC, Call,
5956 Fn: [](const APInt &A, const APInt &B) { return (A & B) == 0; });
5957 case X86::BI__builtin_ia32_ptestc128:
5958 case X86::BI__builtin_ia32_ptestc256:
5959 case X86::BI__builtin_ia32_vtestcps:
5960 case X86::BI__builtin_ia32_vtestcps256:
5961 case X86::BI__builtin_ia32_vtestcpd:
5962 case X86::BI__builtin_ia32_vtestcpd256:
5963 return interp__builtin_ia32_test_op(
5964 S, OpPC, Call,
5965 Fn: [](const APInt &A, const APInt &B) { return (~A & B) == 0; });
5966 case X86::BI__builtin_ia32_ptestnzc128:
5967 case X86::BI__builtin_ia32_ptestnzc256:
5968 case X86::BI__builtin_ia32_vtestnzcps:
5969 case X86::BI__builtin_ia32_vtestnzcps256:
5970 case X86::BI__builtin_ia32_vtestnzcpd:
5971 case X86::BI__builtin_ia32_vtestnzcpd256:
5972 return interp__builtin_ia32_test_op(
5973 S, OpPC, Call, Fn: [](const APInt &A, const APInt &B) {
5974 return ((A & B) != 0) && ((~A & B) != 0);
5975 });
5976 case X86::BI__builtin_ia32_selectb_128:
5977 case X86::BI__builtin_ia32_selectb_256:
5978 case X86::BI__builtin_ia32_selectb_512:
5979 case X86::BI__builtin_ia32_selectw_128:
5980 case X86::BI__builtin_ia32_selectw_256:
5981 case X86::BI__builtin_ia32_selectw_512:
5982 case X86::BI__builtin_ia32_selectd_128:
5983 case X86::BI__builtin_ia32_selectd_256:
5984 case X86::BI__builtin_ia32_selectd_512:
5985 case X86::BI__builtin_ia32_selectq_128:
5986 case X86::BI__builtin_ia32_selectq_256:
5987 case X86::BI__builtin_ia32_selectq_512:
5988 case X86::BI__builtin_ia32_selectph_128:
5989 case X86::BI__builtin_ia32_selectph_256:
5990 case X86::BI__builtin_ia32_selectph_512:
5991 case X86::BI__builtin_ia32_selectpbf_128:
5992 case X86::BI__builtin_ia32_selectpbf_256:
5993 case X86::BI__builtin_ia32_selectpbf_512:
5994 case X86::BI__builtin_ia32_selectps_128:
5995 case X86::BI__builtin_ia32_selectps_256:
5996 case X86::BI__builtin_ia32_selectps_512:
5997 case X86::BI__builtin_ia32_selectpd_128:
5998 case X86::BI__builtin_ia32_selectpd_256:
5999 case X86::BI__builtin_ia32_selectpd_512:
6000 return interp__builtin_ia32_select(S, OpPC, Call);
6001
6002 case X86::BI__builtin_ia32_shufps:
6003 case X86::BI__builtin_ia32_shufps256:
6004 case X86::BI__builtin_ia32_shufps512:
6005 return interp__builtin_ia32_shuffle_generic(
6006 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6007 unsigned NumElemPerLane = 4;
6008 unsigned NumSelectableElems = NumElemPerLane / 2;
6009 unsigned BitsPerElem = 2;
6010 unsigned IndexMask = 0x3;
6011 unsigned MaskBits = 8;
6012 unsigned Lane = DstIdx / NumElemPerLane;
6013 unsigned ElemInLane = DstIdx % NumElemPerLane;
6014 unsigned LaneOffset = Lane * NumElemPerLane;
6015 unsigned SrcIdx = ElemInLane >= NumSelectableElems ? 1 : 0;
6016 unsigned BitIndex = (DstIdx * BitsPerElem) % MaskBits;
6017 unsigned Index = (ShuffleMask >> BitIndex) & IndexMask;
6018 return std::pair<unsigned, int>{SrcIdx,
6019 static_cast<int>(LaneOffset + Index)};
6020 });
6021 case X86::BI__builtin_ia32_shufpd:
6022 case X86::BI__builtin_ia32_shufpd256:
6023 case X86::BI__builtin_ia32_shufpd512:
6024 return interp__builtin_ia32_shuffle_generic(
6025 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6026 unsigned NumElemPerLane = 2;
6027 unsigned NumSelectableElems = NumElemPerLane / 2;
6028 unsigned BitsPerElem = 1;
6029 unsigned IndexMask = 0x1;
6030 unsigned MaskBits = 8;
6031 unsigned Lane = DstIdx / NumElemPerLane;
6032 unsigned ElemInLane = DstIdx % NumElemPerLane;
6033 unsigned LaneOffset = Lane * NumElemPerLane;
6034 unsigned SrcIdx = ElemInLane >= NumSelectableElems ? 1 : 0;
6035 unsigned BitIndex = (DstIdx * BitsPerElem) % MaskBits;
6036 unsigned Index = (ShuffleMask >> BitIndex) & IndexMask;
6037 return std::pair<unsigned, int>{SrcIdx,
6038 static_cast<int>(LaneOffset + Index)};
6039 });
6040
6041 case X86::BI__builtin_ia32_vgf2p8affineinvqb_v16qi:
6042 case X86::BI__builtin_ia32_vgf2p8affineinvqb_v32qi:
6043 case X86::BI__builtin_ia32_vgf2p8affineinvqb_v64qi:
6044 return interp__builtin_ia32_gfni_affine(S, OpPC, Call, Inverse: true);
6045 case X86::BI__builtin_ia32_vgf2p8affineqb_v16qi:
6046 case X86::BI__builtin_ia32_vgf2p8affineqb_v32qi:
6047 case X86::BI__builtin_ia32_vgf2p8affineqb_v64qi:
6048 return interp__builtin_ia32_gfni_affine(S, OpPC, Call, Inverse: false);
6049
6050 case X86::BI__builtin_ia32_vgf2p8mulb_v16qi:
6051 case X86::BI__builtin_ia32_vgf2p8mulb_v32qi:
6052 case X86::BI__builtin_ia32_vgf2p8mulb_v64qi:
6053 return interp__builtin_ia32_gfni_mul(S, OpPC, Call);
6054
6055 case X86::BI__builtin_ia32_bmacor16x16x16_v16hi:
6056 case X86::BI__builtin_ia32_bmacor16x16x16_v32hi:
6057 return interp__builtin_ia32_bmac(S, OpPC, Call, /*IsXor=*/false);
6058 case X86::BI__builtin_ia32_bmacxor16x16x16_v16hi:
6059 case X86::BI__builtin_ia32_bmacxor16x16x16_v32hi:
6060 return interp__builtin_ia32_bmac(S, OpPC, Call, /*IsXor=*/true);
6061
6062 case X86::BI__builtin_ia32_insertps128:
6063 return interp__builtin_ia32_shuffle_generic(
6064 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned Mask) {
6065 // Bits [3:0]: zero mask - if bit is set, zero this element
6066 if ((Mask & (1 << DstIdx)) != 0) {
6067 return std::pair<unsigned, int>{0, -1};
6068 }
6069 // Bits [7:6]: select element from source vector Y (0-3)
6070 // Bits [5:4]: select destination position (0-3)
6071 unsigned SrcElem = (Mask >> 6) & 0x3;
6072 unsigned DstElem = (Mask >> 4) & 0x3;
6073 if (DstIdx == DstElem) {
6074 // Insert element from source vector (B) at this position
6075 return std::pair<unsigned, int>{1, static_cast<int>(SrcElem)};
6076 } else {
6077 // Copy from destination vector (A)
6078 return std::pair<unsigned, int>{0, static_cast<int>(DstIdx)};
6079 }
6080 });
6081 case X86::BI__builtin_ia32_permvarsi256:
6082 case X86::BI__builtin_ia32_permvarsf256:
6083 case X86::BI__builtin_ia32_permvardf512:
6084 case X86::BI__builtin_ia32_permvardi512:
6085 case X86::BI__builtin_ia32_permvarhi128:
6086 return interp__builtin_ia32_shuffle_generic(
6087 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6088 int Offset = ShuffleMask & 0x7;
6089 return std::pair<unsigned, int>{0, Offset};
6090 });
6091 case X86::BI__builtin_ia32_permvarqi128:
6092 case X86::BI__builtin_ia32_permvarhi256:
6093 case X86::BI__builtin_ia32_permvarsi512:
6094 case X86::BI__builtin_ia32_permvarsf512:
6095 return interp__builtin_ia32_shuffle_generic(
6096 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6097 int Offset = ShuffleMask & 0xF;
6098 return std::pair<unsigned, int>{0, Offset};
6099 });
6100 case X86::BI__builtin_ia32_permvardi256:
6101 case X86::BI__builtin_ia32_permvardf256:
6102 return interp__builtin_ia32_shuffle_generic(
6103 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6104 int Offset = ShuffleMask & 0x3;
6105 return std::pair<unsigned, int>{0, Offset};
6106 });
6107 case X86::BI__builtin_ia32_permvarqi256:
6108 case X86::BI__builtin_ia32_permvarhi512:
6109 return interp__builtin_ia32_shuffle_generic(
6110 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6111 int Offset = ShuffleMask & 0x1F;
6112 return std::pair<unsigned, int>{0, Offset};
6113 });
6114 case X86::BI__builtin_ia32_permvarqi512:
6115 return interp__builtin_ia32_shuffle_generic(
6116 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6117 int Offset = ShuffleMask & 0x3F;
6118 return std::pair<unsigned, int>{0, Offset};
6119 });
6120 case X86::BI__builtin_ia32_vpermi2varq128:
6121 case X86::BI__builtin_ia32_vpermi2varpd128:
6122 return interp__builtin_ia32_shuffle_generic(
6123 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6124 int Offset = ShuffleMask & 0x1;
6125 unsigned SrcIdx = (ShuffleMask >> 1) & 0x1;
6126 return std::pair<unsigned, int>{SrcIdx, Offset};
6127 });
6128 case X86::BI__builtin_ia32_vpermi2vard128:
6129 case X86::BI__builtin_ia32_vpermi2varps128:
6130 case X86::BI__builtin_ia32_vpermi2varq256:
6131 case X86::BI__builtin_ia32_vpermi2varpd256:
6132 return interp__builtin_ia32_shuffle_generic(
6133 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6134 int Offset = ShuffleMask & 0x3;
6135 unsigned SrcIdx = (ShuffleMask >> 2) & 0x1;
6136 return std::pair<unsigned, int>{SrcIdx, Offset};
6137 });
6138 case X86::BI__builtin_ia32_vpermi2varhi128:
6139 case X86::BI__builtin_ia32_vpermi2vard256:
6140 case X86::BI__builtin_ia32_vpermi2varps256:
6141 case X86::BI__builtin_ia32_vpermi2varq512:
6142 case X86::BI__builtin_ia32_vpermi2varpd512:
6143 return interp__builtin_ia32_shuffle_generic(
6144 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6145 int Offset = ShuffleMask & 0x7;
6146 unsigned SrcIdx = (ShuffleMask >> 3) & 0x1;
6147 return std::pair<unsigned, int>{SrcIdx, Offset};
6148 });
6149 case X86::BI__builtin_ia32_vpermi2varqi128:
6150 case X86::BI__builtin_ia32_vpermi2varhi256:
6151 case X86::BI__builtin_ia32_vpermi2vard512:
6152 case X86::BI__builtin_ia32_vpermi2varps512:
6153 return interp__builtin_ia32_shuffle_generic(
6154 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6155 int Offset = ShuffleMask & 0xF;
6156 unsigned SrcIdx = (ShuffleMask >> 4) & 0x1;
6157 return std::pair<unsigned, int>{SrcIdx, Offset};
6158 });
6159 case X86::BI__builtin_ia32_vpermi2varqi256:
6160 case X86::BI__builtin_ia32_vpermi2varhi512:
6161 return interp__builtin_ia32_shuffle_generic(
6162 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6163 int Offset = ShuffleMask & 0x1F;
6164 unsigned SrcIdx = (ShuffleMask >> 5) & 0x1;
6165 return std::pair<unsigned, int>{SrcIdx, Offset};
6166 });
6167 case X86::BI__builtin_ia32_vpermi2varqi512:
6168 return interp__builtin_ia32_shuffle_generic(
6169 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6170 int Offset = ShuffleMask & 0x3F;
6171 unsigned SrcIdx = (ShuffleMask >> 6) & 0x1;
6172 return std::pair<unsigned, int>{SrcIdx, Offset};
6173 });
6174 case X86::BI__builtin_ia32_vperm2f128_pd256:
6175 case X86::BI__builtin_ia32_vperm2f128_ps256:
6176 case X86::BI__builtin_ia32_vperm2f128_si256:
6177 case X86::BI__builtin_ia32_permti256: {
6178 unsigned NumElements =
6179 Call->getArg(Arg: 0)->getType()->castAs<VectorType>()->getNumElements();
6180 unsigned PreservedBitsCnt = NumElements >> 2;
6181 return interp__builtin_ia32_shuffle_generic(
6182 S, OpPC, Call,
6183 GetSourceIndex: [PreservedBitsCnt](unsigned DstIdx, unsigned ShuffleMask) {
6184 unsigned ControlBitsCnt = DstIdx >> PreservedBitsCnt << 2;
6185 unsigned ControlBits = ShuffleMask >> ControlBitsCnt;
6186
6187 if (ControlBits & 0b1000)
6188 return std::make_pair(x: 0u, y: -1);
6189
6190 unsigned SrcVecIdx = (ControlBits & 0b10) >> 1;
6191 unsigned PreservedBitsMask = (1 << PreservedBitsCnt) - 1;
6192 int SrcIdx = ((ControlBits & 0b1) << PreservedBitsCnt) |
6193 (DstIdx & PreservedBitsMask);
6194 return std::make_pair(x&: SrcVecIdx, y&: SrcIdx);
6195 });
6196 }
6197 case X86::BI__builtin_ia32_pshufb128:
6198 case X86::BI__builtin_ia32_pshufb256:
6199 case X86::BI__builtin_ia32_pshufb512:
6200 return interp__builtin_ia32_shuffle_generic(
6201 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6202 uint8_t Ctlb = static_cast<uint8_t>(ShuffleMask);
6203 if (Ctlb & 0x80)
6204 return std::make_pair(x: 0, y: -1);
6205
6206 unsigned LaneBase = (DstIdx / 16) * 16;
6207 unsigned SrcOffset = Ctlb & 0x0F;
6208 unsigned SrcIdx = LaneBase + SrcOffset;
6209 return std::make_pair(x: 0, y: static_cast<int>(SrcIdx));
6210 });
6211
6212 case X86::BI__builtin_ia32_pshuflw:
6213 case X86::BI__builtin_ia32_pshuflw256:
6214 case X86::BI__builtin_ia32_pshuflw512:
6215 return interp__builtin_ia32_shuffle_generic(
6216 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6217 unsigned LaneBase = (DstIdx / 8) * 8;
6218 unsigned LaneIdx = DstIdx % 8;
6219 if (LaneIdx < 4) {
6220 unsigned Sel = (ShuffleMask >> (2 * LaneIdx)) & 0x3;
6221 return std::make_pair(x: 0, y: static_cast<int>(LaneBase + Sel));
6222 }
6223
6224 return std::make_pair(x: 0, y: static_cast<int>(DstIdx));
6225 });
6226
6227 case X86::BI__builtin_ia32_pshufhw:
6228 case X86::BI__builtin_ia32_pshufhw256:
6229 case X86::BI__builtin_ia32_pshufhw512:
6230 return interp__builtin_ia32_shuffle_generic(
6231 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6232 unsigned LaneBase = (DstIdx / 8) * 8;
6233 unsigned LaneIdx = DstIdx % 8;
6234 if (LaneIdx >= 4) {
6235 unsigned Sel = (ShuffleMask >> (2 * (LaneIdx - 4))) & 0x3;
6236 return std::make_pair(x: 0, y: static_cast<int>(LaneBase + 4 + Sel));
6237 }
6238
6239 return std::make_pair(x: 0, y: static_cast<int>(DstIdx));
6240 });
6241
6242 case X86::BI__builtin_ia32_pshufd:
6243 case X86::BI__builtin_ia32_pshufd256:
6244 case X86::BI__builtin_ia32_pshufd512:
6245 case X86::BI__builtin_ia32_vpermilps:
6246 case X86::BI__builtin_ia32_vpermilps256:
6247 case X86::BI__builtin_ia32_vpermilps512:
6248 return interp__builtin_ia32_shuffle_generic(
6249 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6250 unsigned LaneBase = (DstIdx / 4) * 4;
6251 unsigned LaneIdx = DstIdx % 4;
6252 unsigned Sel = (ShuffleMask >> (2 * LaneIdx)) & 0x3;
6253 return std::make_pair(x: 0, y: static_cast<int>(LaneBase + Sel));
6254 });
6255
6256 case X86::BI__builtin_ia32_vpermilvarpd:
6257 case X86::BI__builtin_ia32_vpermilvarpd256:
6258 case X86::BI__builtin_ia32_vpermilvarpd512:
6259 return interp__builtin_ia32_shuffle_generic(
6260 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6261 unsigned NumElemPerLane = 2;
6262 unsigned Lane = DstIdx / NumElemPerLane;
6263 unsigned Offset = ShuffleMask & 0b10 ? 1 : 0;
6264 return std::make_pair(
6265 x: 0, y: static_cast<int>(Lane * NumElemPerLane + Offset));
6266 });
6267
6268 case X86::BI__builtin_ia32_vpermilvarps:
6269 case X86::BI__builtin_ia32_vpermilvarps256:
6270 case X86::BI__builtin_ia32_vpermilvarps512:
6271 return interp__builtin_ia32_shuffle_generic(
6272 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
6273 unsigned NumElemPerLane = 4;
6274 unsigned Lane = DstIdx / NumElemPerLane;
6275 unsigned Offset = ShuffleMask & 0b11;
6276 return std::make_pair(
6277 x: 0, y: static_cast<int>(Lane * NumElemPerLane + Offset));
6278 });
6279
6280 case X86::BI__builtin_ia32_vpermilpd:
6281 case X86::BI__builtin_ia32_vpermilpd256:
6282 case X86::BI__builtin_ia32_vpermilpd512:
6283 return interp__builtin_ia32_shuffle_generic(
6284 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned Control) {
6285 unsigned NumElemPerLane = 2;
6286 unsigned BitsPerElem = 1;
6287 unsigned MaskBits = 8;
6288 unsigned IndexMask = 0x1;
6289 unsigned Lane = DstIdx / NumElemPerLane;
6290 unsigned LaneOffset = Lane * NumElemPerLane;
6291 unsigned BitIndex = (DstIdx * BitsPerElem) % MaskBits;
6292 unsigned Index = (Control >> BitIndex) & IndexMask;
6293 return std::make_pair(x: 0, y: static_cast<int>(LaneOffset + Index));
6294 });
6295
6296 case X86::BI__builtin_ia32_permdf256:
6297 case X86::BI__builtin_ia32_permdi256:
6298 return interp__builtin_ia32_shuffle_generic(
6299 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned Control) {
6300 // permute4x64 operates on 4 64-bit elements
6301 // For element i (0-3), extract bits [2*i+1:2*i] from Control
6302 unsigned Index = (Control >> (2 * DstIdx)) & 0x3;
6303 return std::make_pair(x: 0, y: static_cast<int>(Index));
6304 });
6305
6306 case X86::BI__builtin_ia32_vpmultishiftqb128:
6307 case X86::BI__builtin_ia32_vpmultishiftqb256:
6308 case X86::BI__builtin_ia32_vpmultishiftqb512:
6309 return interp__builtin_ia32_multishiftqb(S, OpPC, Call);
6310 case X86::BI__builtin_ia32_kandqi:
6311 case X86::BI__builtin_ia32_kandhi:
6312 case X86::BI__builtin_ia32_kandsi:
6313 case X86::BI__builtin_ia32_kanddi:
6314 return interp__builtin_elementwise_int_binop(
6315 S, OpPC, Call,
6316 Fn: [](const APSInt &LHS, const APSInt &RHS) { return LHS & RHS; });
6317
6318 case X86::BI__builtin_ia32_kandnqi:
6319 case X86::BI__builtin_ia32_kandnhi:
6320 case X86::BI__builtin_ia32_kandnsi:
6321 case X86::BI__builtin_ia32_kandndi:
6322 return interp__builtin_elementwise_int_binop(
6323 S, OpPC, Call,
6324 Fn: [](const APSInt &LHS, const APSInt &RHS) { return ~LHS & RHS; });
6325
6326 case X86::BI__builtin_ia32_korqi:
6327 case X86::BI__builtin_ia32_korhi:
6328 case X86::BI__builtin_ia32_korsi:
6329 case X86::BI__builtin_ia32_kordi:
6330 return interp__builtin_elementwise_int_binop(
6331 S, OpPC, Call,
6332 Fn: [](const APSInt &LHS, const APSInt &RHS) { return LHS | RHS; });
6333
6334 case X86::BI__builtin_ia32_kxnorqi:
6335 case X86::BI__builtin_ia32_kxnorhi:
6336 case X86::BI__builtin_ia32_kxnorsi:
6337 case X86::BI__builtin_ia32_kxnordi:
6338 return interp__builtin_elementwise_int_binop(
6339 S, OpPC, Call,
6340 Fn: [](const APSInt &LHS, const APSInt &RHS) { return ~(LHS ^ RHS); });
6341
6342 case X86::BI__builtin_ia32_kxorqi:
6343 case X86::BI__builtin_ia32_kxorhi:
6344 case X86::BI__builtin_ia32_kxorsi:
6345 case X86::BI__builtin_ia32_kxordi:
6346 return interp__builtin_elementwise_int_binop(
6347 S, OpPC, Call,
6348 Fn: [](const APSInt &LHS, const APSInt &RHS) { return LHS ^ RHS; });
6349
6350 case X86::BI__builtin_ia32_knotqi:
6351 case X86::BI__builtin_ia32_knothi:
6352 case X86::BI__builtin_ia32_knotsi:
6353 case X86::BI__builtin_ia32_knotdi:
6354 return interp__builtin_elementwise_int_unaryop(
6355 S, OpPC, Call, Fn: [](const APSInt &Src) { return ~Src; });
6356
6357 case X86::BI__builtin_ia32_kaddqi:
6358 case X86::BI__builtin_ia32_kaddhi:
6359 case X86::BI__builtin_ia32_kaddsi:
6360 case X86::BI__builtin_ia32_kadddi:
6361 return interp__builtin_elementwise_int_binop(
6362 S, OpPC, Call,
6363 Fn: [](const APSInt &LHS, const APSInt &RHS) { return LHS + RHS; });
6364
6365 case X86::BI__builtin_ia32_kmovb:
6366 case X86::BI__builtin_ia32_kmovw:
6367 case X86::BI__builtin_ia32_kmovd:
6368 case X86::BI__builtin_ia32_kmovq:
6369 return interp__builtin_elementwise_int_unaryop(
6370 S, OpPC, Call, Fn: [](const APSInt &Src) { return Src; });
6371
6372 case X86::BI__builtin_ia32_kunpckhi:
6373 case X86::BI__builtin_ia32_kunpckdi:
6374 case X86::BI__builtin_ia32_kunpcksi:
6375 return interp__builtin_elementwise_int_binop(
6376 S, OpPC, Call, Fn: [](const APSInt &A, const APSInt &B) {
6377 // Generic kunpack: extract lower half of each operand and concatenate
6378 // Result = A[HalfWidth-1:0] concat B[HalfWidth-1:0]
6379 unsigned BW = A.getBitWidth();
6380 return APSInt(A.trunc(width: BW / 2).concat(NewLSB: B.trunc(width: BW / 2)),
6381 A.isUnsigned());
6382 });
6383
6384 case X86::BI__builtin_ia32_phminposuw128:
6385 return interp__builtin_ia32_phminposuw(S, OpPC, Call);
6386
6387 case X86::BI__builtin_ia32_psraq128:
6388 case X86::BI__builtin_ia32_psraq256:
6389 case X86::BI__builtin_ia32_psraq512:
6390 case X86::BI__builtin_ia32_psrad128:
6391 case X86::BI__builtin_ia32_psrad256:
6392 case X86::BI__builtin_ia32_psrad512:
6393 case X86::BI__builtin_ia32_psraw128:
6394 case X86::BI__builtin_ia32_psraw256:
6395 case X86::BI__builtin_ia32_psraw512:
6396 return interp__builtin_ia32_shift_with_count(
6397 S, OpPC, Call,
6398 ShiftOp: [](const APInt &Elt, uint64_t Count) { return Elt.ashr(ShiftAmt: Count); },
6399 OverflowOp: [](const APInt &Elt, unsigned Width) { return Elt.ashr(ShiftAmt: Width - 1); });
6400
6401 case X86::BI__builtin_ia32_psllq128:
6402 case X86::BI__builtin_ia32_psllq256:
6403 case X86::BI__builtin_ia32_psllq512:
6404 case X86::BI__builtin_ia32_pslld128:
6405 case X86::BI__builtin_ia32_pslld256:
6406 case X86::BI__builtin_ia32_pslld512:
6407 case X86::BI__builtin_ia32_psllw128:
6408 case X86::BI__builtin_ia32_psllw256:
6409 case X86::BI__builtin_ia32_psllw512:
6410 return interp__builtin_ia32_shift_with_count(
6411 S, OpPC, Call,
6412 ShiftOp: [](const APInt &Elt, uint64_t Count) { return Elt.shl(shiftAmt: Count); },
6413 OverflowOp: [](const APInt &Elt, unsigned Width) { return APInt::getZero(numBits: Width); });
6414
6415 case X86::BI__builtin_ia32_psrlq128:
6416 case X86::BI__builtin_ia32_psrlq256:
6417 case X86::BI__builtin_ia32_psrlq512:
6418 case X86::BI__builtin_ia32_psrld128:
6419 case X86::BI__builtin_ia32_psrld256:
6420 case X86::BI__builtin_ia32_psrld512:
6421 case X86::BI__builtin_ia32_psrlw128:
6422 case X86::BI__builtin_ia32_psrlw256:
6423 case X86::BI__builtin_ia32_psrlw512:
6424 return interp__builtin_ia32_shift_with_count(
6425 S, OpPC, Call,
6426 ShiftOp: [](const APInt &Elt, uint64_t Count) { return Elt.lshr(shiftAmt: Count); },
6427 OverflowOp: [](const APInt &Elt, unsigned Width) { return APInt::getZero(numBits: Width); });
6428
6429 case X86::BI__builtin_ia32_pternlogd128_mask:
6430 case X86::BI__builtin_ia32_pternlogd256_mask:
6431 case X86::BI__builtin_ia32_pternlogd512_mask:
6432 case X86::BI__builtin_ia32_pternlogq128_mask:
6433 case X86::BI__builtin_ia32_pternlogq256_mask:
6434 case X86::BI__builtin_ia32_pternlogq512_mask:
6435 return interp__builtin_ia32_pternlog(S, OpPC, Call, /*MaskZ=*/false);
6436 case X86::BI__builtin_ia32_pternlogd128_maskz:
6437 case X86::BI__builtin_ia32_pternlogd256_maskz:
6438 case X86::BI__builtin_ia32_pternlogd512_maskz:
6439 case X86::BI__builtin_ia32_pternlogq128_maskz:
6440 case X86::BI__builtin_ia32_pternlogq256_maskz:
6441 case X86::BI__builtin_ia32_pternlogq512_maskz:
6442 return interp__builtin_ia32_pternlog(S, OpPC, Call, /*MaskZ=*/true);
6443 case Builtin::BI__builtin_elementwise_fshl:
6444 return interp__builtin_elementwise_triop(S, OpPC, Call,
6445 Fn: llvm::APIntOps::fshl);
6446 case Builtin::BI__builtin_elementwise_fshr:
6447 return interp__builtin_elementwise_triop(S, OpPC, Call,
6448 Fn: llvm::APIntOps::fshr);
6449
6450 case X86::BI__builtin_ia32_shuf_f32x4_256:
6451 case X86::BI__builtin_ia32_shuf_i32x4_256:
6452 case X86::BI__builtin_ia32_shuf_f64x2_256:
6453 case X86::BI__builtin_ia32_shuf_i64x2_256:
6454 case X86::BI__builtin_ia32_shuf_f32x4:
6455 case X86::BI__builtin_ia32_shuf_i32x4:
6456 case X86::BI__builtin_ia32_shuf_f64x2:
6457 case X86::BI__builtin_ia32_shuf_i64x2: {
6458 // Destination and sources A, B all have the same type.
6459 QualType VecQT = Call->getArg(Arg: 0)->getType();
6460 const auto *VecT = VecQT->castAs<VectorType>();
6461 unsigned NumElems = VecT->getNumElements();
6462 unsigned ElemBits = S.getASTContext().getTypeSize(T: VecT->getElementType());
6463 unsigned LaneBits = 128u;
6464 unsigned NumLanes = (NumElems * ElemBits) / LaneBits;
6465 unsigned NumElemsPerLane = LaneBits / ElemBits;
6466
6467 return interp__builtin_ia32_shuffle_generic(
6468 S, OpPC, Call,
6469 GetSourceIndex: [NumLanes, NumElemsPerLane](unsigned DstIdx, unsigned ShuffleMask) {
6470 // DstIdx determines source. ShuffleMask selects lane in source.
6471 unsigned BitsPerElem = NumLanes / 2;
6472 unsigned IndexMask = (1u << BitsPerElem) - 1;
6473 unsigned Lane = DstIdx / NumElemsPerLane;
6474 unsigned SrcIdx = (Lane < NumLanes / 2) ? 0 : 1;
6475 unsigned BitIdx = BitsPerElem * Lane;
6476 unsigned SrcLaneIdx = (ShuffleMask >> BitIdx) & IndexMask;
6477 unsigned ElemInLane = DstIdx % NumElemsPerLane;
6478 unsigned IdxToPick = SrcLaneIdx * NumElemsPerLane + ElemInLane;
6479 return std::pair<unsigned, int>{SrcIdx, IdxToPick};
6480 });
6481 }
6482
6483 case X86::BI__builtin_ia32_insertf32x4_256:
6484 case X86::BI__builtin_ia32_inserti32x4_256:
6485 case X86::BI__builtin_ia32_insertf64x2_256:
6486 case X86::BI__builtin_ia32_inserti64x2_256:
6487 case X86::BI__builtin_ia32_insertf32x4:
6488 case X86::BI__builtin_ia32_inserti32x4:
6489 case X86::BI__builtin_ia32_insertf64x2_512:
6490 case X86::BI__builtin_ia32_inserti64x2_512:
6491 case X86::BI__builtin_ia32_insertf32x8:
6492 case X86::BI__builtin_ia32_inserti32x8:
6493 case X86::BI__builtin_ia32_insertf64x4:
6494 case X86::BI__builtin_ia32_inserti64x4:
6495 case X86::BI__builtin_ia32_vinsertf128_ps256:
6496 case X86::BI__builtin_ia32_vinsertf128_pd256:
6497 case X86::BI__builtin_ia32_vinsertf128_si256:
6498 case X86::BI__builtin_ia32_insert128i256:
6499 return interp__builtin_ia32_insert_subvector(S, OpPC, Call, ID: BuiltinID);
6500
6501 case clang::X86::BI__builtin_ia32_vcvtps2ph:
6502 case clang::X86::BI__builtin_ia32_vcvtps2ph256:
6503 return interp__builtin_ia32_vcvtps2ph(S, OpPC, Call);
6504
6505 case X86::BI__builtin_ia32_vec_ext_v4hi:
6506 case X86::BI__builtin_ia32_vec_ext_v16qi:
6507 case X86::BI__builtin_ia32_vec_ext_v8hi:
6508 case X86::BI__builtin_ia32_vec_ext_v4si:
6509 case X86::BI__builtin_ia32_vec_ext_v2di:
6510 case X86::BI__builtin_ia32_vec_ext_v32qi:
6511 case X86::BI__builtin_ia32_vec_ext_v16hi:
6512 case X86::BI__builtin_ia32_vec_ext_v8si:
6513 case X86::BI__builtin_ia32_vec_ext_v4di:
6514 case X86::BI__builtin_ia32_vec_ext_v4sf:
6515 return interp__builtin_ia32_vec_ext(S, OpPC, Call, ID: BuiltinID);
6516
6517 case X86::BI__builtin_ia32_vec_set_v4hi:
6518 case X86::BI__builtin_ia32_vec_set_v16qi:
6519 case X86::BI__builtin_ia32_vec_set_v8hi:
6520 case X86::BI__builtin_ia32_vec_set_v4si:
6521 case X86::BI__builtin_ia32_vec_set_v2di:
6522 case X86::BI__builtin_ia32_vec_set_v32qi:
6523 case X86::BI__builtin_ia32_vec_set_v16hi:
6524 case X86::BI__builtin_ia32_vec_set_v8si:
6525 case X86::BI__builtin_ia32_vec_set_v4di:
6526 return interp__builtin_ia32_vec_set(S, OpPC, Call, ID: BuiltinID);
6527
6528 case X86::BI__builtin_ia32_cvtb2mask128:
6529 case X86::BI__builtin_ia32_cvtb2mask256:
6530 case X86::BI__builtin_ia32_cvtb2mask512:
6531 case X86::BI__builtin_ia32_cvtw2mask128:
6532 case X86::BI__builtin_ia32_cvtw2mask256:
6533 case X86::BI__builtin_ia32_cvtw2mask512:
6534 case X86::BI__builtin_ia32_cvtd2mask128:
6535 case X86::BI__builtin_ia32_cvtd2mask256:
6536 case X86::BI__builtin_ia32_cvtd2mask512:
6537 case X86::BI__builtin_ia32_cvtq2mask128:
6538 case X86::BI__builtin_ia32_cvtq2mask256:
6539 case X86::BI__builtin_ia32_cvtq2mask512:
6540 return interp__builtin_ia32_cvt_vec2mask(S, OpPC, Call, ID: BuiltinID);
6541
6542 case X86::BI__builtin_ia32_cvtmask2b128:
6543 case X86::BI__builtin_ia32_cvtmask2b256:
6544 case X86::BI__builtin_ia32_cvtmask2b512:
6545 case X86::BI__builtin_ia32_cvtmask2w128:
6546 case X86::BI__builtin_ia32_cvtmask2w256:
6547 case X86::BI__builtin_ia32_cvtmask2w512:
6548 case X86::BI__builtin_ia32_cvtmask2d128:
6549 case X86::BI__builtin_ia32_cvtmask2d256:
6550 case X86::BI__builtin_ia32_cvtmask2d512:
6551 case X86::BI__builtin_ia32_cvtmask2q128:
6552 case X86::BI__builtin_ia32_cvtmask2q256:
6553 case X86::BI__builtin_ia32_cvtmask2q512:
6554 return interp__builtin_ia32_cvt_mask2vec(S, OpPC, Call, ID: BuiltinID);
6555
6556 case X86::BI__builtin_ia32_cvtsd2ss:
6557 return interp__builtin_ia32_cvtsd2ss(S, OpPC, Call, HasRoundingMask: false);
6558
6559 case X86::BI__builtin_ia32_cvtsd2ss_round_mask:
6560 return interp__builtin_ia32_cvtsd2ss(S, OpPC, Call, HasRoundingMask: true);
6561
6562 case X86::BI__builtin_ia32_cvtpd2ps:
6563 case X86::BI__builtin_ia32_cvtpd2ps256:
6564 return interp__builtin_ia32_cvtpd2ps(S, OpPC, Call, IsMasked: false, HasRounding: false);
6565 case X86::BI__builtin_ia32_cvtpd2ps_mask:
6566 return interp__builtin_ia32_cvtpd2ps(S, OpPC, Call, IsMasked: true, HasRounding: false);
6567 case X86::BI__builtin_ia32_cvtpd2ps512_mask:
6568 return interp__builtin_ia32_cvtpd2ps(S, OpPC, Call, IsMasked: true, HasRounding: true);
6569
6570 case X86::BI__builtin_ia32_cmpb128_mask:
6571 case X86::BI__builtin_ia32_cmpw128_mask:
6572 case X86::BI__builtin_ia32_cmpd128_mask:
6573 case X86::BI__builtin_ia32_cmpq128_mask:
6574 case X86::BI__builtin_ia32_cmpb256_mask:
6575 case X86::BI__builtin_ia32_cmpw256_mask:
6576 case X86::BI__builtin_ia32_cmpd256_mask:
6577 case X86::BI__builtin_ia32_cmpq256_mask:
6578 case X86::BI__builtin_ia32_cmpb512_mask:
6579 case X86::BI__builtin_ia32_cmpw512_mask:
6580 case X86::BI__builtin_ia32_cmpd512_mask:
6581 case X86::BI__builtin_ia32_cmpq512_mask:
6582 return interp__builtin_ia32_cmp_mask(S, OpPC, Call, ID: BuiltinID,
6583 /*IsUnsigned=*/false);
6584
6585 case X86::BI__builtin_ia32_ucmpb128_mask:
6586 case X86::BI__builtin_ia32_ucmpw128_mask:
6587 case X86::BI__builtin_ia32_ucmpd128_mask:
6588 case X86::BI__builtin_ia32_ucmpq128_mask:
6589 case X86::BI__builtin_ia32_ucmpb256_mask:
6590 case X86::BI__builtin_ia32_ucmpw256_mask:
6591 case X86::BI__builtin_ia32_ucmpd256_mask:
6592 case X86::BI__builtin_ia32_ucmpq256_mask:
6593 case X86::BI__builtin_ia32_ucmpb512_mask:
6594 case X86::BI__builtin_ia32_ucmpw512_mask:
6595 case X86::BI__builtin_ia32_ucmpd512_mask:
6596 case X86::BI__builtin_ia32_ucmpq512_mask:
6597 return interp__builtin_ia32_cmp_mask(S, OpPC, Call, ID: BuiltinID,
6598 /*IsUnsigned=*/true);
6599
6600 case X86::BI__builtin_ia32_vpshufbitqmb128_mask:
6601 case X86::BI__builtin_ia32_vpshufbitqmb256_mask:
6602 case X86::BI__builtin_ia32_vpshufbitqmb512_mask:
6603 return interp__builtin_ia32_shufbitqmb_mask(S, OpPC, Call);
6604
6605 case X86::BI__builtin_ia32_pslldqi128_byteshift:
6606 case X86::BI__builtin_ia32_pslldqi256_byteshift:
6607 case X86::BI__builtin_ia32_pslldqi512_byteshift:
6608 // These SLLDQ intrinsics always operate on byte elements (8 bits).
6609 // The lane width is hardcoded to 16 to match the SIMD register size,
6610 // but the algorithm processes one byte per iteration,
6611 // so APInt(8, ...) is correct and intentional.
6612 return interp__builtin_ia32_shuffle_generic(
6613 S, OpPC, Call,
6614 GetSourceIndex: [](unsigned DstIdx, unsigned Shift) -> std::pair<unsigned, int> {
6615 unsigned LaneBase = (DstIdx / 16) * 16;
6616 unsigned LaneIdx = DstIdx % 16;
6617 if (LaneIdx < Shift)
6618 return std::make_pair(x: 0, y: -1);
6619
6620 return std::make_pair(x: 0,
6621 y: static_cast<int>(LaneBase + LaneIdx - Shift));
6622 });
6623
6624 case X86::BI__builtin_ia32_psrldqi128_byteshift:
6625 case X86::BI__builtin_ia32_psrldqi256_byteshift:
6626 case X86::BI__builtin_ia32_psrldqi512_byteshift:
6627 // These SRLDQ intrinsics always operate on byte elements (8 bits).
6628 // The lane width is hardcoded to 16 to match the SIMD register size,
6629 // but the algorithm processes one byte per iteration,
6630 // so APInt(8, ...) is correct and intentional.
6631 return interp__builtin_ia32_shuffle_generic(
6632 S, OpPC, Call,
6633 GetSourceIndex: [](unsigned DstIdx, unsigned Shift) -> std::pair<unsigned, int> {
6634 unsigned LaneBase = (DstIdx / 16) * 16;
6635 unsigned LaneIdx = DstIdx % 16;
6636 if (LaneIdx + Shift < 16)
6637 return std::make_pair(x: 0,
6638 y: static_cast<int>(LaneBase + LaneIdx + Shift));
6639
6640 return std::make_pair(x: 0, y: -1);
6641 });
6642
6643 case X86::BI__builtin_ia32_palignr128:
6644 case X86::BI__builtin_ia32_palignr256:
6645 case X86::BI__builtin_ia32_palignr512:
6646 return interp__builtin_ia32_shuffle_generic(
6647 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned Shift) {
6648 // Default to -1 → zero-fill this destination element
6649 unsigned VecIdx = 1;
6650 int ElemIdx = -1;
6651
6652 int Lane = DstIdx / 16;
6653 int Offset = DstIdx % 16;
6654
6655 // Elements come from VecB first, then VecA after the shift boundary
6656 unsigned ShiftedIdx = Offset + (Shift & 0xFF);
6657 if (ShiftedIdx < 16) { // from VecB
6658 ElemIdx = ShiftedIdx + (Lane * 16);
6659 } else if (ShiftedIdx < 32) { // from VecA
6660 VecIdx = 0;
6661 ElemIdx = (ShiftedIdx - 16) + (Lane * 16);
6662 }
6663
6664 return std::pair<unsigned, int>{VecIdx, ElemIdx};
6665 });
6666
6667 case X86::BI__builtin_ia32_alignd128:
6668 case X86::BI__builtin_ia32_alignd256:
6669 case X86::BI__builtin_ia32_alignd512:
6670 case X86::BI__builtin_ia32_alignq128:
6671 case X86::BI__builtin_ia32_alignq256:
6672 case X86::BI__builtin_ia32_alignq512: {
6673 unsigned NumElems = Call->getType()->castAs<VectorType>()->getNumElements();
6674 return interp__builtin_ia32_shuffle_generic(
6675 S, OpPC, Call, GetSourceIndex: [NumElems](unsigned DstIdx, unsigned Shift) {
6676 unsigned Imm = Shift & 0xFF;
6677 unsigned EffectiveShift = Imm & (NumElems - 1);
6678 unsigned SourcePos = DstIdx + EffectiveShift;
6679 unsigned VecIdx = SourcePos < NumElems ? 1u : 0u;
6680 unsigned ElemIdx = SourcePos & (NumElems - 1);
6681 return std::pair<unsigned, int>{VecIdx, static_cast<int>(ElemIdx)};
6682 });
6683 }
6684
6685 case clang::X86::BI__builtin_ia32_minps:
6686 case clang::X86::BI__builtin_ia32_minpd:
6687 case clang::X86::BI__builtin_ia32_minph128:
6688 case clang::X86::BI__builtin_ia32_minph256:
6689 case clang::X86::BI__builtin_ia32_minps256:
6690 case clang::X86::BI__builtin_ia32_minpd256:
6691 case clang::X86::BI__builtin_ia32_minps512:
6692 case clang::X86::BI__builtin_ia32_minpd512:
6693 case clang::X86::BI__builtin_ia32_minph512:
6694 return interp__builtin_elementwise_fp_binop(
6695 S, OpPC, Call,
6696 Fn: [](const APFloat &A, const APFloat &B,
6697 std::optional<APSInt>) -> std::optional<APFloat> {
6698 if (A.isNaN() || A.isInfinity() || A.isDenormal() || B.isNaN() ||
6699 B.isInfinity() || B.isDenormal())
6700 return std::nullopt;
6701 if (A.isZero() && B.isZero())
6702 return B;
6703 return llvm::minimum(A, B);
6704 });
6705
6706 case clang::X86::BI__builtin_ia32_minss:
6707 case clang::X86::BI__builtin_ia32_minsd:
6708 return interp__builtin_elementwise_fp_binop(
6709 S, OpPC, Call,
6710 Fn: [](const APFloat &A, const APFloat &B,
6711 std::optional<APSInt> RoundingMode) -> std::optional<APFloat> {
6712 return EvalScalarMinMaxFp(A, B, RoundingMode, /*IsMin=*/true);
6713 },
6714 /*IsScalar=*/true);
6715
6716 case clang::X86::BI__builtin_ia32_minsd_round_mask:
6717 case clang::X86::BI__builtin_ia32_minss_round_mask:
6718 case clang::X86::BI__builtin_ia32_minsh_round_mask:
6719 case clang::X86::BI__builtin_ia32_maxsd_round_mask:
6720 case clang::X86::BI__builtin_ia32_maxss_round_mask:
6721 case clang::X86::BI__builtin_ia32_maxsh_round_mask: {
6722 bool IsMin = BuiltinID == clang::X86::BI__builtin_ia32_minsd_round_mask ||
6723 BuiltinID == clang::X86::BI__builtin_ia32_minss_round_mask ||
6724 BuiltinID == clang::X86::BI__builtin_ia32_minsh_round_mask;
6725 return interp__builtin_scalar_fp_round_mask_binop(
6726 S, OpPC, Call,
6727 Fn: [IsMin](const APFloat &A, const APFloat &B,
6728 std::optional<APSInt> RoundingMode) -> std::optional<APFloat> {
6729 return EvalScalarMinMaxFp(A, B, RoundingMode, IsMin);
6730 });
6731 }
6732
6733 case clang::X86::BI__builtin_ia32_maxps:
6734 case clang::X86::BI__builtin_ia32_maxpd:
6735 case clang::X86::BI__builtin_ia32_maxph128:
6736 case clang::X86::BI__builtin_ia32_maxph256:
6737 case clang::X86::BI__builtin_ia32_maxps256:
6738 case clang::X86::BI__builtin_ia32_maxpd256:
6739 case clang::X86::BI__builtin_ia32_maxps512:
6740 case clang::X86::BI__builtin_ia32_maxpd512:
6741 case clang::X86::BI__builtin_ia32_maxph512:
6742 return interp__builtin_elementwise_fp_binop(
6743 S, OpPC, Call,
6744 Fn: [](const APFloat &A, const APFloat &B,
6745 std::optional<APSInt>) -> std::optional<APFloat> {
6746 if (A.isNaN() || A.isInfinity() || A.isDenormal() || B.isNaN() ||
6747 B.isInfinity() || B.isDenormal())
6748 return std::nullopt;
6749 if (A.isZero() && B.isZero())
6750 return B;
6751 return llvm::maximum(A, B);
6752 });
6753
6754 case clang::X86::BI__builtin_ia32_maxss:
6755 case clang::X86::BI__builtin_ia32_maxsd:
6756 return interp__builtin_elementwise_fp_binop(
6757 S, OpPC, Call,
6758 Fn: [](const APFloat &A, const APFloat &B,
6759 std::optional<APSInt> RoundingMode) -> std::optional<APFloat> {
6760 return EvalScalarMinMaxFp(A, B, RoundingMode, /*IsMin=*/false);
6761 },
6762 /*IsScalar=*/true);
6763 case X86::BI__builtin_ia32_vpdpwssd128:
6764 case X86::BI__builtin_ia32_vpdpwssd256:
6765 case X86::BI__builtin_ia32_vpdpwssd512:
6766 case X86::BI__builtin_ia32_vpdpbusd128:
6767 case X86::BI__builtin_ia32_vpdpbusd256:
6768 case X86::BI__builtin_ia32_vpdpbusd512:
6769 return interp__builtin_ia32_vpdp(S, OpPC, Call, IsSaturating: false);
6770 case X86::BI__builtin_ia32_vpdpwssds128:
6771 case X86::BI__builtin_ia32_vpdpwssds256:
6772 case X86::BI__builtin_ia32_vpdpwssds512:
6773 case X86::BI__builtin_ia32_vpdpbusds128:
6774 case X86::BI__builtin_ia32_vpdpbusds256:
6775 case X86::BI__builtin_ia32_vpdpbusds512:
6776 return interp__builtin_ia32_vpdp(S, OpPC, Call, IsSaturating: true);
6777 default:
6778 S.FFDiag(Loc: S.Current->getLocation(PC: OpPC),
6779 DiagId: diag::note_invalid_subexpr_in_const_expr)
6780 << S.Current->getRange(PC: OpPC);
6781
6782 return false;
6783 }
6784
6785 llvm_unreachable("Unhandled builtin ID");
6786}
6787
6788bool InterpretOffsetOf(InterpState &S, CodePtr OpPC, const OffsetOfExpr *E,
6789 ArrayRef<int64_t> ArrayIndices, int64_t &IntResult) {
6790 S.getASTContext().recordOffsetOfEvaluation(E);
6791 CharUnits Result;
6792 unsigned N = E->getNumComponents();
6793 assert(N > 0);
6794
6795 unsigned ArrayIndex = 0;
6796 QualType CurrentType = E->getTypeSourceInfo()->getType();
6797 for (unsigned I = 0; I != N; ++I) {
6798 const OffsetOfNode &Node = E->getComponent(Idx: I);
6799 switch (Node.getKind()) {
6800 case OffsetOfNode::Field: {
6801 const FieldDecl *MemberDecl = Node.getField();
6802 const auto *RD = CurrentType->getAsRecordDecl();
6803 if (!RD || RD->isInvalidDecl())
6804 return false;
6805 const ASTRecordLayout &RL = S.getASTContext().getASTRecordLayout(D: RD);
6806 unsigned FieldIndex = MemberDecl->getFieldIndex();
6807 assert(FieldIndex < RL.getFieldCount() && "offsetof field in wrong type");
6808 Result +=
6809 S.getASTContext().toCharUnitsFromBits(BitSize: RL.getFieldOffset(FieldNo: FieldIndex));
6810 CurrentType = MemberDecl->getType().getNonReferenceType();
6811 break;
6812 }
6813 case OffsetOfNode::Array: {
6814 // When generating bytecode, we put all the index expressions as Sint64 on
6815 // the stack.
6816 int64_t Index = ArrayIndices[ArrayIndex];
6817 if (Index < 0)
6818 return Invalid(S, OpPC);
6819 const ArrayType *AT = S.getASTContext().getAsArrayType(T: CurrentType);
6820 if (!AT)
6821 return false;
6822 CurrentType = AT->getElementType();
6823 CharUnits ElementSize = S.getASTContext().getTypeSizeInChars(T: CurrentType);
6824 int64_t ElemSize = ElementSize.getQuantity();
6825 if (Index != 0 && ElemSize > llvm::maxIntN(N: 64) / Index) {
6826 S.FFDiag(Loc: S.Current->getLocation(PC: OpPC),
6827 DiagId: diag::note_constexpr_offsetof_overflow)
6828 << S.Current->getRange(PC: OpPC);
6829 return false;
6830 }
6831 int64_t Offset = Index * ElemSize;
6832 if (Result.getQuantity() > llvm::maxIntN(N: 64) - Offset) {
6833 S.FFDiag(Loc: S.Current->getLocation(PC: OpPC),
6834 DiagId: diag::note_constexpr_offsetof_overflow)
6835 << S.Current->getRange(PC: OpPC);
6836 return false;
6837 }
6838 Result += CharUnits::fromQuantity(Quantity: Offset);
6839 ++ArrayIndex;
6840 break;
6841 }
6842 case OffsetOfNode::Base: {
6843 const CXXBaseSpecifier *BaseSpec = Node.getBase();
6844 if (BaseSpec->isVirtual())
6845 return false;
6846
6847 // Find the layout of the class whose base we are looking into.
6848 const auto *RD = CurrentType->getAsCXXRecordDecl();
6849 if (!RD || RD->isInvalidDecl())
6850 return false;
6851 const ASTRecordLayout &RL = S.getASTContext().getASTRecordLayout(D: RD);
6852
6853 // Find the base class itself.
6854 CurrentType = BaseSpec->getType();
6855 const auto *BaseRD = CurrentType->getAsCXXRecordDecl();
6856 if (!BaseRD)
6857 return false;
6858
6859 // Add the offset to the base.
6860 Result += RL.getBaseClassOffset(Base: BaseRD);
6861 break;
6862 }
6863 case OffsetOfNode::Identifier:
6864 llvm_unreachable("Dependent OffsetOfExpr?");
6865 }
6866 }
6867
6868 IntResult = Result.getQuantity();
6869
6870 return true;
6871}
6872
6873bool SetThreeWayComparisonField(InterpState &S, CodePtr OpPC,
6874 const Pointer &Ptr, const APSInt &IntValue) {
6875
6876 const Record *R = Ptr.getRecord();
6877 assert(R);
6878 assert(R->getNumFields() == 1);
6879
6880 unsigned FieldOffset = R->getField(I: 0u)->Offset;
6881 PtrView FieldPtr = Ptr.view().atField(Offset: FieldOffset);
6882 PrimType FieldT = FieldPtr.getFieldDesc()->getPrimType();
6883
6884 INT_TYPE_SWITCH(FieldT,
6885 FieldPtr.deref<T>() = T::from(IntValue.getSExtValue()));
6886 FieldPtr.initialize();
6887 return true;
6888}
6889
6890static void zeroAll(PtrView Dest) {
6891 const Descriptor *Desc = Dest.getFieldDesc();
6892
6893 if (Desc->isPrimitive()) {
6894 TYPE_SWITCH(Desc->getPrimType(), {
6895 Dest.deref<T>().~T();
6896 new (&Dest.deref<T>()) T();
6897 });
6898 return;
6899 }
6900
6901 if (Desc->isRecord()) {
6902 const Record *R = Desc->ElemRecord;
6903 for (const Record::Field &F : R->fields()) {
6904 PtrView FieldPtr = Dest.atField(Offset: F.Offset);
6905 zeroAll(Dest: FieldPtr);
6906 }
6907 return;
6908 }
6909
6910 if (Desc->isPrimitiveArray()) {
6911 for (unsigned I = 0, N = Desc->getNumElems(); I != N; ++I) {
6912 TYPE_SWITCH(Desc->getPrimType(), {
6913 Dest.deref<T>().~T();
6914 new (&Dest.deref<T>()) T();
6915 });
6916 }
6917 return;
6918 }
6919
6920 if (Desc->isCompositeArray()) {
6921 for (unsigned I = 0, N = Desc->getNumElems(); I != N; ++I) {
6922 PtrView ElemPtr = Dest.atIndex(Idx: I).narrow();
6923 zeroAll(Dest: ElemPtr);
6924 }
6925 return;
6926 }
6927}
6928
6929static bool copyComposite(InterpState &S, CodePtr OpPC, PtrView Src,
6930 PtrView Dest, bool Activate);
6931static bool copyRecord(InterpState &S, CodePtr OpPC, PtrView Src, PtrView Dest,
6932 bool Activate = false) {
6933 [[maybe_unused]] const Descriptor *SrcDesc = Src.getFieldDesc();
6934 const Descriptor *DestDesc = Dest.getFieldDesc();
6935
6936 auto copyField = [&](const Record::Field &F, bool Activate) -> bool {
6937 PtrView DestField = Dest.atField(Offset: F.Offset);
6938 if (OptPrimType FT = S.Ctx.classify(T: F.Decl->getType())) {
6939 TYPE_SWITCH(*FT, {
6940 DestField.deref<T>() = Src.atField(F.Offset).deref<T>();
6941 if (Src.atField(F.Offset).isInitialized())
6942 DestField.initialize();
6943 if (Activate)
6944 DestField.activate();
6945 });
6946 return true;
6947 }
6948 // Composite field.
6949 return copyComposite(S, OpPC, Src: Src.atField(Offset: F.Offset), Dest: DestField, Activate);
6950 };
6951
6952 assert(SrcDesc->isRecord());
6953 assert(SrcDesc->ElemRecord == DestDesc->ElemRecord);
6954 const Record *R = DestDesc->ElemRecord;
6955 for (const Record::Field &F : R->fields()) {
6956 PtrView FP = Src.atField(Offset: F.Offset);
6957
6958 if (!CheckMutable(S, OpPC, Ptr: FP))
6959 return false;
6960
6961 if (R->isUnion()) {
6962 // For unions, only copy the active field. Zero all others.
6963 if (FP.isActive()) {
6964 if (!copyField(F, /*Activate=*/true))
6965 return false;
6966 } else {
6967 PtrView DestField = Dest.atField(Offset: F.Offset);
6968 zeroAll(Dest: DestField);
6969 }
6970 } else {
6971 if (!copyField(F, Activate))
6972 return false;
6973 }
6974 }
6975
6976 for (const Record::Base &B : R->bases()) {
6977 PtrView DestBase = Dest.atField(Offset: B.Offset);
6978 if (!copyRecord(S, OpPC, Src: Src.atField(Offset: B.Offset), Dest: DestBase, Activate))
6979 return false;
6980 }
6981
6982 Dest.initialize();
6983 return true;
6984}
6985
6986static bool copyComposite(InterpState &S, CodePtr OpPC, PtrView Src,
6987 PtrView Dest, bool Activate = false) {
6988 assert(Src.isLive() && Dest.isLive());
6989
6990 [[maybe_unused]] const Descriptor *SrcDesc = Src.getFieldDesc();
6991 const Descriptor *DestDesc = Dest.getFieldDesc();
6992
6993 assert(!DestDesc->isPrimitive() && !SrcDesc->isPrimitive());
6994
6995 if (DestDesc->isPrimitiveArray()) {
6996 if (!SrcDesc->isPrimitiveArray())
6997 return false;
6998 // For floating types, check the actual QualType so we don't accidentally
6999 // mix up semantics.
7000 if (SrcDesc->getPrimType() == PT_Float) {
7001 if (!S.getASTContext().hasSimilarType(T1: SrcDesc->getElemQualType(),
7002 T2: DestDesc->getElemQualType()))
7003 return false;
7004 }
7005
7006 assert(SrcDesc->isPrimitiveArray());
7007 assert(SrcDesc->getNumElems() == DestDesc->getNumElems());
7008 assert(SrcDesc->getPrimType() == DestDesc->getPrimType());
7009 PrimType ET = DestDesc->getPrimType();
7010 for (unsigned I = 0, N = DestDesc->getNumElems(); I != N; ++I) {
7011 PtrView DestElem = Dest.atIndex(Idx: I);
7012 TYPE_SWITCH(ET, { DestElem.deref<T>() = Src.elem<T>(I); });
7013 DestElem.initializeElement(Index: I);
7014 }
7015 return true;
7016 }
7017
7018 if (DestDesc->isCompositeArray()) {
7019 if (!SrcDesc->isCompositeArray())
7020 return false;
7021 assert(SrcDesc->isCompositeArray());
7022 assert(SrcDesc->getNumElems() == DestDesc->getNumElems());
7023 for (unsigned I = 0, N = DestDesc->getNumElems(); I != N; ++I) {
7024 PtrView SrcElem = Src.atIndex(Idx: I).narrow();
7025 PtrView DestElem = Dest.atIndex(Idx: I).narrow();
7026 if (!copyComposite(S, OpPC, Src: SrcElem, Dest: DestElem, Activate))
7027 return false;
7028 }
7029 return true;
7030 }
7031
7032 if (DestDesc->isRecord()) {
7033 if (!SrcDesc->isRecord())
7034 return false;
7035 return copyRecord(S, OpPC, Src, Dest, Activate);
7036 }
7037 return Invalid(S, OpPC);
7038}
7039
7040bool DoMemcpy(InterpState &S, CodePtr OpPC, const Pointer &Src, Pointer &Dest) {
7041 if (!Src.isBlockPointer() || Src.getFieldDesc()->isPrimitive())
7042 return false;
7043 if (!Dest.isBlockPointer() || Dest.getFieldDesc()->isPrimitive())
7044 return false;
7045
7046 return copyComposite(S, OpPC, Src: Src.view(), Dest: Dest.view());
7047}
7048
7049} // namespace interp
7050} // namespace clang
7051