1//===--- InterpBuiltin.cpp - Interpreter for the constexpr VM ---*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8#include "../ExprConstShared.h"
9#include "Boolean.h"
10#include "EvalEmitter.h"
11#include "InterpBuiltinBitCast.h"
12#include "InterpHelpers.h"
13#include "PrimType.h"
14#include "Program.h"
15#include "clang/AST/InferAlloc.h"
16#include "clang/AST/OSLog.h"
17#include "clang/AST/RecordLayout.h"
18#include "clang/Basic/Builtins.h"
19#include "clang/Basic/TargetBuiltins.h"
20#include "clang/Basic/TargetInfo.h"
21#include "llvm/ADT/StringExtras.h"
22#include "llvm/Support/AllocToken.h"
23#include "llvm/Support/ErrorHandling.h"
24#include "llvm/Support/SipHash.h"
25
26namespace clang {
27namespace interp {
28
29[[maybe_unused]] static bool isNoopBuiltin(unsigned ID) {
30 switch (ID) {
31 case Builtin::BIas_const:
32 case Builtin::BIforward:
33 case Builtin::BIforward_like:
34 case Builtin::BImove:
35 case Builtin::BImove_if_noexcept:
36 case Builtin::BIaddressof:
37 case Builtin::BI__addressof:
38 case Builtin::BI__builtin_addressof:
39 case Builtin::BI__builtin_launder:
40 return true;
41 default:
42 return false;
43 }
44 return false;
45}
46
47static void discard(InterpStack &Stk, PrimType T) {
48 TYPE_SWITCH(T, { Stk.discard<T>(); });
49}
50
51static uint64_t popToUInt64(const InterpState &S, const Expr *E) {
52 INT_TYPE_SWITCH(*S.getContext().classify(E->getType()),
53 return static_cast<uint64_t>(S.Stk.pop<T>()));
54}
55
56static APSInt popToAPSInt(InterpStack &Stk, PrimType T) {
57 INT_TYPE_SWITCH(T, return Stk.pop<T>().toAPSInt());
58}
59
60static APSInt popToAPSInt(InterpState &S, const Expr *E) {
61 return popToAPSInt(Stk&: S.Stk, T: *S.getContext().classify(T: E->getType()));
62}
63static APSInt popToAPSInt(InterpState &S, QualType T) {
64 return popToAPSInt(Stk&: S.Stk, T: *S.getContext().classify(T));
65}
66
67/// Check for common reasons a pointer can't be read from, which
68/// are usually not diagnosed in a builtin function.
69static bool isReadable(const Pointer &P) {
70 if (P.isDummy())
71 return false;
72 if (!P.isBlockPointer())
73 return false;
74 if (!P.isLive())
75 return false;
76 if (P.isOnePastEnd())
77 return false;
78 return true;
79}
80
81/// Pushes \p Val on the stack as the type given by \p QT.
82static void pushInteger(InterpState &S, const APSInt &Val, QualType QT) {
83 assert(QT->isSignedIntegerOrEnumerationType() ||
84 QT->isUnsignedIntegerOrEnumerationType());
85 OptPrimType T = S.getContext().classify(T: QT);
86 assert(T);
87 unsigned BitWidth = S.getASTContext().getIntWidth(T: QT);
88
89 if (T == PT_IntAPS) {
90 auto Result = S.allocAP<IntegralAP<true>>(BitWidth);
91 Result.copy(V: Val);
92 S.Stk.push<IntegralAP<true>>(Args&: Result);
93 return;
94 }
95
96 if (T == PT_IntAP) {
97 auto Result = S.allocAP<IntegralAP<false>>(BitWidth);
98 Result.copy(V: Val);
99 S.Stk.push<IntegralAP<false>>(Args&: Result);
100 return;
101 }
102
103 if (QT->isSignedIntegerOrEnumerationType()) {
104 int64_t V = Val.getSExtValue();
105 INT_TYPE_SWITCH(*T, { S.Stk.push<T>(T::from(V, BitWidth)); });
106 } else {
107 assert(QT->isUnsignedIntegerOrEnumerationType());
108 uint64_t V = Val.getZExtValue();
109 INT_TYPE_SWITCH(*T, { S.Stk.push<T>(T::from(V, BitWidth)); });
110 }
111}
112
113template <typename T>
114static void pushInteger(InterpState &S, T Val, QualType QT) {
115 if constexpr (std::is_same_v<T, APInt>)
116 pushInteger(S, Val: APSInt(Val, !std::is_signed_v<T>), QT);
117 else if constexpr (std::is_same_v<T, APSInt>)
118 pushInteger(S, Val, QT);
119 else
120 pushInteger(S,
121 Val: APSInt(APInt(sizeof(T) * 8, static_cast<uint64_t>(Val),
122 std::is_signed_v<T>),
123 !std::is_signed_v<T>),
124 QT);
125}
126
127static void assignInteger(InterpState &S, const Pointer &Dest, PrimType ValueT,
128 const APSInt &Value) {
129
130 if (ValueT == PT_IntAPS) {
131 Dest.deref<IntegralAP<true>>() =
132 S.allocAP<IntegralAP<true>>(BitWidth: Value.getBitWidth());
133 Dest.deref<IntegralAP<true>>().copy(V: Value);
134 } else if (ValueT == PT_IntAP) {
135 Dest.deref<IntegralAP<false>>() =
136 S.allocAP<IntegralAP<false>>(BitWidth: Value.getBitWidth());
137 Dest.deref<IntegralAP<false>>().copy(V: Value);
138 } else {
139 INT_TYPE_SWITCH_NO_BOOL(
140 ValueT, { Dest.deref<T>() = T::from(static_cast<T>(Value)); });
141 }
142}
143
144static QualType getElemType(const Pointer &P) {
145 const Descriptor *Desc = P.getFieldDesc();
146 QualType T = Desc->getType();
147 if (Desc->isPrimitive())
148 return T;
149 if (T->isPointerType())
150 return T->castAs<PointerType>()->getPointeeType();
151 if (Desc->isArray())
152 return Desc->getElemQualType();
153 if (const auto *AT = T->getAsArrayTypeUnsafe())
154 return AT->getElementType();
155 return T;
156}
157
158static void diagnoseNonConstexprBuiltin(InterpState &S, CodePtr OpPC,
159 unsigned ID) {
160 if (!S.diagnosing())
161 return;
162
163 auto Loc = S.Current->getSource(PC: OpPC);
164 if (S.getLangOpts().CPlusPlus11)
165 S.CCEDiag(SI: Loc, DiagId: diag::note_constexpr_invalid_function)
166 << /*isConstexpr=*/0 << /*isConstructor=*/0
167 << S.getASTContext().BuiltinInfo.getQuotedName(ID);
168 else
169 S.CCEDiag(SI: Loc, DiagId: diag::note_invalid_subexpr_in_const_expr);
170}
171
172static llvm::APSInt convertBoolVectorToInt(const Pointer &Val) {
173 assert(Val.getFieldDesc()->isPrimitiveArray() &&
174 Val.getFieldDesc()->getElemQualType()->isBooleanType() &&
175 "Not a boolean vector");
176 unsigned NumElems = Val.getNumElems();
177
178 // Each element is one bit, so create an integer with NumElts bits.
179 llvm::APSInt Result(NumElems, 0);
180 for (unsigned I = 0; I != NumElems; ++I) {
181 if (Val.elem<bool>(I))
182 Result.setBit(I);
183 }
184
185 return Result;
186}
187
188// Strict double -> float conversion used for X86 PD2PS/cvtsd2ss intrinsics.
189// Reject NaN/Inf/Subnormal inputs and any lossy/inexact conversions.
190static bool convertDoubleToFloatStrict(APFloat Src, Floating &Dst,
191 InterpState &S, const Expr *DiagExpr) {
192 if (Src.isInfinity()) {
193 if (S.diagnosing())
194 S.CCEDiag(E: DiagExpr, DiagId: diag::note_constexpr_float_arithmetic) << 0;
195 return false;
196 }
197 if (Src.isNaN()) {
198 if (S.diagnosing())
199 S.CCEDiag(E: DiagExpr, DiagId: diag::note_constexpr_float_arithmetic) << 1;
200 return false;
201 }
202 APFloat Val = Src;
203 bool LosesInfo = false;
204 APFloat::opStatus Status = Val.convert(
205 ToSemantics: APFloat::IEEEsingle(), RM: APFloat::rmNearestTiesToEven, losesInfo: &LosesInfo);
206 if (LosesInfo || Val.isDenormal()) {
207 if (S.diagnosing())
208 S.CCEDiag(E: DiagExpr, DiagId: diag::note_constexpr_float_arithmetic_strict);
209 return false;
210 }
211 if (Status != APFloat::opOK) {
212 if (S.diagnosing())
213 S.CCEDiag(E: DiagExpr, DiagId: diag::note_invalid_subexpr_in_const_expr);
214 return false;
215 }
216 Dst.copy(F: Val);
217 return true;
218}
219
220static bool interp__builtin_is_constant_evaluated(InterpState &S, CodePtr OpPC,
221 const InterpFrame *Frame,
222 const CallExpr *Call) {
223 unsigned Depth = S.Current->getDepth();
224 auto isStdCall = [](const FunctionDecl *F) -> bool {
225 return F && F->isInStdNamespace() && F->getIdentifier() &&
226 F->getIdentifier()->isStr(Str: "is_constant_evaluated");
227 };
228 const InterpFrame *Caller = Frame->Caller;
229 // The current frame is the one for __builtin_is_constant_evaluated.
230 // The one above that, potentially the one for std::is_constant_evaluated().
231 if (S.inConstantContext() && !S.checkingPotentialConstantExpression() &&
232 S.getEvalStatus().Diag &&
233 (Depth == 0 || (Depth == 1 && isStdCall(Frame->getCallee())))) {
234 if (Caller && isStdCall(Frame->getCallee())) {
235 const Expr *E = Caller->getExpr(PC: Caller->getRetPC());
236 S.report(Loc: E->getExprLoc(),
237 DiagId: diag::warn_is_constant_evaluated_always_true_constexpr)
238 << "std::is_constant_evaluated" << E->getSourceRange();
239 } else {
240 S.report(Loc: Call->getExprLoc(),
241 DiagId: diag::warn_is_constant_evaluated_always_true_constexpr)
242 << "__builtin_is_constant_evaluated" << Call->getSourceRange();
243 }
244 }
245
246 S.Stk.push<Boolean>(Args: Boolean::from(Value: S.inConstantContext()));
247 return true;
248}
249
250// __builtin_assume
251// __assume (MS extension)
252static bool interp__builtin_assume(InterpState &S, CodePtr OpPC,
253 const InterpFrame *Frame,
254 const CallExpr *Call) {
255 // Nothing to be done here since the argument is NOT evaluated.
256 assert(Call->getNumArgs() == 1);
257 return true;
258}
259
260static bool interp__builtin_strcmp(InterpState &S, CodePtr OpPC,
261 const InterpFrame *Frame,
262 const CallExpr *Call, unsigned ID) {
263 uint64_t Limit = ~static_cast<uint64_t>(0);
264 if (ID == Builtin::BIstrncmp || ID == Builtin::BI__builtin_strncmp ||
265 ID == Builtin::BIwcsncmp || ID == Builtin::BI__builtin_wcsncmp)
266 Limit = popToUInt64(S, E: Call->getArg(Arg: 2));
267
268 const Pointer &B = S.Stk.pop<Pointer>();
269 const Pointer &A = S.Stk.pop<Pointer>();
270 if (ID == Builtin::BIstrcmp || ID == Builtin::BIstrncmp ||
271 ID == Builtin::BIwcscmp || ID == Builtin::BIwcsncmp)
272 diagnoseNonConstexprBuiltin(S, OpPC, ID);
273
274 if (Limit == 0) {
275 pushInteger(S, Val: 0, QT: Call->getType());
276 return true;
277 }
278
279 if (!CheckLive(S, OpPC, Ptr: A, AK: AK_Read) || !CheckLive(S, OpPC, Ptr: B, AK: AK_Read))
280 return false;
281
282 if (A.isDummy() || B.isDummy())
283 return false;
284 if (!A.isBlockPointer() || !B.isBlockPointer())
285 return false;
286
287 bool IsWide = ID == Builtin::BIwcscmp || ID == Builtin::BIwcsncmp ||
288 ID == Builtin::BI__builtin_wcscmp ||
289 ID == Builtin::BI__builtin_wcsncmp;
290 assert(A.getFieldDesc()->isPrimitiveArray());
291 assert(B.getFieldDesc()->isPrimitiveArray());
292
293 // Different element types shouldn't happen, but with casts they can.
294 if (!S.getASTContext().hasSameUnqualifiedType(T1: getElemType(P: A), T2: getElemType(P: B)))
295 return false;
296
297 PrimType ElemT = *S.getContext().classify(T: getElemType(P: A));
298
299 auto returnResult = [&](int V) -> bool {
300 pushInteger(S, Val: V, QT: Call->getType());
301 return true;
302 };
303
304 unsigned IndexA = A.getIndex();
305 unsigned IndexB = B.getIndex();
306 uint64_t Steps = 0;
307 for (;; ++IndexA, ++IndexB, ++Steps) {
308
309 if (Steps >= Limit)
310 break;
311 const Pointer &PA = A.atIndex(Idx: IndexA);
312 const Pointer &PB = B.atIndex(Idx: IndexB);
313 if (!CheckRange(S, OpPC, Ptr: PA, AK: AK_Read) ||
314 !CheckRange(S, OpPC, Ptr: PB, AK: AK_Read)) {
315 return false;
316 }
317
318 if (IsWide) {
319 INT_TYPE_SWITCH(ElemT, {
320 T CA = PA.deref<T>();
321 T CB = PB.deref<T>();
322 if (CA > CB)
323 return returnResult(1);
324 if (CA < CB)
325 return returnResult(-1);
326 if (CA.isZero() || CB.isZero())
327 return returnResult(0);
328 });
329 continue;
330 }
331
332 uint8_t CA = PA.deref<uint8_t>();
333 uint8_t CB = PB.deref<uint8_t>();
334
335 if (CA > CB)
336 return returnResult(1);
337 if (CA < CB)
338 return returnResult(-1);
339 if (CA == 0 || CB == 0)
340 return returnResult(0);
341 }
342
343 return returnResult(0);
344}
345
346static bool interp__builtin_strlen(InterpState &S, CodePtr OpPC,
347 const InterpFrame *Frame,
348 const CallExpr *Call, unsigned ID) {
349 const Pointer &StrPtr = S.Stk.pop<Pointer>().expand();
350
351 if (ID == Builtin::BIstrlen || ID == Builtin::BIwcslen)
352 diagnoseNonConstexprBuiltin(S, OpPC, ID);
353
354 if (!CheckArray(S, OpPC, Ptr: StrPtr))
355 return false;
356
357 if (!CheckLive(S, OpPC, Ptr: StrPtr, AK: AK_Read))
358 return false;
359
360 if (!CheckDummy(S, OpPC, B: StrPtr.block(), AK: AK_Read))
361 return false;
362
363 if (!StrPtr.getFieldDesc()->isPrimitiveArray())
364 return false;
365
366 assert(StrPtr.getFieldDesc()->isPrimitiveArray());
367 unsigned ElemSize = StrPtr.getFieldDesc()->getElemSize();
368
369 if (ID == Builtin::BI__builtin_wcslen || ID == Builtin::BIwcslen) {
370 const ASTContext &AC = S.getASTContext();
371 unsigned WCharSize = AC.getTypeSizeInChars(T: AC.getWCharType()).getQuantity();
372 if (ElemSize != WCharSize)
373 return false;
374 }
375
376 size_t Len = 0;
377 for (size_t I = StrPtr.getIndex();; ++I, ++Len) {
378 const Pointer &ElemPtr = StrPtr.atIndex(Idx: I);
379
380 if (!CheckRange(S, OpPC, Ptr: ElemPtr, AK: AK_Read))
381 return false;
382
383 uint32_t Val;
384 switch (ElemSize) {
385 case 1:
386 Val = ElemPtr.deref<uint8_t>();
387 break;
388 case 2:
389 Val = ElemPtr.deref<uint16_t>();
390 break;
391 case 4:
392 Val = ElemPtr.deref<uint32_t>();
393 break;
394 default:
395 llvm_unreachable("Unsupported char size");
396 }
397 if (Val == 0)
398 break;
399 }
400
401 pushInteger(S, Val: Len, QT: Call->getType());
402
403 return true;
404}
405
406static bool interp__builtin_nan(InterpState &S, CodePtr OpPC,
407 const InterpFrame *Frame, const CallExpr *Call,
408 bool Signaling) {
409 const Pointer &Arg = S.Stk.pop<Pointer>();
410
411 if (!CheckLoad(S, OpPC, Ptr: Arg))
412 return false;
413
414 assert(Arg.getFieldDesc()->isPrimitiveArray());
415
416 // Convert the given string to an integer using StringRef's API.
417 llvm::APInt Fill;
418 std::string Str;
419 assert(Arg.getNumElems() >= 1);
420 for (unsigned I = 0;; ++I) {
421 const Pointer &Elem = Arg.atIndex(Idx: I);
422
423 if (!CheckLoad(S, OpPC, Ptr: Elem))
424 return false;
425
426 if (Elem.deref<int8_t>() == 0)
427 break;
428
429 Str += Elem.deref<char>();
430 }
431
432 // Treat empty strings as if they were zero.
433 if (Str.empty())
434 Fill = llvm::APInt(32, 0);
435 else if (StringRef(Str).getAsInteger(Radix: 0, Result&: Fill))
436 return false;
437
438 const llvm::fltSemantics &TargetSemantics =
439 S.getASTContext().getFloatTypeSemantics(
440 T: Call->getDirectCallee()->getReturnType());
441
442 Floating Result = S.allocFloat(Sem: TargetSemantics);
443 if (S.getASTContext().getTargetInfo().isNan2008()) {
444 if (Signaling)
445 Result.copy(
446 F: llvm::APFloat::getSNaN(Sem: TargetSemantics, /*Negative=*/false, payload: &Fill));
447 else
448 Result.copy(
449 F: llvm::APFloat::getQNaN(Sem: TargetSemantics, /*Negative=*/false, payload: &Fill));
450 } else {
451 // Prior to IEEE 754-2008, architectures were allowed to choose whether
452 // the first bit of their significand was set for qNaN or sNaN. MIPS chose
453 // a different encoding to what became a standard in 2008, and for pre-
454 // 2008 revisions, MIPS interpreted sNaN-2008 as qNan and qNaN-2008 as
455 // sNaN. This is now known as "legacy NaN" encoding.
456 if (Signaling)
457 Result.copy(
458 F: llvm::APFloat::getQNaN(Sem: TargetSemantics, /*Negative=*/false, payload: &Fill));
459 else
460 Result.copy(
461 F: llvm::APFloat::getSNaN(Sem: TargetSemantics, /*Negative=*/false, payload: &Fill));
462 }
463
464 S.Stk.push<Floating>(Args&: Result);
465 return true;
466}
467
468static bool interp__builtin_inf(InterpState &S, CodePtr OpPC,
469 const InterpFrame *Frame,
470 const CallExpr *Call) {
471 const llvm::fltSemantics &TargetSemantics =
472 S.getASTContext().getFloatTypeSemantics(
473 T: Call->getDirectCallee()->getReturnType());
474
475 Floating Result = S.allocFloat(Sem: TargetSemantics);
476 Result.copy(F: APFloat::getInf(Sem: TargetSemantics));
477 S.Stk.push<Floating>(Args&: Result);
478 return true;
479}
480
481static bool interp__builtin_copysign(InterpState &S, CodePtr OpPC,
482 const InterpFrame *Frame) {
483 const Floating &Arg2 = S.Stk.pop<Floating>();
484 const Floating &Arg1 = S.Stk.pop<Floating>();
485 Floating Result = S.allocFloat(Sem: Arg1.getSemantics());
486
487 APFloat Copy = Arg1.getAPFloat();
488 Copy.copySign(RHS: Arg2.getAPFloat());
489 Result.copy(F: Copy);
490 S.Stk.push<Floating>(Args&: Result);
491
492 return true;
493}
494
495static bool interp__builtin_fmin(InterpState &S, CodePtr OpPC,
496 const InterpFrame *Frame, bool IsNumBuiltin) {
497 const Floating &RHS = S.Stk.pop<Floating>();
498 const Floating &LHS = S.Stk.pop<Floating>();
499 Floating Result = S.allocFloat(Sem: LHS.getSemantics());
500
501 if (IsNumBuiltin)
502 Result.copy(F: llvm::minimumnum(A: LHS.getAPFloat(), B: RHS.getAPFloat()));
503 else
504 Result.copy(F: minnum(A: LHS.getAPFloat(), B: RHS.getAPFloat()));
505 S.Stk.push<Floating>(Args&: Result);
506 return true;
507}
508
509static bool interp__builtin_fmax(InterpState &S, CodePtr OpPC,
510 const InterpFrame *Frame, bool IsNumBuiltin) {
511 const Floating &RHS = S.Stk.pop<Floating>();
512 const Floating &LHS = S.Stk.pop<Floating>();
513 Floating Result = S.allocFloat(Sem: LHS.getSemantics());
514
515 if (IsNumBuiltin)
516 Result.copy(F: llvm::maximumnum(A: LHS.getAPFloat(), B: RHS.getAPFloat()));
517 else
518 Result.copy(F: maxnum(A: LHS.getAPFloat(), B: RHS.getAPFloat()));
519 S.Stk.push<Floating>(Args&: Result);
520 return true;
521}
522
523/// Defined as __builtin_isnan(...), to accommodate the fact that it can
524/// take a float, double, long double, etc.
525/// But for us, that's all a Floating anyway.
526static bool interp__builtin_isnan(InterpState &S, CodePtr OpPC,
527 const InterpFrame *Frame,
528 const CallExpr *Call) {
529 const Floating &Arg = S.Stk.pop<Floating>();
530
531 pushInteger(S, Val: Arg.isNan(), QT: Call->getType());
532 return true;
533}
534
535static bool interp__builtin_issignaling(InterpState &S, CodePtr OpPC,
536 const InterpFrame *Frame,
537 const CallExpr *Call) {
538 const Floating &Arg = S.Stk.pop<Floating>();
539
540 pushInteger(S, Val: Arg.isSignaling(), QT: Call->getType());
541 return true;
542}
543
544static bool interp__builtin_isinf(InterpState &S, CodePtr OpPC,
545 const InterpFrame *Frame, bool CheckSign,
546 const CallExpr *Call) {
547 const Floating &Arg = S.Stk.pop<Floating>();
548 APFloat F = Arg.getAPFloat();
549 bool IsInf = F.isInfinity();
550
551 if (CheckSign)
552 pushInteger(S, Val: IsInf ? (F.isNegative() ? -1 : 1) : 0, QT: Call->getType());
553 else
554 pushInteger(S, Val: IsInf, QT: Call->getType());
555 return true;
556}
557
558static bool interp__builtin_isfinite(InterpState &S, CodePtr OpPC,
559 const InterpFrame *Frame,
560 const CallExpr *Call) {
561 const Floating &Arg = S.Stk.pop<Floating>();
562
563 pushInteger(S, Val: Arg.isFinite(), QT: Call->getType());
564 return true;
565}
566
567static bool interp__builtin_isnormal(InterpState &S, CodePtr OpPC,
568 const InterpFrame *Frame,
569 const CallExpr *Call) {
570 const Floating &Arg = S.Stk.pop<Floating>();
571
572 pushInteger(S, Val: Arg.isNormal(), QT: Call->getType());
573 return true;
574}
575
576static bool interp__builtin_issubnormal(InterpState &S, CodePtr OpPC,
577 const InterpFrame *Frame,
578 const CallExpr *Call) {
579 const Floating &Arg = S.Stk.pop<Floating>();
580
581 pushInteger(S, Val: Arg.isDenormal(), QT: Call->getType());
582 return true;
583}
584
585static bool interp__builtin_iszero(InterpState &S, CodePtr OpPC,
586 const InterpFrame *Frame,
587 const CallExpr *Call) {
588 const Floating &Arg = S.Stk.pop<Floating>();
589
590 pushInteger(S, Val: Arg.isZero(), QT: Call->getType());
591 return true;
592}
593
594static bool interp__builtin_signbit(InterpState &S, CodePtr OpPC,
595 const InterpFrame *Frame,
596 const CallExpr *Call) {
597 const Floating &Arg = S.Stk.pop<Floating>();
598
599 pushInteger(S, Val: Arg.isNegative(), QT: Call->getType());
600 return true;
601}
602
603static bool interp_floating_comparison(InterpState &S, CodePtr OpPC,
604 const CallExpr *Call, unsigned ID) {
605 const Floating &RHS = S.Stk.pop<Floating>();
606 const Floating &LHS = S.Stk.pop<Floating>();
607
608 pushInteger(
609 S,
610 Val: [&] {
611 switch (ID) {
612 case Builtin::BI__builtin_isgreater:
613 return LHS > RHS;
614 case Builtin::BI__builtin_isgreaterequal:
615 return LHS >= RHS;
616 case Builtin::BI__builtin_isless:
617 return LHS < RHS;
618 case Builtin::BI__builtin_islessequal:
619 return LHS <= RHS;
620 case Builtin::BI__builtin_islessgreater: {
621 ComparisonCategoryResult Cmp = LHS.compare(RHS);
622 return Cmp == ComparisonCategoryResult::Less ||
623 Cmp == ComparisonCategoryResult::Greater;
624 }
625 case Builtin::BI__builtin_isunordered:
626 return LHS.compare(RHS) == ComparisonCategoryResult::Unordered;
627 default:
628 llvm_unreachable("Unexpected builtin ID: Should be a floating point "
629 "comparison function");
630 }
631 }(),
632 QT: Call->getType());
633 return true;
634}
635
636/// First parameter to __builtin_isfpclass is the floating value, the
637/// second one is an integral value.
638static bool interp__builtin_isfpclass(InterpState &S, CodePtr OpPC,
639 const InterpFrame *Frame,
640 const CallExpr *Call) {
641 APSInt FPClassArg = popToAPSInt(S, E: Call->getArg(Arg: 1));
642 const Floating &F = S.Stk.pop<Floating>();
643
644 int32_t Result = static_cast<int32_t>(
645 (F.classify() & std::move(FPClassArg)).getZExtValue());
646 pushInteger(S, Val: Result, QT: Call->getType());
647
648 return true;
649}
650
651/// Five int values followed by one floating value.
652/// __builtin_fpclassify(int, int, int, int, int, float)
653static bool interp__builtin_fpclassify(InterpState &S, CodePtr OpPC,
654 const InterpFrame *Frame,
655 const CallExpr *Call) {
656 const Floating &Val = S.Stk.pop<Floating>();
657
658 PrimType IntT = *S.getContext().classify(E: Call->getArg(Arg: 0));
659 APSInt Values[5];
660 for (unsigned I = 0; I != 5; ++I)
661 Values[4 - I] = popToAPSInt(Stk&: S.Stk, T: IntT);
662
663 unsigned Index;
664 switch (Val.getCategory()) {
665 case APFloat::fcNaN:
666 Index = 0;
667 break;
668 case APFloat::fcInfinity:
669 Index = 1;
670 break;
671 case APFloat::fcNormal:
672 Index = Val.isDenormal() ? 3 : 2;
673 break;
674 case APFloat::fcZero:
675 Index = 4;
676 break;
677 }
678
679 // The last argument is first on the stack.
680 assert(Index <= 4);
681
682 pushInteger(S, Val: Values[Index], QT: Call->getType());
683 return true;
684}
685
686static inline Floating abs(InterpState &S, const Floating &In) {
687 if (!In.isNegative())
688 return In;
689
690 Floating Output = S.allocFloat(Sem: In.getSemantics());
691 APFloat New = In.getAPFloat();
692 New.changeSign();
693 Output.copy(F: New);
694 return Output;
695}
696
697// The C standard says "fabs raises no floating-point exceptions,
698// even if x is a signaling NaN. The returned value is independent of
699// the current rounding direction mode." Therefore constant folding can
700// proceed without regard to the floating point settings.
701// Reference, WG14 N2478 F.10.4.3
702static bool interp__builtin_fabs(InterpState &S, CodePtr OpPC,
703 const InterpFrame *Frame) {
704 const Floating &Val = S.Stk.pop<Floating>();
705 S.Stk.push<Floating>(Args: abs(S, In: Val));
706 return true;
707}
708
709static bool interp__builtin_abs(InterpState &S, CodePtr OpPC,
710 const InterpFrame *Frame,
711 const CallExpr *Call) {
712 APSInt Val = popToAPSInt(S, E: Call->getArg(Arg: 0));
713 if (Val ==
714 APSInt(APInt::getSignedMinValue(numBits: Val.getBitWidth()), /*IsUnsigned=*/false))
715 return false;
716 if (Val.isNegative())
717 Val.negate();
718 pushInteger(S, Val, QT: Call->getType());
719 return true;
720}
721
722static bool interp__builtin_popcount(InterpState &S, CodePtr OpPC,
723 const InterpFrame *Frame,
724 const CallExpr *Call) {
725 APSInt Val;
726 if (Call->getArg(Arg: 0)->getType()->isExtVectorBoolType()) {
727 const Pointer &Arg = S.Stk.pop<Pointer>();
728 Val = convertBoolVectorToInt(Val: Arg);
729 } else {
730 Val = popToAPSInt(S, E: Call->getArg(Arg: 0));
731 }
732 pushInteger(S, Val: Val.popcount(), QT: Call->getType());
733 return true;
734}
735
736static bool interp__builtin_ia32_crc32(InterpState &S, CodePtr OpPC,
737 const InterpFrame *Frame,
738 const CallExpr *Call,
739 unsigned DataBytes) {
740 uint64_t DataVal = popToUInt64(S, E: Call->getArg(Arg: 1));
741 uint64_t CRCVal = popToUInt64(S, E: Call->getArg(Arg: 0));
742
743 // CRC32C polynomial (iSCSI polynomial, bit-reversed)
744 static const uint32_t CRC32C_POLY = 0x82F63B78;
745
746 // Process each byte
747 uint32_t Result = static_cast<uint32_t>(CRCVal);
748 for (unsigned I = 0; I != DataBytes; ++I) {
749 uint8_t Byte = static_cast<uint8_t>((DataVal >> (I * 8)) & 0xFF);
750 Result ^= Byte;
751 for (int J = 0; J != 8; ++J) {
752 Result = (Result >> 1) ^ ((Result & 1) ? CRC32C_POLY : 0);
753 }
754 }
755
756 pushInteger(S, Val: Result, QT: Call->getType());
757 return true;
758}
759
760static bool interp__builtin_classify_type(InterpState &S, CodePtr OpPC,
761 const InterpFrame *Frame,
762 const CallExpr *Call) {
763 // This is an unevaluated call, so there are no arguments on the stack.
764 assert(Call->getNumArgs() == 1);
765 const Expr *Arg = Call->getArg(Arg: 0);
766
767 GCCTypeClass ResultClass =
768 EvaluateBuiltinClassifyType(T: Arg->getType(), LangOpts: S.getLangOpts());
769 int32_t ReturnVal = static_cast<int32_t>(ResultClass);
770 pushInteger(S, Val: ReturnVal, QT: Call->getType());
771 return true;
772}
773
774// __builtin_expect(long, long)
775// __builtin_expect_with_probability(long, long, double)
776static bool interp__builtin_expect(InterpState &S, CodePtr OpPC,
777 const InterpFrame *Frame,
778 const CallExpr *Call) {
779 // The return value is simply the value of the first parameter.
780 // We ignore the probability.
781 unsigned NumArgs = Call->getNumArgs();
782 assert(NumArgs == 2 || NumArgs == 3);
783
784 PrimType ArgT = *S.getContext().classify(T: Call->getArg(Arg: 0)->getType());
785 if (NumArgs == 3)
786 S.Stk.discard<Floating>();
787 discard(Stk&: S.Stk, T: ArgT);
788
789 APSInt Val = popToAPSInt(Stk&: S.Stk, T: ArgT);
790 pushInteger(S, Val, QT: Call->getType());
791 return true;
792}
793
794static bool interp__builtin_addressof(InterpState &S, CodePtr OpPC,
795 const InterpFrame *Frame,
796 const CallExpr *Call) {
797#ifndef NDEBUG
798 assert(Call->getArg(0)->isLValue());
799 PrimType PtrT = S.getContext().classify(Call->getArg(0)).value_or(PT_Ptr);
800 assert(PtrT == PT_Ptr &&
801 "Unsupported pointer type passed to __builtin_addressof()");
802#endif
803 return true;
804}
805
806static bool interp__builtin_move(InterpState &S, CodePtr OpPC,
807 const InterpFrame *Frame,
808 const CallExpr *Call) {
809 return Call->getDirectCallee()->isConstexpr();
810}
811
812static bool interp__builtin_eh_return_data_regno(InterpState &S, CodePtr OpPC,
813 const InterpFrame *Frame,
814 const CallExpr *Call) {
815 APSInt Arg = popToAPSInt(S, E: Call->getArg(Arg: 0));
816
817 int Result = S.getASTContext().getTargetInfo().getEHDataRegisterNumber(
818 RegNo: Arg.getZExtValue());
819 pushInteger(S, Val: Result, QT: Call->getType());
820 return true;
821}
822
823// Two integral values followed by a pointer (lhs, rhs, resultOut)
824static bool interp__builtin_overflowop(InterpState &S, CodePtr OpPC,
825 const CallExpr *Call,
826 unsigned BuiltinOp) {
827 const Pointer &ResultPtr = S.Stk.pop<Pointer>();
828 if (ResultPtr.isDummy() || !ResultPtr.isBlockPointer())
829 return false;
830
831 PrimType RHST = *S.getContext().classify(T: Call->getArg(Arg: 1)->getType());
832 PrimType LHST = *S.getContext().classify(T: Call->getArg(Arg: 0)->getType());
833 APSInt RHS = popToAPSInt(Stk&: S.Stk, T: RHST);
834 APSInt LHS = popToAPSInt(Stk&: S.Stk, T: LHST);
835 QualType ResultType = Call->getArg(Arg: 2)->getType()->getPointeeType();
836 PrimType ResultT = *S.getContext().classify(T: ResultType);
837 bool Overflow;
838
839 APSInt Result;
840 if (BuiltinOp == Builtin::BI__builtin_add_overflow ||
841 BuiltinOp == Builtin::BI__builtin_sub_overflow ||
842 BuiltinOp == Builtin::BI__builtin_mul_overflow) {
843 bool IsSigned = LHS.isSigned() || RHS.isSigned() ||
844 ResultType->isSignedIntegerOrEnumerationType();
845 bool AllSigned = LHS.isSigned() && RHS.isSigned() &&
846 ResultType->isSignedIntegerOrEnumerationType();
847 uint64_t LHSSize = LHS.getBitWidth();
848 uint64_t RHSSize = RHS.getBitWidth();
849 uint64_t ResultSize = S.getASTContext().getTypeSize(T: ResultType);
850 uint64_t MaxBits = std::max(a: std::max(a: LHSSize, b: RHSSize), b: ResultSize);
851
852 // Add an additional bit if the signedness isn't uniformly agreed to. We
853 // could do this ONLY if there is a signed and an unsigned that both have
854 // MaxBits, but the code to check that is pretty nasty. The issue will be
855 // caught in the shrink-to-result later anyway.
856 if (IsSigned && !AllSigned)
857 ++MaxBits;
858
859 LHS = APSInt(LHS.extOrTrunc(width: MaxBits), !IsSigned);
860 RHS = APSInt(RHS.extOrTrunc(width: MaxBits), !IsSigned);
861 Result = APSInt(MaxBits, !IsSigned);
862 }
863
864 // Find largest int.
865 switch (BuiltinOp) {
866 default:
867 llvm_unreachable("Invalid value for BuiltinOp");
868 case Builtin::BI__builtin_add_overflow:
869 case Builtin::BI__builtin_sadd_overflow:
870 case Builtin::BI__builtin_saddl_overflow:
871 case Builtin::BI__builtin_saddll_overflow:
872 case Builtin::BI__builtin_uadd_overflow:
873 case Builtin::BI__builtin_uaddl_overflow:
874 case Builtin::BI__builtin_uaddll_overflow:
875 Result = LHS.isSigned() ? LHS.sadd_ov(RHS, Overflow)
876 : LHS.uadd_ov(RHS, Overflow);
877 break;
878 case Builtin::BI__builtin_sub_overflow:
879 case Builtin::BI__builtin_ssub_overflow:
880 case Builtin::BI__builtin_ssubl_overflow:
881 case Builtin::BI__builtin_ssubll_overflow:
882 case Builtin::BI__builtin_usub_overflow:
883 case Builtin::BI__builtin_usubl_overflow:
884 case Builtin::BI__builtin_usubll_overflow:
885 Result = LHS.isSigned() ? LHS.ssub_ov(RHS, Overflow)
886 : LHS.usub_ov(RHS, Overflow);
887 break;
888 case Builtin::BI__builtin_mul_overflow:
889 case Builtin::BI__builtin_smul_overflow:
890 case Builtin::BI__builtin_smull_overflow:
891 case Builtin::BI__builtin_smulll_overflow:
892 case Builtin::BI__builtin_umul_overflow:
893 case Builtin::BI__builtin_umull_overflow:
894 case Builtin::BI__builtin_umulll_overflow:
895 Result = LHS.isSigned() ? LHS.smul_ov(RHS, Overflow)
896 : LHS.umul_ov(RHS, Overflow);
897 break;
898 }
899
900 // In the case where multiple sizes are allowed, truncate and see if
901 // the values are the same.
902 if (BuiltinOp == Builtin::BI__builtin_add_overflow ||
903 BuiltinOp == Builtin::BI__builtin_sub_overflow ||
904 BuiltinOp == Builtin::BI__builtin_mul_overflow) {
905 // APSInt doesn't have a TruncOrSelf, so we use extOrTrunc instead,
906 // since it will give us the behavior of a TruncOrSelf in the case where
907 // its parameter <= its size. We previously set Result to be at least the
908 // type-size of the result, so getTypeSize(ResultType) <= Resu
909 APSInt Temp = Result.extOrTrunc(width: S.getASTContext().getTypeSize(T: ResultType));
910 Temp.setIsSigned(ResultType->isSignedIntegerOrEnumerationType());
911
912 if (!APSInt::isSameValue(I1: Temp, I2: Result))
913 Overflow = true;
914 Result = std::move(Temp);
915 }
916
917 // Write Result to ResultPtr and put Overflow on the stack.
918 assignInteger(S, Dest: ResultPtr, ValueT: ResultT, Value: Result);
919 if (ResultPtr.canBeInitialized())
920 ResultPtr.initialize();
921
922 assert(Call->getDirectCallee()->getReturnType()->isBooleanType());
923 S.Stk.push<Boolean>(Args&: Overflow);
924 return true;
925}
926
927/// Three integral values followed by a pointer (lhs, rhs, carry, carryOut).
928static bool interp__builtin_carryop(InterpState &S, CodePtr OpPC,
929 const InterpFrame *Frame,
930 const CallExpr *Call, unsigned BuiltinOp) {
931 const Pointer &CarryOutPtr = S.Stk.pop<Pointer>();
932 PrimType LHST = *S.getContext().classify(T: Call->getArg(Arg: 0)->getType());
933 PrimType RHST = *S.getContext().classify(T: Call->getArg(Arg: 1)->getType());
934 APSInt CarryIn = popToAPSInt(Stk&: S.Stk, T: LHST);
935 APSInt RHS = popToAPSInt(Stk&: S.Stk, T: RHST);
936 APSInt LHS = popToAPSInt(Stk&: S.Stk, T: LHST);
937
938 if (CarryOutPtr.isDummy() || !CarryOutPtr.isBlockPointer())
939 return false;
940
941 APSInt CarryOut;
942
943 APSInt Result;
944 // Copy the number of bits and sign.
945 Result = LHS;
946 CarryOut = LHS;
947
948 bool FirstOverflowed = false;
949 bool SecondOverflowed = false;
950 switch (BuiltinOp) {
951 default:
952 llvm_unreachable("Invalid value for BuiltinOp");
953 case Builtin::BI__builtin_addcb:
954 case Builtin::BI__builtin_addcs:
955 case Builtin::BI__builtin_addc:
956 case Builtin::BI__builtin_addcl:
957 case Builtin::BI__builtin_addcll:
958 Result =
959 LHS.uadd_ov(RHS, Overflow&: FirstOverflowed).uadd_ov(RHS: CarryIn, Overflow&: SecondOverflowed);
960 break;
961 case Builtin::BI__builtin_subcb:
962 case Builtin::BI__builtin_subcs:
963 case Builtin::BI__builtin_subc:
964 case Builtin::BI__builtin_subcl:
965 case Builtin::BI__builtin_subcll:
966 Result =
967 LHS.usub_ov(RHS, Overflow&: FirstOverflowed).usub_ov(RHS: CarryIn, Overflow&: SecondOverflowed);
968 break;
969 }
970 // It is possible for both overflows to happen but CGBuiltin uses an OR so
971 // this is consistent.
972 CarryOut = (uint64_t)(FirstOverflowed | SecondOverflowed);
973
974 QualType CarryOutType = Call->getArg(Arg: 3)->getType()->getPointeeType();
975 PrimType CarryOutT = *S.getContext().classify(T: CarryOutType);
976 assignInteger(S, Dest: CarryOutPtr, ValueT: CarryOutT, Value: CarryOut);
977 CarryOutPtr.initialize();
978
979 assert(Call->getType() == Call->getArg(0)->getType());
980 pushInteger(S, Val: Result, QT: Call->getType());
981 return true;
982}
983
984static bool interp__builtin_clz(InterpState &S, CodePtr OpPC,
985 const InterpFrame *Frame, const CallExpr *Call,
986 unsigned BuiltinOp) {
987
988 std::optional<APSInt> Fallback;
989 if (BuiltinOp == Builtin::BI__builtin_clzg && Call->getNumArgs() == 2)
990 Fallback = popToAPSInt(S, E: Call->getArg(Arg: 1));
991
992 APSInt Val;
993 if (Call->getArg(Arg: 0)->getType()->isExtVectorBoolType()) {
994 const Pointer &Arg = S.Stk.pop<Pointer>();
995 Val = convertBoolVectorToInt(Val: Arg);
996 } else {
997 Val = popToAPSInt(S, E: Call->getArg(Arg: 0));
998 }
999
1000 // When the argument is 0, the result of GCC builtins is undefined, whereas
1001 // for Microsoft intrinsics, the result is the bit-width of the argument.
1002 bool ZeroIsUndefined = BuiltinOp != Builtin::BI__lzcnt16 &&
1003 BuiltinOp != Builtin::BI__lzcnt &&
1004 BuiltinOp != Builtin::BI__lzcnt64;
1005
1006 if (Val == 0) {
1007 if (Fallback) {
1008 pushInteger(S, Val: *Fallback, QT: Call->getType());
1009 return true;
1010 }
1011
1012 if (ZeroIsUndefined)
1013 return false;
1014 }
1015
1016 pushInteger(S, Val: Val.countl_zero(), QT: Call->getType());
1017 return true;
1018}
1019
1020static bool interp__builtin_ctz(InterpState &S, CodePtr OpPC,
1021 const InterpFrame *Frame, const CallExpr *Call,
1022 unsigned BuiltinID) {
1023 std::optional<APSInt> Fallback;
1024 if (BuiltinID == Builtin::BI__builtin_ctzg && Call->getNumArgs() == 2)
1025 Fallback = popToAPSInt(S, E: Call->getArg(Arg: 1));
1026
1027 APSInt Val;
1028 if (Call->getArg(Arg: 0)->getType()->isExtVectorBoolType()) {
1029 const Pointer &Arg = S.Stk.pop<Pointer>();
1030 Val = convertBoolVectorToInt(Val: Arg);
1031 } else {
1032 Val = popToAPSInt(S, E: Call->getArg(Arg: 0));
1033 }
1034
1035 if (Val == 0) {
1036 if (Fallback) {
1037 pushInteger(S, Val: *Fallback, QT: Call->getType());
1038 return true;
1039 }
1040 return false;
1041 }
1042
1043 pushInteger(S, Val: Val.countr_zero(), QT: Call->getType());
1044 return true;
1045}
1046
1047static bool interp__builtin_bswap(InterpState &S, CodePtr OpPC,
1048 const InterpFrame *Frame,
1049 const CallExpr *Call) {
1050 const APSInt &Val = popToAPSInt(S, E: Call->getArg(Arg: 0));
1051 if (Val.getBitWidth() == 8)
1052 pushInteger(S, Val, QT: Call->getType());
1053 else
1054 pushInteger(S, Val: Val.byteSwap(), QT: Call->getType());
1055 return true;
1056}
1057
1058/// bool __atomic_always_lock_free(size_t, void const volatile*)
1059/// bool __atomic_is_lock_free(size_t, void const volatile*)
1060static bool interp__builtin_atomic_lock_free(InterpState &S, CodePtr OpPC,
1061 const InterpFrame *Frame,
1062 const CallExpr *Call,
1063 unsigned BuiltinOp) {
1064 auto returnBool = [&S](bool Value) -> bool {
1065 S.Stk.push<Boolean>(Args&: Value);
1066 return true;
1067 };
1068
1069 const Pointer &Ptr = S.Stk.pop<Pointer>();
1070 uint64_t SizeVal = popToUInt64(S, E: Call->getArg(Arg: 0));
1071
1072 // For __atomic_is_lock_free(sizeof(_Atomic(T))), if the size is a power
1073 // of two less than or equal to the maximum inline atomic width, we know it
1074 // is lock-free. If the size isn't a power of two, or greater than the
1075 // maximum alignment where we promote atomics, we know it is not lock-free
1076 // (at least not in the sense of atomic_is_lock_free). Otherwise,
1077 // the answer can only be determined at runtime; for example, 16-byte
1078 // atomics have lock-free implementations on some, but not all,
1079 // x86-64 processors.
1080
1081 // Check power-of-two.
1082 CharUnits Size = CharUnits::fromQuantity(Quantity: SizeVal);
1083 if (Size.isPowerOfTwo()) {
1084 // Check against inlining width.
1085 unsigned InlineWidthBits =
1086 S.getASTContext().getTargetInfo().getMaxAtomicInlineWidth();
1087 if (Size <= S.getASTContext().toCharUnitsFromBits(BitSize: InlineWidthBits)) {
1088
1089 // OK, we will inline appropriately-aligned operations of this size,
1090 // and _Atomic(T) is appropriately-aligned.
1091 if (Size == CharUnits::One())
1092 return returnBool(true);
1093
1094 // Same for null pointers.
1095 assert(BuiltinOp != Builtin::BI__c11_atomic_is_lock_free);
1096 if (Ptr.isZero())
1097 return returnBool(true);
1098
1099 if (Ptr.isIntegralPointer()) {
1100 uint64_t IntVal = Ptr.getIntegerRepresentation();
1101 if (APSInt(APInt(64, IntVal, false), true).isAligned(A: Size.getAsAlign()))
1102 return returnBool(true);
1103 }
1104
1105 const Expr *PtrArg = Call->getArg(Arg: 1);
1106 // Otherwise, check if the type's alignment against Size.
1107 if (const auto *ICE = dyn_cast<ImplicitCastExpr>(Val: PtrArg)) {
1108 // Drop the potential implicit-cast to 'const volatile void*', getting
1109 // the underlying type.
1110 if (ICE->getCastKind() == CK_BitCast)
1111 PtrArg = ICE->getSubExpr();
1112 }
1113
1114 if (const auto *PtrTy = PtrArg->getType()->getAs<PointerType>()) {
1115 QualType PointeeType = PtrTy->getPointeeType();
1116 if (!PointeeType->isIncompleteType() &&
1117 S.getASTContext().getTypeAlignInChars(T: PointeeType) >= Size) {
1118 // OK, we will inline operations on this object.
1119 return returnBool(true);
1120 }
1121 }
1122 }
1123 }
1124
1125 if (BuiltinOp == Builtin::BI__atomic_always_lock_free)
1126 return returnBool(false);
1127
1128 return false;
1129}
1130
1131/// bool __c11_atomic_is_lock_free(size_t)
1132static bool interp__builtin_c11_atomic_is_lock_free(InterpState &S,
1133 CodePtr OpPC,
1134 const InterpFrame *Frame,
1135 const CallExpr *Call) {
1136 uint64_t SizeVal = popToUInt64(S, E: Call->getArg(Arg: 0));
1137
1138 CharUnits Size = CharUnits::fromQuantity(Quantity: SizeVal);
1139 if (Size.isPowerOfTwo()) {
1140 // Check against inlining width.
1141 unsigned InlineWidthBits =
1142 S.getASTContext().getTargetInfo().getMaxAtomicInlineWidth();
1143 if (Size <= S.getASTContext().toCharUnitsFromBits(BitSize: InlineWidthBits)) {
1144 S.Stk.push<Boolean>(Args: true);
1145 return true;
1146 }
1147 }
1148
1149 return false; // returnBool(false);
1150}
1151
1152/// __builtin_complex(Float A, float B);
1153static bool interp__builtin_complex(InterpState &S, CodePtr OpPC,
1154 const InterpFrame *Frame,
1155 const CallExpr *Call) {
1156 const Floating &Arg2 = S.Stk.pop<Floating>();
1157 const Floating &Arg1 = S.Stk.pop<Floating>();
1158 Pointer &Result = S.Stk.peek<Pointer>();
1159
1160 Result.elem<Floating>(I: 0) = Arg1;
1161 Result.elem<Floating>(I: 1) = Arg2;
1162 Result.initializeAllElements();
1163
1164 return true;
1165}
1166
1167/// __builtin_is_aligned()
1168/// __builtin_align_up()
1169/// __builtin_align_down()
1170/// The first parameter is either an integer or a pointer.
1171/// The second parameter is the requested alignment as an integer.
1172static bool interp__builtin_is_aligned_up_down(InterpState &S, CodePtr OpPC,
1173 const InterpFrame *Frame,
1174 const CallExpr *Call,
1175 unsigned BuiltinOp) {
1176 const APSInt &Alignment = popToAPSInt(S, E: Call->getArg(Arg: 1));
1177
1178 if (Alignment < 0 || !Alignment.isPowerOf2()) {
1179 S.FFDiag(E: Call, DiagId: diag::note_constexpr_invalid_alignment) << Alignment;
1180 return false;
1181 }
1182 unsigned SrcWidth = S.getASTContext().getIntWidth(T: Call->getArg(Arg: 0)->getType());
1183 APSInt MaxValue(APInt::getOneBitSet(numBits: SrcWidth, BitNo: SrcWidth - 1));
1184 if (APSInt::compareValues(I1: Alignment, I2: MaxValue) > 0) {
1185 S.FFDiag(E: Call, DiagId: diag::note_constexpr_alignment_too_big)
1186 << MaxValue << Call->getArg(Arg: 0)->getType() << Alignment;
1187 return false;
1188 }
1189
1190 // The first parameter is either an integer or a pointer.
1191 PrimType FirstArgT = *S.Ctx.classify(E: Call->getArg(Arg: 0));
1192
1193 if (isIntegralType(T: FirstArgT)) {
1194 const APSInt &Src = popToAPSInt(Stk&: S.Stk, T: FirstArgT);
1195 APInt AlignMinusOne = Alignment.extOrTrunc(width: Src.getBitWidth()) - 1;
1196 if (BuiltinOp == Builtin::BI__builtin_align_up) {
1197 APSInt AlignedVal =
1198 APSInt((Src + AlignMinusOne) & ~AlignMinusOne, Src.isUnsigned());
1199 pushInteger(S, Val: AlignedVal, QT: Call->getType());
1200 } else if (BuiltinOp == Builtin::BI__builtin_align_down) {
1201 APSInt AlignedVal = APSInt(Src & ~AlignMinusOne, Src.isUnsigned());
1202 pushInteger(S, Val: AlignedVal, QT: Call->getType());
1203 } else {
1204 assert(*S.Ctx.classify(Call->getType()) == PT_Bool);
1205 S.Stk.push<Boolean>(Args: (Src & AlignMinusOne) == 0);
1206 }
1207 return true;
1208 }
1209 assert(FirstArgT == PT_Ptr);
1210 const Pointer &Ptr = S.Stk.pop<Pointer>();
1211 if (!Ptr.isBlockPointer())
1212 return false;
1213
1214 // For one-past-end pointers, we can't call getIndex() since it asserts.
1215 // Use getNumElems() instead which gives the correct index for past-end.
1216 unsigned PtrOffset =
1217 Ptr.isElementPastEnd() ? Ptr.getNumElems() : Ptr.getIndex();
1218 CharUnits BaseAlignment =
1219 S.getASTContext().getDeclAlign(D: Ptr.getDeclDesc()->asValueDecl());
1220 CharUnits PtrAlign =
1221 BaseAlignment.alignmentAtOffset(offset: CharUnits::fromQuantity(Quantity: PtrOffset));
1222
1223 if (BuiltinOp == Builtin::BI__builtin_is_aligned) {
1224 if (PtrAlign.getQuantity() >= Alignment) {
1225 S.Stk.push<Boolean>(Args: true);
1226 return true;
1227 }
1228 // If the alignment is not known to be sufficient, some cases could still
1229 // be aligned at run time. However, if the requested alignment is less or
1230 // equal to the base alignment and the offset is not aligned, we know that
1231 // the run-time value can never be aligned.
1232 if (BaseAlignment.getQuantity() >= Alignment &&
1233 PtrAlign.getQuantity() < Alignment) {
1234 S.Stk.push<Boolean>(Args: false);
1235 return true;
1236 }
1237
1238 S.FFDiag(E: Call->getArg(Arg: 0), DiagId: diag::note_constexpr_alignment_compute)
1239 << Alignment;
1240 return false;
1241 }
1242
1243 assert(BuiltinOp == Builtin::BI__builtin_align_down ||
1244 BuiltinOp == Builtin::BI__builtin_align_up);
1245
1246 // For align_up/align_down, we can return the same value if the alignment
1247 // is known to be greater or equal to the requested value.
1248 if (PtrAlign.getQuantity() >= Alignment) {
1249 S.Stk.push<Pointer>(Args: Ptr);
1250 return true;
1251 }
1252
1253 // The alignment could be greater than the minimum at run-time, so we cannot
1254 // infer much about the resulting pointer value. One case is possible:
1255 // For `_Alignas(32) char buf[N]; __builtin_align_down(&buf[idx], 32)` we
1256 // can infer the correct index if the requested alignment is smaller than
1257 // the base alignment so we can perform the computation on the offset.
1258 if (BaseAlignment.getQuantity() >= Alignment) {
1259 assert(Alignment.getBitWidth() <= 64 &&
1260 "Cannot handle > 64-bit address-space");
1261 uint64_t Alignment64 = Alignment.getZExtValue();
1262 CharUnits NewOffset =
1263 CharUnits::fromQuantity(Quantity: BuiltinOp == Builtin::BI__builtin_align_down
1264 ? llvm::alignDown(Value: PtrOffset, Align: Alignment64)
1265 : llvm::alignTo(Value: PtrOffset, Align: Alignment64));
1266
1267 S.Stk.push<Pointer>(Args: Ptr.atIndex(Idx: NewOffset.getQuantity()));
1268 return true;
1269 }
1270
1271 // Otherwise, we cannot constant-evaluate the result.
1272 S.FFDiag(E: Call->getArg(Arg: 0), DiagId: diag::note_constexpr_alignment_adjust) << Alignment;
1273 return false;
1274}
1275
1276/// __builtin_assume_aligned(Ptr, Alignment[, ExtraOffset])
1277static bool interp__builtin_assume_aligned(InterpState &S, CodePtr OpPC,
1278 const InterpFrame *Frame,
1279 const CallExpr *Call) {
1280 assert(Call->getNumArgs() == 2 || Call->getNumArgs() == 3);
1281
1282 std::optional<APSInt> ExtraOffset;
1283 if (Call->getNumArgs() == 3)
1284 ExtraOffset = popToAPSInt(Stk&: S.Stk, T: *S.Ctx.classify(E: Call->getArg(Arg: 2)));
1285
1286 APSInt Alignment = popToAPSInt(Stk&: S.Stk, T: *S.Ctx.classify(E: Call->getArg(Arg: 1)));
1287 const Pointer &Ptr = S.Stk.pop<Pointer>();
1288
1289 CharUnits Align = CharUnits::fromQuantity(Quantity: Alignment.getZExtValue());
1290
1291 // If there is a base object, then it must have the correct alignment.
1292 if (Ptr.isBlockPointer()) {
1293 CharUnits BaseAlignment;
1294 if (const auto *VD = Ptr.getDeclDesc()->asValueDecl())
1295 BaseAlignment = S.getASTContext().getDeclAlign(D: VD);
1296 else if (const auto *E = Ptr.getDeclDesc()->asExpr())
1297 BaseAlignment = GetAlignOfExpr(Ctx: S.getASTContext(), E, ExprKind: UETT_AlignOf);
1298
1299 if (BaseAlignment < Align) {
1300 S.CCEDiag(E: Call->getArg(Arg: 0),
1301 DiagId: diag::note_constexpr_baa_insufficient_alignment)
1302 << 0 << BaseAlignment.getQuantity() << Align.getQuantity();
1303 return false;
1304 }
1305 }
1306
1307 APValue AV = Ptr.toAPValue(ASTCtx: S.getASTContext());
1308 CharUnits AVOffset = AV.getLValueOffset();
1309 if (ExtraOffset)
1310 AVOffset -= CharUnits::fromQuantity(Quantity: ExtraOffset->getZExtValue());
1311 if (AVOffset.alignTo(Align) != AVOffset) {
1312 if (Ptr.isBlockPointer())
1313 S.CCEDiag(E: Call->getArg(Arg: 0),
1314 DiagId: diag::note_constexpr_baa_insufficient_alignment)
1315 << 1 << AVOffset.getQuantity() << Align.getQuantity();
1316 else
1317 S.CCEDiag(E: Call->getArg(Arg: 0),
1318 DiagId: diag::note_constexpr_baa_value_insufficient_alignment)
1319 << AVOffset.getQuantity() << Align.getQuantity();
1320 return false;
1321 }
1322
1323 S.Stk.push<Pointer>(Args: Ptr);
1324 return true;
1325}
1326
1327/// (CarryIn, LHS, RHS, Result)
1328static bool interp__builtin_ia32_addcarry_subborrow(InterpState &S,
1329 CodePtr OpPC,
1330 const InterpFrame *Frame,
1331 const CallExpr *Call,
1332 unsigned BuiltinOp) {
1333 if (Call->getNumArgs() != 4 || !Call->getArg(Arg: 0)->getType()->isIntegerType() ||
1334 !Call->getArg(Arg: 1)->getType()->isIntegerType() ||
1335 !Call->getArg(Arg: 2)->getType()->isIntegerType())
1336 return false;
1337
1338 const Pointer &CarryOutPtr = S.Stk.pop<Pointer>();
1339
1340 APSInt RHS = popToAPSInt(S, E: Call->getArg(Arg: 2));
1341 APSInt LHS = popToAPSInt(S, E: Call->getArg(Arg: 1));
1342 APSInt CarryIn = popToAPSInt(S, E: Call->getArg(Arg: 0));
1343
1344 bool IsAdd = BuiltinOp == clang::X86::BI__builtin_ia32_addcarryx_u32 ||
1345 BuiltinOp == clang::X86::BI__builtin_ia32_addcarryx_u64;
1346
1347 unsigned BitWidth = LHS.getBitWidth();
1348 unsigned CarryInBit = CarryIn.ugt(RHS: 0) ? 1 : 0;
1349 APInt ExResult =
1350 IsAdd ? (LHS.zext(width: BitWidth + 1) + (RHS.zext(width: BitWidth + 1) + CarryInBit))
1351 : (LHS.zext(width: BitWidth + 1) - (RHS.zext(width: BitWidth + 1) + CarryInBit));
1352
1353 APInt Result = ExResult.extractBits(numBits: BitWidth, bitPosition: 0);
1354 APSInt CarryOut =
1355 APSInt(ExResult.extractBits(numBits: 1, bitPosition: BitWidth), /*IsUnsigned=*/true);
1356
1357 QualType CarryOutType = Call->getArg(Arg: 3)->getType()->getPointeeType();
1358 PrimType CarryOutT = *S.getContext().classify(T: CarryOutType);
1359 assignInteger(S, Dest: CarryOutPtr, ValueT: CarryOutT, Value: APSInt(std::move(Result), true));
1360
1361 pushInteger(S, Val: CarryOut, QT: Call->getType());
1362
1363 return true;
1364}
1365
1366static bool interp__builtin_os_log_format_buffer_size(InterpState &S,
1367 CodePtr OpPC,
1368 const InterpFrame *Frame,
1369 const CallExpr *Call) {
1370 analyze_os_log::OSLogBufferLayout Layout;
1371 analyze_os_log::computeOSLogBufferLayout(Ctx&: S.getASTContext(), E: Call, layout&: Layout);
1372 pushInteger(S, Val: Layout.size().getQuantity(), QT: Call->getType());
1373 return true;
1374}
1375
1376static bool
1377interp__builtin_ptrauth_string_discriminator(InterpState &S, CodePtr OpPC,
1378 const InterpFrame *Frame,
1379 const CallExpr *Call) {
1380 const auto &Ptr = S.Stk.pop<Pointer>();
1381 assert(Ptr.getFieldDesc()->isPrimitiveArray());
1382
1383 // This should be created for a StringLiteral, so should alway shold at least
1384 // one array element.
1385 assert(Ptr.getFieldDesc()->getNumElems() >= 1);
1386 StringRef R(&Ptr.deref<char>(), Ptr.getFieldDesc()->getNumElems() - 1);
1387 uint64_t Result = getPointerAuthStableSipHash(S: R);
1388 pushInteger(S, Val: Result, QT: Call->getType());
1389 return true;
1390}
1391
1392static bool interp__builtin_infer_alloc_token(InterpState &S, CodePtr OpPC,
1393 const InterpFrame *Frame,
1394 const CallExpr *Call) {
1395 const ASTContext &ASTCtx = S.getASTContext();
1396 uint64_t BitWidth = ASTCtx.getTypeSize(T: ASTCtx.getSizeType());
1397 auto Mode =
1398 ASTCtx.getLangOpts().AllocTokenMode.value_or(u: llvm::DefaultAllocTokenMode);
1399 auto MaxTokensOpt = ASTCtx.getLangOpts().AllocTokenMax;
1400 uint64_t MaxTokens =
1401 MaxTokensOpt.value_or(u: 0) ? *MaxTokensOpt : (~0ULL >> (64 - BitWidth));
1402
1403 // We do not read any of the arguments; discard them.
1404 for (int I = Call->getNumArgs() - 1; I >= 0; --I)
1405 discard(Stk&: S.Stk, T: S.getContext().classify(E: Call->getArg(Arg: I)).value_or(PT: PT_Ptr));
1406
1407 // Note: Type inference from a surrounding cast is not supported in
1408 // constexpr evaluation.
1409 QualType AllocType = infer_alloc::inferPossibleType(E: Call, Ctx: ASTCtx, CastE: nullptr);
1410 if (AllocType.isNull()) {
1411 S.CCEDiag(E: Call,
1412 DiagId: diag::note_constexpr_infer_alloc_token_type_inference_failed);
1413 return false;
1414 }
1415
1416 auto ATMD = infer_alloc::getAllocTokenMetadata(T: AllocType, Ctx: ASTCtx);
1417 if (!ATMD) {
1418 S.CCEDiag(E: Call, DiagId: diag::note_constexpr_infer_alloc_token_no_metadata);
1419 return false;
1420 }
1421
1422 auto MaybeToken = llvm::getAllocToken(Mode, Metadata: *ATMD, MaxTokens);
1423 if (!MaybeToken) {
1424 S.CCEDiag(E: Call, DiagId: diag::note_constexpr_infer_alloc_token_stateful_mode);
1425 return false;
1426 }
1427
1428 pushInteger(S, Val: llvm::APInt(BitWidth, *MaybeToken), QT: ASTCtx.getSizeType());
1429 return true;
1430}
1431
1432static bool interp__builtin_operator_new(InterpState &S, CodePtr OpPC,
1433 const InterpFrame *Frame,
1434 const CallExpr *Call) {
1435 // A call to __operator_new is only valid within std::allocate<>::allocate.
1436 // Walk up the call stack to find the appropriate caller and get the
1437 // element type from it.
1438 auto [NewCall, ElemType] = S.getStdAllocatorCaller(Name: "allocate");
1439
1440 if (ElemType.isNull()) {
1441 S.FFDiag(E: Call, DiagId: S.getLangOpts().CPlusPlus20
1442 ? diag::note_constexpr_new_untyped
1443 : diag::note_constexpr_new);
1444 return false;
1445 }
1446 assert(NewCall);
1447
1448 if (ElemType->isIncompleteType() || ElemType->isFunctionType()) {
1449 S.FFDiag(E: Call, DiagId: diag::note_constexpr_new_not_complete_object_type)
1450 << (ElemType->isIncompleteType() ? 0 : 1) << ElemType;
1451 return false;
1452 }
1453
1454 // We only care about the first parameter (the size), so discard all the
1455 // others.
1456 {
1457 unsigned NumArgs = Call->getNumArgs();
1458 assert(NumArgs >= 1);
1459
1460 // The std::nothrow_t arg never gets put on the stack.
1461 if (Call->getArg(Arg: NumArgs - 1)->getType()->isNothrowT())
1462 --NumArgs;
1463 auto Args = ArrayRef(Call->getArgs(), Call->getNumArgs());
1464 // First arg is needed.
1465 Args = Args.drop_front();
1466
1467 // Discard the rest.
1468 for (const Expr *Arg : Args)
1469 discard(Stk&: S.Stk, T: *S.getContext().classify(E: Arg));
1470 }
1471
1472 APSInt Bytes = popToAPSInt(S, E: Call->getArg(Arg: 0));
1473 CharUnits ElemSize = S.getASTContext().getTypeSizeInChars(T: ElemType);
1474 assert(!ElemSize.isZero());
1475 // Divide the number of bytes by sizeof(ElemType), so we get the number of
1476 // elements we should allocate.
1477 APInt NumElems, Remainder;
1478 APInt ElemSizeAP(Bytes.getBitWidth(), ElemSize.getQuantity());
1479 APInt::udivrem(LHS: Bytes, RHS: ElemSizeAP, Quotient&: NumElems, Remainder);
1480 if (Remainder != 0) {
1481 // This likely indicates a bug in the implementation of 'std::allocator'.
1482 S.FFDiag(E: Call, DiagId: diag::note_constexpr_operator_new_bad_size)
1483 << Bytes << APSInt(ElemSizeAP, true) << ElemType;
1484 return false;
1485 }
1486
1487 // NB: The same check we're using in CheckArraySize()
1488 if (NumElems.getActiveBits() >
1489 ConstantArrayType::getMaxSizeBits(Context: S.getASTContext()) ||
1490 NumElems.ugt(RHS: Descriptor::MaxArrayElemBytes / ElemSize.getQuantity())) {
1491 // FIXME: NoThrow check?
1492 const SourceInfo &Loc = S.Current->getSource(PC: OpPC);
1493 S.FFDiag(SI: Loc, DiagId: diag::note_constexpr_new_too_large)
1494 << NumElems.getZExtValue();
1495 return false;
1496 }
1497
1498 if (!CheckArraySize(S, OpPC, NumElems: NumElems.getZExtValue()))
1499 return false;
1500
1501 bool IsArray = NumElems.ugt(RHS: 1);
1502 OptPrimType ElemT = S.getContext().classify(T: ElemType);
1503 DynamicAllocator &Allocator = S.getAllocator();
1504 if (ElemT) {
1505 Block *B =
1506 Allocator.allocate(Source: NewCall, T: *ElemT, NumElements: NumElems.getZExtValue(),
1507 EvalID: S.Ctx.getEvalID(), AllocForm: DynamicAllocator::Form::Operator);
1508 assert(B);
1509 S.Stk.push<Pointer>(Args: Pointer(B).atIndex(Idx: 0));
1510 return true;
1511 }
1512
1513 assert(!ElemT);
1514
1515 // Composite arrays
1516 if (IsArray) {
1517 const Descriptor *Desc =
1518 S.P.createDescriptor(D: NewCall, Ty: ElemType.getTypePtr(), MDSize: std::nullopt);
1519 Block *B =
1520 Allocator.allocate(D: Desc, NumElements: NumElems.getZExtValue(), EvalID: S.Ctx.getEvalID(),
1521 AllocForm: DynamicAllocator::Form::Operator);
1522 assert(B);
1523 S.Stk.push<Pointer>(Args: Pointer(B).atIndex(Idx: 0).narrow());
1524 return true;
1525 }
1526
1527 // Records. Still allocate them as single-element arrays.
1528 QualType AllocType = S.getASTContext().getConstantArrayType(
1529 EltTy: ElemType, ArySize: NumElems, SizeExpr: nullptr, ASM: ArraySizeModifier::Normal, IndexTypeQuals: 0);
1530
1531 const Descriptor *Desc = S.P.createDescriptor(D: NewCall, Ty: AllocType.getTypePtr(),
1532 MDSize: Descriptor::InlineDescMD);
1533 Block *B = Allocator.allocate(D: Desc, EvalID: S.getContext().getEvalID(),
1534 AllocForm: DynamicAllocator::Form::Operator);
1535 assert(B);
1536 S.Stk.push<Pointer>(Args: Pointer(B).atIndex(Idx: 0).narrow());
1537 return true;
1538}
1539
1540static bool interp__builtin_operator_delete(InterpState &S, CodePtr OpPC,
1541 const InterpFrame *Frame,
1542 const CallExpr *Call) {
1543 const Expr *Source = nullptr;
1544 const Block *BlockToDelete = nullptr;
1545
1546 if (S.checkingPotentialConstantExpression()) {
1547 S.Stk.discard<Pointer>();
1548 return false;
1549 }
1550
1551 // This is permitted only within a call to std::allocator<T>::deallocate.
1552 if (!S.getStdAllocatorCaller(Name: "deallocate")) {
1553 S.FFDiag(E: Call);
1554 S.Stk.discard<Pointer>();
1555 return true;
1556 }
1557
1558 {
1559 const Pointer &Ptr = S.Stk.pop<Pointer>();
1560
1561 if (Ptr.isZero()) {
1562 S.CCEDiag(E: Call, DiagId: diag::note_constexpr_deallocate_null);
1563 return true;
1564 }
1565
1566 Source = Ptr.getDeclDesc()->asExpr();
1567 BlockToDelete = Ptr.block();
1568
1569 if (!BlockToDelete->isDynamic()) {
1570 S.FFDiag(E: Call, DiagId: diag::note_constexpr_delete_not_heap_alloc)
1571 << Ptr.toDiagnosticString(Ctx: S.getASTContext());
1572 if (const auto *D = Ptr.getFieldDesc()->asDecl())
1573 S.Note(Loc: D->getLocation(), DiagId: diag::note_declared_at);
1574 }
1575 }
1576 assert(BlockToDelete);
1577
1578 DynamicAllocator &Allocator = S.getAllocator();
1579 const Descriptor *BlockDesc = BlockToDelete->getDescriptor();
1580 std::optional<DynamicAllocator::Form> AllocForm =
1581 Allocator.getAllocationForm(Source);
1582
1583 if (!Allocator.deallocate(Source, BlockToDelete, S)) {
1584 // Nothing has been deallocated, this must be a double-delete.
1585 const SourceInfo &Loc = S.Current->getSource(PC: OpPC);
1586 S.FFDiag(SI: Loc, DiagId: diag::note_constexpr_double_delete);
1587 return false;
1588 }
1589 assert(AllocForm);
1590
1591 return CheckNewDeleteForms(
1592 S, OpPC, AllocForm: *AllocForm, DeleteForm: DynamicAllocator::Form::Operator, D: BlockDesc, NewExpr: Source);
1593}
1594
1595static bool interp__builtin_arithmetic_fence(InterpState &S, CodePtr OpPC,
1596 const InterpFrame *Frame,
1597 const CallExpr *Call) {
1598 const Floating &Arg0 = S.Stk.pop<Floating>();
1599 S.Stk.push<Floating>(Args: Arg0);
1600 return true;
1601}
1602
1603static bool interp__builtin_vector_reduce(InterpState &S, CodePtr OpPC,
1604 const CallExpr *Call, unsigned ID) {
1605 const Pointer &Arg = S.Stk.pop<Pointer>();
1606 assert(Arg.getFieldDesc()->isPrimitiveArray());
1607
1608 QualType ElemType = Arg.getFieldDesc()->getElemQualType();
1609 assert(Call->getType() == ElemType);
1610 PrimType ElemT = *S.getContext().classify(T: ElemType);
1611 unsigned NumElems = Arg.getNumElems();
1612
1613 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
1614 T Result = Arg.elem<T>(0);
1615 unsigned BitWidth = Result.bitWidth();
1616 for (unsigned I = 1; I != NumElems; ++I) {
1617 T Elem = Arg.elem<T>(I);
1618 T PrevResult = Result;
1619
1620 if (ID == Builtin::BI__builtin_reduce_add) {
1621 if (T::add(Result, Elem, BitWidth, &Result)) {
1622 unsigned OverflowBits = BitWidth + 1;
1623 (void)handleOverflow(S, OpPC,
1624 (PrevResult.toAPSInt(OverflowBits) +
1625 Elem.toAPSInt(OverflowBits)));
1626 return false;
1627 }
1628 } else if (ID == Builtin::BI__builtin_reduce_mul) {
1629 if (T::mul(Result, Elem, BitWidth, &Result)) {
1630 unsigned OverflowBits = BitWidth * 2;
1631 (void)handleOverflow(S, OpPC,
1632 (PrevResult.toAPSInt(OverflowBits) *
1633 Elem.toAPSInt(OverflowBits)));
1634 return false;
1635 }
1636
1637 } else if (ID == Builtin::BI__builtin_reduce_and) {
1638 (void)T::bitAnd(Result, Elem, BitWidth, &Result);
1639 } else if (ID == Builtin::BI__builtin_reduce_or) {
1640 (void)T::bitOr(Result, Elem, BitWidth, &Result);
1641 } else if (ID == Builtin::BI__builtin_reduce_xor) {
1642 (void)T::bitXor(Result, Elem, BitWidth, &Result);
1643 } else if (ID == Builtin::BI__builtin_reduce_min) {
1644 if (Elem < Result)
1645 Result = Elem;
1646 } else if (ID == Builtin::BI__builtin_reduce_max) {
1647 if (Elem > Result)
1648 Result = Elem;
1649 } else {
1650 llvm_unreachable("Unhandled vector reduce builtin");
1651 }
1652 }
1653 pushInteger(S, Result.toAPSInt(), Call->getType());
1654 });
1655
1656 return true;
1657}
1658
1659static bool interp__builtin_elementwise_abs(InterpState &S, CodePtr OpPC,
1660 const InterpFrame *Frame,
1661 const CallExpr *Call,
1662 unsigned BuiltinID) {
1663 assert(Call->getNumArgs() == 1);
1664 QualType Ty = Call->getArg(Arg: 0)->getType();
1665 if (Ty->isIntegerType()) {
1666 APSInt Val = popToAPSInt(S, E: Call->getArg(Arg: 0));
1667 pushInteger(S, Val: Val.abs(), QT: Call->getType());
1668 return true;
1669 }
1670
1671 if (Ty->isFloatingType()) {
1672 Floating Val = S.Stk.pop<Floating>();
1673 Floating Result = abs(S, In: Val);
1674 S.Stk.push<Floating>(Args&: Result);
1675 return true;
1676 }
1677
1678 // Otherwise, the argument must be a vector.
1679 assert(Call->getArg(0)->getType()->isVectorType());
1680 const Pointer &Arg = S.Stk.pop<Pointer>();
1681 assert(Arg.getFieldDesc()->isPrimitiveArray());
1682 const Pointer &Dst = S.Stk.peek<Pointer>();
1683 assert(Dst.getFieldDesc()->isPrimitiveArray());
1684 assert(Arg.getFieldDesc()->getNumElems() ==
1685 Dst.getFieldDesc()->getNumElems());
1686
1687 QualType ElemType = Arg.getFieldDesc()->getElemQualType();
1688 PrimType ElemT = *S.getContext().classify(T: ElemType);
1689 unsigned NumElems = Arg.getNumElems();
1690 // we can either have a vector of integer or a vector of floating point
1691 for (unsigned I = 0; I != NumElems; ++I) {
1692 if (ElemType->isIntegerType()) {
1693 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
1694 Dst.elem<T>(I) = T::from(static_cast<T>(
1695 APSInt(Arg.elem<T>(I).toAPSInt().abs(),
1696 ElemType->isUnsignedIntegerOrEnumerationType())));
1697 });
1698 } else {
1699 Floating Val = Arg.elem<Floating>(I);
1700 Dst.elem<Floating>(I) = abs(S, In: Val);
1701 }
1702 }
1703 Dst.initializeAllElements();
1704
1705 return true;
1706}
1707
1708/// Can be called with an integer or vector as the first and only parameter.
1709static bool interp__builtin_elementwise_countzeroes(InterpState &S,
1710 CodePtr OpPC,
1711 const InterpFrame *Frame,
1712 const CallExpr *Call,
1713 unsigned BuiltinID) {
1714 bool HasZeroArg = Call->getNumArgs() == 2;
1715 bool IsCTTZ = BuiltinID == Builtin::BI__builtin_elementwise_ctzg;
1716 assert(Call->getNumArgs() == 1 || HasZeroArg);
1717 if (Call->getArg(Arg: 0)->getType()->isIntegerType()) {
1718 PrimType ArgT = *S.getContext().classify(T: Call->getArg(Arg: 0)->getType());
1719 APSInt Val = popToAPSInt(Stk&: S.Stk, T: ArgT);
1720 std::optional<APSInt> ZeroVal;
1721 if (HasZeroArg) {
1722 ZeroVal = Val;
1723 Val = popToAPSInt(Stk&: S.Stk, T: ArgT);
1724 }
1725
1726 if (Val.isZero()) {
1727 if (ZeroVal) {
1728 pushInteger(S, Val: *ZeroVal, QT: Call->getType());
1729 return true;
1730 }
1731 // If we haven't been provided the second argument, the result is
1732 // undefined
1733 S.FFDiag(SI: S.Current->getSource(PC: OpPC),
1734 DiagId: diag::note_constexpr_countzeroes_zero)
1735 << /*IsTrailing=*/IsCTTZ;
1736 return false;
1737 }
1738
1739 if (BuiltinID == Builtin::BI__builtin_elementwise_clzg) {
1740 pushInteger(S, Val: Val.countLeadingZeros(), QT: Call->getType());
1741 } else {
1742 pushInteger(S, Val: Val.countTrailingZeros(), QT: Call->getType());
1743 }
1744 return true;
1745 }
1746 // Otherwise, the argument must be a vector.
1747 const ASTContext &ASTCtx = S.getASTContext();
1748 Pointer ZeroArg;
1749 if (HasZeroArg) {
1750 assert(Call->getArg(1)->getType()->isVectorType() &&
1751 ASTCtx.hasSameUnqualifiedType(Call->getArg(0)->getType(),
1752 Call->getArg(1)->getType()));
1753 (void)ASTCtx;
1754 ZeroArg = S.Stk.pop<Pointer>();
1755 assert(ZeroArg.getFieldDesc()->isPrimitiveArray());
1756 }
1757 assert(Call->getArg(0)->getType()->isVectorType());
1758 const Pointer &Arg = S.Stk.pop<Pointer>();
1759 assert(Arg.getFieldDesc()->isPrimitiveArray());
1760 const Pointer &Dst = S.Stk.peek<Pointer>();
1761 assert(Dst.getFieldDesc()->isPrimitiveArray());
1762 assert(Arg.getFieldDesc()->getNumElems() ==
1763 Dst.getFieldDesc()->getNumElems());
1764
1765 QualType ElemType = Arg.getFieldDesc()->getElemQualType();
1766 PrimType ElemT = *S.getContext().classify(T: ElemType);
1767 unsigned NumElems = Arg.getNumElems();
1768
1769 // FIXME: Reading from uninitialized vector elements?
1770 for (unsigned I = 0; I != NumElems; ++I) {
1771 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
1772 APInt EltVal = Arg.atIndex(I).deref<T>().toAPSInt();
1773 if (EltVal.isZero()) {
1774 if (HasZeroArg) {
1775 Dst.atIndex(I).deref<T>() = ZeroArg.atIndex(I).deref<T>();
1776 } else {
1777 // If we haven't been provided the second argument, the result is
1778 // undefined
1779 S.FFDiag(S.Current->getSource(OpPC),
1780 diag::note_constexpr_countzeroes_zero)
1781 << /*IsTrailing=*/IsCTTZ;
1782 return false;
1783 }
1784 } else if (IsCTTZ) {
1785 Dst.atIndex(I).deref<T>() = T::from(EltVal.countTrailingZeros());
1786 } else {
1787 Dst.atIndex(I).deref<T>() = T::from(EltVal.countLeadingZeros());
1788 }
1789 Dst.atIndex(I).initialize();
1790 });
1791 }
1792
1793 return true;
1794}
1795
1796static bool interp__builtin_memcpy(InterpState &S, CodePtr OpPC,
1797 const InterpFrame *Frame,
1798 const CallExpr *Call, unsigned ID) {
1799 assert(Call->getNumArgs() == 3);
1800 const ASTContext &ASTCtx = S.getASTContext();
1801 uint64_t Size = popToUInt64(S, E: Call->getArg(Arg: 2));
1802 Pointer SrcPtr = S.Stk.pop<Pointer>().expand();
1803 Pointer DestPtr = S.Stk.pop<Pointer>().expand();
1804
1805 if (ID == Builtin::BImemcpy || ID == Builtin::BImemmove)
1806 diagnoseNonConstexprBuiltin(S, OpPC, ID);
1807
1808 bool Move =
1809 (ID == Builtin::BI__builtin_memmove || ID == Builtin::BImemmove ||
1810 ID == Builtin::BI__builtin_wmemmove || ID == Builtin::BIwmemmove);
1811 bool WChar = ID == Builtin::BIwmemcpy || ID == Builtin::BIwmemmove ||
1812 ID == Builtin::BI__builtin_wmemcpy ||
1813 ID == Builtin::BI__builtin_wmemmove;
1814
1815 // If the size is zero, we treat this as always being a valid no-op.
1816 if (Size == 0) {
1817 S.Stk.push<Pointer>(Args&: DestPtr);
1818 return true;
1819 }
1820
1821 if (SrcPtr.isZero() || DestPtr.isZero()) {
1822 Pointer DiagPtr = (SrcPtr.isZero() ? SrcPtr : DestPtr);
1823 S.FFDiag(SI: S.Current->getSource(PC: OpPC), DiagId: diag::note_constexpr_memcpy_null)
1824 << /*IsMove=*/Move << /*IsWchar=*/WChar << !SrcPtr.isZero()
1825 << DiagPtr.toDiagnosticString(Ctx: ASTCtx);
1826 return false;
1827 }
1828
1829 // Diagnose integral src/dest pointers specially.
1830 if (SrcPtr.isIntegralPointer() || DestPtr.isIntegralPointer()) {
1831 std::string DiagVal = "(void *)";
1832 DiagVal += SrcPtr.isIntegralPointer()
1833 ? std::to_string(val: SrcPtr.getIntegerRepresentation())
1834 : std::to_string(val: DestPtr.getIntegerRepresentation());
1835 S.FFDiag(SI: S.Current->getSource(PC: OpPC), DiagId: diag::note_constexpr_memcpy_null)
1836 << Move << WChar << DestPtr.isIntegralPointer() << DiagVal;
1837 return false;
1838 }
1839
1840 if (!isReadable(P: DestPtr) || !isReadable(P: SrcPtr))
1841 return false;
1842
1843 if (DestPtr.getType()->isIncompleteType()) {
1844 S.FFDiag(SI: S.Current->getSource(PC: OpPC),
1845 DiagId: diag::note_constexpr_memcpy_incomplete_type)
1846 << Move << DestPtr.getType();
1847 return false;
1848 }
1849 if (SrcPtr.getType()->isIncompleteType()) {
1850 S.FFDiag(SI: S.Current->getSource(PC: OpPC),
1851 DiagId: diag::note_constexpr_memcpy_incomplete_type)
1852 << Move << SrcPtr.getType();
1853 return false;
1854 }
1855
1856 QualType DestElemType = getElemType(P: DestPtr);
1857 if (DestElemType->isIncompleteType()) {
1858 S.FFDiag(SI: S.Current->getSource(PC: OpPC),
1859 DiagId: diag::note_constexpr_memcpy_incomplete_type)
1860 << Move << DestElemType;
1861 return false;
1862 }
1863
1864 size_t RemainingDestElems;
1865 if (DestPtr.getFieldDesc()->isArray()) {
1866 RemainingDestElems = DestPtr.isUnknownSizeArray()
1867 ? 0
1868 : (DestPtr.getNumElems() - DestPtr.getIndex());
1869 } else {
1870 RemainingDestElems = 1;
1871 }
1872 unsigned DestElemSize = ASTCtx.getTypeSizeInChars(T: DestElemType).getQuantity();
1873
1874 if (WChar) {
1875 uint64_t WCharSize =
1876 ASTCtx.getTypeSizeInChars(T: ASTCtx.getWCharType()).getQuantity();
1877 Size *= WCharSize;
1878 }
1879
1880 if (Size % DestElemSize != 0) {
1881 S.FFDiag(SI: S.Current->getSource(PC: OpPC),
1882 DiagId: diag::note_constexpr_memcpy_unsupported)
1883 << Move << WChar << 0 << DestElemType << Size << DestElemSize;
1884 return false;
1885 }
1886
1887 QualType SrcElemType = getElemType(P: SrcPtr);
1888 size_t RemainingSrcElems;
1889 if (SrcPtr.getFieldDesc()->isArray()) {
1890 RemainingSrcElems = SrcPtr.isUnknownSizeArray()
1891 ? 0
1892 : (SrcPtr.getNumElems() - SrcPtr.getIndex());
1893 } else {
1894 RemainingSrcElems = 1;
1895 }
1896 unsigned SrcElemSize = ASTCtx.getTypeSizeInChars(T: SrcElemType).getQuantity();
1897
1898 if (!ASTCtx.hasSameUnqualifiedType(T1: DestElemType, T2: SrcElemType)) {
1899 S.FFDiag(SI: S.Current->getSource(PC: OpPC), DiagId: diag::note_constexpr_memcpy_type_pun)
1900 << Move << SrcElemType << DestElemType;
1901 return false;
1902 }
1903
1904 if (!DestElemType.isTriviallyCopyableType(Context: ASTCtx)) {
1905 S.FFDiag(SI: S.Current->getSource(PC: OpPC), DiagId: diag::note_constexpr_memcpy_nontrivial)
1906 << Move << DestElemType;
1907 return false;
1908 }
1909
1910 // Check if we have enough elements to read from and write to.
1911 size_t RemainingDestBytes = RemainingDestElems * DestElemSize;
1912 size_t RemainingSrcBytes = RemainingSrcElems * SrcElemSize;
1913 if (Size > RemainingDestBytes || Size > RemainingSrcBytes) {
1914 APInt N = APInt(64, Size / DestElemSize);
1915 S.FFDiag(SI: S.Current->getSource(PC: OpPC),
1916 DiagId: diag::note_constexpr_memcpy_unsupported)
1917 << Move << WChar << (Size > RemainingSrcBytes ? 1 : 2) << DestElemType
1918 << toString(I: N, Radix: 10, /*Signed=*/false);
1919 return false;
1920 }
1921
1922 // Check for overlapping memory regions.
1923 if (!Move && Pointer::pointToSameBlock(A: SrcPtr, B: DestPtr)) {
1924 // Remove base casts.
1925 Pointer SrcP = SrcPtr.stripBaseCasts();
1926 Pointer DestP = DestPtr.stripBaseCasts();
1927
1928 unsigned SrcIndex = SrcP.expand().getIndex() * SrcP.elemSize();
1929 unsigned DstIndex = DestP.expand().getIndex() * DestP.elemSize();
1930
1931 if ((SrcIndex <= DstIndex && (SrcIndex + Size) > DstIndex) ||
1932 (DstIndex <= SrcIndex && (DstIndex + Size) > SrcIndex)) {
1933 S.FFDiag(SI: S.Current->getSource(PC: OpPC), DiagId: diag::note_constexpr_memcpy_overlap)
1934 << /*IsWChar=*/false;
1935 return false;
1936 }
1937 }
1938
1939 assert(Size % DestElemSize == 0);
1940 if (!DoMemcpy(S, OpPC, SrcPtr, DestPtr, Size: Bytes(Size).toBits()))
1941 return false;
1942
1943 S.Stk.push<Pointer>(Args&: DestPtr);
1944 return true;
1945}
1946
1947/// Determine if T is a character type for which we guarantee that
1948/// sizeof(T) == 1.
1949static bool isOneByteCharacterType(QualType T) {
1950 return T->isCharType() || T->isChar8Type();
1951}
1952
1953static bool interp__builtin_memcmp(InterpState &S, CodePtr OpPC,
1954 const InterpFrame *Frame,
1955 const CallExpr *Call, unsigned ID) {
1956 assert(Call->getNumArgs() == 3);
1957 uint64_t Size = popToUInt64(S, E: Call->getArg(Arg: 2));
1958 const Pointer &PtrB = S.Stk.pop<Pointer>();
1959 const Pointer &PtrA = S.Stk.pop<Pointer>();
1960
1961 if (ID == Builtin::BImemcmp || ID == Builtin::BIbcmp ||
1962 ID == Builtin::BIwmemcmp)
1963 diagnoseNonConstexprBuiltin(S, OpPC, ID);
1964
1965 if (Size == 0) {
1966 pushInteger(S, Val: 0, QT: Call->getType());
1967 return true;
1968 }
1969
1970 if (!PtrA.isBlockPointer() || !PtrB.isBlockPointer())
1971 return false;
1972
1973 bool IsWide =
1974 (ID == Builtin::BIwmemcmp || ID == Builtin::BI__builtin_wmemcmp);
1975
1976 const ASTContext &ASTCtx = S.getASTContext();
1977 QualType ElemTypeA = getElemType(P: PtrA);
1978 QualType ElemTypeB = getElemType(P: PtrB);
1979 // FIXME: This is an arbitrary limitation the current constant interpreter
1980 // had. We could remove this.
1981 if (!IsWide && (!isOneByteCharacterType(T: ElemTypeA) ||
1982 !isOneByteCharacterType(T: ElemTypeB))) {
1983 S.FFDiag(SI: S.Current->getSource(PC: OpPC),
1984 DiagId: diag::note_constexpr_memcmp_unsupported)
1985 << ASTCtx.BuiltinInfo.getQuotedName(ID) << PtrA.getType()
1986 << PtrB.getType();
1987 return false;
1988 }
1989
1990 if (PtrA.isDummy() || PtrB.isDummy())
1991 return false;
1992
1993 if (!CheckRange(S, OpPC, Ptr: PtrA, AK: AK_Read) ||
1994 !CheckRange(S, OpPC, Ptr: PtrB, AK: AK_Read))
1995 return false;
1996
1997 // Now, read both pointers to a buffer and compare those.
1998 BitcastBuffer BufferA(
1999 Bits(ASTCtx.getTypeSize(T: ElemTypeA) * PtrA.getNumElems()));
2000 readPointerToBuffer(Ctx: S.getContext(), FromPtr: PtrA, Buffer&: BufferA, ReturnOnUninit: false);
2001 // FIXME: The swapping here is UNDOING something we do when reading the
2002 // data into the buffer.
2003 if (ASTCtx.getTargetInfo().isBigEndian())
2004 swapBytes(M: BufferA.Data.get(), N: BufferA.byteSize().getQuantity());
2005
2006 BitcastBuffer BufferB(
2007 Bits(ASTCtx.getTypeSize(T: ElemTypeB) * PtrB.getNumElems()));
2008 readPointerToBuffer(Ctx: S.getContext(), FromPtr: PtrB, Buffer&: BufferB, ReturnOnUninit: false);
2009 // FIXME: The swapping here is UNDOING something we do when reading the
2010 // data into the buffer.
2011 if (ASTCtx.getTargetInfo().isBigEndian())
2012 swapBytes(M: BufferB.Data.get(), N: BufferB.byteSize().getQuantity());
2013
2014 size_t MinBufferSize = std::min(a: BufferA.byteSize().getQuantity(),
2015 b: BufferB.byteSize().getQuantity());
2016
2017 unsigned ElemSize = 1;
2018 if (IsWide)
2019 ElemSize = ASTCtx.getTypeSizeInChars(T: ASTCtx.getWCharType()).getQuantity();
2020 // The Size given for the wide variants is in wide-char units. Convert it
2021 // to bytes.
2022 size_t ByteSize = Size * ElemSize;
2023 size_t CmpSize = std::min(a: MinBufferSize, b: ByteSize);
2024
2025 for (size_t I = 0; I != CmpSize; I += ElemSize) {
2026 if (IsWide) {
2027 INT_TYPE_SWITCH(*S.getContext().classify(ASTCtx.getWCharType()), {
2028 T A = *reinterpret_cast<T *>(BufferA.atByte(I));
2029 T B = *reinterpret_cast<T *>(BufferB.atByte(I));
2030 if (A < B) {
2031 pushInteger(S, -1, Call->getType());
2032 return true;
2033 }
2034 if (A > B) {
2035 pushInteger(S, 1, Call->getType());
2036 return true;
2037 }
2038 });
2039 } else {
2040 std::byte A = BufferA.deref<std::byte>(Offset: Bytes(I));
2041 std::byte B = BufferB.deref<std::byte>(Offset: Bytes(I));
2042
2043 if (A < B) {
2044 pushInteger(S, Val: -1, QT: Call->getType());
2045 return true;
2046 }
2047 if (A > B) {
2048 pushInteger(S, Val: 1, QT: Call->getType());
2049 return true;
2050 }
2051 }
2052 }
2053
2054 // We compared CmpSize bytes above. If the limiting factor was the Size
2055 // passed, we're done and the result is equality (0).
2056 if (ByteSize <= CmpSize) {
2057 pushInteger(S, Val: 0, QT: Call->getType());
2058 return true;
2059 }
2060
2061 // However, if we read all the available bytes but were instructed to read
2062 // even more, diagnose this as a "read of dereferenced one-past-the-end
2063 // pointer". This is what would happen if we called CheckLoad() on every array
2064 // element.
2065 S.FFDiag(SI: S.Current->getSource(PC: OpPC), DiagId: diag::note_constexpr_access_past_end)
2066 << AK_Read << S.Current->getRange(PC: OpPC);
2067 return false;
2068}
2069
2070// __builtin_memchr(ptr, int, int)
2071// __builtin_strchr(ptr, int)
2072static bool interp__builtin_memchr(InterpState &S, CodePtr OpPC,
2073 const CallExpr *Call, unsigned ID) {
2074 if (ID == Builtin::BImemchr || ID == Builtin::BIwcschr ||
2075 ID == Builtin::BIstrchr || ID == Builtin::BIwmemchr)
2076 diagnoseNonConstexprBuiltin(S, OpPC, ID);
2077
2078 std::optional<APSInt> MaxLength;
2079 if (Call->getNumArgs() == 3)
2080 MaxLength = popToAPSInt(S, E: Call->getArg(Arg: 2));
2081
2082 APSInt Desired = popToAPSInt(S, E: Call->getArg(Arg: 1));
2083 const Pointer &Ptr = S.Stk.pop<Pointer>();
2084
2085 if (MaxLength && MaxLength->isZero()) {
2086 S.Stk.push<Pointer>();
2087 return true;
2088 }
2089
2090 if (Ptr.isDummy()) {
2091 if (Ptr.getType()->isIncompleteType())
2092 S.FFDiag(SI: S.Current->getSource(PC: OpPC),
2093 DiagId: diag::note_constexpr_ltor_incomplete_type)
2094 << Ptr.getType();
2095 return false;
2096 }
2097
2098 // Null is only okay if the given size is 0.
2099 if (Ptr.isZero()) {
2100 S.FFDiag(SI: S.Current->getSource(PC: OpPC), DiagId: diag::note_constexpr_access_null)
2101 << AK_Read;
2102 return false;
2103 }
2104
2105 if (!Ptr.isBlockPointer())
2106 return false;
2107
2108 QualType ElemTy = Ptr.getFieldDesc()->isArray()
2109 ? Ptr.getFieldDesc()->getElemQualType()
2110 : Ptr.getFieldDesc()->getType();
2111 bool IsRawByte = ID == Builtin::BImemchr || ID == Builtin::BI__builtin_memchr;
2112
2113 // Give up on byte-oriented matching against multibyte elements.
2114 if (IsRawByte && !isOneByteCharacterType(T: ElemTy)) {
2115 S.FFDiag(SI: S.Current->getSource(PC: OpPC),
2116 DiagId: diag::note_constexpr_memchr_unsupported)
2117 << S.getASTContext().BuiltinInfo.getQuotedName(ID) << ElemTy;
2118 return false;
2119 }
2120
2121 if (!isReadable(P: Ptr))
2122 return false;
2123
2124 if (ID == Builtin::BIstrchr || ID == Builtin::BI__builtin_strchr) {
2125 int64_t DesiredTrunc;
2126 if (S.getASTContext().CharTy->isSignedIntegerType())
2127 DesiredTrunc =
2128 Desired.trunc(width: S.getASTContext().getCharWidth()).getSExtValue();
2129 else
2130 DesiredTrunc =
2131 Desired.trunc(width: S.getASTContext().getCharWidth()).getZExtValue();
2132 // strchr compares directly to the passed integer, and therefore
2133 // always fails if given an int that is not a char.
2134 if (Desired != DesiredTrunc) {
2135 S.Stk.push<Pointer>();
2136 return true;
2137 }
2138 }
2139
2140 uint64_t DesiredVal;
2141 if (ID == Builtin::BIwmemchr || ID == Builtin::BI__builtin_wmemchr ||
2142 ID == Builtin::BIwcschr || ID == Builtin::BI__builtin_wcschr) {
2143 // wcschr and wmemchr are given a wchar_t to look for. Just use it.
2144 DesiredVal = Desired.getZExtValue();
2145 } else {
2146 DesiredVal = Desired.trunc(width: S.getASTContext().getCharWidth()).getZExtValue();
2147 }
2148
2149 bool StopAtZero =
2150 (ID == Builtin::BIstrchr || ID == Builtin::BI__builtin_strchr ||
2151 ID == Builtin::BIwcschr || ID == Builtin::BI__builtin_wcschr);
2152
2153 PrimType ElemT =
2154 IsRawByte ? PT_Sint8 : *S.getContext().classify(T: getElemType(P: Ptr));
2155
2156 size_t Index = Ptr.getIndex();
2157 size_t Step = 0;
2158 for (;;) {
2159 const Pointer &ElemPtr =
2160 (Index + Step) > 0 ? Ptr.atIndex(Idx: Index + Step) : Ptr;
2161
2162 if (!CheckLoad(S, OpPC, Ptr: ElemPtr))
2163 return false;
2164
2165 uint64_t V;
2166 INT_TYPE_SWITCH_NO_BOOL(
2167 ElemT, { V = static_cast<uint64_t>(ElemPtr.deref<T>().toUnsigned()); });
2168
2169 if (V == DesiredVal) {
2170 S.Stk.push<Pointer>(Args: ElemPtr);
2171 return true;
2172 }
2173
2174 if (StopAtZero && V == 0)
2175 break;
2176
2177 ++Step;
2178 if (MaxLength && Step == MaxLength->getZExtValue())
2179 break;
2180 }
2181
2182 S.Stk.push<Pointer>();
2183 return true;
2184}
2185
2186static std::optional<unsigned> computeFullDescSize(const ASTContext &ASTCtx,
2187 const Descriptor *Desc) {
2188 if (Desc->isPrimitive())
2189 return ASTCtx.getTypeSizeInChars(T: Desc->getType()).getQuantity();
2190 if (Desc->isArray())
2191 return ASTCtx.getTypeSizeInChars(T: Desc->getElemQualType()).getQuantity() *
2192 Desc->getNumElems();
2193 if (Desc->isRecord()) {
2194 // Can't use Descriptor::getType() as that may return a pointer type. Look
2195 // at the decl directly.
2196 return ASTCtx
2197 .getTypeSizeInChars(
2198 T: ASTCtx.getCanonicalTagType(TD: Desc->ElemRecord->getDecl()))
2199 .getQuantity();
2200 }
2201
2202 return std::nullopt;
2203}
2204
2205/// Compute the byte offset of \p Ptr in the full declaration.
2206static unsigned computePointerOffset(const ASTContext &ASTCtx,
2207 const Pointer &Ptr) {
2208 unsigned Result = 0;
2209
2210 Pointer P = Ptr;
2211 while (P.isField() || P.isArrayElement()) {
2212 P = P.expand();
2213 const Descriptor *D = P.getFieldDesc();
2214
2215 if (P.isArrayElement()) {
2216 unsigned ElemSize =
2217 ASTCtx.getTypeSizeInChars(T: D->getElemQualType()).getQuantity();
2218 if (P.isOnePastEnd())
2219 Result += ElemSize * P.getNumElems();
2220 else
2221 Result += ElemSize * P.getIndex();
2222 P = P.expand().getArray();
2223 } else if (P.isBaseClass()) {
2224 const auto *RD = cast<CXXRecordDecl>(Val: D->asDecl());
2225 bool IsVirtual = Ptr.isVirtualBaseClass();
2226 P = P.getBase();
2227 const Record *BaseRecord = P.getRecord();
2228
2229 const ASTRecordLayout &Layout =
2230 ASTCtx.getASTRecordLayout(D: cast<CXXRecordDecl>(Val: BaseRecord->getDecl()));
2231 if (IsVirtual)
2232 Result += Layout.getVBaseClassOffset(VBase: RD).getQuantity();
2233 else
2234 Result += Layout.getBaseClassOffset(Base: RD).getQuantity();
2235 } else if (P.isField()) {
2236 const FieldDecl *FD = P.getField();
2237 const ASTRecordLayout &Layout =
2238 ASTCtx.getASTRecordLayout(D: FD->getParent());
2239 unsigned FieldIndex = FD->getFieldIndex();
2240 uint64_t FieldOffset =
2241 ASTCtx.toCharUnitsFromBits(BitSize: Layout.getFieldOffset(FieldNo: FieldIndex))
2242 .getQuantity();
2243 Result += FieldOffset;
2244 P = P.getBase();
2245 } else
2246 llvm_unreachable("Unhandled descriptor type");
2247 }
2248
2249 return Result;
2250}
2251
2252/// Does Ptr point to the last subobject?
2253static bool pointsToLastObject(const Pointer &Ptr) {
2254 Pointer P = Ptr;
2255 while (!P.isRoot()) {
2256
2257 if (P.isArrayElement()) {
2258 P = P.expand().getArray();
2259 continue;
2260 }
2261 if (P.isBaseClass()) {
2262 if (P.getRecord()->getNumFields() > 0)
2263 return false;
2264 P = P.getBase();
2265 continue;
2266 }
2267
2268 Pointer Base = P.getBase();
2269 if (const Record *R = Base.getRecord()) {
2270 assert(P.getField());
2271 if (P.getField()->getFieldIndex() != R->getNumFields() - 1)
2272 return false;
2273 }
2274 P = Base;
2275 }
2276
2277 return true;
2278}
2279
2280/// Does Ptr point to the last object AND to a flexible array member?
2281static bool isUserWritingOffTheEnd(const ASTContext &Ctx, const Pointer &Ptr) {
2282 auto isFlexibleArrayMember = [&](const Descriptor *FieldDesc) {
2283 using FAMKind = LangOptions::StrictFlexArraysLevelKind;
2284 FAMKind StrictFlexArraysLevel =
2285 Ctx.getLangOpts().getStrictFlexArraysLevel();
2286
2287 if (StrictFlexArraysLevel == FAMKind::Default)
2288 return true;
2289
2290 unsigned NumElems = FieldDesc->getNumElems();
2291 if (NumElems == 0 && StrictFlexArraysLevel != FAMKind::IncompleteOnly)
2292 return true;
2293
2294 if (NumElems == 1 && StrictFlexArraysLevel == FAMKind::OneZeroOrIncomplete)
2295 return true;
2296 return false;
2297 };
2298
2299 const Descriptor *FieldDesc = Ptr.getFieldDesc();
2300 if (!FieldDesc->isArray())
2301 return false;
2302
2303 return Ptr.isDummy() && pointsToLastObject(Ptr) &&
2304 isFlexibleArrayMember(FieldDesc);
2305}
2306
2307UnsignedOrNone evaluateBuiltinObjectSize(const ASTContext &ASTCtx,
2308 unsigned Kind, Pointer &Ptr) {
2309 if (Ptr.isZero() || !Ptr.isBlockPointer())
2310 return std::nullopt;
2311
2312 if (Ptr.isDummy() && Ptr.getType()->isPointerType())
2313 return std::nullopt;
2314
2315 // According to the GCC documentation, we want the size of the subobject
2316 // denoted by the pointer. But that's not quite right -- what we actually
2317 // want is the size of the immediately-enclosing array, if there is one.
2318 if (Ptr.isArrayElement())
2319 Ptr = Ptr.expand();
2320
2321 bool DetermineForCompleteObject = Ptr.getFieldDesc() == Ptr.getDeclDesc();
2322 const Descriptor *DeclDesc = Ptr.getDeclDesc();
2323 assert(DeclDesc);
2324
2325 bool UseFieldDesc = (Kind & 1u);
2326 bool ReportMinimum = (Kind & 2u);
2327 if (!UseFieldDesc || DetermineForCompleteObject) {
2328 // Lower bound, so we can't fall back to this.
2329 if (ReportMinimum && !DetermineForCompleteObject)
2330 return std::nullopt;
2331
2332 // Can't read beyond the pointer decl desc.
2333 if (!UseFieldDesc && !ReportMinimum && DeclDesc->getType()->isPointerType())
2334 return std::nullopt;
2335 } else {
2336 if (isUserWritingOffTheEnd(Ctx: ASTCtx, Ptr)) {
2337 // If we cannot determine the size of the initial allocation, then we
2338 // can't given an accurate upper-bound. However, we are still able to give
2339 // conservative lower-bounds for Type=3.
2340 if (Kind == 1)
2341 return std::nullopt;
2342 }
2343 }
2344
2345 // The "closest surrounding subobject" is NOT a base class,
2346 // so strip the base class casts.
2347 if (UseFieldDesc && Ptr.isBaseClass())
2348 Ptr = Ptr.stripBaseCasts();
2349
2350 const Descriptor *Desc = UseFieldDesc ? Ptr.getFieldDesc() : DeclDesc;
2351 assert(Desc);
2352
2353 std::optional<unsigned> FullSize = computeFullDescSize(ASTCtx, Desc);
2354 if (!FullSize)
2355 return std::nullopt;
2356
2357 unsigned ByteOffset;
2358 if (UseFieldDesc) {
2359 if (Ptr.isBaseClass()) {
2360 assert(computePointerOffset(ASTCtx, Ptr.getBase()) <=
2361 computePointerOffset(ASTCtx, Ptr));
2362 ByteOffset = computePointerOffset(ASTCtx, Ptr: Ptr.getBase()) -
2363 computePointerOffset(ASTCtx, Ptr);
2364 } else {
2365 if (Ptr.inArray())
2366 ByteOffset =
2367 computePointerOffset(ASTCtx, Ptr) -
2368 computePointerOffset(ASTCtx, Ptr: Ptr.expand().atIndex(Idx: 0).narrow());
2369 else
2370 ByteOffset = 0;
2371 }
2372 } else
2373 ByteOffset = computePointerOffset(ASTCtx, Ptr);
2374
2375 assert(ByteOffset <= *FullSize);
2376 return *FullSize - ByteOffset;
2377}
2378
2379static bool interp__builtin_object_size(InterpState &S, CodePtr OpPC,
2380 const InterpFrame *Frame,
2381 const CallExpr *Call) {
2382 const ASTContext &ASTCtx = S.getASTContext();
2383 // From the GCC docs:
2384 // Kind is an integer constant from 0 to 3. If the least significant bit is
2385 // clear, objects are whole variables. If it is set, a closest surrounding
2386 // subobject is considered the object a pointer points to. The second bit
2387 // determines if maximum or minimum of remaining bytes is computed.
2388 unsigned Kind = popToUInt64(S, E: Call->getArg(Arg: 1));
2389 assert(Kind <= 3 && "unexpected kind");
2390 Pointer Ptr = S.Stk.pop<Pointer>();
2391
2392 if (Call->getArg(Arg: 0)->HasSideEffects(Ctx: ASTCtx)) {
2393 // "If there are any side effects in them, it returns (size_t) -1
2394 // for type 0 or 1 and (size_t) 0 for type 2 or 3."
2395 pushInteger(S, Val: Kind <= 1 ? -1 : 0, QT: Call->getType());
2396 return true;
2397 }
2398
2399 if (auto Result = evaluateBuiltinObjectSize(ASTCtx, Kind, Ptr)) {
2400 pushInteger(S, Val: *Result, QT: Call->getType());
2401 return true;
2402 }
2403 return false;
2404}
2405
2406static bool interp__builtin_is_within_lifetime(InterpState &S, CodePtr OpPC,
2407 const CallExpr *Call) {
2408
2409 if (!S.inConstantContext())
2410 return false;
2411
2412 const Pointer &Ptr = S.Stk.pop<Pointer>();
2413
2414 auto Error = [&](int Diag) {
2415 bool CalledFromStd = false;
2416 const auto *Callee = S.Current->getCallee();
2417 if (Callee && Callee->isInStdNamespace()) {
2418 const IdentifierInfo *Identifier = Callee->getIdentifier();
2419 CalledFromStd = Identifier && Identifier->isStr(Str: "is_within_lifetime");
2420 }
2421 S.CCEDiag(SI: CalledFromStd
2422 ? S.Current->Caller->getSource(PC: S.Current->getRetPC())
2423 : S.Current->getSource(PC: OpPC),
2424 DiagId: diag::err_invalid_is_within_lifetime)
2425 << (CalledFromStd ? "std::is_within_lifetime"
2426 : "__builtin_is_within_lifetime")
2427 << Diag;
2428 return false;
2429 };
2430
2431 if (Ptr.isZero())
2432 return Error(0);
2433 if (Ptr.isOnePastEnd())
2434 return Error(1);
2435
2436 bool Result = Ptr.getLifetime() != Lifetime::Ended;
2437 if (!Ptr.isActive()) {
2438 Result = false;
2439 } else {
2440 if (!CheckLive(S, OpPC, Ptr, AK: AK_Read))
2441 return false;
2442 if (!CheckMutable(S, OpPC, Ptr))
2443 return false;
2444 if (!CheckDummy(S, OpPC, B: Ptr.block(), AK: AK_Read))
2445 return false;
2446 }
2447
2448 // Check if we're currently running an initializer.
2449 if (llvm::is_contained(Range&: S.InitializingBlocks, Element: Ptr.block()))
2450 return Error(2);
2451 if (S.EvaluatingDecl && Ptr.getDeclDesc()->asVarDecl() == S.EvaluatingDecl)
2452 return Error(2);
2453
2454 pushInteger(S, Val: Result, QT: Call->getType());
2455 return true;
2456}
2457
2458static bool interp__builtin_elementwise_int_unaryop(
2459 InterpState &S, CodePtr OpPC, const CallExpr *Call,
2460 llvm::function_ref<APInt(const APSInt &)> Fn) {
2461 assert(Call->getNumArgs() == 1);
2462
2463 // Single integer case.
2464 if (!Call->getArg(Arg: 0)->getType()->isVectorType()) {
2465 assert(Call->getType()->isIntegerType());
2466 APSInt Src = popToAPSInt(S, E: Call->getArg(Arg: 0));
2467 APInt Result = Fn(Src);
2468 pushInteger(S, Val: APSInt(std::move(Result), !Src.isSigned()), QT: Call->getType());
2469 return true;
2470 }
2471
2472 // Vector case.
2473 const Pointer &Arg = S.Stk.pop<Pointer>();
2474 assert(Arg.getFieldDesc()->isPrimitiveArray());
2475 const Pointer &Dst = S.Stk.peek<Pointer>();
2476 assert(Dst.getFieldDesc()->isPrimitiveArray());
2477 assert(Arg.getFieldDesc()->getNumElems() ==
2478 Dst.getFieldDesc()->getNumElems());
2479
2480 QualType ElemType = Arg.getFieldDesc()->getElemQualType();
2481 PrimType ElemT = *S.getContext().classify(T: ElemType);
2482 unsigned NumElems = Arg.getNumElems();
2483 bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
2484
2485 for (unsigned I = 0; I != NumElems; ++I) {
2486 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
2487 APSInt Src = Arg.elem<T>(I).toAPSInt();
2488 APInt Result = Fn(Src);
2489 Dst.elem<T>(I) = static_cast<T>(APSInt(std::move(Result), DestUnsigned));
2490 });
2491 }
2492 Dst.initializeAllElements();
2493
2494 return true;
2495}
2496
2497static bool interp__builtin_elementwise_fp_binop(
2498 InterpState &S, CodePtr OpPC, const CallExpr *Call,
2499 llvm::function_ref<std::optional<APFloat>(
2500 const APFloat &, const APFloat &, std::optional<APSInt> RoundingMode)>
2501 Fn) {
2502 assert((Call->getNumArgs() == 2) || (Call->getNumArgs() == 3));
2503 const auto *VT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
2504 assert(VT->getElementType()->isFloatingType());
2505 unsigned NumElems = VT->getNumElements();
2506
2507 // Vector case.
2508 assert(Call->getArg(0)->getType()->isVectorType() &&
2509 Call->getArg(1)->getType()->isVectorType());
2510 assert(VT->getElementType() ==
2511 Call->getArg(1)->getType()->castAs<VectorType>()->getElementType());
2512 assert(VT->getNumElements() ==
2513 Call->getArg(1)->getType()->castAs<VectorType>()->getNumElements());
2514
2515 std::optional<APSInt> RoundingMode = std::nullopt;
2516 if (Call->getNumArgs() == 3)
2517 RoundingMode = popToAPSInt(S, E: Call->getArg(Arg: 2));
2518
2519 const Pointer &BPtr = S.Stk.pop<Pointer>();
2520 const Pointer &APtr = S.Stk.pop<Pointer>();
2521 const Pointer &Dst = S.Stk.peek<Pointer>();
2522 for (unsigned ElemIdx = 0; ElemIdx != NumElems; ++ElemIdx) {
2523 using T = PrimConv<PT_Float>::T;
2524 APFloat ElemA = APtr.elem<T>(I: ElemIdx).getAPFloat();
2525 APFloat ElemB = BPtr.elem<T>(I: ElemIdx).getAPFloat();
2526 std::optional<APFloat> Result = Fn(ElemA, ElemB, RoundingMode);
2527 if (!Result)
2528 return false;
2529 Dst.elem<T>(I: ElemIdx) = static_cast<T>(*Result);
2530 }
2531
2532 Dst.initializeAllElements();
2533
2534 return true;
2535}
2536
2537static bool interp__builtin_elementwise_int_binop(
2538 InterpState &S, CodePtr OpPC, const CallExpr *Call,
2539 llvm::function_ref<APInt(const APSInt &, const APSInt &)> Fn) {
2540 assert(Call->getNumArgs() == 2);
2541
2542 // Single integer case.
2543 if (!Call->getArg(Arg: 0)->getType()->isVectorType()) {
2544 assert(!Call->getArg(1)->getType()->isVectorType());
2545 APSInt RHS = popToAPSInt(S, E: Call->getArg(Arg: 1));
2546 APSInt LHS = popToAPSInt(S, E: Call->getArg(Arg: 0));
2547 APInt Result = Fn(LHS, RHS);
2548 pushInteger(S, Val: APSInt(std::move(Result), !LHS.isSigned()), QT: Call->getType());
2549 return true;
2550 }
2551
2552 const auto *VT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
2553 assert(VT->getElementType()->isIntegralOrEnumerationType());
2554 PrimType ElemT = *S.getContext().classify(T: VT->getElementType());
2555 unsigned NumElems = VT->getNumElements();
2556 bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
2557
2558 // Vector + Scalar case.
2559 if (!Call->getArg(Arg: 1)->getType()->isVectorType()) {
2560 assert(Call->getArg(1)->getType()->isIntegralOrEnumerationType());
2561
2562 APSInt RHS = popToAPSInt(S, E: Call->getArg(Arg: 1));
2563 const Pointer &LHS = S.Stk.pop<Pointer>();
2564 const Pointer &Dst = S.Stk.peek<Pointer>();
2565
2566 for (unsigned I = 0; I != NumElems; ++I) {
2567 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
2568 Dst.elem<T>(I) = static_cast<T>(
2569 APSInt(Fn(LHS.elem<T>(I).toAPSInt(), RHS), DestUnsigned));
2570 });
2571 }
2572 Dst.initializeAllElements();
2573 return true;
2574 }
2575
2576 // Vector case.
2577 assert(Call->getArg(0)->getType()->isVectorType() &&
2578 Call->getArg(1)->getType()->isVectorType());
2579 assert(VT->getElementType() ==
2580 Call->getArg(1)->getType()->castAs<VectorType>()->getElementType());
2581 assert(VT->getNumElements() ==
2582 Call->getArg(1)->getType()->castAs<VectorType>()->getNumElements());
2583 assert(VT->getElementType()->isIntegralOrEnumerationType());
2584
2585 const Pointer &RHS = S.Stk.pop<Pointer>();
2586 const Pointer &LHS = S.Stk.pop<Pointer>();
2587 const Pointer &Dst = S.Stk.peek<Pointer>();
2588 for (unsigned I = 0; I != NumElems; ++I) {
2589 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
2590 APSInt Elem1 = LHS.elem<T>(I).toAPSInt();
2591 APSInt Elem2 = RHS.elem<T>(I).toAPSInt();
2592 Dst.elem<T>(I) = static_cast<T>(APSInt(Fn(Elem1, Elem2), DestUnsigned));
2593 });
2594 }
2595 Dst.initializeAllElements();
2596
2597 return true;
2598}
2599
2600static bool
2601interp__builtin_x86_pack(InterpState &S, CodePtr, const CallExpr *E,
2602 llvm::function_ref<APInt(const APSInt &)> PackFn) {
2603 const auto *VT0 = E->getArg(Arg: 0)->getType()->castAs<VectorType>();
2604 [[maybe_unused]] const auto *VT1 =
2605 E->getArg(Arg: 1)->getType()->castAs<VectorType>();
2606 assert(VT0 && VT1 && "pack builtin VT0 and VT1 must be VectorType");
2607 assert(VT0->getElementType() == VT1->getElementType() &&
2608 VT0->getNumElements() == VT1->getNumElements() &&
2609 "pack builtin VT0 and VT1 ElementType must be same");
2610
2611 const Pointer &RHS = S.Stk.pop<Pointer>();
2612 const Pointer &LHS = S.Stk.pop<Pointer>();
2613 const Pointer &Dst = S.Stk.peek<Pointer>();
2614
2615 const ASTContext &ASTCtx = S.getASTContext();
2616 unsigned SrcBits = ASTCtx.getIntWidth(T: VT0->getElementType());
2617 unsigned LHSVecLen = VT0->getNumElements();
2618 unsigned SrcPerLane = 128 / SrcBits;
2619 unsigned Lanes = LHSVecLen * SrcBits / 128;
2620
2621 PrimType SrcT = *S.getContext().classify(T: VT0->getElementType());
2622 PrimType DstT = *S.getContext().classify(T: getElemType(P: Dst));
2623 bool IsUnsigend = getElemType(P: Dst)->isUnsignedIntegerType();
2624
2625 for (unsigned Lane = 0; Lane != Lanes; ++Lane) {
2626 unsigned BaseSrc = Lane * SrcPerLane;
2627 unsigned BaseDst = Lane * (2 * SrcPerLane);
2628
2629 for (unsigned I = 0; I != SrcPerLane; ++I) {
2630 INT_TYPE_SWITCH_NO_BOOL(SrcT, {
2631 APSInt A = LHS.elem<T>(BaseSrc + I).toAPSInt();
2632 APSInt B = RHS.elem<T>(BaseSrc + I).toAPSInt();
2633
2634 assignInteger(S, Dst.atIndex(BaseDst + I), DstT,
2635 APSInt(PackFn(A), IsUnsigend));
2636 assignInteger(S, Dst.atIndex(BaseDst + SrcPerLane + I), DstT,
2637 APSInt(PackFn(B), IsUnsigend));
2638 });
2639 }
2640 }
2641
2642 Dst.initializeAllElements();
2643 return true;
2644}
2645
2646static bool interp__builtin_elementwise_maxmin(InterpState &S, CodePtr OpPC,
2647 const CallExpr *Call,
2648 unsigned BuiltinID) {
2649 assert(Call->getNumArgs() == 2);
2650
2651 QualType Arg0Type = Call->getArg(Arg: 0)->getType();
2652
2653 // TODO: Support floating-point types.
2654 if (!(Arg0Type->isIntegerType() ||
2655 (Arg0Type->isVectorType() &&
2656 Arg0Type->castAs<VectorType>()->getElementType()->isIntegerType())))
2657 return false;
2658
2659 if (!Arg0Type->isVectorType()) {
2660 assert(!Call->getArg(1)->getType()->isVectorType());
2661 APSInt RHS = popToAPSInt(S, E: Call->getArg(Arg: 1));
2662 APSInt LHS = popToAPSInt(S, T: Arg0Type);
2663 APInt Result;
2664 if (BuiltinID == Builtin::BI__builtin_elementwise_max) {
2665 Result = std::max(a: LHS, b: RHS);
2666 } else if (BuiltinID == Builtin::BI__builtin_elementwise_min) {
2667 Result = std::min(a: LHS, b: RHS);
2668 } else {
2669 llvm_unreachable("Wrong builtin ID");
2670 }
2671
2672 pushInteger(S, Val: APSInt(Result, !LHS.isSigned()), QT: Call->getType());
2673 return true;
2674 }
2675
2676 // Vector case.
2677 assert(Call->getArg(0)->getType()->isVectorType() &&
2678 Call->getArg(1)->getType()->isVectorType());
2679 const auto *VT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
2680 assert(VT->getElementType() ==
2681 Call->getArg(1)->getType()->castAs<VectorType>()->getElementType());
2682 assert(VT->getNumElements() ==
2683 Call->getArg(1)->getType()->castAs<VectorType>()->getNumElements());
2684 assert(VT->getElementType()->isIntegralOrEnumerationType());
2685
2686 const Pointer &RHS = S.Stk.pop<Pointer>();
2687 const Pointer &LHS = S.Stk.pop<Pointer>();
2688 const Pointer &Dst = S.Stk.peek<Pointer>();
2689 PrimType ElemT = *S.getContext().classify(T: VT->getElementType());
2690 unsigned NumElems = VT->getNumElements();
2691 for (unsigned I = 0; I != NumElems; ++I) {
2692 APSInt Elem1;
2693 APSInt Elem2;
2694 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
2695 Elem1 = LHS.elem<T>(I).toAPSInt();
2696 Elem2 = RHS.elem<T>(I).toAPSInt();
2697 });
2698
2699 APSInt Result;
2700 if (BuiltinID == Builtin::BI__builtin_elementwise_max) {
2701 Result = APSInt(std::max(a: Elem1, b: Elem2),
2702 Call->getType()->isUnsignedIntegerOrEnumerationType());
2703 } else if (BuiltinID == Builtin::BI__builtin_elementwise_min) {
2704 Result = APSInt(std::min(a: Elem1, b: Elem2),
2705 Call->getType()->isUnsignedIntegerOrEnumerationType());
2706 } else {
2707 llvm_unreachable("Wrong builtin ID");
2708 }
2709
2710 INT_TYPE_SWITCH_NO_BOOL(ElemT,
2711 { Dst.elem<T>(I) = static_cast<T>(Result); });
2712 }
2713 Dst.initializeAllElements();
2714
2715 return true;
2716}
2717
2718static bool interp__builtin_ia32_pmul(
2719 InterpState &S, CodePtr OpPC, const CallExpr *Call,
2720 llvm::function_ref<APInt(const APSInt &, const APSInt &, const APSInt &,
2721 const APSInt &)>
2722 Fn) {
2723 assert(Call->getArg(0)->getType()->isVectorType() &&
2724 Call->getArg(1)->getType()->isVectorType());
2725 const Pointer &RHS = S.Stk.pop<Pointer>();
2726 const Pointer &LHS = S.Stk.pop<Pointer>();
2727 const Pointer &Dst = S.Stk.peek<Pointer>();
2728
2729 const auto *VT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
2730 PrimType ElemT = *S.getContext().classify(T: VT->getElementType());
2731 unsigned NumElems = VT->getNumElements();
2732 const auto *DestVT = Call->getType()->castAs<VectorType>();
2733 PrimType DestElemT = *S.getContext().classify(T: DestVT->getElementType());
2734 bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
2735
2736 unsigned DstElem = 0;
2737 for (unsigned I = 0; I != NumElems; I += 2) {
2738 APSInt Result;
2739 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
2740 APSInt LoLHS = LHS.elem<T>(I).toAPSInt();
2741 APSInt HiLHS = LHS.elem<T>(I + 1).toAPSInt();
2742 APSInt LoRHS = RHS.elem<T>(I).toAPSInt();
2743 APSInt HiRHS = RHS.elem<T>(I + 1).toAPSInt();
2744 Result = APSInt(Fn(LoLHS, HiLHS, LoRHS, HiRHS), DestUnsigned);
2745 });
2746
2747 INT_TYPE_SWITCH_NO_BOOL(DestElemT,
2748 { Dst.elem<T>(DstElem) = static_cast<T>(Result); });
2749 ++DstElem;
2750 }
2751
2752 Dst.initializeAllElements();
2753 return true;
2754}
2755
2756static bool interp_builtin_horizontal_int_binop(
2757 InterpState &S, CodePtr OpPC, const CallExpr *Call,
2758 llvm::function_ref<APInt(const APSInt &, const APSInt &)> Fn) {
2759 const auto *VT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
2760 PrimType ElemT = *S.getContext().classify(T: VT->getElementType());
2761 bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
2762
2763 const Pointer &RHS = S.Stk.pop<Pointer>();
2764 const Pointer &LHS = S.Stk.pop<Pointer>();
2765 const Pointer &Dst = S.Stk.peek<Pointer>();
2766 unsigned NumElts = VT->getNumElements();
2767 unsigned EltBits = S.getASTContext().getIntWidth(T: VT->getElementType());
2768 unsigned EltsPerLane = 128 / EltBits;
2769 unsigned Lanes = NumElts * EltBits / 128;
2770 unsigned DestIndex = 0;
2771
2772 for (unsigned Lane = 0; Lane < Lanes; ++Lane) {
2773 unsigned LaneStart = Lane * EltsPerLane;
2774 for (unsigned I = 0; I < EltsPerLane; I += 2) {
2775 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
2776 APSInt Elem1 = LHS.elem<T>(LaneStart + I).toAPSInt();
2777 APSInt Elem2 = LHS.elem<T>(LaneStart + I + 1).toAPSInt();
2778 APSInt ResL = APSInt(Fn(Elem1, Elem2), DestUnsigned);
2779 Dst.elem<T>(DestIndex++) = static_cast<T>(ResL);
2780 });
2781 }
2782
2783 for (unsigned I = 0; I < EltsPerLane; I += 2) {
2784 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
2785 APSInt Elem1 = RHS.elem<T>(LaneStart + I).toAPSInt();
2786 APSInt Elem2 = RHS.elem<T>(LaneStart + I + 1).toAPSInt();
2787 APSInt ResR = APSInt(Fn(Elem1, Elem2), DestUnsigned);
2788 Dst.elem<T>(DestIndex++) = static_cast<T>(ResR);
2789 });
2790 }
2791 }
2792 Dst.initializeAllElements();
2793 return true;
2794}
2795
2796static bool interp_builtin_horizontal_fp_binop(
2797 InterpState &S, CodePtr OpPC, const CallExpr *Call,
2798 llvm::function_ref<APFloat(const APFloat &, const APFloat &,
2799 llvm::RoundingMode)>
2800 Fn) {
2801 const Pointer &RHS = S.Stk.pop<Pointer>();
2802 const Pointer &LHS = S.Stk.pop<Pointer>();
2803 const Pointer &Dst = S.Stk.peek<Pointer>();
2804 FPOptions FPO = Call->getFPFeaturesInEffect(LO: S.Ctx.getLangOpts());
2805 llvm::RoundingMode RM = getRoundingMode(FPO);
2806 const auto *VT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
2807
2808 unsigned NumElts = VT->getNumElements();
2809 unsigned EltBits = S.getASTContext().getTypeSize(T: VT->getElementType());
2810 unsigned NumLanes = NumElts * EltBits / 128;
2811 unsigned NumElemsPerLane = NumElts / NumLanes;
2812 unsigned HalfElemsPerLane = NumElemsPerLane / 2;
2813
2814 for (unsigned L = 0; L != NumElts; L += NumElemsPerLane) {
2815 using T = PrimConv<PT_Float>::T;
2816 for (unsigned E = 0; E != HalfElemsPerLane; ++E) {
2817 APFloat Elem1 = LHS.elem<T>(I: L + (2 * E) + 0).getAPFloat();
2818 APFloat Elem2 = LHS.elem<T>(I: L + (2 * E) + 1).getAPFloat();
2819 Dst.elem<T>(I: L + E) = static_cast<T>(Fn(Elem1, Elem2, RM));
2820 }
2821 for (unsigned E = 0; E != HalfElemsPerLane; ++E) {
2822 APFloat Elem1 = RHS.elem<T>(I: L + (2 * E) + 0).getAPFloat();
2823 APFloat Elem2 = RHS.elem<T>(I: L + (2 * E) + 1).getAPFloat();
2824 Dst.elem<T>(I: L + E + HalfElemsPerLane) =
2825 static_cast<T>(Fn(Elem1, Elem2, RM));
2826 }
2827 }
2828 Dst.initializeAllElements();
2829 return true;
2830}
2831
2832static bool interp__builtin_ia32_addsub(InterpState &S, CodePtr OpPC,
2833 const CallExpr *Call) {
2834 // Addsub: alternates between subtraction and addition
2835 // Result[i] = (i % 2 == 0) ? (a[i] - b[i]) : (a[i] + b[i])
2836 const Pointer &RHS = S.Stk.pop<Pointer>();
2837 const Pointer &LHS = S.Stk.pop<Pointer>();
2838 const Pointer &Dst = S.Stk.peek<Pointer>();
2839 FPOptions FPO = Call->getFPFeaturesInEffect(LO: S.Ctx.getLangOpts());
2840 llvm::RoundingMode RM = getRoundingMode(FPO);
2841 const auto *VT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
2842 unsigned NumElems = VT->getNumElements();
2843
2844 using T = PrimConv<PT_Float>::T;
2845 for (unsigned I = 0; I != NumElems; ++I) {
2846 APFloat LElem = LHS.elem<T>(I).getAPFloat();
2847 APFloat RElem = RHS.elem<T>(I).getAPFloat();
2848 if (I % 2 == 0) {
2849 // Even indices: subtract
2850 LElem.subtract(RHS: RElem, RM);
2851 } else {
2852 // Odd indices: add
2853 LElem.add(RHS: RElem, RM);
2854 }
2855 Dst.elem<T>(I) = static_cast<T>(LElem);
2856 }
2857 Dst.initializeAllElements();
2858 return true;
2859}
2860
2861static bool interp__builtin_ia32_pclmulqdq(InterpState &S, CodePtr OpPC,
2862 const CallExpr *Call) {
2863 // PCLMULQDQ: carry-less multiplication of selected 64-bit halves
2864 // imm8 bit 0: selects lower (0) or upper (1) 64 bits of first operand
2865 // imm8 bit 4: selects lower (0) or upper (1) 64 bits of second operand
2866 assert(Call->getArg(0)->getType()->isVectorType() &&
2867 Call->getArg(1)->getType()->isVectorType());
2868
2869 // Extract imm8 argument
2870 APSInt Imm8 = popToAPSInt(S, E: Call->getArg(Arg: 2));
2871 bool SelectUpperA = (Imm8 & 0x01) != 0;
2872 bool SelectUpperB = (Imm8 & 0x10) != 0;
2873
2874 const Pointer &RHS = S.Stk.pop<Pointer>();
2875 const Pointer &LHS = S.Stk.pop<Pointer>();
2876 const Pointer &Dst = S.Stk.peek<Pointer>();
2877
2878 const auto *VT = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
2879 PrimType ElemT = *S.getContext().classify(T: VT->getElementType());
2880 unsigned NumElems = VT->getNumElements();
2881 const auto *DestVT = Call->getType()->castAs<VectorType>();
2882 PrimType DestElemT = *S.getContext().classify(T: DestVT->getElementType());
2883 bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
2884
2885 // Process each 128-bit lane (2 elements at a time)
2886 for (unsigned Lane = 0; Lane < NumElems; Lane += 2) {
2887 APSInt A0, A1, B0, B1;
2888 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
2889 A0 = LHS.elem<T>(Lane + 0).toAPSInt();
2890 A1 = LHS.elem<T>(Lane + 1).toAPSInt();
2891 B0 = RHS.elem<T>(Lane + 0).toAPSInt();
2892 B1 = RHS.elem<T>(Lane + 1).toAPSInt();
2893 });
2894
2895 // Select the appropriate 64-bit values based on imm8
2896 APInt A = SelectUpperA ? A1 : A0;
2897 APInt B = SelectUpperB ? B1 : B0;
2898
2899 // Extend both operands to 128 bits for carry-less multiplication
2900 APInt A128 = A.zext(width: 128);
2901 APInt B128 = B.zext(width: 128);
2902
2903 // Use APIntOps::clmul for carry-less multiplication
2904 APInt Result = llvm::APIntOps::clmul(LHS: A128, RHS: B128);
2905
2906 // Split the 128-bit result into two 64-bit halves
2907 APSInt ResultLow(Result.extractBits(numBits: 64, bitPosition: 0), DestUnsigned);
2908 APSInt ResultHigh(Result.extractBits(numBits: 64, bitPosition: 64), DestUnsigned);
2909
2910 INT_TYPE_SWITCH_NO_BOOL(DestElemT, {
2911 Dst.elem<T>(Lane + 0) = static_cast<T>(ResultLow);
2912 Dst.elem<T>(Lane + 1) = static_cast<T>(ResultHigh);
2913 });
2914 }
2915
2916 Dst.initializeAllElements();
2917 return true;
2918}
2919
2920static bool interp__builtin_elementwise_triop_fp(
2921 InterpState &S, CodePtr OpPC, const CallExpr *Call,
2922 llvm::function_ref<APFloat(const APFloat &, const APFloat &,
2923 const APFloat &, llvm::RoundingMode)>
2924 Fn) {
2925 assert(Call->getNumArgs() == 3);
2926
2927 FPOptions FPO = Call->getFPFeaturesInEffect(LO: S.Ctx.getLangOpts());
2928 llvm::RoundingMode RM = getRoundingMode(FPO);
2929 QualType Arg1Type = Call->getArg(Arg: 0)->getType();
2930 QualType Arg2Type = Call->getArg(Arg: 1)->getType();
2931 QualType Arg3Type = Call->getArg(Arg: 2)->getType();
2932
2933 // Non-vector floating point types.
2934 if (!Arg1Type->isVectorType()) {
2935 assert(!Arg2Type->isVectorType());
2936 assert(!Arg3Type->isVectorType());
2937 (void)Arg2Type;
2938 (void)Arg3Type;
2939
2940 const Floating &Z = S.Stk.pop<Floating>();
2941 const Floating &Y = S.Stk.pop<Floating>();
2942 const Floating &X = S.Stk.pop<Floating>();
2943 APFloat F = Fn(X.getAPFloat(), Y.getAPFloat(), Z.getAPFloat(), RM);
2944 Floating Result = S.allocFloat(Sem: X.getSemantics());
2945 Result.copy(F);
2946 S.Stk.push<Floating>(Args&: Result);
2947 return true;
2948 }
2949
2950 // Vector type.
2951 assert(Arg1Type->isVectorType() && Arg2Type->isVectorType() &&
2952 Arg3Type->isVectorType());
2953
2954 const VectorType *VecTy = Arg1Type->castAs<VectorType>();
2955 QualType ElemQT = VecTy->getElementType();
2956 unsigned NumElems = VecTy->getNumElements();
2957
2958 assert(ElemQT == Arg2Type->castAs<VectorType>()->getElementType() &&
2959 ElemQT == Arg3Type->castAs<VectorType>()->getElementType());
2960 assert(NumElems == Arg2Type->castAs<VectorType>()->getNumElements() &&
2961 NumElems == Arg3Type->castAs<VectorType>()->getNumElements());
2962 assert(ElemQT->isRealFloatingType());
2963 (void)ElemQT;
2964
2965 const Pointer &VZ = S.Stk.pop<Pointer>();
2966 const Pointer &VY = S.Stk.pop<Pointer>();
2967 const Pointer &VX = S.Stk.pop<Pointer>();
2968 const Pointer &Dst = S.Stk.peek<Pointer>();
2969 for (unsigned I = 0; I != NumElems; ++I) {
2970 using T = PrimConv<PT_Float>::T;
2971 APFloat X = VX.elem<T>(I).getAPFloat();
2972 APFloat Y = VY.elem<T>(I).getAPFloat();
2973 APFloat Z = VZ.elem<T>(I).getAPFloat();
2974 APFloat F = Fn(X, Y, Z, RM);
2975 Dst.elem<Floating>(I) = Floating(F);
2976 }
2977 Dst.initializeAllElements();
2978 return true;
2979}
2980
2981/// AVX512 predicated move: "Result = Mask[] ? LHS[] : RHS[]".
2982static bool interp__builtin_select(InterpState &S, CodePtr OpPC,
2983 const CallExpr *Call) {
2984 const Pointer &RHS = S.Stk.pop<Pointer>();
2985 const Pointer &LHS = S.Stk.pop<Pointer>();
2986 APSInt Mask = popToAPSInt(S, E: Call->getArg(Arg: 0));
2987 const Pointer &Dst = S.Stk.peek<Pointer>();
2988
2989 assert(LHS.getNumElems() == RHS.getNumElems());
2990 assert(LHS.getNumElems() == Dst.getNumElems());
2991 unsigned NumElems = LHS.getNumElems();
2992 PrimType ElemT = LHS.getFieldDesc()->getPrimType();
2993 PrimType DstElemT = Dst.getFieldDesc()->getPrimType();
2994
2995 for (unsigned I = 0; I != NumElems; ++I) {
2996 if (ElemT == PT_Float) {
2997 assert(DstElemT == PT_Float);
2998 Dst.elem<Floating>(I) =
2999 Mask[I] ? LHS.elem<Floating>(I) : RHS.elem<Floating>(I);
3000 } else {
3001 APSInt Elem;
3002 INT_TYPE_SWITCH(ElemT, {
3003 Elem = Mask[I] ? LHS.elem<T>(I).toAPSInt() : RHS.elem<T>(I).toAPSInt();
3004 });
3005 INT_TYPE_SWITCH_NO_BOOL(DstElemT,
3006 { Dst.elem<T>(I) = static_cast<T>(Elem); });
3007 }
3008 }
3009 Dst.initializeAllElements();
3010
3011 return true;
3012}
3013
3014/// Scalar variant of AVX512 predicated select:
3015/// Result[i] = (Mask bit 0) ? LHS[i] : RHS[i], but only element 0 may change.
3016/// All other elements are taken from RHS.
3017static bool interp__builtin_select_scalar(InterpState &S,
3018 const CallExpr *Call) {
3019 unsigned N =
3020 Call->getArg(Arg: 1)->getType()->castAs<VectorType>()->getNumElements();
3021
3022 const Pointer &W = S.Stk.pop<Pointer>();
3023 const Pointer &A = S.Stk.pop<Pointer>();
3024 APSInt U = popToAPSInt(S, E: Call->getArg(Arg: 0));
3025 const Pointer &Dst = S.Stk.peek<Pointer>();
3026
3027 bool TakeA0 = U.getZExtValue() & 1ULL;
3028
3029 for (unsigned I = TakeA0; I != N; ++I)
3030 Dst.elem<Floating>(I) = W.elem<Floating>(I);
3031 if (TakeA0)
3032 Dst.elem<Floating>(I: 0) = A.elem<Floating>(I: 0);
3033
3034 Dst.initializeAllElements();
3035 return true;
3036}
3037
3038static bool interp__builtin_ia32_test_op(
3039 InterpState &S, CodePtr OpPC, const CallExpr *Call,
3040 llvm::function_ref<bool(const APInt &A, const APInt &B)> Fn) {
3041 const Pointer &RHS = S.Stk.pop<Pointer>();
3042 const Pointer &LHS = S.Stk.pop<Pointer>();
3043
3044 assert(LHS.getNumElems() == RHS.getNumElems());
3045
3046 unsigned SourceLen = LHS.getNumElems();
3047 QualType ElemQT = getElemType(P: LHS);
3048 OptPrimType ElemPT = S.getContext().classify(T: ElemQT);
3049 unsigned LaneWidth = S.getASTContext().getTypeSize(T: ElemQT);
3050
3051 APInt AWide(LaneWidth * SourceLen, 0);
3052 APInt BWide(LaneWidth * SourceLen, 0);
3053
3054 for (unsigned I = 0; I != SourceLen; ++I) {
3055 APInt ALane;
3056 APInt BLane;
3057
3058 if (ElemQT->isIntegerType()) { // Get value.
3059 INT_TYPE_SWITCH_NO_BOOL(*ElemPT, {
3060 ALane = LHS.elem<T>(I).toAPSInt();
3061 BLane = RHS.elem<T>(I).toAPSInt();
3062 });
3063 } else if (ElemQT->isFloatingType()) { // Get only sign bit.
3064 using T = PrimConv<PT_Float>::T;
3065 ALane = LHS.elem<T>(I).getAPFloat().bitcastToAPInt().isNegative();
3066 BLane = RHS.elem<T>(I).getAPFloat().bitcastToAPInt().isNegative();
3067 } else { // Must be integer or floating type.
3068 return false;
3069 }
3070 AWide.insertBits(SubBits: ALane, bitPosition: I * LaneWidth);
3071 BWide.insertBits(SubBits: BLane, bitPosition: I * LaneWidth);
3072 }
3073 pushInteger(S, Val: Fn(AWide, BWide), QT: Call->getType());
3074 return true;
3075}
3076
3077static bool interp__builtin_ia32_movmsk_op(InterpState &S, CodePtr OpPC,
3078 const CallExpr *Call) {
3079 assert(Call->getNumArgs() == 1);
3080
3081 const Pointer &Source = S.Stk.pop<Pointer>();
3082
3083 unsigned SourceLen = Source.getNumElems();
3084 QualType ElemQT = getElemType(P: Source);
3085 OptPrimType ElemT = S.getContext().classify(T: ElemQT);
3086 unsigned ResultLen =
3087 S.getASTContext().getTypeSize(T: Call->getType()); // Always 32-bit integer.
3088 APInt Result(ResultLen, 0);
3089
3090 for (unsigned I = 0; I != SourceLen; ++I) {
3091 APInt Elem;
3092 if (ElemQT->isIntegerType()) {
3093 INT_TYPE_SWITCH_NO_BOOL(*ElemT, { Elem = Source.elem<T>(I).toAPSInt(); });
3094 } else if (ElemQT->isRealFloatingType()) {
3095 using T = PrimConv<PT_Float>::T;
3096 Elem = Source.elem<T>(I).getAPFloat().bitcastToAPInt();
3097 } else {
3098 return false;
3099 }
3100 Result.setBitVal(BitPosition: I, BitValue: Elem.isNegative());
3101 }
3102 pushInteger(S, Val: Result, QT: Call->getType());
3103 return true;
3104}
3105
3106static bool interp__builtin_elementwise_triop(
3107 InterpState &S, CodePtr OpPC, const CallExpr *Call,
3108 llvm::function_ref<APInt(const APSInt &, const APSInt &, const APSInt &)>
3109 Fn) {
3110 assert(Call->getNumArgs() == 3);
3111
3112 QualType Arg0Type = Call->getArg(Arg: 0)->getType();
3113 QualType Arg2Type = Call->getArg(Arg: 2)->getType();
3114 // Non-vector integer types.
3115 if (!Arg0Type->isVectorType()) {
3116 const APSInt &Op2 = popToAPSInt(S, T: Arg2Type);
3117 const APSInt &Op1 = popToAPSInt(S, E: Call->getArg(Arg: 1));
3118 const APSInt &Op0 = popToAPSInt(S, T: Arg0Type);
3119 APSInt Result = APSInt(Fn(Op0, Op1, Op2), Op0.isUnsigned());
3120 pushInteger(S, Val: Result, QT: Call->getType());
3121 return true;
3122 }
3123
3124 const auto *VecT = Arg0Type->castAs<VectorType>();
3125 PrimType ElemT = *S.getContext().classify(T: VecT->getElementType());
3126 unsigned NumElems = VecT->getNumElements();
3127 bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
3128
3129 // Vector + Vector + Scalar case.
3130 if (!Arg2Type->isVectorType()) {
3131 APSInt Op2 = popToAPSInt(S, T: Arg2Type);
3132
3133 const Pointer &Op1 = S.Stk.pop<Pointer>();
3134 const Pointer &Op0 = S.Stk.pop<Pointer>();
3135 const Pointer &Dst = S.Stk.peek<Pointer>();
3136 for (unsigned I = 0; I != NumElems; ++I) {
3137 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
3138 Dst.elem<T>(I) = static_cast<T>(APSInt(
3139 Fn(Op0.elem<T>(I).toAPSInt(), Op1.elem<T>(I).toAPSInt(), Op2),
3140 DestUnsigned));
3141 });
3142 }
3143 Dst.initializeAllElements();
3144
3145 return true;
3146 }
3147
3148 // Vector type.
3149 const Pointer &Op2 = S.Stk.pop<Pointer>();
3150 const Pointer &Op1 = S.Stk.pop<Pointer>();
3151 const Pointer &Op0 = S.Stk.pop<Pointer>();
3152 const Pointer &Dst = S.Stk.peek<Pointer>();
3153 for (unsigned I = 0; I != NumElems; ++I) {
3154 APSInt Val0, Val1, Val2;
3155 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
3156 Val0 = Op0.elem<T>(I).toAPSInt();
3157 Val1 = Op1.elem<T>(I).toAPSInt();
3158 Val2 = Op2.elem<T>(I).toAPSInt();
3159 });
3160 APSInt Result = APSInt(Fn(Val0, Val1, Val2), Val0.isUnsigned());
3161 INT_TYPE_SWITCH_NO_BOOL(ElemT,
3162 { Dst.elem<T>(I) = static_cast<T>(Result); });
3163 }
3164 Dst.initializeAllElements();
3165
3166 return true;
3167}
3168
3169static bool interp__builtin_x86_extract_vector(InterpState &S, CodePtr OpPC,
3170 const CallExpr *Call,
3171 unsigned ID) {
3172 assert(Call->getNumArgs() == 2);
3173
3174 APSInt ImmAPS = popToAPSInt(S, E: Call->getArg(Arg: 1));
3175 uint64_t Index = ImmAPS.getZExtValue();
3176
3177 const Pointer &Src = S.Stk.pop<Pointer>();
3178 if (!Src.getFieldDesc()->isPrimitiveArray())
3179 return false;
3180
3181 const Pointer &Dst = S.Stk.peek<Pointer>();
3182 if (!Dst.getFieldDesc()->isPrimitiveArray())
3183 return false;
3184
3185 unsigned SrcElems = Src.getNumElems();
3186 unsigned DstElems = Dst.getNumElems();
3187
3188 unsigned NumLanes = SrcElems / DstElems;
3189 unsigned Lane = static_cast<unsigned>(Index % NumLanes);
3190 unsigned ExtractPos = Lane * DstElems;
3191
3192 PrimType ElemT = Src.getFieldDesc()->getPrimType();
3193
3194 TYPE_SWITCH(ElemT, {
3195 for (unsigned I = 0; I != DstElems; ++I) {
3196 Dst.elem<T>(I) = Src.elem<T>(ExtractPos + I);
3197 }
3198 });
3199
3200 Dst.initializeAllElements();
3201 return true;
3202}
3203
3204static bool interp__builtin_x86_extract_vector_masked(InterpState &S,
3205 CodePtr OpPC,
3206 const CallExpr *Call,
3207 unsigned ID) {
3208 assert(Call->getNumArgs() == 4);
3209
3210 APSInt MaskAPS = popToAPSInt(S, E: Call->getArg(Arg: 3));
3211 const Pointer &Merge = S.Stk.pop<Pointer>();
3212 APSInt ImmAPS = popToAPSInt(S, E: Call->getArg(Arg: 1));
3213 const Pointer &Src = S.Stk.pop<Pointer>();
3214
3215 if (!Src.getFieldDesc()->isPrimitiveArray() ||
3216 !Merge.getFieldDesc()->isPrimitiveArray())
3217 return false;
3218
3219 const Pointer &Dst = S.Stk.peek<Pointer>();
3220 if (!Dst.getFieldDesc()->isPrimitiveArray())
3221 return false;
3222
3223 unsigned SrcElems = Src.getNumElems();
3224 unsigned DstElems = Dst.getNumElems();
3225
3226 unsigned NumLanes = SrcElems / DstElems;
3227 unsigned Lane = static_cast<unsigned>(ImmAPS.getZExtValue() % NumLanes);
3228 unsigned Base = Lane * DstElems;
3229
3230 PrimType ElemT = Src.getFieldDesc()->getPrimType();
3231
3232 TYPE_SWITCH(ElemT, {
3233 for (unsigned I = 0; I != DstElems; ++I) {
3234 if (MaskAPS[I])
3235 Dst.elem<T>(I) = Src.elem<T>(Base + I);
3236 else
3237 Dst.elem<T>(I) = Merge.elem<T>(I);
3238 }
3239 });
3240
3241 Dst.initializeAllElements();
3242 return true;
3243}
3244
3245static bool interp__builtin_x86_insert_subvector(InterpState &S, CodePtr OpPC,
3246 const CallExpr *Call,
3247 unsigned ID) {
3248 assert(Call->getNumArgs() == 3);
3249
3250 APSInt ImmAPS = popToAPSInt(S, E: Call->getArg(Arg: 2));
3251 uint64_t Index = ImmAPS.getZExtValue();
3252
3253 const Pointer &SubVec = S.Stk.pop<Pointer>();
3254 if (!SubVec.getFieldDesc()->isPrimitiveArray())
3255 return false;
3256
3257 const Pointer &BaseVec = S.Stk.pop<Pointer>();
3258 if (!BaseVec.getFieldDesc()->isPrimitiveArray())
3259 return false;
3260
3261 const Pointer &Dst = S.Stk.peek<Pointer>();
3262
3263 unsigned BaseElements = BaseVec.getNumElems();
3264 unsigned SubElements = SubVec.getNumElems();
3265
3266 assert(SubElements != 0 && BaseElements != 0 &&
3267 (BaseElements % SubElements) == 0);
3268
3269 unsigned NumLanes = BaseElements / SubElements;
3270 unsigned Lane = static_cast<unsigned>(Index % NumLanes);
3271 unsigned InsertPos = Lane * SubElements;
3272
3273 PrimType ElemT = BaseVec.getFieldDesc()->getPrimType();
3274
3275 TYPE_SWITCH(ElemT, {
3276 for (unsigned I = 0; I != BaseElements; ++I)
3277 Dst.elem<T>(I) = BaseVec.elem<T>(I);
3278 for (unsigned I = 0; I != SubElements; ++I)
3279 Dst.elem<T>(InsertPos + I) = SubVec.elem<T>(I);
3280 });
3281
3282 Dst.initializeAllElements();
3283 return true;
3284}
3285
3286static bool interp__builtin_ia32_phminposuw(InterpState &S, CodePtr OpPC,
3287 const CallExpr *Call) {
3288 assert(Call->getNumArgs() == 1);
3289
3290 const Pointer &Source = S.Stk.pop<Pointer>();
3291 const Pointer &Dest = S.Stk.peek<Pointer>();
3292
3293 unsigned SourceLen = Source.getNumElems();
3294 QualType ElemQT = getElemType(P: Source);
3295 OptPrimType ElemT = S.getContext().classify(T: ElemQT);
3296 unsigned ElemBitWidth = S.getASTContext().getTypeSize(T: ElemQT);
3297
3298 bool DestUnsigned = Call->getCallReturnType(Ctx: S.getASTContext())
3299 ->castAs<VectorType>()
3300 ->getElementType()
3301 ->isUnsignedIntegerOrEnumerationType();
3302
3303 INT_TYPE_SWITCH_NO_BOOL(*ElemT, {
3304 APSInt MinIndex(ElemBitWidth, DestUnsigned);
3305 APSInt MinVal = Source.elem<T>(0).toAPSInt();
3306
3307 for (unsigned I = 1; I != SourceLen; ++I) {
3308 APSInt Val = Source.elem<T>(I).toAPSInt();
3309 if (MinVal.ugt(Val)) {
3310 MinVal = Val;
3311 MinIndex = I;
3312 }
3313 }
3314
3315 Dest.elem<T>(0) = static_cast<T>(MinVal);
3316 Dest.elem<T>(1) = static_cast<T>(MinIndex);
3317 for (unsigned I = 2; I != SourceLen; ++I) {
3318 Dest.elem<T>(I) = static_cast<T>(APSInt(ElemBitWidth, DestUnsigned));
3319 }
3320 });
3321 Dest.initializeAllElements();
3322 return true;
3323}
3324
3325static bool interp__builtin_ia32_pternlog(InterpState &S, CodePtr OpPC,
3326 const CallExpr *Call, bool MaskZ) {
3327 assert(Call->getNumArgs() == 5);
3328
3329 APInt U = popToAPSInt(S, E: Call->getArg(Arg: 4)); // Lane mask
3330 APInt Imm = popToAPSInt(S, E: Call->getArg(Arg: 3)); // Ternary truth table
3331 const Pointer &C = S.Stk.pop<Pointer>();
3332 const Pointer &B = S.Stk.pop<Pointer>();
3333 const Pointer &A = S.Stk.pop<Pointer>();
3334 const Pointer &Dst = S.Stk.peek<Pointer>();
3335
3336 unsigned DstLen = A.getNumElems();
3337 QualType ElemQT = getElemType(P: A);
3338 OptPrimType ElemT = S.getContext().classify(T: ElemQT);
3339 unsigned LaneWidth = S.getASTContext().getTypeSize(T: ElemQT);
3340 bool DstUnsigned = ElemQT->isUnsignedIntegerOrEnumerationType();
3341
3342 INT_TYPE_SWITCH_NO_BOOL(*ElemT, {
3343 for (unsigned I = 0; I != DstLen; ++I) {
3344 APInt ALane = A.elem<T>(I).toAPSInt();
3345 APInt BLane = B.elem<T>(I).toAPSInt();
3346 APInt CLane = C.elem<T>(I).toAPSInt();
3347 APInt RLane(LaneWidth, 0);
3348 if (U[I]) { // If lane not masked, compute ternary logic.
3349 for (unsigned Bit = 0; Bit != LaneWidth; ++Bit) {
3350 unsigned ABit = ALane[Bit];
3351 unsigned BBit = BLane[Bit];
3352 unsigned CBit = CLane[Bit];
3353 unsigned Idx = (ABit << 2) | (BBit << 1) | (CBit);
3354 RLane.setBitVal(Bit, Imm[Idx]);
3355 }
3356 Dst.elem<T>(I) = static_cast<T>(APSInt(RLane, DstUnsigned));
3357 } else if (MaskZ) { // If zero masked, zero the lane.
3358 Dst.elem<T>(I) = static_cast<T>(APSInt(RLane, DstUnsigned));
3359 } else { // Just masked, put in A lane.
3360 Dst.elem<T>(I) = static_cast<T>(APSInt(ALane, DstUnsigned));
3361 }
3362 }
3363 });
3364 Dst.initializeAllElements();
3365 return true;
3366}
3367
3368static bool interp__builtin_vec_ext(InterpState &S, CodePtr OpPC,
3369 const CallExpr *Call, unsigned ID) {
3370 assert(Call->getNumArgs() == 2);
3371
3372 APSInt ImmAPS = popToAPSInt(S, E: Call->getArg(Arg: 1));
3373 const Pointer &Vec = S.Stk.pop<Pointer>();
3374 if (!Vec.getFieldDesc()->isPrimitiveArray())
3375 return false;
3376
3377 unsigned NumElems = Vec.getNumElems();
3378 unsigned Index =
3379 static_cast<unsigned>(ImmAPS.getZExtValue() & (NumElems - 1));
3380
3381 PrimType ElemT = Vec.getFieldDesc()->getPrimType();
3382 // FIXME(#161685): Replace float+int split with a numeric-only type switch
3383 if (ElemT == PT_Float) {
3384 S.Stk.push<Floating>(Args&: Vec.elem<Floating>(I: Index));
3385 return true;
3386 }
3387 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
3388 APSInt V = Vec.elem<T>(Index).toAPSInt();
3389 pushInteger(S, V, Call->getType());
3390 });
3391
3392 return true;
3393}
3394
3395static bool interp__builtin_vec_set(InterpState &S, CodePtr OpPC,
3396 const CallExpr *Call, unsigned ID) {
3397 assert(Call->getNumArgs() == 3);
3398
3399 APSInt ImmAPS = popToAPSInt(S, E: Call->getArg(Arg: 2));
3400 APSInt ValAPS = popToAPSInt(S, E: Call->getArg(Arg: 1));
3401
3402 const Pointer &Base = S.Stk.pop<Pointer>();
3403 if (!Base.getFieldDesc()->isPrimitiveArray())
3404 return false;
3405
3406 const Pointer &Dst = S.Stk.peek<Pointer>();
3407
3408 unsigned NumElems = Base.getNumElems();
3409 unsigned Index =
3410 static_cast<unsigned>(ImmAPS.getZExtValue() & (NumElems - 1));
3411
3412 PrimType ElemT = Base.getFieldDesc()->getPrimType();
3413 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
3414 for (unsigned I = 0; I != NumElems; ++I)
3415 Dst.elem<T>(I) = Base.elem<T>(I);
3416 Dst.elem<T>(Index) = static_cast<T>(ValAPS);
3417 });
3418
3419 Dst.initializeAllElements();
3420 return true;
3421}
3422
3423static bool evalICmpImm(uint8_t Imm, const APSInt &A, const APSInt &B,
3424 bool IsUnsigned) {
3425 switch (Imm & 0x7) {
3426 case 0x00: // _MM_CMPINT_EQ
3427 return (A == B);
3428 case 0x01: // _MM_CMPINT_LT
3429 return IsUnsigned ? A.ult(RHS: B) : A.slt(RHS: B);
3430 case 0x02: // _MM_CMPINT_LE
3431 return IsUnsigned ? A.ule(RHS: B) : A.sle(RHS: B);
3432 case 0x03: // _MM_CMPINT_FALSE
3433 return false;
3434 case 0x04: // _MM_CMPINT_NE
3435 return (A != B);
3436 case 0x05: // _MM_CMPINT_NLT
3437 return IsUnsigned ? A.ugt(RHS: B) : A.sgt(RHS: B);
3438 case 0x06: // _MM_CMPINT_NLE
3439 return IsUnsigned ? A.uge(RHS: B) : A.sge(RHS: B);
3440 case 0x07: // _MM_CMPINT_TRUE
3441 return true;
3442 default:
3443 llvm_unreachable("Invalid Op");
3444 }
3445}
3446
3447static bool interp__builtin_ia32_cmp_mask(InterpState &S, CodePtr OpPC,
3448 const CallExpr *Call, unsigned ID,
3449 bool IsUnsigned) {
3450 assert(Call->getNumArgs() == 4);
3451
3452 APSInt Mask = popToAPSInt(S, E: Call->getArg(Arg: 3));
3453 APSInt Opcode = popToAPSInt(S, E: Call->getArg(Arg: 2));
3454 unsigned CmpOp = static_cast<unsigned>(Opcode.getZExtValue());
3455 const Pointer &RHS = S.Stk.pop<Pointer>();
3456 const Pointer &LHS = S.Stk.pop<Pointer>();
3457
3458 assert(LHS.getNumElems() == RHS.getNumElems());
3459
3460 APInt RetMask = APInt::getZero(numBits: LHS.getNumElems());
3461 unsigned VectorLen = LHS.getNumElems();
3462 PrimType ElemT = LHS.getFieldDesc()->getPrimType();
3463
3464 for (unsigned ElemNum = 0; ElemNum < VectorLen; ++ElemNum) {
3465 APSInt A, B;
3466 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
3467 A = LHS.elem<T>(ElemNum).toAPSInt();
3468 B = RHS.elem<T>(ElemNum).toAPSInt();
3469 });
3470 RetMask.setBitVal(BitPosition: ElemNum,
3471 BitValue: Mask[ElemNum] && evalICmpImm(Imm: CmpOp, A, B, IsUnsigned));
3472 }
3473 pushInteger(S, Val: RetMask, QT: Call->getType());
3474 return true;
3475}
3476
3477static bool interp__builtin_ia32_vpconflict(InterpState &S, CodePtr OpPC,
3478 const CallExpr *Call) {
3479 assert(Call->getNumArgs() == 1);
3480
3481 QualType Arg0Type = Call->getArg(Arg: 0)->getType();
3482 const auto *VecT = Arg0Type->castAs<VectorType>();
3483 PrimType ElemT = *S.getContext().classify(T: VecT->getElementType());
3484 unsigned NumElems = VecT->getNumElements();
3485 bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
3486 const Pointer &Src = S.Stk.pop<Pointer>();
3487 const Pointer &Dst = S.Stk.peek<Pointer>();
3488
3489 for (unsigned I = 0; I != NumElems; ++I) {
3490 INT_TYPE_SWITCH_NO_BOOL(ElemT, {
3491 APSInt ElemI = Src.elem<T>(I).toAPSInt();
3492 APInt ConflictMask(ElemI.getBitWidth(), 0);
3493 for (unsigned J = 0; J != I; ++J) {
3494 APSInt ElemJ = Src.elem<T>(J).toAPSInt();
3495 ConflictMask.setBitVal(J, ElemI == ElemJ);
3496 }
3497 Dst.elem<T>(I) = static_cast<T>(APSInt(ConflictMask, DestUnsigned));
3498 });
3499 }
3500 Dst.initializeAllElements();
3501 return true;
3502}
3503
3504static bool interp__builtin_ia32_cvt_vec2mask(InterpState &S, CodePtr OpPC,
3505 const CallExpr *Call,
3506 unsigned ID) {
3507 assert(Call->getNumArgs() == 1);
3508
3509 const Pointer &Vec = S.Stk.pop<Pointer>();
3510 unsigned RetWidth = S.getASTContext().getIntWidth(T: Call->getType());
3511 APInt RetMask(RetWidth, 0);
3512
3513 unsigned VectorLen = Vec.getNumElems();
3514 PrimType ElemT = Vec.getFieldDesc()->getPrimType();
3515
3516 for (unsigned ElemNum = 0; ElemNum != VectorLen; ++ElemNum) {
3517 APSInt A;
3518 INT_TYPE_SWITCH_NO_BOOL(ElemT, { A = Vec.elem<T>(ElemNum).toAPSInt(); });
3519 unsigned MSB = A[A.getBitWidth() - 1];
3520 RetMask.setBitVal(BitPosition: ElemNum, BitValue: MSB);
3521 }
3522 pushInteger(S, Val: RetMask, QT: Call->getType());
3523 return true;
3524}
3525
3526static bool interp__builtin_ia32_cvt_mask2vec(InterpState &S, CodePtr OpPC,
3527 const CallExpr *Call,
3528 unsigned ID) {
3529 assert(Call->getNumArgs() == 1);
3530
3531 APSInt Mask = popToAPSInt(S, E: Call->getArg(Arg: 0));
3532
3533 const Pointer &Vec = S.Stk.peek<Pointer>();
3534 unsigned NumElems = Vec.getNumElems();
3535 PrimType ElemT = Vec.getFieldDesc()->getPrimType();
3536
3537 for (unsigned I = 0; I != NumElems; ++I) {
3538 bool BitSet = Mask[I];
3539
3540 INT_TYPE_SWITCH_NO_BOOL(
3541 ElemT, { Vec.elem<T>(I) = BitSet ? T::from(-1) : T::from(0); });
3542 }
3543
3544 Vec.initializeAllElements();
3545
3546 return true;
3547}
3548
3549static bool interp__builtin_ia32_cvtsd2ss(InterpState &S, CodePtr OpPC,
3550 const CallExpr *Call,
3551 bool HasRoundingMask) {
3552 APSInt Rounding, MaskInt;
3553 Pointer Src, B, A;
3554
3555 if (HasRoundingMask) {
3556 assert(Call->getNumArgs() == 5);
3557 Rounding = popToAPSInt(S, E: Call->getArg(Arg: 4));
3558 MaskInt = popToAPSInt(S, E: Call->getArg(Arg: 3));
3559 Src = S.Stk.pop<Pointer>();
3560 B = S.Stk.pop<Pointer>();
3561 A = S.Stk.pop<Pointer>();
3562 if (!CheckLoad(S, OpPC, Ptr: A) || !CheckLoad(S, OpPC, Ptr: B) ||
3563 !CheckLoad(S, OpPC, Ptr: Src))
3564 return false;
3565 } else {
3566 assert(Call->getNumArgs() == 2);
3567 B = S.Stk.pop<Pointer>();
3568 A = S.Stk.pop<Pointer>();
3569 if (!CheckLoad(S, OpPC, Ptr: A) || !CheckLoad(S, OpPC, Ptr: B))
3570 return false;
3571 }
3572
3573 const auto *DstVTy = Call->getType()->castAs<VectorType>();
3574 unsigned NumElems = DstVTy->getNumElements();
3575 const Pointer &Dst = S.Stk.peek<Pointer>();
3576
3577 // Copy all elements except lane 0 (overwritten below) from A to Dst.
3578 for (unsigned I = 1; I != NumElems; ++I)
3579 Dst.elem<Floating>(I) = A.elem<Floating>(I);
3580
3581 // Convert element 0 from double to float, or use Src if masked off.
3582 if (!HasRoundingMask || (MaskInt.getZExtValue() & 0x1)) {
3583 assert(S.getASTContext().FloatTy == DstVTy->getElementType() &&
3584 "cvtsd2ss requires float element type in destination vector");
3585
3586 Floating Conv = S.allocFloat(
3587 Sem: S.getASTContext().getFloatTypeSemantics(T: DstVTy->getElementType()));
3588 APFloat SrcVal = B.elem<Floating>(I: 0).getAPFloat();
3589 if (!convertDoubleToFloatStrict(Src: SrcVal, Dst&: Conv, S, DiagExpr: Call))
3590 return false;
3591 Dst.elem<Floating>(I: 0) = Conv;
3592 } else {
3593 Dst.elem<Floating>(I: 0) = Src.elem<Floating>(I: 0);
3594 }
3595
3596 Dst.initializeAllElements();
3597 return true;
3598}
3599
3600static bool interp__builtin_ia32_cvtpd2ps(InterpState &S, CodePtr OpPC,
3601 const CallExpr *Call, bool IsMasked,
3602 bool HasRounding) {
3603
3604 APSInt MaskVal;
3605 Pointer PassThrough;
3606 Pointer Src;
3607 APSInt Rounding;
3608
3609 if (IsMasked) {
3610 // Pop in reverse order.
3611 if (HasRounding) {
3612 Rounding = popToAPSInt(S, E: Call->getArg(Arg: 3));
3613 MaskVal = popToAPSInt(S, E: Call->getArg(Arg: 2));
3614 PassThrough = S.Stk.pop<Pointer>();
3615 Src = S.Stk.pop<Pointer>();
3616 } else {
3617 MaskVal = popToAPSInt(S, E: Call->getArg(Arg: 2));
3618 PassThrough = S.Stk.pop<Pointer>();
3619 Src = S.Stk.pop<Pointer>();
3620 }
3621
3622 if (!CheckLoad(S, OpPC, Ptr: PassThrough))
3623 return false;
3624 } else {
3625 // Pop source only.
3626 Src = S.Stk.pop<Pointer>();
3627 }
3628
3629 if (!CheckLoad(S, OpPC, Ptr: Src))
3630 return false;
3631
3632 const auto *RetVTy = Call->getType()->castAs<VectorType>();
3633 unsigned RetElems = RetVTy->getNumElements();
3634 unsigned SrcElems = Src.getNumElems();
3635 const Pointer &Dst = S.Stk.peek<Pointer>();
3636
3637 // Initialize destination with passthrough or zeros.
3638 for (unsigned I = 0; I != RetElems; ++I)
3639 if (IsMasked)
3640 Dst.elem<Floating>(I) = PassThrough.elem<Floating>(I);
3641 else
3642 Dst.elem<Floating>(I) = Floating(APFloat(0.0f));
3643
3644 assert(S.getASTContext().FloatTy == RetVTy->getElementType() &&
3645 "cvtpd2ps requires float element type in return vector");
3646
3647 // Convert double to float for enabled elements (only process source elements
3648 // that exist).
3649 for (unsigned I = 0; I != SrcElems; ++I) {
3650 if (IsMasked && !MaskVal[I])
3651 continue;
3652
3653 APFloat SrcVal = Src.elem<Floating>(I).getAPFloat();
3654
3655 Floating Conv = S.allocFloat(
3656 Sem: S.getASTContext().getFloatTypeSemantics(T: RetVTy->getElementType()));
3657 if (!convertDoubleToFloatStrict(Src: SrcVal, Dst&: Conv, S, DiagExpr: Call))
3658 return false;
3659 Dst.elem<Floating>(I) = Conv;
3660 }
3661
3662 Dst.initializeAllElements();
3663 return true;
3664}
3665
3666static bool interp__builtin_ia32_shuffle_generic(
3667 InterpState &S, CodePtr OpPC, const CallExpr *Call,
3668 llvm::function_ref<std::pair<unsigned, int>(unsigned, unsigned)>
3669 GetSourceIndex) {
3670
3671 assert(Call->getNumArgs() == 2 || Call->getNumArgs() == 3);
3672
3673 unsigned ShuffleMask = 0;
3674 Pointer A, MaskVector, B;
3675 bool IsVectorMask = false;
3676 bool IsSingleOperand = (Call->getNumArgs() == 2);
3677
3678 if (IsSingleOperand) {
3679 QualType MaskType = Call->getArg(Arg: 1)->getType();
3680 if (MaskType->isVectorType()) {
3681 IsVectorMask = true;
3682 MaskVector = S.Stk.pop<Pointer>();
3683 A = S.Stk.pop<Pointer>();
3684 B = A;
3685 } else if (MaskType->isIntegerType()) {
3686 ShuffleMask = popToAPSInt(S, E: Call->getArg(Arg: 1)).getZExtValue();
3687 A = S.Stk.pop<Pointer>();
3688 B = A;
3689 } else {
3690 return false;
3691 }
3692 } else {
3693 QualType Arg2Type = Call->getArg(Arg: 2)->getType();
3694 if (Arg2Type->isVectorType()) {
3695 IsVectorMask = true;
3696 B = S.Stk.pop<Pointer>();
3697 MaskVector = S.Stk.pop<Pointer>();
3698 A = S.Stk.pop<Pointer>();
3699 } else if (Arg2Type->isIntegerType()) {
3700 ShuffleMask = popToAPSInt(S, E: Call->getArg(Arg: 2)).getZExtValue();
3701 B = S.Stk.pop<Pointer>();
3702 A = S.Stk.pop<Pointer>();
3703 } else {
3704 return false;
3705 }
3706 }
3707
3708 QualType Arg0Type = Call->getArg(Arg: 0)->getType();
3709 const auto *VecT = Arg0Type->castAs<VectorType>();
3710 PrimType ElemT = *S.getContext().classify(T: VecT->getElementType());
3711 unsigned NumElems = VecT->getNumElements();
3712
3713 const Pointer &Dst = S.Stk.peek<Pointer>();
3714
3715 PrimType MaskElemT = PT_Uint32;
3716 if (IsVectorMask) {
3717 QualType Arg1Type = Call->getArg(Arg: 1)->getType();
3718 const auto *MaskVecT = Arg1Type->castAs<VectorType>();
3719 QualType MaskElemType = MaskVecT->getElementType();
3720 MaskElemT = *S.getContext().classify(T: MaskElemType);
3721 }
3722
3723 for (unsigned DstIdx = 0; DstIdx != NumElems; ++DstIdx) {
3724 if (IsVectorMask) {
3725 INT_TYPE_SWITCH(MaskElemT, {
3726 ShuffleMask = static_cast<unsigned>(MaskVector.elem<T>(DstIdx));
3727 });
3728 }
3729
3730 auto [SrcVecIdx, SrcIdx] = GetSourceIndex(DstIdx, ShuffleMask);
3731
3732 if (SrcIdx < 0) {
3733 // Zero out this element
3734 if (ElemT == PT_Float) {
3735 Dst.elem<Floating>(I: DstIdx) = Floating(
3736 S.getASTContext().getFloatTypeSemantics(T: VecT->getElementType()));
3737 } else {
3738 INT_TYPE_SWITCH_NO_BOOL(ElemT, { Dst.elem<T>(DstIdx) = T::from(0); });
3739 }
3740 } else {
3741 const Pointer &Src = (SrcVecIdx == 0) ? A : B;
3742 TYPE_SWITCH(ElemT, { Dst.elem<T>(DstIdx) = Src.elem<T>(SrcIdx); });
3743 }
3744 }
3745 Dst.initializeAllElements();
3746
3747 return true;
3748}
3749
3750static bool interp__builtin_ia32_shift_with_count(
3751 InterpState &S, CodePtr OpPC, const CallExpr *Call,
3752 llvm::function_ref<APInt(const APInt &, uint64_t)> ShiftOp,
3753 llvm::function_ref<APInt(const APInt &, unsigned)> OverflowOp) {
3754
3755 assert(Call->getNumArgs() == 2);
3756
3757 const Pointer &Count = S.Stk.pop<Pointer>();
3758 const Pointer &Source = S.Stk.pop<Pointer>();
3759
3760 QualType SourceType = Call->getArg(Arg: 0)->getType();
3761 QualType CountType = Call->getArg(Arg: 1)->getType();
3762 assert(SourceType->isVectorType() && CountType->isVectorType());
3763
3764 const auto *SourceVecT = SourceType->castAs<VectorType>();
3765 const auto *CountVecT = CountType->castAs<VectorType>();
3766 PrimType SourceElemT = *S.getContext().classify(T: SourceVecT->getElementType());
3767 PrimType CountElemT = *S.getContext().classify(T: CountVecT->getElementType());
3768
3769 const Pointer &Dst = S.Stk.peek<Pointer>();
3770
3771 unsigned DestEltWidth =
3772 S.getASTContext().getTypeSize(T: SourceVecT->getElementType());
3773 bool IsDestUnsigned = SourceVecT->getElementType()->isUnsignedIntegerType();
3774 unsigned DestLen = SourceVecT->getNumElements();
3775 unsigned CountEltWidth =
3776 S.getASTContext().getTypeSize(T: CountVecT->getElementType());
3777 unsigned NumBitsInQWord = 64;
3778 unsigned NumCountElts = NumBitsInQWord / CountEltWidth;
3779
3780 uint64_t CountLQWord = 0;
3781 for (unsigned EltIdx = 0; EltIdx != NumCountElts; ++EltIdx) {
3782 uint64_t Elt = 0;
3783 INT_TYPE_SWITCH(CountElemT,
3784 { Elt = static_cast<uint64_t>(Count.elem<T>(EltIdx)); });
3785 CountLQWord |= (Elt << (EltIdx * CountEltWidth));
3786 }
3787
3788 for (unsigned EltIdx = 0; EltIdx != DestLen; ++EltIdx) {
3789 APSInt Elt;
3790 INT_TYPE_SWITCH(SourceElemT, { Elt = Source.elem<T>(EltIdx).toAPSInt(); });
3791
3792 APInt Result;
3793 if (CountLQWord < DestEltWidth) {
3794 Result = ShiftOp(Elt, CountLQWord);
3795 } else {
3796 Result = OverflowOp(Elt, DestEltWidth);
3797 }
3798 if (IsDestUnsigned) {
3799 INT_TYPE_SWITCH(SourceElemT, {
3800 Dst.elem<T>(EltIdx) = T::from(Result.getZExtValue());
3801 });
3802 } else {
3803 INT_TYPE_SWITCH(SourceElemT, {
3804 Dst.elem<T>(EltIdx) = T::from(Result.getSExtValue());
3805 });
3806 }
3807 }
3808
3809 Dst.initializeAllElements();
3810 return true;
3811}
3812
3813static bool interp__builtin_ia32_shufbitqmb_mask(InterpState &S, CodePtr OpPC,
3814 const CallExpr *Call) {
3815
3816 assert(Call->getNumArgs() == 3);
3817
3818 QualType SourceType = Call->getArg(Arg: 0)->getType();
3819 QualType ShuffleMaskType = Call->getArg(Arg: 1)->getType();
3820 QualType ZeroMaskType = Call->getArg(Arg: 2)->getType();
3821 if (!SourceType->isVectorType() || !ShuffleMaskType->isVectorType() ||
3822 !ZeroMaskType->isIntegerType()) {
3823 return false;
3824 }
3825
3826 Pointer Source, ShuffleMask;
3827 APSInt ZeroMask = popToAPSInt(S, E: Call->getArg(Arg: 2));
3828 ShuffleMask = S.Stk.pop<Pointer>();
3829 Source = S.Stk.pop<Pointer>();
3830
3831 const auto *SourceVecT = SourceType->castAs<VectorType>();
3832 const auto *ShuffleMaskVecT = ShuffleMaskType->castAs<VectorType>();
3833 assert(SourceVecT->getNumElements() == ShuffleMaskVecT->getNumElements());
3834 assert(ZeroMask.getBitWidth() == SourceVecT->getNumElements());
3835
3836 PrimType SourceElemT = *S.getContext().classify(T: SourceVecT->getElementType());
3837 PrimType ShuffleMaskElemT =
3838 *S.getContext().classify(T: ShuffleMaskVecT->getElementType());
3839
3840 unsigned NumBytesInQWord = 8;
3841 unsigned NumBitsInByte = 8;
3842 unsigned NumBytes = SourceVecT->getNumElements();
3843 unsigned NumQWords = NumBytes / NumBytesInQWord;
3844 unsigned RetWidth = ZeroMask.getBitWidth();
3845 APSInt RetMask(llvm::APInt(RetWidth, 0), /*isUnsigned=*/true);
3846
3847 for (unsigned QWordId = 0; QWordId != NumQWords; ++QWordId) {
3848 APInt SourceQWord(64, 0);
3849 for (unsigned ByteIdx = 0; ByteIdx != NumBytesInQWord; ++ByteIdx) {
3850 uint64_t Byte = 0;
3851 INT_TYPE_SWITCH(SourceElemT, {
3852 Byte = static_cast<uint64_t>(
3853 Source.elem<T>(QWordId * NumBytesInQWord + ByteIdx));
3854 });
3855 SourceQWord.insertBits(SubBits: APInt(8, Byte & 0xFF), bitPosition: ByteIdx * NumBitsInByte);
3856 }
3857
3858 for (unsigned ByteIdx = 0; ByteIdx != NumBytesInQWord; ++ByteIdx) {
3859 unsigned SelIdx = QWordId * NumBytesInQWord + ByteIdx;
3860 unsigned M = 0;
3861 INT_TYPE_SWITCH(ShuffleMaskElemT, {
3862 M = static_cast<unsigned>(ShuffleMask.elem<T>(SelIdx)) & 0x3F;
3863 });
3864
3865 if (ZeroMask[SelIdx]) {
3866 RetMask.setBitVal(BitPosition: SelIdx, BitValue: SourceQWord[M]);
3867 }
3868 }
3869 }
3870
3871 pushInteger(S, Val: RetMask, QT: Call->getType());
3872 return true;
3873}
3874
3875static bool interp__builtin_ia32_vcvtps2ph(InterpState &S, CodePtr OpPC,
3876 const CallExpr *Call) {
3877 // Arguments are: vector of floats, rounding immediate
3878 assert(Call->getNumArgs() == 2);
3879
3880 APSInt Imm = popToAPSInt(S, E: Call->getArg(Arg: 1));
3881 const Pointer &Src = S.Stk.pop<Pointer>();
3882 const Pointer &Dst = S.Stk.peek<Pointer>();
3883
3884 assert(Src.getFieldDesc()->isPrimitiveArray());
3885 assert(Dst.getFieldDesc()->isPrimitiveArray());
3886
3887 const auto *SrcVTy = Call->getArg(Arg: 0)->getType()->castAs<VectorType>();
3888 unsigned SrcNumElems = SrcVTy->getNumElements();
3889 const auto *DstVTy = Call->getType()->castAs<VectorType>();
3890 unsigned DstNumElems = DstVTy->getNumElements();
3891
3892 const llvm::fltSemantics &HalfSem =
3893 S.getASTContext().getFloatTypeSemantics(T: S.getASTContext().HalfTy);
3894
3895 // imm[2] == 1 means use MXCSR rounding mode.
3896 // In that case, we can only evaluate if the conversion is exact.
3897 int ImmVal = Imm.getZExtValue();
3898 bool UseMXCSR = (ImmVal & 4) != 0;
3899 bool IsFPConstrained =
3900 Call->getFPFeaturesInEffect(LO: S.getASTContext().getLangOpts())
3901 .isFPConstrained();
3902
3903 llvm::RoundingMode RM;
3904 if (!UseMXCSR) {
3905 switch (ImmVal & 3) {
3906 case 0:
3907 RM = llvm::RoundingMode::NearestTiesToEven;
3908 break;
3909 case 1:
3910 RM = llvm::RoundingMode::TowardNegative;
3911 break;
3912 case 2:
3913 RM = llvm::RoundingMode::TowardPositive;
3914 break;
3915 case 3:
3916 RM = llvm::RoundingMode::TowardZero;
3917 break;
3918 default:
3919 llvm_unreachable("Invalid immediate rounding mode");
3920 }
3921 } else {
3922 // For MXCSR, we must check for exactness. We can use any rounding mode
3923 // for the trial conversion since the result is the same if it's exact.
3924 RM = llvm::RoundingMode::NearestTiesToEven;
3925 }
3926
3927 QualType DstElemQT = Dst.getFieldDesc()->getElemQualType();
3928 PrimType DstElemT = *S.getContext().classify(T: DstElemQT);
3929
3930 for (unsigned I = 0; I != SrcNumElems; ++I) {
3931 Floating SrcVal = Src.elem<Floating>(I);
3932 APFloat DstVal = SrcVal.getAPFloat();
3933
3934 bool LostInfo;
3935 APFloat::opStatus St = DstVal.convert(ToSemantics: HalfSem, RM, losesInfo: &LostInfo);
3936
3937 if (UseMXCSR && IsFPConstrained && St != APFloat::opOK) {
3938 S.FFDiag(SI: S.Current->getSource(PC: OpPC),
3939 DiagId: diag::note_constexpr_dynamic_rounding);
3940 return false;
3941 }
3942
3943 INT_TYPE_SWITCH_NO_BOOL(DstElemT, {
3944 // Convert the destination value's bit pattern to an unsigned integer,
3945 // then reconstruct the element using the target type's 'from' method.
3946 uint64_t RawBits = DstVal.bitcastToAPInt().getZExtValue();
3947 Dst.elem<T>(I) = T::from(RawBits);
3948 });
3949 }
3950
3951 // Zero out remaining elements if the destination has more elements
3952 // (e.g., vcvtps2ph converting 4 floats to 8 shorts).
3953 if (DstNumElems > SrcNumElems) {
3954 for (unsigned I = SrcNumElems; I != DstNumElems; ++I) {
3955 INT_TYPE_SWITCH_NO_BOOL(DstElemT, { Dst.elem<T>(I) = T::from(0); });
3956 }
3957 }
3958
3959 Dst.initializeAllElements();
3960 return true;
3961}
3962
3963static bool interp__builtin_ia32_multishiftqb(InterpState &S, CodePtr OpPC,
3964 const CallExpr *Call) {
3965 assert(Call->getNumArgs() == 2);
3966
3967 QualType ATy = Call->getArg(Arg: 0)->getType();
3968 QualType BTy = Call->getArg(Arg: 1)->getType();
3969 if (!ATy->isVectorType() || !BTy->isVectorType()) {
3970 return false;
3971 }
3972
3973 const Pointer &BPtr = S.Stk.pop<Pointer>();
3974 const Pointer &APtr = S.Stk.pop<Pointer>();
3975 const auto *AVecT = ATy->castAs<VectorType>();
3976 assert(AVecT->getNumElements() ==
3977 BTy->castAs<VectorType>()->getNumElements());
3978
3979 PrimType ElemT = *S.getContext().classify(T: AVecT->getElementType());
3980
3981 unsigned NumBytesInQWord = 8;
3982 unsigned NumBitsInByte = 8;
3983 unsigned NumBytes = AVecT->getNumElements();
3984 unsigned NumQWords = NumBytes / NumBytesInQWord;
3985 const Pointer &Dst = S.Stk.peek<Pointer>();
3986
3987 for (unsigned QWordId = 0; QWordId != NumQWords; ++QWordId) {
3988 APInt BQWord(64, 0);
3989 for (unsigned ByteIdx = 0; ByteIdx != NumBytesInQWord; ++ByteIdx) {
3990 unsigned Idx = QWordId * NumBytesInQWord + ByteIdx;
3991 INT_TYPE_SWITCH(ElemT, {
3992 uint64_t Byte = static_cast<uint64_t>(BPtr.elem<T>(Idx));
3993 BQWord.insertBits(APInt(8, Byte & 0xFF), ByteIdx * NumBitsInByte);
3994 });
3995 }
3996
3997 for (unsigned ByteIdx = 0; ByteIdx != NumBytesInQWord; ++ByteIdx) {
3998 unsigned Idx = QWordId * NumBytesInQWord + ByteIdx;
3999 uint64_t Ctrl = 0;
4000 INT_TYPE_SWITCH(
4001 ElemT, { Ctrl = static_cast<uint64_t>(APtr.elem<T>(Idx)) & 0x3F; });
4002
4003 APInt Byte(8, 0);
4004 for (unsigned BitIdx = 0; BitIdx != NumBitsInByte; ++BitIdx) {
4005 Byte.setBitVal(BitPosition: BitIdx, BitValue: BQWord[(Ctrl + BitIdx) & 0x3F]);
4006 }
4007 INT_TYPE_SWITCH(ElemT,
4008 { Dst.elem<T>(Idx) = T::from(Byte.getZExtValue()); });
4009 }
4010 }
4011
4012 Dst.initializeAllElements();
4013
4014 return true;
4015}
4016
4017static bool interp_builtin_ia32_gfni_affine(InterpState &S, CodePtr OpPC,
4018 const CallExpr *Call,
4019 bool Inverse) {
4020 assert(Call->getNumArgs() == 3);
4021 QualType XType = Call->getArg(Arg: 0)->getType();
4022 QualType AType = Call->getArg(Arg: 1)->getType();
4023 QualType ImmType = Call->getArg(Arg: 2)->getType();
4024 if (!XType->isVectorType() || !AType->isVectorType() ||
4025 !ImmType->isIntegerType()) {
4026 return false;
4027 }
4028
4029 Pointer X, A;
4030 APSInt Imm = popToAPSInt(S, E: Call->getArg(Arg: 2));
4031 A = S.Stk.pop<Pointer>();
4032 X = S.Stk.pop<Pointer>();
4033
4034 const Pointer &Dst = S.Stk.peek<Pointer>();
4035 const auto *AVecT = AType->castAs<VectorType>();
4036 assert(XType->castAs<VectorType>()->getNumElements() ==
4037 AVecT->getNumElements());
4038 unsigned NumBytesInQWord = 8;
4039 unsigned NumBytes = AVecT->getNumElements();
4040 unsigned NumBitsInQWord = 64;
4041 unsigned NumQWords = NumBytes / NumBytesInQWord;
4042 unsigned NumBitsInByte = 8;
4043 PrimType AElemT = *S.getContext().classify(T: AVecT->getElementType());
4044
4045 // computing A*X + Imm
4046 for (unsigned QWordIdx = 0; QWordIdx != NumQWords; ++QWordIdx) {
4047 // Extract the QWords from X, A
4048 APInt XQWord(NumBitsInQWord, 0);
4049 APInt AQWord(NumBitsInQWord, 0);
4050 for (unsigned ByteIdx = 0; ByteIdx != NumBytesInQWord; ++ByteIdx) {
4051 unsigned Idx = QWordIdx * NumBytesInQWord + ByteIdx;
4052 uint8_t XByte;
4053 uint8_t AByte;
4054 INT_TYPE_SWITCH(AElemT, {
4055 XByte = static_cast<uint8_t>(X.elem<T>(Idx));
4056 AByte = static_cast<uint8_t>(A.elem<T>(Idx));
4057 });
4058
4059 XQWord.insertBits(SubBits: APInt(NumBitsInByte, XByte), bitPosition: ByteIdx * NumBitsInByte);
4060 AQWord.insertBits(SubBits: APInt(NumBitsInByte, AByte), bitPosition: ByteIdx * NumBitsInByte);
4061 }
4062
4063 for (unsigned ByteIdx = 0; ByteIdx != NumBytesInQWord; ++ByteIdx) {
4064 unsigned Idx = QWordIdx * NumBytesInQWord + ByteIdx;
4065 uint8_t XByte =
4066 XQWord.lshr(shiftAmt: ByteIdx * NumBitsInByte).getLoBits(numBits: 8).getZExtValue();
4067 INT_TYPE_SWITCH(AElemT, {
4068 Dst.elem<T>(Idx) = T::from(GFNIAffine(XByte, AQWord, Imm, Inverse));
4069 });
4070 }
4071 }
4072 Dst.initializeAllElements();
4073 return true;
4074}
4075
4076static bool interp__builtin_ia32_gfni_mul(InterpState &S, CodePtr OpPC,
4077 const CallExpr *Call) {
4078 assert(Call->getNumArgs() == 2);
4079
4080 QualType AType = Call->getArg(Arg: 0)->getType();
4081 QualType BType = Call->getArg(Arg: 1)->getType();
4082 if (!AType->isVectorType() || !BType->isVectorType()) {
4083 return false;
4084 }
4085
4086 Pointer A, B;
4087 B = S.Stk.pop<Pointer>();
4088 A = S.Stk.pop<Pointer>();
4089
4090 const Pointer &Dst = S.Stk.peek<Pointer>();
4091 const auto *AVecT = AType->castAs<VectorType>();
4092 assert(AVecT->getNumElements() ==
4093 BType->castAs<VectorType>()->getNumElements());
4094
4095 PrimType AElemT = *S.getContext().classify(T: AVecT->getElementType());
4096 unsigned NumBytes = A.getNumElems();
4097
4098 for (unsigned ByteIdx = 0; ByteIdx != NumBytes; ++ByteIdx) {
4099 uint8_t AByte, BByte;
4100 INT_TYPE_SWITCH(AElemT, {
4101 AByte = static_cast<uint8_t>(A.elem<T>(ByteIdx));
4102 BByte = static_cast<uint8_t>(B.elem<T>(ByteIdx));
4103 Dst.elem<T>(ByteIdx) = T::from(GFNIMul(AByte, BByte));
4104 });
4105 }
4106
4107 Dst.initializeAllElements();
4108 return true;
4109}
4110
4111bool InterpretBuiltin(InterpState &S, CodePtr OpPC, const CallExpr *Call,
4112 uint32_t BuiltinID) {
4113 if (!S.getASTContext().BuiltinInfo.isConstantEvaluated(ID: BuiltinID))
4114 return Invalid(S, OpPC);
4115
4116 const InterpFrame *Frame = S.Current;
4117 switch (BuiltinID) {
4118 case Builtin::BI__builtin_is_constant_evaluated:
4119 return interp__builtin_is_constant_evaluated(S, OpPC, Frame, Call);
4120
4121 case Builtin::BI__builtin_assume:
4122 case Builtin::BI__assume:
4123 return interp__builtin_assume(S, OpPC, Frame, Call);
4124
4125 case Builtin::BI__builtin_strcmp:
4126 case Builtin::BIstrcmp:
4127 case Builtin::BI__builtin_strncmp:
4128 case Builtin::BIstrncmp:
4129 case Builtin::BI__builtin_wcsncmp:
4130 case Builtin::BIwcsncmp:
4131 case Builtin::BI__builtin_wcscmp:
4132 case Builtin::BIwcscmp:
4133 return interp__builtin_strcmp(S, OpPC, Frame, Call, ID: BuiltinID);
4134
4135 case Builtin::BI__builtin_strlen:
4136 case Builtin::BIstrlen:
4137 case Builtin::BI__builtin_wcslen:
4138 case Builtin::BIwcslen:
4139 return interp__builtin_strlen(S, OpPC, Frame, Call, ID: BuiltinID);
4140
4141 case Builtin::BI__builtin_nan:
4142 case Builtin::BI__builtin_nanf:
4143 case Builtin::BI__builtin_nanl:
4144 case Builtin::BI__builtin_nanf16:
4145 case Builtin::BI__builtin_nanf128:
4146 return interp__builtin_nan(S, OpPC, Frame, Call, /*Signaling=*/false);
4147
4148 case Builtin::BI__builtin_nans:
4149 case Builtin::BI__builtin_nansf:
4150 case Builtin::BI__builtin_nansl:
4151 case Builtin::BI__builtin_nansf16:
4152 case Builtin::BI__builtin_nansf128:
4153 return interp__builtin_nan(S, OpPC, Frame, Call, /*Signaling=*/true);
4154
4155 case Builtin::BI__builtin_huge_val:
4156 case Builtin::BI__builtin_huge_valf:
4157 case Builtin::BI__builtin_huge_vall:
4158 case Builtin::BI__builtin_huge_valf16:
4159 case Builtin::BI__builtin_huge_valf128:
4160 case Builtin::BI__builtin_inf:
4161 case Builtin::BI__builtin_inff:
4162 case Builtin::BI__builtin_infl:
4163 case Builtin::BI__builtin_inff16:
4164 case Builtin::BI__builtin_inff128:
4165 return interp__builtin_inf(S, OpPC, Frame, Call);
4166
4167 case Builtin::BI__builtin_copysign:
4168 case Builtin::BI__builtin_copysignf:
4169 case Builtin::BI__builtin_copysignl:
4170 case Builtin::BI__builtin_copysignf128:
4171 return interp__builtin_copysign(S, OpPC, Frame);
4172
4173 case Builtin::BI__builtin_fmin:
4174 case Builtin::BI__builtin_fminf:
4175 case Builtin::BI__builtin_fminl:
4176 case Builtin::BI__builtin_fminf16:
4177 case Builtin::BI__builtin_fminf128:
4178 return interp__builtin_fmin(S, OpPC, Frame, /*IsNumBuiltin=*/false);
4179
4180 case Builtin::BI__builtin_fminimum_num:
4181 case Builtin::BI__builtin_fminimum_numf:
4182 case Builtin::BI__builtin_fminimum_numl:
4183 case Builtin::BI__builtin_fminimum_numf16:
4184 case Builtin::BI__builtin_fminimum_numf128:
4185 return interp__builtin_fmin(S, OpPC, Frame, /*IsNumBuiltin=*/true);
4186
4187 case Builtin::BI__builtin_fmax:
4188 case Builtin::BI__builtin_fmaxf:
4189 case Builtin::BI__builtin_fmaxl:
4190 case Builtin::BI__builtin_fmaxf16:
4191 case Builtin::BI__builtin_fmaxf128:
4192 return interp__builtin_fmax(S, OpPC, Frame, /*IsNumBuiltin=*/false);
4193
4194 case Builtin::BI__builtin_fmaximum_num:
4195 case Builtin::BI__builtin_fmaximum_numf:
4196 case Builtin::BI__builtin_fmaximum_numl:
4197 case Builtin::BI__builtin_fmaximum_numf16:
4198 case Builtin::BI__builtin_fmaximum_numf128:
4199 return interp__builtin_fmax(S, OpPC, Frame, /*IsNumBuiltin=*/true);
4200
4201 case Builtin::BI__builtin_isnan:
4202 return interp__builtin_isnan(S, OpPC, Frame, Call);
4203
4204 case Builtin::BI__builtin_issignaling:
4205 return interp__builtin_issignaling(S, OpPC, Frame, Call);
4206
4207 case Builtin::BI__builtin_isinf:
4208 return interp__builtin_isinf(S, OpPC, Frame, /*Sign=*/CheckSign: false, Call);
4209
4210 case Builtin::BI__builtin_isinf_sign:
4211 return interp__builtin_isinf(S, OpPC, Frame, /*Sign=*/CheckSign: true, Call);
4212
4213 case Builtin::BI__builtin_isfinite:
4214 return interp__builtin_isfinite(S, OpPC, Frame, Call);
4215
4216 case Builtin::BI__builtin_isnormal:
4217 return interp__builtin_isnormal(S, OpPC, Frame, Call);
4218
4219 case Builtin::BI__builtin_issubnormal:
4220 return interp__builtin_issubnormal(S, OpPC, Frame, Call);
4221
4222 case Builtin::BI__builtin_iszero:
4223 return interp__builtin_iszero(S, OpPC, Frame, Call);
4224
4225 case Builtin::BI__builtin_signbit:
4226 case Builtin::BI__builtin_signbitf:
4227 case Builtin::BI__builtin_signbitl:
4228 return interp__builtin_signbit(S, OpPC, Frame, Call);
4229
4230 case Builtin::BI__builtin_isgreater:
4231 case Builtin::BI__builtin_isgreaterequal:
4232 case Builtin::BI__builtin_isless:
4233 case Builtin::BI__builtin_islessequal:
4234 case Builtin::BI__builtin_islessgreater:
4235 case Builtin::BI__builtin_isunordered:
4236 return interp_floating_comparison(S, OpPC, Call, ID: BuiltinID);
4237
4238 case Builtin::BI__builtin_isfpclass:
4239 return interp__builtin_isfpclass(S, OpPC, Frame, Call);
4240
4241 case Builtin::BI__builtin_fpclassify:
4242 return interp__builtin_fpclassify(S, OpPC, Frame, Call);
4243
4244 case Builtin::BI__builtin_fabs:
4245 case Builtin::BI__builtin_fabsf:
4246 case Builtin::BI__builtin_fabsl:
4247 case Builtin::BI__builtin_fabsf128:
4248 return interp__builtin_fabs(S, OpPC, Frame);
4249
4250 case Builtin::BI__builtin_abs:
4251 case Builtin::BI__builtin_labs:
4252 case Builtin::BI__builtin_llabs:
4253 return interp__builtin_abs(S, OpPC, Frame, Call);
4254
4255 case Builtin::BI__builtin_popcount:
4256 case Builtin::BI__builtin_popcountl:
4257 case Builtin::BI__builtin_popcountll:
4258 case Builtin::BI__builtin_popcountg:
4259 case Builtin::BI__popcnt16: // Microsoft variants of popcount
4260 case Builtin::BI__popcnt:
4261 case Builtin::BI__popcnt64:
4262 return interp__builtin_popcount(S, OpPC, Frame, Call);
4263
4264 case Builtin::BI__builtin_parity:
4265 case Builtin::BI__builtin_parityl:
4266 case Builtin::BI__builtin_parityll:
4267 return interp__builtin_elementwise_int_unaryop(
4268 S, OpPC, Call, Fn: [](const APSInt &Val) {
4269 return APInt(Val.getBitWidth(), Val.popcount() % 2);
4270 });
4271 case Builtin::BI__builtin_clrsb:
4272 case Builtin::BI__builtin_clrsbl:
4273 case Builtin::BI__builtin_clrsbll:
4274 return interp__builtin_elementwise_int_unaryop(
4275 S, OpPC, Call, Fn: [](const APSInt &Val) {
4276 return APInt(Val.getBitWidth(),
4277 Val.getBitWidth() - Val.getSignificantBits());
4278 });
4279 case Builtin::BI__builtin_bitreverse8:
4280 case Builtin::BI__builtin_bitreverse16:
4281 case Builtin::BI__builtin_bitreverse32:
4282 case Builtin::BI__builtin_bitreverse64:
4283 return interp__builtin_elementwise_int_unaryop(
4284 S, OpPC, Call, Fn: [](const APSInt &Val) { return Val.reverseBits(); });
4285
4286 case Builtin::BI__builtin_classify_type:
4287 return interp__builtin_classify_type(S, OpPC, Frame, Call);
4288
4289 case Builtin::BI__builtin_expect:
4290 case Builtin::BI__builtin_expect_with_probability:
4291 return interp__builtin_expect(S, OpPC, Frame, Call);
4292
4293 case Builtin::BI__builtin_rotateleft8:
4294 case Builtin::BI__builtin_rotateleft16:
4295 case Builtin::BI__builtin_rotateleft32:
4296 case Builtin::BI__builtin_rotateleft64:
4297 case Builtin::BI__builtin_stdc_rotate_left:
4298 case Builtin::BI_rotl8: // Microsoft variants of rotate left
4299 case Builtin::BI_rotl16:
4300 case Builtin::BI_rotl:
4301 case Builtin::BI_lrotl:
4302 case Builtin::BI_rotl64:
4303 case Builtin::BI__builtin_rotateright8:
4304 case Builtin::BI__builtin_rotateright16:
4305 case Builtin::BI__builtin_rotateright32:
4306 case Builtin::BI__builtin_rotateright64:
4307 case Builtin::BI__builtin_stdc_rotate_right:
4308 case Builtin::BI_rotr8: // Microsoft variants of rotate right
4309 case Builtin::BI_rotr16:
4310 case Builtin::BI_rotr:
4311 case Builtin::BI_lrotr:
4312 case Builtin::BI_rotr64: {
4313 // Determine if this is a rotate right operation
4314 bool IsRotateRight;
4315 switch (BuiltinID) {
4316 case Builtin::BI__builtin_rotateright8:
4317 case Builtin::BI__builtin_rotateright16:
4318 case Builtin::BI__builtin_rotateright32:
4319 case Builtin::BI__builtin_rotateright64:
4320 case Builtin::BI__builtin_stdc_rotate_right:
4321 case Builtin::BI_rotr8:
4322 case Builtin::BI_rotr16:
4323 case Builtin::BI_rotr:
4324 case Builtin::BI_lrotr:
4325 case Builtin::BI_rotr64:
4326 IsRotateRight = true;
4327 break;
4328 default:
4329 IsRotateRight = false;
4330 break;
4331 }
4332
4333 return interp__builtin_elementwise_int_binop(
4334 S, OpPC, Call, Fn: [IsRotateRight](const APSInt &Value, APSInt Amount) {
4335 Amount = NormalizeRotateAmount(Value, Amount);
4336 return IsRotateRight ? Value.rotr(rotateAmt: Amount.getZExtValue())
4337 : Value.rotl(rotateAmt: Amount.getZExtValue());
4338 });
4339 }
4340
4341 case Builtin::BI__builtin_ffs:
4342 case Builtin::BI__builtin_ffsl:
4343 case Builtin::BI__builtin_ffsll:
4344 return interp__builtin_elementwise_int_unaryop(
4345 S, OpPC, Call, Fn: [](const APSInt &Val) {
4346 return APInt(Val.getBitWidth(),
4347 Val.isZero() ? 0u : Val.countTrailingZeros() + 1u);
4348 });
4349
4350 case Builtin::BIaddressof:
4351 case Builtin::BI__addressof:
4352 case Builtin::BI__builtin_addressof:
4353 assert(isNoopBuiltin(BuiltinID));
4354 return interp__builtin_addressof(S, OpPC, Frame, Call);
4355
4356 case Builtin::BIas_const:
4357 case Builtin::BIforward:
4358 case Builtin::BIforward_like:
4359 case Builtin::BImove:
4360 case Builtin::BImove_if_noexcept:
4361 assert(isNoopBuiltin(BuiltinID));
4362 return interp__builtin_move(S, OpPC, Frame, Call);
4363
4364 case Builtin::BI__builtin_eh_return_data_regno:
4365 return interp__builtin_eh_return_data_regno(S, OpPC, Frame, Call);
4366
4367 case Builtin::BI__builtin_launder:
4368 assert(isNoopBuiltin(BuiltinID));
4369 return true;
4370
4371 case Builtin::BI__builtin_add_overflow:
4372 case Builtin::BI__builtin_sub_overflow:
4373 case Builtin::BI__builtin_mul_overflow:
4374 case Builtin::BI__builtin_sadd_overflow:
4375 case Builtin::BI__builtin_uadd_overflow:
4376 case Builtin::BI__builtin_uaddl_overflow:
4377 case Builtin::BI__builtin_uaddll_overflow:
4378 case Builtin::BI__builtin_usub_overflow:
4379 case Builtin::BI__builtin_usubl_overflow:
4380 case Builtin::BI__builtin_usubll_overflow:
4381 case Builtin::BI__builtin_umul_overflow:
4382 case Builtin::BI__builtin_umull_overflow:
4383 case Builtin::BI__builtin_umulll_overflow:
4384 case Builtin::BI__builtin_saddl_overflow:
4385 case Builtin::BI__builtin_saddll_overflow:
4386 case Builtin::BI__builtin_ssub_overflow:
4387 case Builtin::BI__builtin_ssubl_overflow:
4388 case Builtin::BI__builtin_ssubll_overflow:
4389 case Builtin::BI__builtin_smul_overflow:
4390 case Builtin::BI__builtin_smull_overflow:
4391 case Builtin::BI__builtin_smulll_overflow:
4392 return interp__builtin_overflowop(S, OpPC, Call, BuiltinOp: BuiltinID);
4393
4394 case Builtin::BI__builtin_addcb:
4395 case Builtin::BI__builtin_addcs:
4396 case Builtin::BI__builtin_addc:
4397 case Builtin::BI__builtin_addcl:
4398 case Builtin::BI__builtin_addcll:
4399 case Builtin::BI__builtin_subcb:
4400 case Builtin::BI__builtin_subcs:
4401 case Builtin::BI__builtin_subc:
4402 case Builtin::BI__builtin_subcl:
4403 case Builtin::BI__builtin_subcll:
4404 return interp__builtin_carryop(S, OpPC, Frame, Call, BuiltinOp: BuiltinID);
4405
4406 case Builtin::BI__builtin_clz:
4407 case Builtin::BI__builtin_clzl:
4408 case Builtin::BI__builtin_clzll:
4409 case Builtin::BI__builtin_clzs:
4410 case Builtin::BI__builtin_clzg:
4411 case Builtin::BI__lzcnt16: // Microsoft variants of count leading-zeroes
4412 case Builtin::BI__lzcnt:
4413 case Builtin::BI__lzcnt64:
4414 return interp__builtin_clz(S, OpPC, Frame, Call, BuiltinOp: BuiltinID);
4415
4416 case Builtin::BI__builtin_ctz:
4417 case Builtin::BI__builtin_ctzl:
4418 case Builtin::BI__builtin_ctzll:
4419 case Builtin::BI__builtin_ctzs:
4420 case Builtin::BI__builtin_ctzg:
4421 return interp__builtin_ctz(S, OpPC, Frame, Call, BuiltinID);
4422
4423 case Builtin::BI__builtin_elementwise_clzg:
4424 case Builtin::BI__builtin_elementwise_ctzg:
4425 return interp__builtin_elementwise_countzeroes(S, OpPC, Frame, Call,
4426 BuiltinID);
4427 case Builtin::BI__builtin_bswapg:
4428 case Builtin::BI__builtin_bswap16:
4429 case Builtin::BI__builtin_bswap32:
4430 case Builtin::BI__builtin_bswap64:
4431 return interp__builtin_bswap(S, OpPC, Frame, Call);
4432
4433 case Builtin::BI__atomic_always_lock_free:
4434 case Builtin::BI__atomic_is_lock_free:
4435 return interp__builtin_atomic_lock_free(S, OpPC, Frame, Call, BuiltinOp: BuiltinID);
4436
4437 case Builtin::BI__c11_atomic_is_lock_free:
4438 return interp__builtin_c11_atomic_is_lock_free(S, OpPC, Frame, Call);
4439
4440 case Builtin::BI__builtin_complex:
4441 return interp__builtin_complex(S, OpPC, Frame, Call);
4442
4443 case Builtin::BI__builtin_is_aligned:
4444 case Builtin::BI__builtin_align_up:
4445 case Builtin::BI__builtin_align_down:
4446 return interp__builtin_is_aligned_up_down(S, OpPC, Frame, Call, BuiltinOp: BuiltinID);
4447
4448 case Builtin::BI__builtin_assume_aligned:
4449 return interp__builtin_assume_aligned(S, OpPC, Frame, Call);
4450
4451 case clang::X86::BI__builtin_ia32_crc32qi:
4452 return interp__builtin_ia32_crc32(S, OpPC, Frame, Call, DataBytes: 1);
4453 case clang::X86::BI__builtin_ia32_crc32hi:
4454 return interp__builtin_ia32_crc32(S, OpPC, Frame, Call, DataBytes: 2);
4455 case clang::X86::BI__builtin_ia32_crc32si:
4456 return interp__builtin_ia32_crc32(S, OpPC, Frame, Call, DataBytes: 4);
4457 case clang::X86::BI__builtin_ia32_crc32di:
4458 return interp__builtin_ia32_crc32(S, OpPC, Frame, Call, DataBytes: 8);
4459
4460 case clang::X86::BI__builtin_ia32_bextr_u32:
4461 case clang::X86::BI__builtin_ia32_bextr_u64:
4462 case clang::X86::BI__builtin_ia32_bextri_u32:
4463 case clang::X86::BI__builtin_ia32_bextri_u64:
4464 return interp__builtin_elementwise_int_binop(
4465 S, OpPC, Call, Fn: [](const APSInt &Val, const APSInt &Idx) {
4466 unsigned BitWidth = Val.getBitWidth();
4467 uint64_t Shift = Idx.extractBitsAsZExtValue(numBits: 8, bitPosition: 0);
4468 uint64_t Length = Idx.extractBitsAsZExtValue(numBits: 8, bitPosition: 8);
4469 if (Length > BitWidth) {
4470 Length = BitWidth;
4471 }
4472
4473 // Handle out of bounds cases.
4474 if (Length == 0 || Shift >= BitWidth)
4475 return APInt(BitWidth, 0);
4476
4477 uint64_t Result = Val.getZExtValue() >> Shift;
4478 Result &= llvm::maskTrailingOnes<uint64_t>(N: Length);
4479 return APInt(BitWidth, Result);
4480 });
4481
4482 case clang::X86::BI__builtin_ia32_bzhi_si:
4483 case clang::X86::BI__builtin_ia32_bzhi_di:
4484 return interp__builtin_elementwise_int_binop(
4485 S, OpPC, Call, Fn: [](const APSInt &Val, const APSInt &Idx) {
4486 unsigned BitWidth = Val.getBitWidth();
4487 uint64_t Index = Idx.extractBitsAsZExtValue(numBits: 8, bitPosition: 0);
4488 APSInt Result = Val;
4489
4490 if (Index < BitWidth)
4491 Result.clearHighBits(hiBits: BitWidth - Index);
4492
4493 return Result;
4494 });
4495
4496 case clang::X86::BI__builtin_ia32_ktestcqi:
4497 case clang::X86::BI__builtin_ia32_ktestchi:
4498 case clang::X86::BI__builtin_ia32_ktestcsi:
4499 case clang::X86::BI__builtin_ia32_ktestcdi:
4500 return interp__builtin_elementwise_int_binop(
4501 S, OpPC, Call, Fn: [](const APSInt &A, const APSInt &B) {
4502 return APInt(sizeof(unsigned char) * 8, (~A & B) == 0);
4503 });
4504
4505 case clang::X86::BI__builtin_ia32_ktestzqi:
4506 case clang::X86::BI__builtin_ia32_ktestzhi:
4507 case clang::X86::BI__builtin_ia32_ktestzsi:
4508 case clang::X86::BI__builtin_ia32_ktestzdi:
4509 return interp__builtin_elementwise_int_binop(
4510 S, OpPC, Call, Fn: [](const APSInt &A, const APSInt &B) {
4511 return APInt(sizeof(unsigned char) * 8, (A & B) == 0);
4512 });
4513
4514 case clang::X86::BI__builtin_ia32_kortestcqi:
4515 case clang::X86::BI__builtin_ia32_kortestchi:
4516 case clang::X86::BI__builtin_ia32_kortestcsi:
4517 case clang::X86::BI__builtin_ia32_kortestcdi:
4518 return interp__builtin_elementwise_int_binop(
4519 S, OpPC, Call, Fn: [](const APSInt &A, const APSInt &B) {
4520 return APInt(sizeof(unsigned char) * 8, ~(A | B) == 0);
4521 });
4522
4523 case clang::X86::BI__builtin_ia32_kortestzqi:
4524 case clang::X86::BI__builtin_ia32_kortestzhi:
4525 case clang::X86::BI__builtin_ia32_kortestzsi:
4526 case clang::X86::BI__builtin_ia32_kortestzdi:
4527 return interp__builtin_elementwise_int_binop(
4528 S, OpPC, Call, Fn: [](const APSInt &A, const APSInt &B) {
4529 return APInt(sizeof(unsigned char) * 8, (A | B) == 0);
4530 });
4531
4532 case clang::X86::BI__builtin_ia32_kshiftliqi:
4533 case clang::X86::BI__builtin_ia32_kshiftlihi:
4534 case clang::X86::BI__builtin_ia32_kshiftlisi:
4535 case clang::X86::BI__builtin_ia32_kshiftlidi:
4536 return interp__builtin_elementwise_int_binop(
4537 S, OpPC, Call, Fn: [](const APSInt &LHS, const APSInt &RHS) {
4538 unsigned Amt = RHS.getZExtValue() & 0xFF;
4539 if (Amt >= LHS.getBitWidth())
4540 return APInt::getZero(numBits: LHS.getBitWidth());
4541 return LHS.shl(shiftAmt: Amt);
4542 });
4543
4544 case clang::X86::BI__builtin_ia32_kshiftriqi:
4545 case clang::X86::BI__builtin_ia32_kshiftrihi:
4546 case clang::X86::BI__builtin_ia32_kshiftrisi:
4547 case clang::X86::BI__builtin_ia32_kshiftridi:
4548 return interp__builtin_elementwise_int_binop(
4549 S, OpPC, Call, Fn: [](const APSInt &LHS, const APSInt &RHS) {
4550 unsigned Amt = RHS.getZExtValue() & 0xFF;
4551 if (Amt >= LHS.getBitWidth())
4552 return APInt::getZero(numBits: LHS.getBitWidth());
4553 return LHS.lshr(shiftAmt: Amt);
4554 });
4555
4556 case clang::X86::BI__builtin_ia32_lzcnt_u16:
4557 case clang::X86::BI__builtin_ia32_lzcnt_u32:
4558 case clang::X86::BI__builtin_ia32_lzcnt_u64:
4559 return interp__builtin_elementwise_int_unaryop(
4560 S, OpPC, Call, Fn: [](const APSInt &Src) {
4561 return APInt(Src.getBitWidth(), Src.countLeadingZeros());
4562 });
4563
4564 case clang::X86::BI__builtin_ia32_tzcnt_u16:
4565 case clang::X86::BI__builtin_ia32_tzcnt_u32:
4566 case clang::X86::BI__builtin_ia32_tzcnt_u64:
4567 return interp__builtin_elementwise_int_unaryop(
4568 S, OpPC, Call, Fn: [](const APSInt &Src) {
4569 return APInt(Src.getBitWidth(), Src.countTrailingZeros());
4570 });
4571
4572 case clang::X86::BI__builtin_ia32_pdep_si:
4573 case clang::X86::BI__builtin_ia32_pdep_di:
4574 return interp__builtin_elementwise_int_binop(
4575 S, OpPC, Call, Fn: [](const APSInt &Val, const APSInt &Mask) {
4576 unsigned BitWidth = Val.getBitWidth();
4577 APInt Result = APInt::getZero(numBits: BitWidth);
4578
4579 for (unsigned I = 0, P = 0; I != BitWidth; ++I) {
4580 if (Mask[I])
4581 Result.setBitVal(BitPosition: I, BitValue: Val[P++]);
4582 }
4583
4584 return Result;
4585 });
4586
4587 case clang::X86::BI__builtin_ia32_pext_si:
4588 case clang::X86::BI__builtin_ia32_pext_di:
4589 return interp__builtin_elementwise_int_binop(
4590 S, OpPC, Call, Fn: [](const APSInt &Val, const APSInt &Mask) {
4591 unsigned BitWidth = Val.getBitWidth();
4592 APInt Result = APInt::getZero(numBits: BitWidth);
4593
4594 for (unsigned I = 0, P = 0; I != BitWidth; ++I) {
4595 if (Mask[I])
4596 Result.setBitVal(BitPosition: P++, BitValue: Val[I]);
4597 }
4598
4599 return Result;
4600 });
4601
4602 case clang::X86::BI__builtin_ia32_addcarryx_u32:
4603 case clang::X86::BI__builtin_ia32_addcarryx_u64:
4604 case clang::X86::BI__builtin_ia32_subborrow_u32:
4605 case clang::X86::BI__builtin_ia32_subborrow_u64:
4606 return interp__builtin_ia32_addcarry_subborrow(S, OpPC, Frame, Call,
4607 BuiltinOp: BuiltinID);
4608
4609 case Builtin::BI__builtin_os_log_format_buffer_size:
4610 return interp__builtin_os_log_format_buffer_size(S, OpPC, Frame, Call);
4611
4612 case Builtin::BI__builtin_ptrauth_string_discriminator:
4613 return interp__builtin_ptrauth_string_discriminator(S, OpPC, Frame, Call);
4614
4615 case Builtin::BI__builtin_infer_alloc_token:
4616 return interp__builtin_infer_alloc_token(S, OpPC, Frame, Call);
4617
4618 case Builtin::BI__noop:
4619 pushInteger(S, Val: 0, QT: Call->getType());
4620 return true;
4621
4622 case Builtin::BI__builtin_operator_new:
4623 return interp__builtin_operator_new(S, OpPC, Frame, Call);
4624
4625 case Builtin::BI__builtin_operator_delete:
4626 return interp__builtin_operator_delete(S, OpPC, Frame, Call);
4627
4628 case Builtin::BI__arithmetic_fence:
4629 return interp__builtin_arithmetic_fence(S, OpPC, Frame, Call);
4630
4631 case Builtin::BI__builtin_reduce_add:
4632 case Builtin::BI__builtin_reduce_mul:
4633 case Builtin::BI__builtin_reduce_and:
4634 case Builtin::BI__builtin_reduce_or:
4635 case Builtin::BI__builtin_reduce_xor:
4636 case Builtin::BI__builtin_reduce_min:
4637 case Builtin::BI__builtin_reduce_max:
4638 return interp__builtin_vector_reduce(S, OpPC, Call, ID: BuiltinID);
4639
4640 case Builtin::BI__builtin_elementwise_popcount:
4641 return interp__builtin_elementwise_int_unaryop(
4642 S, OpPC, Call, Fn: [](const APSInt &Src) {
4643 return APInt(Src.getBitWidth(), Src.popcount());
4644 });
4645 case Builtin::BI__builtin_elementwise_bitreverse:
4646 return interp__builtin_elementwise_int_unaryop(
4647 S, OpPC, Call, Fn: [](const APSInt &Src) { return Src.reverseBits(); });
4648
4649 case Builtin::BI__builtin_elementwise_abs:
4650 return interp__builtin_elementwise_abs(S, OpPC, Frame, Call, BuiltinID);
4651
4652 case Builtin::BI__builtin_memcpy:
4653 case Builtin::BImemcpy:
4654 case Builtin::BI__builtin_wmemcpy:
4655 case Builtin::BIwmemcpy:
4656 case Builtin::BI__builtin_memmove:
4657 case Builtin::BImemmove:
4658 case Builtin::BI__builtin_wmemmove:
4659 case Builtin::BIwmemmove:
4660 return interp__builtin_memcpy(S, OpPC, Frame, Call, ID: BuiltinID);
4661
4662 case Builtin::BI__builtin_memcmp:
4663 case Builtin::BImemcmp:
4664 case Builtin::BI__builtin_bcmp:
4665 case Builtin::BIbcmp:
4666 case Builtin::BI__builtin_wmemcmp:
4667 case Builtin::BIwmemcmp:
4668 return interp__builtin_memcmp(S, OpPC, Frame, Call, ID: BuiltinID);
4669
4670 case Builtin::BImemchr:
4671 case Builtin::BI__builtin_memchr:
4672 case Builtin::BIstrchr:
4673 case Builtin::BI__builtin_strchr:
4674 case Builtin::BIwmemchr:
4675 case Builtin::BI__builtin_wmemchr:
4676 case Builtin::BIwcschr:
4677 case Builtin::BI__builtin_wcschr:
4678 case Builtin::BI__builtin_char_memchr:
4679 return interp__builtin_memchr(S, OpPC, Call, ID: BuiltinID);
4680
4681 case Builtin::BI__builtin_object_size:
4682 case Builtin::BI__builtin_dynamic_object_size:
4683 return interp__builtin_object_size(S, OpPC, Frame, Call);
4684
4685 case Builtin::BI__builtin_is_within_lifetime:
4686 return interp__builtin_is_within_lifetime(S, OpPC, Call);
4687
4688 case Builtin::BI__builtin_elementwise_add_sat:
4689 return interp__builtin_elementwise_int_binop(
4690 S, OpPC, Call, Fn: [](const APSInt &LHS, const APSInt &RHS) {
4691 return LHS.isSigned() ? LHS.sadd_sat(RHS) : LHS.uadd_sat(RHS);
4692 });
4693
4694 case Builtin::BI__builtin_elementwise_sub_sat:
4695 return interp__builtin_elementwise_int_binop(
4696 S, OpPC, Call, Fn: [](const APSInt &LHS, const APSInt &RHS) {
4697 return LHS.isSigned() ? LHS.ssub_sat(RHS) : LHS.usub_sat(RHS);
4698 });
4699 case X86::BI__builtin_ia32_extract128i256:
4700 case X86::BI__builtin_ia32_vextractf128_pd256:
4701 case X86::BI__builtin_ia32_vextractf128_ps256:
4702 case X86::BI__builtin_ia32_vextractf128_si256:
4703 return interp__builtin_x86_extract_vector(S, OpPC, Call, ID: BuiltinID);
4704
4705 case X86::BI__builtin_ia32_extractf32x4_256_mask:
4706 case X86::BI__builtin_ia32_extractf32x4_mask:
4707 case X86::BI__builtin_ia32_extractf32x8_mask:
4708 case X86::BI__builtin_ia32_extractf64x2_256_mask:
4709 case X86::BI__builtin_ia32_extractf64x2_512_mask:
4710 case X86::BI__builtin_ia32_extractf64x4_mask:
4711 case X86::BI__builtin_ia32_extracti32x4_256_mask:
4712 case X86::BI__builtin_ia32_extracti32x4_mask:
4713 case X86::BI__builtin_ia32_extracti32x8_mask:
4714 case X86::BI__builtin_ia32_extracti64x2_256_mask:
4715 case X86::BI__builtin_ia32_extracti64x2_512_mask:
4716 case X86::BI__builtin_ia32_extracti64x4_mask:
4717 return interp__builtin_x86_extract_vector_masked(S, OpPC, Call, ID: BuiltinID);
4718
4719 case clang::X86::BI__builtin_ia32_pmulhrsw128:
4720 case clang::X86::BI__builtin_ia32_pmulhrsw256:
4721 case clang::X86::BI__builtin_ia32_pmulhrsw512:
4722 return interp__builtin_elementwise_int_binop(
4723 S, OpPC, Call, Fn: [](const APSInt &LHS, const APSInt &RHS) {
4724 return (llvm::APIntOps::mulsExtended(C1: LHS, C2: RHS).ashr(ShiftAmt: 14) + 1)
4725 .extractBits(numBits: 16, bitPosition: 1);
4726 });
4727
4728 case clang::X86::BI__builtin_ia32_movmskps:
4729 case clang::X86::BI__builtin_ia32_movmskpd:
4730 case clang::X86::BI__builtin_ia32_pmovmskb128:
4731 case clang::X86::BI__builtin_ia32_pmovmskb256:
4732 case clang::X86::BI__builtin_ia32_movmskps256:
4733 case clang::X86::BI__builtin_ia32_movmskpd256: {
4734 return interp__builtin_ia32_movmsk_op(S, OpPC, Call);
4735 }
4736
4737 case X86::BI__builtin_ia32_psignb128:
4738 case X86::BI__builtin_ia32_psignb256:
4739 case X86::BI__builtin_ia32_psignw128:
4740 case X86::BI__builtin_ia32_psignw256:
4741 case X86::BI__builtin_ia32_psignd128:
4742 case X86::BI__builtin_ia32_psignd256:
4743 return interp__builtin_elementwise_int_binop(
4744 S, OpPC, Call, Fn: [](const APInt &AElem, const APInt &BElem) {
4745 if (BElem.isZero())
4746 return APInt::getZero(numBits: AElem.getBitWidth());
4747 if (BElem.isNegative())
4748 return -AElem;
4749 return AElem;
4750 });
4751
4752 case clang::X86::BI__builtin_ia32_pavgb128:
4753 case clang::X86::BI__builtin_ia32_pavgw128:
4754 case clang::X86::BI__builtin_ia32_pavgb256:
4755 case clang::X86::BI__builtin_ia32_pavgw256:
4756 case clang::X86::BI__builtin_ia32_pavgb512:
4757 case clang::X86::BI__builtin_ia32_pavgw512:
4758 return interp__builtin_elementwise_int_binop(S, OpPC, Call,
4759 Fn: llvm::APIntOps::avgCeilU);
4760
4761 case clang::X86::BI__builtin_ia32_pmaddubsw128:
4762 case clang::X86::BI__builtin_ia32_pmaddubsw256:
4763 case clang::X86::BI__builtin_ia32_pmaddubsw512:
4764 return interp__builtin_ia32_pmul(
4765 S, OpPC, Call,
4766 Fn: [](const APSInt &LoLHS, const APSInt &HiLHS, const APSInt &LoRHS,
4767 const APSInt &HiRHS) {
4768 unsigned BitWidth = 2 * LoLHS.getBitWidth();
4769 return (LoLHS.zext(width: BitWidth) * LoRHS.sext(width: BitWidth))
4770 .sadd_sat(RHS: (HiLHS.zext(width: BitWidth) * HiRHS.sext(width: BitWidth)));
4771 });
4772
4773 case clang::X86::BI__builtin_ia32_pmaddwd128:
4774 case clang::X86::BI__builtin_ia32_pmaddwd256:
4775 case clang::X86::BI__builtin_ia32_pmaddwd512:
4776 return interp__builtin_ia32_pmul(
4777 S, OpPC, Call,
4778 Fn: [](const APSInt &LoLHS, const APSInt &HiLHS, const APSInt &LoRHS,
4779 const APSInt &HiRHS) {
4780 unsigned BitWidth = 2 * LoLHS.getBitWidth();
4781 return (LoLHS.sext(width: BitWidth) * LoRHS.sext(width: BitWidth)) +
4782 (HiLHS.sext(width: BitWidth) * HiRHS.sext(width: BitWidth));
4783 });
4784
4785 case clang::X86::BI__builtin_ia32_pmulhuw128:
4786 case clang::X86::BI__builtin_ia32_pmulhuw256:
4787 case clang::X86::BI__builtin_ia32_pmulhuw512:
4788 return interp__builtin_elementwise_int_binop(S, OpPC, Call,
4789 Fn: llvm::APIntOps::mulhu);
4790
4791 case clang::X86::BI__builtin_ia32_pmulhw128:
4792 case clang::X86::BI__builtin_ia32_pmulhw256:
4793 case clang::X86::BI__builtin_ia32_pmulhw512:
4794 return interp__builtin_elementwise_int_binop(S, OpPC, Call,
4795 Fn: llvm::APIntOps::mulhs);
4796
4797 case clang::X86::BI__builtin_ia32_psllv2di:
4798 case clang::X86::BI__builtin_ia32_psllv4di:
4799 case clang::X86::BI__builtin_ia32_psllv4si:
4800 case clang::X86::BI__builtin_ia32_psllv8di:
4801 case clang::X86::BI__builtin_ia32_psllv8hi:
4802 case clang::X86::BI__builtin_ia32_psllv8si:
4803 case clang::X86::BI__builtin_ia32_psllv16hi:
4804 case clang::X86::BI__builtin_ia32_psllv16si:
4805 case clang::X86::BI__builtin_ia32_psllv32hi:
4806 case clang::X86::BI__builtin_ia32_psllwi128:
4807 case clang::X86::BI__builtin_ia32_psllwi256:
4808 case clang::X86::BI__builtin_ia32_psllwi512:
4809 case clang::X86::BI__builtin_ia32_pslldi128:
4810 case clang::X86::BI__builtin_ia32_pslldi256:
4811 case clang::X86::BI__builtin_ia32_pslldi512:
4812 case clang::X86::BI__builtin_ia32_psllqi128:
4813 case clang::X86::BI__builtin_ia32_psllqi256:
4814 case clang::X86::BI__builtin_ia32_psllqi512:
4815 return interp__builtin_elementwise_int_binop(
4816 S, OpPC, Call, Fn: [](const APSInt &LHS, const APSInt &RHS) {
4817 if (RHS.uge(RHS: LHS.getBitWidth())) {
4818 return APInt::getZero(numBits: LHS.getBitWidth());
4819 }
4820 return LHS.shl(shiftAmt: RHS.getZExtValue());
4821 });
4822
4823 case clang::X86::BI__builtin_ia32_psrav4si:
4824 case clang::X86::BI__builtin_ia32_psrav8di:
4825 case clang::X86::BI__builtin_ia32_psrav8hi:
4826 case clang::X86::BI__builtin_ia32_psrav8si:
4827 case clang::X86::BI__builtin_ia32_psrav16hi:
4828 case clang::X86::BI__builtin_ia32_psrav16si:
4829 case clang::X86::BI__builtin_ia32_psrav32hi:
4830 case clang::X86::BI__builtin_ia32_psravq128:
4831 case clang::X86::BI__builtin_ia32_psravq256:
4832 case clang::X86::BI__builtin_ia32_psrawi128:
4833 case clang::X86::BI__builtin_ia32_psrawi256:
4834 case clang::X86::BI__builtin_ia32_psrawi512:
4835 case clang::X86::BI__builtin_ia32_psradi128:
4836 case clang::X86::BI__builtin_ia32_psradi256:
4837 case clang::X86::BI__builtin_ia32_psradi512:
4838 case clang::X86::BI__builtin_ia32_psraqi128:
4839 case clang::X86::BI__builtin_ia32_psraqi256:
4840 case clang::X86::BI__builtin_ia32_psraqi512:
4841 return interp__builtin_elementwise_int_binop(
4842 S, OpPC, Call, Fn: [](const APSInt &LHS, const APSInt &RHS) {
4843 if (RHS.uge(RHS: LHS.getBitWidth())) {
4844 return LHS.ashr(ShiftAmt: LHS.getBitWidth() - 1);
4845 }
4846 return LHS.ashr(ShiftAmt: RHS.getZExtValue());
4847 });
4848
4849 case clang::X86::BI__builtin_ia32_psrlv2di:
4850 case clang::X86::BI__builtin_ia32_psrlv4di:
4851 case clang::X86::BI__builtin_ia32_psrlv4si:
4852 case clang::X86::BI__builtin_ia32_psrlv8di:
4853 case clang::X86::BI__builtin_ia32_psrlv8hi:
4854 case clang::X86::BI__builtin_ia32_psrlv8si:
4855 case clang::X86::BI__builtin_ia32_psrlv16hi:
4856 case clang::X86::BI__builtin_ia32_psrlv16si:
4857 case clang::X86::BI__builtin_ia32_psrlv32hi:
4858 case clang::X86::BI__builtin_ia32_psrlwi128:
4859 case clang::X86::BI__builtin_ia32_psrlwi256:
4860 case clang::X86::BI__builtin_ia32_psrlwi512:
4861 case clang::X86::BI__builtin_ia32_psrldi128:
4862 case clang::X86::BI__builtin_ia32_psrldi256:
4863 case clang::X86::BI__builtin_ia32_psrldi512:
4864 case clang::X86::BI__builtin_ia32_psrlqi128:
4865 case clang::X86::BI__builtin_ia32_psrlqi256:
4866 case clang::X86::BI__builtin_ia32_psrlqi512:
4867 return interp__builtin_elementwise_int_binop(
4868 S, OpPC, Call, Fn: [](const APSInt &LHS, const APSInt &RHS) {
4869 if (RHS.uge(RHS: LHS.getBitWidth())) {
4870 return APInt::getZero(numBits: LHS.getBitWidth());
4871 }
4872 return LHS.lshr(shiftAmt: RHS.getZExtValue());
4873 });
4874 case clang::X86::BI__builtin_ia32_packsswb128:
4875 case clang::X86::BI__builtin_ia32_packsswb256:
4876 case clang::X86::BI__builtin_ia32_packsswb512:
4877 case clang::X86::BI__builtin_ia32_packssdw128:
4878 case clang::X86::BI__builtin_ia32_packssdw256:
4879 case clang::X86::BI__builtin_ia32_packssdw512:
4880 return interp__builtin_x86_pack(S, OpPC, E: Call, PackFn: [](const APSInt &Src) {
4881 return APInt(Src).truncSSat(width: Src.getBitWidth() / 2);
4882 });
4883 case clang::X86::BI__builtin_ia32_packusdw128:
4884 case clang::X86::BI__builtin_ia32_packusdw256:
4885 case clang::X86::BI__builtin_ia32_packusdw512:
4886 case clang::X86::BI__builtin_ia32_packuswb128:
4887 case clang::X86::BI__builtin_ia32_packuswb256:
4888 case clang::X86::BI__builtin_ia32_packuswb512:
4889 return interp__builtin_x86_pack(S, OpPC, E: Call, PackFn: [](const APSInt &Src) {
4890 return APInt(Src).truncSSatU(width: Src.getBitWidth() / 2);
4891 });
4892
4893 case clang::X86::BI__builtin_ia32_selectss_128:
4894 case clang::X86::BI__builtin_ia32_selectsd_128:
4895 case clang::X86::BI__builtin_ia32_selectsh_128:
4896 case clang::X86::BI__builtin_ia32_selectsbf_128:
4897 return interp__builtin_select_scalar(S, Call);
4898 case clang::X86::BI__builtin_ia32_vprotbi:
4899 case clang::X86::BI__builtin_ia32_vprotdi:
4900 case clang::X86::BI__builtin_ia32_vprotqi:
4901 case clang::X86::BI__builtin_ia32_vprotwi:
4902 case clang::X86::BI__builtin_ia32_prold128:
4903 case clang::X86::BI__builtin_ia32_prold256:
4904 case clang::X86::BI__builtin_ia32_prold512:
4905 case clang::X86::BI__builtin_ia32_prolq128:
4906 case clang::X86::BI__builtin_ia32_prolq256:
4907 case clang::X86::BI__builtin_ia32_prolq512:
4908 return interp__builtin_elementwise_int_binop(
4909 S, OpPC, Call,
4910 Fn: [](const APSInt &LHS, const APSInt &RHS) { return LHS.rotl(rotateAmt: RHS); });
4911
4912 case clang::X86::BI__builtin_ia32_prord128:
4913 case clang::X86::BI__builtin_ia32_prord256:
4914 case clang::X86::BI__builtin_ia32_prord512:
4915 case clang::X86::BI__builtin_ia32_prorq128:
4916 case clang::X86::BI__builtin_ia32_prorq256:
4917 case clang::X86::BI__builtin_ia32_prorq512:
4918 return interp__builtin_elementwise_int_binop(
4919 S, OpPC, Call,
4920 Fn: [](const APSInt &LHS, const APSInt &RHS) { return LHS.rotr(rotateAmt: RHS); });
4921
4922 case Builtin::BI__builtin_elementwise_max:
4923 case Builtin::BI__builtin_elementwise_min:
4924 return interp__builtin_elementwise_maxmin(S, OpPC, Call, BuiltinID);
4925
4926 case clang::X86::BI__builtin_ia32_phaddw128:
4927 case clang::X86::BI__builtin_ia32_phaddw256:
4928 case clang::X86::BI__builtin_ia32_phaddd128:
4929 case clang::X86::BI__builtin_ia32_phaddd256:
4930 return interp_builtin_horizontal_int_binop(
4931 S, OpPC, Call,
4932 Fn: [](const APSInt &LHS, const APSInt &RHS) { return LHS + RHS; });
4933 case clang::X86::BI__builtin_ia32_phaddsw128:
4934 case clang::X86::BI__builtin_ia32_phaddsw256:
4935 return interp_builtin_horizontal_int_binop(
4936 S, OpPC, Call,
4937 Fn: [](const APSInt &LHS, const APSInt &RHS) { return LHS.sadd_sat(RHS); });
4938 case clang::X86::BI__builtin_ia32_phsubw128:
4939 case clang::X86::BI__builtin_ia32_phsubw256:
4940 case clang::X86::BI__builtin_ia32_phsubd128:
4941 case clang::X86::BI__builtin_ia32_phsubd256:
4942 return interp_builtin_horizontal_int_binop(
4943 S, OpPC, Call,
4944 Fn: [](const APSInt &LHS, const APSInt &RHS) { return LHS - RHS; });
4945 case clang::X86::BI__builtin_ia32_phsubsw128:
4946 case clang::X86::BI__builtin_ia32_phsubsw256:
4947 return interp_builtin_horizontal_int_binop(
4948 S, OpPC, Call,
4949 Fn: [](const APSInt &LHS, const APSInt &RHS) { return LHS.ssub_sat(RHS); });
4950 case clang::X86::BI__builtin_ia32_haddpd:
4951 case clang::X86::BI__builtin_ia32_haddps:
4952 case clang::X86::BI__builtin_ia32_haddpd256:
4953 case clang::X86::BI__builtin_ia32_haddps256:
4954 return interp_builtin_horizontal_fp_binop(
4955 S, OpPC, Call,
4956 Fn: [](const APFloat &LHS, const APFloat &RHS, llvm::RoundingMode RM) {
4957 APFloat F = LHS;
4958 F.add(RHS, RM);
4959 return F;
4960 });
4961 case clang::X86::BI__builtin_ia32_hsubpd:
4962 case clang::X86::BI__builtin_ia32_hsubps:
4963 case clang::X86::BI__builtin_ia32_hsubpd256:
4964 case clang::X86::BI__builtin_ia32_hsubps256:
4965 return interp_builtin_horizontal_fp_binop(
4966 S, OpPC, Call,
4967 Fn: [](const APFloat &LHS, const APFloat &RHS, llvm::RoundingMode RM) {
4968 APFloat F = LHS;
4969 F.subtract(RHS, RM);
4970 return F;
4971 });
4972 case clang::X86::BI__builtin_ia32_addsubpd:
4973 case clang::X86::BI__builtin_ia32_addsubps:
4974 case clang::X86::BI__builtin_ia32_addsubpd256:
4975 case clang::X86::BI__builtin_ia32_addsubps256:
4976 return interp__builtin_ia32_addsub(S, OpPC, Call);
4977
4978 case clang::X86::BI__builtin_ia32_pmuldq128:
4979 case clang::X86::BI__builtin_ia32_pmuldq256:
4980 case clang::X86::BI__builtin_ia32_pmuldq512:
4981 return interp__builtin_ia32_pmul(
4982 S, OpPC, Call,
4983 Fn: [](const APSInt &LoLHS, const APSInt &HiLHS, const APSInt &LoRHS,
4984 const APSInt &HiRHS) {
4985 return llvm::APIntOps::mulsExtended(C1: LoLHS, C2: LoRHS);
4986 });
4987
4988 case clang::X86::BI__builtin_ia32_pmuludq128:
4989 case clang::X86::BI__builtin_ia32_pmuludq256:
4990 case clang::X86::BI__builtin_ia32_pmuludq512:
4991 return interp__builtin_ia32_pmul(
4992 S, OpPC, Call,
4993 Fn: [](const APSInt &LoLHS, const APSInt &HiLHS, const APSInt &LoRHS,
4994 const APSInt &HiRHS) {
4995 return llvm::APIntOps::muluExtended(C1: LoLHS, C2: LoRHS);
4996 });
4997
4998 case clang::X86::BI__builtin_ia32_pclmulqdq128:
4999 case clang::X86::BI__builtin_ia32_pclmulqdq256:
5000 case clang::X86::BI__builtin_ia32_pclmulqdq512:
5001 return interp__builtin_ia32_pclmulqdq(S, OpPC, Call);
5002
5003 case Builtin::BI__builtin_elementwise_fma:
5004 return interp__builtin_elementwise_triop_fp(
5005 S, OpPC, Call,
5006 Fn: [](const APFloat &X, const APFloat &Y, const APFloat &Z,
5007 llvm::RoundingMode RM) {
5008 APFloat F = X;
5009 F.fusedMultiplyAdd(Multiplicand: Y, Addend: Z, RM);
5010 return F;
5011 });
5012
5013 case X86::BI__builtin_ia32_vpmadd52luq128:
5014 case X86::BI__builtin_ia32_vpmadd52luq256:
5015 case X86::BI__builtin_ia32_vpmadd52luq512:
5016 return interp__builtin_elementwise_triop(
5017 S, OpPC, Call, Fn: [](const APSInt &A, const APSInt &B, const APSInt &C) {
5018 return A + (B.trunc(width: 52) * C.trunc(width: 52)).zext(width: 64);
5019 });
5020 case X86::BI__builtin_ia32_vpmadd52huq128:
5021 case X86::BI__builtin_ia32_vpmadd52huq256:
5022 case X86::BI__builtin_ia32_vpmadd52huq512:
5023 return interp__builtin_elementwise_triop(
5024 S, OpPC, Call, Fn: [](const APSInt &A, const APSInt &B, const APSInt &C) {
5025 return A + llvm::APIntOps::mulhu(C1: B.trunc(width: 52), C2: C.trunc(width: 52)).zext(width: 64);
5026 });
5027
5028 case X86::BI__builtin_ia32_vpshldd128:
5029 case X86::BI__builtin_ia32_vpshldd256:
5030 case X86::BI__builtin_ia32_vpshldd512:
5031 case X86::BI__builtin_ia32_vpshldq128:
5032 case X86::BI__builtin_ia32_vpshldq256:
5033 case X86::BI__builtin_ia32_vpshldq512:
5034 case X86::BI__builtin_ia32_vpshldw128:
5035 case X86::BI__builtin_ia32_vpshldw256:
5036 case X86::BI__builtin_ia32_vpshldw512:
5037 return interp__builtin_elementwise_triop(
5038 S, OpPC, Call,
5039 Fn: [](const APSInt &Hi, const APSInt &Lo, const APSInt &Amt) {
5040 return llvm::APIntOps::fshl(Hi, Lo, Shift: Amt);
5041 });
5042
5043 case X86::BI__builtin_ia32_vpshrdd128:
5044 case X86::BI__builtin_ia32_vpshrdd256:
5045 case X86::BI__builtin_ia32_vpshrdd512:
5046 case X86::BI__builtin_ia32_vpshrdq128:
5047 case X86::BI__builtin_ia32_vpshrdq256:
5048 case X86::BI__builtin_ia32_vpshrdq512:
5049 case X86::BI__builtin_ia32_vpshrdw128:
5050 case X86::BI__builtin_ia32_vpshrdw256:
5051 case X86::BI__builtin_ia32_vpshrdw512:
5052 // NOTE: Reversed Hi/Lo operands.
5053 return interp__builtin_elementwise_triop(
5054 S, OpPC, Call,
5055 Fn: [](const APSInt &Lo, const APSInt &Hi, const APSInt &Amt) {
5056 return llvm::APIntOps::fshr(Hi, Lo, Shift: Amt);
5057 });
5058 case X86::BI__builtin_ia32_vpconflictsi_128:
5059 case X86::BI__builtin_ia32_vpconflictsi_256:
5060 case X86::BI__builtin_ia32_vpconflictsi_512:
5061 case X86::BI__builtin_ia32_vpconflictdi_128:
5062 case X86::BI__builtin_ia32_vpconflictdi_256:
5063 case X86::BI__builtin_ia32_vpconflictdi_512:
5064 return interp__builtin_ia32_vpconflict(S, OpPC, Call);
5065 case clang::X86::BI__builtin_ia32_blendpd:
5066 case clang::X86::BI__builtin_ia32_blendpd256:
5067 case clang::X86::BI__builtin_ia32_blendps:
5068 case clang::X86::BI__builtin_ia32_blendps256:
5069 case clang::X86::BI__builtin_ia32_pblendw128:
5070 case clang::X86::BI__builtin_ia32_pblendw256:
5071 case clang::X86::BI__builtin_ia32_pblendd128:
5072 case clang::X86::BI__builtin_ia32_pblendd256:
5073 return interp__builtin_ia32_shuffle_generic(
5074 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
5075 // Bit index for mask.
5076 unsigned MaskBit = (ShuffleMask >> (DstIdx % 8)) & 0x1;
5077 unsigned SrcVecIdx = MaskBit ? 1 : 0; // 1 = TrueVec, 0 = FalseVec
5078 return std::pair<unsigned, int>{SrcVecIdx, static_cast<int>(DstIdx)};
5079 });
5080
5081
5082
5083 case clang::X86::BI__builtin_ia32_blendvpd:
5084 case clang::X86::BI__builtin_ia32_blendvpd256:
5085 case clang::X86::BI__builtin_ia32_blendvps:
5086 case clang::X86::BI__builtin_ia32_blendvps256:
5087 return interp__builtin_elementwise_triop_fp(
5088 S, OpPC, Call,
5089 Fn: [](const APFloat &F, const APFloat &T, const APFloat &C,
5090 llvm::RoundingMode) { return C.isNegative() ? T : F; });
5091
5092 case clang::X86::BI__builtin_ia32_pblendvb128:
5093 case clang::X86::BI__builtin_ia32_pblendvb256:
5094 return interp__builtin_elementwise_triop(
5095 S, OpPC, Call, Fn: [](const APSInt &F, const APSInt &T, const APSInt &C) {
5096 return ((APInt)C).isNegative() ? T : F;
5097 });
5098 case X86::BI__builtin_ia32_ptestz128:
5099 case X86::BI__builtin_ia32_ptestz256:
5100 case X86::BI__builtin_ia32_vtestzps:
5101 case X86::BI__builtin_ia32_vtestzps256:
5102 case X86::BI__builtin_ia32_vtestzpd:
5103 case X86::BI__builtin_ia32_vtestzpd256:
5104 return interp__builtin_ia32_test_op(
5105 S, OpPC, Call,
5106 Fn: [](const APInt &A, const APInt &B) { return (A & B) == 0; });
5107 case X86::BI__builtin_ia32_ptestc128:
5108 case X86::BI__builtin_ia32_ptestc256:
5109 case X86::BI__builtin_ia32_vtestcps:
5110 case X86::BI__builtin_ia32_vtestcps256:
5111 case X86::BI__builtin_ia32_vtestcpd:
5112 case X86::BI__builtin_ia32_vtestcpd256:
5113 return interp__builtin_ia32_test_op(
5114 S, OpPC, Call,
5115 Fn: [](const APInt &A, const APInt &B) { return (~A & B) == 0; });
5116 case X86::BI__builtin_ia32_ptestnzc128:
5117 case X86::BI__builtin_ia32_ptestnzc256:
5118 case X86::BI__builtin_ia32_vtestnzcps:
5119 case X86::BI__builtin_ia32_vtestnzcps256:
5120 case X86::BI__builtin_ia32_vtestnzcpd:
5121 case X86::BI__builtin_ia32_vtestnzcpd256:
5122 return interp__builtin_ia32_test_op(
5123 S, OpPC, Call, Fn: [](const APInt &A, const APInt &B) {
5124 return ((A & B) != 0) && ((~A & B) != 0);
5125 });
5126 case X86::BI__builtin_ia32_selectb_128:
5127 case X86::BI__builtin_ia32_selectb_256:
5128 case X86::BI__builtin_ia32_selectb_512:
5129 case X86::BI__builtin_ia32_selectw_128:
5130 case X86::BI__builtin_ia32_selectw_256:
5131 case X86::BI__builtin_ia32_selectw_512:
5132 case X86::BI__builtin_ia32_selectd_128:
5133 case X86::BI__builtin_ia32_selectd_256:
5134 case X86::BI__builtin_ia32_selectd_512:
5135 case X86::BI__builtin_ia32_selectq_128:
5136 case X86::BI__builtin_ia32_selectq_256:
5137 case X86::BI__builtin_ia32_selectq_512:
5138 case X86::BI__builtin_ia32_selectph_128:
5139 case X86::BI__builtin_ia32_selectph_256:
5140 case X86::BI__builtin_ia32_selectph_512:
5141 case X86::BI__builtin_ia32_selectpbf_128:
5142 case X86::BI__builtin_ia32_selectpbf_256:
5143 case X86::BI__builtin_ia32_selectpbf_512:
5144 case X86::BI__builtin_ia32_selectps_128:
5145 case X86::BI__builtin_ia32_selectps_256:
5146 case X86::BI__builtin_ia32_selectps_512:
5147 case X86::BI__builtin_ia32_selectpd_128:
5148 case X86::BI__builtin_ia32_selectpd_256:
5149 case X86::BI__builtin_ia32_selectpd_512:
5150 return interp__builtin_select(S, OpPC, Call);
5151
5152 case X86::BI__builtin_ia32_shufps:
5153 case X86::BI__builtin_ia32_shufps256:
5154 case X86::BI__builtin_ia32_shufps512:
5155 return interp__builtin_ia32_shuffle_generic(
5156 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
5157 unsigned NumElemPerLane = 4;
5158 unsigned NumSelectableElems = NumElemPerLane / 2;
5159 unsigned BitsPerElem = 2;
5160 unsigned IndexMask = 0x3;
5161 unsigned MaskBits = 8;
5162 unsigned Lane = DstIdx / NumElemPerLane;
5163 unsigned ElemInLane = DstIdx % NumElemPerLane;
5164 unsigned LaneOffset = Lane * NumElemPerLane;
5165 unsigned SrcIdx = ElemInLane >= NumSelectableElems ? 1 : 0;
5166 unsigned BitIndex = (DstIdx * BitsPerElem) % MaskBits;
5167 unsigned Index = (ShuffleMask >> BitIndex) & IndexMask;
5168 return std::pair<unsigned, int>{SrcIdx,
5169 static_cast<int>(LaneOffset + Index)};
5170 });
5171 case X86::BI__builtin_ia32_shufpd:
5172 case X86::BI__builtin_ia32_shufpd256:
5173 case X86::BI__builtin_ia32_shufpd512:
5174 return interp__builtin_ia32_shuffle_generic(
5175 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
5176 unsigned NumElemPerLane = 2;
5177 unsigned NumSelectableElems = NumElemPerLane / 2;
5178 unsigned BitsPerElem = 1;
5179 unsigned IndexMask = 0x1;
5180 unsigned MaskBits = 8;
5181 unsigned Lane = DstIdx / NumElemPerLane;
5182 unsigned ElemInLane = DstIdx % NumElemPerLane;
5183 unsigned LaneOffset = Lane * NumElemPerLane;
5184 unsigned SrcIdx = ElemInLane >= NumSelectableElems ? 1 : 0;
5185 unsigned BitIndex = (DstIdx * BitsPerElem) % MaskBits;
5186 unsigned Index = (ShuffleMask >> BitIndex) & IndexMask;
5187 return std::pair<unsigned, int>{SrcIdx,
5188 static_cast<int>(LaneOffset + Index)};
5189 });
5190
5191 case X86::BI__builtin_ia32_vgf2p8affineinvqb_v16qi:
5192 case X86::BI__builtin_ia32_vgf2p8affineinvqb_v32qi:
5193 case X86::BI__builtin_ia32_vgf2p8affineinvqb_v64qi:
5194 return interp_builtin_ia32_gfni_affine(S, OpPC, Call, Inverse: true);
5195 case X86::BI__builtin_ia32_vgf2p8affineqb_v16qi:
5196 case X86::BI__builtin_ia32_vgf2p8affineqb_v32qi:
5197 case X86::BI__builtin_ia32_vgf2p8affineqb_v64qi:
5198 return interp_builtin_ia32_gfni_affine(S, OpPC, Call, Inverse: false);
5199
5200 case X86::BI__builtin_ia32_vgf2p8mulb_v16qi:
5201 case X86::BI__builtin_ia32_vgf2p8mulb_v32qi:
5202 case X86::BI__builtin_ia32_vgf2p8mulb_v64qi:
5203 return interp__builtin_ia32_gfni_mul(S, OpPC, Call);
5204
5205 case X86::BI__builtin_ia32_insertps128:
5206 return interp__builtin_ia32_shuffle_generic(
5207 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned Mask) {
5208 // Bits [3:0]: zero mask - if bit is set, zero this element
5209 if ((Mask & (1 << DstIdx)) != 0) {
5210 return std::pair<unsigned, int>{0, -1};
5211 }
5212 // Bits [7:6]: select element from source vector Y (0-3)
5213 // Bits [5:4]: select destination position (0-3)
5214 unsigned SrcElem = (Mask >> 6) & 0x3;
5215 unsigned DstElem = (Mask >> 4) & 0x3;
5216 if (DstIdx == DstElem) {
5217 // Insert element from source vector (B) at this position
5218 return std::pair<unsigned, int>{1, static_cast<int>(SrcElem)};
5219 } else {
5220 // Copy from destination vector (A)
5221 return std::pair<unsigned, int>{0, static_cast<int>(DstIdx)};
5222 }
5223 });
5224 case X86::BI__builtin_ia32_permvarsi256:
5225 case X86::BI__builtin_ia32_permvarsf256:
5226 case X86::BI__builtin_ia32_permvardf512:
5227 case X86::BI__builtin_ia32_permvardi512:
5228 case X86::BI__builtin_ia32_permvarhi128:
5229 return interp__builtin_ia32_shuffle_generic(
5230 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
5231 int Offset = ShuffleMask & 0x7;
5232 return std::pair<unsigned, int>{0, Offset};
5233 });
5234 case X86::BI__builtin_ia32_permvarqi128:
5235 case X86::BI__builtin_ia32_permvarhi256:
5236 case X86::BI__builtin_ia32_permvarsi512:
5237 case X86::BI__builtin_ia32_permvarsf512:
5238 return interp__builtin_ia32_shuffle_generic(
5239 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
5240 int Offset = ShuffleMask & 0xF;
5241 return std::pair<unsigned, int>{0, Offset};
5242 });
5243 case X86::BI__builtin_ia32_permvardi256:
5244 case X86::BI__builtin_ia32_permvardf256:
5245 return interp__builtin_ia32_shuffle_generic(
5246 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
5247 int Offset = ShuffleMask & 0x3;
5248 return std::pair<unsigned, int>{0, Offset};
5249 });
5250 case X86::BI__builtin_ia32_permvarqi256:
5251 case X86::BI__builtin_ia32_permvarhi512:
5252 return interp__builtin_ia32_shuffle_generic(
5253 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
5254 int Offset = ShuffleMask & 0x1F;
5255 return std::pair<unsigned, int>{0, Offset};
5256 });
5257 case X86::BI__builtin_ia32_permvarqi512:
5258 return interp__builtin_ia32_shuffle_generic(
5259 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
5260 int Offset = ShuffleMask & 0x3F;
5261 return std::pair<unsigned, int>{0, Offset};
5262 });
5263 case X86::BI__builtin_ia32_vpermi2varq128:
5264 case X86::BI__builtin_ia32_vpermi2varpd128:
5265 return interp__builtin_ia32_shuffle_generic(
5266 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
5267 int Offset = ShuffleMask & 0x1;
5268 unsigned SrcIdx = (ShuffleMask >> 1) & 0x1;
5269 return std::pair<unsigned, int>{SrcIdx, Offset};
5270 });
5271 case X86::BI__builtin_ia32_vpermi2vard128:
5272 case X86::BI__builtin_ia32_vpermi2varps128:
5273 case X86::BI__builtin_ia32_vpermi2varq256:
5274 case X86::BI__builtin_ia32_vpermi2varpd256:
5275 return interp__builtin_ia32_shuffle_generic(
5276 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
5277 int Offset = ShuffleMask & 0x3;
5278 unsigned SrcIdx = (ShuffleMask >> 2) & 0x1;
5279 return std::pair<unsigned, int>{SrcIdx, Offset};
5280 });
5281 case X86::BI__builtin_ia32_vpermi2varhi128:
5282 case X86::BI__builtin_ia32_vpermi2vard256:
5283 case X86::BI__builtin_ia32_vpermi2varps256:
5284 case X86::BI__builtin_ia32_vpermi2varq512:
5285 case X86::BI__builtin_ia32_vpermi2varpd512:
5286 return interp__builtin_ia32_shuffle_generic(
5287 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
5288 int Offset = ShuffleMask & 0x7;
5289 unsigned SrcIdx = (ShuffleMask >> 3) & 0x1;
5290 return std::pair<unsigned, int>{SrcIdx, Offset};
5291 });
5292 case X86::BI__builtin_ia32_vpermi2varqi128:
5293 case X86::BI__builtin_ia32_vpermi2varhi256:
5294 case X86::BI__builtin_ia32_vpermi2vard512:
5295 case X86::BI__builtin_ia32_vpermi2varps512:
5296 return interp__builtin_ia32_shuffle_generic(
5297 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
5298 int Offset = ShuffleMask & 0xF;
5299 unsigned SrcIdx = (ShuffleMask >> 4) & 0x1;
5300 return std::pair<unsigned, int>{SrcIdx, Offset};
5301 });
5302 case X86::BI__builtin_ia32_vpermi2varqi256:
5303 case X86::BI__builtin_ia32_vpermi2varhi512:
5304 return interp__builtin_ia32_shuffle_generic(
5305 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
5306 int Offset = ShuffleMask & 0x1F;
5307 unsigned SrcIdx = (ShuffleMask >> 5) & 0x1;
5308 return std::pair<unsigned, int>{SrcIdx, Offset};
5309 });
5310 case X86::BI__builtin_ia32_vpermi2varqi512:
5311 return interp__builtin_ia32_shuffle_generic(
5312 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
5313 int Offset = ShuffleMask & 0x3F;
5314 unsigned SrcIdx = (ShuffleMask >> 6) & 0x1;
5315 return std::pair<unsigned, int>{SrcIdx, Offset};
5316 });
5317 case X86::BI__builtin_ia32_vperm2f128_pd256:
5318 case X86::BI__builtin_ia32_vperm2f128_ps256:
5319 case X86::BI__builtin_ia32_vperm2f128_si256:
5320 case X86::BI__builtin_ia32_permti256: {
5321 unsigned NumElements =
5322 Call->getArg(Arg: 0)->getType()->castAs<VectorType>()->getNumElements();
5323 unsigned PreservedBitsCnt = NumElements >> 2;
5324 return interp__builtin_ia32_shuffle_generic(
5325 S, OpPC, Call,
5326 GetSourceIndex: [PreservedBitsCnt](unsigned DstIdx, unsigned ShuffleMask) {
5327 unsigned ControlBitsCnt = DstIdx >> PreservedBitsCnt << 2;
5328 unsigned ControlBits = ShuffleMask >> ControlBitsCnt;
5329
5330 if (ControlBits & 0b1000)
5331 return std::make_pair(x: 0u, y: -1);
5332
5333 unsigned SrcVecIdx = (ControlBits & 0b10) >> 1;
5334 unsigned PreservedBitsMask = (1 << PreservedBitsCnt) - 1;
5335 int SrcIdx = ((ControlBits & 0b1) << PreservedBitsCnt) |
5336 (DstIdx & PreservedBitsMask);
5337 return std::make_pair(x&: SrcVecIdx, y&: SrcIdx);
5338 });
5339 }
5340 case X86::BI__builtin_ia32_pshufb128:
5341 case X86::BI__builtin_ia32_pshufb256:
5342 case X86::BI__builtin_ia32_pshufb512:
5343 return interp__builtin_ia32_shuffle_generic(
5344 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
5345 uint8_t Ctlb = static_cast<uint8_t>(ShuffleMask);
5346 if (Ctlb & 0x80)
5347 return std::make_pair(x: 0, y: -1);
5348
5349 unsigned LaneBase = (DstIdx / 16) * 16;
5350 unsigned SrcOffset = Ctlb & 0x0F;
5351 unsigned SrcIdx = LaneBase + SrcOffset;
5352 return std::make_pair(x: 0, y: static_cast<int>(SrcIdx));
5353 });
5354
5355 case X86::BI__builtin_ia32_pshuflw:
5356 case X86::BI__builtin_ia32_pshuflw256:
5357 case X86::BI__builtin_ia32_pshuflw512:
5358 return interp__builtin_ia32_shuffle_generic(
5359 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
5360 unsigned LaneBase = (DstIdx / 8) * 8;
5361 unsigned LaneIdx = DstIdx % 8;
5362 if (LaneIdx < 4) {
5363 unsigned Sel = (ShuffleMask >> (2 * LaneIdx)) & 0x3;
5364 return std::make_pair(x: 0, y: static_cast<int>(LaneBase + Sel));
5365 }
5366
5367 return std::make_pair(x: 0, y: static_cast<int>(DstIdx));
5368 });
5369
5370 case X86::BI__builtin_ia32_pshufhw:
5371 case X86::BI__builtin_ia32_pshufhw256:
5372 case X86::BI__builtin_ia32_pshufhw512:
5373 return interp__builtin_ia32_shuffle_generic(
5374 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
5375 unsigned LaneBase = (DstIdx / 8) * 8;
5376 unsigned LaneIdx = DstIdx % 8;
5377 if (LaneIdx >= 4) {
5378 unsigned Sel = (ShuffleMask >> (2 * (LaneIdx - 4))) & 0x3;
5379 return std::make_pair(x: 0, y: static_cast<int>(LaneBase + 4 + Sel));
5380 }
5381
5382 return std::make_pair(x: 0, y: static_cast<int>(DstIdx));
5383 });
5384
5385 case X86::BI__builtin_ia32_pshufd:
5386 case X86::BI__builtin_ia32_pshufd256:
5387 case X86::BI__builtin_ia32_pshufd512:
5388 case X86::BI__builtin_ia32_vpermilps:
5389 case X86::BI__builtin_ia32_vpermilps256:
5390 case X86::BI__builtin_ia32_vpermilps512:
5391 return interp__builtin_ia32_shuffle_generic(
5392 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
5393 unsigned LaneBase = (DstIdx / 4) * 4;
5394 unsigned LaneIdx = DstIdx % 4;
5395 unsigned Sel = (ShuffleMask >> (2 * LaneIdx)) & 0x3;
5396 return std::make_pair(x: 0, y: static_cast<int>(LaneBase + Sel));
5397 });
5398
5399 case X86::BI__builtin_ia32_vpermilvarpd:
5400 case X86::BI__builtin_ia32_vpermilvarpd256:
5401 case X86::BI__builtin_ia32_vpermilvarpd512:
5402 return interp__builtin_ia32_shuffle_generic(
5403 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
5404 unsigned NumElemPerLane = 2;
5405 unsigned Lane = DstIdx / NumElemPerLane;
5406 unsigned Offset = ShuffleMask & 0b10 ? 1 : 0;
5407 return std::make_pair(
5408 x: 0, y: static_cast<int>(Lane * NumElemPerLane + Offset));
5409 });
5410
5411 case X86::BI__builtin_ia32_vpermilvarps:
5412 case X86::BI__builtin_ia32_vpermilvarps256:
5413 case X86::BI__builtin_ia32_vpermilvarps512:
5414 return interp__builtin_ia32_shuffle_generic(
5415 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned ShuffleMask) {
5416 unsigned NumElemPerLane = 4;
5417 unsigned Lane = DstIdx / NumElemPerLane;
5418 unsigned Offset = ShuffleMask & 0b11;
5419 return std::make_pair(
5420 x: 0, y: static_cast<int>(Lane * NumElemPerLane + Offset));
5421 });
5422
5423 case X86::BI__builtin_ia32_vpermilpd:
5424 case X86::BI__builtin_ia32_vpermilpd256:
5425 case X86::BI__builtin_ia32_vpermilpd512:
5426 return interp__builtin_ia32_shuffle_generic(
5427 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned Control) {
5428 unsigned NumElemPerLane = 2;
5429 unsigned BitsPerElem = 1;
5430 unsigned MaskBits = 8;
5431 unsigned IndexMask = 0x1;
5432 unsigned Lane = DstIdx / NumElemPerLane;
5433 unsigned LaneOffset = Lane * NumElemPerLane;
5434 unsigned BitIndex = (DstIdx * BitsPerElem) % MaskBits;
5435 unsigned Index = (Control >> BitIndex) & IndexMask;
5436 return std::make_pair(x: 0, y: static_cast<int>(LaneOffset + Index));
5437 });
5438
5439 case X86::BI__builtin_ia32_permdf256:
5440 case X86::BI__builtin_ia32_permdi256:
5441 return interp__builtin_ia32_shuffle_generic(
5442 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned Control) {
5443 // permute4x64 operates on 4 64-bit elements
5444 // For element i (0-3), extract bits [2*i+1:2*i] from Control
5445 unsigned Index = (Control >> (2 * DstIdx)) & 0x3;
5446 return std::make_pair(x: 0, y: static_cast<int>(Index));
5447 });
5448
5449 case X86::BI__builtin_ia32_vpmultishiftqb128:
5450 case X86::BI__builtin_ia32_vpmultishiftqb256:
5451 case X86::BI__builtin_ia32_vpmultishiftqb512:
5452 return interp__builtin_ia32_multishiftqb(S, OpPC, Call);
5453 case X86::BI__builtin_ia32_kandqi:
5454 case X86::BI__builtin_ia32_kandhi:
5455 case X86::BI__builtin_ia32_kandsi:
5456 case X86::BI__builtin_ia32_kanddi:
5457 return interp__builtin_elementwise_int_binop(
5458 S, OpPC, Call,
5459 Fn: [](const APSInt &LHS, const APSInt &RHS) { return LHS & RHS; });
5460
5461 case X86::BI__builtin_ia32_kandnqi:
5462 case X86::BI__builtin_ia32_kandnhi:
5463 case X86::BI__builtin_ia32_kandnsi:
5464 case X86::BI__builtin_ia32_kandndi:
5465 return interp__builtin_elementwise_int_binop(
5466 S, OpPC, Call,
5467 Fn: [](const APSInt &LHS, const APSInt &RHS) { return ~LHS & RHS; });
5468
5469 case X86::BI__builtin_ia32_korqi:
5470 case X86::BI__builtin_ia32_korhi:
5471 case X86::BI__builtin_ia32_korsi:
5472 case X86::BI__builtin_ia32_kordi:
5473 return interp__builtin_elementwise_int_binop(
5474 S, OpPC, Call,
5475 Fn: [](const APSInt &LHS, const APSInt &RHS) { return LHS | RHS; });
5476
5477 case X86::BI__builtin_ia32_kxnorqi:
5478 case X86::BI__builtin_ia32_kxnorhi:
5479 case X86::BI__builtin_ia32_kxnorsi:
5480 case X86::BI__builtin_ia32_kxnordi:
5481 return interp__builtin_elementwise_int_binop(
5482 S, OpPC, Call,
5483 Fn: [](const APSInt &LHS, const APSInt &RHS) { return ~(LHS ^ RHS); });
5484
5485 case X86::BI__builtin_ia32_kxorqi:
5486 case X86::BI__builtin_ia32_kxorhi:
5487 case X86::BI__builtin_ia32_kxorsi:
5488 case X86::BI__builtin_ia32_kxordi:
5489 return interp__builtin_elementwise_int_binop(
5490 S, OpPC, Call,
5491 Fn: [](const APSInt &LHS, const APSInt &RHS) { return LHS ^ RHS; });
5492
5493 case X86::BI__builtin_ia32_knotqi:
5494 case X86::BI__builtin_ia32_knothi:
5495 case X86::BI__builtin_ia32_knotsi:
5496 case X86::BI__builtin_ia32_knotdi:
5497 return interp__builtin_elementwise_int_unaryop(
5498 S, OpPC, Call, Fn: [](const APSInt &Src) { return ~Src; });
5499
5500 case X86::BI__builtin_ia32_kaddqi:
5501 case X86::BI__builtin_ia32_kaddhi:
5502 case X86::BI__builtin_ia32_kaddsi:
5503 case X86::BI__builtin_ia32_kadddi:
5504 return interp__builtin_elementwise_int_binop(
5505 S, OpPC, Call,
5506 Fn: [](const APSInt &LHS, const APSInt &RHS) { return LHS + RHS; });
5507
5508 case X86::BI__builtin_ia32_kmovb:
5509 case X86::BI__builtin_ia32_kmovw:
5510 case X86::BI__builtin_ia32_kmovd:
5511 case X86::BI__builtin_ia32_kmovq:
5512 return interp__builtin_elementwise_int_unaryop(
5513 S, OpPC, Call, Fn: [](const APSInt &Src) { return Src; });
5514
5515 case X86::BI__builtin_ia32_kunpckhi:
5516 case X86::BI__builtin_ia32_kunpckdi:
5517 case X86::BI__builtin_ia32_kunpcksi:
5518 return interp__builtin_elementwise_int_binop(
5519 S, OpPC, Call, Fn: [](const APSInt &A, const APSInt &B) {
5520 // Generic kunpack: extract lower half of each operand and concatenate
5521 // Result = A[HalfWidth-1:0] concat B[HalfWidth-1:0]
5522 unsigned BW = A.getBitWidth();
5523 return APSInt(A.trunc(width: BW / 2).concat(NewLSB: B.trunc(width: BW / 2)),
5524 A.isUnsigned());
5525 });
5526
5527 case X86::BI__builtin_ia32_phminposuw128:
5528 return interp__builtin_ia32_phminposuw(S, OpPC, Call);
5529
5530 case X86::BI__builtin_ia32_psraq128:
5531 case X86::BI__builtin_ia32_psraq256:
5532 case X86::BI__builtin_ia32_psraq512:
5533 case X86::BI__builtin_ia32_psrad128:
5534 case X86::BI__builtin_ia32_psrad256:
5535 case X86::BI__builtin_ia32_psrad512:
5536 case X86::BI__builtin_ia32_psraw128:
5537 case X86::BI__builtin_ia32_psraw256:
5538 case X86::BI__builtin_ia32_psraw512:
5539 return interp__builtin_ia32_shift_with_count(
5540 S, OpPC, Call,
5541 ShiftOp: [](const APInt &Elt, uint64_t Count) { return Elt.ashr(ShiftAmt: Count); },
5542 OverflowOp: [](const APInt &Elt, unsigned Width) { return Elt.ashr(ShiftAmt: Width - 1); });
5543
5544 case X86::BI__builtin_ia32_psllq128:
5545 case X86::BI__builtin_ia32_psllq256:
5546 case X86::BI__builtin_ia32_psllq512:
5547 case X86::BI__builtin_ia32_pslld128:
5548 case X86::BI__builtin_ia32_pslld256:
5549 case X86::BI__builtin_ia32_pslld512:
5550 case X86::BI__builtin_ia32_psllw128:
5551 case X86::BI__builtin_ia32_psllw256:
5552 case X86::BI__builtin_ia32_psllw512:
5553 return interp__builtin_ia32_shift_with_count(
5554 S, OpPC, Call,
5555 ShiftOp: [](const APInt &Elt, uint64_t Count) { return Elt.shl(shiftAmt: Count); },
5556 OverflowOp: [](const APInt &Elt, unsigned Width) { return APInt::getZero(numBits: Width); });
5557
5558 case X86::BI__builtin_ia32_psrlq128:
5559 case X86::BI__builtin_ia32_psrlq256:
5560 case X86::BI__builtin_ia32_psrlq512:
5561 case X86::BI__builtin_ia32_psrld128:
5562 case X86::BI__builtin_ia32_psrld256:
5563 case X86::BI__builtin_ia32_psrld512:
5564 case X86::BI__builtin_ia32_psrlw128:
5565 case X86::BI__builtin_ia32_psrlw256:
5566 case X86::BI__builtin_ia32_psrlw512:
5567 return interp__builtin_ia32_shift_with_count(
5568 S, OpPC, Call,
5569 ShiftOp: [](const APInt &Elt, uint64_t Count) { return Elt.lshr(shiftAmt: Count); },
5570 OverflowOp: [](const APInt &Elt, unsigned Width) { return APInt::getZero(numBits: Width); });
5571
5572 case X86::BI__builtin_ia32_pternlogd128_mask:
5573 case X86::BI__builtin_ia32_pternlogd256_mask:
5574 case X86::BI__builtin_ia32_pternlogd512_mask:
5575 case X86::BI__builtin_ia32_pternlogq128_mask:
5576 case X86::BI__builtin_ia32_pternlogq256_mask:
5577 case X86::BI__builtin_ia32_pternlogq512_mask:
5578 return interp__builtin_ia32_pternlog(S, OpPC, Call, /*MaskZ=*/false);
5579 case X86::BI__builtin_ia32_pternlogd128_maskz:
5580 case X86::BI__builtin_ia32_pternlogd256_maskz:
5581 case X86::BI__builtin_ia32_pternlogd512_maskz:
5582 case X86::BI__builtin_ia32_pternlogq128_maskz:
5583 case X86::BI__builtin_ia32_pternlogq256_maskz:
5584 case X86::BI__builtin_ia32_pternlogq512_maskz:
5585 return interp__builtin_ia32_pternlog(S, OpPC, Call, /*MaskZ=*/true);
5586 case Builtin::BI__builtin_elementwise_fshl:
5587 return interp__builtin_elementwise_triop(S, OpPC, Call,
5588 Fn: llvm::APIntOps::fshl);
5589 case Builtin::BI__builtin_elementwise_fshr:
5590 return interp__builtin_elementwise_triop(S, OpPC, Call,
5591 Fn: llvm::APIntOps::fshr);
5592
5593 case X86::BI__builtin_ia32_shuf_f32x4_256:
5594 case X86::BI__builtin_ia32_shuf_i32x4_256:
5595 case X86::BI__builtin_ia32_shuf_f64x2_256:
5596 case X86::BI__builtin_ia32_shuf_i64x2_256:
5597 case X86::BI__builtin_ia32_shuf_f32x4:
5598 case X86::BI__builtin_ia32_shuf_i32x4:
5599 case X86::BI__builtin_ia32_shuf_f64x2:
5600 case X86::BI__builtin_ia32_shuf_i64x2: {
5601 // Destination and sources A, B all have the same type.
5602 QualType VecQT = Call->getArg(Arg: 0)->getType();
5603 const auto *VecT = VecQT->castAs<VectorType>();
5604 unsigned NumElems = VecT->getNumElements();
5605 unsigned ElemBits = S.getASTContext().getTypeSize(T: VecT->getElementType());
5606 unsigned LaneBits = 128u;
5607 unsigned NumLanes = (NumElems * ElemBits) / LaneBits;
5608 unsigned NumElemsPerLane = LaneBits / ElemBits;
5609
5610 return interp__builtin_ia32_shuffle_generic(
5611 S, OpPC, Call,
5612 GetSourceIndex: [NumLanes, NumElemsPerLane](unsigned DstIdx, unsigned ShuffleMask) {
5613 // DstIdx determines source. ShuffleMask selects lane in source.
5614 unsigned BitsPerElem = NumLanes / 2;
5615 unsigned IndexMask = (1u << BitsPerElem) - 1;
5616 unsigned Lane = DstIdx / NumElemsPerLane;
5617 unsigned SrcIdx = (Lane < NumLanes / 2) ? 0 : 1;
5618 unsigned BitIdx = BitsPerElem * Lane;
5619 unsigned SrcLaneIdx = (ShuffleMask >> BitIdx) & IndexMask;
5620 unsigned ElemInLane = DstIdx % NumElemsPerLane;
5621 unsigned IdxToPick = SrcLaneIdx * NumElemsPerLane + ElemInLane;
5622 return std::pair<unsigned, int>{SrcIdx, IdxToPick};
5623 });
5624 }
5625
5626 case X86::BI__builtin_ia32_insertf32x4_256:
5627 case X86::BI__builtin_ia32_inserti32x4_256:
5628 case X86::BI__builtin_ia32_insertf64x2_256:
5629 case X86::BI__builtin_ia32_inserti64x2_256:
5630 case X86::BI__builtin_ia32_insertf32x4:
5631 case X86::BI__builtin_ia32_inserti32x4:
5632 case X86::BI__builtin_ia32_insertf64x2_512:
5633 case X86::BI__builtin_ia32_inserti64x2_512:
5634 case X86::BI__builtin_ia32_insertf32x8:
5635 case X86::BI__builtin_ia32_inserti32x8:
5636 case X86::BI__builtin_ia32_insertf64x4:
5637 case X86::BI__builtin_ia32_inserti64x4:
5638 case X86::BI__builtin_ia32_vinsertf128_ps256:
5639 case X86::BI__builtin_ia32_vinsertf128_pd256:
5640 case X86::BI__builtin_ia32_vinsertf128_si256:
5641 case X86::BI__builtin_ia32_insert128i256:
5642 return interp__builtin_x86_insert_subvector(S, OpPC, Call, ID: BuiltinID);
5643
5644 case clang::X86::BI__builtin_ia32_vcvtps2ph:
5645 case clang::X86::BI__builtin_ia32_vcvtps2ph256:
5646 return interp__builtin_ia32_vcvtps2ph(S, OpPC, Call);
5647
5648 case X86::BI__builtin_ia32_vec_ext_v4hi:
5649 case X86::BI__builtin_ia32_vec_ext_v16qi:
5650 case X86::BI__builtin_ia32_vec_ext_v8hi:
5651 case X86::BI__builtin_ia32_vec_ext_v4si:
5652 case X86::BI__builtin_ia32_vec_ext_v2di:
5653 case X86::BI__builtin_ia32_vec_ext_v32qi:
5654 case X86::BI__builtin_ia32_vec_ext_v16hi:
5655 case X86::BI__builtin_ia32_vec_ext_v8si:
5656 case X86::BI__builtin_ia32_vec_ext_v4di:
5657 case X86::BI__builtin_ia32_vec_ext_v4sf:
5658 return interp__builtin_vec_ext(S, OpPC, Call, ID: BuiltinID);
5659
5660 case X86::BI__builtin_ia32_vec_set_v4hi:
5661 case X86::BI__builtin_ia32_vec_set_v16qi:
5662 case X86::BI__builtin_ia32_vec_set_v8hi:
5663 case X86::BI__builtin_ia32_vec_set_v4si:
5664 case X86::BI__builtin_ia32_vec_set_v2di:
5665 case X86::BI__builtin_ia32_vec_set_v32qi:
5666 case X86::BI__builtin_ia32_vec_set_v16hi:
5667 case X86::BI__builtin_ia32_vec_set_v8si:
5668 case X86::BI__builtin_ia32_vec_set_v4di:
5669 return interp__builtin_vec_set(S, OpPC, Call, ID: BuiltinID);
5670
5671 case X86::BI__builtin_ia32_cvtb2mask128:
5672 case X86::BI__builtin_ia32_cvtb2mask256:
5673 case X86::BI__builtin_ia32_cvtb2mask512:
5674 case X86::BI__builtin_ia32_cvtw2mask128:
5675 case X86::BI__builtin_ia32_cvtw2mask256:
5676 case X86::BI__builtin_ia32_cvtw2mask512:
5677 case X86::BI__builtin_ia32_cvtd2mask128:
5678 case X86::BI__builtin_ia32_cvtd2mask256:
5679 case X86::BI__builtin_ia32_cvtd2mask512:
5680 case X86::BI__builtin_ia32_cvtq2mask128:
5681 case X86::BI__builtin_ia32_cvtq2mask256:
5682 case X86::BI__builtin_ia32_cvtq2mask512:
5683 return interp__builtin_ia32_cvt_vec2mask(S, OpPC, Call, ID: BuiltinID);
5684
5685 case X86::BI__builtin_ia32_cvtmask2b128:
5686 case X86::BI__builtin_ia32_cvtmask2b256:
5687 case X86::BI__builtin_ia32_cvtmask2b512:
5688 case X86::BI__builtin_ia32_cvtmask2w128:
5689 case X86::BI__builtin_ia32_cvtmask2w256:
5690 case X86::BI__builtin_ia32_cvtmask2w512:
5691 case X86::BI__builtin_ia32_cvtmask2d128:
5692 case X86::BI__builtin_ia32_cvtmask2d256:
5693 case X86::BI__builtin_ia32_cvtmask2d512:
5694 case X86::BI__builtin_ia32_cvtmask2q128:
5695 case X86::BI__builtin_ia32_cvtmask2q256:
5696 case X86::BI__builtin_ia32_cvtmask2q512:
5697 return interp__builtin_ia32_cvt_mask2vec(S, OpPC, Call, ID: BuiltinID);
5698
5699 case X86::BI__builtin_ia32_cvtsd2ss:
5700 return interp__builtin_ia32_cvtsd2ss(S, OpPC, Call, HasRoundingMask: false);
5701
5702 case X86::BI__builtin_ia32_cvtsd2ss_round_mask:
5703 return interp__builtin_ia32_cvtsd2ss(S, OpPC, Call, HasRoundingMask: true);
5704
5705 case X86::BI__builtin_ia32_cvtpd2ps:
5706 case X86::BI__builtin_ia32_cvtpd2ps256:
5707 return interp__builtin_ia32_cvtpd2ps(S, OpPC, Call, IsMasked: false, HasRounding: false);
5708 case X86::BI__builtin_ia32_cvtpd2ps_mask:
5709 return interp__builtin_ia32_cvtpd2ps(S, OpPC, Call, IsMasked: true, HasRounding: false);
5710 case X86::BI__builtin_ia32_cvtpd2ps512_mask:
5711 return interp__builtin_ia32_cvtpd2ps(S, OpPC, Call, IsMasked: true, HasRounding: true);
5712
5713 case X86::BI__builtin_ia32_cmpb128_mask:
5714 case X86::BI__builtin_ia32_cmpw128_mask:
5715 case X86::BI__builtin_ia32_cmpd128_mask:
5716 case X86::BI__builtin_ia32_cmpq128_mask:
5717 case X86::BI__builtin_ia32_cmpb256_mask:
5718 case X86::BI__builtin_ia32_cmpw256_mask:
5719 case X86::BI__builtin_ia32_cmpd256_mask:
5720 case X86::BI__builtin_ia32_cmpq256_mask:
5721 case X86::BI__builtin_ia32_cmpb512_mask:
5722 case X86::BI__builtin_ia32_cmpw512_mask:
5723 case X86::BI__builtin_ia32_cmpd512_mask:
5724 case X86::BI__builtin_ia32_cmpq512_mask:
5725 return interp__builtin_ia32_cmp_mask(S, OpPC, Call, ID: BuiltinID,
5726 /*IsUnsigned=*/false);
5727
5728 case X86::BI__builtin_ia32_ucmpb128_mask:
5729 case X86::BI__builtin_ia32_ucmpw128_mask:
5730 case X86::BI__builtin_ia32_ucmpd128_mask:
5731 case X86::BI__builtin_ia32_ucmpq128_mask:
5732 case X86::BI__builtin_ia32_ucmpb256_mask:
5733 case X86::BI__builtin_ia32_ucmpw256_mask:
5734 case X86::BI__builtin_ia32_ucmpd256_mask:
5735 case X86::BI__builtin_ia32_ucmpq256_mask:
5736 case X86::BI__builtin_ia32_ucmpb512_mask:
5737 case X86::BI__builtin_ia32_ucmpw512_mask:
5738 case X86::BI__builtin_ia32_ucmpd512_mask:
5739 case X86::BI__builtin_ia32_ucmpq512_mask:
5740 return interp__builtin_ia32_cmp_mask(S, OpPC, Call, ID: BuiltinID,
5741 /*IsUnsigned=*/true);
5742
5743 case X86::BI__builtin_ia32_vpshufbitqmb128_mask:
5744 case X86::BI__builtin_ia32_vpshufbitqmb256_mask:
5745 case X86::BI__builtin_ia32_vpshufbitqmb512_mask:
5746 return interp__builtin_ia32_shufbitqmb_mask(S, OpPC, Call);
5747
5748 case X86::BI__builtin_ia32_pslldqi128_byteshift:
5749 case X86::BI__builtin_ia32_pslldqi256_byteshift:
5750 case X86::BI__builtin_ia32_pslldqi512_byteshift:
5751 // These SLLDQ intrinsics always operate on byte elements (8 bits).
5752 // The lane width is hardcoded to 16 to match the SIMD register size,
5753 // but the algorithm processes one byte per iteration,
5754 // so APInt(8, ...) is correct and intentional.
5755 return interp__builtin_ia32_shuffle_generic(
5756 S, OpPC, Call,
5757 GetSourceIndex: [](unsigned DstIdx, unsigned Shift) -> std::pair<unsigned, int> {
5758 unsigned LaneBase = (DstIdx / 16) * 16;
5759 unsigned LaneIdx = DstIdx % 16;
5760 if (LaneIdx < Shift)
5761 return std::make_pair(x: 0, y: -1);
5762
5763 return std::make_pair(x: 0,
5764 y: static_cast<int>(LaneBase + LaneIdx - Shift));
5765 });
5766
5767 case X86::BI__builtin_ia32_psrldqi128_byteshift:
5768 case X86::BI__builtin_ia32_psrldqi256_byteshift:
5769 case X86::BI__builtin_ia32_psrldqi512_byteshift:
5770 // These SRLDQ intrinsics always operate on byte elements (8 bits).
5771 // The lane width is hardcoded to 16 to match the SIMD register size,
5772 // but the algorithm processes one byte per iteration,
5773 // so APInt(8, ...) is correct and intentional.
5774 return interp__builtin_ia32_shuffle_generic(
5775 S, OpPC, Call,
5776 GetSourceIndex: [](unsigned DstIdx, unsigned Shift) -> std::pair<unsigned, int> {
5777 unsigned LaneBase = (DstIdx / 16) * 16;
5778 unsigned LaneIdx = DstIdx % 16;
5779 if (LaneIdx + Shift < 16)
5780 return std::make_pair(x: 0,
5781 y: static_cast<int>(LaneBase + LaneIdx + Shift));
5782
5783 return std::make_pair(x: 0, y: -1);
5784 });
5785
5786 case X86::BI__builtin_ia32_palignr128:
5787 case X86::BI__builtin_ia32_palignr256:
5788 case X86::BI__builtin_ia32_palignr512:
5789 return interp__builtin_ia32_shuffle_generic(
5790 S, OpPC, Call, GetSourceIndex: [](unsigned DstIdx, unsigned Shift) {
5791 // Default to -1 → zero-fill this destination element
5792 unsigned VecIdx = 1;
5793 int ElemIdx = -1;
5794
5795 int Lane = DstIdx / 16;
5796 int Offset = DstIdx % 16;
5797
5798 // Elements come from VecB first, then VecA after the shift boundary
5799 unsigned ShiftedIdx = Offset + (Shift & 0xFF);
5800 if (ShiftedIdx < 16) { // from VecB
5801 ElemIdx = ShiftedIdx + (Lane * 16);
5802 } else if (ShiftedIdx < 32) { // from VecA
5803 VecIdx = 0;
5804 ElemIdx = (ShiftedIdx - 16) + (Lane * 16);
5805 }
5806
5807 return std::pair<unsigned, int>{VecIdx, ElemIdx};
5808 });
5809
5810 case X86::BI__builtin_ia32_alignd128:
5811 case X86::BI__builtin_ia32_alignd256:
5812 case X86::BI__builtin_ia32_alignd512:
5813 case X86::BI__builtin_ia32_alignq128:
5814 case X86::BI__builtin_ia32_alignq256:
5815 case X86::BI__builtin_ia32_alignq512: {
5816 unsigned NumElems = Call->getType()->castAs<VectorType>()->getNumElements();
5817 return interp__builtin_ia32_shuffle_generic(
5818 S, OpPC, Call, GetSourceIndex: [NumElems](unsigned DstIdx, unsigned Shift) {
5819 unsigned Imm = Shift & 0xFF;
5820 unsigned EffectiveShift = Imm & (NumElems - 1);
5821 unsigned SourcePos = DstIdx + EffectiveShift;
5822 unsigned VecIdx = SourcePos < NumElems ? 1u : 0u;
5823 unsigned ElemIdx = SourcePos & (NumElems - 1);
5824 return std::pair<unsigned, int>{VecIdx, static_cast<int>(ElemIdx)};
5825 });
5826 }
5827
5828 case clang::X86::BI__builtin_ia32_minps:
5829 case clang::X86::BI__builtin_ia32_minpd:
5830 case clang::X86::BI__builtin_ia32_minph128:
5831 case clang::X86::BI__builtin_ia32_minph256:
5832 case clang::X86::BI__builtin_ia32_minps256:
5833 case clang::X86::BI__builtin_ia32_minpd256:
5834 case clang::X86::BI__builtin_ia32_minps512:
5835 case clang::X86::BI__builtin_ia32_minpd512:
5836 case clang::X86::BI__builtin_ia32_minph512:
5837 return interp__builtin_elementwise_fp_binop(
5838 S, OpPC, Call,
5839 Fn: [](const APFloat &A, const APFloat &B,
5840 std::optional<APSInt>) -> std::optional<APFloat> {
5841 if (A.isNaN() || A.isInfinity() || A.isDenormal() || B.isNaN() ||
5842 B.isInfinity() || B.isDenormal())
5843 return std::nullopt;
5844 if (A.isZero() && B.isZero())
5845 return B;
5846 return llvm::minimum(A, B);
5847 });
5848
5849 case clang::X86::BI__builtin_ia32_maxps:
5850 case clang::X86::BI__builtin_ia32_maxpd:
5851 case clang::X86::BI__builtin_ia32_maxph128:
5852 case clang::X86::BI__builtin_ia32_maxph256:
5853 case clang::X86::BI__builtin_ia32_maxps256:
5854 case clang::X86::BI__builtin_ia32_maxpd256:
5855 case clang::X86::BI__builtin_ia32_maxps512:
5856 case clang::X86::BI__builtin_ia32_maxpd512:
5857 case clang::X86::BI__builtin_ia32_maxph512:
5858 return interp__builtin_elementwise_fp_binop(
5859 S, OpPC, Call,
5860 Fn: [](const APFloat &A, const APFloat &B,
5861 std::optional<APSInt>) -> std::optional<APFloat> {
5862 if (A.isNaN() || A.isInfinity() || A.isDenormal() || B.isNaN() ||
5863 B.isInfinity() || B.isDenormal())
5864 return std::nullopt;
5865 if (A.isZero() && B.isZero())
5866 return B;
5867 return llvm::maximum(A, B);
5868 });
5869
5870 default:
5871 S.FFDiag(Loc: S.Current->getLocation(PC: OpPC),
5872 DiagId: diag::note_invalid_subexpr_in_const_expr)
5873 << S.Current->getRange(PC: OpPC);
5874
5875 return false;
5876 }
5877
5878 llvm_unreachable("Unhandled builtin ID");
5879}
5880
5881bool InterpretOffsetOf(InterpState &S, CodePtr OpPC, const OffsetOfExpr *E,
5882 ArrayRef<int64_t> ArrayIndices, int64_t &IntResult) {
5883 CharUnits Result;
5884 unsigned N = E->getNumComponents();
5885 assert(N > 0);
5886
5887 unsigned ArrayIndex = 0;
5888 QualType CurrentType = E->getTypeSourceInfo()->getType();
5889 for (unsigned I = 0; I != N; ++I) {
5890 const OffsetOfNode &Node = E->getComponent(Idx: I);
5891 switch (Node.getKind()) {
5892 case OffsetOfNode::Field: {
5893 const FieldDecl *MemberDecl = Node.getField();
5894 const auto *RD = CurrentType->getAsRecordDecl();
5895 if (!RD || RD->isInvalidDecl())
5896 return false;
5897 const ASTRecordLayout &RL = S.getASTContext().getASTRecordLayout(D: RD);
5898 unsigned FieldIndex = MemberDecl->getFieldIndex();
5899 assert(FieldIndex < RL.getFieldCount() && "offsetof field in wrong type");
5900 Result +=
5901 S.getASTContext().toCharUnitsFromBits(BitSize: RL.getFieldOffset(FieldNo: FieldIndex));
5902 CurrentType = MemberDecl->getType().getNonReferenceType();
5903 break;
5904 }
5905 case OffsetOfNode::Array: {
5906 // When generating bytecode, we put all the index expressions as Sint64 on
5907 // the stack.
5908 int64_t Index = ArrayIndices[ArrayIndex];
5909 const ArrayType *AT = S.getASTContext().getAsArrayType(T: CurrentType);
5910 if (!AT)
5911 return false;
5912 CurrentType = AT->getElementType();
5913 CharUnits ElementSize = S.getASTContext().getTypeSizeInChars(T: CurrentType);
5914 Result += Index * ElementSize;
5915 ++ArrayIndex;
5916 break;
5917 }
5918 case OffsetOfNode::Base: {
5919 const CXXBaseSpecifier *BaseSpec = Node.getBase();
5920 if (BaseSpec->isVirtual())
5921 return false;
5922
5923 // Find the layout of the class whose base we are looking into.
5924 const auto *RD = CurrentType->getAsCXXRecordDecl();
5925 if (!RD || RD->isInvalidDecl())
5926 return false;
5927 const ASTRecordLayout &RL = S.getASTContext().getASTRecordLayout(D: RD);
5928
5929 // Find the base class itself.
5930 CurrentType = BaseSpec->getType();
5931 const auto *BaseRD = CurrentType->getAsCXXRecordDecl();
5932 if (!BaseRD)
5933 return false;
5934
5935 // Add the offset to the base.
5936 Result += RL.getBaseClassOffset(Base: BaseRD);
5937 break;
5938 }
5939 case OffsetOfNode::Identifier:
5940 llvm_unreachable("Dependent OffsetOfExpr?");
5941 }
5942 }
5943
5944 IntResult = Result.getQuantity();
5945
5946 return true;
5947}
5948
5949bool SetThreeWayComparisonField(InterpState &S, CodePtr OpPC,
5950 const Pointer &Ptr, const APSInt &IntValue) {
5951
5952 const Record *R = Ptr.getRecord();
5953 assert(R);
5954 assert(R->getNumFields() == 1);
5955
5956 unsigned FieldOffset = R->getField(I: 0u)->Offset;
5957 const Pointer &FieldPtr = Ptr.atField(Off: FieldOffset);
5958 PrimType FieldT = *S.getContext().classify(T: FieldPtr.getType());
5959
5960 INT_TYPE_SWITCH(FieldT,
5961 FieldPtr.deref<T>() = T::from(IntValue.getSExtValue()));
5962 FieldPtr.initialize();
5963 return true;
5964}
5965
5966static void zeroAll(Pointer &Dest) {
5967 const Descriptor *Desc = Dest.getFieldDesc();
5968
5969 if (Desc->isPrimitive()) {
5970 TYPE_SWITCH(Desc->getPrimType(), {
5971 Dest.deref<T>().~T();
5972 new (&Dest.deref<T>()) T();
5973 });
5974 return;
5975 }
5976
5977 if (Desc->isRecord()) {
5978 const Record *R = Desc->ElemRecord;
5979 for (const Record::Field &F : R->fields()) {
5980 Pointer FieldPtr = Dest.atField(Off: F.Offset);
5981 zeroAll(Dest&: FieldPtr);
5982 }
5983 return;
5984 }
5985
5986 if (Desc->isPrimitiveArray()) {
5987 for (unsigned I = 0, N = Desc->getNumElems(); I != N; ++I) {
5988 TYPE_SWITCH(Desc->getPrimType(), {
5989 Dest.deref<T>().~T();
5990 new (&Dest.deref<T>()) T();
5991 });
5992 }
5993 return;
5994 }
5995
5996 if (Desc->isCompositeArray()) {
5997 for (unsigned I = 0, N = Desc->getNumElems(); I != N; ++I) {
5998 Pointer ElemPtr = Dest.atIndex(Idx: I).narrow();
5999 zeroAll(Dest&: ElemPtr);
6000 }
6001 return;
6002 }
6003}
6004
6005static bool copyComposite(InterpState &S, CodePtr OpPC, const Pointer &Src,
6006 Pointer &Dest, bool Activate);
6007static bool copyRecord(InterpState &S, CodePtr OpPC, const Pointer &Src,
6008 Pointer &Dest, bool Activate = false) {
6009 [[maybe_unused]] const Descriptor *SrcDesc = Src.getFieldDesc();
6010 const Descriptor *DestDesc = Dest.getFieldDesc();
6011
6012 auto copyField = [&](const Record::Field &F, bool Activate) -> bool {
6013 Pointer DestField = Dest.atField(Off: F.Offset);
6014 if (OptPrimType FT = S.Ctx.classify(T: F.Decl->getType())) {
6015 TYPE_SWITCH(*FT, {
6016 DestField.deref<T>() = Src.atField(F.Offset).deref<T>();
6017 if (Src.atField(F.Offset).isInitialized())
6018 DestField.initialize();
6019 if (Activate)
6020 DestField.activate();
6021 });
6022 return true;
6023 }
6024 // Composite field.
6025 return copyComposite(S, OpPC, Src: Src.atField(Off: F.Offset), Dest&: DestField, Activate);
6026 };
6027
6028 assert(SrcDesc->isRecord());
6029 assert(SrcDesc->ElemRecord == DestDesc->ElemRecord);
6030 const Record *R = DestDesc->ElemRecord;
6031 for (const Record::Field &F : R->fields()) {
6032 if (R->isUnion()) {
6033 // For unions, only copy the active field. Zero all others.
6034 const Pointer &SrcField = Src.atField(Off: F.Offset);
6035 if (SrcField.isActive()) {
6036 if (!copyField(F, /*Activate=*/true))
6037 return false;
6038 } else {
6039 if (!CheckMutable(S, OpPC, Ptr: Src.atField(Off: F.Offset)))
6040 return false;
6041 Pointer DestField = Dest.atField(Off: F.Offset);
6042 zeroAll(Dest&: DestField);
6043 }
6044 } else {
6045 if (!copyField(F, Activate))
6046 return false;
6047 }
6048 }
6049
6050 for (const Record::Base &B : R->bases()) {
6051 Pointer DestBase = Dest.atField(Off: B.Offset);
6052 if (!copyRecord(S, OpPC, Src: Src.atField(Off: B.Offset), Dest&: DestBase, Activate))
6053 return false;
6054 }
6055
6056 Dest.initialize();
6057 return true;
6058}
6059
6060static bool copyComposite(InterpState &S, CodePtr OpPC, const Pointer &Src,
6061 Pointer &Dest, bool Activate = false) {
6062 assert(Src.isLive() && Dest.isLive());
6063
6064 [[maybe_unused]] const Descriptor *SrcDesc = Src.getFieldDesc();
6065 const Descriptor *DestDesc = Dest.getFieldDesc();
6066
6067 assert(!DestDesc->isPrimitive() && !SrcDesc->isPrimitive());
6068
6069 if (DestDesc->isPrimitiveArray()) {
6070 assert(SrcDesc->isPrimitiveArray());
6071 assert(SrcDesc->getNumElems() == DestDesc->getNumElems());
6072 PrimType ET = DestDesc->getPrimType();
6073 for (unsigned I = 0, N = DestDesc->getNumElems(); I != N; ++I) {
6074 Pointer DestElem = Dest.atIndex(Idx: I);
6075 TYPE_SWITCH(ET, {
6076 DestElem.deref<T>() = Src.elem<T>(I);
6077 DestElem.initialize();
6078 });
6079 }
6080 return true;
6081 }
6082
6083 if (DestDesc->isCompositeArray()) {
6084 assert(SrcDesc->isCompositeArray());
6085 assert(SrcDesc->getNumElems() == DestDesc->getNumElems());
6086 for (unsigned I = 0, N = DestDesc->getNumElems(); I != N; ++I) {
6087 const Pointer &SrcElem = Src.atIndex(Idx: I).narrow();
6088 Pointer DestElem = Dest.atIndex(Idx: I).narrow();
6089 if (!copyComposite(S, OpPC, Src: SrcElem, Dest&: DestElem, Activate))
6090 return false;
6091 }
6092 return true;
6093 }
6094
6095 if (DestDesc->isRecord())
6096 return copyRecord(S, OpPC, Src, Dest, Activate);
6097 return Invalid(S, OpPC);
6098}
6099
6100bool DoMemcpy(InterpState &S, CodePtr OpPC, const Pointer &Src, Pointer &Dest) {
6101 return copyComposite(S, OpPC, Src, Dest);
6102}
6103
6104} // namespace interp
6105} // namespace clang
6106