1//===- Interpreter.cpp - Interpreter Loop for llubi -----------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the evaluation loop for each kind of instruction.
10//
11//===----------------------------------------------------------------------===//
12
13#include "Context.h"
14#include "ExecutorBase.h"
15#include "Library.h"
16#include "Value.h"
17#include "llvm/ADT/STLExtras.h"
18#include "llvm/ADT/SmallVector.h"
19#include "llvm/Analysis/VectorUtils.h"
20#include "llvm/IR/InlineAsm.h"
21#include "llvm/IR/InstVisitor.h"
22#include "llvm/IR/Intrinsics.h"
23#include "llvm/IR/Operator.h"
24#include "llvm/IR/PatternMatch.h"
25#include "llvm/Support/Allocator.h"
26#include "llvm/TargetParser/Triple.h"
27
28#include <cassert>
29#include <cstring>
30#include <limits>
31
32namespace llvm::ubi {
33
34using namespace PatternMatch;
35
36/// Visit the scalar values recursively. The callback function may modify the
37/// value in-place.
38static void forEachScalarValue(AnyValue &V,
39 function_ref<void(AnyValue &)> Visit) {
40 if (V.isNone())
41 return;
42
43 if (V.isAggregate()) {
44 for (auto &SubValue : V.asAggregate())
45 forEachScalarValue(V&: SubValue, Visit);
46 return;
47 }
48
49 Visit(V);
50}
51
52static void applyRangeAttr(AnyValue &V, const ConstantRange &CR) {
53 forEachScalarValue(V, Visit: [&](AnyValue &Scalar) {
54 if (Scalar.isInteger() && !CR.contains(Val: Scalar.asInteger()))
55 Scalar = AnyValue::poison();
56 });
57}
58
59static void applyNoFPClassAttr(AnyValue &V, FPClassTest NoFPClass) {
60 forEachScalarValue(V, Visit: [NoFPClass](AnyValue &Scalar) {
61 if (Scalar.isFloat() && (Scalar.asFloat().classify() & NoFPClass))
62 Scalar = AnyValue::poison();
63 });
64}
65
66static void applyNonNullAttr(AnyValue &V, unsigned AS, const DataLayout &DL) {
67 if (V.isPointer() && V.asPointer().isNullPtr(AS, DL))
68 V = AnyValue::poison();
69}
70
71static void applyAlignAttr(AnyValue &V, Align Alignment) {
72 forEachScalarValue(V, Visit: [Alignment](AnyValue &Scalar) {
73 if (Scalar.isPointer() &&
74 Scalar.asPointer().address().countr_zero() < Log2(A: Alignment))
75 Scalar = AnyValue::poison();
76 });
77}
78
79static bool violatesNoUndefAttr(AnyValue &V) {
80 bool ContainsPoison = false;
81 forEachScalarValue(
82 V, Visit: [&](AnyValue &Scalar) { ContainsPoison |= Scalar.isPoison(); });
83 return ContainsPoison;
84}
85
86/// Assumes V is either a poison or a pointer.
87static bool violatesDereferenceableBytesAttr(const AnyValue &V, uint64_t Bytes,
88 bool OrNull, unsigned AS,
89 Context &Ctx) {
90 if (V.isPoison())
91 return true;
92
93 auto &Ptr = V.asPointer();
94 if (Ptr.isNullPtr(AS, DL: Ctx.getDataLayout())) {
95 if (OrNull)
96 return false;
97 return true;
98 }
99
100 auto *MO = Ctx.checkProvenance(Ptr, Check: [&](const Provenance &) {
101 // TODO: check read_provenance
102 // TODO: check nofree for attributes/metadata.
103 return true;
104 });
105 if (!MO)
106 return true;
107
108 const APInt &PtrAddr = Ptr.address();
109 return Bytes > MO->getSize() || PtrAddr.ult(RHS: MO->getAddress()) ||
110 PtrAddr.ugt(RHS: MO->getAddress() + MO->getSize() - Bytes);
111}
112
113/// Instruction executor using the visitor pattern.
114/// Unlike the Context class that manages the global state,
115/// InstExecutor only maintains the state for call frames.
116class InstExecutor : public InstVisitor<InstExecutor, void>,
117 public ExecutorBase {
118 const DataLayout &DL;
119 std::list<Frame> CallStack;
120 AnyValue None;
121 std::list<AnyValue> UnsupportedConstantValues;
122 Library Lib;
123
124 const AnyValue &getValue(Value *V) {
125 if (auto *C = dyn_cast<Constant>(Val: V)) {
126 if (const AnyValue *Val = Ctx.getConstantValue(C))
127 return *Val;
128 reportError() << "Unsupported constant: " << *C << ".";
129 UnsupportedConstantValues.push_back(
130 x: AnyValue::getPoisonValue(Ctx, Ty: C->getType()));
131 return UnsupportedConstantValues.back();
132 }
133 if (isa<MetadataAsValue>(Val: V))
134 return None;
135 return CurrentFrame->ValueMap.at(Val: V);
136 }
137
138 void setResult(Instruction &I, AnyValue V) {
139 if (!hasProgramExited() && !Handler.onInstructionExecuted(I, Result: V))
140 setFailed();
141 if (hasProgramExited())
142 return;
143 assert(V.isCompatibleWith(I.getType()) && "Unexpected value storage kind.");
144 if (!V.isNone())
145 CurrentFrame->ValueMap.insert_or_assign(Key: &I, Val: std::move(V));
146 }
147
148 APFloat handleDenormal(APFloat Val, DenormalMode::DenormalModeKind Mode,
149 bool IsInput) {
150 if (!Val.isDenormal())
151 return Val;
152 if (IsInput) {
153 // Non-deterministically choose between flushing or preserving the
154 // denormal value.
155 if (Ctx.getRandomBool())
156 return Val;
157 }
158 if (Mode == DenormalMode::PositiveZero)
159 return APFloat::getZero(Sem: Val.getSemantics(), Negative: false);
160 if (Mode == DenormalMode::PreserveSign)
161 return APFloat::getZero(Sem: Val.getSemantics(), Negative: Val.isNegative());
162 // Default case for IEEE, Dynamic, and Invalid
163 // Currently we treat Dynamic the same as IEEE, since we don't support
164 // changing the mode at this point.
165 return Val;
166 }
167
168 AnyValue handleFMFFlags(AnyValue Val, FastMathFlags FMF, bool IsInput) {
169 if (Val.isPoison())
170 return AnyValue::poison();
171
172 if (Val.isAggregate()) {
173 std::vector<AnyValue> ResVec;
174 ResVec.reserve(n: Val.asAggregate().size());
175 for (const auto &A : Val.asAggregate())
176 ResVec.push_back(x: handleFMFFlags(Val: A, FMF, IsInput));
177 return AnyValue(ResVec);
178 }
179
180 const APFloat &APVal = Val.asFloat();
181 if (FMF.noNaNs() && APVal.isNaN())
182 return AnyValue::poison();
183 if (FMF.noInfs() && APVal.isInfinity())
184 return AnyValue::poison();
185 if (IsInput && FMF.noSignedZeros() && APVal.isZero())
186 return AnyValue(APFloat::getZero(
187 Sem: APVal.getSemantics(), Negative: APVal.isNegative() ^ Ctx.getRandomBool()));
188 return Val;
189 }
190
191 void addNaNCandidate(SmallVectorImpl<APFloat> &Candidates,
192 APFloat Candidate) {
193 if (any_of(Range&: Candidates, P: [&](const APFloat &Existing) {
194 return Existing.bitwiseIsEqual(RHS: Candidate);
195 }))
196 return;
197 Candidates.push_back(Elt: std::move(Candidate));
198 }
199
200 APFloat pickNaNCandidate(ArrayRef<APFloat> Candidates) {
201 assert(!Candidates.empty() && "Need at least one NaN candidate.");
202 return Candidates[Ctx.getRandomUInt64() % Candidates.size()];
203 }
204
205 APInt getRandomNaNPayload(const fltSemantics &Sem) {
206 const unsigned NumBits = Sem.precision - 1;
207 SmallVector<APInt::WordType, 2> RandomWords;
208 const unsigned NumWords = APInt::getNumWords(BitWidth: NumBits);
209 RandomWords.reserve(N: NumWords);
210 for (unsigned I = 0; I != NumWords; ++I)
211 RandomWords.push_back(Elt: Ctx.getRandomUInt64());
212 return APInt(NumBits, RandomWords);
213 }
214
215 bool isPreferredNaN(const APFloat &Val) {
216 assert(Val.isNaN() && "Expected NaN.");
217 const APFloat Preferred =
218 APFloat::getQNaN(Sem: Val.getSemantics(), Negative: Val.isNegative());
219 return Val.bitwiseIsEqual(RHS: Preferred);
220 }
221
222 bool wasmMayProduceExtraNaNPayload(ArrayRef<const APFloat *> Inputs) {
223 for (const APFloat *Input : Inputs) {
224 if (!Input->isNaN())
225 continue;
226 if (Input->isSignaling() || !isPreferredNaN(Val: *Input))
227 return true;
228 }
229 return false;
230 }
231
232 APFloat propagateInputNaN(const APFloat &InputNaN, const fltSemantics &DstSem,
233 bool QuietingMode, bool FlipSign) {
234 APFloat Res = InputNaN;
235 bool LosesInfo;
236 Res.convert(ToSemantics: DstSem, RM: APFloat::rmNearestTiesToEven, losesInfo: &LosesInfo);
237 if (FlipSign)
238 Res.changeSign();
239 if (QuietingMode && Res.isSignaling())
240 Res = Res.makeQuiet();
241 return Res;
242 }
243
244 APFloat maybeQuietSNaN(APFloat Val) const {
245 if (Val.isSignaling() && Ctx.getRandomBool())
246 return Val.makeQuiet();
247 return Val;
248 }
249
250 APFloat maxnumWithSNaNQuieting(const APFloat &LHS, const APFloat &RHS) {
251 return maxnum(A: maybeQuietSNaN(Val: LHS), B: maybeQuietSNaN(Val: RHS));
252 }
253
254 APFloat minnumWithSNaNQuieting(const APFloat &LHS, const APFloat &RHS) {
255 return minnum(A: maybeQuietSNaN(Val: LHS), B: maybeQuietSNaN(Val: RHS));
256 }
257
258 void addPropagatedNaNCandidates(SmallVectorImpl<APFloat> &Candidates,
259 ArrayRef<const APFloat *> Inputs,
260 const fltSemantics &DstSem, bool QuietingMode,
261 bool SignChoice) {
262 for (const APFloat *Input : Inputs) {
263 if (!Input->isNaN())
264 continue;
265 addNaNCandidate(Candidates, Candidate: propagateInputNaN(InputNaN: *Input, DstSem,
266 QuietingMode, FlipSign: SignChoice));
267 }
268 }
269
270 void addTargetSpecificNaNCandidates(SmallVectorImpl<APFloat> &Candidates,
271 const APFloat &Result,
272 ArrayRef<const APFloat *> Inputs,
273 bool SignChoice) {
274 const Triple &TT = Ctx.getTargetTriple();
275 if (TT.isWasm()) {
276 if (!wasmMayProduceExtraNaNPayload(Inputs))
277 return;
278 APInt Payload = getRandomNaNPayload(Sem: Result.getSemantics());
279 addNaNCandidate(Candidates, Candidate: APFloat::getQNaN(Sem: Result.getSemantics(),
280 Negative: SignChoice, payload: &Payload));
281 return;
282 }
283
284 if (TT.isSPARC32() || TT.isSPARC64()) {
285 APInt Payload = APInt::getAllOnes(numBits: Result.getSemantics().precision - 1);
286 addNaNCandidate(Candidates, Candidate: APFloat::getQNaN(Sem: Result.getSemantics(),
287 Negative: SignChoice, payload: &Payload));
288 }
289 }
290
291 APFloat applyNaNPropagation(const APFloat &Result,
292 ArrayRef<const APFloat *> Inputs) {
293 if (!Result.isNaN())
294 return Result;
295
296 const NaNPropagationBehavior Choice =
297 Ctx.getEffectiveNaNPropagationBehavior();
298 const bool SignChoice = Ctx.getRandomBool();
299 const fltSemantics &ResultSem = Result.getSemantics();
300 auto PreferredNaN = [&]() {
301 return APFloat::getQNaN(Sem: ResultSem, Negative: SignChoice);
302 };
303
304 SmallVector<APFloat, 4> Candidates;
305 switch (Choice) {
306 case NaNPropagationBehavior::PreferredNaN:
307 return PreferredNaN();
308 case NaNPropagationBehavior::QuietingNaN:
309 addPropagatedNaNCandidates(Candidates, Inputs, DstSem: ResultSem,
310 /*QuietingMode=*/true, SignChoice);
311 return Candidates.empty() ? PreferredNaN() : pickNaNCandidate(Candidates);
312 case NaNPropagationBehavior::UnchangedNaN:
313 addPropagatedNaNCandidates(Candidates, Inputs, DstSem: ResultSem,
314 /*QuietingMode=*/false, SignChoice);
315 return Candidates.empty() ? PreferredNaN() : pickNaNCandidate(Candidates);
316 case NaNPropagationBehavior::TargetSpecificNaN:
317 addTargetSpecificNaNCandidates(Candidates, Result, Inputs, SignChoice);
318 return Candidates.empty() ? PreferredNaN() : pickNaNCandidate(Candidates);
319 case NaNPropagationBehavior::NonDeterministic:
320 addNaNCandidate(Candidates, Candidate: PreferredNaN());
321 addPropagatedNaNCandidates(Candidates, Inputs, DstSem: ResultSem,
322 /*QuietingMode=*/true, SignChoice);
323 addPropagatedNaNCandidates(Candidates, Inputs, DstSem: ResultSem,
324 /*QuietingMode=*/false, SignChoice);
325 addTargetSpecificNaNCandidates(Candidates, Result, Inputs, SignChoice);
326 return pickNaNCandidate(Candidates);
327 }
328 llvm_unreachable("Unhandled NaN propagation behavior.");
329 }
330
331 AnyValue computeUnOp(Type *Ty, const AnyValue &Operand,
332 function_ref<AnyValue(const AnyValue &)> ScalarFn) {
333 if (Ty->isVectorTy()) {
334 auto &OperandVec = Operand.asAggregate();
335 std::vector<AnyValue> ResVec;
336 ResVec.reserve(n: OperandVec.size());
337 for (const auto &Scalar : OperandVec)
338 ResVec.push_back(x: ScalarFn(Scalar));
339 return std::move(ResVec);
340 }
341 return ScalarFn(Operand);
342 }
343
344 void visitUnOp(Instruction &I,
345 function_ref<AnyValue(const AnyValue &)> ScalarFn) {
346 setResult(I, V: computeUnOp(Ty: I.getType(), Operand: getValue(V: I.getOperand(i: 0)), ScalarFn));
347 }
348
349 void visitIntUnOp(Instruction &I,
350 function_ref<AnyValue(const APInt &)> ScalarFn) {
351 visitUnOp(I, ScalarFn: [&](const AnyValue &Operand) -> AnyValue {
352 if (Operand.isPoison())
353 return AnyValue::poison();
354 return ScalarFn(Operand.asInteger());
355 });
356 }
357
358 void visitBitwiseFPUnOp(Instruction &I,
359 function_ref<APFloat(const APFloat &)> ScalarFn) {
360 setResult(I, V: visitBitwiseFPUnOpWithResult(
361 RetTy: I.getType(), FMF: cast<FPMathOperator>(Val&: I).getFastMathFlags(),
362 Operand: getValue(V: I.getOperand(i: 0)), ScalarFn));
363 }
364
365 AnyValue
366 visitIntUnOpWithResult(Type *RetTy, const AnyValue &Operand,
367 function_ref<AnyValue(const APInt &)> ScalarFn) {
368 return computeUnOp(Ty: RetTy, Operand,
369 ScalarFn: [&](const AnyValue &OperandInner) -> AnyValue {
370 if (OperandInner.isPoison())
371 return AnyValue::poison();
372 return ScalarFn(OperandInner.asInteger());
373 });
374 }
375
376 AnyValue visitBitwiseFPUnOpWithResult(
377 Type *RetTy, const FastMathFlags &FMF, const AnyValue &Operand,
378 function_ref<APFloat(const APFloat &)> ScalarFn) {
379 return computeUnOp(
380 Ty: RetTy, Operand, ScalarFn: [&](const AnyValue &OperandInner) -> AnyValue {
381 if (OperandInner.isPoison())
382 return AnyValue::poison();
383
384 // We don't flush denormals here since bitwise floating-point
385 // operations only manipulate on certain bits of the operand.
386
387 AnyValue ValidatedOperand =
388 handleFMFFlags(Val: OperandInner, FMF, /*IsInput=*/true);
389 if (ValidatedOperand.isPoison())
390 return ValidatedOperand;
391
392 APFloat Result = ScalarFn(ValidatedOperand.asFloat());
393
394 return handleFMFFlags(Val: Result, FMF, /*IsInput=*/false);
395 });
396 }
397
398 AnyValue computeBinOp(
399 Type *Ty, const AnyValue &LHS, const AnyValue &RHS,
400 function_ref<AnyValue(const AnyValue &, const AnyValue &)> ScalarFn) {
401 if (Ty->isVectorTy()) {
402 auto &LHSVec = LHS.asAggregate();
403 auto &RHSVec = RHS.asAggregate();
404 std::vector<AnyValue> ResVec;
405 ResVec.reserve(n: LHSVec.size());
406 for (const auto &[ScalarLHS, ScalarRHS] : zip(t: LHSVec, u: RHSVec))
407 ResVec.push_back(x: ScalarFn(ScalarLHS, ScalarRHS));
408 return std::move(ResVec);
409 }
410 return ScalarFn(LHS, RHS);
411 }
412
413 void visitBinOp(
414 Instruction &I,
415 function_ref<AnyValue(const AnyValue &, const AnyValue &)> ScalarFn) {
416 setResult(I, V: computeBinOp(Ty: I.getType(), LHS: getValue(V: I.getOperand(i: 0)),
417 RHS: getValue(V: I.getOperand(i: 1)), ScalarFn));
418 }
419
420 void
421 visitIntBinOp(Instruction &I,
422 function_ref<AnyValue(const APInt &, const APInt &)> ScalarFn) {
423 visitBinOp(I, ScalarFn: [&](const AnyValue &LHS, const AnyValue &RHS) -> AnyValue {
424 if (LHS.isPoison() || RHS.isPoison())
425 return AnyValue::poison();
426 return ScalarFn(LHS.asInteger(), RHS.asInteger());
427 });
428 }
429
430 void visitFPBinOp(
431 Instruction &I,
432 function_ref<APFloat(const APFloat &, const APFloat &)> ScalarFn) {
433 setResult(I, V: visitFPBinOpWithResult(
434 RetTy: I.getType(), FMF: cast<FPMathOperator>(Val&: I).getFastMathFlags(),
435 LHS: getValue(V: I.getOperand(i: 0)), RHS: getValue(V: I.getOperand(i: 1)),
436 ScalarFn));
437 }
438
439 AnyValue visitIntBinOpWithResult(
440 Type *RetTy, const AnyValue &LHS, const AnyValue &RHS,
441 function_ref<AnyValue(const APInt &, const APInt &)> ScalarFn) {
442 return computeBinOp(
443 Ty: RetTy, LHS, RHS,
444 ScalarFn: [&](const AnyValue &LHSInner, const AnyValue &RHSInner) -> AnyValue {
445 if (LHSInner.isPoison() || RHSInner.isPoison())
446 return AnyValue::poison();
447 return ScalarFn(LHSInner.asInteger(), RHSInner.asInteger());
448 });
449 }
450
451 AnyValue visitOverflowIntBinOpWithResult(
452 Type *RetTy, const AnyValue &LHS, const AnyValue &RHS,
453 function_ref<std::pair<APInt, bool>(const APInt &, const APInt &)>
454 ScalarFn) {
455 if (!LHS.isAggregate()) {
456 if (LHS.isPoison() || RHS.isPoison())
457 return std::vector<AnyValue>{AnyValue::poison(), AnyValue::poison()};
458 auto [Res, Overflow] = ScalarFn(LHS.asInteger(), RHS.asInteger());
459 return std::vector<AnyValue>{AnyValue(Res), AnyValue::boolean(Val: Overflow)};
460 }
461
462 auto &LHSVec = LHS.asAggregate();
463 auto &RHSVec = RHS.asAggregate();
464 std::vector<AnyValue> ResVec;
465 std::vector<AnyValue> OverflowVec;
466 ResVec.reserve(n: LHSVec.size());
467 OverflowVec.reserve(n: LHSVec.size());
468 for (const auto &[ScalarLHS, ScalarRHS] : zip(t: LHSVec, u: RHSVec)) {
469 if (ScalarLHS.isPoison() || ScalarRHS.isPoison()) {
470 ResVec.push_back(x: AnyValue::poison());
471 OverflowVec.push_back(x: AnyValue::poison());
472 continue;
473 }
474 auto [Res, Overflow] =
475 ScalarFn(ScalarLHS.asInteger(), ScalarRHS.asInteger());
476 ResVec.push_back(x: AnyValue(Res));
477 OverflowVec.push_back(x: AnyValue::boolean(Val: Overflow));
478 }
479 return std::vector<AnyValue>{AnyValue(std::move(ResVec)),
480 AnyValue(std::move(OverflowVec))};
481 }
482
483 AnyValue visitFPBinOpWithResult(
484 Type *RetTy, const FastMathFlags &FMF, const AnyValue &LHS,
485 const AnyValue &RHS,
486 function_ref<APFloat(const APFloat &, const APFloat &)> ScalarFn) {
487 DenormalMode DenormMode = getCurrentDenormalMode(Ty: RetTy);
488
489 if (!Ctx.isDefaultFPEnv())
490 reportImmediateUB() << "Non-constrained floating-point operation assumes "
491 "default floating-point environment";
492
493 return computeBinOp(
494 Ty: RetTy, LHS, RHS,
495 ScalarFn: [&](const AnyValue &LHSInner, const AnyValue &RHSInner) -> AnyValue {
496 if (LHSInner.isPoison() || RHSInner.isPoison())
497 return AnyValue::poison();
498
499 AnyValue ValidatedLHS =
500 handleFMFFlags(Val: LHSInner, FMF, /*IsInput=*/true);
501 AnyValue ValidatedRHS =
502 handleFMFFlags(Val: RHSInner, FMF, /*IsInput=*/true);
503 if (ValidatedLHS.isPoison())
504 return ValidatedLHS;
505 if (ValidatedRHS.isPoison())
506 return ValidatedRHS;
507
508 // Flush input denormals
509 APFloat FLHS = handleDenormal(Val: ValidatedLHS.asFloat(),
510 Mode: DenormMode.Input, /*IsInput=*/true);
511 APFloat FRHS = handleDenormal(Val: ValidatedRHS.asFloat(),
512 Mode: DenormMode.Input, /*IsInput=*/true);
513
514 APFloat RawResult = ScalarFn(FLHS, FRHS);
515
516 // Flush output denormals and handle fast-math flags.
517 AnyValue FResult = handleFMFFlags(
518 Val: handleDenormal(Val: RawResult, Mode: DenormMode.Output, /*IsInput=*/false),
519 FMF,
520 /*IsInput=*/false);
521
522 if (FResult.isPoison())
523 return FResult;
524
525 APFloat Result = FResult.asFloat();
526 return applyNaNPropagation(Result, Inputs: {&FLHS, &FRHS});
527 });
528 }
529
530 AnyValue
531 computeTriOp(Type *Ty, const AnyValue &Op1, const AnyValue &Op2,
532 const AnyValue &Op3,
533 function_ref<AnyValue(const AnyValue &, const AnyValue &,
534 const AnyValue &)>
535 ScalarFn) {
536 if (Ty->isVectorTy()) {
537 auto &Op1Vec = Op1.asAggregate();
538 auto &Op2Vec = Op2.asAggregate();
539 auto &Op3Vec = Op3.asAggregate();
540 std::vector<AnyValue> ResVec;
541 ResVec.reserve(n: Op1Vec.size());
542 for (const auto &[ScalarOp1, ScalarOp2, ScalarOp3] :
543 zip(t: Op1Vec, u: Op2Vec, args: Op3Vec))
544 ResVec.push_back(x: ScalarFn(ScalarOp1, ScalarOp2, ScalarOp3));
545 return std::move(ResVec);
546 }
547 return ScalarFn(Op1, Op2, Op3);
548 }
549
550 void visitTriOp(Instruction &I,
551 function_ref<AnyValue(const AnyValue &, const AnyValue &,
552 const AnyValue &)>
553 ScalarFn) {
554 setResult(I, V: computeTriOp(Ty: I.getType(), Op1: getValue(V: I.getOperand(i: 0)),
555 Op2: getValue(V: I.getOperand(i: 1)),
556 Op3: getValue(V: I.getOperand(i: 2)), ScalarFn));
557 }
558
559 void visitIntTriOp(
560 Instruction &I,
561 function_ref<AnyValue(const APInt &, const APInt &, const APInt &)>
562 ScalarFn) {
563 visitTriOp(I,
564 ScalarFn: [&](const AnyValue &Op1, const AnyValue &Op2,
565 const AnyValue &Op3) -> AnyValue {
566 if (Op1.isPoison() || Op2.isPoison() || Op3.isPoison())
567 return AnyValue::poison();
568 return ScalarFn(Op1.asInteger(), Op2.asInteger(),
569 Op3.asInteger());
570 });
571 }
572
573 AnyValue visitIntTriOpWithResult(
574 Type *RetTy, const AnyValue &Op1, const AnyValue &Op2,
575 const AnyValue &Op3,
576 function_ref<AnyValue(const APInt &, const APInt &, const APInt &)>
577 ScalarFn) {
578 return computeTriOp(
579 Ty: RetTy, Op1, Op2, Op3,
580 ScalarFn: [&](const AnyValue &Op1Inner, const AnyValue &Op2Inner,
581 const AnyValue &Op3Inner) -> AnyValue {
582 if (Op1Inner.isPoison() || Op2Inner.isPoison() || Op3Inner.isPoison())
583 return AnyValue::poison();
584 return ScalarFn(Op1Inner.asInteger(), Op2Inner.asInteger(),
585 Op3Inner.asInteger());
586 });
587 }
588
589 AnyValue visitFPTriOpWithResult(
590 Type *RetTy, const FastMathFlags &FMF, const AnyValue &Op1,
591 const AnyValue &Op2, const AnyValue &Op3,
592 function_ref<APFloat(const APFloat &, const APFloat &, const APFloat &)>
593 ScalarFn) {
594 DenormalMode DenormMode = getCurrentDenormalMode(Ty: RetTy);
595
596 if (!Ctx.isDefaultFPEnv())
597 reportImmediateUB() << "Non-constrained floating-point operation assumes "
598 "default floating-point environment";
599
600 return computeTriOp(
601 Ty: RetTy, Op1, Op2, Op3,
602 ScalarFn: [&](const AnyValue &Op1Inner, const AnyValue &Op2Inner,
603 const AnyValue &Op3Inner) -> AnyValue {
604 if (Op1Inner.isPoison() || Op2Inner.isPoison() || Op3Inner.isPoison())
605 return AnyValue::poison();
606
607 AnyValue ValidatedOp1 =
608 handleFMFFlags(Val: Op1Inner, FMF, /*IsInput=*/true);
609 AnyValue ValidatedOp2 =
610 handleFMFFlags(Val: Op2Inner, FMF, /*IsInput=*/true);
611 AnyValue ValidatedOp3 =
612 handleFMFFlags(Val: Op3Inner, FMF, /*IsInput=*/true);
613 if (ValidatedOp1.isPoison())
614 return ValidatedOp1;
615 if (ValidatedOp2.isPoison())
616 return ValidatedOp2;
617 if (ValidatedOp3.isPoison())
618 return ValidatedOp3;
619
620 // Flush input denormals
621 APFloat FOp1 = handleDenormal(Val: ValidatedOp1.asFloat(),
622 Mode: DenormMode.Input, /*IsInput=*/true);
623 APFloat FOp2 = handleDenormal(Val: ValidatedOp2.asFloat(),
624 Mode: DenormMode.Input, /*IsInput=*/true);
625 APFloat FOp3 = handleDenormal(Val: ValidatedOp3.asFloat(),
626 Mode: DenormMode.Input, /*IsInput=*/true);
627
628 APFloat RawResult = ScalarFn(FOp1, FOp2, FOp3);
629
630 // Flush output denormals and handle fast-math flags.
631 AnyValue FResult = handleFMFFlags(
632 Val: handleDenormal(Val: RawResult, Mode: DenormMode.Output, /*IsInput=*/false),
633 FMF,
634 /*IsInput=*/false);
635
636 if (FResult.isPoison())
637 return FResult;
638
639 APFloat Result = FResult.asFloat();
640 return applyNaNPropagation(Result, Inputs: {&FOp1, &FOp2, &FOp3});
641 });
642 }
643
644 void jumpTo(Instruction &Terminator, BasicBlock *DestBB) {
645 if (!Handler.onBBJump(I&: Terminator, To&: *DestBB)) {
646 setFailed();
647 return;
648 }
649 BasicBlock *From = CurrentFrame->BB;
650 CurrentFrame->BB = DestBB;
651 CurrentFrame->PC = DestBB->begin();
652 // Update PHI nodes in batch to avoid the interference between PHI nodes.
653 // We need to store the incoming values into a temporary buffer.
654 // Otherwise, the incoming value may be overwritten before it is
655 // used by other PHI nodes.
656 SmallVector<std::pair<PHINode *, AnyValue>> IncomingValues;
657 PHINode *PHI = nullptr;
658 while ((PHI = dyn_cast<PHINode>(Val&: CurrentFrame->PC))) {
659 AnyValue IncomingVal = getValue(V: PHI->getIncomingValueForBlock(BB: From));
660
661 // Fast-math flags validation
662 if (isa<FPMathOperator>(Val: PHI)) {
663 FastMathFlags FMF = PHI->getFastMathFlags();
664 if (FMF.any())
665 IncomingVal =
666 handleFMFFlags(Val: std::move(IncomingVal), FMF, /*IsInput=*/true);
667 }
668
669 IncomingValues.emplace_back(Args&: PHI, Args&: IncomingVal);
670 ++CurrentFrame->PC;
671 }
672 for (auto &[K, V] : IncomingValues)
673 setResult(I&: *K, V: std::move(V));
674 }
675
676 /// Helper function to determine whether an inline asm is a no-op, which is
677 /// used to implement black_box style optimization blockers.
678 bool isNoopInlineAsm(Value *V, Type *RetTy) {
679 if (auto *Asm = dyn_cast<InlineAsm>(Val: V))
680 return Asm->getAsmString().empty() && RetTy->isVoidTy();
681 return false;
682 }
683
684 DenormalMode getCurrentDenormalMode(Type *Ty) {
685 return CurrentFrame->Func.getDenormalMode(
686 FPType: Ty->getScalarType()->getFltSemantics());
687 }
688
689 // Helper function to convert BooleanKind to bool. Report an immediate UB if
690 // a poison is found.
691 bool getBooleanNonPoison(BooleanKind Boolean) {
692 if (Boolean == BooleanKind::Poison)
693 reportImmediateUB() << "Unexpected poison boolean value";
694 return Boolean == BooleanKind::True;
695 }
696
697 APInt getIntNonPoison(const AnyValue &V) {
698 if (V.isPoison()) {
699 reportImmediateUB() << "Unexpected poison integer value.";
700 return APInt::getZero(numBits: 64);
701 }
702 return V.asInteger();
703 }
704
705 AnyValue callMemTransferIntrinsic(CallBase &CB, ArrayRef<AnyValue> Args,
706 Intrinsic::ID IID) {
707 const AnyValue &Dest = Args[0];
708 const AnyValue &Src = Args[1];
709 const AnyValue &Length = Args[2];
710 // TODO: Handle isvolatile argument.
711 if (Length.isPoison()) {
712 reportImmediateUB() << "Memory transfer intrinsic with poison length.";
713 return AnyValue();
714 }
715
716 const APInt &LengthInt = Args[2].asInteger();
717 if (LengthInt.getActiveBits() > 64) {
718 reportImmediateUB()
719 << "Memory transfer intrinsic length overflows uint64_t.";
720 return AnyValue();
721 }
722
723 const uint64_t Len = LengthInt.getZExtValue();
724 if (Len == 0)
725 return AnyValue();
726
727 if (Dest.isPoison()) {
728 reportImmediateUB()
729 << "Memory transfer intrinsic with poison destination pointer.";
730 return AnyValue();
731 }
732
733 if (Src.isPoison()) {
734 reportImmediateUB()
735 << "Memory transfer intrinsic with poison source pointer.";
736 return AnyValue();
737 }
738
739 const Pointer &DstPtr = Dest.asPointer();
740 const Pointer &SrcPtr = Src.asPointer();
741
742 Align DstAlign = CB.getParamAlign(ArgNo: 0).valueOrOne();
743 Align SrcAlign = CB.getParamAlign(ArgNo: 1).valueOrOne();
744
745 auto [SrcMO, SrcOffset] =
746 verifyMemAccess(Ptr: SrcPtr, AccessSize: Len, Alignment: SrcAlign, /*IsStore=*/false);
747 if (!SrcMO)
748 return AnyValue();
749
750 auto [DstMO, DstOffset] =
751 verifyMemAccess(Ptr: DstPtr, AccessSize: Len, Alignment: DstAlign, /*IsStore=*/true);
752 if (!DstMO)
753 return AnyValue();
754
755 if (IID == Intrinsic::memcpy || IID == Intrinsic::memcpy_inline) {
756 if (SrcMO == DstMO && SrcOffset != DstOffset) {
757 const uint64_t SrcEnd = SrcOffset + Len;
758 const uint64_t DstEnd = DstOffset + Len;
759 if (SrcOffset < DstEnd && DstOffset < SrcEnd) {
760 reportImmediateUB()
761 << "memcpy with overlapping source and destination.";
762 return AnyValue();
763 }
764 }
765 }
766
767 MutableArrayRef<Byte> DstBytes = DstMO->getBytes().slice(N: DstOffset, M: Len);
768 ArrayRef<Byte> SrcBytes = SrcMO->getBytes().slice(N: SrcOffset, M: Len);
769 std::memmove(dest: DstBytes.data(), src: SrcBytes.data(), n: Len * sizeof(Byte));
770 return AnyValue();
771 }
772
773 AnyValue callMemSetIntrinsic(CallBase &CB, ArrayRef<AnyValue> Args) {
774 const AnyValue &Dest = Args[0];
775 const AnyValue &Val = Args[1];
776 const AnyValue &Length = Args[2];
777
778 if (Length.isPoison()) {
779 reportImmediateUB() << "memset called with poison length.";
780 return AnyValue();
781 }
782
783 const APInt &LengthInt = Length.asInteger();
784 if (LengthInt.getActiveBits() > 64) {
785 reportImmediateUB() << "memset called with length overflows uint64_t.";
786 return AnyValue();
787 }
788
789 const uint64_t Len = LengthInt.getZExtValue();
790 if (Len == 0)
791 return AnyValue();
792
793 if (Dest.isPoison()) {
794 reportImmediateUB() << "memset called with poison destination pointer.";
795 return AnyValue();
796 }
797
798 const Pointer &DstPtr = Dest.asPointer();
799
800 Align DstAlign = CB.getParamAlign(ArgNo: 0).valueOrOne();
801 auto [DstMO, DstOffset] =
802 verifyMemAccess(Ptr: DstPtr, AccessSize: Len, Alignment: DstAlign, /*IsStore=*/true);
803 if (!DstMO)
804 return AnyValue();
805
806 Byte FillByte = Val.isPoison()
807 ? Byte::poison()
808 : Byte::concrete(Val: Val.asInteger().getZExtValue());
809 fill(Range: DstMO->getBytes().slice(N: DstOffset, M: Len), Value&: FillByte);
810 return AnyValue();
811 }
812
813 static BooleanKind getMaskLane(const AnyValue &Mask, size_t I) {
814 return Mask.asAggregate()[I].asBoolean();
815 }
816
817 AnyValue callExperimentalVectorHistogramIntrinsic(CallBase &CB,
818 ArrayRef<AnyValue> Args,
819 Intrinsic::ID IID) {
820 struct LaneUpdate {
821 MemoryObject *MO;
822 uint64_t Offset;
823 uint64_t Count;
824 AnyValue Old;
825 AnyValue New;
826 };
827
828 const auto &Ptrs = Args[0].asAggregate();
829 const AnyValue &Update = Args[1];
830 const AnyValue &Mask = Args[2];
831 Type *ElemTy = CB.getArgOperand(i: 1)->getType();
832 const uint64_t AccessSize = Ctx.getEffectiveTypeStoreSize(Ty: ElemTy);
833
834 SmallVector<LaneUpdate, 8> Lanes;
835 Lanes.reserve(N: Ptrs.size());
836 for (size_t I = 0, E = Ptrs.size(); I != E; ++I) {
837 switch (getMaskLane(Mask, I)) {
838 case BooleanKind::False:
839 continue;
840 case BooleanKind::Poison:
841 reportImmediateUB()
842 << "Poison mask lane in experimental vector histogram intrinsic.";
843 return AnyValue();
844 case BooleanKind::True:
845 break;
846 }
847
848 if (Ptrs[I].isPoison()) {
849 reportImmediateUB() << "Poison pointer lane in experimental vector "
850 "histogram intrinsic.";
851 return AnyValue();
852 }
853
854 auto [MO, Offset] =
855 verifyMemAccess(Ptr: Ptrs[I].asPointer(), AccessSize, Alignment: Align(1),
856 /*IsStore=*/true);
857 if (!MO)
858 return AnyValue();
859
860 Lanes.push_back(Elt: {.MO: MO, .Offset: Offset, .Count: 0, .Old: AnyValue(), .New: AnyValue()});
861 }
862
863 for (LaneUpdate &Lane : Lanes) {
864 Lane.Count = count_if(Range&: Lanes, P: [&](const LaneUpdate &Other) {
865 return Other.MO == Lane.MO && Other.Offset == Lane.Offset;
866 });
867 Lane.Old = Ctx.load(MO&: *Lane.MO, Offset: Lane.Offset, ValTy: ElemTy);
868 }
869
870 for (LaneUpdate &Lane : Lanes) {
871 const AnyValue &Old = Lane.Old;
872 AnyValue &New = Lane.New;
873
874 if (Old.isPoison() || Update.isPoison()) {
875 New = AnyValue::poison();
876 } else {
877 const APInt &OldInt = Old.asInteger();
878 const APInt &UpdateInt = Update.asInteger();
879
880 switch (IID) {
881 case Intrinsic::experimental_vector_histogram_add:
882 New = OldInt + UpdateInt * APInt(UpdateInt.getBitWidth(), Lane.Count,
883 /*isSigned=*/false,
884 /*implicitTrunc=*/true);
885 break;
886 case Intrinsic::experimental_vector_histogram_uadd_sat: {
887 APInt Acc = OldInt;
888 for (uint64_t I = 0; I != Lane.Count; ++I)
889 Acc = Acc.uadd_sat(RHS: UpdateInt);
890 New = Acc;
891 break;
892 }
893 case Intrinsic::experimental_vector_histogram_umax:
894 New = APIntOps::umax(A: OldInt, B: UpdateInt);
895 break;
896 case Intrinsic::experimental_vector_histogram_umin:
897 New = APIntOps::umin(A: OldInt, B: UpdateInt);
898 break;
899 default:
900 llvm_unreachable("Unexpected histogram intrinsic ID");
901 }
902 }
903 }
904
905 for (const LaneUpdate &Lane : Lanes)
906 Ctx.store(MO&: *Lane.MO, Offset: Lane.Offset, Val: Lane.New, ValTy: ElemTy);
907
908 return AnyValue();
909 }
910
911public:
912 InstExecutor(Context &C, EventHandler &H, Function &F,
913 ArrayRef<AnyValue> Args, AnyValue &RetVal)
914 : ExecutorBase(C, H), DL(Ctx.getDataLayout()),
915 Lib(Ctx, Handler, DL, static_cast<ExecutorBase &>(*this)) {
916 CallStack.emplace_back(args&: F, /*CallSite=*/args: nullptr, /*LastFrame=*/args: nullptr, args&: Args,
917 args&: RetVal, args: Ctx.getTLIImpl());
918 }
919
920 void visitReturnInst(ReturnInst &RI) {
921 if (auto *RV = RI.getReturnValue())
922 CurrentFrame->RetVal = getValue(V: RV);
923 else
924 CurrentFrame->RetVal = AnyValue();
925 CurrentFrame->State = FrameState::Exit;
926 if (!Handler.onInstructionExecuted(I&: RI, Result: None))
927 setFailed();
928 }
929
930 void visitUncondBrInst(UncondBrInst &BI) { jumpTo(Terminator&: BI, DestBB: BI.getSuccessor()); }
931
932 void visitCondBrInst(CondBrInst &BI) {
933 switch (getValue(V: BI.getCondition()).asBoolean()) {
934 case BooleanKind::True:
935 jumpTo(Terminator&: BI, DestBB: BI.getSuccessor(i: 0));
936 return;
937 case BooleanKind::False:
938 jumpTo(Terminator&: BI, DestBB: BI.getSuccessor(i: 1));
939 return;
940 case BooleanKind::Poison:
941 reportImmediateUB() << "Branch on poison condition.";
942 return;
943 }
944 }
945
946 void visitSwitchInst(SwitchInst &SI) {
947 auto &Cond = getValue(V: SI.getCondition());
948 if (Cond.isPoison()) {
949 reportImmediateUB() << "Switch on poison condition.";
950 return;
951 }
952 for (auto &Case : SI.cases()) {
953 if (Case.getCaseValue()->getValue() == Cond.asInteger()) {
954 jumpTo(Terminator&: SI, DestBB: Case.getCaseSuccessor());
955 return;
956 }
957 }
958 jumpTo(Terminator&: SI, DestBB: SI.getDefaultDest());
959 }
960
961 void visitUnreachableInst(UnreachableInst &) {
962 reportImmediateUB() << "Unreachable code.";
963 }
964
965 void visitCallBrInst(CallBrInst &CI) {
966 if (isNoopInlineAsm(V: CI.getCalledOperand(), RetTy: CI.getType())) {
967 jumpTo(Terminator&: CI, DestBB: CI.getDefaultDest());
968 return;
969 }
970
971 Handler.onUnrecognizedInstruction(I&: CI);
972 setFailed();
973 }
974
975 void visitIndirectBrInst(IndirectBrInst &IBI) {
976 auto &Target = getValue(V: IBI.getAddress());
977 if (Target.isPoison()) {
978 reportImmediateUB() << "Indirect branch on poison.";
979 return;
980 }
981 if (BasicBlock *DestBB = Ctx.getTargetBlock(Ptr: Target.asPointer())) {
982 if (any_of(Range: IBI.successors(),
983 P: [DestBB](BasicBlock *Succ) { return Succ == DestBB; }))
984 jumpTo(Terminator&: IBI, DestBB);
985 else
986 reportImmediateUB() << "Indirect branch on unlisted target BB.";
987
988 return;
989 }
990 reportImmediateUB() << "Indirect branch on invalid target BB.";
991 }
992
993 void returnFromCallee() {
994 auto &CB = cast<CallBase>(Val&: *CurrentFrame->PC);
995 CurrentFrame->CalleeArgs.clear();
996 AnyValue &RetVal = CurrentFrame->CalleeRetVal;
997 if (Type *RetTy = CB.getType(); !RetTy->isVoidTy()) {
998 // Handle attributes on the return value (Attributes from resolved callee
999 // should be applied if available).
1000 AttributeSet AttrsAtCallSite = CB.getRetAttributes();
1001 AttributeSet AttrsAtCallee =
1002 CurrentFrame->ResolvedCallee->getAttributes().getRetAttrs();
1003 handleAttributes(Ty: RetTy, V&: RetVal, AttrsAtCallSite, AttrsAtCallee);
1004 handleMetadata(Ty: RetTy, V&: RetVal, I&: CB);
1005 }
1006 setResult(I&: CB, V: std::move(RetVal));
1007
1008 for (auto &ByValArg : CurrentFrame->CalleeByValArgs)
1009 Ctx.free(Obj: *ByValArg);
1010 CurrentFrame->CalleeByValArgs.clear();
1011
1012 if (auto *II = dyn_cast<InvokeInst>(Val: &CB))
1013 jumpTo(Terminator&: *II, DestBB: II->getNormalDest());
1014 else if (CurrentFrame->State == FrameState::Pending)
1015 ++CurrentFrame->PC;
1016 }
1017
1018 AnyValue callIntrinsic(CallBase &CB, ArrayRef<AnyValue> Args) {
1019 Intrinsic::ID IID = CB.getIntrinsicID();
1020 Type *RetTy = CB.getType();
1021 const FastMathFlags FMF = CB.getFastMathFlagsOrNone();
1022
1023 switch (IID) {
1024 case Intrinsic::assume:
1025 switch (Args[0].asBoolean()) {
1026 case BooleanKind::True:
1027 for (unsigned Idx = 0; Idx < CB.getNumOperandBundles(); Idx++) {
1028 OperandBundleUse OBU = CB.getOperandBundleAt(Index: Idx);
1029 auto GetBundleArg = [&](uint32_t Offset) -> Value * {
1030 return OBU.Inputs[Offset];
1031 };
1032 if (OBU.Inputs.empty())
1033 continue;
1034 Value *WasOnVal = GetBundleArg(0);
1035 // Bail out on unrecognized operand bundles.
1036 if (!WasOnVal->getType()->isPointerTy())
1037 continue;
1038 unsigned AS = WasOnVal->getType()->getPointerAddressSpace();
1039 const AnyValue &WasOn = getValue(V: WasOnVal);
1040 if (WasOn.isPoison()) {
1041 reportImmediateUB() << "Assume on poison pointer.";
1042 break;
1043 }
1044 const Pointer &WasOnPtr = WasOn.asPointer();
1045 Attribute::AttrKind Kind =
1046 Attribute::getAttrKindFromName(AttrName: OBU.getTagName());
1047 switch (Kind) {
1048 case Attribute::Alignment: {
1049 // Alignment assumptions should have 2 or 3 arguments.
1050 APInt Alignment = getIntNonPoison(V: getValue(V: GetBundleArg(1)));
1051 APInt CheckedAddr = WasOnPtr.address();
1052 if (OBU.Inputs.size() == 3) {
1053 APInt Offset = getIntNonPoison(V: getValue(V: GetBundleArg(2)));
1054 CheckedAddr -= Offset.sextOrTrunc(width: CheckedAddr.getBitWidth());
1055 }
1056 if (!Alignment.isPowerOf2()) {
1057 if (!CheckedAddr.isZero())
1058 reportImmediateUB() << "Assume on pointer " << WasOn
1059 << " with a nonzero adjusted address and a "
1060 "non-power-of-two alignment "
1061 << Alignment << '.';
1062 break;
1063 }
1064 if (CheckedAddr.countr_zero() < Alignment.logBase2())
1065 reportImmediateUB()
1066 << "The pointer " << WasOn << " violates align(" << Alignment
1067 << ") assumption.";
1068 break;
1069 }
1070 case Attribute::NonNull:
1071 if (WasOnPtr.isNullPtr(AS, DL))
1072 reportImmediateUB()
1073 << "The pointer " << WasOn << " violates nonnull assumption.";
1074 break;
1075 case Attribute::Dereferenceable:
1076 case Attribute::DereferenceableOrNull: {
1077 APInt DereferenceableBytes =
1078 getIntNonPoison(V: getValue(V: GetBundleArg(1)));
1079 // Only n > 0 implies that the pointer is dereferenceable.
1080 if (DereferenceableBytes.isZero())
1081 break;
1082 if (violatesDereferenceableBytesAttr(
1083 V: WasOn, Bytes: DereferenceableBytes.getLimitedValue(),
1084 OrNull: Kind == Attribute::DereferenceableOrNull, AS, Ctx))
1085 reportImmediateUB() << "The pointer " << WasOn << " violates "
1086 << (Kind == Attribute::DereferenceableOrNull
1087 ? "dereferenceable_or_null("
1088 : "dereferenceable(")
1089 << DereferenceableBytes << ") assumption.";
1090 break;
1091 }
1092 default:
1093 // TODO: handle other operand bundles like separate_storage.
1094 break;
1095 }
1096 }
1097 break;
1098 case BooleanKind::False:
1099 case BooleanKind::Poison:
1100 reportImmediateUB() << "Assume on false or poison condition.";
1101 break;
1102 }
1103 return AnyValue();
1104 case Intrinsic::lifetime_start:
1105 case Intrinsic::lifetime_end: {
1106 auto Ptr = Args[0];
1107 if (Ptr.isPoison())
1108 return AnyValue();
1109 auto *MO = Ctx.checkProvenance(Ptr: Ptr.asPointer(),
1110 Check: [](const Provenance &) { return true; });
1111 assert(MO && "Memory object accessed by lifetime intrinsic should be "
1112 "always valid.");
1113 if (IID == Intrinsic::lifetime_start) {
1114 MO->setState(MemoryObjectState::Alive);
1115 fill(Range: MO->getBytes(), Value: Byte::undef());
1116 } else {
1117 fill(Range: MO->getBytes(), Value: Byte::poison());
1118 MO->setState(MemoryObjectState::Dead);
1119 }
1120 return AnyValue();
1121 }
1122 case Intrinsic::ssa_copy:
1123 case Intrinsic::expect:
1124 case Intrinsic::expect_with_probability:
1125 return Args[0];
1126 case Intrinsic::donothing:
1127 return AnyValue();
1128 case Intrinsic::vscale: {
1129 const unsigned BitWidth = RetTy->getScalarSizeInBits();
1130 const APInt VScale(64, Ctx.getVScale());
1131 if (!VScale.isIntN(N: BitWidth))
1132 return AnyValue::poison();
1133 return VScale.zextOrTrunc(width: BitWidth);
1134 }
1135 case Intrinsic::abs: {
1136 const bool IsIntMinPoison = getBooleanNonPoison(Boolean: Args[1].asBoolean());
1137 return visitIntUnOpWithResult(
1138 RetTy, Operand: Args[0], ScalarFn: [&](const APInt &Operand) -> AnyValue {
1139 if (IsIntMinPoison && Operand.isMinSignedValue())
1140 return AnyValue::poison();
1141 return Operand.abs();
1142 });
1143 }
1144 case Intrinsic::smax: {
1145 return visitIntBinOpWithResult(
1146 RetTy, LHS: Args[0], RHS: Args[1],
1147 ScalarFn: [](const APInt &LHS, const APInt &RHS) -> AnyValue {
1148 return APIntOps::smax(A: LHS, B: RHS);
1149 });
1150 }
1151 case Intrinsic::smin: {
1152 return visitIntBinOpWithResult(
1153 RetTy, LHS: Args[0], RHS: Args[1],
1154 ScalarFn: [](const APInt &LHS, const APInt &RHS) -> AnyValue {
1155 return APIntOps::smin(A: LHS, B: RHS);
1156 });
1157 }
1158 case Intrinsic::umax: {
1159 return visitIntBinOpWithResult(
1160 RetTy, LHS: Args[0], RHS: Args[1],
1161 ScalarFn: [](const APInt &LHS, const APInt &RHS) -> AnyValue {
1162 return APIntOps::umax(A: LHS, B: RHS);
1163 });
1164 }
1165 case Intrinsic::umin: {
1166 return visitIntBinOpWithResult(
1167 RetTy, LHS: Args[0], RHS: Args[1],
1168 ScalarFn: [](const APInt &LHS, const APInt &RHS) -> AnyValue {
1169 return APIntOps::umin(A: LHS, B: RHS);
1170 });
1171 }
1172 case Intrinsic::scmp:
1173 case Intrinsic::ucmp: {
1174 const unsigned BitWidth = RetTy->getScalarSizeInBits();
1175 return visitIntBinOpWithResult(
1176 RetTy, LHS: Args[0], RHS: Args[1],
1177 ScalarFn: [&](const APInt &LHS, const APInt &RHS) -> AnyValue {
1178 if (LHS == RHS)
1179 return APInt::getZero(numBits: BitWidth);
1180 if (IID == Intrinsic::scmp)
1181 return LHS.slt(RHS) ? APInt::getAllOnes(numBits: BitWidth)
1182 : APInt(BitWidth, 1);
1183 return LHS.ult(RHS) ? APInt::getAllOnes(numBits: BitWidth)
1184 : APInt(BitWidth, 1);
1185 });
1186 }
1187 case Intrinsic::bitreverse: {
1188 return visitIntUnOpWithResult(RetTy, Operand: Args[0],
1189 ScalarFn: [](const APInt &Operand) -> AnyValue {
1190 return Operand.reverseBits();
1191 });
1192 }
1193 case Intrinsic::bswap: {
1194 return visitIntUnOpWithResult(
1195 RetTy, Operand: Args[0],
1196 ScalarFn: [](const APInt &Operand) -> AnyValue { return Operand.byteSwap(); });
1197 }
1198 case Intrinsic::ctpop: {
1199 return visitIntUnOpWithResult(
1200 RetTy, Operand: Args[0], ScalarFn: [](const APInt &Operand) -> AnyValue {
1201 return APInt(Operand.getBitWidth(), Operand.popcount());
1202 });
1203 }
1204 case Intrinsic::ctlz:
1205 case Intrinsic::cttz: {
1206 const bool IsZeroPoison = getBooleanNonPoison(Boolean: Args[1].asBoolean());
1207 return visitIntUnOpWithResult(
1208 RetTy, Operand: Args[0], ScalarFn: [&](const APInt &Operand) -> AnyValue {
1209 if (IsZeroPoison && Operand.isZero())
1210 return AnyValue::poison();
1211 if (IID == Intrinsic::ctlz)
1212 return APInt(Operand.getBitWidth(), Operand.countl_zero());
1213 return APInt(Operand.getBitWidth(), Operand.countr_zero());
1214 });
1215 }
1216 case Intrinsic::fshl:
1217 case Intrinsic::fshr: {
1218 return visitIntTriOpWithResult(
1219 RetTy, Op1: Args[0], Op2: Args[1], Op3: Args[2],
1220 ScalarFn: [IID](const APInt &Op1, const APInt &Op2,
1221 const APInt &Op3) -> AnyValue {
1222 const unsigned BitWidth = Op1.getBitWidth();
1223 const uint64_t ShiftAmount = Op3.urem(RHS: BitWidth);
1224 const bool IsFShr = IID == Intrinsic::fshr;
1225 if (ShiftAmount == 0)
1226 return IsFShr ? Op2 : Op1;
1227 const uint64_t LShrAmount =
1228 IsFShr ? ShiftAmount : BitWidth - ShiftAmount;
1229 const uint64_t ShlAmount =
1230 !IsFShr ? ShiftAmount : BitWidth - ShiftAmount;
1231 return Op1.shl(shiftAmt: ShlAmount) | Op2.lshr(shiftAmt: LShrAmount);
1232 });
1233 }
1234 case Intrinsic::clmul: {
1235 return visitIntBinOpWithResult(
1236 RetTy, LHS: Args[0], RHS: Args[1],
1237 ScalarFn: [](const APInt &LHS, const APInt &RHS) -> AnyValue {
1238 return APIntOps::clmul(LHS, RHS);
1239 });
1240 }
1241 case Intrinsic::sadd_with_overflow:
1242 case Intrinsic::uadd_with_overflow:
1243 case Intrinsic::ssub_with_overflow:
1244 case Intrinsic::usub_with_overflow:
1245 case Intrinsic::smul_with_overflow:
1246 case Intrinsic::umul_with_overflow: {
1247 return visitOverflowIntBinOpWithResult(
1248 RetTy, LHS: Args[0], RHS: Args[1],
1249 ScalarFn: [IID](const APInt &LHS, const APInt &RHS) -> std::pair<APInt, bool> {
1250 APInt Res;
1251 bool Overflow = false;
1252 switch (IID) {
1253 case Intrinsic::sadd_with_overflow:
1254 Res = LHS.sadd_ov(RHS, Overflow);
1255 break;
1256 case Intrinsic::uadd_with_overflow:
1257 Res = LHS.uadd_ov(RHS, Overflow);
1258 break;
1259 case Intrinsic::ssub_with_overflow:
1260 Res = LHS.ssub_ov(RHS, Overflow);
1261 break;
1262 case Intrinsic::usub_with_overflow:
1263 Res = LHS.usub_ov(RHS, Overflow);
1264 break;
1265 case Intrinsic::smul_with_overflow:
1266 Res = LHS.smul_ov(RHS, Overflow);
1267 break;
1268 case Intrinsic::umul_with_overflow:
1269 Res = LHS.umul_ov(RHS, Overflow);
1270 break;
1271 default:
1272 llvm_unreachable("Unexpected intrinsic ID");
1273 }
1274 return {Res, Overflow};
1275 });
1276 }
1277 case Intrinsic::sadd_sat:
1278 case Intrinsic::uadd_sat:
1279 case Intrinsic::ssub_sat:
1280 case Intrinsic::usub_sat:
1281 case Intrinsic::sshl_sat:
1282 case Intrinsic::ushl_sat: {
1283 return visitIntBinOpWithResult(
1284 RetTy, LHS: Args[0], RHS: Args[1],
1285 ScalarFn: [IID](const APInt &LHS, const APInt &RHS) -> AnyValue {
1286 switch (IID) {
1287 case Intrinsic::sadd_sat:
1288 return LHS.sadd_sat(RHS);
1289 case Intrinsic::uadd_sat:
1290 return LHS.uadd_sat(RHS);
1291 case Intrinsic::ssub_sat:
1292 return LHS.ssub_sat(RHS);
1293 case Intrinsic::usub_sat:
1294 return LHS.usub_sat(RHS);
1295 case Intrinsic::sshl_sat: {
1296 if (RHS.uge(RHS: LHS.getBitWidth()))
1297 return AnyValue::poison();
1298 return LHS.sshl_sat(RHS);
1299 }
1300 case Intrinsic::ushl_sat: {
1301 if (RHS.uge(RHS: LHS.getBitWidth()))
1302 return AnyValue::poison();
1303 return LHS.ushl_sat(RHS);
1304 }
1305 default:
1306 llvm_unreachable("Unexpected intrinsic ID");
1307 }
1308 });
1309 }
1310 case Intrinsic::vector_reduce_add:
1311 case Intrinsic::vector_reduce_mul:
1312 case Intrinsic::vector_reduce_and:
1313 case Intrinsic::vector_reduce_or:
1314 case Intrinsic::vector_reduce_xor:
1315 case Intrinsic::vector_reduce_smax:
1316 case Intrinsic::vector_reduce_smin:
1317 case Intrinsic::vector_reduce_umax:
1318 case Intrinsic::vector_reduce_umin: {
1319 std::optional<APInt> Res;
1320 for (const auto &V : Args[0].asAggregate()) {
1321 if (V.isPoison()) {
1322 Res.reset();
1323 break;
1324 }
1325 const auto &IntV = V.asInteger();
1326 if (!Res) {
1327 Res = IntV;
1328 continue;
1329 }
1330 switch (IID) {
1331 case Intrinsic::vector_reduce_add:
1332 *Res += IntV;
1333 break;
1334 case Intrinsic::vector_reduce_mul:
1335 *Res *= IntV;
1336 break;
1337 case Intrinsic::vector_reduce_and:
1338 *Res &= IntV;
1339 break;
1340 case Intrinsic::vector_reduce_or:
1341 *Res |= IntV;
1342 break;
1343 case Intrinsic::vector_reduce_xor:
1344 *Res ^= IntV;
1345 break;
1346 case Intrinsic::vector_reduce_smax:
1347 *Res = APIntOps::smax(A: *Res, B: IntV);
1348 break;
1349 case Intrinsic::vector_reduce_smin:
1350 *Res = APIntOps::smin(A: *Res, B: IntV);
1351 break;
1352 case Intrinsic::vector_reduce_umax:
1353 *Res = APIntOps::umax(A: *Res, B: IntV);
1354 break;
1355 case Intrinsic::vector_reduce_umin:
1356 *Res = APIntOps::umin(A: *Res, B: IntV);
1357 break;
1358 default:
1359 llvm_unreachable("Unexpected intrinsic ID");
1360 }
1361 }
1362 return Res ? *Res : AnyValue::poison();
1363 }
1364 case Intrinsic::vector_insert: {
1365 assert(!Args[2].isPoison() &&
1366 "Verifier should reject poison vector_insert immarg.");
1367 const auto &Vec = Args[0].asAggregate();
1368 const auto &SubVec = Args[1].asAggregate();
1369 const auto &Idx = Args[2].asInteger();
1370 auto EC =
1371 cast<VectorType>(Val: CB.getArgOperand(i: 1)->getType())->getElementCount();
1372 const uint64_t RawOffset = Idx.getZExtValue();
1373 const uint32_t MinSize = EC.getKnownMinValue();
1374 assert(RawOffset % MinSize == 0 &&
1375 "Verifier should reject misaligned vector_insert index.");
1376 const uint64_t Chunk = RawOffset / MinSize;
1377 const uint64_t EVL = Ctx.getEVL(EC);
1378 if (Chunk > std::numeric_limits<uint64_t>::max() / EVL)
1379 return AnyValue::getPoisonValue(Ctx, Ty: RetTy);
1380 const uint64_t Offset = Chunk * EVL;
1381 if (Offset > Vec.size() || SubVec.size() > Vec.size() - Offset)
1382 return AnyValue::getPoisonValue(Ctx, Ty: RetTy);
1383 std::vector<AnyValue> Res;
1384 Res.reserve(n: Vec.size());
1385 for (size_t I = 0; I != Vec.size(); ++I) {
1386 if (I >= Offset && I < Offset + SubVec.size())
1387 Res.push_back(x: SubVec[I - Offset]);
1388 else
1389 Res.push_back(x: Vec[I]);
1390 }
1391 return std::move(Res);
1392 }
1393 case Intrinsic::vector_extract: {
1394 assert(!Args[1].isPoison() &&
1395 "Verifier should reject poison vector_extract immarg.");
1396 const auto &Vec = Args[0].asAggregate();
1397 const auto &Idx = Args[1].asInteger();
1398 auto EC = cast<VectorType>(Val: RetTy)->getElementCount();
1399 const uint64_t RawOffset = Idx.getZExtValue();
1400 const uint32_t MinSize = EC.getKnownMinValue();
1401 assert(RawOffset % MinSize == 0 &&
1402 "Verifier should reject misaligned vector_extract index.");
1403 const uint64_t Chunk = RawOffset / MinSize;
1404 const uint64_t EVL = Ctx.getEVL(EC);
1405 if (Chunk > std::numeric_limits<uint64_t>::max() / EVL)
1406 return AnyValue::getPoisonValue(Ctx, Ty: RetTy);
1407 const uint64_t Offset = Chunk * EVL;
1408 if (Offset > Vec.size() || EVL > Vec.size() - Offset)
1409 return AnyValue::getPoisonValue(Ctx, Ty: RetTy);
1410 return std::vector<AnyValue>(Vec.begin() + Offset,
1411 Vec.begin() + Offset + EVL);
1412 }
1413 case Intrinsic::vector_reverse: {
1414 auto Vec = Args[0].asAggregate();
1415 std::reverse(first: Vec.begin(), last: Vec.end());
1416 return std::move(Vec);
1417 }
1418 case Intrinsic::vector_deinterleave2:
1419 case Intrinsic::vector_deinterleave3:
1420 case Intrinsic::vector_deinterleave4:
1421 case Intrinsic::vector_deinterleave5:
1422 case Intrinsic::vector_deinterleave6:
1423 case Intrinsic::vector_deinterleave7:
1424 case Intrinsic::vector_deinterleave8: {
1425 const unsigned Factor = getDeinterleaveIntrinsicFactor(ID: IID);
1426 if (Factor == 0)
1427 llvm_unreachable("Unexpected intrinsic ID");
1428 const auto &Vec = Args[0].asAggregate();
1429 std::vector<std::vector<AnyValue>> Res(Factor);
1430 for (auto &SubVec : Res)
1431 SubVec.reserve(n: Vec.size() / Factor);
1432 for (size_t I = 0, E = Vec.size(); I != E; ++I)
1433 Res[I % Factor].push_back(x: Vec[I]);
1434
1435 std::vector<AnyValue> AggRes;
1436 AggRes.reserve(n: Factor);
1437 for (auto &SubVec : Res)
1438 AggRes.emplace_back(args: std::move(SubVec));
1439 return AnyValue(std::move(AggRes));
1440 }
1441 case Intrinsic::vector_interleave2:
1442 case Intrinsic::vector_interleave3:
1443 case Intrinsic::vector_interleave4:
1444 case Intrinsic::vector_interleave5:
1445 case Intrinsic::vector_interleave6:
1446 case Intrinsic::vector_interleave7:
1447 case Intrinsic::vector_interleave8: {
1448 const unsigned Factor = getInterleaveIntrinsicFactor(ID: IID);
1449 if (Factor == 0)
1450 llvm_unreachable("Unexpected intrinsic ID");
1451 const auto &Vec = Args[0].asAggregate();
1452 std::vector<AnyValue> Res;
1453 Res.reserve(n: Vec.size() * Factor);
1454 for (size_t I = 0, E = Vec.size(); I != E; ++I) {
1455 for (unsigned J = 0; J != Factor; ++J)
1456 Res.push_back(x: Args[J].asAggregate()[I]);
1457 }
1458 return std::move(Res);
1459 }
1460 case Intrinsic::vector_splice_left: {
1461 if (Args[2].isPoison())
1462 return AnyValue::getPoisonValue(Ctx, Ty: RetTy);
1463 const auto &LHS = Args[0].asAggregate();
1464 const auto &RHS = Args[1].asAggregate();
1465 const auto &Off = Args[2].asInteger();
1466 const size_t Len = LHS.size();
1467 if (Off.ugt(RHS: Len))
1468 return AnyValue::getPoisonValue(Ctx, Ty: RetTy);
1469 uint64_t Offset = Off.getZExtValue();
1470 std::vector<AnyValue> Res;
1471 Res.reserve(n: Len);
1472 for (size_t I = 0; I != Len; ++I) {
1473 size_t Pos = I + Offset;
1474 Res.push_back(x: Pos < Len ? LHS[Pos] : RHS[Pos - Len]);
1475 }
1476 return std::move(Res);
1477 }
1478 case Intrinsic::vector_splice_right: {
1479 if (Args[2].isPoison())
1480 return AnyValue::getPoisonValue(Ctx, Ty: RetTy);
1481 const auto &LHS = Args[0].asAggregate();
1482 const auto &RHS = Args[1].asAggregate();
1483 const auto &Off = Args[2].asInteger();
1484 const size_t Len = LHS.size();
1485 if (Off.ugt(RHS: Len))
1486 return AnyValue::getPoisonValue(Ctx, Ty: RetTy);
1487 uint64_t Offset = Len - Off.getZExtValue();
1488 std::vector<AnyValue> Res;
1489 Res.reserve(n: Len);
1490 for (size_t I = 0; I != Len; ++I) {
1491 size_t Pos = I + Offset;
1492 Res.push_back(x: Pos < Len ? LHS[Pos] : RHS[Pos - Len]);
1493 }
1494 return std::move(Res);
1495 }
1496 case Intrinsic::stepvector: {
1497 std::vector<AnyValue> Res;
1498 const uint32_t Len =
1499 Ctx.getEVL(EC: cast<VectorType>(Val: RetTy)->getElementCount());
1500 const unsigned BitWidth = RetTy->getScalarSizeInBits();
1501 Res.reserve(n: Len);
1502 for (uint64_t I = 0; I != Len; ++I) {
1503 Res.push_back(
1504 x: APInt(BitWidth, I, /*IsSigned=*/false, /*ImplicitTrunc=*/true));
1505 }
1506 return std::move(Res);
1507 }
1508 case Intrinsic::vector_reduce_fadd:
1509 case Intrinsic::vector_reduce_fmul:
1510 case Intrinsic::vector_reduce_fmaximum:
1511 case Intrinsic::vector_reduce_fminimum: {
1512 const auto DenormMode = getCurrentDenormalMode(Ty: RetTy);
1513 const bool HasStart = IID == Intrinsic::vector_reduce_fadd ||
1514 IID == Intrinsic::vector_reduce_fmul;
1515 const AnyValue &Vector = HasStart ? Args[1] : Args[0];
1516 std::optional<APFloat> Res;
1517 if (HasStart) {
1518 if (Args[0].isPoison())
1519 return AnyValue::poison();
1520 const AnyValue ValidatedStart =
1521 handleFMFFlags(Val: Args[0], FMF, /*IsInput=*/true);
1522 if (ValidatedStart.isPoison())
1523 return AnyValue::poison();
1524 Res = handleDenormal(Val: ValidatedStart.asFloat(), Mode: DenormMode.Input,
1525 /*IsInput=*/true);
1526 }
1527 for (const auto &V : Vector.asAggregate()) {
1528 if (V.isPoison())
1529 return AnyValue::poison();
1530 const AnyValue ValidatedOp = handleFMFFlags(Val: V, FMF, /*IsInput=*/true);
1531 if (ValidatedOp.isPoison())
1532 return AnyValue::poison();
1533 APFloat Op = handleDenormal(Val: ValidatedOp.asFloat(), Mode: DenormMode.Input,
1534 /*IsInput=*/true);
1535 if (!Res) {
1536 Res = std::move(Op);
1537 continue;
1538 }
1539 switch (IID) {
1540 case Intrinsic::vector_reduce_fadd:
1541 *Res = *Res + Op;
1542 break;
1543 case Intrinsic::vector_reduce_fmul:
1544 *Res = *Res * Op;
1545 break;
1546 case Intrinsic::vector_reduce_fmaximum:
1547 *Res = maximum(A: *Res, B: Op);
1548 break;
1549 case Intrinsic::vector_reduce_fminimum:
1550 *Res = minimum(A: *Res, B: Op);
1551 break;
1552 default:
1553 llvm_unreachable("Unexpected intrinsic ID");
1554 }
1555 }
1556 assert(Res.has_value());
1557 const AnyValue ValidatedRes =
1558 handleFMFFlags(Val: *Res, FMF, /*IsInput=*/false);
1559 if (ValidatedRes.isPoison())
1560 return AnyValue::poison();
1561 const APFloat FRes =
1562 handleDenormal(Val: ValidatedRes.asFloat(), Mode: DenormMode.Output,
1563 /*IsInput=*/false);
1564 SmallVector<const APFloat *, 8> InputVec;
1565 InputVec.reserve(N: Vector.asAggregate().size());
1566 transform(
1567 Range: Vector.asAggregate(), d_first: std::back_inserter(x&: InputVec),
1568 F: [](const AnyValue &V) -> const APFloat * { return &V.asFloat(); });
1569 return applyNaNPropagation(Result: FRes, Inputs: InputVec);
1570 }
1571 case Intrinsic::vector_reduce_fmax:
1572 case Intrinsic::vector_reduce_fmin: {
1573 const auto DenormMode = getCurrentDenormalMode(Ty: RetTy);
1574 const auto &Vector = Args[0].asAggregate();
1575 SmallVector<APFloat, 8> InputFloats;
1576 SmallVector<const APFloat *, 8> InputVec;
1577 InputFloats.reserve(N: Vector.size());
1578 InputVec.reserve(N: Vector.size());
1579 for (const auto &V : Vector) {
1580 if (V.isPoison())
1581 return AnyValue::poison();
1582 const AnyValue ValidatedOp = handleFMFFlags(Val: V, FMF, /*IsInput=*/true);
1583 if (ValidatedOp.isPoison())
1584 return AnyValue::poison();
1585 InputFloats.push_back(Elt: handleDenormal(Val: ValidatedOp.asFloat(),
1586 Mode: DenormMode.Input,
1587 /*IsInput=*/true));
1588 InputVec.push_back(Elt: &InputFloats.back());
1589 }
1590 assert(!InputVec.empty());
1591 SmallVector<APFloat, 8> Worklist(InputFloats);
1592 const bool HasSNaN =
1593 any_of(Range&: InputVec, P: [](const APFloat *V) { return V->isSignaling(); });
1594 while (Worklist.size() > 1) {
1595 size_t LHSIdx = 0;
1596 size_t RHSIdx = 1;
1597 if (HasSNaN) {
1598 LHSIdx = Ctx.getRandomUInt64() % Worklist.size();
1599 RHSIdx = Ctx.getRandomUInt64() % (Worklist.size() - 1);
1600 if (RHSIdx >= LHSIdx)
1601 ++RHSIdx;
1602 }
1603
1604 APFloat Res =
1605 IID == Intrinsic::vector_reduce_fmax
1606 ? maxnumWithSNaNQuieting(LHS: Worklist[LHSIdx], RHS: Worklist[RHSIdx])
1607 : minnumWithSNaNQuieting(LHS: Worklist[LHSIdx], RHS: Worklist[RHSIdx]);
1608 if (LHSIdx < RHSIdx)
1609 std::swap(a&: LHSIdx, b&: RHSIdx);
1610 Worklist.erase(CI: Worklist.begin() + LHSIdx);
1611 Worklist.erase(CI: Worklist.begin() + RHSIdx);
1612 Worklist.push_back(Elt: std::move(Res));
1613 }
1614
1615 AnyValue ValidatedRes =
1616 handleFMFFlags(Val: Worklist.front(), FMF, /*IsInput=*/false);
1617 if (ValidatedRes.isPoison())
1618 return AnyValue::poison();
1619 APFloat FRes = handleDenormal(Val: ValidatedRes.asFloat(), Mode: DenormMode.Output,
1620 /*IsInput=*/false);
1621
1622 return applyNaNPropagation(Result: FRes, Inputs: InputVec);
1623 }
1624 case Intrinsic::fabs: {
1625 return visitBitwiseFPUnOpWithResult(
1626 RetTy, FMF, Operand: Args[0],
1627 ScalarFn: [](const APFloat &Operand) -> APFloat { return abs(X: Operand); });
1628 }
1629 case Intrinsic::fma: {
1630 return visitFPTriOpWithResult(
1631 RetTy, FMF, Op1: Args[0], Op2: Args[1], Op3: Args[2],
1632 ScalarFn: [](const APFloat &Op1, const APFloat &Op2,
1633 const APFloat &Op3) -> APFloat {
1634 auto Res = Op1;
1635 Res.fusedMultiplyAdd(Multiplicand: Op2, Addend: Op3, RM: RoundingMode::NearestTiesToEven);
1636 return Res;
1637 });
1638 }
1639 case Intrinsic::fmuladd: {
1640 return visitFPTriOpWithResult(
1641 RetTy, FMF, Op1: Args[0], Op2: Args[1], Op3: Args[2],
1642 ScalarFn: [&](const APFloat &Op1, const APFloat &Op2,
1643 const APFloat &Op3) -> APFloat {
1644 if (Ctx.fuseMultiplyAdd()) {
1645 auto Res = Op1;
1646 Res.fusedMultiplyAdd(Multiplicand: Op2, Addend: Op3, RM: RoundingMode::NearestTiesToEven);
1647 return Res;
1648 }
1649 return Op1 * Op2 + Op3;
1650 });
1651 }
1652 case Intrinsic::is_fpclass: {
1653 const FPClassTest Mask =
1654 static_cast<FPClassTest>(Args[1].asInteger().getZExtValue());
1655 return computeUnOp(Ty: RetTy, Operand: Args[0], ScalarFn: [&](const AnyValue &Op) -> AnyValue {
1656 if (Op.isPoison())
1657 return AnyValue::poison();
1658 return AnyValue::boolean(
1659 Val: static_cast<bool>(Op.asFloat().classify() & Mask));
1660 });
1661 }
1662 case Intrinsic::copysign: {
1663 return computeBinOp(
1664 Ty: RetTy, LHS: Args[0], RHS: Args[1],
1665 ScalarFn: [&](const AnyValue &LHS, const AnyValue &RHS) -> AnyValue {
1666 if (LHS.isPoison() || RHS.isPoison())
1667 return AnyValue::poison();
1668 const AnyValue ValidatedLHS =
1669 handleFMFFlags(Val: LHS, FMF, /*IsInput=*/true);
1670 const AnyValue ValidatedRHS =
1671 handleFMFFlags(Val: RHS, FMF, /*IsInput=*/true);
1672 if (ValidatedLHS.isPoison() || ValidatedRHS.isPoison())
1673 return AnyValue::poison();
1674
1675 return handleFMFFlags(Val: APFloat::copySign(Value: ValidatedLHS.asFloat(),
1676 Sign: ValidatedRHS.asFloat()),
1677 FMF, /*IsInput=*/false);
1678 });
1679 }
1680 case Intrinsic::maxnum:
1681 case Intrinsic::minnum:
1682 case Intrinsic::maximum:
1683 case Intrinsic::minimum:
1684 case Intrinsic::maximumnum:
1685 case Intrinsic::minimumnum: {
1686 return visitFPBinOpWithResult(
1687 RetTy, FMF, LHS: Args[0], RHS: Args[1],
1688 ScalarFn: [&](const APFloat &LHS, const APFloat &RHS) -> APFloat {
1689 switch (IID) {
1690 case Intrinsic::maximum:
1691 return maximum(A: LHS, B: RHS);
1692 case Intrinsic::minimum:
1693 return minimum(A: LHS, B: RHS);
1694 case Intrinsic::maximumnum:
1695 return maximumnum(A: LHS, B: RHS);
1696 case Intrinsic::minimumnum:
1697 return minimumnum(A: LHS, B: RHS);
1698 case Intrinsic::maxnum:
1699 return maxnumWithSNaNQuieting(LHS, RHS);
1700 case Intrinsic::minnum:
1701 return minnumWithSNaNQuieting(LHS, RHS);
1702 default:
1703 llvm_unreachable("Unexpected intrinsic ID");
1704 }
1705 });
1706 }
1707 case Intrinsic::fptosi_sat:
1708 case Intrinsic::fptoui_sat: {
1709 const auto BitWidth = RetTy->getScalarSizeInBits();
1710 return computeUnOp(Ty: RetTy, Operand: Args[0], ScalarFn: [&](const AnyValue &Op) -> AnyValue {
1711 if (Op.isPoison())
1712 return AnyValue::poison();
1713 const APFloat &Operand = Op.asFloat();
1714 APSInt V(BitWidth, IID == Intrinsic::fptoui_sat);
1715 [[maybe_unused]] bool IsExact;
1716 Operand.convertToInteger(Result&: V, RM: APFloat::rmTowardZero, IsExact: &IsExact);
1717 return V;
1718 });
1719 }
1720 case Intrinsic::memcpy:
1721 case Intrinsic::memcpy_inline:
1722 case Intrinsic::memmove:
1723 return callMemTransferIntrinsic(CB, Args, IID);
1724 case Intrinsic::memset:
1725 case Intrinsic::memset_inline:
1726 return callMemSetIntrinsic(CB, Args);
1727 case Intrinsic::experimental_noalias_scope_decl:
1728 // FIXME: Not implemented yet. Currently it acts as a noop.
1729 return AnyValue();
1730 case Intrinsic::experimental_cttz_elts: {
1731 auto *IsZeroPoisonC = cast<ConstantInt>(Val: CB.getArgOperand(i: 1));
1732 const bool IsZeroPoison = IsZeroPoisonC->isOne();
1733
1734 const auto &Vec = Args[0].asAggregate();
1735 const unsigned RetBW = RetTy->getIntegerBitWidth();
1736
1737 if (!isUIntN(N: RetBW, x: Vec.size()))
1738 return AnyValue::poison();
1739
1740 uint64_t Count = 0;
1741 for (const AnyValue &V : Vec) {
1742 if (V.isPoison())
1743 return AnyValue::poison();
1744 if (!V.asInteger().isZero())
1745 break;
1746 ++Count;
1747 }
1748
1749 if (Count == Vec.size() && IsZeroPoison)
1750 return AnyValue::poison();
1751 return APInt(RetBW, Count);
1752 }
1753 case Intrinsic::experimental_get_vector_length: {
1754 auto *VFC = cast<ConstantInt>(Val: CB.getArgOperand(i: 1));
1755 auto *ScalableC = cast<ConstantInt>(Val: CB.getArgOperand(i: 2));
1756
1757 if (Args[0].isPoison())
1758 return AnyValue::poison();
1759
1760 const APInt &Cnt = Args[0].asInteger();
1761 const uint64_t VF = VFC->getZExtValue();
1762 const bool Scalable = ScalableC->isOne();
1763
1764 const uint64_t MaxLanes = Ctx.getEVL(EC: ElementCount::get(MinVal: VF, Scalable));
1765
1766 uint64_t Res = 0;
1767 if (!Cnt.isZero()) {
1768 if (Cnt.getActiveBits() <= 64 && Cnt.getZExtValue() <= MaxLanes) {
1769 Res = Cnt.getZExtValue();
1770 } else {
1771 APInt Max(Cnt.getBitWidth(), MaxLanes);
1772 APInt NumIters =
1773 APIntOps::RoundingUDiv(A: Cnt, B: Max, RM: APInt::Rounding::UP);
1774 uint64_t Lower =
1775 APIntOps::RoundingUDiv(A: Cnt, B: NumIters, RM: APInt::Rounding::UP)
1776 .getZExtValue();
1777 uint64_t Range = MaxLanes - Lower + 1;
1778 Res = Lower + Ctx.getRandomUInt64() % Range;
1779 }
1780 }
1781
1782 if (isIntN(N: 32, x: Res))
1783 return APInt(32, Res);
1784 return AnyValue::poison();
1785 }
1786
1787 case Intrinsic::experimental_vector_extract_last_active: {
1788 const auto &Data = Args[0].asAggregate();
1789 const AnyValue &Mask = Args[1];
1790
1791 for (size_t I = Data.size(); I != 0; --I) {
1792 switch (getMaskLane(Mask, I: I - 1)) {
1793 case BooleanKind::True:
1794 return Data[I - 1];
1795 case BooleanKind::False:
1796 break;
1797 case BooleanKind::Poison:
1798 return AnyValue::poison();
1799 }
1800 }
1801
1802 return Args[2];
1803 }
1804
1805 case Intrinsic::experimental_vector_compress: {
1806 const auto &Val = Args[0].asAggregate();
1807 const AnyValue &Mask = Args[1];
1808 const auto &Passthru = Args[2].asAggregate();
1809
1810 std::vector<AnyValue> Res;
1811 Res.reserve(n: Val.size());
1812
1813 for (size_t I = 0, E = Val.size(); I != E; ++I) {
1814 switch (getMaskLane(Mask, I)) {
1815 case BooleanKind::True:
1816 Res.push_back(x: Val[I]);
1817 break;
1818 case BooleanKind::False:
1819 break;
1820 case BooleanKind::Poison:
1821 return AnyValue::getPoisonValue(Ctx, Ty: RetTy);
1822 }
1823 }
1824
1825 for (size_t I = Res.size(), E = Val.size(); I != E; ++I)
1826 Res.push_back(x: Passthru[I]);
1827 return std::move(Res);
1828 }
1829
1830 case Intrinsic::experimental_vector_match: {
1831 const auto &Search = Args[0].asAggregate();
1832 const auto &Needles = Args[1].asAggregate();
1833 const auto &Mask = Args[2].asAggregate();
1834
1835 std::vector<AnyValue> Res;
1836 Res.reserve(n: Search.size());
1837
1838 for (size_t I = 0, E = Search.size(); I != E; ++I) {
1839 switch (Mask[I].asBoolean()) {
1840 case BooleanKind::False:
1841 Res.push_back(x: AnyValue::boolean(Val: false));
1842 continue;
1843 case BooleanKind::Poison:
1844 Res.push_back(x: AnyValue::poison());
1845 continue;
1846 case BooleanKind::True:
1847 break;
1848 }
1849
1850 if (Search[I].isPoison()) {
1851 Res.push_back(x: AnyValue::poison());
1852 continue;
1853 }
1854
1855 bool Found = false;
1856 bool SawPoison = false;
1857 for (const AnyValue &Needle : Needles) {
1858 if (Needle.isPoison()) {
1859 SawPoison = true;
1860 break;
1861 }
1862 if (Search[I].asInteger() == Needle.asInteger())
1863 Found = true;
1864 }
1865
1866 if (SawPoison)
1867 Res.push_back(x: AnyValue::poison());
1868 else
1869 Res.push_back(x: AnyValue::boolean(Val: Found));
1870 }
1871
1872 return std::move(Res);
1873 }
1874 case Intrinsic::experimental_vector_histogram_add:
1875 case Intrinsic::experimental_vector_histogram_uadd_sat:
1876 case Intrinsic::experimental_vector_histogram_umax:
1877 case Intrinsic::experimental_vector_histogram_umin:
1878 return callExperimentalVectorHistogramIntrinsic(CB, Args, IID);
1879 default:
1880 Handler.onUnrecognizedInstruction(I&: CB);
1881 setFailed();
1882 return AnyValue();
1883 }
1884 }
1885
1886 AnyValue callLibFunc(CallBase &CB, Function *ResolvedCallee,
1887 ArrayRef<AnyValue> CalleeArgs) {
1888 LibFunc LF;
1889 // Respect nobuiltin attributes on call site.
1890 if (CB.isNoBuiltin() ||
1891 !CurrentFrame->TLI.getLibFunc(FDecl: *ResolvedCallee, F&: LF)) {
1892 Handler.onUnrecognizedInstruction(I&: CB);
1893 setFailed();
1894 return AnyValue();
1895 }
1896
1897 if (auto LibCallRes =
1898 Lib.executeLibcall(LF, Name: CB.getName(), Type: CB.getType(), Args: CalleeArgs))
1899 return *LibCallRes;
1900
1901 if (ExitInfo)
1902 return AnyValue();
1903
1904 Handler.onUnrecognizedInstruction(I&: CB);
1905 setFailed();
1906 return AnyValue();
1907 }
1908
1909 /// Handle both poison-generating and UB-implying attributes for parameters
1910 /// and return values.
1911 void handleAttributes(Type *Ty, AnyValue &V, AttributeSet AttrsAtCallSite,
1912 AttributeSet AttrsAtCallee) {
1913 if (Ty->isIntOrIntVectorTy()) {
1914 if (auto CRAttr = AttrsAtCallSite.getAttribute(Kind: Attribute::Range);
1915 CRAttr.isValid())
1916 applyRangeAttr(V, CR: CRAttr.getRange());
1917 if (auto CRAttr = AttrsAtCallee.getAttribute(Kind: Attribute::Range);
1918 CRAttr.isValid())
1919 applyRangeAttr(V, CR: CRAttr.getRange());
1920 }
1921 if (AttributeFuncs::isNoFPClassCompatibleType(Ty)) {
1922 if (auto CRAttr = AttrsAtCallSite.getAttribute(Kind: Attribute::NoFPClass);
1923 CRAttr.isValid())
1924 applyNoFPClassAttr(V, NoFPClass: CRAttr.getNoFPClass());
1925 if (auto CRAttr = AttrsAtCallee.getAttribute(Kind: Attribute::NoFPClass);
1926 CRAttr.isValid())
1927 applyNoFPClassAttr(V, NoFPClass: CRAttr.getNoFPClass());
1928 }
1929 if (Ty->isPointerTy()) {
1930 if (AttrsAtCallSite.hasAttribute(Kind: Attribute::NonNull) ||
1931 AttrsAtCallee.hasAttribute(Kind: Attribute::NonNull))
1932 applyNonNullAttr(V, AS: Ty->getPointerAddressSpace(), DL);
1933 }
1934 if (Ty->isPtrOrPtrVectorTy()) {
1935 if (MaybeAlign Align = AttrsAtCallSite.getAlignment())
1936 applyAlignAttr(V, Alignment: *Align);
1937 if (MaybeAlign Align = AttrsAtCallee.getAlignment())
1938 applyAlignAttr(V, Alignment: *Align);
1939 }
1940 if ((AttrsAtCallSite.hasAttribute(Kind: Attribute::NoUndef) ||
1941 AttrsAtCallee.hasAttribute(Kind: Attribute::NoUndef)) &&
1942 violatesNoUndefAttr(V)) {
1943 reportImmediateUB() << "The value " << V
1944 << " violates noundef attribute.";
1945 return;
1946 }
1947 if (Ty->isPointerTy()) {
1948 unsigned AS = Ty->getPointerAddressSpace();
1949 if (uint64_t DereferenceableBytes =
1950 std::max(a: AttrsAtCallSite.getDereferenceableBytes(),
1951 b: AttrsAtCallee.getDereferenceableBytes())) {
1952 if (violatesDereferenceableBytesAttr(V, Bytes: DereferenceableBytes,
1953 /*OrNull=*/false, AS, Ctx))
1954 reportImmediateUB()
1955 << "The value " << V << " violates dereferenceable("
1956 << DereferenceableBytes << ") attribute.";
1957 } else if (uint64_t DereferenceableOrNullBytes =
1958 std::max(a: AttrsAtCallSite.getDereferenceableOrNullBytes(),
1959 b: AttrsAtCallee.getDereferenceableOrNullBytes())) {
1960 if (violatesDereferenceableBytesAttr(V, Bytes: DereferenceableOrNullBytes,
1961 /*OrNull=*/true, AS, Ctx))
1962 reportImmediateUB() << "The value " << V
1963 << " violates "
1964 "dereferenceable_or_null("
1965 << DereferenceableOrNullBytes << ") attribute.";
1966 }
1967 }
1968 }
1969
1970 /// Handle both poison-generating and UB-implying metadata on instructions.
1971 void handleMetadata(Type *Ty, AnyValue &V, Instruction &I) {
1972 auto ExtractFirstIntOperand = [](const MDNode *Node) {
1973 return mdconst::extract<ConstantInt>(MD: Node->getOperand(I: 0))->getZExtValue();
1974 };
1975
1976 if (Ty->isIntOrIntVectorTy()) {
1977 if (MDNode *Ranges = I.getMetadata(KindID: LLVMContext::MD_range)) {
1978 SmallVector<ConstantRange> RangeList;
1979 for (uint32_t I = 0; I < Ranges->getNumOperands(); I += 2) {
1980 RangeList.emplace_back(
1981 Args: mdconst::extract<ConstantInt>(MD: Ranges->getOperand(I))->getValue(),
1982 Args: mdconst::extract<ConstantInt>(MD: Ranges->getOperand(I: I + 1))
1983 ->getValue());
1984 }
1985 forEachScalarValue(V, Visit: [&](AnyValue &Scalar) {
1986 if (!Scalar.isInteger())
1987 return;
1988 for (auto &CR : RangeList)
1989 if (CR.contains(Val: Scalar.asInteger()))
1990 return;
1991 Scalar = AnyValue::poison();
1992 });
1993 }
1994 }
1995 if (AttributeFuncs::isNoFPClassCompatibleType(Ty)) {
1996 if (const MDNode *NoFPClass = I.getMetadata(KindID: LLVMContext::MD_nofpclass)) {
1997 applyNoFPClassAttr(
1998 V, NoFPClass: static_cast<FPClassTest>(ExtractFirstIntOperand(NoFPClass)));
1999 }
2000 }
2001 if (Ty->isPointerTy()) {
2002 if (I.hasMetadata(KindID: LLVMContext::MD_nonnull))
2003 applyNonNullAttr(V, AS: Ty->getPointerAddressSpace(), DL);
2004 // Unlike align attributes, !align is only defined for pointer types.
2005 if (const MDNode *Alignment = I.getMetadata(KindID: LLVMContext::MD_align))
2006 applyAlignAttr(V, Alignment: Align(ExtractFirstIntOperand(Alignment)));
2007 }
2008 if (I.hasMetadata(KindID: LLVMContext::MD_noundef) && violatesNoUndefAttr(V)) {
2009 reportImmediateUB() << "The value " << V
2010 << " violates !noundef metadata.";
2011 return;
2012 }
2013 if (Ty->isPointerTy()) {
2014 unsigned AS = Ty->getPointerAddressSpace();
2015 if (const MDNode *DereferenceableBytes =
2016 I.getMetadata(KindID: LLVMContext::MD_dereferenceable)) {
2017 uint64_t Bytes = ExtractFirstIntOperand(DereferenceableBytes);
2018 if (violatesDereferenceableBytesAttr(V, Bytes,
2019 /*OrNull=*/false, AS, Ctx))
2020 reportImmediateUB()
2021 << "The value " << V << " violates !dereferenceable !{i64 "
2022 << Bytes << "} metadata.";
2023 } else if (const MDNode *DereferenceableOrNullBytes =
2024 I.getMetadata(KindID: LLVMContext::MD_dereferenceable_or_null)) {
2025 uint64_t Bytes = ExtractFirstIntOperand(DereferenceableOrNullBytes);
2026 if (violatesDereferenceableBytesAttr(V, Bytes,
2027 /*OrNull=*/true, AS, Ctx))
2028 reportImmediateUB()
2029 << "The value " << V << " violates !dereferenceable_or_null!{i64 "
2030 << Bytes << "} metadata.";
2031 }
2032 }
2033 }
2034
2035 void enterCall(CallBase &CB) {
2036 Function *Callee = CB.getCalledFunction();
2037 // TODO: handle initializes
2038 auto &CalleeArgs = CurrentFrame->CalleeArgs;
2039 assert(CalleeArgs.empty() &&
2040 "Forgot to call returnFromCallee before entering a new call.");
2041 for (Value *Arg : CB.args())
2042 CalleeArgs.push_back(Elt: getValue(V: Arg));
2043
2044 if (!Callee) {
2045 Value *CalledOperand = CB.getCalledOperand();
2046 if (isNoopInlineAsm(V: CalledOperand, RetTy: CB.getType())) {
2047 CurrentFrame->ResolvedCallee = nullptr;
2048 returnFromCallee();
2049 return;
2050 }
2051
2052 if (isa<InlineAsm>(Val: CalledOperand)) {
2053 Handler.onUnrecognizedInstruction(I&: CB);
2054 setFailed();
2055 return;
2056 }
2057
2058 auto &CalleeVal = getValue(V: CalledOperand);
2059 if (CalleeVal.isPoison()) {
2060 reportImmediateUB() << "Indirect call through poison function pointer.";
2061 return;
2062 }
2063 Callee = Ctx.getTargetFunction(Ptr: CalleeVal.asPointer());
2064 if (!Callee) {
2065 reportImmediateUB()
2066 << "Indirect call through invalid function pointer.";
2067 return;
2068 }
2069 if (Callee->getFunctionType() != CB.getFunctionType()) {
2070 reportImmediateUB() << "Indirect call through a function pointer with "
2071 "mismatched signature. Expected: "
2072 << *CB.getFunctionType()
2073 << ", Actual: " << *Callee->getFunctionType();
2074 return;
2075 }
2076 }
2077
2078 assert(Callee && "Expected a resolved callee function.");
2079 assert(
2080 Callee->getFunctionType() == CB.getFunctionType() &&
2081 "Expected the callee function type to match the call site signature.");
2082
2083 // Handle parameter attributes (Attributes from resolved callee should be
2084 // applied if available).
2085 for (auto [I, Arg] : enumerate(First: CB.args())) {
2086 Type *ArgTy = Arg->getType();
2087 AnyValue &ArgVal = CalleeArgs[I];
2088
2089 // CallBase::paramHasAttr also checks parameter attributes at known
2090 // callee. We do it explicitly to avoid duplication.
2091 AttributeSet AttrsAtCallSite = CB.getParamAttributes(ArgNo: I);
2092 AttributeSet AttrsAtCallee = Callee->getAttributes().getParamAttrs(ArgNo: I);
2093
2094 if (ArgTy->isPointerTy()) {
2095 auto *ByValTy = AttrsAtCallSite.getByValType();
2096 auto *ByValTyFromCallee = AttrsAtCallee.getByValType();
2097 if (ByValTy != ByValTyFromCallee) {
2098 reportImmediateUB()
2099 << "Mismatched byval attribute between callee and callsite.";
2100 return;
2101 }
2102 if (ByValTy) {
2103 if (ArgVal.isPoison()) {
2104 reportImmediateUB() << "Invalid poison byval pointer argument.";
2105 return;
2106 }
2107
2108 uint64_t Size = Ctx.getEffectiveTypeAllocSize(Ty: ByValTy);
2109 MaybeAlign AllocAlign = AttrsAtCallSite.getAlignment();
2110 // Ignore the alignment at the callsite when it is set on the callee.
2111 if (MaybeAlign CalleeAlign = AttrsAtCallee.getAlignment())
2112 AllocAlign = CalleeAlign;
2113 if (!AllocAlign.has_value()) {
2114 // If the alignment is not specified, we use the default ABI
2115 // alignment. This is the default behavior of
2116 // TargetLoweringBase::getByValTypeAlignment.
2117 AllocAlign = DL.getABITypeAlign(Ty: ByValTy);
2118 }
2119 assert(I < Callee->arg_size() &&
2120 "Byval pointers cannot be passed via variadic arguments.");
2121 auto Obj = Ctx.allocate(
2122 Size, Align: AllocAlign->value(), Name: Callee->getArg(i: I)->getName(),
2123 AS: ArgTy->getPointerAddressSpace(), InitKind: MemInitKind::Uninitialized,
2124 AllocKind: MemAllocKind::Stack);
2125 if (!Obj) {
2126 reportError()
2127 << "Insufficient stack space for byval pointer argument.";
2128 return;
2129 }
2130 if (auto [MO, Offset] = verifyMemAccess(
2131 Ptr: ArgVal.asPointer(), AccessSize: Size,
2132 Alignment: std::max(a: AllocAlign.value(),
2133 b: AttrsAtCallSite.getAlignment().valueOrOne()),
2134 /*IsStore=*/false);
2135 MO)
2136 copy(Range: MO->getBytes().slice(N: Offset, M: Size), Out: Obj->getBytes().begin());
2137 else
2138 return;
2139 CurrentFrame->CalleeByValArgs.push_back(Elt: Obj);
2140 ArgVal = Ctx.deriveFromMemoryObject(Obj: std::move(Obj));
2141 }
2142 }
2143 handleAttributes(Ty: ArgTy, V&: ArgVal, AttrsAtCallSite, AttrsAtCallee);
2144 }
2145
2146 CurrentFrame->ResolvedCallee = Callee;
2147 if (Callee->isIntrinsic()) {
2148 CurrentFrame->CalleeRetVal = callIntrinsic(CB, Args: CalleeArgs);
2149 returnFromCallee();
2150 return;
2151 } else if (Callee->isDeclaration()) {
2152 CurrentFrame->CalleeRetVal = callLibFunc(CB, ResolvedCallee: Callee, CalleeArgs);
2153 returnFromCallee();
2154 return;
2155 } else {
2156 uint32_t MaxStackDepth = Ctx.getMaxStackDepth();
2157 if (MaxStackDepth && CallStack.size() >= MaxStackDepth) {
2158 reportError() << "Maximum stack depth exceeded.";
2159 return;
2160 }
2161 assert(!Callee->empty() && "Expected a defined function.");
2162 // Suspend the current frame and push the callee frame onto the stack.
2163 ArrayRef<AnyValue> Args = CurrentFrame->CalleeArgs;
2164 AnyValue &RetVal = CurrentFrame->CalleeRetVal;
2165 CurrentFrame->State = FrameState::Pending;
2166 CallStack.emplace_back(args&: *Callee, args: &CB, args&: CurrentFrame, args&: Args, args&: RetVal,
2167 args: Ctx.getTLIImpl());
2168 }
2169 }
2170
2171 void visitCallInst(CallInst &CI) { enterCall(CB&: CI); }
2172
2173 void visitInvokeInst(InvokeInst &II) {
2174 // TODO: handle exceptions
2175 enterCall(CB&: II);
2176 }
2177
2178 void visitAdd(BinaryOperator &I) {
2179 visitIntBinOp(I, ScalarFn: [&](const APInt &LHS, const APInt &RHS) {
2180 return addNoWrap(LHS, RHS, HasNSW: I.hasNoSignedWrap(), HasNUW: I.hasNoUnsignedWrap());
2181 });
2182 }
2183
2184 void visitSub(BinaryOperator &I) {
2185 visitIntBinOp(I, ScalarFn: [&](const APInt &LHS, const APInt &RHS) {
2186 return subNoWrap(LHS, RHS, HasNSW: I.hasNoSignedWrap(), HasNUW: I.hasNoUnsignedWrap());
2187 });
2188 }
2189
2190 void visitMul(BinaryOperator &I) {
2191 visitIntBinOp(I, ScalarFn: [&](const APInt &LHS, const APInt &RHS) {
2192 return mulNoWrap(LHS, RHS, HasNSW: I.hasNoSignedWrap(), HasNUW: I.hasNoUnsignedWrap());
2193 });
2194 }
2195
2196 void visitSDiv(BinaryOperator &I) {
2197 visitBinOp(I, ScalarFn: [&](const AnyValue &LHS, const AnyValue &RHS) -> AnyValue {
2198 // Priority: Immediate UB > poison > normal value
2199 if (RHS.isPoison()) {
2200 reportImmediateUB() << "Division by zero (refine RHS to 0).";
2201 return AnyValue::poison();
2202 }
2203 const APInt &RHSVal = RHS.asInteger();
2204 if (RHSVal.isZero()) {
2205 reportImmediateUB() << "Division by zero.";
2206 return AnyValue::poison();
2207 }
2208 if (LHS.isPoison()) {
2209 if (RHSVal.isAllOnes())
2210 reportImmediateUB()
2211 << "Signed division overflow (refine LHS to INT_MIN).";
2212 return AnyValue::poison();
2213 }
2214 const APInt &LHSVal = LHS.asInteger();
2215 if (LHSVal.isMinSignedValue() && RHSVal.isAllOnes()) {
2216 reportImmediateUB() << "Signed division overflow.";
2217 return AnyValue::poison();
2218 }
2219
2220 if (I.isExact()) {
2221 APInt Q, R;
2222 APInt::sdivrem(LHS: LHSVal, RHS: RHSVal, Quotient&: Q, Remainder&: R);
2223 if (!R.isZero())
2224 return AnyValue::poison();
2225 return Q;
2226 } else {
2227 return LHSVal.sdiv(RHS: RHSVal);
2228 }
2229 });
2230 }
2231
2232 void visitSRem(BinaryOperator &I) {
2233 visitBinOp(I, ScalarFn: [&](const AnyValue &LHS, const AnyValue &RHS) -> AnyValue {
2234 // Priority: Immediate UB > poison > normal value
2235 if (RHS.isPoison()) {
2236 reportImmediateUB() << "Division by zero (refine RHS to 0).";
2237 return AnyValue::poison();
2238 }
2239 const APInt &RHSVal = RHS.asInteger();
2240 if (RHSVal.isZero()) {
2241 reportImmediateUB() << "Division by zero.";
2242 return AnyValue::poison();
2243 }
2244 if (LHS.isPoison()) {
2245 if (RHSVal.isAllOnes())
2246 reportImmediateUB()
2247 << "Signed division overflow (refine LHS to INT_MIN).";
2248 return AnyValue::poison();
2249 }
2250 const APInt &LHSVal = LHS.asInteger();
2251 if (LHSVal.isMinSignedValue() && RHSVal.isAllOnes()) {
2252 reportImmediateUB() << "Signed division overflow. LHS: " << LHSVal
2253 << ", RHS: " << RHSVal;
2254 return AnyValue::poison();
2255 }
2256
2257 return LHSVal.srem(RHS: RHSVal);
2258 });
2259 }
2260
2261 void visitUDiv(BinaryOperator &I) {
2262 visitBinOp(I, ScalarFn: [&](const AnyValue &LHS, const AnyValue &RHS) -> AnyValue {
2263 // Priority: Immediate UB > poison > normal value
2264 if (RHS.isPoison()) {
2265 reportImmediateUB() << "Division by zero (refine RHS to 0).";
2266 return AnyValue::poison();
2267 }
2268 const APInt &RHSVal = RHS.asInteger();
2269 if (RHSVal.isZero()) {
2270 reportImmediateUB() << "Division by zero.";
2271 return AnyValue::poison();
2272 }
2273 if (LHS.isPoison())
2274 return AnyValue::poison();
2275 const APInt &LHSVal = LHS.asInteger();
2276
2277 if (I.isExact()) {
2278 APInt Q, R;
2279 APInt::udivrem(LHS: LHSVal, RHS: RHSVal, Quotient&: Q, Remainder&: R);
2280 if (!R.isZero())
2281 return AnyValue::poison();
2282 return Q;
2283 } else {
2284 return LHSVal.udiv(RHS: RHSVal);
2285 }
2286 });
2287 }
2288
2289 void visitURem(BinaryOperator &I) {
2290 visitBinOp(I, ScalarFn: [&](const AnyValue &LHS, const AnyValue &RHS) -> AnyValue {
2291 // Priority: Immediate UB > poison > normal value
2292 if (RHS.isPoison()) {
2293 reportImmediateUB() << "Division by zero (refine RHS to 0).";
2294 return AnyValue::poison();
2295 }
2296 const APInt &RHSVal = RHS.asInteger();
2297 if (RHSVal.isZero()) {
2298 reportImmediateUB() << "Division by zero.";
2299 return AnyValue::poison();
2300 }
2301 if (LHS.isPoison())
2302 return AnyValue::poison();
2303 const APInt &LHSVal = LHS.asInteger();
2304 return LHSVal.urem(RHS: RHSVal);
2305 });
2306 }
2307
2308 void visitFAdd(BinaryOperator &I) {
2309 visitFPBinOp(I, ScalarFn: [](const APFloat &LHS, const APFloat &RHS) -> APFloat {
2310 APFloat Res = LHS;
2311 Res.add(RHS, RM: APFloat::rmNearestTiesToEven);
2312 return Res;
2313 });
2314 }
2315
2316 void visitFSub(BinaryOperator &I) {
2317 visitFPBinOp(I, ScalarFn: [](const APFloat &LHS, const APFloat &RHS) -> APFloat {
2318 APFloat Res = LHS;
2319 Res.subtract(RHS, RM: APFloat::rmNearestTiesToEven);
2320 return Res;
2321 });
2322 }
2323
2324 void visitFMul(BinaryOperator &I) {
2325 visitFPBinOp(I, ScalarFn: [](const APFloat &LHS, const APFloat &RHS) -> APFloat {
2326 APFloat Res = LHS;
2327 Res.multiply(RHS, RM: APFloat::rmNearestTiesToEven);
2328 return Res;
2329 });
2330 }
2331
2332 void visitFDiv(BinaryOperator &I) {
2333 visitFPBinOp(I, ScalarFn: [](const APFloat &LHS, const APFloat &RHS) -> APFloat {
2334 APFloat Res = LHS;
2335 Res.divide(RHS, RM: APFloat::rmNearestTiesToEven);
2336 return Res;
2337 });
2338 }
2339
2340 void visitFRem(BinaryOperator &I) {
2341 visitFPBinOp(I, ScalarFn: [](const APFloat &LHS, const APFloat &RHS) -> APFloat {
2342 APFloat Res = LHS;
2343 Res.mod(RHS);
2344 return Res;
2345 });
2346 }
2347
2348 void visitFNeg(UnaryOperator &I) {
2349 visitBitwiseFPUnOp(
2350 I, ScalarFn: [](const APFloat &Operand) -> APFloat { return -Operand; });
2351 }
2352
2353 void visitTruncInst(TruncInst &Trunc) {
2354 visitIntUnOp(I&: Trunc, ScalarFn: [&](const APInt &Operand) -> AnyValue {
2355 unsigned DestBW = Trunc.getType()->getScalarSizeInBits();
2356 if (Trunc.hasNoSignedWrap() && Operand.getSignificantBits() > DestBW)
2357 return AnyValue::poison();
2358 if (Trunc.hasNoUnsignedWrap() && Operand.getActiveBits() > DestBW)
2359 return AnyValue::poison();
2360 return Operand.trunc(width: DestBW);
2361 });
2362 }
2363
2364 void visitZExtInst(ZExtInst &ZExt) {
2365 visitIntUnOp(I&: ZExt, ScalarFn: [&](const APInt &Operand) -> AnyValue {
2366 uint32_t DestBW = ZExt.getDestTy()->getScalarSizeInBits();
2367 if (ZExt.hasNonNeg() && Operand.isNegative())
2368 return AnyValue::poison();
2369 return Operand.zext(width: DestBW);
2370 });
2371 }
2372
2373 void visitSExtInst(SExtInst &SExt) {
2374 visitIntUnOp(I&: SExt, ScalarFn: [&](const APInt &Operand) -> AnyValue {
2375 uint32_t DestBW = SExt.getDestTy()->getScalarSizeInBits();
2376 return Operand.sext(width: DestBW);
2377 });
2378 }
2379
2380 void visitFPExtInst(FPExtInst &FPExt) { visitFPConvInst(I&: FPExt); }
2381
2382 void visitFPTruncInst(FPTruncInst &FPTrunc) { visitFPConvInst(I&: FPTrunc); }
2383
2384 void visitFPConvInst(Instruction &I) {
2385 if (!Ctx.isDefaultFPEnv())
2386 reportImmediateUB() << "Non-constrained floating-point operation assumes "
2387 "default floating-point environment";
2388
2389 const fltSemantics &DstSem =
2390 I.getType()->getScalarType()->getFltSemantics();
2391
2392 visitUnOp(I, ScalarFn: [&](const AnyValue &Operand) -> AnyValue {
2393 if (Operand.isPoison())
2394 return AnyValue::poison();
2395
2396 FastMathFlags FMF = cast<FPMathOperator>(Val&: I).getFastMathFlags();
2397 DenormalMode DenormMode =
2398 getCurrentDenormalMode(Ty: I.getOperand(i: 0)->getType());
2399
2400 auto ValidatedOperand = handleFMFFlags(Val: Operand, FMF, /*IsInput=*/true);
2401 if (ValidatedOperand.isPoison())
2402 return ValidatedOperand;
2403
2404 APFloat FOperand = handleDenormal(Val: ValidatedOperand.asFloat(),
2405 Mode: DenormMode.Input, /*IsInput=*/true);
2406 APFloat SourceNaN = FOperand;
2407
2408 bool LosesInfo;
2409 FOperand.convert(ToSemantics: DstSem, RM: Ctx.getCurrentRoundingMode(), losesInfo: &LosesInfo);
2410
2411 if (auto ValidateRes = handleFMFFlags(Val: FOperand, FMF, /*IsInput=*/false);
2412 ValidateRes.isPoison())
2413 return ValidateRes;
2414
2415 FOperand = handleDenormal(Val: std::move(FOperand), Mode: DenormMode.Output, IsInput: true);
2416
2417 return AnyValue(applyNaNPropagation(Result: FOperand, Inputs: {&SourceNaN}));
2418 });
2419 }
2420
2421 void visitFPToSIInst(FPToSIInst &FPToSI) {
2422 visitFPToIntInst(I&: FPToSI, /*IsUnsigned=*/false);
2423 }
2424
2425 void visitFPToUIInst(FPToUIInst &FPToUI) {
2426 visitFPToIntInst(I&: FPToUI, /*IsUnsigned=*/true);
2427 }
2428
2429 void visitFPToIntInst(Instruction &I, bool IsUnsigned) {
2430 // Note: We DO NOT use CurrentRoundingMode here.
2431 // Language specs require truncation towards zero for FP-to-Int conversions.
2432 visitUnOp(I, ScalarFn: [&](const AnyValue &Operand) -> AnyValue {
2433 if (Operand.isPoison())
2434 return AnyValue::poison();
2435
2436 APSInt Res(I.getType()->getScalarSizeInBits(), /*isUnsigned=*/IsUnsigned);
2437 bool IsExact;
2438 APFloat::opStatus Status = Operand.asFloat().convertToInteger(
2439 Result&: Res, RM: APFloat::rmTowardZero, IsExact: &IsExact);
2440
2441 if (Status == APFloat::opInvalidOp)
2442 return AnyValue::poison();
2443
2444 return AnyValue(Res);
2445 });
2446 }
2447
2448 void visitSIToFPInst(SIToFPInst &SIToFP) {
2449 visitIntToFPInst(I&: SIToFP, /*IsSigned=*/true);
2450 }
2451
2452 void visitUIToFPInst(UIToFPInst &UIToFP) {
2453 visitIntToFPInst(I&: UIToFP, /*IsSigned=*/false);
2454 }
2455
2456 void visitIntToFPInst(Instruction &I, bool IsSigned) {
2457 const fltSemantics &DstSem =
2458 I.getType()->getScalarType()->getFltSemantics();
2459
2460 visitUnOp(I, ScalarFn: [&](const AnyValue &Operand) -> AnyValue {
2461 if (Operand.isPoison())
2462 return AnyValue::poison();
2463
2464 APInt IOperand = Operand.asInteger();
2465
2466 if (isa<UIToFPInst>(Val: I) && I.hasNonNeg() && IOperand.isNegative())
2467 return AnyValue::poison();
2468
2469 APFloat Res(DstSem);
2470
2471 Res.convertFromAPInt(Input: Operand.asInteger(), /*IsSigned=*/IsSigned,
2472 RM: Ctx.getCurrentRoundingMode());
2473
2474 return AnyValue(Res);
2475 });
2476 }
2477
2478 void visitAnd(BinaryOperator &I) {
2479 visitIntBinOp(I, ScalarFn: [](const APInt &LHS, const APInt &RHS) -> AnyValue {
2480 return LHS & RHS;
2481 });
2482 }
2483
2484 void visitXor(BinaryOperator &I) {
2485 visitIntBinOp(I, ScalarFn: [](const APInt &LHS, const APInt &RHS) -> AnyValue {
2486 return LHS ^ RHS;
2487 });
2488 }
2489
2490 void visitOr(BinaryOperator &I) {
2491 visitIntBinOp(I, ScalarFn: [&](const APInt &LHS, const APInt &RHS) -> AnyValue {
2492 if (cast<PossiblyDisjointInst>(Val&: I).isDisjoint() && LHS.intersects(RHS))
2493 return AnyValue::poison();
2494 return LHS | RHS;
2495 });
2496 }
2497
2498 void visitShl(BinaryOperator &I) {
2499 visitIntBinOp(I, ScalarFn: [&](const APInt &LHS, const APInt &RHS) -> AnyValue {
2500 if (RHS.uge(RHS: LHS.getBitWidth()))
2501 return AnyValue::poison();
2502 if (I.hasNoSignedWrap() && RHS.uge(RHS: LHS.getNumSignBits()))
2503 return AnyValue::poison();
2504 if (I.hasNoUnsignedWrap() && RHS.ugt(RHS: LHS.countl_zero()))
2505 return AnyValue::poison();
2506 return LHS.shl(ShiftAmt: RHS);
2507 });
2508 }
2509
2510 void visitLShr(BinaryOperator &I) {
2511 visitIntBinOp(I, ScalarFn: [&](const APInt &LHS, const APInt &RHS) -> AnyValue {
2512 if (RHS.uge(RHS: LHS.getBitWidth()) ||
2513 (cast<PossiblyExactOperator>(Val&: I).isExact() &&
2514 RHS.ugt(RHS: LHS.countr_zero())))
2515 return AnyValue::poison();
2516 return LHS.lshr(ShiftAmt: RHS);
2517 });
2518 }
2519
2520 void visitAShr(BinaryOperator &I) {
2521 visitIntBinOp(I, ScalarFn: [&](const APInt &LHS, const APInt &RHS) -> AnyValue {
2522 if (RHS.uge(RHS: LHS.getBitWidth()) ||
2523 (cast<PossiblyExactOperator>(Val&: I).isExact() &&
2524 RHS.ugt(RHS: LHS.countr_zero())))
2525 return AnyValue::poison();
2526 return LHS.ashr(ShiftAmt: RHS);
2527 });
2528 }
2529
2530 void visitICmpInst(ICmpInst &I) {
2531 visitBinOp(I, ScalarFn: [&](const AnyValue &LHS, const AnyValue &RHS) -> AnyValue {
2532 if (LHS.isPoison() || RHS.isPoison())
2533 return AnyValue::poison();
2534 const APInt &LHSVal =
2535 LHS.isPointer() ? LHS.asPointer().address() : LHS.asInteger();
2536 const APInt &RHSVal =
2537 RHS.isPointer() ? RHS.asPointer().address() : RHS.asInteger();
2538 if (I.hasSameSign() && LHSVal.isNonNegative() != RHSVal.isNonNegative())
2539 return AnyValue::poison();
2540 return AnyValue::boolean(
2541 Val: ICmpInst::compare(LHS: LHSVal, RHS: RHSVal, Pred: I.getPredicate()));
2542 });
2543 }
2544
2545 void visitFCmpInst(FCmpInst &I) {
2546 DenormalMode DenormMode =
2547 getCurrentDenormalMode(Ty: I.getOperand(i_nocapture: 0)->getType());
2548 FastMathFlags FMF = I.getFastMathFlags();
2549
2550 visitBinOp(I, ScalarFn: [&](const AnyValue &LHS, const AnyValue &RHS) -> AnyValue {
2551 if (LHS.isPoison() || RHS.isPoison())
2552 return AnyValue::poison();
2553
2554 if (auto ValidateRes = handleFMFFlags(Val: LHS, FMF, /*IsInput=*/true);
2555 ValidateRes.isPoison())
2556 return ValidateRes;
2557 if (auto ValidateRes = handleFMFFlags(Val: RHS, FMF, /*IsInput=*/true);
2558 ValidateRes.isPoison())
2559 return ValidateRes;
2560
2561 APFloat FLHS =
2562 handleDenormal(Val: LHS.asFloat(), Mode: DenormMode.Input, /*IsInput=*/true);
2563 APFloat FRHS =
2564 handleDenormal(Val: RHS.asFloat(), Mode: DenormMode.Input, /*IsInput=*/true);
2565
2566 return AnyValue::boolean(Val: FCmpInst::compare(LHS: FLHS, RHS: FRHS, Pred: I.getPredicate()));
2567 });
2568 }
2569
2570 void visitSelect(SelectInst &SI) {
2571 AnyValue Res;
2572
2573 if (SI.getCondition()->getType()->isIntegerTy(BitWidth: 1)) {
2574 switch (getValue(V: SI.getCondition()).asBoolean()) {
2575 case BooleanKind::True:
2576 Res = getValue(V: SI.getTrueValue());
2577 break;
2578 case BooleanKind::False:
2579 Res = getValue(V: SI.getFalseValue());
2580 break;
2581 case BooleanKind::Poison:
2582 Res = AnyValue::getPoisonValue(Ctx, Ty: SI.getType());
2583 break;
2584 }
2585 } else {
2586 auto &Cond = getValue(V: SI.getCondition()).asAggregate();
2587 auto &TV = getValue(V: SI.getTrueValue()).asAggregate();
2588 auto &FV = getValue(V: SI.getFalseValue()).asAggregate();
2589 std::vector<AnyValue> ResVec;
2590 size_t Len = Cond.size();
2591 ResVec.reserve(n: Len);
2592 for (uint32_t I = 0; I != Len; ++I) {
2593 switch (Cond[I].asBoolean()) {
2594 case BooleanKind::True:
2595 ResVec.push_back(x: TV[I]);
2596 break;
2597 case BooleanKind::False:
2598 ResVec.push_back(x: FV[I]);
2599 break;
2600 case BooleanKind::Poison:
2601 ResVec.push_back(x: AnyValue::poison());
2602 break;
2603 }
2604 }
2605 Res = AnyValue(std::move(ResVec));
2606 }
2607
2608 // Handle fast-math flags
2609 if (auto *FPMO = dyn_cast<FPMathOperator>(Val: &SI)) {
2610 if (FastMathFlags FMF = FPMO->getFastMathFlags(); FMF.any())
2611 Res = handleFMFFlags(Val: std::move(Res), FMF, /*IsInput=*/true);
2612 }
2613
2614 setResult(I&: SI, V: std::move(Res));
2615 }
2616
2617 void visitAllocaInst(AllocaInst &AI) {
2618 uint64_t AllocSize = Ctx.getEffectiveTypeAllocSize(Ty: AI.getAllocatedType());
2619 if (AI.isArrayAllocation()) {
2620 auto &Size = getValue(V: AI.getArraySize());
2621 if (Size.isPoison()) {
2622 reportImmediateUB() << "Alloca with poison array size.";
2623 return;
2624 }
2625 if (Size.asInteger().getActiveBits() > 64) {
2626 reportImmediateUB()
2627 << "Alloca with large array size that overflows uint64_t. Size: "
2628 << Size.asInteger();
2629 return;
2630 }
2631 bool Overflowed = false;
2632 AllocSize = SaturatingMultiply(X: AllocSize, Y: Size.asInteger().getZExtValue(),
2633 ResultOverflowed: &Overflowed);
2634 if (Overflowed) {
2635 reportImmediateUB()
2636 << "Alloca with allocation size that overflows uint64_t. Size: "
2637 << Size.asInteger();
2638 return;
2639 }
2640 }
2641 // If it is used by llvm.lifetime.start, it should be initially dead.
2642 bool IsInitiallyDead = any_of(Range: AI.users(), P: [](User *U) {
2643 return match(V: U, P: m_Intrinsic<Intrinsic::lifetime_start>());
2644 });
2645 auto Obj = Ctx.allocate(Size: AllocSize, Align: AI.getPointerAlignment(DL).value(),
2646 Name: AI.getName(), AS: AI.getAddressSpace(),
2647 InitKind: IsInitiallyDead ? MemInitKind::Poisoned
2648 : MemInitKind::Uninitialized,
2649 AllocKind: MemAllocKind::Stack);
2650 if (!Obj) {
2651 reportError() << "Insufficient stack space.";
2652 return;
2653 }
2654 CurrentFrame->Allocas.push_back(Elt: Obj);
2655 setResult(I&: AI, V: Ctx.deriveFromMemoryObject(Obj));
2656 }
2657
2658 void visitGetElementPtrInst(GetElementPtrInst &GEP) {
2659 setResult(I&: GEP, V: Ctx.computeGEP(GEP&: cast<GEPOperator>(Val&: GEP),
2660 GetValue: [this](Value *V) -> const AnyValue & {
2661 return getValue(V);
2662 }));
2663 }
2664
2665 void visitPtrToInt(PtrToIntInst &I) {
2666 return visitUnOp(I, ScalarFn: [&](const AnyValue &V) -> AnyValue {
2667 if (V.isPoison())
2668 return AnyValue::poison();
2669 Ctx.exposeProvenance(Prov&: V.asPointer().provenance());
2670 return V.asPointer().address();
2671 });
2672 }
2673
2674 void visitIntToPtr(IntToPtrInst &I) {
2675 return visitUnOp(I, ScalarFn: [&](const AnyValue &V) -> AnyValue {
2676 if (V.isPoison())
2677 return AnyValue::poison();
2678 auto Prov = Ctx.getWildcardProvenance();
2679 // TODO: check metadata
2680 return Pointer(std::move(Prov),
2681 V.asInteger().zextOrTrunc(width: DL.getPointerSizeInBits(
2682 AS: I.getType()->getPointerAddressSpace())));
2683 });
2684 }
2685
2686 void visitPtrToAddr(PtrToAddrInst &I) {
2687 unsigned BitWidth = I.getType()->getScalarSizeInBits();
2688 return visitUnOp(I, ScalarFn: [&](const AnyValue &V) -> AnyValue {
2689 if (V.isPoison())
2690 return AnyValue::poison();
2691 return V.asPointer().address().trunc(width: BitWidth);
2692 });
2693 }
2694
2695 void visitLoadInst(LoadInst &LI) {
2696 auto RetVal = load(Ptr: getValue(V: LI.getPointerOperand()), Alignment: LI.getAlign(),
2697 ValTy: LI.getType(), NoUndef: LI.hasMetadata(KindID: LLVMContext::MD_noundef));
2698 // TODO: track volatile loads
2699 handleMetadata(Ty: LI.getType(), V&: RetVal, I&: LI);
2700 setResult(I&: LI, V: std::move(RetVal));
2701 }
2702
2703 void visitStoreInst(StoreInst &SI) {
2704 auto &Ptr = getValue(V: SI.getPointerOperand());
2705 auto &Val = getValue(V: SI.getValueOperand());
2706 // TODO: track volatile stores
2707 // TODO: handle metadata
2708 store(Ptr, Alignment: SI.getAlign(), Val, ValTy: SI.getValueOperand()->getType());
2709 if (!hasProgramExited() && !Handler.onInstructionExecuted(I&: SI, Result: AnyValue()))
2710 setFailed();
2711 }
2712
2713 void visitInstruction(Instruction &I) {
2714 Handler.onUnrecognizedInstruction(I);
2715 setFailed();
2716 }
2717
2718 void visitExtractValueInst(ExtractValueInst &EVI) {
2719 auto &Res = getValue(V: EVI.getAggregateOperand());
2720 const AnyValue *Pos = &Res;
2721 for (unsigned Idx : EVI.indices())
2722 Pos = &Pos->asAggregate()[Idx];
2723 setResult(I&: EVI, V: *Pos);
2724 }
2725
2726 void visitInsertValueInst(InsertValueInst &IVI) {
2727 AnyValue Res = getValue(V: IVI.getAggregateOperand());
2728 AnyValue *Pos = &Res;
2729 for (unsigned Idx : IVI.indices())
2730 Pos = &Pos->asAggregate()[Idx];
2731 *Pos = getValue(V: IVI.getInsertedValueOperand());
2732 setResult(I&: IVI, V: std::move(Res));
2733 }
2734
2735 void visitInsertElementInst(InsertElementInst &IEI) {
2736 auto Res = getValue(V: IEI.getOperand(i_nocapture: 0));
2737 auto &ResVec = Res.asAggregate();
2738 auto &Idx = getValue(V: IEI.getOperand(i_nocapture: 2));
2739 if (Idx.isPoison() || Idx.asInteger().uge(RHS: ResVec.size())) {
2740 setResult(I&: IEI, V: AnyValue::getPoisonValue(Ctx, Ty: IEI.getType()));
2741 return;
2742 }
2743 ResVec[Idx.asInteger().getZExtValue()] = getValue(V: IEI.getOperand(i_nocapture: 1));
2744 setResult(I&: IEI, V: std::move(Res));
2745 }
2746
2747 void visitExtractElementInst(ExtractElementInst &EEI) {
2748 auto &SrcVec = getValue(V: EEI.getOperand(i_nocapture: 0)).asAggregate();
2749 auto &Idx = getValue(V: EEI.getOperand(i_nocapture: 1));
2750 if (Idx.isPoison() || Idx.asInteger().uge(RHS: SrcVec.size())) {
2751 setResult(I&: EEI, V: AnyValue::getPoisonValue(Ctx, Ty: EEI.getType()));
2752 return;
2753 }
2754 setResult(I&: EEI, V: SrcVec[Idx.asInteger().getZExtValue()]);
2755 }
2756
2757 void visitShuffleVectorInst(ShuffleVectorInst &SVI) {
2758 auto &LHSVec = getValue(V: SVI.getOperand(i_nocapture: 0)).asAggregate();
2759 auto &RHSVec = getValue(V: SVI.getOperand(i_nocapture: 1)).asAggregate();
2760 uint32_t Size = cast<VectorType>(Val: SVI.getOperand(i_nocapture: 0)->getType())
2761 ->getElementCount()
2762 .getKnownMinValue();
2763 std::vector<AnyValue> Res;
2764 uint32_t DstLen = Ctx.getEVL(EC: SVI.getType()->getElementCount());
2765 Res.reserve(n: DstLen);
2766 uint32_t Stride = SVI.getShuffleMask().size();
2767 // For scalable vectors, we need to repeat the shuffle mask until we fill
2768 // the destination vector.
2769 for (uint32_t Off = 0; Off != DstLen; Off += Stride) {
2770 for (int Idx : SVI.getShuffleMask()) {
2771 if (Idx == PoisonMaskElem)
2772 Res.push_back(x: AnyValue::poison());
2773 else if (Idx < static_cast<int>(Size))
2774 Res.push_back(x: LHSVec[Idx]);
2775 else
2776 Res.push_back(x: RHSVec[Idx - Size]);
2777 }
2778 }
2779 setResult(I&: SVI, V: std::move(Res));
2780 }
2781
2782 void visitBitCastInst(BitCastInst &BCI) {
2783 // The conversion is done as if the value had been stored to memory and read
2784 // back as the target type.
2785 SmallVector<Byte> Bytes;
2786 Bytes.resize(N: Ctx.getEffectiveTypeStoreSize(Ty: BCI.getType()),
2787 NV: Byte::concrete(Val: 0));
2788 Ctx.toBytes(Val: getValue(V: BCI.getOperand(i_nocapture: 0)), Ty: BCI.getOperand(i_nocapture: 0)->getType(),
2789 Bytes);
2790 setResult(I&: BCI, V: Ctx.fromBytes(Bytes, Ty: BCI.getType()));
2791 }
2792
2793 void visitFreezeInst(FreezeInst &FI) {
2794 AnyValue Val = getValue(V: FI.getOperand(i_nocapture: 0));
2795 Ctx.freeze(Val, Ty: FI.getType());
2796 setResult(I&: FI, V: std::move(Val));
2797 }
2798
2799 /// This function implements the main interpreter loop.
2800 /// It handles function calls in a non-recursive manner to avoid stack
2801 /// overflows.
2802 ProgramExitInfo runMainLoop() {
2803 uint32_t MaxSteps = Ctx.getMaxSteps();
2804 uint32_t Steps = 0;
2805 while (!hasProgramExited() && !CallStack.empty()) {
2806 Frame &Top = CallStack.back();
2807 CurrentFrame = &Top;
2808 if (Top.State == FrameState::Entry) {
2809 Handler.onFunctionEntry(F&: Top.Func, Args: Top.Args, CallSite: Top.CallSite);
2810 } else {
2811 assert(Top.State == FrameState::Pending &&
2812 "Expected to return from a callee.");
2813 returnFromCallee();
2814 }
2815
2816 Top.State = FrameState::Running;
2817 // Interpreter loop inside a function
2818 while (!hasProgramExited()) {
2819 assert(Top.State == FrameState::Running &&
2820 "Expected to be in running state.");
2821 if (MaxSteps != 0 && Steps >= MaxSteps) {
2822 reportError() << "Exceeded maximum number of execution steps.";
2823 break;
2824 }
2825 ++Steps;
2826
2827 Instruction &I = *Top.PC;
2828 visit(I: &I);
2829 Ctx.resetNoncacheableConstantBuffer();
2830 if (hasProgramExited())
2831 break;
2832
2833 // A function call or return has occurred.
2834 // We need to exit the inner loop and switch to a different frame.
2835 if (Top.State != FrameState::Running)
2836 break;
2837
2838 // Otherwise, move to the next instruction if it is not a terminator.
2839 // For terminators, the PC is updated in the visit* method.
2840 if (!I.isTerminator())
2841 ++Top.PC;
2842 }
2843
2844 if (hasProgramExited())
2845 break;
2846
2847 if (Top.State == FrameState::Exit) {
2848 assert((Top.Func.getReturnType()->isVoidTy() || !Top.RetVal.isNone()) &&
2849 "Expected return value to be set on function exit.");
2850 Handler.onFunctionExit(F&: Top.Func, RetVal: Top.RetVal);
2851 // Free stack objects allocated in this frame.
2852 for (auto &Obj : Top.Allocas)
2853 Ctx.free(Obj: *Obj);
2854 CallStack.pop_back();
2855 } else {
2856 assert(Top.State == FrameState::Pending &&
2857 "Expected to enter a callee.");
2858 }
2859 }
2860 if (!hasProgramExited())
2861 requestProgramExit(Kind: ProgramExitInfo::ProgramExitKind::Returned);
2862 return *getExitInfo();
2863 }
2864};
2865
2866ProgramExitInfo Context::runFunction(Function &F, ArrayRef<AnyValue> Args,
2867 AnyValue &RetVal, EventHandler &Handler) {
2868 InstExecutor Executor(*this, Handler, F, Args, RetVal);
2869 return Executor.runMainLoop();
2870}
2871
2872} // namespace llvm::ubi
2873