1//===- InstCombineSimplifyDemanded.cpp ------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains logic for simplifying instructions based on information
10// about how they are used.
11//
12//===----------------------------------------------------------------------===//
13
14#include "InstCombineInternal.h"
15#include "llvm/ADT/SmallBitVector.h"
16#include "llvm/Analysis/ValueTracking.h"
17#include "llvm/Analysis/VectorUtils.h"
18#include "llvm/IR/GetElementPtrTypeIterator.h"
19#include "llvm/IR/IntrinsicInst.h"
20#include "llvm/IR/PatternMatch.h"
21#include "llvm/Support/KnownBits.h"
22#include "llvm/Transforms/InstCombine/InstCombiner.h"
23
24using namespace llvm;
25using namespace llvm::PatternMatch;
26
27#define DEBUG_TYPE "instcombine"
28
29/// Check to see if the specified operand of the specified instruction is a
30/// constant integer. If so, check to see if there are any bits set in the
31/// constant that are not demanded. If so, shrink the constant and return true.
32static bool ShrinkDemandedConstant(Instruction *I, unsigned OpNo,
33 const APInt &Demanded) {
34 assert(I && "No instruction?");
35 assert(OpNo < I->getNumOperands() && "Operand index too large");
36
37 // The operand must be a constant integer or splat integer.
38 Value *Op = I->getOperand(i: OpNo);
39 const APInt *C;
40 if (!match(V: Op, P: m_APInt(Res&: C)))
41 return false;
42
43 // If there are no bits set that aren't demanded, nothing to do.
44 if (C->isSubsetOf(RHS: Demanded))
45 return false;
46
47 // This instruction is producing bits that are not demanded. Shrink the RHS.
48 I->setOperand(i: OpNo, Val: ConstantInt::get(Ty: Op->getType(), V: *C & Demanded));
49
50 return true;
51}
52
53/// Let N = 2 * M.
54/// Given an N-bit integer representing a pack of two M-bit integers,
55/// we can select one of the packed integers by right-shifting by either
56/// zero or M (which is the most straightforward to check if M is a power
57/// of 2), and then isolating the lower M bits. In this case, we can
58/// represent the shift as a select on whether the shr amount is nonzero.
59static Value *simplifyShiftSelectingPackedElement(Instruction *I,
60 const APInt &DemandedMask,
61 InstCombinerImpl &IC,
62 unsigned Depth) {
63 assert(I->getOpcode() == Instruction::LShr &&
64 "Only lshr instruction supported");
65
66 uint64_t ShlAmt;
67 Value *Upper, *Lower;
68 if (!match(V: I->getOperand(i: 0),
69 P: m_OneUse(SubPattern: m_c_DisjointOr(
70 L: m_OneUse(SubPattern: m_Shl(L: m_Value(V&: Upper), R: m_ConstantInt(V&: ShlAmt))),
71 R: m_Value(V&: Lower)))))
72 return nullptr;
73
74 if (!isPowerOf2_64(Value: ShlAmt))
75 return nullptr;
76
77 const uint64_t DemandedBitWidth = DemandedMask.getActiveBits();
78 if (DemandedBitWidth > ShlAmt)
79 return nullptr;
80
81 // Check that upper demanded bits are not lost from lshift.
82 if (Upper->getType()->getScalarSizeInBits() < ShlAmt + DemandedBitWidth)
83 return nullptr;
84
85 KnownBits KnownLowerBits = IC.computeKnownBits(V: Lower, CtxI: I, Depth);
86 if (!KnownLowerBits.getMaxValue().isIntN(N: ShlAmt))
87 return nullptr;
88
89 Value *ShrAmt = I->getOperand(i: 1);
90 KnownBits KnownShrBits = IC.computeKnownBits(V: ShrAmt, CtxI: I, Depth);
91
92 // Verify that ShrAmt is either exactly ShlAmt (which is a power of 2) or
93 // zero.
94 if (~KnownShrBits.Zero != ShlAmt)
95 return nullptr;
96
97 IRBuilderBase::InsertPointGuard Guard(IC.Builder);
98 IC.Builder.SetInsertPoint(I);
99 Value *ShrAmtZ =
100 IC.Builder.CreateICmpEQ(LHS: ShrAmt, RHS: Constant::getNullValue(Ty: ShrAmt->getType()),
101 Name: ShrAmt->getName() + ".z");
102 // There is no existing !prof metadata we can derive the !prof metadata for
103 // this select.
104 Value *Select = IC.Builder.CreateSelectWithUnknownProfile(C: ShrAmtZ, True: Lower,
105 False: Upper, DEBUG_TYPE);
106 Select->takeName(V: I);
107 return Select;
108}
109
110/// Returns the bitwidth of the given scalar or pointer type. For vector types,
111/// returns the element type's bitwidth.
112static unsigned getBitWidth(Type *Ty, const DataLayout &DL) {
113 if (unsigned BitWidth = Ty->getScalarSizeInBits())
114 return BitWidth;
115
116 return DL.getPointerTypeSizeInBits(Ty);
117}
118
119/// Inst is an integer instruction that SimplifyDemandedBits knows about. See if
120/// the instruction has any properties that allow us to simplify its operands.
121bool InstCombinerImpl::SimplifyDemandedInstructionBits(Instruction &Inst,
122 KnownBits &Known) {
123 APInt DemandedMask(APInt::getAllOnes(numBits: Known.getBitWidth()));
124 Value *V = SimplifyDemandedUseBits(I: &Inst, DemandedMask, Known,
125 Q: SQ.getWithInstruction(I: &Inst));
126 if (!V) return false;
127 if (V == &Inst) return true;
128 replaceInstUsesWith(I&: Inst, V);
129 return true;
130}
131
132/// Inst is an integer instruction that SimplifyDemandedBits knows about. See if
133/// the instruction has any properties that allow us to simplify its operands.
134bool InstCombinerImpl::SimplifyDemandedInstructionBits(Instruction &Inst) {
135 KnownBits Known(getBitWidth(Ty: Inst.getType(), DL));
136 return SimplifyDemandedInstructionBits(Inst, Known);
137}
138
139bool InstCombinerImpl::SimplifyDemandedInstructionFPClass(Instruction &Inst) {
140 KnownFPClass Known;
141
142 Value *V = SimplifyDemandedUseFPClass(I: &Inst, DemandedMask: fcAllFlags, Known,
143 Q: SQ.getWithInstruction(I: &Inst));
144 if (!V)
145 return false;
146 if (V == &Inst)
147 return true;
148 replaceInstUsesWith(I&: Inst, V);
149 return true;
150}
151
152/// This form of SimplifyDemandedBits simplifies the specified instruction
153/// operand if possible, updating it in place. It returns true if it made any
154/// change and false otherwise.
155bool InstCombinerImpl::SimplifyDemandedBits(Instruction *I, unsigned OpNo,
156 const APInt &DemandedMask,
157 KnownBits &Known,
158 const SimplifyQuery &Q,
159 unsigned Depth) {
160 Use &U = I->getOperandUse(i: OpNo);
161 Value *V = U.get();
162 if (isa<Constant>(Val: V)) {
163 llvm::computeKnownBits(V, Known, Q, Depth);
164 return false;
165 }
166
167 Known.resetAll();
168 if (DemandedMask.isZero()) {
169 // Not demanding any bits from V.
170 replaceUse(U, NewValue: UndefValue::get(T: V->getType()));
171 return true;
172 }
173
174 Instruction *VInst = dyn_cast<Instruction>(Val: V);
175 if (!VInst) {
176 llvm::computeKnownBits(V, Known, Q, Depth);
177 return false;
178 }
179
180 if (Depth == MaxAnalysisRecursionDepth)
181 return false;
182
183 Value *NewVal;
184 if (VInst->hasOneUse()) {
185 // If the instruction has one use, we can directly simplify it.
186 NewVal = SimplifyDemandedUseBits(I: VInst, DemandedMask, Known, Q, Depth);
187 } else {
188 // If there are multiple uses of this instruction, then we can simplify
189 // VInst to some other value, but not modify the instruction.
190 NewVal =
191 SimplifyMultipleUseDemandedBits(I: VInst, DemandedMask, Known, Q, Depth);
192 }
193 if (!NewVal) return false;
194 if (Instruction* OpInst = dyn_cast<Instruction>(Val&: U))
195 salvageDebugInfo(I&: *OpInst);
196
197 replaceUse(U, NewValue: NewVal);
198 return true;
199}
200
201/// This function attempts to replace V with a simpler value based on the
202/// demanded bits. When this function is called, it is known that only the bits
203/// set in DemandedMask of the result of V are ever used downstream.
204/// Consequently, depending on the mask and V, it may be possible to replace V
205/// with a constant or one of its operands. In such cases, this function does
206/// the replacement and returns true. In all other cases, it returns false after
207/// analyzing the expression and setting KnownOne and known to be one in the
208/// expression. Known.Zero contains all the bits that are known to be zero in
209/// the expression. These are provided to potentially allow the caller (which
210/// might recursively be SimplifyDemandedBits itself) to simplify the
211/// expression.
212/// Known.One and Known.Zero always follow the invariant that:
213/// Known.One & Known.Zero == 0.
214/// That is, a bit can't be both 1 and 0. The bits in Known.One and Known.Zero
215/// are accurate even for bits not in DemandedMask. Note
216/// also that the bitwidth of V, DemandedMask, Known.Zero and Known.One must all
217/// be the same.
218///
219/// This returns null if it did not change anything and it permits no
220/// simplification. This returns V itself if it did some simplification of V's
221/// operands based on the information about what bits are demanded. This returns
222/// some other non-null value if it found out that V is equal to another value
223/// in the context where the specified bits are demanded, but not for all users.
224Value *InstCombinerImpl::SimplifyDemandedUseBits(Instruction *I,
225 const APInt &DemandedMask,
226 KnownBits &Known,
227 const SimplifyQuery &Q,
228 unsigned Depth) {
229 assert(I != nullptr && "Null pointer of Value???");
230 assert(Depth <= MaxAnalysisRecursionDepth && "Limit Search Depth");
231 uint32_t BitWidth = DemandedMask.getBitWidth();
232 Type *VTy = I->getType();
233 assert(
234 (!VTy->isIntOrIntVectorTy() || VTy->getScalarSizeInBits() == BitWidth) &&
235 Known.getBitWidth() == BitWidth &&
236 "Value *V, DemandedMask and Known must have same BitWidth");
237
238 KnownBits LHSKnown(BitWidth), RHSKnown(BitWidth);
239
240 // Update flags after simplifying an operand based on the fact that some high
241 // order bits are not demanded.
242 auto disableWrapFlagsBasedOnUnusedHighBits = [](Instruction *I,
243 unsigned NLZ) {
244 if (NLZ > 0) {
245 // Disable the nsw and nuw flags here: We can no longer guarantee that
246 // we won't wrap after simplification. Removing the nsw/nuw flags is
247 // legal here because the top bit is not demanded.
248 I->setHasNoSignedWrap(false);
249 I->setHasNoUnsignedWrap(false);
250 }
251 return I;
252 };
253
254 // If the high-bits of an ADD/SUB/MUL are not demanded, then we do not care
255 // about the high bits of the operands.
256 auto simplifyOperandsBasedOnUnusedHighBits = [&](APInt &DemandedFromOps) {
257 unsigned NLZ = DemandedMask.countl_zero();
258 // Right fill the mask of bits for the operands to demand the most
259 // significant bit and all those below it.
260 DemandedFromOps = APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: BitWidth - NLZ);
261 if (ShrinkDemandedConstant(I, OpNo: 0, Demanded: DemandedFromOps) ||
262 SimplifyDemandedBits(I, OpNo: 0, DemandedMask: DemandedFromOps, Known&: LHSKnown, Q, Depth: Depth + 1) ||
263 ShrinkDemandedConstant(I, OpNo: 1, Demanded: DemandedFromOps) ||
264 SimplifyDemandedBits(I, OpNo: 1, DemandedMask: DemandedFromOps, Known&: RHSKnown, Q, Depth: Depth + 1)) {
265 disableWrapFlagsBasedOnUnusedHighBits(I, NLZ);
266 return true;
267 }
268 return false;
269 };
270
271 switch (I->getOpcode()) {
272 default:
273 llvm::computeKnownBits(V: I, Known, Q, Depth);
274 break;
275 case Instruction::And: {
276 // If either the LHS or the RHS are Zero, the result is zero.
277 if (SimplifyDemandedBits(I, OpNo: 1, DemandedMask, Known&: RHSKnown, Q, Depth: Depth + 1) ||
278 SimplifyDemandedBits(I, OpNo: 0, DemandedMask: DemandedMask & ~RHSKnown.Zero, Known&: LHSKnown, Q,
279 Depth: Depth + 1))
280 return I;
281
282 Known = analyzeKnownBitsFromAndXorOr(I: cast<Operator>(Val: I), KnownLHS: LHSKnown, KnownRHS: RHSKnown,
283 SQ: Q, Depth);
284
285 // If the client is only demanding bits that we know, return the known
286 // constant.
287 if (DemandedMask.isSubsetOf(RHS: Known.Zero | Known.One))
288 return Constant::getIntegerValue(Ty: VTy, V: Known.One);
289
290 // If all of the demanded bits are known 1 on one side, return the other.
291 // These bits cannot contribute to the result of the 'and'.
292 if (DemandedMask.isSubsetOf(RHS: LHSKnown.Zero | RHSKnown.One))
293 return I->getOperand(i: 0);
294 if (DemandedMask.isSubsetOf(RHS: RHSKnown.Zero | LHSKnown.One))
295 return I->getOperand(i: 1);
296
297 // If the RHS is a constant, see if we can simplify it.
298 if (ShrinkDemandedConstant(I, OpNo: 1, Demanded: DemandedMask & ~LHSKnown.Zero))
299 return I;
300
301 break;
302 }
303 case Instruction::Or: {
304 // If either the LHS or the RHS are One, the result is One.
305 if (SimplifyDemandedBits(I, OpNo: 1, DemandedMask, Known&: RHSKnown, Q, Depth: Depth + 1) ||
306 SimplifyDemandedBits(I, OpNo: 0, DemandedMask: DemandedMask & ~RHSKnown.One, Known&: LHSKnown, Q,
307 Depth: Depth + 1)) {
308 // Disjoint flag may not longer hold.
309 I->dropPoisonGeneratingFlags();
310 return I;
311 }
312
313 Known = analyzeKnownBitsFromAndXorOr(I: cast<Operator>(Val: I), KnownLHS: LHSKnown, KnownRHS: RHSKnown,
314 SQ: Q, Depth);
315
316 // If the client is only demanding bits that we know, return the known
317 // constant.
318 if (DemandedMask.isSubsetOf(RHS: Known.Zero | Known.One))
319 return Constant::getIntegerValue(Ty: VTy, V: Known.One);
320
321 // If all of the demanded bits are known zero on one side, return the other.
322 // These bits cannot contribute to the result of the 'or'.
323 if (DemandedMask.isSubsetOf(RHS: LHSKnown.One | RHSKnown.Zero))
324 return I->getOperand(i: 0);
325 if (DemandedMask.isSubsetOf(RHS: RHSKnown.One | LHSKnown.Zero))
326 return I->getOperand(i: 1);
327
328 // If the RHS is a constant, see if we can simplify it.
329 if (ShrinkDemandedConstant(I, OpNo: 1, Demanded: DemandedMask))
330 return I;
331
332 // Infer disjoint flag if no common bits are set.
333 if (!cast<PossiblyDisjointInst>(Val: I)->isDisjoint()) {
334 WithCache<const Value *> LHSCache(I->getOperand(i: 0), LHSKnown),
335 RHSCache(I->getOperand(i: 1), RHSKnown);
336 if (haveNoCommonBitsSet(LHSCache, RHSCache, SQ: Q)) {
337 cast<PossiblyDisjointInst>(Val: I)->setIsDisjoint(true);
338 return I;
339 }
340 }
341
342 break;
343 }
344 case Instruction::Xor: {
345 if (SimplifyDemandedBits(I, OpNo: 1, DemandedMask, Known&: RHSKnown, Q, Depth: Depth + 1) ||
346 SimplifyDemandedBits(I, OpNo: 0, DemandedMask, Known&: LHSKnown, Q, Depth: Depth + 1))
347 return I;
348 Value *LHS, *RHS;
349 if (DemandedMask == 1 && match(V: I->getOperand(i: 0), P: m_Ctpop(Op0: m_Value(V&: LHS))) &&
350 match(V: I->getOperand(i: 1), P: m_Ctpop(Op0: m_Value(V&: RHS)))) {
351 // (ctpop(X) ^ ctpop(Y)) & 1 --> ctpop(X^Y) & 1
352 IRBuilderBase::InsertPointGuard Guard(Builder);
353 Builder.SetInsertPoint(I);
354 auto *Xor = Builder.CreateXor(LHS, RHS);
355 return Builder.CreateUnaryIntrinsic(ID: Intrinsic::ctpop, Op: Xor);
356 }
357
358 Known = analyzeKnownBitsFromAndXorOr(I: cast<Operator>(Val: I), KnownLHS: LHSKnown, KnownRHS: RHSKnown,
359 SQ: Q, Depth);
360
361 // If the client is only demanding bits that we know, return the known
362 // constant.
363 if (DemandedMask.isSubsetOf(RHS: Known.Zero | Known.One))
364 return Constant::getIntegerValue(Ty: VTy, V: Known.One);
365
366 // If all of the demanded bits are known zero on one side, return the other.
367 // These bits cannot contribute to the result of the 'xor'.
368 if (DemandedMask.isSubsetOf(RHS: RHSKnown.Zero))
369 return I->getOperand(i: 0);
370 if (DemandedMask.isSubsetOf(RHS: LHSKnown.Zero))
371 return I->getOperand(i: 1);
372
373 // If all of the demanded bits are known to be zero on one side or the
374 // other, turn this into an *inclusive* or.
375 // e.g. (A & C1)^(B & C2) -> (A & C1)|(B & C2) iff C1&C2 == 0
376 if (DemandedMask.isSubsetOf(RHS: RHSKnown.Zero | LHSKnown.Zero)) {
377 Instruction *Or =
378 BinaryOperator::CreateOr(V1: I->getOperand(i: 0), V2: I->getOperand(i: 1));
379 if (DemandedMask.isAllOnes())
380 cast<PossiblyDisjointInst>(Val: Or)->setIsDisjoint(true);
381 Or->takeName(V: I);
382 return InsertNewInstWith(New: Or, Old: I->getIterator());
383 }
384
385 // If all of the demanded bits on one side are known, and all of the set
386 // bits on that side are also known to be set on the other side, turn this
387 // into an AND, as we know the bits will be cleared.
388 // e.g. (X | C1) ^ C2 --> (X | C1) & ~C2 iff (C1&C2) == C2
389 if (DemandedMask.isSubsetOf(RHS: RHSKnown.Zero|RHSKnown.One) &&
390 RHSKnown.One.isSubsetOf(RHS: LHSKnown.One)) {
391 Constant *AndC = Constant::getIntegerValue(Ty: VTy,
392 V: ~RHSKnown.One & DemandedMask);
393 Instruction *And = BinaryOperator::CreateAnd(V1: I->getOperand(i: 0), V2: AndC);
394 return InsertNewInstWith(New: And, Old: I->getIterator());
395 }
396
397 // If the RHS is a constant, see if we can change it. Don't alter a -1
398 // constant because that's a canonical 'not' op, and that is better for
399 // combining, SCEV, and codegen.
400 const APInt *C;
401 if (match(V: I->getOperand(i: 1), P: m_APInt(Res&: C)) && !C->isAllOnes()) {
402 if ((*C | ~DemandedMask).isAllOnes()) {
403 // Force bits to 1 to create a 'not' op.
404 I->setOperand(i: 1, Val: ConstantInt::getAllOnesValue(Ty: VTy));
405 return I;
406 }
407 // If we can't turn this into a 'not', try to shrink the constant.
408 if (ShrinkDemandedConstant(I, OpNo: 1, Demanded: DemandedMask))
409 return I;
410 }
411
412 // If our LHS is an 'and' and if it has one use, and if any of the bits we
413 // are flipping are known to be set, then the xor is just resetting those
414 // bits to zero. We can just knock out bits from the 'and' and the 'xor',
415 // simplifying both of them.
416 if (Instruction *LHSInst = dyn_cast<Instruction>(Val: I->getOperand(i: 0))) {
417 ConstantInt *AndRHS, *XorRHS;
418 if (LHSInst->getOpcode() == Instruction::And && LHSInst->hasOneUse() &&
419 match(V: I->getOperand(i: 1), P: m_ConstantInt(CI&: XorRHS)) &&
420 match(V: LHSInst->getOperand(i: 1), P: m_ConstantInt(CI&: AndRHS)) &&
421 (LHSKnown.One & RHSKnown.One & DemandedMask) != 0) {
422 APInt NewMask = ~(LHSKnown.One & RHSKnown.One & DemandedMask);
423
424 Constant *AndC = ConstantInt::get(Ty: VTy, V: NewMask & AndRHS->getValue());
425 Instruction *NewAnd = BinaryOperator::CreateAnd(V1: I->getOperand(i: 0), V2: AndC);
426 InsertNewInstWith(New: NewAnd, Old: I->getIterator());
427
428 Constant *XorC = ConstantInt::get(Ty: VTy, V: NewMask & XorRHS->getValue());
429 Instruction *NewXor = BinaryOperator::CreateXor(V1: NewAnd, V2: XorC);
430 return InsertNewInstWith(New: NewXor, Old: I->getIterator());
431 }
432 }
433 break;
434 }
435 case Instruction::Select: {
436 if (SimplifyDemandedBits(I, OpNo: 2, DemandedMask, Known&: RHSKnown, Q, Depth: Depth + 1) ||
437 SimplifyDemandedBits(I, OpNo: 1, DemandedMask, Known&: LHSKnown, Q, Depth: Depth + 1))
438 return I;
439
440 // If the operands are constants, see if we can simplify them.
441 // This is similar to ShrinkDemandedConstant, but for a select we want to
442 // try to keep the selected constants the same as icmp value constants, if
443 // we can. This helps not break apart (or helps put back together)
444 // canonical patterns like min and max.
445 auto CanonicalizeSelectConstant = [](Instruction *I, unsigned OpNo,
446 const APInt &DemandedMask) {
447 const APInt *SelC;
448 if (!match(V: I->getOperand(i: OpNo), P: m_APInt(Res&: SelC)))
449 return false;
450
451 // Get the constant out of the ICmp, if there is one.
452 // Only try this when exactly 1 operand is a constant (if both operands
453 // are constant, the icmp should eventually simplify). Otherwise, we may
454 // invert the transform that reduces set bits and infinite-loop.
455 Value *X;
456 const APInt *CmpC;
457 if (!match(V: I->getOperand(i: 0), P: m_ICmp(L: m_Value(V&: X), R: m_APInt(Res&: CmpC))) ||
458 isa<Constant>(Val: X) || CmpC->getBitWidth() != SelC->getBitWidth())
459 return ShrinkDemandedConstant(I, OpNo, Demanded: DemandedMask);
460
461 // If the constant is already the same as the ICmp, leave it as-is.
462 if (*CmpC == *SelC)
463 return false;
464 // If the constants are not already the same, but can be with the demand
465 // mask, use the constant value from the ICmp.
466 if ((*CmpC & DemandedMask) == (*SelC & DemandedMask)) {
467 I->setOperand(i: OpNo, Val: ConstantInt::get(Ty: I->getType(), V: *CmpC));
468 return true;
469 }
470 return ShrinkDemandedConstant(I, OpNo, Demanded: DemandedMask);
471 };
472 if (CanonicalizeSelectConstant(I, 1, DemandedMask) ||
473 CanonicalizeSelectConstant(I, 2, DemandedMask))
474 return I;
475
476 // Only known if known in both the LHS and RHS.
477 adjustKnownBitsForSelectArm(Known&: LHSKnown, Cond: I->getOperand(i: 0), Arm: I->getOperand(i: 1),
478 /*Invert=*/false, Q, Depth);
479 adjustKnownBitsForSelectArm(Known&: RHSKnown, Cond: I->getOperand(i: 0), Arm: I->getOperand(i: 2),
480 /*Invert=*/true, Q, Depth);
481 Known = LHSKnown.intersectWith(RHS: RHSKnown);
482 break;
483 }
484 case Instruction::Trunc: {
485 // If we do not demand the high bits of a right-shifted and truncated value,
486 // then we may be able to truncate it before the shift.
487 Value *X;
488 const APInt *C;
489 if (match(V: I->getOperand(i: 0), P: m_OneUse(SubPattern: m_LShr(L: m_Value(V&: X), R: m_APInt(Res&: C))))) {
490 // The shift amount must be valid (not poison) in the narrow type, and
491 // it must not be greater than the high bits demanded of the result.
492 if (C->ult(RHS: VTy->getScalarSizeInBits()) &&
493 C->ule(RHS: DemandedMask.countl_zero())) {
494 // trunc (lshr X, C) --> lshr (trunc X), C
495 IRBuilderBase::InsertPointGuard Guard(Builder);
496 Builder.SetInsertPoint(I);
497 Value *Trunc = Builder.CreateTrunc(V: X, DestTy: VTy);
498 return Builder.CreateLShr(LHS: Trunc, RHS: C->getZExtValue());
499 }
500 }
501 }
502 [[fallthrough]];
503 case Instruction::ZExt: {
504 unsigned SrcBitWidth = I->getOperand(i: 0)->getType()->getScalarSizeInBits();
505
506 APInt InputDemandedMask = DemandedMask.zextOrTrunc(width: SrcBitWidth);
507 KnownBits InputKnown(SrcBitWidth);
508 if (SimplifyDemandedBits(I, OpNo: 0, DemandedMask: InputDemandedMask, Known&: InputKnown, Q,
509 Depth: Depth + 1)) {
510 // For zext nneg, we may have dropped the instruction which made the
511 // input non-negative.
512 I->dropPoisonGeneratingFlags();
513 return I;
514 }
515 assert(InputKnown.getBitWidth() == SrcBitWidth && "Src width changed?");
516 if (I->getOpcode() == Instruction::ZExt && I->hasNonNeg() &&
517 !InputKnown.isNegative())
518 InputKnown.makeNonNegative();
519 Known = InputKnown.zextOrTrunc(BitWidth);
520
521 break;
522 }
523 case Instruction::SExt: {
524 // Compute the bits in the result that are not present in the input.
525 unsigned SrcBitWidth = I->getOperand(i: 0)->getType()->getScalarSizeInBits();
526
527 APInt InputDemandedBits = DemandedMask.trunc(width: SrcBitWidth);
528
529 // If any of the sign extended bits are demanded, we know that the sign
530 // bit is demanded.
531 if (DemandedMask.getActiveBits() > SrcBitWidth)
532 InputDemandedBits.setBit(SrcBitWidth-1);
533
534 KnownBits InputKnown(SrcBitWidth);
535 if (SimplifyDemandedBits(I, OpNo: 0, DemandedMask: InputDemandedBits, Known&: InputKnown, Q, Depth: Depth + 1))
536 return I;
537
538 // If the input sign bit is known zero, or if the NewBits are not demanded
539 // convert this into a zero extension.
540 if (InputKnown.isNonNegative() ||
541 DemandedMask.getActiveBits() <= SrcBitWidth) {
542 // Convert to ZExt cast.
543 CastInst *NewCast = new ZExtInst(I->getOperand(i: 0), VTy);
544 NewCast->takeName(V: I);
545 return InsertNewInstWith(New: NewCast, Old: I->getIterator());
546 }
547
548 // If the sign bit of the input is known set or clear, then we know the
549 // top bits of the result.
550 Known = InputKnown.sext(BitWidth);
551 break;
552 }
553 case Instruction::Add: {
554 if ((DemandedMask & 1) == 0) {
555 // If we do not need the low bit, try to convert bool math to logic:
556 // add iN (zext i1 X), (sext i1 Y) --> sext (~X & Y) to iN
557 Value *X, *Y;
558 if (match(V: I, P: m_c_Add(L: m_OneUse(SubPattern: m_ZExt(Op: m_Value(V&: X))),
559 R: m_OneUse(SubPattern: m_SExt(Op: m_Value(V&: Y))))) &&
560 X->getType()->isIntOrIntVectorTy(BitWidth: 1) && X->getType() == Y->getType()) {
561 // Truth table for inputs and output signbits:
562 // X:0 | X:1
563 // ----------
564 // Y:0 | 0 | 0 |
565 // Y:1 | -1 | 0 |
566 // ----------
567 IRBuilderBase::InsertPointGuard Guard(Builder);
568 Builder.SetInsertPoint(I);
569 Value *AndNot = Builder.CreateAnd(LHS: Builder.CreateNot(V: X), RHS: Y);
570 return Builder.CreateSExt(V: AndNot, DestTy: VTy);
571 }
572
573 // add iN (sext i1 X), (sext i1 Y) --> sext (X | Y) to iN
574 if (match(V: I, P: m_Add(L: m_SExt(Op: m_Value(V&: X)), R: m_SExt(Op: m_Value(V&: Y)))) &&
575 X->getType()->isIntOrIntVectorTy(BitWidth: 1) && X->getType() == Y->getType() &&
576 (I->getOperand(i: 0)->hasOneUse() || I->getOperand(i: 1)->hasOneUse())) {
577
578 // Truth table for inputs and output signbits:
579 // X:0 | X:1
580 // -----------
581 // Y:0 | 0 | -1 |
582 // Y:1 | -1 | -1 |
583 // -----------
584 IRBuilderBase::InsertPointGuard Guard(Builder);
585 Builder.SetInsertPoint(I);
586 Value *Or = Builder.CreateOr(LHS: X, RHS: Y);
587 return Builder.CreateSExt(V: Or, DestTy: VTy);
588 }
589 }
590
591 // Right fill the mask of bits for the operands to demand the most
592 // significant bit and all those below it.
593 unsigned NLZ = DemandedMask.countl_zero();
594 APInt DemandedFromOps = APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: BitWidth - NLZ);
595 if (ShrinkDemandedConstant(I, OpNo: 1, Demanded: DemandedFromOps) ||
596 SimplifyDemandedBits(I, OpNo: 1, DemandedMask: DemandedFromOps, Known&: RHSKnown, Q, Depth: Depth + 1))
597 return disableWrapFlagsBasedOnUnusedHighBits(I, NLZ);
598
599 // If low order bits are not demanded and known to be zero in one operand,
600 // then we don't need to demand them from the other operand, since they
601 // can't cause overflow into any bits that are demanded in the result.
602 unsigned NTZ = (~DemandedMask & RHSKnown.Zero).countr_one();
603 APInt DemandedFromLHS = DemandedFromOps;
604 DemandedFromLHS.clearLowBits(loBits: NTZ);
605 if (ShrinkDemandedConstant(I, OpNo: 0, Demanded: DemandedFromLHS) ||
606 SimplifyDemandedBits(I, OpNo: 0, DemandedMask: DemandedFromLHS, Known&: LHSKnown, Q, Depth: Depth + 1))
607 return disableWrapFlagsBasedOnUnusedHighBits(I, NLZ);
608
609 unsigned NtzLHS = (~DemandedMask & LHSKnown.Zero).countr_one();
610 APInt DemandedFromRHS = DemandedFromOps;
611 DemandedFromRHS.clearLowBits(loBits: NtzLHS);
612 if (ShrinkDemandedConstant(I, OpNo: 1, Demanded: DemandedFromRHS))
613 return disableWrapFlagsBasedOnUnusedHighBits(I, NLZ);
614
615 // If we are known to be adding zeros to every bit below
616 // the highest demanded bit, we just return the other side.
617 if (DemandedFromOps.isSubsetOf(RHS: RHSKnown.Zero))
618 return I->getOperand(i: 0);
619 if (DemandedFromOps.isSubsetOf(RHS: LHSKnown.Zero))
620 return I->getOperand(i: 1);
621
622 // (add X, C) --> (xor X, C) IFF C is equal to the top bit of the DemandMask
623 {
624 const APInt *C;
625 if (match(V: I->getOperand(i: 1), P: m_APInt(Res&: C)) &&
626 C->isOneBitSet(BitNo: DemandedMask.getActiveBits() - 1)) {
627 IRBuilderBase::InsertPointGuard Guard(Builder);
628 Builder.SetInsertPoint(I);
629 return Builder.CreateXor(LHS: I->getOperand(i: 0), RHS: ConstantInt::get(Ty: VTy, V: *C));
630 }
631 }
632
633 // Otherwise just compute the known bits of the result.
634 bool NSW = cast<OverflowingBinaryOperator>(Val: I)->hasNoSignedWrap();
635 bool NUW = cast<OverflowingBinaryOperator>(Val: I)->hasNoUnsignedWrap();
636 Known = KnownBits::add(LHS: LHSKnown, RHS: RHSKnown, NSW, NUW);
637 break;
638 }
639 case Instruction::Sub: {
640 // Right fill the mask of bits for the operands to demand the most
641 // significant bit and all those below it.
642 unsigned NLZ = DemandedMask.countl_zero();
643 APInt DemandedFromOps = APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: BitWidth - NLZ);
644 if (ShrinkDemandedConstant(I, OpNo: 1, Demanded: DemandedFromOps) ||
645 SimplifyDemandedBits(I, OpNo: 1, DemandedMask: DemandedFromOps, Known&: RHSKnown, Q, Depth: Depth + 1))
646 return disableWrapFlagsBasedOnUnusedHighBits(I, NLZ);
647
648 // If low order bits are not demanded and are known to be zero in RHS,
649 // then we don't need to demand them from LHS, since they can't cause a
650 // borrow from any bits that are demanded in the result.
651 unsigned NTZ = (~DemandedMask & RHSKnown.Zero).countr_one();
652 APInt DemandedFromLHS = DemandedFromOps;
653 DemandedFromLHS.clearLowBits(loBits: NTZ);
654 if (ShrinkDemandedConstant(I, OpNo: 0, Demanded: DemandedFromLHS) ||
655 SimplifyDemandedBits(I, OpNo: 0, DemandedMask: DemandedFromLHS, Known&: LHSKnown, Q, Depth: Depth + 1))
656 return disableWrapFlagsBasedOnUnusedHighBits(I, NLZ);
657
658 // If we are known to be subtracting zeros from every bit below
659 // the highest demanded bit, we just return the other side.
660 if (DemandedFromOps.isSubsetOf(RHS: RHSKnown.Zero))
661 return I->getOperand(i: 0);
662 // We can't do this with the LHS for subtraction, unless we are only
663 // demanding the LSB.
664 if (DemandedFromOps.isOne() && DemandedFromOps.isSubsetOf(RHS: LHSKnown.Zero))
665 return I->getOperand(i: 1);
666
667 // Canonicalize sub mask, X -> ~X
668 const APInt *LHSC;
669 if (match(V: I->getOperand(i: 0), P: m_LowBitMask(V&: LHSC)) &&
670 DemandedFromOps.isSubsetOf(RHS: *LHSC)) {
671 IRBuilderBase::InsertPointGuard Guard(Builder);
672 Builder.SetInsertPoint(I);
673 return Builder.CreateNot(V: I->getOperand(i: 1));
674 }
675
676 // Otherwise just compute the known bits of the result.
677 bool NSW = cast<OverflowingBinaryOperator>(Val: I)->hasNoSignedWrap();
678 bool NUW = cast<OverflowingBinaryOperator>(Val: I)->hasNoUnsignedWrap();
679 Known = KnownBits::sub(LHS: LHSKnown, RHS: RHSKnown, NSW, NUW);
680 break;
681 }
682 case Instruction::Mul: {
683 APInt DemandedFromOps;
684 if (simplifyOperandsBasedOnUnusedHighBits(DemandedFromOps))
685 return I;
686
687 if (DemandedMask.isPowerOf2()) {
688 // The LSB of X*Y is set only if (X & 1) == 1 and (Y & 1) == 1.
689 // If we demand exactly one bit N and we have "X * (C' << N)" where C' is
690 // odd (has LSB set), then the left-shifted low bit of X is the answer.
691 unsigned CTZ = DemandedMask.countr_zero();
692 const APInt *C;
693 if (match(V: I->getOperand(i: 1), P: m_APInt(Res&: C)) && C->countr_zero() == CTZ) {
694 Constant *ShiftC = ConstantInt::get(Ty: VTy, V: CTZ);
695 Instruction *Shl = BinaryOperator::CreateShl(V1: I->getOperand(i: 0), V2: ShiftC);
696 return InsertNewInstWith(New: Shl, Old: I->getIterator());
697 }
698 }
699 // For a squared value "X * X", the bottom 2 bits are 0 and X[0] because:
700 // X * X is odd iff X is odd.
701 // 'Quadratic Reciprocity': X * X -> 0 for bit[1]
702 if (I->getOperand(i: 0) == I->getOperand(i: 1) && DemandedMask.ult(RHS: 4)) {
703 Constant *One = ConstantInt::get(Ty: VTy, V: 1);
704 Instruction *And1 = BinaryOperator::CreateAnd(V1: I->getOperand(i: 0), V2: One);
705 return InsertNewInstWith(New: And1, Old: I->getIterator());
706 }
707
708 llvm::computeKnownBits(V: I, Known, Q, Depth);
709 break;
710 }
711 case Instruction::Shl: {
712 const APInt *SA;
713 if (match(V: I->getOperand(i: 1), P: m_APInt(Res&: SA))) {
714 const APInt *ShrAmt;
715 if (match(V: I->getOperand(i: 0), P: m_Shr(L: m_Value(), R: m_APInt(Res&: ShrAmt))))
716 if (Instruction *Shr = dyn_cast<Instruction>(Val: I->getOperand(i: 0)))
717 if (Value *R = simplifyShrShlDemandedBits(Shr, ShrOp1: *ShrAmt, Shl: I, ShlOp1: *SA,
718 DemandedMask, Known))
719 return R;
720
721 // Do not simplify if shl is part of funnel-shift pattern
722 if (I->hasOneUse()) {
723 Instruction *Inst = I->user_back();
724 if (Inst->getOpcode() == BinaryOperator::Or) {
725 if (auto Opt = convertOrOfShiftsToFunnelShift(Or&: *Inst)) {
726 auto [IID, FShiftArgs] = *Opt;
727 if ((IID == Intrinsic::fshl || IID == Intrinsic::fshr) &&
728 FShiftArgs[0] == FShiftArgs[1]) {
729 llvm::computeKnownBits(V: I, Known, Q, Depth);
730 break;
731 }
732 }
733 }
734 }
735
736 // We only want bits that already match the signbit then we don't
737 // need to shift.
738 uint64_t ShiftAmt = SA->getLimitedValue(Limit: BitWidth - 1);
739 if (DemandedMask.countr_zero() >= ShiftAmt) {
740 if (I->hasNoSignedWrap()) {
741 unsigned NumHiDemandedBits = BitWidth - DemandedMask.countr_zero();
742 unsigned SignBits =
743 ComputeNumSignBits(Op: I->getOperand(i: 0), CtxI: Q.CtxI, Depth: Depth + 1);
744 if (SignBits > ShiftAmt && SignBits - ShiftAmt >= NumHiDemandedBits)
745 return I->getOperand(i: 0);
746 }
747
748 // If we can pre-shift a right-shifted constant to the left without
749 // losing any high bits and we don't demand the low bits, then eliminate
750 // the left-shift:
751 // (C >> X) << LeftShiftAmtC --> (C << LeftShiftAmtC) >> X
752 Value *X;
753 Constant *C;
754 if (match(V: I->getOperand(i: 0), P: m_LShr(L: m_ImmConstant(C), R: m_Value(V&: X)))) {
755 Constant *LeftShiftAmtC = ConstantInt::get(Ty: VTy, V: ShiftAmt);
756 Constant *NewC = ConstantFoldBinaryOpOperands(Opcode: Instruction::Shl, LHS: C,
757 RHS: LeftShiftAmtC, DL);
758 if (ConstantFoldBinaryOpOperands(Opcode: Instruction::LShr, LHS: NewC,
759 RHS: LeftShiftAmtC, DL) == C) {
760 Instruction *Lshr = BinaryOperator::CreateLShr(V1: NewC, V2: X);
761 return InsertNewInstWith(New: Lshr, Old: I->getIterator());
762 }
763 }
764 }
765
766 APInt DemandedMaskIn(DemandedMask.lshr(shiftAmt: ShiftAmt));
767
768 // If the shift is NUW/NSW, then it does demand the high bits.
769 ShlOperator *IOp = cast<ShlOperator>(Val: I);
770 if (IOp->hasNoSignedWrap())
771 DemandedMaskIn.setHighBits(ShiftAmt+1);
772 else if (IOp->hasNoUnsignedWrap())
773 DemandedMaskIn.setHighBits(ShiftAmt);
774
775 if (SimplifyDemandedBits(I, OpNo: 0, DemandedMask: DemandedMaskIn, Known, Q, Depth: Depth + 1))
776 return I;
777
778 Known = KnownBits::shl(LHS: Known,
779 RHS: KnownBits::makeConstant(C: APInt(BitWidth, ShiftAmt)),
780 /* NUW */ IOp->hasNoUnsignedWrap(),
781 /* NSW */ IOp->hasNoSignedWrap());
782 } else {
783 // This is a variable shift, so we can't shift the demand mask by a known
784 // amount. But if we are not demanding high bits, then we are not
785 // demanding those bits from the pre-shifted operand either.
786 if (unsigned CTLZ = DemandedMask.countl_zero()) {
787 APInt DemandedFromOp(APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: BitWidth - CTLZ));
788 if (SimplifyDemandedBits(I, OpNo: 0, DemandedMask: DemandedFromOp, Known, Q, Depth: Depth + 1)) {
789 // We can't guarantee that nsw/nuw hold after simplifying the operand.
790 I->dropPoisonGeneratingFlags();
791 return I;
792 }
793 }
794 llvm::computeKnownBits(V: I, Known, Q, Depth);
795 }
796 break;
797 }
798 case Instruction::LShr: {
799 const APInt *SA;
800 if (match(V: I->getOperand(i: 1), P: m_APInt(Res&: SA))) {
801 uint64_t ShiftAmt = SA->getLimitedValue(Limit: BitWidth-1);
802
803 // Do not simplify if lshr is part of funnel-shift pattern
804 if (I->hasOneUse()) {
805 Instruction *Inst = I->user_back();
806 if (Inst->getOpcode() == BinaryOperator::Or) {
807 if (auto Opt = convertOrOfShiftsToFunnelShift(Or&: *Inst)) {
808 auto [IID, FShiftArgs] = *Opt;
809 if ((IID == Intrinsic::fshl || IID == Intrinsic::fshr) &&
810 FShiftArgs[0] == FShiftArgs[1]) {
811 llvm::computeKnownBits(V: I, Known, Q, Depth);
812 break;
813 }
814 }
815 }
816 }
817
818 // If we are just demanding the shifted sign bit and below, then this can
819 // be treated as an ASHR in disguise.
820 if (DemandedMask.countl_zero() >= ShiftAmt) {
821 // If we only want bits that already match the signbit then we don't
822 // need to shift.
823 unsigned NumHiDemandedBits = BitWidth - DemandedMask.countr_zero();
824 unsigned SignBits =
825 ComputeNumSignBits(Op: I->getOperand(i: 0), CtxI: Q.CtxI, Depth: Depth + 1);
826 if (SignBits >= NumHiDemandedBits)
827 return I->getOperand(i: 0);
828
829 // If we can pre-shift a left-shifted constant to the right without
830 // losing any low bits (we already know we don't demand the high bits),
831 // then eliminate the right-shift:
832 // (C << X) >> RightShiftAmtC --> (C >> RightShiftAmtC) << X
833 Value *X;
834 Constant *C;
835 if (match(V: I->getOperand(i: 0), P: m_Shl(L: m_ImmConstant(C), R: m_Value(V&: X)))) {
836 Constant *RightShiftAmtC = ConstantInt::get(Ty: VTy, V: ShiftAmt);
837 Constant *NewC = ConstantFoldBinaryOpOperands(Opcode: Instruction::LShr, LHS: C,
838 RHS: RightShiftAmtC, DL);
839 if (ConstantFoldBinaryOpOperands(Opcode: Instruction::Shl, LHS: NewC,
840 RHS: RightShiftAmtC, DL) == C) {
841 Instruction *Shl = BinaryOperator::CreateShl(V1: NewC, V2: X);
842 return InsertNewInstWith(New: Shl, Old: I->getIterator());
843 }
844 }
845
846 const APInt *Factor;
847 if (match(V: I->getOperand(i: 0),
848 P: m_OneUse(SubPattern: m_Mul(L: m_Value(V&: X), R: m_APInt(Res&: Factor)))) &&
849 Factor->countr_zero() >= ShiftAmt) {
850 BinaryOperator *Mul = BinaryOperator::CreateMul(
851 V1: X, V2: ConstantInt::get(Ty: X->getType(), V: Factor->lshr(shiftAmt: ShiftAmt)));
852 return InsertNewInstWith(New: Mul, Old: I->getIterator());
853 }
854 }
855
856 // Unsigned shift right.
857 APInt DemandedMaskIn(DemandedMask.shl(shiftAmt: ShiftAmt));
858 if (SimplifyDemandedBits(I, OpNo: 0, DemandedMask: DemandedMaskIn, Known, Q, Depth: Depth + 1)) {
859 // exact flag may not longer hold.
860 I->dropPoisonGeneratingFlags();
861 return I;
862 }
863 Known >>= ShiftAmt;
864 if (ShiftAmt)
865 Known.Zero.setHighBits(ShiftAmt); // high bits known zero.
866 break;
867 }
868 if (Value *V =
869 simplifyShiftSelectingPackedElement(I, DemandedMask, IC&: *this, Depth))
870 return V;
871
872 llvm::computeKnownBits(V: I, Known, Q, Depth);
873 break;
874 }
875 case Instruction::AShr: {
876 unsigned SignBits = ComputeNumSignBits(Op: I->getOperand(i: 0), CtxI: Q.CtxI, Depth: Depth + 1);
877
878 // If we only want bits that already match the signbit then we don't need
879 // to shift.
880 unsigned NumHiDemandedBits = BitWidth - DemandedMask.countr_zero();
881 if (SignBits >= NumHiDemandedBits)
882 return I->getOperand(i: 0);
883
884 // If this is an arithmetic shift right and only the low-bit is set, we can
885 // always convert this into a logical shr, even if the shift amount is
886 // variable. The low bit of the shift cannot be an input sign bit unless
887 // the shift amount is >= the size of the datatype, which is undefined.
888 if (DemandedMask.isOne()) {
889 // Perform the logical shift right.
890 Instruction *NewVal = BinaryOperator::CreateLShr(
891 V1: I->getOperand(i: 0), V2: I->getOperand(i: 1), Name: I->getName());
892 return InsertNewInstWith(New: NewVal, Old: I->getIterator());
893 }
894
895 const APInt *SA;
896 if (match(V: I->getOperand(i: 1), P: m_APInt(Res&: SA))) {
897 uint32_t ShiftAmt = SA->getLimitedValue(Limit: BitWidth-1);
898
899 // Signed shift right.
900 APInt DemandedMaskIn(DemandedMask.shl(shiftAmt: ShiftAmt));
901 // If any of the bits being shifted in are demanded, then we should set
902 // the sign bit as demanded.
903 bool ShiftedInBitsDemanded = DemandedMask.countl_zero() < ShiftAmt;
904 if (ShiftedInBitsDemanded)
905 DemandedMaskIn.setSignBit();
906 if (SimplifyDemandedBits(I, OpNo: 0, DemandedMask: DemandedMaskIn, Known, Q, Depth: Depth + 1)) {
907 // exact flag may not longer hold.
908 I->dropPoisonGeneratingFlags();
909 return I;
910 }
911
912 // If the input sign bit is known to be zero, or if none of the shifted in
913 // bits are demanded, turn this into an unsigned shift right.
914 if (Known.Zero[BitWidth - 1] || !ShiftedInBitsDemanded) {
915 BinaryOperator *LShr = BinaryOperator::CreateLShr(V1: I->getOperand(i: 0),
916 V2: I->getOperand(i: 1));
917 LShr->setIsExact(cast<BinaryOperator>(Val: I)->isExact());
918 LShr->takeName(V: I);
919 return InsertNewInstWith(New: LShr, Old: I->getIterator());
920 }
921
922 Known = KnownBits::ashr(
923 LHS: Known, RHS: KnownBits::makeConstant(C: APInt(BitWidth, ShiftAmt)),
924 ShAmtNonZero: ShiftAmt != 0, Exact: I->isExact());
925 } else {
926 llvm::computeKnownBits(V: I, Known, Q, Depth);
927 }
928 break;
929 }
930 case Instruction::UDiv: {
931 // UDiv doesn't demand low bits that are zero in the divisor.
932 const APInt *SA;
933 if (match(V: I->getOperand(i: 1), P: m_APInt(Res&: SA))) {
934 // TODO: Take the demanded mask of the result into account.
935 unsigned RHSTrailingZeros = SA->countr_zero();
936 APInt DemandedMaskIn =
937 APInt::getHighBitsSet(numBits: BitWidth, hiBitsSet: BitWidth - RHSTrailingZeros);
938 if (SimplifyDemandedBits(I, OpNo: 0, DemandedMask: DemandedMaskIn, Known&: LHSKnown, Q, Depth: Depth + 1)) {
939 // We can't guarantee that "exact" is still true after changing the
940 // the dividend.
941 I->dropPoisonGeneratingFlags();
942 return I;
943 }
944
945 Known = KnownBits::udiv(LHS: LHSKnown, RHS: KnownBits::makeConstant(C: *SA),
946 Exact: cast<BinaryOperator>(Val: I)->isExact());
947 } else {
948 llvm::computeKnownBits(V: I, Known, Q, Depth);
949 }
950 break;
951 }
952 case Instruction::SRem: {
953 const APInt *Rem;
954 if (match(V: I->getOperand(i: 1), P: m_APInt(Res&: Rem)) && Rem->isPowerOf2()) {
955 if (DemandedMask.ult(RHS: *Rem)) // srem won't affect demanded bits
956 return I->getOperand(i: 0);
957
958 APInt LowBits = *Rem - 1;
959 APInt Mask2 = LowBits | APInt::getSignMask(BitWidth);
960 if (SimplifyDemandedBits(I, OpNo: 0, DemandedMask: Mask2, Known&: LHSKnown, Q, Depth: Depth + 1))
961 return I;
962 Known = KnownBits::srem(LHS: LHSKnown, RHS: KnownBits::makeConstant(C: *Rem));
963 break;
964 }
965
966 llvm::computeKnownBits(V: I, Known, Q, Depth);
967 break;
968 }
969 case Instruction::Call: {
970 bool KnownBitsComputed = false;
971 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: I)) {
972 switch (II->getIntrinsicID()) {
973 case Intrinsic::abs: {
974 if (DemandedMask == 1)
975 return II->getArgOperand(i: 0);
976 break;
977 }
978 case Intrinsic::ctpop: {
979 // Checking if the number of clear bits is odd (parity)? If the type has
980 // an even number of bits, that's the same as checking if the number of
981 // set bits is odd, so we can eliminate the 'not' op.
982 Value *X;
983 if (DemandedMask == 1 && VTy->getScalarSizeInBits() % 2 == 0 &&
984 match(V: II->getArgOperand(i: 0), P: m_Not(V: m_Value(V&: X)))) {
985 Function *Ctpop = Intrinsic::getOrInsertDeclaration(
986 M: II->getModule(), id: Intrinsic::ctpop, OverloadTys: VTy);
987 return InsertNewInstWith(New: CallInst::Create(Func: Ctpop, Args: {X}), Old: I->getIterator());
988 }
989 break;
990 }
991 case Intrinsic::bswap: {
992 // If the only bits demanded come from one byte of the bswap result,
993 // just shift the input byte into position to eliminate the bswap.
994 unsigned NLZ = DemandedMask.countl_zero();
995 unsigned NTZ = DemandedMask.countr_zero();
996
997 // Round NTZ down to the next byte. If we have 11 trailing zeros, then
998 // we need all the bits down to bit 8. Likewise, round NLZ. If we
999 // have 14 leading zeros, round to 8.
1000 NLZ = alignDown(Value: NLZ, Align: 8);
1001 NTZ = alignDown(Value: NTZ, Align: 8);
1002 // If we need exactly one byte, we can do this transformation.
1003 if (BitWidth - NLZ - NTZ == 8) {
1004 // Replace this with either a left or right shift to get the byte into
1005 // the right place.
1006 Instruction *NewVal;
1007 if (NLZ > NTZ)
1008 NewVal = BinaryOperator::CreateLShr(
1009 V1: II->getArgOperand(i: 0), V2: ConstantInt::get(Ty: VTy, V: NLZ - NTZ));
1010 else
1011 NewVal = BinaryOperator::CreateShl(
1012 V1: II->getArgOperand(i: 0), V2: ConstantInt::get(Ty: VTy, V: NTZ - NLZ));
1013 NewVal->takeName(V: I);
1014 return InsertNewInstWith(New: NewVal, Old: I->getIterator());
1015 }
1016 break;
1017 }
1018 case Intrinsic::ptrmask: {
1019 unsigned MaskWidth = I->getOperand(i: 1)->getType()->getScalarSizeInBits();
1020 RHSKnown = KnownBits(MaskWidth);
1021 // If either the LHS or the RHS are Zero, the result is zero.
1022 if (SimplifyDemandedBits(I, OpNo: 0, DemandedMask, Known&: LHSKnown, Q, Depth: Depth + 1) ||
1023 SimplifyDemandedBits(
1024 I, OpNo: 1, DemandedMask: (DemandedMask & ~LHSKnown.Zero).zextOrTrunc(width: MaskWidth),
1025 Known&: RHSKnown, Q, Depth: Depth + 1))
1026 return I;
1027
1028 // TODO: Should be 1-extend
1029 RHSKnown = RHSKnown.anyextOrTrunc(BitWidth);
1030
1031 Known = LHSKnown & RHSKnown;
1032 KnownBitsComputed = true;
1033
1034 // If the client is only demanding bits we know to be zero, return
1035 // `llvm.ptrmask(p, 0)`. We can't return `null` here due to pointer
1036 // provenance, but making the mask zero will be easily optimizable in
1037 // the backend.
1038 if (DemandedMask.isSubsetOf(RHS: Known.Zero) &&
1039 !match(V: I->getOperand(i: 1), P: m_Zero()))
1040 return replaceOperand(
1041 I&: *I, OpNum: 1, V: Constant::getNullValue(Ty: I->getOperand(i: 1)->getType()));
1042
1043 // Mask in demanded space does nothing.
1044 // NOTE: We may have attributes associated with the return value of the
1045 // llvm.ptrmask intrinsic that will be lost when we just return the
1046 // operand. We should try to preserve them.
1047 if (DemandedMask.isSubsetOf(RHS: RHSKnown.One | LHSKnown.Zero))
1048 return I->getOperand(i: 0);
1049
1050 // If the RHS is a constant, see if we can simplify it.
1051 if (ShrinkDemandedConstant(
1052 I, OpNo: 1, Demanded: (DemandedMask & ~LHSKnown.Zero).zextOrTrunc(width: MaskWidth)))
1053 return I;
1054
1055 // Combine:
1056 // (ptrmask (getelementptr i8, ptr p, imm i), imm mask)
1057 // -> (ptrmask (getelementptr i8, ptr p, imm (i & mask)), imm mask)
1058 // where only the low bits known to be zero in the pointer are changed
1059 Value *InnerPtr;
1060 uint64_t GEPIndex;
1061 uint64_t PtrMaskImmediate;
1062 if (match(V: I,
1063 P: m_Intrinsic<Intrinsic::ptrmask>(
1064 Ops: m_PtrAdd(DL, PointerOp: m_Value(V&: InnerPtr), OffsetOp: m_ConstantInt(V&: GEPIndex)),
1065 Ops: m_ConstantInt(V&: PtrMaskImmediate)))) {
1066
1067 LHSKnown = computeKnownBits(V: InnerPtr, CtxI: I, Depth: Depth + 1);
1068 if (!LHSKnown.isZero()) {
1069 const unsigned trailingZeros = LHSKnown.countMinTrailingZeros();
1070 uint64_t PointerAlignBits = (uint64_t(1) << trailingZeros) - 1;
1071
1072 uint64_t HighBitsGEPIndex = GEPIndex & ~PointerAlignBits;
1073 uint64_t MaskedLowBitsGEPIndex =
1074 GEPIndex & PointerAlignBits & PtrMaskImmediate;
1075
1076 uint64_t MaskedGEPIndex = HighBitsGEPIndex | MaskedLowBitsGEPIndex;
1077
1078 if (MaskedGEPIndex != GEPIndex) {
1079 auto *GEP = cast<GEPOperator>(Val: II->getArgOperand(i: 0));
1080 Builder.SetInsertPoint(I);
1081 Type *GEPIndexType =
1082 DL.getIndexType(PtrTy: GEP->getPointerOperand()->getType());
1083 Value *MaskedGEP = Builder.CreateGEP(
1084 Ty: GEP->getSourceElementType(), Ptr: InnerPtr,
1085 IdxList: ConstantInt::get(Ty: GEPIndexType, V: MaskedGEPIndex),
1086 Name: GEP->getName(), NW: GEP->isInBounds());
1087
1088 replaceOperand(I&: *I, OpNum: 0, V: MaskedGEP);
1089 return I;
1090 }
1091 }
1092 }
1093
1094 break;
1095 }
1096
1097 case Intrinsic::fshr:
1098 case Intrinsic::fshl: {
1099 const APInt *SA;
1100 if (!match(V: I->getOperand(i: 2), P: m_APInt(Res&: SA)))
1101 break;
1102
1103 // Normalize to funnel shift left. APInt shifts of BitWidth are well-
1104 // defined, so no need to special-case zero shifts here.
1105 uint64_t ShiftAmt = SA->urem(RHS: BitWidth);
1106 if (II->getIntrinsicID() == Intrinsic::fshr)
1107 ShiftAmt = BitWidth - ShiftAmt;
1108
1109 APInt DemandedMaskLHS(DemandedMask.lshr(shiftAmt: ShiftAmt));
1110 APInt DemandedMaskRHS(DemandedMask.shl(shiftAmt: BitWidth - ShiftAmt));
1111 if (I->getOperand(i: 0) != I->getOperand(i: 1)) {
1112 if (SimplifyDemandedBits(I, OpNo: 0, DemandedMask: DemandedMaskLHS, Known&: LHSKnown, Q,
1113 Depth: Depth + 1) ||
1114 SimplifyDemandedBits(I, OpNo: 1, DemandedMask: DemandedMaskRHS, Known&: RHSKnown, Q,
1115 Depth: Depth + 1)) {
1116 // Range attribute or metadata may no longer hold.
1117 I->dropPoisonGeneratingAnnotations();
1118 return I;
1119 }
1120 } else { // fshl is a rotate
1121 // Avoid converting rotate into funnel shift.
1122 // Only simplify if one operand is constant.
1123 LHSKnown = computeKnownBits(V: I->getOperand(i: 0), CtxI: I, Depth: Depth + 1);
1124 if (DemandedMaskLHS.isSubsetOf(RHS: LHSKnown.Zero | LHSKnown.One) &&
1125 !match(V: I->getOperand(i: 0), P: m_SpecificInt(V: LHSKnown.One))) {
1126 replaceOperand(I&: *I, OpNum: 0, V: Constant::getIntegerValue(Ty: VTy, V: LHSKnown.One));
1127 // Range attribute or metadata may no longer hold.
1128 I->dropPoisonGeneratingAnnotations();
1129 return I;
1130 }
1131
1132 RHSKnown = computeKnownBits(V: I->getOperand(i: 1), CtxI: I, Depth: Depth + 1);
1133 if (DemandedMaskRHS.isSubsetOf(RHS: RHSKnown.Zero | RHSKnown.One) &&
1134 !match(V: I->getOperand(i: 1), P: m_SpecificInt(V: RHSKnown.One))) {
1135 replaceOperand(I&: *I, OpNum: 1, V: Constant::getIntegerValue(Ty: VTy, V: RHSKnown.One));
1136 // Range attribute or metadata may no longer hold.
1137 I->dropPoisonGeneratingAnnotations();
1138 return I;
1139 }
1140 }
1141
1142 LHSKnown <<= ShiftAmt;
1143 RHSKnown >>= BitWidth - ShiftAmt;
1144 Known = LHSKnown.unionWith(RHS: RHSKnown);
1145 KnownBitsComputed = true;
1146 break;
1147 }
1148 case Intrinsic::umax: {
1149 // UMax(A, C) == A if ...
1150 // The lowest non-zero bit of DemandMask is higher than the highest
1151 // non-zero bit of C.
1152 const APInt *C;
1153 unsigned CTZ = DemandedMask.countr_zero();
1154 if (match(V: II->getArgOperand(i: 1), P: m_APInt(Res&: C)) &&
1155 CTZ >= C->getActiveBits())
1156 return II->getArgOperand(i: 0);
1157 break;
1158 }
1159 case Intrinsic::umin: {
1160 // UMin(A, C) == A if ...
1161 // The lowest non-zero bit of DemandMask is higher than the highest
1162 // non-one bit of C.
1163 // This comes from using DeMorgans on the above umax example.
1164 const APInt *C;
1165 unsigned CTZ = DemandedMask.countr_zero();
1166 if (match(V: II->getArgOperand(i: 1), P: m_APInt(Res&: C)) &&
1167 CTZ >= C->getBitWidth() - C->countl_one())
1168 return II->getArgOperand(i: 0);
1169 break;
1170 }
1171 default: {
1172 // Handle target specific intrinsics
1173 std::optional<Value *> V = targetSimplifyDemandedUseBitsIntrinsic(
1174 II&: *II, DemandedMask, Known, KnownBitsComputed);
1175 if (V)
1176 return *V;
1177 break;
1178 }
1179 }
1180 }
1181
1182 if (!KnownBitsComputed)
1183 llvm::computeKnownBits(V: I, Known, Q, Depth);
1184 break;
1185 }
1186 }
1187
1188 if (I->getType()->isPointerTy()) {
1189 Align Alignment = I->getPointerAlignment(DL);
1190 Known.Zero.setLowBits(Log2(A: Alignment));
1191 }
1192
1193 // If the client is only demanding bits that we know, return the known
1194 // constant. We can't directly simplify pointers as a constant because of
1195 // pointer provenance.
1196 // TODO: We could return `(inttoptr const)` for pointers.
1197 if (!I->getType()->isPointerTy() &&
1198 DemandedMask.isSubsetOf(RHS: Known.Zero | Known.One))
1199 return Constant::getIntegerValue(Ty: VTy, V: Known.One);
1200
1201 if (CLOpts.verify_known_bits) {
1202 KnownBits ReferenceKnown = llvm::computeKnownBits(V: I, Q, Depth);
1203 if (Known != ReferenceKnown) {
1204 errs() << "Mismatched known bits for " << *I << " in "
1205 << I->getFunction()->getName() << "\n";
1206 errs() << "computeKnownBits(): " << ReferenceKnown << "\n";
1207 errs() << "SimplifyDemandedBits(): " << Known << "\n";
1208 std::abort();
1209 }
1210 }
1211
1212 return nullptr;
1213}
1214
1215/// Helper routine of SimplifyDemandedUseBits. It computes Known
1216/// bits. It also tries to handle simplifications that can be done based on
1217/// DemandedMask, but without modifying the Instruction.
1218Value *InstCombinerImpl::SimplifyMultipleUseDemandedBits(
1219 Instruction *I, const APInt &DemandedMask, KnownBits &Known,
1220 const SimplifyQuery &Q, unsigned Depth) {
1221 unsigned BitWidth = DemandedMask.getBitWidth();
1222 Type *ITy = I->getType();
1223
1224 KnownBits LHSKnown(BitWidth);
1225 KnownBits RHSKnown(BitWidth);
1226
1227 // Despite the fact that we can't simplify this instruction in all User's
1228 // context, we can at least compute the known bits, and we can
1229 // do simplifications that apply to *just* the one user if we know that
1230 // this instruction has a simpler value in that context.
1231 switch (I->getOpcode()) {
1232 case Instruction::And: {
1233 llvm::computeKnownBits(V: I->getOperand(i: 1), Known&: RHSKnown, Q, Depth: Depth + 1);
1234 llvm::computeKnownBits(V: I->getOperand(i: 0), Known&: LHSKnown, Q, Depth: Depth + 1);
1235 Known = analyzeKnownBitsFromAndXorOr(I: cast<Operator>(Val: I), KnownLHS: LHSKnown, KnownRHS: RHSKnown,
1236 SQ: Q, Depth);
1237 computeKnownBitsFromContext(V: I, Known, Q, Depth);
1238
1239 // If the client is only demanding bits that we know, return the known
1240 // constant.
1241 if (DemandedMask.isSubsetOf(RHS: Known.Zero | Known.One))
1242 return Constant::getIntegerValue(Ty: ITy, V: Known.One);
1243
1244 // If all of the demanded bits are known 1 on one side, return the other.
1245 // These bits cannot contribute to the result of the 'and' in this context.
1246 if (DemandedMask.isSubsetOf(RHS: LHSKnown.Zero | RHSKnown.One))
1247 return I->getOperand(i: 0);
1248 if (DemandedMask.isSubsetOf(RHS: RHSKnown.Zero | LHSKnown.One))
1249 return I->getOperand(i: 1);
1250
1251 break;
1252 }
1253 case Instruction::Or: {
1254 llvm::computeKnownBits(V: I->getOperand(i: 1), Known&: RHSKnown, Q, Depth: Depth + 1);
1255 llvm::computeKnownBits(V: I->getOperand(i: 0), Known&: LHSKnown, Q, Depth: Depth + 1);
1256 Known = analyzeKnownBitsFromAndXorOr(I: cast<Operator>(Val: I), KnownLHS: LHSKnown, KnownRHS: RHSKnown,
1257 SQ: Q, Depth);
1258 computeKnownBitsFromContext(V: I, Known, Q, Depth);
1259
1260 // If the client is only demanding bits that we know, return the known
1261 // constant.
1262 if (DemandedMask.isSubsetOf(RHS: Known.Zero | Known.One))
1263 return Constant::getIntegerValue(Ty: ITy, V: Known.One);
1264
1265 // We can simplify (X|Y) -> X or Y in the user's context if we know that
1266 // only bits from X or Y are demanded.
1267 // If all of the demanded bits are known zero on one side, return the other.
1268 // These bits cannot contribute to the result of the 'or' in this context.
1269 if (DemandedMask.isSubsetOf(RHS: LHSKnown.One | RHSKnown.Zero))
1270 return I->getOperand(i: 0);
1271 if (DemandedMask.isSubsetOf(RHS: RHSKnown.One | LHSKnown.Zero))
1272 return I->getOperand(i: 1);
1273
1274 break;
1275 }
1276 case Instruction::Xor: {
1277 llvm::computeKnownBits(V: I->getOperand(i: 1), Known&: RHSKnown, Q, Depth: Depth + 1);
1278 llvm::computeKnownBits(V: I->getOperand(i: 0), Known&: LHSKnown, Q, Depth: Depth + 1);
1279 Known = analyzeKnownBitsFromAndXorOr(I: cast<Operator>(Val: I), KnownLHS: LHSKnown, KnownRHS: RHSKnown,
1280 SQ: Q, Depth);
1281 computeKnownBitsFromContext(V: I, Known, Q, Depth);
1282
1283 // If the client is only demanding bits that we know, return the known
1284 // constant.
1285 if (DemandedMask.isSubsetOf(RHS: Known.Zero | Known.One))
1286 return Constant::getIntegerValue(Ty: ITy, V: Known.One);
1287
1288 // We can simplify (X^Y) -> X or Y in the user's context if we know that
1289 // only bits from X or Y are demanded.
1290 // If all of the demanded bits are known zero on one side, return the other.
1291 if (DemandedMask.isSubsetOf(RHS: RHSKnown.Zero))
1292 return I->getOperand(i: 0);
1293 if (DemandedMask.isSubsetOf(RHS: LHSKnown.Zero))
1294 return I->getOperand(i: 1);
1295
1296 break;
1297 }
1298 case Instruction::Add: {
1299 unsigned NLZ = DemandedMask.countl_zero();
1300 APInt DemandedFromOps = APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: BitWidth - NLZ);
1301
1302 // If an operand adds zeros to every bit below the highest demanded bit,
1303 // that operand doesn't change the result. Return the other side.
1304 llvm::computeKnownBits(V: I->getOperand(i: 1), Known&: RHSKnown, Q, Depth: Depth + 1);
1305 if (DemandedFromOps.isSubsetOf(RHS: RHSKnown.Zero))
1306 return I->getOperand(i: 0);
1307
1308 llvm::computeKnownBits(V: I->getOperand(i: 0), Known&: LHSKnown, Q, Depth: Depth + 1);
1309 if (DemandedFromOps.isSubsetOf(RHS: LHSKnown.Zero))
1310 return I->getOperand(i: 1);
1311
1312 bool NSW = cast<OverflowingBinaryOperator>(Val: I)->hasNoSignedWrap();
1313 bool NUW = cast<OverflowingBinaryOperator>(Val: I)->hasNoUnsignedWrap();
1314 Known = KnownBits::add(LHS: LHSKnown, RHS: RHSKnown, NSW, NUW);
1315 computeKnownBitsFromContext(V: I, Known, Q, Depth);
1316 break;
1317 }
1318 case Instruction::Sub: {
1319 unsigned NLZ = DemandedMask.countl_zero();
1320 APInt DemandedFromOps = APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: BitWidth - NLZ);
1321
1322 // If an operand subtracts zeros from every bit below the highest demanded
1323 // bit, that operand doesn't change the result. Return the other side.
1324 llvm::computeKnownBits(V: I->getOperand(i: 1), Known&: RHSKnown, Q, Depth: Depth + 1);
1325 if (DemandedFromOps.isSubsetOf(RHS: RHSKnown.Zero))
1326 return I->getOperand(i: 0);
1327
1328 bool NSW = cast<OverflowingBinaryOperator>(Val: I)->hasNoSignedWrap();
1329 bool NUW = cast<OverflowingBinaryOperator>(Val: I)->hasNoUnsignedWrap();
1330 llvm::computeKnownBits(V: I->getOperand(i: 0), Known&: LHSKnown, Q, Depth: Depth + 1);
1331 Known = KnownBits::sub(LHS: LHSKnown, RHS: RHSKnown, NSW, NUW);
1332 computeKnownBitsFromContext(V: I, Known, Q, Depth);
1333 break;
1334 }
1335 case Instruction::AShr: {
1336 // Compute the Known bits to simplify things downstream.
1337 llvm::computeKnownBits(V: I, Known, Q, Depth);
1338
1339 // If this user is only demanding bits that we know, return the known
1340 // constant.
1341 if (DemandedMask.isSubsetOf(RHS: Known.Zero | Known.One))
1342 return Constant::getIntegerValue(Ty: ITy, V: Known.One);
1343
1344 // If the right shift operand 0 is a result of a left shift by the same
1345 // amount, this is probably a zero/sign extension, which may be unnecessary,
1346 // if we do not demand any of the new sign bits. So, return the original
1347 // operand instead.
1348 const APInt *ShiftRC;
1349 const APInt *ShiftLC;
1350 Value *X;
1351 unsigned BitWidth = DemandedMask.getBitWidth();
1352 if (match(V: I,
1353 P: m_AShr(L: m_Shl(L: m_Value(V&: X), R: m_APInt(Res&: ShiftLC)), R: m_APInt(Res&: ShiftRC))) &&
1354 ShiftLC == ShiftRC && ShiftLC->ult(RHS: BitWidth) &&
1355 DemandedMask.isSubsetOf(RHS: APInt::getLowBitsSet(
1356 numBits: BitWidth, loBitsSet: BitWidth - ShiftRC->getZExtValue()))) {
1357 return X;
1358 }
1359
1360 break;
1361 }
1362 default:
1363 // Compute the Known bits to simplify things downstream.
1364 llvm::computeKnownBits(V: I, Known, Q, Depth);
1365
1366 // If this user is only demanding bits that we know, return the known
1367 // constant.
1368 if (DemandedMask.isSubsetOf(RHS: Known.Zero|Known.One))
1369 return Constant::getIntegerValue(Ty: ITy, V: Known.One);
1370
1371 break;
1372 }
1373
1374 return nullptr;
1375}
1376
1377/// Helper routine of SimplifyDemandedUseBits. It tries to simplify
1378/// "E1 = (X lsr C1) << C2", where the C1 and C2 are constant, into
1379/// "E2 = X << (C2 - C1)" or "E2 = X >> (C1 - C2)", depending on the sign
1380/// of "C2-C1".
1381///
1382/// Suppose E1 and E2 are generally different in bits S={bm, bm+1,
1383/// ..., bn}, without considering the specific value X is holding.
1384/// This transformation is legal iff one of following conditions is hold:
1385/// 1) All the bit in S are 0, in this case E1 == E2.
1386/// 2) We don't care those bits in S, per the input DemandedMask.
1387/// 3) Combination of 1) and 2). Some bits in S are 0, and we don't care the
1388/// rest bits.
1389///
1390/// Currently we only test condition 2).
1391///
1392/// As with SimplifyDemandedUseBits, it returns NULL if the simplification was
1393/// not successful.
1394Value *InstCombinerImpl::simplifyShrShlDemandedBits(
1395 Instruction *Shr, const APInt &ShrOp1, Instruction *Shl,
1396 const APInt &ShlOp1, const APInt &DemandedMask, KnownBits &Known) {
1397 if (!ShlOp1 || !ShrOp1)
1398 return nullptr; // No-op.
1399
1400 Value *VarX = Shr->getOperand(i: 0);
1401 Type *Ty = VarX->getType();
1402 unsigned BitWidth = Ty->getScalarSizeInBits();
1403 if (ShlOp1.uge(RHS: BitWidth) || ShrOp1.uge(RHS: BitWidth))
1404 return nullptr; // Undef.
1405
1406 unsigned ShlAmt = ShlOp1.getZExtValue();
1407 unsigned ShrAmt = ShrOp1.getZExtValue();
1408
1409 Known.One.clearAllBits();
1410 Known.Zero.setLowBits(ShlAmt - 1);
1411 Known.Zero &= DemandedMask;
1412
1413 APInt BitMask1(APInt::getAllOnes(numBits: BitWidth));
1414 APInt BitMask2(APInt::getAllOnes(numBits: BitWidth));
1415
1416 bool isLshr = (Shr->getOpcode() == Instruction::LShr);
1417 BitMask1 = isLshr ? (BitMask1.lshr(shiftAmt: ShrAmt) << ShlAmt) :
1418 (BitMask1.ashr(ShiftAmt: ShrAmt) << ShlAmt);
1419
1420 if (ShrAmt <= ShlAmt) {
1421 BitMask2 <<= (ShlAmt - ShrAmt);
1422 } else {
1423 BitMask2 = isLshr ? BitMask2.lshr(shiftAmt: ShrAmt - ShlAmt):
1424 BitMask2.ashr(ShiftAmt: ShrAmt - ShlAmt);
1425 }
1426
1427 // Check if condition-2 (see the comment to this function) is satified.
1428 if ((BitMask1 & DemandedMask) == (BitMask2 & DemandedMask)) {
1429 if (ShrAmt == ShlAmt)
1430 return VarX;
1431
1432 if (!Shr->hasOneUse())
1433 return nullptr;
1434
1435 BinaryOperator *New;
1436 if (ShrAmt < ShlAmt) {
1437 Constant *Amt = ConstantInt::get(Ty: VarX->getType(), V: ShlAmt - ShrAmt);
1438 New = BinaryOperator::CreateShl(V1: VarX, V2: Amt);
1439 BinaryOperator *Orig = cast<BinaryOperator>(Val: Shl);
1440 New->setHasNoSignedWrap(Orig->hasNoSignedWrap());
1441 New->setHasNoUnsignedWrap(Orig->hasNoUnsignedWrap());
1442 } else {
1443 Constant *Amt = ConstantInt::get(Ty: VarX->getType(), V: ShrAmt - ShlAmt);
1444 New = isLshr ? BinaryOperator::CreateLShr(V1: VarX, V2: Amt) :
1445 BinaryOperator::CreateAShr(V1: VarX, V2: Amt);
1446 if (cast<BinaryOperator>(Val: Shr)->isExact())
1447 New->setIsExact(true);
1448 }
1449
1450 return InsertNewInstWith(New, Old: Shl->getIterator());
1451 }
1452
1453 return nullptr;
1454}
1455
1456/// Return true if the top-level all-lanes demanded-elements query can be
1457/// skipped for an intermediate insertelement chain node. This is limited to a
1458/// bounded one-use chain with distinct in-range constant indices, where SDVE
1459/// cannot remove a dead insert before hitting its depth limit.
1460static bool canSkipDemandedEltsInInsertChain(InsertElementInst &IE,
1461 unsigned VWidth,
1462 unsigned DepthLimit) {
1463 // Only skip chain nodes that feed another insertelement; the final chain root
1464 // still runs the full query.
1465 if (!IE.hasOneUse())
1466 return false;
1467 auto *UserIE = dyn_cast<InsertElementInst>(Val: IE.user_back());
1468 if (!UserIE || UserIE->getOperand(i_nocapture: 0) != &IE)
1469 return false;
1470
1471 SmallBitVector SeenIndices(VWidth);
1472 auto HasNewIndexInRange = [&](InsertElementInst &Insert) {
1473 auto *Idx = dyn_cast<ConstantInt>(Val: Insert.getOperand(i_nocapture: 2));
1474 // Let the normal SDVE path handle variable or out-of-range indices. The
1475 // latter may simplify the chain and must not be passed to getZExtValue().
1476 if (!Idx || Idx->getValue().uge(RHS: VWidth))
1477 return false;
1478
1479 unsigned Index = Idx->getZExtValue();
1480 if (SeenIndices.test(Idx: Index))
1481 return false;
1482
1483 SeenIndices.set(Index);
1484 return true;
1485 };
1486
1487 auto *Cur = &IE;
1488 for (unsigned I = 0; I != DepthLimit; ++I) {
1489 // This loop scans the same base-chain window that the SDVE query would
1490 // inspect before hitting its depth limit. With distinct insert indices in
1491 // that window, the all-lanes query cannot remove a dead insert; with
1492 // VWidth > DepthLimit, it also cannot narrow demand to a single lane.
1493 if (!HasNewIndexInRange(*Cur))
1494 return false;
1495
1496 Value *Base = Cur->getOperand(i_nocapture: 0);
1497 if (match(V: Base, P: m_Poison()))
1498 return true;
1499
1500 Cur = dyn_cast<InsertElementInst>(Val: Base);
1501 if (!Cur || !Cur->hasOneUse())
1502 return false;
1503 }
1504
1505 return true;
1506}
1507
1508/// The specified value produces a vector with any number of elements.
1509/// This method analyzes which elements of the operand are poison and
1510/// returns that information in PoisonElts.
1511///
1512/// DemandedElts contains the set of elements that are actually used by the
1513/// caller, and by default (AllowMultipleUsers equals false) the value is
1514/// simplified only if it has a single caller. If AllowMultipleUsers is set
1515/// to true, DemandedElts refers to the union of sets of elements that are
1516/// used by all callers.
1517///
1518/// If the information about demanded elements can be used to simplify the
1519/// operation, the operation is simplified, then the resultant value is
1520/// returned. This returns null if no change was made.
1521Value *InstCombinerImpl::SimplifyDemandedVectorElts(Value *V,
1522 APInt DemandedElts,
1523 APInt &PoisonElts,
1524 unsigned Depth,
1525 bool AllowMultipleUsers) {
1526 // Cannot analyze scalable type. The number of vector elements is not a
1527 // compile-time constant.
1528 if (isa<ScalableVectorType>(Val: V->getType()))
1529 return nullptr;
1530
1531 unsigned VWidth = cast<FixedVectorType>(Val: V->getType())->getNumElements();
1532 APInt EltMask(APInt::getAllOnes(numBits: VWidth));
1533 assert((DemandedElts & ~EltMask) == 0 && "Invalid DemandedElts!");
1534
1535 if (match(V, P: m_Poison())) {
1536 // If the entire vector is poison, just return this info.
1537 PoisonElts = EltMask;
1538 return nullptr;
1539 }
1540
1541 if (DemandedElts.isZero()) { // If nothing is demanded, provide poison.
1542 PoisonElts = EltMask;
1543 return PoisonValue::get(T: V->getType());
1544 }
1545
1546 PoisonElts = 0;
1547
1548 if (auto *C = dyn_cast<Constant>(Val: V)) {
1549 // Check if this is identity. If so, return 0 since we are not simplifying
1550 // anything.
1551 if (DemandedElts.isAllOnes())
1552 return nullptr;
1553
1554 Type *EltTy = cast<VectorType>(Val: V->getType())->getElementType();
1555 Constant *Poison = PoisonValue::get(T: EltTy);
1556 SmallVector<Constant*, 16> Elts;
1557 for (unsigned i = 0; i != VWidth; ++i) {
1558 if (!DemandedElts[i]) { // If not demanded, set to poison.
1559 Elts.push_back(Elt: Poison);
1560 PoisonElts.setBit(i);
1561 continue;
1562 }
1563
1564 Constant *Elt = C->getAggregateElement(Elt: i);
1565 if (!Elt) return nullptr;
1566
1567 Elts.push_back(Elt);
1568 if (isa<PoisonValue>(Val: Elt)) // Already poison.
1569 PoisonElts.setBit(i);
1570 }
1571
1572 // If we changed the constant, return it.
1573 Constant *NewCV = ConstantVector::get(V: Elts);
1574 return NewCV != C ? NewCV : nullptr;
1575 }
1576
1577 // Limit search depth.
1578 if (Depth == CLOpts.simplify_vector_elts_depth)
1579 return nullptr;
1580
1581 if (!AllowMultipleUsers) {
1582 // If multiple users are using the root value, proceed with
1583 // simplification conservatively assuming that all elements
1584 // are needed.
1585 if (!V->hasOneUse()) {
1586 // Quit if we find multiple users of a non-root value though.
1587 // They'll be handled when it's their turn to be visited by
1588 // the main instcombine process.
1589 if (Depth != 0)
1590 // TODO: Just compute the PoisonElts information recursively.
1591 return nullptr;
1592
1593 // Conservatively assume that all elements are needed.
1594 DemandedElts = EltMask;
1595 }
1596 }
1597
1598 Instruction *I = dyn_cast<Instruction>(Val: V);
1599 if (!I) return nullptr; // Only analyze instructions.
1600
1601 bool MadeChange = false;
1602 auto simplifyAndSetOp = [&](Instruction *Inst, unsigned OpNum,
1603 APInt Demanded, APInt &Undef) {
1604 auto *II = dyn_cast<IntrinsicInst>(Val: Inst);
1605 Value *Op = II ? II->getArgOperand(i: OpNum) : Inst->getOperand(i: OpNum);
1606 if (Value *V = SimplifyDemandedVectorElts(V: Op, DemandedElts: Demanded, PoisonElts&: Undef, Depth: Depth + 1)) {
1607 replaceOperand(I&: *Inst, OpNum, V);
1608 MadeChange = true;
1609 }
1610 };
1611
1612 APInt PoisonElts2(VWidth, 0);
1613 APInt PoisonElts3(VWidth, 0);
1614 switch (I->getOpcode()) {
1615 default: break;
1616
1617 case Instruction::GetElementPtr: {
1618 // The LangRef requires that struct geps have all constant indices. As
1619 // such, we can't convert any operand to partial undef.
1620 auto mayIndexStructType = [](GetElementPtrInst &GEP) {
1621 for (auto I = gep_type_begin(GEP), E = gep_type_end(GEP);
1622 I != E; I++)
1623 if (I.isStruct())
1624 return true;
1625 return false;
1626 };
1627 if (mayIndexStructType(cast<GetElementPtrInst>(Val&: *I)))
1628 break;
1629
1630 // Conservatively track the demanded elements back through any vector
1631 // operands we may have. We know there must be at least one, or we
1632 // wouldn't have a vector result to get here. Note that we intentionally
1633 // merge the undef bits here since gepping with either an poison base or
1634 // index results in poison.
1635 for (unsigned i = 0; i < I->getNumOperands(); i++) {
1636 if (i == 0 ? match(V: I->getOperand(i), P: m_Undef())
1637 : match(V: I->getOperand(i), P: m_Poison())) {
1638 // If the entire vector is undefined, just return this info.
1639 PoisonElts = EltMask;
1640 return nullptr;
1641 }
1642 if (I->getOperand(i)->getType()->isVectorTy()) {
1643 APInt PoisonEltsOp(VWidth, 0);
1644 simplifyAndSetOp(I, i, DemandedElts, PoisonEltsOp);
1645 // gep(x, undef) is not undef, so skip considering idx ops here
1646 // Note that we could propagate poison, but we can't distinguish between
1647 // undef & poison bits ATM
1648 if (i == 0)
1649 PoisonElts |= PoisonEltsOp;
1650 }
1651 }
1652
1653 break;
1654 }
1655 case Instruction::InsertElement: {
1656 unsigned DepthLimit = CLOpts.simplify_vector_elts_depth;
1657 auto *IE = cast<InsertElementInst>(Val: I);
1658 // Skip only when SDVE cannot simplify this insert chain before the limit.
1659 if (Depth == 0 && DemandedElts.isAllOnes() && VWidth > DepthLimit &&
1660 canSkipDemandedEltsInInsertChain(IE&: *IE, VWidth, DepthLimit))
1661 return nullptr;
1662
1663 // If this is a variable index, we don't know which element it overwrites.
1664 // demand exactly the same input as we produce.
1665 ConstantInt *Idx = dyn_cast<ConstantInt>(Val: I->getOperand(i: 2));
1666 if (!Idx) {
1667 // Note that we can't propagate undef elt info, because we don't know
1668 // which elt is getting updated.
1669 simplifyAndSetOp(I, 0, DemandedElts, PoisonElts2);
1670 break;
1671 }
1672
1673 // The element inserted overwrites whatever was there, so the input demanded
1674 // set is simpler than the output set.
1675 unsigned IdxNo = Idx->getZExtValue();
1676 APInt PreInsertDemandedElts = DemandedElts;
1677 if (IdxNo < VWidth)
1678 PreInsertDemandedElts.clearBit(BitPosition: IdxNo);
1679
1680 // If we only demand the element that is being inserted and that element
1681 // was extracted from the same index in another vector with the same type,
1682 // replace this insert with that other vector.
1683 // Note: This is attempted before the call to simplifyAndSetOp because that
1684 // may change PoisonElts to a value that does not match with Vec.
1685 Value *Vec;
1686 if (PreInsertDemandedElts == 0 &&
1687 match(V: I->getOperand(i: 1),
1688 P: m_ExtractElt(Val: m_Value(V&: Vec), Idx: m_SpecificInt(V: IdxNo))) &&
1689 Vec->getType() == I->getType()) {
1690 return Vec;
1691 }
1692
1693 simplifyAndSetOp(I, 0, PreInsertDemandedElts, PoisonElts);
1694
1695 // If this is inserting an element that isn't demanded, remove this
1696 // insertelement.
1697 if (IdxNo >= VWidth || !DemandedElts[IdxNo]) {
1698 Worklist.push(I);
1699 return I->getOperand(i: 0);
1700 }
1701
1702 // The inserted element is defined.
1703 PoisonElts.clearBit(BitPosition: IdxNo);
1704 break;
1705 }
1706 case Instruction::ShuffleVector: {
1707 auto *Shuffle = cast<ShuffleVectorInst>(Val: I);
1708 assert(Shuffle->getOperand(0)->getType() ==
1709 Shuffle->getOperand(1)->getType() &&
1710 "Expected shuffle operands to have same type");
1711 unsigned OpWidth = cast<FixedVectorType>(Val: Shuffle->getOperand(i_nocapture: 0)->getType())
1712 ->getNumElements();
1713 // Handle trivial case of a splat. Only check the first element of LHS
1714 // operand.
1715 if (all_of(Range: Shuffle->getShuffleMask(), P: equal_to(Arg: 0)) &&
1716 DemandedElts.isAllOnes()) {
1717 if (!isa<PoisonValue>(Val: I->getOperand(i: 1))) {
1718 I->setOperand(i: 1, Val: PoisonValue::get(T: I->getOperand(i: 1)->getType()));
1719 MadeChange = true;
1720 }
1721 APInt LeftDemanded(OpWidth, 1);
1722 APInt LHSPoisonElts(OpWidth, 0);
1723 simplifyAndSetOp(I, 0, LeftDemanded, LHSPoisonElts);
1724 if (LHSPoisonElts[0])
1725 PoisonElts = EltMask;
1726 else
1727 PoisonElts.clearAllBits();
1728 break;
1729 }
1730
1731 APInt LeftDemanded(OpWidth, 0), RightDemanded(OpWidth, 0);
1732 for (unsigned i = 0; i < VWidth; i++) {
1733 if (DemandedElts[i]) {
1734 unsigned MaskVal = Shuffle->getMaskValue(Elt: i);
1735 if (MaskVal != -1u) {
1736 assert(MaskVal < OpWidth * 2 &&
1737 "shufflevector mask index out of range!");
1738 if (MaskVal < OpWidth)
1739 LeftDemanded.setBit(MaskVal);
1740 else
1741 RightDemanded.setBit(MaskVal - OpWidth);
1742 }
1743 }
1744 }
1745
1746 APInt LHSPoisonElts(OpWidth, 0);
1747 simplifyAndSetOp(I, 0, LeftDemanded, LHSPoisonElts);
1748
1749 APInt RHSPoisonElts(OpWidth, 0);
1750 simplifyAndSetOp(I, 1, RightDemanded, RHSPoisonElts);
1751
1752 // If this shuffle does not change the vector length and the elements
1753 // demanded by this shuffle are an identity mask, then this shuffle is
1754 // unnecessary.
1755 //
1756 // We are assuming canonical form for the mask, so the source vector is
1757 // operand 0 and operand 1 is not used.
1758 //
1759 // Note that if an element is demanded and this shuffle mask is undefined
1760 // for that element, then the shuffle is not considered an identity
1761 // operation. The shuffle prevents poison from the operand vector from
1762 // leaking to the result by replacing poison with an undefined value.
1763 if (VWidth == OpWidth) {
1764 bool IsIdentityShuffle = true;
1765 for (unsigned i = 0; i < VWidth; i++) {
1766 unsigned MaskVal = Shuffle->getMaskValue(Elt: i);
1767 if (DemandedElts[i] && i != MaskVal) {
1768 IsIdentityShuffle = false;
1769 break;
1770 }
1771 }
1772 if (IsIdentityShuffle)
1773 return Shuffle->getOperand(i_nocapture: 0);
1774 }
1775
1776 bool NewPoisonElts = false;
1777 unsigned LHSIdx = -1u, LHSValIdx = -1u;
1778 unsigned RHSIdx = -1u, RHSValIdx = -1u;
1779 bool LHSUniform = true;
1780 bool RHSUniform = true;
1781 for (unsigned i = 0; i < VWidth; i++) {
1782 unsigned MaskVal = Shuffle->getMaskValue(Elt: i);
1783 if (MaskVal == -1u) {
1784 PoisonElts.setBit(i);
1785 } else if (!DemandedElts[i]) {
1786 NewPoisonElts = true;
1787 PoisonElts.setBit(i);
1788 } else if (MaskVal < OpWidth) {
1789 if (LHSPoisonElts[MaskVal]) {
1790 NewPoisonElts = true;
1791 PoisonElts.setBit(i);
1792 } else {
1793 LHSIdx = LHSIdx == -1u ? i : OpWidth;
1794 LHSValIdx = LHSValIdx == -1u ? MaskVal : OpWidth;
1795 LHSUniform = LHSUniform && (MaskVal == i);
1796 }
1797 } else {
1798 if (RHSPoisonElts[MaskVal - OpWidth]) {
1799 NewPoisonElts = true;
1800 PoisonElts.setBit(i);
1801 } else {
1802 RHSIdx = RHSIdx == -1u ? i : OpWidth;
1803 RHSValIdx = RHSValIdx == -1u ? MaskVal - OpWidth : OpWidth;
1804 RHSUniform = RHSUniform && (MaskVal - OpWidth == i);
1805 }
1806 }
1807 }
1808
1809 // Try to transform shuffle with constant vector and single element from
1810 // this constant vector to single insertelement instruction.
1811 // shufflevector V, C, <v1, v2, .., ci, .., vm> ->
1812 // insertelement V, C[ci], ci-n
1813 if (OpWidth ==
1814 cast<FixedVectorType>(Val: Shuffle->getType())->getNumElements()) {
1815 Value *Op = nullptr;
1816 Constant *Value = nullptr;
1817 unsigned Idx = -1u;
1818
1819 // Find constant vector with the single element in shuffle (LHS or RHS).
1820 if (LHSIdx < OpWidth && RHSUniform) {
1821 if (auto *CV = dyn_cast<ConstantVector>(Val: Shuffle->getOperand(i_nocapture: 0))) {
1822 Op = Shuffle->getOperand(i_nocapture: 1);
1823 Value = CV->getOperand(i_nocapture: LHSValIdx);
1824 Idx = LHSIdx;
1825 }
1826 }
1827 if (RHSIdx < OpWidth && LHSUniform) {
1828 if (auto *CV = dyn_cast<ConstantVector>(Val: Shuffle->getOperand(i_nocapture: 1))) {
1829 Op = Shuffle->getOperand(i_nocapture: 0);
1830 Value = CV->getOperand(i_nocapture: RHSValIdx);
1831 Idx = RHSIdx;
1832 }
1833 }
1834 // Found constant vector with single element - convert to insertelement.
1835 if (Op && Value) {
1836 Instruction *New = InsertElementInst::Create(
1837 Vec: Op, NewElt: Value, Idx: ConstantInt::get(Ty: Type::getInt64Ty(C&: I->getContext()), V: Idx),
1838 NameStr: Shuffle->getName());
1839 InsertNewInstWith(New, Old: Shuffle->getIterator());
1840 return New;
1841 }
1842 }
1843 if (NewPoisonElts) {
1844 // Add additional discovered undefs.
1845 SmallVector<int, 16> Elts;
1846 for (unsigned i = 0; i < VWidth; ++i) {
1847 if (PoisonElts[i])
1848 Elts.push_back(Elt: PoisonMaskElem);
1849 else
1850 Elts.push_back(Elt: Shuffle->getMaskValue(Elt: i));
1851 }
1852 Shuffle->setShuffleMask(Elts);
1853 MadeChange = true;
1854 }
1855 break;
1856 }
1857 case Instruction::Select: {
1858 // If this is a vector select, try to transform the select condition based
1859 // on the current demanded elements.
1860 SelectInst *Sel = cast<SelectInst>(Val: I);
1861 if (Sel->getCondition()->getType()->isVectorTy()) {
1862 // TODO: We are not doing anything with PoisonElts based on this call.
1863 // It is overwritten below based on the other select operands. If an
1864 // element of the select condition is known undef, then we are free to
1865 // choose the output value from either arm of the select. If we know that
1866 // one of those values is undef, then the output can be undef.
1867 simplifyAndSetOp(I, 0, DemandedElts, PoisonElts);
1868 }
1869
1870 // Next, see if we can transform the arms of the select.
1871 APInt DemandedLHS(DemandedElts), DemandedRHS(DemandedElts);
1872 if (auto *CV = dyn_cast<ConstantVector>(Val: Sel->getCondition())) {
1873 for (unsigned i = 0; i < VWidth; i++) {
1874 Constant *CElt = CV->getAggregateElement(Elt: i);
1875
1876 // isNullValue() always returns false when called on a ConstantExpr.
1877 if (CElt->isNullValue())
1878 DemandedLHS.clearBit(BitPosition: i);
1879 else if (CElt->isOneValue())
1880 DemandedRHS.clearBit(BitPosition: i);
1881 }
1882 }
1883
1884 simplifyAndSetOp(I, 1, DemandedLHS, PoisonElts2);
1885 simplifyAndSetOp(I, 2, DemandedRHS, PoisonElts3);
1886
1887 // Output elements are undefined if the element from each arm is undefined.
1888 // TODO: This can be improved. See comment in select condition handling.
1889 PoisonElts = PoisonElts2 & PoisonElts3;
1890 break;
1891 }
1892 case Instruction::BitCast: {
1893 // Vector->vector casts only.
1894 VectorType *VTy = dyn_cast<VectorType>(Val: I->getOperand(i: 0)->getType());
1895 if (!VTy) break;
1896 unsigned InVWidth = cast<FixedVectorType>(Val: VTy)->getNumElements();
1897 APInt InputDemandedElts(InVWidth, 0);
1898 PoisonElts2 = APInt(InVWidth, 0);
1899 unsigned Ratio;
1900
1901 if (VWidth == InVWidth) {
1902 // If we are converting from <4 x i32> -> <4 x f32>, we demand the same
1903 // elements as are demanded of us.
1904 Ratio = 1;
1905 InputDemandedElts = DemandedElts;
1906 } else if ((VWidth % InVWidth) == 0) {
1907 // If the number of elements in the output is a multiple of the number of
1908 // elements in the input then an input element is live if any of the
1909 // corresponding output elements are live.
1910 Ratio = VWidth / InVWidth;
1911 for (unsigned OutIdx = 0; OutIdx != VWidth; ++OutIdx)
1912 if (DemandedElts[OutIdx])
1913 InputDemandedElts.setBit(OutIdx / Ratio);
1914 } else if ((InVWidth % VWidth) == 0) {
1915 // If the number of elements in the input is a multiple of the number of
1916 // elements in the output then an input element is live if the
1917 // corresponding output element is live.
1918 Ratio = InVWidth / VWidth;
1919 for (unsigned InIdx = 0; InIdx != InVWidth; ++InIdx)
1920 if (DemandedElts[InIdx / Ratio])
1921 InputDemandedElts.setBit(InIdx);
1922 } else {
1923 // Unsupported so far.
1924 break;
1925 }
1926
1927 simplifyAndSetOp(I, 0, InputDemandedElts, PoisonElts2);
1928
1929 if (VWidth == InVWidth) {
1930 PoisonElts = PoisonElts2;
1931 } else if ((VWidth % InVWidth) == 0) {
1932 // If the number of elements in the output is a multiple of the number of
1933 // elements in the input then an output element is undef if the
1934 // corresponding input element is undef.
1935 for (unsigned OutIdx = 0; OutIdx != VWidth; ++OutIdx)
1936 if (PoisonElts2[OutIdx / Ratio])
1937 PoisonElts.setBit(OutIdx);
1938 } else if ((InVWidth % VWidth) == 0) {
1939 // If the number of elements in the input is a multiple of the number of
1940 // elements in the output then an output element is undef if all of the
1941 // corresponding input elements are undef.
1942 for (unsigned OutIdx = 0; OutIdx != VWidth; ++OutIdx) {
1943 APInt SubUndef = PoisonElts2.lshr(shiftAmt: OutIdx * Ratio).zextOrTrunc(width: Ratio);
1944 if (SubUndef.popcount() == Ratio)
1945 PoisonElts.setBit(OutIdx);
1946 }
1947 } else {
1948 llvm_unreachable("Unimp");
1949 }
1950 break;
1951 }
1952 case Instruction::FPTrunc:
1953 case Instruction::FPExt:
1954 simplifyAndSetOp(I, 0, DemandedElts, PoisonElts);
1955 break;
1956
1957 case Instruction::Call: {
1958 IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: I);
1959 if (!II) break;
1960 switch (II->getIntrinsicID()) {
1961 case Intrinsic::masked_gather: // fallthrough
1962 case Intrinsic::masked_load: {
1963 // Subtlety: If we load from a pointer, the pointer must be valid
1964 // regardless of whether the element is demanded. Doing otherwise risks
1965 // segfaults which didn't exist in the original program.
1966 APInt DemandedPtrs(APInt::getAllOnes(numBits: VWidth)),
1967 DemandedPassThrough(DemandedElts);
1968 if (auto *CMask = dyn_cast<Constant>(Val: II->getOperand(i_nocapture: 1))) {
1969 for (unsigned i = 0; i < VWidth; i++) {
1970 if (Constant *CElt = CMask->getAggregateElement(Elt: i)) {
1971 if (CElt->isNullValue())
1972 DemandedPtrs.clearBit(BitPosition: i);
1973 else if (CElt->isAllOnesValue())
1974 DemandedPassThrough.clearBit(BitPosition: i);
1975 }
1976 }
1977 }
1978
1979 if (II->getIntrinsicID() == Intrinsic::masked_gather)
1980 simplifyAndSetOp(II, 0, DemandedPtrs, PoisonElts2);
1981 simplifyAndSetOp(II, 2, DemandedPassThrough, PoisonElts3);
1982
1983 // Output elements are undefined if the element from both sources are.
1984 // TODO: can strengthen via mask as well.
1985 PoisonElts = PoisonElts2 & PoisonElts3;
1986 break;
1987 }
1988 default: {
1989 // Handle target specific intrinsics
1990 std::optional<Value *> V = targetSimplifyDemandedVectorEltsIntrinsic(
1991 II&: *II, DemandedElts, UndefElts&: PoisonElts, UndefElts2&: PoisonElts2, UndefElts3&: PoisonElts3,
1992 SimplifyAndSetOp: simplifyAndSetOp);
1993 if (V)
1994 return *V;
1995
1996 // Trivially vectorizable intrinsics operate elementwise: each result lane
1997 // uses only the matching lane of the (vector) operands, so the demand
1998 // passes through unchanged to every vector operand.
1999 Intrinsic::ID IID = II->getIntrinsicID();
2000 if (isTriviallyVectorizable(ID: IID)) {
2001 APInt PoisonEltsAcc(VWidth, 0);
2002 for (Use &Arg : II->args()) {
2003 unsigned OpNo = Arg.getOperandNo();
2004 // Scalar operands do not carry per-lane demand.
2005 if (isVectorIntrinsicWithScalarOpAtArg(ID: IID, ScalarOpdIdx: OpNo, /*TTI=*/nullptr))
2006 continue;
2007 APInt OpPoisonElts(VWidth, 0);
2008 simplifyAndSetOp(II, OpNo, DemandedElts, OpPoisonElts);
2009 PoisonEltsAcc |= OpPoisonElts;
2010 }
2011 // A result lane is poison if any operand lane is poison, but only for
2012 // intrinsics that are known to propagate poison elementwise.
2013 if (intrinsicPropagatesPoison(IID))
2014 PoisonElts = PoisonEltsAcc;
2015 }
2016 break;
2017 }
2018 } // switch on IntrinsicID
2019 break;
2020 } // case Call
2021 } // switch on Opcode
2022
2023 // TODO: We bail completely on integer div/rem and shifts because they have
2024 // UB/poison potential, but that should be refined.
2025 BinaryOperator *BO;
2026 if (match(V: I, P: m_BinOp(I&: BO)) && !BO->isIntDivRem() && !BO->isShift()) {
2027 Value *X = BO->getOperand(i_nocapture: 0);
2028 Value *Y = BO->getOperand(i_nocapture: 1);
2029
2030 // Look for an equivalent binop except that one operand has been shuffled.
2031 // If the demand for this binop only includes elements that are the same as
2032 // the other binop, then we may be able to replace this binop with a use of
2033 // the earlier one.
2034 //
2035 // Example:
2036 // %other_bo = bo (shuf X, {0}), Y
2037 // %this_extracted_bo = extelt (bo X, Y), 0
2038 // -->
2039 // %other_bo = bo (shuf X, {0}), Y
2040 // %this_extracted_bo = extelt %other_bo, 0
2041 //
2042 // TODO: Handle demand of an arbitrary single element or more than one
2043 // element instead of just element 0.
2044 // TODO: Unlike general demanded elements transforms, this should be safe
2045 // for any (div/rem/shift) opcode too.
2046 if (DemandedElts == 1 && !X->hasOneUse() && !Y->hasOneUse() &&
2047 BO->hasOneUse() ) {
2048
2049 auto findShufBO = [&](bool MatchShufAsOp0) -> User * {
2050 // Try to use shuffle-of-operand in place of an operand:
2051 // bo X, Y --> bo (shuf X), Y
2052 // bo X, Y --> bo X, (shuf Y)
2053
2054 Value *OtherOp = MatchShufAsOp0 ? Y : X;
2055 if (!OtherOp->hasUseList())
2056 return nullptr;
2057
2058 BinaryOperator::BinaryOps Opcode = BO->getOpcode();
2059 Value *ShufOp = MatchShufAsOp0 ? X : Y;
2060
2061 for (User *U : OtherOp->users()) {
2062 ArrayRef<int> Mask;
2063 auto Shuf = m_Shuffle(v1: m_Specific(V: ShufOp), v2: m_Value(), mask: m_Mask(Mask));
2064 if (BO->isCommutative()
2065 ? match(V: U, P: m_c_BinOp(Opcode, L: Shuf, R: m_Specific(V: OtherOp)))
2066 : MatchShufAsOp0
2067 ? match(V: U, P: m_BinOp(Opcode, L: Shuf, R: m_Specific(V: OtherOp)))
2068 : match(V: U, P: m_BinOp(Opcode, L: m_Specific(V: OtherOp), R: Shuf)))
2069 if (match(Mask, P: m_ZeroMask()) && Mask[0] != PoisonMaskElem)
2070 if (DT.dominates(Def: U, User: I))
2071 return U;
2072 }
2073 return nullptr;
2074 };
2075
2076 User *ShufBO = findShufBO(/* MatchShufAsOp0 */ true);
2077 if (!ShufBO)
2078 ShufBO = findShufBO(/* MatchShufAsOp0 */ false);
2079 if (ShufBO) {
2080 auto *ShufBOI = cast<Instruction>(Val: ShufBO);
2081 ShufBOI->andIRFlags(V: BO);
2082 Worklist.add(I: ShufBOI);
2083 return ShufBO;
2084 }
2085 }
2086
2087 simplifyAndSetOp(I, 0, DemandedElts, PoisonElts);
2088 simplifyAndSetOp(I, 1, DemandedElts, PoisonElts2);
2089
2090 // Output elements are undefined if both are undefined. Consider things
2091 // like undef & 0. The result is known zero, not undef.
2092 PoisonElts &= PoisonElts2;
2093 }
2094
2095 // If we've proven all of the lanes poison, return a poison value.
2096 // TODO: Intersect w/demanded lanes
2097 if (PoisonElts.isAllOnes())
2098 return PoisonValue::get(T: I->getType());
2099
2100 return MadeChange ? I : nullptr;
2101}
2102
2103/// For floating-point classes that resolve to a single bit pattern, return that
2104/// value.
2105static Constant *getFPClassConstant(Type *Ty, FPClassTest Mask,
2106 bool IsCanonicalizing = false) {
2107 if (Mask == fcNone)
2108 return PoisonValue::get(T: Ty);
2109
2110 if (Mask == fcPosZero)
2111 return Constant::getNullValue(Ty);
2112
2113 // TODO: Support aggregate types that are allowed by FPMathOperator.
2114 if (Ty->isAggregateType())
2115 return nullptr;
2116
2117 // Turn any possible snans into quiet if we can.
2118 if (Mask == fcNan && IsCanonicalizing)
2119 return ConstantFP::getQNaN(Ty);
2120
2121 switch (Mask) {
2122 case fcNegZero:
2123 return ConstantFP::getZero(Ty, Negative: true);
2124 case fcPosInf:
2125 return ConstantFP::getInfinity(Ty);
2126 case fcNegInf:
2127 return ConstantFP::getInfinity(Ty, Negative: true);
2128 case fcQNan:
2129 // Payload bits cannot be dropped for pure signbit operations.
2130 return IsCanonicalizing ? ConstantFP::getQNaN(Ty) : nullptr;
2131 default:
2132 return nullptr;
2133 }
2134}
2135
2136/// Perform multiple-use aware simplfications for fabs(\p Src). Returns a
2137/// replacement value if it's simplified, otherwise nullptr. Updates \p Known
2138/// with the known fpclass if not simplified.
2139static Value *simplifyDemandedFPClassFabs(KnownFPClass &Known, Value *Src,
2140 FPClassTest DemandedMask,
2141 KnownFPClass KnownSrc, bool NSZ) {
2142 if ((DemandedMask & fcNan) == fcNone)
2143 KnownSrc.knownNot(RuleOut: fcNan);
2144 if ((DemandedMask & fcInf) == fcNone)
2145 KnownSrc.knownNot(RuleOut: fcInf);
2146
2147 if (KnownSrc.getSignBit() == false ||
2148 ((DemandedMask & fcNan) == fcNone && KnownSrc.isKnownNever(Mask: fcNegative)))
2149 return Src;
2150
2151 // If the only sign bit difference is due to -0, ignore it with nsz
2152 if (NSZ &&
2153 KnownSrc.isKnownNever(Mask: KnownFPClass::OrderedLessThanZeroMask | fcNan))
2154 return Src;
2155
2156 Known = KnownFPClass::fabs(Src: KnownSrc);
2157 Known.knownNot(RuleOut: ~DemandedMask);
2158 return nullptr;
2159}
2160
2161/// Try to set an inferred no-nans or no-infs in \p FMF. \p ValidResults is a
2162/// mask of known valid results for the operator (already computed from the
2163/// result, and the known operand inputs in \p Known)
2164static FastMathFlags inferFastMathValueFlags(FastMathFlags FMF,
2165 FPClassTest ValidResults,
2166 ArrayRef<KnownFPClass> Known) {
2167 if (!FMF.noNaNs() && (ValidResults & fcNan) == fcNone) {
2168 if (all_of(Range&: Known, P: [](const KnownFPClass KnownSrc) {
2169 return KnownSrc.isKnownNeverNaN();
2170 }))
2171 FMF.setNoNaNs();
2172 }
2173
2174 if (!FMF.noInfs() && (ValidResults & fcInf) == fcNone) {
2175 if (all_of(Range&: Known, P: [](const KnownFPClass KnownSrc) {
2176 return KnownSrc.isKnownNeverInfinity();
2177 }))
2178 FMF.setNoInfs();
2179 }
2180
2181 return FMF;
2182}
2183
2184static FPClassTest adjustDemandedMaskFromFlags(FPClassTest DemandedMask,
2185 FastMathFlags FMF) {
2186 if (FMF.noNaNs())
2187 DemandedMask &= ~fcNan;
2188
2189 if (FMF.noInfs())
2190 DemandedMask &= ~fcInf;
2191 return DemandedMask;
2192}
2193
2194/// Apply epilog fixups to a floating-point intrinsic. See if the result can
2195/// fold to a constant, or apply fast math flags.
2196static Value *simplifyDemandedFPClassResult(Instruction *FPOp,
2197 FastMathFlags FMF,
2198 FPClassTest DemandedMask,
2199 KnownFPClass &Known,
2200 ArrayRef<KnownFPClass> KnownSrcs) {
2201 FPClassTest ValidResults = DemandedMask & Known.getKnownFPClasses();
2202 Constant *SingleVal = getFPClassConstant(Ty: FPOp->getType(), Mask: ValidResults,
2203 /*IsCanonicalizing=*/true);
2204 if (SingleVal)
2205 return SingleVal;
2206
2207 FastMathFlags InferredFMF =
2208 inferFastMathValueFlags(FMF, ValidResults, Known: KnownSrcs);
2209 if (InferredFMF != FMF) {
2210 FPOp->dropUBImplyingAttrsAndMetadata();
2211 FPOp->setFastMathFlags(InferredFMF);
2212 return FPOp;
2213 }
2214
2215 return nullptr;
2216}
2217
2218/// Perform multiple-use aware simplfications for fneg(fabs(\p Src)). Returns a
2219/// replacement value if it's simplified, otherwise nullptr. Updates \p Known
2220/// with the known fpclass if not simplified.
2221static Value *simplifyDemandedFPClassFnegFabs(KnownFPClass &Known, Value *Src,
2222 FPClassTest DemandedMask,
2223 KnownFPClass KnownSrc, bool NSZ) {
2224 if ((DemandedMask & fcNan) == fcNone)
2225 KnownSrc.knownNot(RuleOut: fcNan);
2226 if ((DemandedMask & fcInf) == fcNone)
2227 KnownSrc.knownNot(RuleOut: fcInf);
2228
2229 // If the source value is known negative, we can directly fold to it.
2230 if (KnownSrc.getSignBit() == true)
2231 return Src;
2232
2233 // If the only sign bit difference is for 0, ignore it with nsz.
2234 if (NSZ &&
2235 KnownSrc.isKnownNever(Mask: KnownFPClass::OrderedGreaterThanZeroMask | fcNan))
2236 return Src;
2237
2238 Known = KnownFPClass::fneg(Src: KnownFPClass::fabs(Src: KnownSrc));
2239 Known.knownNot(RuleOut: ~DemandedMask);
2240 return nullptr;
2241}
2242
2243static Value *simplifyDemandedFPClassCopysignMag(Value *MagSrc,
2244 FPClassTest DemandedMask,
2245 KnownFPClass KnownSrc,
2246 bool NSZ) {
2247 if (NSZ) {
2248 constexpr FPClassTest NegOrZero = fcNegative | fcPosZero;
2249 constexpr FPClassTest PosOrZero = fcPositive | fcNegZero;
2250
2251 if ((DemandedMask & ~NegOrZero) == fcNone &&
2252 KnownSrc.isKnownAlways(Mask: NegOrZero))
2253 return MagSrc;
2254
2255 if ((DemandedMask & ~PosOrZero) == fcNone &&
2256 KnownSrc.isKnownAlways(Mask: PosOrZero))
2257 return MagSrc;
2258 } else {
2259 if ((DemandedMask & ~fcNegative) == fcNone && KnownSrc.getSignBit() == true)
2260 return MagSrc;
2261
2262 if ((DemandedMask & ~fcPositive) == fcNone &&
2263 KnownSrc.getSignBit() == false)
2264 return MagSrc;
2265 }
2266
2267 return nullptr;
2268}
2269
2270static Value *
2271simplifyDemandedFPClassMinMax(KnownFPClass &Known, Intrinsic::ID IID,
2272 const CallInst *CI, FPClassTest DemandedMask,
2273 KnownFPClass KnownLHS, KnownFPClass KnownRHS,
2274 const Function &F, bool NSZ) {
2275 bool OrderedZeroSign = !NSZ;
2276
2277 KnownFPClass::MinMaxKind OpKind;
2278 switch (IID) {
2279 case Intrinsic::maximum: {
2280 OpKind = KnownFPClass::MinMaxKind::maximum;
2281
2282 // If one operand is known greater than the other, it must be that
2283 // operand unless the other is a nan.
2284 if (cannotOrderStrictlyLess(LHS: KnownLHS.getKnownFPClasses(),
2285 RHS: KnownRHS.getKnownFPClasses(),
2286 OrderedZeroSign) &&
2287 KnownRHS.isKnownNever(Mask: fcNan))
2288 return CI->getArgOperand(i: 0);
2289
2290 if (cannotOrderStrictlyGreater(LHS: KnownLHS.getKnownFPClasses(),
2291 RHS: KnownRHS.getKnownFPClasses(),
2292 OrderedZeroSign) &&
2293 KnownLHS.isKnownNever(Mask: fcNan))
2294 return CI->getArgOperand(i: 1);
2295
2296 break;
2297 }
2298 case Intrinsic::minimum: {
2299 OpKind = KnownFPClass::MinMaxKind::minimum;
2300
2301 // If one operand is known less than the other, it must be that operand
2302 // unless the other is a nan.
2303 if (cannotOrderStrictlyGreater(LHS: KnownLHS.getKnownFPClasses(),
2304 RHS: KnownRHS.getKnownFPClasses(),
2305 OrderedZeroSign) &&
2306 KnownRHS.isKnownNever(Mask: fcNan))
2307 return CI->getArgOperand(i: 0);
2308
2309 if (cannotOrderStrictlyLess(LHS: KnownLHS.getKnownFPClasses(),
2310 RHS: KnownRHS.getKnownFPClasses(),
2311 OrderedZeroSign) &&
2312 KnownLHS.isKnownNever(Mask: fcNan))
2313 return CI->getArgOperand(i: 1);
2314
2315 break;
2316 }
2317 case Intrinsic::maxnum:
2318 case Intrinsic::maximumnum: {
2319 OpKind = IID == Intrinsic::maxnum ? KnownFPClass::MinMaxKind::maxnum
2320 : KnownFPClass::MinMaxKind::maximumnum;
2321
2322 if (cannotOrderStrictlyLess(LHS: KnownLHS.getKnownFPClasses(),
2323 RHS: KnownRHS.getKnownFPClasses(),
2324 OrderedZeroSign) &&
2325 KnownLHS.isKnownNever(Mask: fcNan))
2326 return CI->getArgOperand(i: 0);
2327
2328 if (cannotOrderStrictlyGreater(LHS: KnownLHS.getKnownFPClasses(),
2329 RHS: KnownRHS.getKnownFPClasses(),
2330 OrderedZeroSign) &&
2331 KnownRHS.isKnownNever(Mask: fcNan))
2332 return CI->getArgOperand(i: 1);
2333
2334 break;
2335 }
2336 case Intrinsic::minnum:
2337 case Intrinsic::minimumnum: {
2338 OpKind = IID == Intrinsic::minnum ? KnownFPClass::MinMaxKind::minnum
2339 : KnownFPClass::MinMaxKind::minimumnum;
2340
2341 if (cannotOrderStrictlyGreater(LHS: KnownLHS.getKnownFPClasses(),
2342 RHS: KnownRHS.getKnownFPClasses(),
2343 OrderedZeroSign) &&
2344 KnownLHS.isKnownNever(Mask: fcNan))
2345 return CI->getArgOperand(i: 0);
2346
2347 if (cannotOrderStrictlyLess(LHS: KnownLHS.getKnownFPClasses(),
2348 RHS: KnownRHS.getKnownFPClasses(),
2349 OrderedZeroSign) &&
2350 KnownRHS.isKnownNever(Mask: fcNan))
2351 return CI->getArgOperand(i: 1);
2352
2353 break;
2354 }
2355 default:
2356 llvm_unreachable("not a min/max intrinsic");
2357 }
2358
2359 Type *EltTy = CI->getType()->getScalarType();
2360 DenormalMode Mode = F.getDenormalMode(FPType: EltTy->getFltSemantics());
2361 Known = KnownFPClass::minMaxLike(LHS: KnownLHS, RHS: KnownRHS, Kind: OpKind, Mode);
2362 Known.knownNot(RuleOut: ~DemandedMask);
2363
2364 return getFPClassConstant(Ty: CI->getType(), Mask: Known.getKnownFPClasses(),
2365 /*IsCanonicalizing=*/true);
2366}
2367
2368static Value *
2369simplifyDemandedUseFPClassFPTrunc(InstCombinerImpl &IC, Instruction &I,
2370 FastMathFlags FMF, FPClassTest DemandedMask,
2371 KnownFPClass &Known, const SimplifyQuery &SQ,
2372 unsigned Depth) {
2373 const fltSemantics &DstTy = I.getType()->getScalarType()->getFltSemantics();
2374 const fltSemantics &SrcTy =
2375 I.getOperand(i: 0)->getType()->getScalarType()->getFltSemantics();
2376
2377 const Function *Fn = I.getFunction();
2378 DenormalMode DstMode =
2379 Fn ? Fn->getDenormalMode(FPType: DstTy) : DenormalMode::getDynamic();
2380 DenormalMode SrcMode =
2381 Fn ? Fn->getDenormalMode(FPType: SrcTy) : DenormalMode::getDynamic();
2382
2383 FPClassTest SrcDemandedMask = DemandedMask;
2384 if (DemandedMask & fcNan)
2385 SrcDemandedMask |= fcNan;
2386
2387 // Zero results may have been rounded from subnormal or normal sources.
2388 if (DemandedMask & fcNegZero)
2389 SrcDemandedMask |= fcNegSubnormal | fcNegNormal;
2390 if (DemandedMask & fcPosZero)
2391 SrcDemandedMask |= fcPosSubnormal | fcPosNormal;
2392
2393 // Subnormal results may have been normal in the source type
2394 if (DemandedMask & fcNegSubnormal)
2395 SrcDemandedMask |= fcNegNormal;
2396 if (DemandedMask & fcPosSubnormal)
2397 SrcDemandedMask |= fcPosNormal;
2398
2399 const bool SubnormalNeverProducesNormal =
2400 DstTy.minExponent > SrcTy.minExponent;
2401
2402 // A subnormal input may round to a normal value in the destination type.
2403 // For example: bf16 = fptrunc(f32) and f80 = fptrunc(f128).
2404 if (!SubnormalNeverProducesNormal) {
2405 // TODO: Refine based on the rounding mode.
2406 if (DemandedMask & fcNegNormal)
2407 SrcDemandedMask |= fcNegSubnormal;
2408 if (DemandedMask & fcPosNormal)
2409 SrcDemandedMask |= fcPosSubnormal;
2410 }
2411
2412 if (DemandedMask & fcPosInf)
2413 SrcDemandedMask |= fcPosNormal;
2414 if (DemandedMask & fcNegInf)
2415 SrcDemandedMask |= fcNegNormal;
2416
2417 // TODO: This is a conservative approach for determining the actual input and
2418 // output denormal mode used by fptrunc.
2419 if (DstMode != DenormalMode::getIEEE() ||
2420 SrcMode != DenormalMode::getIEEE()) {
2421 // Input subnormals may flush to zero, and output subnormals may also flush
2422 // to zero. The result can be subnormal (and flushed to zero) even if the
2423 // input is normal.
2424 if ((DemandedMask & fcZero) != fcNone)
2425 SrcDemandedMask |= fcSubnormal | fcNormal;
2426 }
2427
2428 KnownFPClass KnownSrc;
2429 if (IC.SimplifyDemandedFPClass(I: &I, Op: 0, DemandedMask: SrcDemandedMask, Known&: KnownSrc, Q: SQ,
2430 Depth: Depth + 1))
2431 return &I;
2432
2433 Known = KnownFPClass::fptrunc(KnownSrc, DstTy, SrcTy, DstMode, SrcMode);
2434 Known.knownNot(RuleOut: ~DemandedMask);
2435
2436 return simplifyDemandedFPClassResult(FPOp: &I, FMF, DemandedMask, Known,
2437 KnownSrcs: {KnownSrc});
2438}
2439
2440Value *InstCombinerImpl::SimplifyDemandedUseFPClass(Instruction *I,
2441 FPClassTest DemandedMask,
2442 KnownFPClass &Known,
2443 const SimplifyQuery &SQ,
2444 unsigned Depth) {
2445 assert(Depth <= MaxAnalysisRecursionDepth && "Limit Search Depth");
2446 assert(Known == KnownFPClass() && "expected uninitialized state");
2447
2448 Type *VTy = I->getType();
2449
2450 FastMathFlags FMF;
2451 if (auto *FPOp = dyn_cast<FPMathOperator>(Val: I)) {
2452 FMF = FPOp->getFastMathFlags();
2453 DemandedMask = adjustDemandedMaskFromFlags(DemandedMask, FMF);
2454 }
2455
2456 switch (I->getOpcode()) {
2457 case Instruction::FNeg: {
2458 // Special case fneg(fabs(x))
2459
2460 Value *FNegSrc = I->getOperand(i: 0);
2461 Value *FNegFAbsSrc;
2462 if (match(V: FNegSrc, P: m_OneUse(SubPattern: m_FAbs(Op0: m_Value(V&: FNegFAbsSrc))))) {
2463 KnownFPClass KnownSrc;
2464 if (SimplifyDemandedFPClass(I: cast<Instruction>(Val: FNegSrc), Op: 0,
2465 DemandedMask: llvm::unknown_sign(Mask: DemandedMask), Known&: KnownSrc,
2466 Q: SQ, Depth: Depth + 1))
2467 return I;
2468
2469 FastMathFlags FabsFMF = cast<FPMathOperator>(Val: FNegSrc)->getFastMathFlags();
2470 FPClassTest ThisDemandedMask =
2471 adjustDemandedMaskFromFlags(DemandedMask, FMF: FabsFMF);
2472
2473 bool IsNSZ = FMF.noSignedZeros() || FabsFMF.noSignedZeros();
2474 if (Value *Simplified = simplifyDemandedFPClassFnegFabs(
2475 Known, Src: FNegFAbsSrc, DemandedMask: ThisDemandedMask, KnownSrc, NSZ: IsNSZ))
2476 return Simplified;
2477
2478 if ((ThisDemandedMask & fcNan) == fcNone)
2479 KnownSrc.knownNot(RuleOut: fcNan);
2480 if ((ThisDemandedMask & fcInf) == fcNone)
2481 KnownSrc.knownNot(RuleOut: fcInf);
2482
2483 // fneg(fabs(x)) => fneg(x)
2484 if (KnownSrc.getSignBit() == false)
2485 return replaceOperand(I&: *I, OpNum: 0, V: FNegFAbsSrc);
2486
2487 // fneg(fabs(x)) => fneg(x), ignoring -0 if nsz.
2488 if (IsNSZ &&
2489 KnownSrc.isKnownNever(Mask: KnownFPClass::OrderedLessThanZeroMask | fcNan))
2490 return replaceOperand(I&: *I, OpNum: 0, V: FNegFAbsSrc);
2491
2492 break;
2493 }
2494
2495 if (SimplifyDemandedFPClass(I, Op: 0, DemandedMask: llvm::fneg(Mask: DemandedMask), Known, Q: SQ,
2496 Depth: Depth + 1))
2497 return I;
2498 Known.fneg();
2499 Known.knownNot(RuleOut: ~DemandedMask);
2500 break;
2501 }
2502 case Instruction::FAdd:
2503 case Instruction::FSub: {
2504 KnownFPClass KnownLHS, KnownRHS;
2505
2506 // fadd x, x can be handled more aggressively.
2507 if (I->getOperand(i: 0) == I->getOperand(i: 1) &&
2508 I->getOpcode() == Instruction::FAdd &&
2509 isGuaranteedNotToBeUndef(V: I->getOperand(i: 0), AC: SQ.AC, CtxI: SQ.CtxI, DT: SQ.DT,
2510 Depth: Depth + 1)) {
2511 Type *EltTy = VTy->getScalarType();
2512 DenormalMode Mode = F.getDenormalMode(FPType: EltTy->getFltSemantics());
2513
2514 FPClassTest SrcDemandedMask = DemandedMask;
2515 if (DemandedMask & fcNan)
2516 SrcDemandedMask |= fcNan;
2517
2518 // Doubling a subnormal could have resulted in a normal value.
2519 if (DemandedMask & fcPosNormal)
2520 SrcDemandedMask |= fcPosSubnormal;
2521 if (DemandedMask & fcNegNormal)
2522 SrcDemandedMask |= fcNegSubnormal;
2523
2524 // Doubling a subnormal may produce 0 if FTZ/DAZ.
2525 if (Mode != DenormalMode::getIEEE()) {
2526 if (DemandedMask & fcPosZero) {
2527 SrcDemandedMask |= fcPosSubnormal;
2528
2529 if (Mode.inputsMayBePositiveZero() || Mode.outputsMayBePositiveZero())
2530 SrcDemandedMask |= fcNegSubnormal;
2531 }
2532
2533 if (DemandedMask & fcNegZero)
2534 SrcDemandedMask |= fcNegSubnormal;
2535 }
2536
2537 // Doubling a normal could have resulted in an infinity.
2538 if (DemandedMask & fcPosInf)
2539 SrcDemandedMask |= fcPosNormal;
2540 if (DemandedMask & fcNegInf)
2541 SrcDemandedMask |= fcNegNormal;
2542
2543 if (SimplifyDemandedFPClass(I, Op: 0, DemandedMask: SrcDemandedMask, Known&: KnownLHS, Q: SQ,
2544 Depth: Depth + 1))
2545 return I;
2546
2547 Known = KnownFPClass::fadd_self(Src: KnownLHS, RM: RoundingMode::NearestTiesToEven,
2548 Mode);
2549 KnownRHS = KnownLHS;
2550 } else {
2551 FPClassTest SrcDemandedMask = fcFinite;
2552
2553 // inf + (-inf) = nan
2554 if (DemandedMask & fcNan)
2555 SrcDemandedMask |= fcNan | fcInf;
2556
2557 if (DemandedMask & fcInf)
2558 SrcDemandedMask |= fcInf;
2559
2560 if (SimplifyDemandedFPClass(I, Op: 1, DemandedMask: SrcDemandedMask, Known&: KnownRHS, Q: SQ,
2561 Depth: Depth + 1) ||
2562 SimplifyDemandedFPClass(I, Op: 0, DemandedMask: SrcDemandedMask, Known&: KnownLHS, Q: SQ,
2563 Depth: Depth + 1))
2564 return I;
2565
2566 Type *EltTy = VTy->getScalarType();
2567 DenormalMode Mode = F.getDenormalMode(FPType: EltTy->getFltSemantics());
2568
2569 Known = I->getOpcode() == Instruction::FAdd
2570 ? KnownFPClass::fadd(LHS: KnownLHS, RHS: KnownRHS,
2571 RM: RoundingMode::NearestTiesToEven, Mode)
2572 : KnownFPClass::fsub(LHS: KnownLHS, RHS: KnownRHS,
2573 RM: RoundingMode::NearestTiesToEven, Mode);
2574 }
2575
2576 Known.knownNot(RuleOut: ~DemandedMask);
2577
2578 if (Constant *SingleVal = getFPClassConstant(Ty: VTy, Mask: Known.getKnownFPClasses(),
2579 /*IsCanonicalizing=*/true))
2580 return SingleVal;
2581
2582 // Propagate known result to simplify edge case checks.
2583 bool ResultNotNan = (DemandedMask & fcNan) == fcNone;
2584
2585 // With nnan: X + {+/-}Inf --> {+/-}Inf
2586 if (ResultNotNan && I->getOpcode() == Instruction::FAdd &&
2587 KnownRHS.isKnownAlways(Mask: fcInf | fcNan) && KnownLHS.isKnownNever(Mask: fcNan))
2588 return I->getOperand(i: 1);
2589
2590 // With nnan: {+/-}Inf + X --> {+/-}Inf
2591 // With nnan: {+/-}Inf - X --> {+/-}Inf
2592 if (ResultNotNan && KnownLHS.isKnownAlways(Mask: fcInf | fcNan) &&
2593 KnownRHS.isKnownNever(Mask: fcNan))
2594 return I->getOperand(i: 0);
2595
2596 FastMathFlags InferredFMF = inferFastMathValueFlags(
2597 FMF, ValidResults: Known.getKnownFPClasses(), Known: {KnownLHS, KnownRHS});
2598 if (InferredFMF != FMF) {
2599 I->setFastMathFlags(InferredFMF);
2600 return I;
2601 }
2602
2603 return nullptr;
2604 }
2605 case Instruction::FMul: {
2606 KnownFPClass KnownLHS, KnownRHS;
2607
2608 Value *X = I->getOperand(i: 0);
2609 Value *Y = I->getOperand(i: 1);
2610
2611 FPClassTest SrcDemandedMask =
2612 DemandedMask & (fcNan | fcZero | fcSubnormal | fcNormal);
2613
2614 if (DemandedMask & fcInf) {
2615 // mul x, inf = inf
2616 // mul large_x, large_y = inf
2617 SrcDemandedMask |= fcSubnormal | fcNormal | fcInf;
2618 }
2619
2620 if (DemandedMask & fcNan) {
2621 // mul +/-inf, 0 => nan
2622 SrcDemandedMask |= fcZero | fcInf | fcNan;
2623
2624 // TODO: Mode check
2625 // mul +/-inf, sub => nan if daz
2626 SrcDemandedMask |= fcSubnormal;
2627 }
2628
2629 // mul normal, subnormal = normal
2630 // Normal inputs may result in underflow.
2631 if (DemandedMask & (fcNormal | fcSubnormal))
2632 SrcDemandedMask |= fcNormal | fcSubnormal;
2633
2634 if (DemandedMask & fcZero)
2635 SrcDemandedMask |= fcNormal | fcSubnormal;
2636
2637 if (X == Y &&
2638 isGuaranteedNotToBeUndef(V: X, AC: SQ.AC, CtxI: SQ.CtxI, DT: SQ.DT, Depth: Depth + 1)) {
2639 if (SimplifyDemandedFPClass(I, Op: 0, DemandedMask: SrcDemandedMask, Known&: KnownLHS, Q: SQ,
2640 Depth: Depth + 1))
2641 return I;
2642 Type *EltTy = VTy->getScalarType();
2643
2644 DenormalMode Mode = F.getDenormalMode(FPType: EltTy->getFltSemantics());
2645 Known = KnownFPClass::square(Src: KnownLHS, Mode);
2646 Known.knownNot(RuleOut: ~DemandedMask);
2647
2648 if (Constant *Folded = getFPClassConstant(Ty: VTy, Mask: Known.getKnownFPClasses(),
2649 /*IsCanonicalizing=*/true))
2650 return Folded;
2651
2652 if (Known.isKnownAlways(Mask: fcPosZero | fcPosInf | fcNan) &&
2653 KnownLHS.isKnownNever(Mask: fcSubnormal | fcNormal)) {
2654 // We can skip the fabs if the source was already known positive.
2655 if (KnownLHS.isKnownAlways(Mask: fcPositive))
2656 return X;
2657
2658 // => fabs(x), in case this was a -inf or -0.
2659 // Note: Dropping canonicalize.
2660 IRBuilderBase::InsertPointGuard Guard(Builder);
2661 Builder.SetInsertPoint(I);
2662 Value *Fabs = Builder.CreateFAbs(V: X, FMFSource: FMF);
2663 Fabs->takeName(V: I);
2664 return Fabs;
2665 }
2666
2667 return nullptr;
2668 }
2669
2670 if (SimplifyDemandedFPClass(I, Op: 1, DemandedMask: SrcDemandedMask, Known&: KnownRHS, Q: SQ,
2671 Depth: Depth + 1) ||
2672 SimplifyDemandedFPClass(I, Op: 0, DemandedMask: SrcDemandedMask, Known&: KnownLHS, Q: SQ, Depth: Depth + 1))
2673 return I;
2674
2675 if (FMF.noInfs()) {
2676 // Flag implies inputs cannot be infinity.
2677 KnownLHS.knownNot(RuleOut: fcInf);
2678 KnownRHS.knownNot(RuleOut: fcInf);
2679 }
2680
2681 bool NonNanResult = (DemandedMask & fcNan) == fcNone;
2682
2683 // With no-nans/no-infs:
2684 // X * 0.0 --> copysign(0.0, X)
2685 // X * -0.0 --> copysign(0.0, -X)
2686 if ((NonNanResult || KnownLHS.isKnownNeverInfOrNaN()) &&
2687 KnownRHS.isKnownAlways(Mask: fcPosZero | fcNan)) {
2688 IRBuilderBase::InsertPointGuard Guard(Builder);
2689 Builder.SetInsertPoint(I);
2690
2691 // => copysign(+0, lhs)
2692 // Note: Dropping canonicalize
2693 Value *Copysign = Builder.CreateCopySign(LHS: Y, RHS: X, FMFSource: FMF);
2694 Copysign->takeName(V: I);
2695 return Copysign;
2696 }
2697
2698 if (KnownLHS.isKnownAlways(Mask: fcPosZero | fcNan) &&
2699 (NonNanResult || KnownRHS.isKnownNeverInfOrNaN())) {
2700 IRBuilderBase::InsertPointGuard Guard(Builder);
2701 Builder.SetInsertPoint(I);
2702
2703 // => copysign(+0, rhs)
2704 // Note: Dropping canonicalize
2705 Value *Copysign = Builder.CreateCopySign(LHS: X, RHS: Y, FMFSource: FMF);
2706 Copysign->takeName(V: I);
2707 return Copysign;
2708 }
2709
2710 if ((NonNanResult || KnownLHS.isKnownNeverInfOrNaN()) &&
2711 KnownRHS.isKnownAlways(Mask: fcNegZero | fcNan)) {
2712 IRBuilderBase::InsertPointGuard Guard(Builder);
2713 Builder.SetInsertPoint(I);
2714
2715 // => copysign(0, fneg(lhs))
2716 // Note: Dropping canonicalize
2717 Value *Copysign =
2718 Builder.CreateCopySign(LHS: Y, RHS: Builder.CreateFNegFMF(V: X, FMFSource: FMF), FMFSource: FMF);
2719 Copysign->takeName(V: I);
2720 return Copysign;
2721 }
2722
2723 if (KnownLHS.isKnownAlways(Mask: fcNegZero | fcNan) &&
2724 (NonNanResult || KnownRHS.isKnownNeverInfOrNaN())) {
2725 IRBuilderBase::InsertPointGuard Guard(Builder);
2726 Builder.SetInsertPoint(I);
2727
2728 // => copysign(+0, fneg(rhs))
2729 // Note: Dropping canonicalize
2730 Value *Copysign =
2731 Builder.CreateCopySign(LHS: X, RHS: Builder.CreateFNegFMF(V: Y, FMFSource: FMF), FMFSource: FMF);
2732 Copysign->takeName(V: I);
2733 return Copysign;
2734 }
2735
2736 Type *EltTy = VTy->getScalarType();
2737 DenormalMode Mode = F.getDenormalMode(FPType: EltTy->getFltSemantics());
2738
2739 if (KnownLHS.isKnownAlways(Mask: fcInf | fcNan) &&
2740 (KnownRHS.isKnownNeverNaN() &&
2741 KnownRHS.cannotBeOrderedGreaterEqZero(Mode))) {
2742 IRBuilderBase::InsertPointGuard Guard(Builder);
2743 Builder.SetInsertPoint(I);
2744
2745 // Note: Dropping canonicalize
2746 Value *Neg = Builder.CreateFNegFMF(V: X, FMFSource: FMF);
2747 Neg->takeName(V: I);
2748 return Neg;
2749 }
2750
2751 if (KnownRHS.isKnownAlways(Mask: fcInf | fcNan) &&
2752 (KnownLHS.isKnownNeverNaN() &&
2753 KnownLHS.cannotBeOrderedGreaterEqZero(Mode))) {
2754 IRBuilderBase::InsertPointGuard Guard(Builder);
2755 Builder.SetInsertPoint(I);
2756
2757 // Note: Dropping canonicalize
2758 Value *Neg = Builder.CreateFNegFMF(V: Y, FMFSource: FMF);
2759 Neg->takeName(V: I);
2760 return Neg;
2761 }
2762
2763 Known = KnownFPClass::fmul(LHS: KnownLHS, RHS: KnownRHS, Mode);
2764 Known.knownNot(RuleOut: ~DemandedMask);
2765
2766 if (Constant *SingleVal = getFPClassConstant(Ty: VTy, Mask: Known.getKnownFPClasses(),
2767 /*IsCanonicalizing=*/true))
2768 return SingleVal;
2769
2770 FastMathFlags InferredFMF = inferFastMathValueFlags(
2771 FMF, ValidResults: Known.getKnownFPClasses(), Known: {KnownLHS, KnownRHS});
2772 if (InferredFMF != FMF) {
2773 I->setFastMathFlags(InferredFMF);
2774 return I;
2775 }
2776
2777 return nullptr;
2778 }
2779 case Instruction::FDiv: {
2780 Value *X = I->getOperand(i: 0);
2781 Value *Y = I->getOperand(i: 1);
2782 if (X == Y &&
2783 isGuaranteedNotToBeUndef(V: X, AC: SQ.AC, CtxI: SQ.CtxI, DT: SQ.DT, Depth: Depth + 1)) {
2784 // If the source is 0, inf or nan, the result is a nan
2785 IRBuilderBase::InsertPointGuard Guard(Builder);
2786 Builder.SetInsertPoint(I);
2787
2788 Value *IsZeroOrNan = Builder.CreateFCmpFMF(
2789 P: FCmpInst::FCMP_UEQ, LHS: I->getOperand(i: 0), RHS: ConstantFP::getZero(Ty: VTy), FMFSource: FMF);
2790
2791 Value *Fabs = Builder.CreateFAbs(V: I->getOperand(i: 0), FMFSource: FMF);
2792 Value *IsInfOrNan = Builder.CreateFCmpFMF(
2793 P: FCmpInst::FCMP_UEQ, LHS: Fabs, RHS: ConstantFP::getInfinity(Ty: VTy), FMFSource: FMF);
2794
2795 Value *IsInfOrZeroOrNan = Builder.CreateOr(LHS: IsInfOrNan, RHS: IsZeroOrNan);
2796
2797 return Builder.CreateSelectFMFWithUnknownProfile(
2798 C: IsInfOrZeroOrNan, True: ConstantFP::getQNaN(Ty: VTy),
2799 False: ConstantFP::get(
2800 Ty: VTy, V: APFloat::getOne(Sem: VTy->getScalarType()->getFltSemantics())),
2801 FMFSource: FMF, DEBUG_TYPE);
2802 }
2803
2804 Type *EltTy = VTy->getScalarType();
2805 DenormalMode Mode = F.getDenormalMode(FPType: EltTy->getFltSemantics());
2806
2807 // Every output class could require denormal inputs (except for the
2808 // degenerate case of only-nan results, without DAZ).
2809 FPClassTest SrcDemandedMask = (DemandedMask & fcNan) | fcSubnormal;
2810
2811 // Normal inputs may result in underflow.
2812 // x / x = 1.0 for non0/inf/nan
2813 // -x = +y / -z
2814 // -x = -y / +z
2815 if (DemandedMask & (fcSubnormal | fcNormal))
2816 SrcDemandedMask |= fcNormal;
2817
2818 if (DemandedMask & fcNan) {
2819 // 0 / 0 = nan
2820 // inf / inf = nan
2821
2822 // Subnormal is added in case of DAZ, but this isn't strictly
2823 // necessary. Every other input class implies a possible subnormal source,
2824 // so this only could matter in the degenerate case of only-nan results.
2825 SrcDemandedMask |= fcZero | fcInf | fcNan;
2826 }
2827
2828 // Zero outputs may be the result of underflow.
2829 if (DemandedMask & fcZero)
2830 SrcDemandedMask |= fcNormal | fcSubnormal;
2831
2832 FPClassTest LHSDemandedMask = SrcDemandedMask;
2833 FPClassTest RHSDemandedMask = SrcDemandedMask;
2834
2835 // 0 / inf = 0
2836 if (DemandedMask & fcZero) {
2837 assert((LHSDemandedMask & fcSubnormal) &&
2838 "should not have to worry about daz here");
2839 LHSDemandedMask |= fcZero;
2840 RHSDemandedMask |= fcInf;
2841 }
2842
2843 // x / 0 = inf
2844 // large_normal / small_normal = inf
2845 // inf / 1 = inf
2846 // large_normal / subnormal = inf
2847 if (DemandedMask & fcInf) {
2848 LHSDemandedMask |= fcInf | fcNormal | fcSubnormal;
2849 RHSDemandedMask |= fcZero | fcSubnormal | fcNormal;
2850 }
2851
2852 KnownFPClass KnownLHS, KnownRHS;
2853 if (SimplifyDemandedFPClass(I, Op: 0, DemandedMask: LHSDemandedMask, Known&: KnownLHS, Q: SQ,
2854 Depth: Depth + 1) ||
2855 SimplifyDemandedFPClass(I, Op: 1, DemandedMask: RHSDemandedMask, Known&: KnownRHS, Q: SQ, Depth: Depth + 1))
2856 return I;
2857
2858 bool ResultNotNan = (DemandedMask & fcNan) == fcNone;
2859 bool ResultNotInf = (DemandedMask & fcInf) == fcNone;
2860
2861 // Replacing 0/x with a zero is only valid when the divisor can't be
2862 // (logical) zero, since 0/0 is NaN -- unless NaN results aren't demanded. A
2863 // subnormal divisor can flush to zero under a flushing denormal mode.
2864 bool CanIgnoreZeroByZeroNan =
2865 ResultNotNan || KnownRHS.isKnownNeverLogicalZero(Mode);
2866
2867 // nsz [+-]0 / x -> 0
2868 if (FMF.noSignedZeros() && KnownLHS.isKnownAlways(Mask: fcZero) &&
2869 KnownRHS.isKnownNeverNaN() && CanIgnoreZeroByZeroNan)
2870 return ConstantFP::getZero(Ty: VTy);
2871
2872 if (KnownLHS.isKnownAlways(Mask: fcPosZero) && KnownRHS.isKnownNeverNaN() &&
2873 CanIgnoreZeroByZeroNan) {
2874 IRBuilderBase::InsertPointGuard Guard(Builder);
2875 Builder.SetInsertPoint(I);
2876
2877 // nnan +0 / x -> copysign(0, rhs)
2878 // TODO: -0 / x => copysign(0, fneg(rhs))
2879 Value *Copysign = Builder.CreateCopySign(LHS: X, RHS: Y, FMFSource: FMF);
2880 Copysign->takeName(V: I);
2881 return Copysign;
2882 }
2883
2884 if (!ResultNotInf &&
2885 ((ResultNotNan || (KnownLHS.isKnownNeverNaN() &&
2886 KnownLHS.isKnownNeverLogicalZero(Mode))) &&
2887 (KnownRHS.isKnownAlways(Mask: fcPosZero) ||
2888 (FMF.noSignedZeros() && KnownRHS.isKnownAlways(Mask: fcZero))))) {
2889 IRBuilderBase::InsertPointGuard Guard(Builder);
2890 Builder.SetInsertPoint(I);
2891
2892 // nnan x / 0 => copysign(inf, x);
2893 // nnan nsz x / -0 => copysign(inf, x);
2894 Value *Copysign =
2895 Builder.CreateCopySign(LHS: ConstantFP::getInfinity(Ty: VTy), RHS: X, FMFSource: FMF);
2896 Copysign->takeName(V: I);
2897 return Copysign;
2898 }
2899
2900 // nnan ninf X / [-]0.0 -> poison
2901 if (ResultNotNan && ResultNotInf && KnownRHS.isKnownAlways(Mask: fcZero))
2902 return PoisonValue::get(T: VTy);
2903
2904 Known = KnownFPClass::fdiv(LHS: KnownLHS, RHS: KnownRHS, Mode);
2905 Known.knownNot(RuleOut: ~DemandedMask);
2906
2907 if (Constant *SingleVal = getFPClassConstant(Ty: VTy, Mask: Known.getKnownFPClasses(),
2908 /*IsCanonicalizing=*/true))
2909 return SingleVal;
2910
2911 FastMathFlags InferredFMF = inferFastMathValueFlags(
2912 FMF, ValidResults: Known.getKnownFPClasses(), Known: {KnownLHS, KnownRHS});
2913 if (InferredFMF != FMF) {
2914 I->setFastMathFlags(InferredFMF);
2915 return I;
2916 }
2917
2918 return nullptr;
2919 }
2920 case Instruction::FPTrunc:
2921 return simplifyDemandedUseFPClassFPTrunc(IC&: *this, I&: *I, FMF, DemandedMask,
2922 Known, SQ, Depth);
2923 case Instruction::FPExt: {
2924 const fltSemantics &DstTy = VTy->getScalarType()->getFltSemantics();
2925 const fltSemantics &SrcTy =
2926 I->getOperand(i: 0)->getType()->getScalarType()->getFltSemantics();
2927
2928 const DenormalMode DstMode = F.getDenormalMode(FPType: DstTy);
2929 const DenormalMode SrcMode = F.getDenormalMode(FPType: SrcTy);
2930
2931 FPClassTest SrcDemandedMask = DemandedMask;
2932 if (DemandedMask & fcNan)
2933 SrcDemandedMask |= fcNan;
2934
2935 // No subnormal result does not imply not-subnormal in the source type.
2936 if ((DemandedMask & fcNegNormal) != fcNone)
2937 SrcDemandedMask |= fcNegSubnormal;
2938 if ((DemandedMask & fcPosNormal) != fcNone)
2939 SrcDemandedMask |= fcPosSubnormal;
2940
2941 // TODO: This is a conservative approach for determining the actual input
2942 // and output denormal mode used by fpext.
2943 if (DstMode != DenormalMode::getIEEE() ||
2944 SrcMode != DenormalMode::getIEEE()) {
2945 // We are assuming that normal values will never produce subnormal results
2946 // for fpext. This assumption is incorrect for ppcf128 = fpext(f80).
2947 if ((DemandedMask & fcZero) != fcNone)
2948 SrcDemandedMask |= fcSubnormal;
2949 }
2950
2951 // TODO: Properly handle ppcf128 = fpext(f80) (since f80 has a larger
2952 // exponent range than ppcf128).
2953
2954 KnownFPClass KnownSrc;
2955 if (SimplifyDemandedFPClass(I, Op: 0, DemandedMask: SrcDemandedMask, Known&: KnownSrc, Q: SQ, Depth: Depth + 1))
2956 return I;
2957
2958 Known = KnownFPClass::fpext(KnownSrc, DstTy, SrcTy, DstMode, SrcMode);
2959 Known.knownNot(RuleOut: ~DemandedMask);
2960
2961 return simplifyDemandedFPClassResult(FPOp: I, FMF, DemandedMask, Known,
2962 KnownSrcs: {KnownSrc});
2963 }
2964 case Instruction::Call: {
2965 CallInst *CI = cast<CallInst>(Val: I);
2966 const Intrinsic::ID IID = CI->getIntrinsicID();
2967 switch (IID) {
2968 case Intrinsic::fabs: {
2969 KnownFPClass KnownSrc;
2970 if (SimplifyDemandedFPClass(I, Op: 0, DemandedMask: llvm::inverse_fabs(Mask: DemandedMask),
2971 Known&: KnownSrc, Q: SQ, Depth: Depth + 1))
2972 return I;
2973
2974 if (Value *Simplified = simplifyDemandedFPClassFabs(
2975 Known, Src: CI->getArgOperand(i: 0), DemandedMask, KnownSrc,
2976 NSZ: FMF.noSignedZeros()))
2977 return Simplified;
2978 break;
2979 }
2980 case Intrinsic::arithmetic_fence:
2981 if (SimplifyDemandedFPClass(I, Op: 0, DemandedMask, Known, Q: SQ, Depth: Depth + 1))
2982 return I;
2983 break;
2984 case Intrinsic::copysign: {
2985 // Flip on more potentially demanded classes
2986 const FPClassTest DemandedMaskAnySign = llvm::unknown_sign(Mask: DemandedMask);
2987 KnownFPClass KnownMag;
2988 if (SimplifyDemandedFPClass(I: CI, Op: 0, DemandedMask: DemandedMaskAnySign, Known&: KnownMag, Q: SQ,
2989 Depth: Depth + 1))
2990 return I;
2991
2992 if ((DemandedMask & fcNegative) == DemandedMask &&
2993 CI->getArgOperand(i: 1) != ConstantFP::get(Ty: VTy, V: -1.0)) {
2994 // Roundabout way of replacing with fneg(fabs)
2995 CI->setOperand(i_nocapture: 1, Val_nocapture: ConstantFP::get(Ty: VTy, V: -1.0));
2996 return I;
2997 }
2998
2999 if ((DemandedMask & fcPositive) == DemandedMask &&
3000 CI->getArgOperand(i: 1) != ConstantFP::getZero(Ty: VTy)) {
3001 // Roundabout way of replacing with fabs
3002 CI->setOperand(i_nocapture: 1, Val_nocapture: ConstantFP::getZero(Ty: VTy));
3003 return I;
3004 }
3005
3006 if (Value *Simplified = simplifyDemandedFPClassCopysignMag(
3007 MagSrc: CI->getArgOperand(i: 0), DemandedMask, KnownSrc: KnownMag,
3008 NSZ: FMF.noSignedZeros()))
3009 return Simplified;
3010
3011 KnownFPClass KnownSign =
3012 computeKnownFPClass(V: CI->getArgOperand(i: 1), InterestedClasses: fcAllFlags, SQ, Depth: Depth + 1);
3013 if (KnownMag.getSignBit() && KnownSign.getSignBit() &&
3014 *KnownMag.getSignBit() == *KnownSign.getSignBit())
3015 return CI->getOperand(i_nocapture: 0);
3016
3017 // TODO: Call argument attribute not considered
3018 // Input implied not-nan from flag.
3019 if (FMF.noNaNs())
3020 KnownSign.knownNot(RuleOut: fcNan);
3021
3022 if (KnownSign.getSignBit() == false &&
3023 CI->getArgOperand(i: 1) != ConstantFP::getZero(Ty: VTy)) {
3024 CI->dropUBImplyingAttrsAndMetadata();
3025 CI->setOperand(i_nocapture: 1, Val_nocapture: ConstantFP::getZero(Ty: VTy));
3026 return I;
3027 }
3028
3029 if (KnownSign.getSignBit() == true &&
3030 CI->getArgOperand(i: 1) != ConstantFP::get(Ty: VTy, V: -1.0)) {
3031 CI->dropUBImplyingAttrsAndMetadata();
3032 CI->setOperand(i_nocapture: 1, Val_nocapture: ConstantFP::get(Ty: VTy, V: -1.0));
3033 return I;
3034 }
3035
3036 Known = KnownFPClass::copysign(KnownMag, KnownSign);
3037 Known.knownNot(RuleOut: ~DemandedMask);
3038 break;
3039 }
3040 case Intrinsic::fma:
3041 case Intrinsic::fmuladd: {
3042 // Keep in mind that fma can rule out NaN in places that fmuladd cannot.
3043
3044 // We can't do any simplification on the source besides stripping out
3045 // unneeded nans.
3046 FPClassTest SrcDemandedMask = DemandedMask | ~fcNan;
3047 if (DemandedMask & fcNan)
3048 SrcDemandedMask |= fcNan;
3049
3050 KnownFPClass KnownSrc[3];
3051
3052 const bool IsFMA = IID == Intrinsic::fma;
3053
3054 Type *EltTy = VTy->getScalarType();
3055 if (CI->getArgOperand(i: 0) == CI->getArgOperand(i: 1) &&
3056 isGuaranteedNotToBeUndef(V: CI->getArgOperand(i: 0), AC: SQ.AC, CtxI: SQ.CtxI, DT: SQ.DT,
3057 Depth: Depth + 1)) {
3058 if (SimplifyDemandedFPClass(I: CI, Op: 0, DemandedMask: SrcDemandedMask, Known&: KnownSrc[0], Q: SQ,
3059 Depth: Depth + 1) ||
3060 SimplifyDemandedFPClass(I: CI, Op: 2, DemandedMask: SrcDemandedMask, Known&: KnownSrc[2], Q: SQ,
3061 Depth: Depth + 1))
3062 return I;
3063
3064 KnownSrc[1] = KnownSrc[0];
3065 DenormalMode Mode = F.getDenormalMode(FPType: EltTy->getFltSemantics());
3066 Known = IsFMA ? KnownFPClass::fma_square(Squared: KnownSrc[0], Addend: KnownSrc[2], Mode)
3067 : KnownFPClass::fmuladd_square(Squared: KnownSrc[0], Addend: KnownSrc[2],
3068 Mode);
3069 } else {
3070 for (int OpIdx = 0; OpIdx != 3; ++OpIdx) {
3071 if (SimplifyDemandedFPClass(I: CI, Op: OpIdx, DemandedMask: SrcDemandedMask,
3072 Known&: KnownSrc[OpIdx], Q: SQ, Depth: Depth + 1))
3073 return CI;
3074 }
3075
3076 DenormalMode Mode = F.getDenormalMode(FPType: EltTy->getFltSemantics());
3077 Known = IsFMA ? KnownFPClass::fma(LHS: KnownSrc[0], RHS: KnownSrc[1], Addend: KnownSrc[2],
3078 Mode)
3079 : KnownFPClass::fmuladd(LHS: KnownSrc[0], RHS: KnownSrc[1],
3080 Addend: KnownSrc[2], Mode);
3081 }
3082
3083 return simplifyDemandedFPClassResult(FPOp: CI, FMF, DemandedMask, Known,
3084 KnownSrcs: {KnownSrc});
3085 }
3086 case Intrinsic::maximum:
3087 case Intrinsic::minimum:
3088 case Intrinsic::maximumnum:
3089 case Intrinsic::minimumnum:
3090 case Intrinsic::maxnum:
3091 case Intrinsic::minnum: {
3092 const bool PropagateNaN =
3093 IID == Intrinsic::maximum || IID == Intrinsic::minimum;
3094
3095 // We can't tell much based on the demanded result without inspecting the
3096 // operands (e.g., a known-positive result could have been clamped), but
3097 // we can still prune known-nan inputs.
3098 FPClassTest SrcDemandedMask =
3099 PropagateNaN && ((DemandedMask & fcNan) == fcNone)
3100 ? DemandedMask | ~fcNan
3101 : fcAllFlags;
3102
3103 KnownFPClass KnownLHS, KnownRHS;
3104 if (SimplifyDemandedFPClass(I: CI, Op: 1, DemandedMask: SrcDemandedMask, Known&: KnownRHS, Q: SQ,
3105 Depth: Depth + 1) ||
3106 SimplifyDemandedFPClass(I: CI, Op: 0, DemandedMask: SrcDemandedMask, Known&: KnownLHS, Q: SQ,
3107 Depth: Depth + 1))
3108 return I;
3109
3110 Value *Simplified =
3111 simplifyDemandedFPClassMinMax(Known, IID, CI, DemandedMask, KnownLHS,
3112 KnownRHS, F, NSZ: FMF.noSignedZeros());
3113 if (Simplified)
3114 return Simplified;
3115
3116 auto *FPOp = cast<FPMathOperator>(Val: CI);
3117
3118 FPClassTest ValidResults = DemandedMask & Known.getKnownFPClasses();
3119 FastMathFlags InferredFMF = FMF;
3120
3121 if (!FMF.noSignedZeros()) {
3122 // Add NSZ flag if we know the result will not be sensitive to the sign
3123 // of 0.
3124 FPClassTest ZeroMask = fcZero;
3125
3126 Type *EltTy = VTy->getScalarType();
3127 DenormalMode Mode = F.getDenormalMode(FPType: EltTy->getFltSemantics());
3128 if (Mode != DenormalMode::getIEEE())
3129 ZeroMask |= fcSubnormal;
3130
3131 bool ResultNotLogical0 = (ValidResults & ZeroMask) == fcNone;
3132 if (ResultNotLogical0 || ((KnownLHS.isKnownNeverLogicalNegZero(Mode) ||
3133 KnownRHS.isKnownNeverLogicalPosZero(Mode)) &&
3134 (KnownLHS.isKnownNeverLogicalPosZero(Mode) ||
3135 KnownRHS.isKnownNeverLogicalNegZero(Mode))))
3136 InferredFMF.setNoSignedZeros(true);
3137 }
3138
3139 if (!FMF.noNaNs() &&
3140 ((PropagateNaN && (ValidResults & fcNan) == fcNone) ||
3141 (KnownLHS.isKnownNeverNaN() && KnownRHS.isKnownNeverNaN()))) {
3142 CI->dropUBImplyingAttrsAndMetadata();
3143 InferredFMF.setNoNaNs(true);
3144 }
3145
3146 if (InferredFMF != FMF) {
3147 CI->setFastMathFlags(InferredFMF);
3148 return FPOp;
3149 }
3150
3151 return nullptr;
3152 }
3153 case Intrinsic::exp:
3154 case Intrinsic::exp2:
3155 case Intrinsic::exp10: {
3156 if ((DemandedMask & fcPositive) == fcNone) {
3157 // Only returns positive values or nans.
3158 if ((DemandedMask & fcNan) == fcNone)
3159 return PoisonValue::get(T: VTy);
3160
3161 // Only need nan propagation.
3162 if ((DemandedMask & ~fcNan) == fcNone)
3163 return ConstantFP::getQNaN(Ty: VTy);
3164
3165 return CI->getArgOperand(i: 0);
3166 }
3167
3168 FPClassTest SrcDemandedMask = DemandedMask & fcNan;
3169 if (DemandedMask & fcNan)
3170 SrcDemandedMask |= fcNan;
3171
3172 if (DemandedMask & fcZero) {
3173 // exp(-infinity) = 0
3174 SrcDemandedMask |= fcNegInf;
3175
3176 // exp(-largest_normal) = 0
3177 //
3178 // Negative numbers of sufficiently large magnitude underflow to 0. No
3179 // subnormal input has a 0 result.
3180 SrcDemandedMask |= fcNegNormal;
3181 }
3182
3183 if (DemandedMask & fcPosSubnormal) {
3184 // Negative numbers of sufficiently large magnitude underflow to 0. No
3185 // subnormal input has a 0 result.
3186 SrcDemandedMask |= fcNegNormal;
3187 }
3188
3189 if (DemandedMask & fcPosNormal) {
3190 // exp(0) = 1
3191 // exp(+/- smallest_normal) = 1
3192 // exp(+/- largest_denormal) = 1
3193 // exp(+/- smallest_denormal) = 1
3194 // exp(-1) = pos normal
3195 SrcDemandedMask |= fcNormal | fcSubnormal | fcZero;
3196 }
3197
3198 // exp(inf), exp(largest_normal) = inf
3199 if (DemandedMask & fcPosInf)
3200 SrcDemandedMask |= fcPosInf | fcPosNormal;
3201
3202 KnownFPClass KnownSrc;
3203
3204 // TODO: This could really make use of KnownFPClass of specific value
3205 // range, (i.e., close enough to 1)
3206 if (SimplifyDemandedFPClass(I, Op: 0, DemandedMask: SrcDemandedMask, Known&: KnownSrc, Q: SQ,
3207 Depth: Depth + 1))
3208 return I;
3209
3210 // exp(+/-0) = 1
3211 if (KnownSrc.isKnownAlways(Mask: fcZero))
3212 return ConstantFP::get(Ty: VTy, V: 1.0);
3213
3214 // Only perform nan propagation.
3215 // Note: Dropping canonicalize / quiet of signaling nan.
3216 if (KnownSrc.isKnownAlways(Mask: fcNan))
3217 return CI->getArgOperand(i: 0);
3218
3219 // exp(0 | nan) => x == 0.0 ? 1.0 : x
3220 if (KnownSrc.isKnownAlways(Mask: fcZero | fcNan)) {
3221 IRBuilderBase::InsertPointGuard Guard(Builder);
3222 Builder.SetInsertPoint(CI);
3223
3224 // fadd +/-0, 1.0 => 1.0
3225 // fadd nan, 1.0 => nan
3226 return Builder.CreateFAddFMF(L: CI->getArgOperand(i: 0),
3227 R: ConstantFP::get(Ty: VTy, V: 1.0), FMFSource: FMF);
3228 }
3229
3230 if (KnownSrc.isKnownAlways(Mask: fcInf | fcNan)) {
3231 // exp(-inf) = 0
3232 // exp(+inf) = +inf
3233 IRBuilderBase::InsertPointGuard Guard(Builder);
3234 Builder.SetInsertPoint(CI);
3235
3236 // Note: Dropping canonicalize / quiet of signaling nan.
3237 Value *X = CI->getArgOperand(i: 0);
3238 Value *IsPosInfOrNan = Builder.CreateFCmpFMF(
3239 P: FCmpInst::FCMP_UEQ, LHS: X, RHS: ConstantFP::getInfinity(Ty: VTy), FMFSource: FMF);
3240 // We do not know whether an infinity or a NaN is more likely here,
3241 // so mark the branch weights as unkown.
3242 Value *ZeroOrInf = Builder.CreateSelectFMFWithUnknownProfile(
3243 C: IsPosInfOrNan, True: X, False: ConstantFP::getZero(Ty: VTy), FMFSource: FMF, DEBUG_TYPE);
3244 return ZeroOrInf;
3245 }
3246
3247 Known = KnownFPClass::exp(Src: KnownSrc);
3248 Known.knownNot(RuleOut: ~DemandedMask);
3249
3250 return simplifyDemandedFPClassResult(FPOp: CI, FMF, DemandedMask, Known,
3251 KnownSrcs: KnownSrc);
3252 }
3253 case Intrinsic::log:
3254 case Intrinsic::log2:
3255 case Intrinsic::log10: {
3256 Type *EltTy = VTy->getScalarType();
3257 bool IsMultiUnitFPType = EltTy->isMultiUnitFPType();
3258
3259 FPClassTest DemandedSrcMask = DemandedMask & (fcNan | fcPosInf);
3260 if (DemandedMask & fcNan)
3261 DemandedSrcMask |= fcNan;
3262
3263 DenormalMode Mode = F.getDenormalMode(FPType: EltTy->getFltSemantics());
3264
3265 // log(x) can be negative zero or subnormal if x is close to +1.0 for the
3266 // PPCDoubleDouble type.
3267 if (IsMultiUnitFPType && (DemandedMask & (fcNegZero | fcSubnormal))) {
3268 DemandedSrcMask |= fcPosNormal;
3269 // Note that DoubleAPFloat::isDenormal considers 1.0 + DBL_TRUE_MIN to
3270 // be denormal, which APFloat::classify treats as fcSubnormal.
3271 DemandedSrcMask |= fcPosSubnormal;
3272 }
3273
3274 // log(x < 0) = nan
3275 if (DemandedMask & fcNan)
3276 DemandedSrcMask |= (fcNegative & ~fcNegZero);
3277
3278 // log(0) = -inf
3279 if (DemandedMask & fcNegInf) {
3280 DemandedSrcMask |= fcZero;
3281
3282 // Subnormals may be treated as logical zero.
3283 if (Mode.inputsMayBeZero())
3284 DemandedSrcMask |= fcSubnormal;
3285 }
3286
3287 if (DemandedMask & fcNormal)
3288 DemandedSrcMask |= fcNormal | fcSubnormal;
3289
3290 // log(1) = 0
3291 if (DemandedMask & fcZero)
3292 DemandedSrcMask |= fcPosNormal;
3293
3294 KnownFPClass KnownSrc;
3295 if (SimplifyDemandedFPClass(I, Op: 0, DemandedMask: DemandedSrcMask, Known&: KnownSrc, Q: SQ,
3296 Depth: Depth + 1))
3297 return I;
3298
3299 Known = KnownFPClass::log(Src: KnownSrc, Mode, IsMultiUnitFPType);
3300 Known.knownNot(RuleOut: ~DemandedMask);
3301
3302 return simplifyDemandedFPClassResult(FPOp: CI, FMF, DemandedMask, Known,
3303 KnownSrcs: KnownSrc);
3304 }
3305 case Intrinsic::sqrt: {
3306 Type *EltTy = VTy->getScalarType();
3307 DenormalMode Mode = F.getDenormalMode(FPType: EltTy->getFltSemantics());
3308
3309 // Be explicit about the DemandedSrcMask so we do not miss anything.
3310 FPClassTest DemandedSrcMask = fcNone;
3311 if (DemandedMask & fcSNan)
3312 DemandedSrcMask |= fcSNan;
3313 if (DemandedMask & fcQNan)
3314 DemandedSrcMask |=
3315 fcQNan | fcSNan | fcNegInf | fcNegNormal | fcNegSubnormal;
3316 if (DemandedMask & fcNegZero) {
3317 DemandedSrcMask |= fcNegZero;
3318 if (Mode.Input == DenormalMode::PreserveSign ||
3319 Mode.Input == DenormalMode::Dynamic)
3320 DemandedSrcMask |= fcNegSubnormal;
3321 }
3322 if (DemandedMask & fcPosZero) {
3323 DemandedSrcMask |= fcPosZero;
3324 if (Mode.inputsMayBeZero())
3325 DemandedSrcMask |= fcPosSubnormal;
3326 if (Mode.inputsMayBePositiveZero())
3327 DemandedSrcMask |= fcNegSubnormal;
3328 }
3329 if (DemandedMask & fcPosNormal)
3330 DemandedSrcMask |= fcPosSubnormal | fcPosNormal;
3331 if (DemandedMask & fcPosInf)
3332 DemandedSrcMask |= fcPosInf;
3333
3334 KnownFPClass KnownSrc;
3335 if (SimplifyDemandedFPClass(I, Op: 0, DemandedMask: DemandedSrcMask, Known&: KnownSrc, Q: SQ,
3336 Depth: Depth + 1))
3337 return I;
3338
3339 KnownSrc.knownNot(RuleOut: ~DemandedSrcMask);
3340
3341 // sqrt(-x) = nan, but be careful of negative subnormals flushed to 0.
3342 // TODO: This becomes redundant if KnownFPClass::sqrt rules out
3343 // fcPosNormal.
3344 if (KnownSrc.isKnownNever(Mask: fcPositive) &&
3345 KnownSrc.isKnownNeverLogicalZero(Mode))
3346 return ConstantFP::getQNaN(Ty: VTy);
3347
3348 Known = KnownFPClass::sqrt(Src: KnownSrc, Mode);
3349 Known.knownNot(RuleOut: ~DemandedMask);
3350
3351 if (Known.getKnownFPClasses() == fcZero) {
3352 if (FMF.noSignedZeros())
3353 return ConstantFP::getZero(Ty: VTy);
3354 IRBuilderBase::InsertPointGuard Guard(Builder);
3355 Builder.SetInsertPoint(CI);
3356
3357 Value *Copysign = Builder.CreateCopySign(LHS: ConstantFP::getZero(Ty: VTy),
3358 RHS: CI->getArgOperand(i: 0), FMFSource: FMF);
3359 Copysign->takeName(V: CI);
3360 return Copysign;
3361 }
3362
3363 // TODO: fold sqrt(x) -> x when x is nan | pinf | zero
3364
3365 return simplifyDemandedFPClassResult(FPOp: CI, FMF, DemandedMask, Known,
3366 KnownSrcs: {KnownSrc});
3367 }
3368 case Intrinsic::ldexp: {
3369 Type *EltTy = VTy->getScalarType();
3370 const fltSemantics &FltSem = EltTy->getFltSemantics();
3371 DenormalMode Mode = F.getDenormalMode(FPType: FltSem);
3372
3373 FPClassTest SrcDemandedMask = fcNone;
3374 if (DemandedMask & fcNan)
3375 SrcDemandedMask |= fcNan;
3376
3377 if (DemandedMask & fcPosInf)
3378 SrcDemandedMask |= fcPosInf | fcPosNormal | fcPosSubnormal;
3379 if (DemandedMask & fcNegInf)
3380 SrcDemandedMask |= fcNegInf | fcNegNormal | fcNegSubnormal;
3381
3382 if (DemandedMask & (fcPosNormal | fcPosSubnormal))
3383 SrcDemandedMask |= fcPosNormal | fcPosSubnormal;
3384 if (DemandedMask & (fcNegNormal | fcNegSubnormal))
3385 SrcDemandedMask |= fcNegNormal | fcNegSubnormal;
3386
3387 // Note: If we refine based on KnownExpBits in the future, then this
3388 // denormal handling code will need to be updated.
3389 if (DemandedMask & fcPosZero) {
3390 SrcDemandedMask |= fcPosFinite;
3391 if (Mode.inputsMayBePositiveZero())
3392 SrcDemandedMask |= fcNegSubnormal;
3393 if (Mode.outputsMayBePositiveZero())
3394 SrcDemandedMask |= fcNegNormal | fcNegSubnormal;
3395 }
3396 if (DemandedMask & fcNegZero)
3397 SrcDemandedMask |= fcNegFinite;
3398
3399 KnownFPClass KnownSrc;
3400 if (SimplifyDemandedFPClass(I: CI, Op: 0, DemandedMask: SrcDemandedMask, Known&: KnownSrc, Q: SQ,
3401 Depth: Depth + 1))
3402 return CI;
3403
3404 KnownBits KnownExpBits =
3405 ::computeKnownBits(V: CI->getArgOperand(i: 1), Q: SQ, Depth: Depth + 1);
3406
3407 Known = KnownFPClass::ldexp(Src: KnownSrc, ExpBits: KnownExpBits, Flt: FltSem, Mode);
3408 Known.knownNot(RuleOut: ~DemandedMask);
3409
3410 return simplifyDemandedFPClassResult(FPOp: CI, FMF, DemandedMask, Known,
3411 KnownSrcs: {KnownSrc});
3412 }
3413 case Intrinsic::trunc:
3414 case Intrinsic::floor:
3415 case Intrinsic::ceil:
3416 case Intrinsic::rint:
3417 case Intrinsic::nearbyint:
3418 case Intrinsic::round:
3419 case Intrinsic::roundeven: {
3420 Type *EltTy = VTy->getScalarType();
3421 DenormalMode Mode = F.getDenormalMode(FPType: EltTy->getFltSemantics());
3422
3423 FPClassTest DemandedSrcMask = DemandedMask;
3424 if (DemandedMask & fcNan)
3425 DemandedSrcMask |= fcNan;
3426
3427 const bool IsMultiUnitFPType = EltTy->isMultiUnitFPType();
3428 // Rounding a finite multi-unit value may overflow to an infinity.
3429 if (IsMultiUnitFPType) {
3430 if (IID != Intrinsic::trunc && IID != Intrinsic::floor &&
3431 (DemandedMask & fcPosInf))
3432 DemandedSrcMask |= fcPosNormal;
3433 if (IID != Intrinsic::trunc && IID != Intrinsic::ceil &&
3434 (DemandedMask & fcNegInf))
3435 DemandedSrcMask |= fcNegNormal;
3436 }
3437
3438 // Zero results imply valid subnormal sources.
3439 if (DemandedMask & fcNegZero)
3440 DemandedSrcMask |= fcNegSubnormal | fcNegNormal;
3441
3442 if (DemandedMask & fcPosZero) {
3443 DemandedSrcMask |= fcPosSubnormal | fcPosNormal;
3444 if (Mode.inputsMayBePositiveZero())
3445 DemandedSrcMask |= fcNegSubnormal;
3446 }
3447
3448 // Rounding a subnormal away from zero may produce a normal value.
3449 if (DemandedMask & fcNegNormal)
3450 DemandedSrcMask |= fcNegSubnormal;
3451 if (DemandedMask & fcPosNormal)
3452 DemandedSrcMask |= fcPosSubnormal;
3453
3454 KnownFPClass KnownSrc;
3455 if (SimplifyDemandedFPClass(I: CI, Op: 0, DemandedMask: DemandedSrcMask, Known&: KnownSrc, Q: SQ,
3456 Depth: Depth + 1))
3457 return I;
3458
3459 // Note: Possibly dropping snan quiet.
3460 if (KnownSrc.isKnownAlways(Mask: fcInf | fcNan | fcZero))
3461 return CI->getArgOperand(i: 0);
3462
3463 bool IsRoundNearestOrTrunc =
3464 IID == Intrinsic::round || IID == Intrinsic::roundeven ||
3465 IID == Intrinsic::nearbyint || IID == Intrinsic::rint ||
3466 IID == Intrinsic::trunc;
3467
3468 if ((IID == Intrinsic::floor || IsRoundNearestOrTrunc) &&
3469 KnownSrc.isKnownAlways(Mask: fcPosZero | fcPosSubnormal))
3470 return ConstantFP::getZero(Ty: VTy);
3471
3472 if ((IID == Intrinsic::ceil || IsRoundNearestOrTrunc) &&
3473 KnownSrc.isKnownAlways(Mask: fcNegZero | fcNegSubnormal) &&
3474 !Mode.inputsMayBePositiveZero())
3475 return ConstantFP::getZero(Ty: VTy, Negative: true);
3476
3477 if (IID == Intrinsic::floor && KnownSrc.isKnownAlways(Mask: fcNegSubnormal) &&
3478 !Mode.inputsMayBeZero())
3479 return ConstantFP::get(Ty: VTy, V: -1.0);
3480
3481 if (IID == Intrinsic::ceil && KnownSrc.isKnownAlways(Mask: fcPosSubnormal) &&
3482 !Mode.inputsMayBeZero())
3483 return ConstantFP::get(Ty: VTy, V: 1.0);
3484
3485 switch (IID) {
3486 case Intrinsic::trunc:
3487 Known = KnownFPClass::trunc(Src: KnownSrc, Mode);
3488 break;
3489 case Intrinsic::floor:
3490 Known = KnownFPClass::floor(Src: KnownSrc, IsMultiUnitFPType, Mode);
3491 break;
3492 case Intrinsic::ceil:
3493 Known = KnownFPClass::ceil(Src: KnownSrc, IsMultiUnitFPType, Mode);
3494 break;
3495 default:
3496 Known =
3497 KnownFPClass::roundToIntegral(Src: KnownSrc, IsMultiUnitFPType, Mode);
3498 break;
3499 }
3500
3501 Known.knownNot(RuleOut: ~DemandedMask);
3502
3503 if (Constant *SingleVal =
3504 getFPClassConstant(Ty: VTy, Mask: Known.getKnownFPClasses(),
3505 /*IsCanonicalizing=*/true))
3506 return SingleVal;
3507
3508 if ((IID == Intrinsic::trunc || IsRoundNearestOrTrunc) &&
3509 KnownSrc.isKnownAlways(Mask: fcZero | fcSubnormal) &&
3510 (!Mode.inputsMayBePositiveZero() ||
3511 KnownSrc.isKnownNever(Mask: fcNegSubnormal))) {
3512 IRBuilderBase::InsertPointGuard Guard(Builder);
3513 Builder.SetInsertPoint(CI);
3514
3515 Value *Copysign = Builder.CreateCopySign(LHS: ConstantFP::getZero(Ty: VTy),
3516 RHS: CI->getArgOperand(i: 0));
3517 Copysign->takeName(V: CI);
3518 return Copysign;
3519 }
3520
3521 FastMathFlags InferredFMF =
3522 inferFastMathValueFlags(FMF, ValidResults: Known.getKnownFPClasses(), Known: KnownSrc);
3523 if (InferredFMF != FMF) {
3524 CI->dropUBImplyingAttrsAndMetadata();
3525 CI->setFastMathFlags(InferredFMF);
3526 return CI;
3527 }
3528
3529 return nullptr;
3530 }
3531 case Intrinsic::fptrunc_round:
3532 return simplifyDemandedUseFPClassFPTrunc(IC&: *this, I&: *CI, FMF, DemandedMask,
3533 Known, SQ, Depth);
3534 case Intrinsic::canonicalize: {
3535 Type *EltTy = VTy->getScalarType();
3536
3537 if (EltTy->isIEEELikeFPTy()) {
3538 const fltSemantics &Flt = EltTy->getFltSemantics();
3539 DenormalMode Mode = F.getDenormalMode(FPType: Flt);
3540
3541 FPClassTest SrcDemandedMask = DemandedMask;
3542
3543 // A demanded quiet nan result may have come from a signaling nan, so we
3544 // need to expand the demanded mask.
3545 if ((DemandedMask & fcQNan) != fcNone)
3546 SrcDemandedMask |= fcSNan;
3547
3548 if (Mode != DenormalMode::getIEEE()) {
3549 // Any zero results may have come from flushed denormals.
3550 if (DemandedMask & fcPosZero) {
3551 SrcDemandedMask |= fcPosSubnormal;
3552 if (Mode.inputsMayBePositiveZero() ||
3553 Mode.outputsMayBePositiveZero())
3554 SrcDemandedMask |= fcNegSubnormal;
3555 }
3556 if (DemandedMask & fcNegZero)
3557 SrcDemandedMask |= fcNegSubnormal;
3558 }
3559
3560 if (Mode == DenormalMode::getPreserveSign()) {
3561 // If a denormal input will be flushed, and we don't need zeros, we
3562 // don't need denormals either.
3563 if ((DemandedMask & fcPosZero) == fcNone)
3564 SrcDemandedMask &= ~fcPosSubnormal;
3565
3566 if ((DemandedMask & fcNegZero) == fcNone)
3567 SrcDemandedMask &= ~fcNegSubnormal;
3568 }
3569
3570 KnownFPClass KnownSrc;
3571
3572 // Simplify upstream operations before trying to simplify this call.
3573 if (SimplifyDemandedFPClass(I, Op: 0, DemandedMask: SrcDemandedMask, Known&: KnownSrc, Q: SQ,
3574 Depth: Depth + 1))
3575 return I;
3576
3577 // Perform the canonicalization to see if this folded to a constant.
3578 Known = KnownFPClass::canonicalize(Src: KnownSrc, Flt, Mode);
3579 Known.knownNot(RuleOut: ~DemandedMask);
3580
3581 if (Constant *SingleVal =
3582 getFPClassConstant(Ty: VTy, Mask: Known.getKnownFPClasses()))
3583 return SingleVal;
3584
3585 // For IEEE handling, there is only a bit change for nan inputs, so we
3586 // can drop it if we do not demand nan results or we know the input
3587 // isn't a nan.
3588 // Otherwise, we also need to avoid denormal inputs to drop the
3589 // canonicalize.
3590 if (KnownSrc.isKnownNeverNaN() && (Mode == DenormalMode::getIEEE() ||
3591 KnownSrc.isKnownNeverSubnormal()))
3592 return CI->getArgOperand(i: 0);
3593
3594 FastMathFlags InferredFMF =
3595 inferFastMathValueFlags(FMF, ValidResults: Known.getKnownFPClasses(), Known: KnownSrc);
3596 if (InferredFMF != FMF) {
3597 CI->dropUBImplyingAttrsAndMetadata();
3598 CI->setFastMathFlags(InferredFMF);
3599 return CI;
3600 }
3601
3602 return nullptr;
3603 }
3604
3605 [[fallthrough]];
3606 }
3607 default:
3608 Known = computeKnownFPClass(V: I, InterestedClasses: DemandedMask, SQ, Depth: Depth + 1);
3609 Known.knownNot(RuleOut: ~DemandedMask);
3610 break;
3611 }
3612
3613 break;
3614 }
3615 case Instruction::Select: {
3616 KnownFPClass KnownLHS, KnownRHS;
3617 if (SimplifyDemandedFPClass(I, Op: 2, DemandedMask, Known&: KnownRHS, Q: SQ, Depth: Depth + 1) ||
3618 SimplifyDemandedFPClass(I, Op: 1, DemandedMask, Known&: KnownLHS, Q: SQ, Depth: Depth + 1))
3619 return I;
3620
3621 if (KnownLHS.isKnownNever(Mask: DemandedMask))
3622 return I->getOperand(i: 2);
3623 if (KnownRHS.isKnownNever(Mask: DemandedMask))
3624 return I->getOperand(i: 1);
3625
3626 adjustKnownFPClassForSelectArm(Known&: KnownLHS, Cond: I->getOperand(i: 0), Arm: I->getOperand(i: 1),
3627 /*Invert=*/false, Q: SQ, Depth);
3628 adjustKnownFPClassForSelectArm(Known&: KnownRHS, Cond: I->getOperand(i: 0), Arm: I->getOperand(i: 2),
3629 /*Invert=*/true, Q: SQ, Depth);
3630 Known = KnownLHS.intersectWith(RHS: KnownRHS);
3631 Known.knownNot(RuleOut: ~DemandedMask);
3632 break;
3633 }
3634 case Instruction::ExtractElement: {
3635 // TODO: Handle demanded element mask
3636 if (SimplifyDemandedFPClass(I, Op: 0, DemandedMask, Known, Q: SQ, Depth: Depth + 1))
3637 return I;
3638 Known.knownNot(RuleOut: ~DemandedMask);
3639 break;
3640 }
3641 case Instruction::InsertElement: {
3642 KnownFPClass KnownInserted, KnownVec;
3643 if (SimplifyDemandedFPClass(I, Op: 1, DemandedMask, Known&: KnownInserted, Q: SQ,
3644 Depth: Depth + 1) ||
3645 SimplifyDemandedFPClass(I, Op: 0, DemandedMask, Known&: KnownVec, Q: SQ, Depth: Depth + 1))
3646 return I;
3647
3648 // TODO: Use demanded elements logic from computeKnownFPClass
3649 Known = KnownVec | KnownInserted;
3650 Known.knownNot(RuleOut: ~DemandedMask);
3651 break;
3652 }
3653 case Instruction::ShuffleVector: {
3654 KnownFPClass KnownLHS, KnownRHS;
3655 if (SimplifyDemandedFPClass(I, Op: 1, DemandedMask, Known&: KnownRHS, Q: SQ, Depth: Depth + 1) ||
3656 SimplifyDemandedFPClass(I, Op: 0, DemandedMask, Known&: KnownLHS, Q: SQ, Depth: Depth + 1))
3657 return I;
3658
3659 // TODO: This is overly conservative and should consider demanded elements,
3660 // and splats.
3661 Known = KnownLHS | KnownRHS;
3662 Known.knownNot(RuleOut: ~DemandedMask);
3663 break;
3664 }
3665 case Instruction::InsertValue: {
3666 KnownFPClass KnownAgg, KnownElt;
3667 if (SimplifyDemandedFPClass(I, Op: 0, DemandedMask, Known&: KnownAgg, Q: SQ, Depth: Depth + 1) ||
3668 SimplifyDemandedFPClass(I, Op: 1, DemandedMask, Known&: KnownElt, Q: SQ, Depth: Depth + 1))
3669 return I;
3670
3671 Known = KnownAgg | KnownElt;
3672 break;
3673 }
3674 case Instruction::ExtractValue: {
3675 Value *ExtractSrc;
3676 if (match(V: I, P: m_ExtractValue<0>(V: m_OneUse(SubPattern: m_Value(V&: ExtractSrc))))) {
3677 if (auto *II = dyn_cast<IntrinsicInst>(Val: ExtractSrc)) {
3678 const Intrinsic::ID IID = II->getIntrinsicID();
3679 switch (IID) {
3680 case Intrinsic::frexp: {
3681 FPClassTest SrcDemandedMask = fcNone;
3682
3683 if (DemandedMask & fcNan)
3684 SrcDemandedMask |= fcNan;
3685
3686 // Positive subnormals and negative subnormals could become positive
3687 // zero.
3688 if (DemandedMask & fcPosZero)
3689 SrcDemandedMask |= fcPosZero | fcSubnormal;
3690
3691 // Negative subnormals could become negative zero.
3692 if (DemandedMask & fcNegZero)
3693 SrcDemandedMask |= fcNegZero | fcNegSubnormal;
3694
3695 if (DemandedMask & (fcNegNormal | fcNegSubnormal))
3696 SrcDemandedMask |= fcNegNormal | fcNegSubnormal;
3697 if (DemandedMask & (fcPosNormal | fcPosSubnormal))
3698 SrcDemandedMask |= fcPosNormal | fcPosSubnormal;
3699
3700 if (DemandedMask & fcPosInf)
3701 SrcDemandedMask |= fcPosInf;
3702 if (DemandedMask & fcNegInf)
3703 SrcDemandedMask |= fcNegInf;
3704
3705 KnownFPClass KnownSrc;
3706 if (SimplifyDemandedFPClass(I: II, Op: 0, DemandedMask: SrcDemandedMask, Known&: KnownSrc, Q: SQ,
3707 Depth: Depth + 1))
3708 return I;
3709
3710 Type *EltTy = VTy->getScalarType();
3711 DenormalMode Mode = F.getDenormalMode(FPType: EltTy->getFltSemantics());
3712
3713 Known = KnownFPClass::frexp_mant(Src: KnownSrc, Mode);
3714 Known.setKnownFPClasses(Known.getKnownFPClasses() & DemandedMask);
3715
3716 if (Constant *SingleVal =
3717 getFPClassConstant(Ty: VTy, Mask: Known.getKnownFPClasses(),
3718 /*IsCanonicalizing=*/true))
3719 return SingleVal;
3720
3721 // frexp returns zero, infinity, and NaN inputs unchanged.
3722 if (KnownSrc.isKnownAlways(Mask: fcZero | fcInf | fcNan))
3723 return II->getArgOperand(i: 0);
3724
3725 return nullptr;
3726 }
3727 default:
3728 break;
3729 }
3730 }
3731 }
3732
3733 KnownFPClass KnownSrc;
3734 if (SimplifyDemandedFPClass(I, Op: 0, DemandedMask, Known&: KnownSrc, Q: SQ, Depth: Depth + 1))
3735 return I;
3736 Known = KnownSrc;
3737 break;
3738 }
3739 case Instruction::PHI: {
3740 const unsigned PhiRecursionLimit = MaxAnalysisRecursionDepth - 2;
3741 if (Depth >= PhiRecursionLimit)
3742 break;
3743
3744 PHINode *P = cast<PHINode>(Val: I);
3745 SimplifyQuery ContextSQ = SQ.getWithoutCondContext();
3746
3747 bool First = true;
3748 bool Changed = false;
3749 for (unsigned I = 0, E = P->getNumIncomingValues(); I != E; ++I) {
3750 // TODO: Better support for self recursive phi
3751 BasicBlock *PredBB = P->getIncomingBlock(i: I);
3752 const Instruction *CtxI = PredBB->getTerminator();
3753
3754 // Attempt to simplify all incoming edges at a time. If we simplify one
3755 // incoming edge, the phi may fold away, losing information on a later
3756 // visit.
3757 KnownFPClass KnownSrc;
3758 if (SimplifyDemandedFPClass(
3759 I: P, Op: P->getOperandNumForIncomingValue(i: I), DemandedMask, Known&: KnownSrc,
3760 Q: ContextSQ.getWithInstruction(I: CtxI), Depth: Depth + 1)) {
3761 // Fixup the other block references to the simplified value.
3762 P->setIncomingValueForBlock(BB: PredBB, V: P->getIncomingValue(i: I));
3763 Changed = true;
3764 }
3765
3766 if (First) {
3767 Known = KnownSrc;
3768 First = false;
3769 } else {
3770 Known |= KnownSrc;
3771 }
3772 }
3773
3774 if (Changed)
3775 return P;
3776
3777 Known.knownNot(RuleOut: ~DemandedMask);
3778 break;
3779 }
3780 default:
3781 Known = computeKnownFPClass(V: I, InterestedClasses: DemandedMask, SQ, Depth: Depth + 1);
3782 Known.knownNot(RuleOut: ~DemandedMask);
3783 break;
3784 }
3785
3786 return getFPClassConstant(Ty: VTy, Mask: Known.getKnownFPClasses());
3787}
3788
3789/// Helper routine of SimplifyDemandedUseFPClass. It computes Known
3790/// floating-point classes. It also tries to handle simplifications that can be
3791/// done based on DemandedMask, but without modifying the Instruction.
3792Value *InstCombinerImpl::SimplifyMultipleUseDemandedFPClass(
3793 Instruction *I, FPClassTest DemandedMask, KnownFPClass &Known,
3794 const SimplifyQuery &SQ, unsigned Depth) {
3795 FastMathFlags FMF;
3796 if (auto *FPOp = dyn_cast<FPMathOperator>(Val: I)) {
3797 FMF = FPOp->getFastMathFlags();
3798 DemandedMask = adjustDemandedMaskFromFlags(DemandedMask, FMF);
3799 }
3800
3801 switch (I->getOpcode()) {
3802 case Instruction::Select: {
3803 // TODO: Can we infer which side it came from based on adjusted result
3804 // class?
3805 KnownFPClass KnownRHS =
3806 computeKnownFPClass(V: I->getOperand(i: 2), InterestedClasses: DemandedMask, SQ, Depth: Depth + 1);
3807 if (KnownRHS.isKnownNever(Mask: DemandedMask))
3808 return I->getOperand(i: 1);
3809
3810 KnownFPClass KnownLHS =
3811 computeKnownFPClass(V: I->getOperand(i: 1), InterestedClasses: DemandedMask, SQ, Depth: Depth + 1);
3812 if (KnownLHS.isKnownNever(Mask: DemandedMask))
3813 return I->getOperand(i: 2);
3814
3815 adjustKnownFPClassForSelectArm(Known&: KnownLHS, Cond: I->getOperand(i: 0), Arm: I->getOperand(i: 1),
3816 /*Invert=*/false, Q: SQ, Depth);
3817 adjustKnownFPClassForSelectArm(Known&: KnownRHS, Cond: I->getOperand(i: 0), Arm: I->getOperand(i: 2),
3818 /*Invert=*/true, Q: SQ, Depth);
3819 Known = KnownLHS.intersectWith(RHS: KnownRHS);
3820 Known.knownNot(RuleOut: ~DemandedMask);
3821 break;
3822 }
3823 case Instruction::FNeg: {
3824 // Special case fneg(fabs(x))
3825 Value *Src;
3826
3827 Value *FNegSrc = I->getOperand(i: 0);
3828 if (!match(V: FNegSrc, P: m_FAbs(Op0: m_Value(V&: Src)))) {
3829 Known = computeKnownFPClass(V: I, InterestedClasses: DemandedMask, SQ, Depth: Depth + 1);
3830 break;
3831 }
3832
3833 KnownFPClass KnownSrc = computeKnownFPClass(V: Src, InterestedClasses: fcAllFlags, SQ, Depth: Depth + 1);
3834
3835 FastMathFlags FabsFMF = cast<FPMathOperator>(Val: FNegSrc)->getFastMathFlags();
3836 FPClassTest ThisDemandedMask =
3837 adjustDemandedMaskFromFlags(DemandedMask, FMF: FabsFMF);
3838
3839 // We cannot apply the NSZ logic with multiple uses. We can apply it if the
3840 // inner fabs has it and this is the only use.
3841 if (Value *Simplified = simplifyDemandedFPClassFnegFabs(
3842 Known, Src, DemandedMask: ThisDemandedMask, KnownSrc, /*NSZ=*/false))
3843 return Simplified;
3844 break;
3845 }
3846 case Instruction::Call: {
3847 const CallInst *CI = cast<CallInst>(Val: I);
3848 const Intrinsic::ID IID = CI->getIntrinsicID();
3849 switch (IID) {
3850 case Intrinsic::fabs: {
3851 Value *Src = CI->getArgOperand(i: 0);
3852 KnownFPClass KnownSrc =
3853 computeKnownFPClass(V: Src, InterestedClasses: fcAllFlags, SQ, Depth: Depth + 1);
3854
3855 // NSZ cannot be applied in multiple use case (maybe it could if all uses
3856 // were known nsz)
3857 if (Value *Simplified = simplifyDemandedFPClassFabs(
3858 Known, Src: CI->getArgOperand(i: 0), DemandedMask, KnownSrc,
3859 /*NSZ=*/false))
3860 return Simplified;
3861 break;
3862 }
3863 case Intrinsic::copysign: {
3864 Value *Mag = CI->getArgOperand(i: 0);
3865 Value *Sign = CI->getArgOperand(i: 1);
3866 KnownFPClass KnownMag =
3867 computeKnownFPClass(V: Mag, InterestedClasses: fcAllFlags, SQ, Depth: Depth + 1);
3868
3869 // Rule out some cases by magnitude, which may help prove the sign bit is
3870 // one direction or the other.
3871 KnownMag.knownNot(RuleOut: ~llvm::unknown_sign(Mask: DemandedMask));
3872
3873 // Cannot use nsz in the multiple use case.
3874 if (Value *Simplified = simplifyDemandedFPClassCopysignMag(
3875 MagSrc: Mag, DemandedMask, KnownSrc: KnownMag, /*NSZ=*/false))
3876 return Simplified;
3877
3878 KnownFPClass KnownSign =
3879 computeKnownFPClass(V: Sign, InterestedClasses: fcAllFlags, SQ, Depth: Depth + 1);
3880
3881 if (FMF.noInfs())
3882 KnownSign.knownNot(RuleOut: fcInf);
3883 if (FMF.noNaNs())
3884 KnownSign.knownNot(RuleOut: fcNan);
3885
3886 if (KnownSign.getSignBit() && KnownMag.getSignBit() &&
3887 *KnownSign.getSignBit() == *KnownMag.getSignBit())
3888 return Mag;
3889
3890 Known = KnownFPClass::copysign(KnownMag, KnownSign);
3891 break;
3892 }
3893 case Intrinsic::maxnum:
3894 case Intrinsic::minnum:
3895 case Intrinsic::maximum:
3896 case Intrinsic::minimum:
3897 case Intrinsic::maximumnum:
3898 case Intrinsic::minimumnum: {
3899 KnownFPClass KnownRHS = computeKnownFPClass(V: CI->getArgOperand(i: 1),
3900 InterestedClasses: DemandedMask, SQ, Depth: Depth + 1);
3901 if (KnownRHS.isUnknown())
3902 return nullptr;
3903
3904 KnownFPClass KnownLHS = computeKnownFPClass(V: CI->getArgOperand(i: 0),
3905 InterestedClasses: DemandedMask, SQ, Depth: Depth + 1);
3906
3907 // Cannot use NSZ in the multiple use case.
3908 return simplifyDemandedFPClassMinMax(Known, IID, CI, DemandedMask,
3909 KnownLHS, KnownRHS, F,
3910 /*NSZ=*/false);
3911 }
3912 default:
3913 break;
3914 }
3915
3916 [[fallthrough]];
3917 }
3918 default:
3919 Known = computeKnownFPClass(V: I, InterestedClasses: DemandedMask, SQ, Depth: Depth + 1);
3920 Known.knownNot(RuleOut: ~DemandedMask);
3921 break;
3922 }
3923
3924 return getFPClassConstant(Ty: I->getType(), Mask: Known.getKnownFPClasses());
3925}
3926
3927bool InstCombinerImpl::SimplifyDemandedFPClass(Instruction *I, unsigned OpNo,
3928 FPClassTest DemandedMask,
3929 KnownFPClass &Known,
3930 const SimplifyQuery &SQ,
3931 unsigned Depth) {
3932 Use &U = I->getOperandUse(i: OpNo);
3933 Value *V = U.get();
3934 Type *VTy = V->getType();
3935
3936 if (DemandedMask == fcNone) {
3937 if (isa<PoisonValue>(Val: V))
3938 return false;
3939 replaceUse(U, NewValue: PoisonValue::get(T: VTy));
3940 return true;
3941 }
3942
3943 // Handle constant
3944 Instruction *VInst = dyn_cast<Instruction>(Val: V);
3945 if (!VInst) {
3946 // Handle constants and arguments
3947 Known = computeKnownFPClass(V, InterestedClasses: fcAllFlags, SQ, Depth);
3948 Known.knownNot(RuleOut: ~DemandedMask);
3949
3950 if (Known.getKnownFPClasses() == fcNone) {
3951 if (isa<PoisonValue>(Val: V))
3952 return false;
3953 replaceUse(U, NewValue: PoisonValue::get(T: VTy));
3954 return true;
3955 }
3956
3957 // Do not try to replace values which are already constants (unless we are
3958 // folding to poison). Doing so could promote poison elements to non-poison
3959 // constants.
3960 if (isa<Constant>(Val: V))
3961 return false;
3962
3963 Value *FoldedToConst = getFPClassConstant(Ty: VTy, Mask: Known.getKnownFPClasses());
3964 if (!FoldedToConst || FoldedToConst == V)
3965 return false;
3966
3967 replaceUse(U, NewValue: FoldedToConst);
3968 return true;
3969 }
3970
3971 if (Depth == MaxAnalysisRecursionDepth) {
3972 Known.knownNot(RuleOut: ~DemandedMask);
3973 return false;
3974 }
3975
3976 Value *NewVal;
3977
3978 if (VInst->hasOneUse()) {
3979 // If the instruction has one use, we can directly simplify it.
3980 NewVal = SimplifyDemandedUseFPClass(I: VInst, DemandedMask, Known, SQ, Depth);
3981 } else {
3982 // If there are multiple uses of this instruction, then we can simplify
3983 // VInst to some other value, but not modify the instruction.
3984 NewVal = SimplifyMultipleUseDemandedFPClass(I: VInst, DemandedMask, Known, SQ,
3985 Depth);
3986 }
3987
3988 if (!NewVal)
3989 return false;
3990 // Recursion through a PHI may have already replaced this use. If the
3991 // recursive simplification only changed V in place, do not restore it.
3992 if (NewVal == V && U.get() != V)
3993 return true;
3994 if (Instruction *OpInst = dyn_cast<Instruction>(Val&: U))
3995 salvageDebugInfo(I&: *OpInst);
3996
3997 replaceUse(U, NewValue: NewVal);
3998 return true;
3999}
4000