1//===- SelectionDAG.cpp - Implement the SelectionDAG data structures ------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements the SelectionDAG class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/CodeGen/SelectionDAG.h"
14#include "SDNodeDbgValue.h"
15#include "SelectionDAGOptions.h"
16#include "llvm/ADT/APFloat.h"
17#include "llvm/ADT/APInt.h"
18#include "llvm/ADT/APSInt.h"
19#include "llvm/ADT/ArrayRef.h"
20#include "llvm/ADT/BitVector.h"
21#include "llvm/ADT/DenseSet.h"
22#include "llvm/ADT/FoldingSet.h"
23#include "llvm/ADT/STLExtras.h"
24#include "llvm/ADT/SmallPtrSet.h"
25#include "llvm/ADT/SmallVector.h"
26#include "llvm/ADT/Twine.h"
27#include "llvm/Analysis/AliasAnalysis.h"
28#include "llvm/Analysis/MemoryLocation.h"
29#include "llvm/Analysis/TargetLibraryInfo.h"
30#include "llvm/Analysis/ValueTracking.h"
31#include "llvm/Analysis/VectorUtils.h"
32#include "llvm/BinaryFormat/Dwarf.h"
33#include "llvm/CodeGen/Analysis.h"
34#include "llvm/CodeGen/CodeGenCommonISel.h"
35#include "llvm/CodeGen/FunctionLoweringInfo.h"
36#include "llvm/CodeGen/ISDOpcodes.h"
37#include "llvm/CodeGen/MachineBasicBlock.h"
38#include "llvm/CodeGen/MachineConstantPool.h"
39#include "llvm/CodeGen/MachineFrameInfo.h"
40#include "llvm/CodeGen/MachineFunction.h"
41#include "llvm/CodeGen/MachineMemOperand.h"
42#include "llvm/CodeGen/RuntimeLibcallUtil.h"
43#include "llvm/CodeGen/SDPatternMatch.h"
44#include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
45#include "llvm/CodeGen/SelectionDAGNodes.h"
46#include "llvm/CodeGen/SelectionDAGTargetInfo.h"
47#include "llvm/CodeGen/TargetFrameLowering.h"
48#include "llvm/CodeGen/TargetLowering.h"
49#include "llvm/CodeGen/TargetRegisterInfo.h"
50#include "llvm/CodeGen/TargetSubtargetInfo.h"
51#include "llvm/CodeGen/ValueTypes.h"
52#include "llvm/CodeGenTypes/MachineValueType.h"
53#include "llvm/IR/Constant.h"
54#include "llvm/IR/Constants.h"
55#include "llvm/IR/DataLayout.h"
56#include "llvm/IR/DebugInfoMetadata.h"
57#include "llvm/IR/DebugLoc.h"
58#include "llvm/IR/DerivedTypes.h"
59#include "llvm/IR/Function.h"
60#include "llvm/IR/GlobalValue.h"
61#include "llvm/IR/MemCacheHint.h"
62#include "llvm/IR/Metadata.h"
63#include "llvm/IR/Type.h"
64#include "llvm/Support/Casting.h"
65#include "llvm/Support/CodeGen.h"
66#include "llvm/Support/Compiler.h"
67#include "llvm/Support/Debug.h"
68#include "llvm/Support/ErrorHandling.h"
69#include "llvm/Support/KnownBits.h"
70#include "llvm/Support/KnownFPClass.h"
71#include "llvm/Support/MathExtras.h"
72#include "llvm/Support/raw_ostream.h"
73#include "llvm/Target/TargetMachine.h"
74#include "llvm/Target/TargetOptions.h"
75#include "llvm/TargetParser/Triple.h"
76#include "llvm/Transforms/Utils/SizeOpts.h"
77#include <algorithm>
78#include <cassert>
79#include <cstdint>
80#include <cstdlib>
81#include <limits>
82#include <optional>
83#include <string>
84#include <utility>
85#include <vector>
86
87using namespace llvm;
88using namespace llvm::SDPatternMatch;
89
90/// makeVTList - Return an instance of the SDVTList struct initialized with the
91/// specified members.
92static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) {
93 SDVTList Res = {.VTs: VTs, .NumVTs: NumVTs};
94 return Res;
95}
96
97// Default null implementations of the callbacks.
98void SelectionDAG::DAGUpdateListener::NodeDeleted(SDNode*, SDNode*) {}
99void SelectionDAG::DAGUpdateListener::NodeUpdated(SDNode*) {}
100void SelectionDAG::DAGUpdateListener::NodeInserted(SDNode *) {}
101
102void SelectionDAG::DAGNodeDeletedListener::anchor() {}
103void SelectionDAG::DAGNodeInsertedListener::anchor() {}
104
105#define DEBUG_TYPE "selectiondag"
106
107static void NewSDValueDbgMsg(SDValue V, StringRef Msg, SelectionDAG *G) {
108 LLVM_DEBUG(dbgs() << Msg; V.getNode()->dump(G););
109}
110
111unsigned SelectionDAG::getHasPredecessorMaxSteps() const {
112 return getSelectionDAGOptions().has_predecessor_max_steps;
113}
114
115//===----------------------------------------------------------------------===//
116// ConstantFPSDNode Class
117//===----------------------------------------------------------------------===//
118
119/// isExactlyValue - We don't rely on operator== working on double values, as
120/// it returns true for things that are clearly not equal, like -0.0 and 0.0.
121/// As such, this method can be used to do an exact bit-for-bit comparison of
122/// two floating point values.
123bool ConstantFPSDNode::isExactlyValue(const APFloat& V) const {
124 return getValueAPF().bitwiseIsEqual(RHS: V);
125}
126
127bool ConstantFPSDNode::isValueValidForType(EVT VT,
128 const APFloat& Val) {
129 assert(VT.isFloatingPoint() && "Can only convert between FP types");
130
131 // convert modifies in place, so make a copy.
132 APFloat Val2 = APFloat(Val);
133 bool losesInfo;
134 (void)Val2.convert(ToSemantics: VT.getFltSemantics(), RM: APFloat::rmNearestTiesToEven,
135 losesInfo: &losesInfo);
136 return !losesInfo;
137}
138
139//===----------------------------------------------------------------------===//
140// ISD Namespace
141//===----------------------------------------------------------------------===//
142
143bool ISD::isConstantSplatVector(SDValue V, APInt &SplatVal) {
144 if (V.getOpcode() == ISD::SPLAT_VECTOR) {
145 if (auto OptAPInt = V.getOperand(i: 0)->bitcastToAPInt()) {
146 unsigned EltSize =
147 V.getValueType().getVectorElementType().getSizeInBits();
148 SplatVal = OptAPInt->trunc(width: EltSize);
149 return true;
150 }
151 }
152
153 auto *BV = dyn_cast<BuildVectorSDNode>(Val&: V);
154 if (!BV)
155 return false;
156
157 APInt SplatUndef;
158 unsigned SplatBitSize;
159 bool HasUndefs;
160 unsigned EltSize = V.getValueType().getVectorElementType().getSizeInBits();
161 // Endianness does not matter here. We are checking for a splat given the
162 // element size of the vector, and if we find such a splat for little endian
163 // layout, then that should be valid also for big endian (as the full vector
164 // size is known to be a multiple of the element size).
165 const bool IsBigEndian = false;
166 return BV->isConstantSplat(SplatValue&: SplatVal, SplatUndef, SplatBitSize, HasAnyUndefs&: HasUndefs,
167 MinSplatBits: EltSize, isBigEndian: IsBigEndian) &&
168 EltSize == SplatBitSize;
169}
170
171// FIXME: AllOnes and AllZeros duplicate a lot of code. Could these be
172// specializations of the more general isConstantSplatVector()?
173
174bool ISD::isConstantSplatVectorAllOnes(SDValue V, bool BuildVectorOnly) {
175 // Look through a bit convert.
176 while (V.getOpcode() == ISD::BITCAST)
177 V = V.getOperand(i: 0);
178
179 if (!BuildVectorOnly && V.getOpcode() == ISD::SPLAT_VECTOR) {
180 APInt SplatVal;
181 return isConstantSplatVector(V, SplatVal) && SplatVal.isAllOnes();
182 }
183
184 if (V.getOpcode() != ISD::BUILD_VECTOR)
185 return false;
186
187 unsigned i = 0, e = V.getNumOperands();
188
189 // Skip over all of the undef values.
190 while (i != e && V.getOperand(i).isUndef())
191 ++i;
192
193 // Do not accept an all-undef vector.
194 if (i == e) return false;
195
196 // Do not accept build_vectors that aren't all constants or which have non-~0
197 // elements. We have to be a bit careful here, as the type of the constant
198 // may not be the same as the type of the vector elements due to type
199 // legalization (the elements are promoted to a legal type for the target and
200 // a vector of a type may be legal when the base element type is not).
201 // We only want to check enough bits to cover the vector elements, because
202 // we care if the resultant vector is all ones, not whether the individual
203 // constants are.
204 SDValue NotZero = V.getOperand(i);
205 if (auto OptAPInt = NotZero->bitcastToAPInt()) {
206 unsigned EltSize = V.getValueType().getScalarSizeInBits();
207 if (OptAPInt->countr_one() < EltSize)
208 return false;
209 } else
210 return false;
211
212 // Okay, we have at least one ~0 value, check to see if the rest match or are
213 // undefs. Even with the above element type twiddling, this should be OK, as
214 // the same type legalization should have applied to all the elements.
215 for (++i; i != e; ++i)
216 if (V.getOperand(i) != NotZero && !V.getOperand(i).isUndef())
217 return false;
218 return true;
219}
220
221bool ISD::isConstantSplatVectorAllZeros(SDValue V, bool BuildVectorOnly) {
222 // Look through a bit convert.
223 while (V.getOpcode() == ISD::BITCAST)
224 V = V.getOperand(i: 0);
225
226 if (!BuildVectorOnly && V.getOpcode() == ISD::SPLAT_VECTOR) {
227 APInt SplatVal;
228 return isConstantSplatVector(V, SplatVal) && SplatVal.isZero();
229 }
230
231 if (V.getOpcode() != ISD::BUILD_VECTOR)
232 return false;
233
234 bool IsAllUndef = true;
235 for (const SDValue &Op : V->op_values()) {
236 if (Op.isUndef())
237 continue;
238 IsAllUndef = false;
239 // Do not accept build_vectors that aren't all constants or which have non-0
240 // elements. We have to be a bit careful here, as the type of the constant
241 // may not be the same as the type of the vector elements due to type
242 // legalization (the elements are promoted to a legal type for the target
243 // and a vector of a type may be legal when the base element type is not).
244 // We only want to check enough bits to cover the vector elements, because
245 // we care if the resultant vector is all zeros, not whether the individual
246 // constants are.
247 if (auto OptAPInt = Op->bitcastToAPInt()) {
248 unsigned EltSize = V.getValueType().getScalarSizeInBits();
249 if (OptAPInt->countr_zero() < EltSize)
250 return false;
251 } else
252 return false;
253 }
254
255 // Do not accept an all-undef vector.
256 if (IsAllUndef)
257 return false;
258 return true;
259}
260
261bool ISD::isBuildVectorAllOnes(SDValue V) {
262 return isConstantSplatVectorAllOnes(V, /*BuildVectorOnly*/ true);
263}
264
265bool ISD::isBuildVectorAllZeros(SDValue V) {
266 return isConstantSplatVectorAllZeros(V, /*BuildVectorOnly*/ true);
267}
268
269bool ISD::isBuildVectorOfConstantSDNodes(SDValue V) {
270 if (V.getOpcode() != ISD::BUILD_VECTOR)
271 return false;
272
273 for (const SDValue &Op : V->op_values()) {
274 if (Op.isUndef())
275 continue;
276 if (!isa<ConstantSDNode>(Val: Op))
277 return false;
278 }
279 return true;
280}
281
282bool ISD::isBuildVectorOfConstantFPSDNodes(SDValue V) {
283 if (V.getOpcode() != ISD::BUILD_VECTOR)
284 return false;
285
286 for (const SDValue &Op : V->op_values()) {
287 if (Op.isUndef())
288 continue;
289 if (!isa<ConstantFPSDNode>(Val: Op))
290 return false;
291 }
292 return true;
293}
294
295bool ISD::isVectorShrinkable(SDValue V, unsigned NewEltSize, bool Signed) {
296 assert(V.getValueType().isVector() && "Expected a vector!");
297
298 unsigned EltSize = V.getValueType().getScalarSizeInBits();
299 if (EltSize <= NewEltSize)
300 return false;
301
302 if (V.getOpcode() == ISD::ZERO_EXTEND) {
303 return (V.getOperand(i: 0).getValueType().getScalarSizeInBits() <=
304 NewEltSize) &&
305 !Signed;
306 }
307 if (V.getOpcode() == ISD::SIGN_EXTEND) {
308 return (V.getOperand(i: 0).getValueType().getScalarSizeInBits() <=
309 NewEltSize) &&
310 Signed;
311 }
312 if (V.getOpcode() != ISD::BUILD_VECTOR)
313 return false;
314
315 for (const SDValue &Op : V->op_values()) {
316 if (Op.isUndef())
317 continue;
318 if (!isa<ConstantSDNode>(Val: Op))
319 return false;
320
321 APInt C = Op->getAsAPIntVal().trunc(width: EltSize);
322 if (Signed && C.trunc(width: NewEltSize).sext(width: EltSize) != C)
323 return false;
324 if (!Signed && C.trunc(width: NewEltSize).zext(width: EltSize) != C)
325 return false;
326 }
327
328 return true;
329}
330
331bool ISD::allOperandsUndef(SDValue V) {
332 // Return false if the node has no operands.
333 // This is "logically inconsistent" with the definition of "all" but
334 // is probably the desired behavior.
335 if (V.getNumOperands() == 0)
336 return false;
337 return all_of(Range: V->op_values(), P: [](SDValue Op) { return Op.isUndef(); });
338}
339
340bool ISD::isFreezeUndef(SDValue V) {
341 return V.getOpcode() == ISD::FREEZE && V.getOperand(i: 0).isUndef();
342}
343
344template <typename ConstNodeType>
345bool ISD::matchUnaryPredicateImpl(SDValue Op, const APInt &DemandedElts,
346 std::function<bool(ConstNodeType *)> Match,
347 bool AllowUndefs, bool AllowTruncation) {
348 // FIXME: Add support for scalar UNDEF cases?
349 if (auto *C = dyn_cast<ConstNodeType>(Op))
350 return Match(C);
351
352 // FIXME: Add support for vector UNDEF cases?
353 if (ISD::BUILD_VECTOR != Op.getOpcode() &&
354 ISD::SPLAT_VECTOR != Op.getOpcode())
355 return false;
356
357 if (ISD::SPLAT_VECTOR == Op.getOpcode() && !DemandedElts)
358 return true;
359
360 EVT SVT = Op.getValueType().getScalarType();
361 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
362 if (ISD::SPLAT_VECTOR != Op.getOpcode() && !DemandedElts[i])
363 continue;
364
365 if (AllowUndefs && Op.getOperand(i).isUndef()) {
366 if (!Match(nullptr))
367 return false;
368 continue;
369 }
370
371 auto *Cst = dyn_cast<ConstNodeType>(Op.getOperand(i));
372 if (!Cst || (!AllowTruncation && Cst->getValueType(0) != SVT) ||
373 !Match(Cst))
374 return false;
375 }
376 return true;
377}
378// Build used template types.
379template bool ISD::matchUnaryPredicateImpl<ConstantSDNode>(
380 SDValue, const APInt &, std::function<bool(ConstantSDNode *)>, bool, bool);
381template bool ISD::matchUnaryPredicateImpl<ConstantFPSDNode>(
382 SDValue, const APInt &, std::function<bool(ConstantFPSDNode *)>, bool,
383 bool);
384
385bool ISD::matchBinaryPredicate(
386 SDValue LHS, SDValue RHS, const APInt &DemandedElts,
387 std::function<bool(ConstantSDNode *, ConstantSDNode *)> Match,
388 bool AllowUndefs, bool AllowTypeMismatch) {
389 if (!AllowTypeMismatch && LHS.getValueType() != RHS.getValueType())
390 return false;
391
392 // TODO: Add support for scalar UNDEF cases?
393 if (auto *LHSCst = dyn_cast<ConstantSDNode>(Val&: LHS))
394 if (auto *RHSCst = dyn_cast<ConstantSDNode>(Val&: RHS))
395 return Match(LHSCst, RHSCst);
396
397 // TODO: Add support for vector UNDEF cases?
398 if (LHS.getOpcode() != RHS.getOpcode() ||
399 (LHS.getOpcode() != ISD::BUILD_VECTOR &&
400 LHS.getOpcode() != ISD::SPLAT_VECTOR))
401 return false;
402
403 if (ISD::SPLAT_VECTOR == LHS.getOpcode() && !DemandedElts)
404 return true;
405
406 EVT SVT = LHS.getValueType().getScalarType();
407 for (unsigned i = 0, e = LHS.getNumOperands(); i != e; ++i) {
408 if (ISD::SPLAT_VECTOR != LHS.getOpcode() && !DemandedElts[i])
409 continue;
410 SDValue LHSOp = LHS.getOperand(i);
411 SDValue RHSOp = RHS.getOperand(i);
412 bool LHSUndef = AllowUndefs && LHSOp.isUndef();
413 bool RHSUndef = AllowUndefs && RHSOp.isUndef();
414 auto *LHSCst = dyn_cast<ConstantSDNode>(Val&: LHSOp);
415 auto *RHSCst = dyn_cast<ConstantSDNode>(Val&: RHSOp);
416 if ((!LHSCst && !LHSUndef) || (!RHSCst && !RHSUndef))
417 return false;
418 if (!AllowTypeMismatch && (LHSOp.getValueType() != SVT ||
419 LHSOp.getValueType() != RHSOp.getValueType()))
420 return false;
421 if (!Match(LHSCst, RHSCst))
422 return false;
423 }
424 return true;
425}
426
427ISD::NodeType ISD::getInverseMinMaxOpcode(unsigned MinMaxOpc) {
428 switch (MinMaxOpc) {
429 default:
430 llvm_unreachable("unrecognized opcode");
431 case ISD::UMIN:
432 return ISD::UMAX;
433 case ISD::UMAX:
434 return ISD::UMIN;
435 case ISD::SMIN:
436 return ISD::SMAX;
437 case ISD::SMAX:
438 return ISD::SMIN;
439 }
440}
441
442ISD::NodeType ISD::getOppositeSignednessMinMaxOpcode(unsigned MinMaxOpc) {
443 switch (MinMaxOpc) {
444 default:
445 llvm_unreachable("unrecognized min/max opcode");
446 case ISD::SMIN:
447 return ISD::UMIN;
448 case ISD::SMAX:
449 return ISD::UMAX;
450 case ISD::UMIN:
451 return ISD::SMIN;
452 case ISD::UMAX:
453 return ISD::SMAX;
454 }
455}
456
457ISD::NodeType ISD::getVecReduceBaseOpcode(unsigned VecReduceOpcode) {
458 switch (VecReduceOpcode) {
459 default:
460 llvm_unreachable("Expected VECREDUCE opcode");
461 case ISD::VECREDUCE_FADD:
462 case ISD::VECREDUCE_SEQ_FADD:
463 case ISD::VP_REDUCE_FADD:
464 case ISD::VP_REDUCE_SEQ_FADD:
465 return ISD::FADD;
466 case ISD::VECREDUCE_FMUL:
467 case ISD::VECREDUCE_SEQ_FMUL:
468 case ISD::VP_REDUCE_FMUL:
469 case ISD::VP_REDUCE_SEQ_FMUL:
470 return ISD::FMUL;
471 case ISD::VECREDUCE_ADD:
472 case ISD::VP_REDUCE_ADD:
473 return ISD::ADD;
474 case ISD::VECREDUCE_MUL:
475 case ISD::VP_REDUCE_MUL:
476 return ISD::MUL;
477 case ISD::VECREDUCE_AND:
478 case ISD::VP_REDUCE_AND:
479 return ISD::AND;
480 case ISD::VECREDUCE_OR:
481 case ISD::VP_REDUCE_OR:
482 return ISD::OR;
483 case ISD::VECREDUCE_XOR:
484 case ISD::VP_REDUCE_XOR:
485 return ISD::XOR;
486 case ISD::VECREDUCE_SMAX:
487 case ISD::VP_REDUCE_SMAX:
488 return ISD::SMAX;
489 case ISD::VECREDUCE_SMIN:
490 case ISD::VP_REDUCE_SMIN:
491 return ISD::SMIN;
492 case ISD::VECREDUCE_UMAX:
493 case ISD::VP_REDUCE_UMAX:
494 return ISD::UMAX;
495 case ISD::VECREDUCE_UMIN:
496 case ISD::VP_REDUCE_UMIN:
497 return ISD::UMIN;
498 case ISD::VECREDUCE_FMAX:
499 case ISD::VP_REDUCE_FMAX:
500 return ISD::FMAXNUM;
501 case ISD::VECREDUCE_FMIN:
502 case ISD::VP_REDUCE_FMIN:
503 return ISD::FMINNUM;
504 case ISD::VECREDUCE_FMAXIMUM:
505 case ISD::VP_REDUCE_FMAXIMUM:
506 return ISD::FMAXIMUM;
507 case ISD::VECREDUCE_FMINIMUM:
508 case ISD::VP_REDUCE_FMINIMUM:
509 return ISD::FMINIMUM;
510 case ISD::VECREDUCE_FMAXIMUMNUM:
511 return ISD::FMAXIMUMNUM;
512 case ISD::VECREDUCE_FMINIMUMNUM:
513 return ISD::FMINIMUMNUM;
514 }
515}
516
517ISD::NodeType ISD::getUnmaskedBinOpOpcode(unsigned MaskedOpc) {
518 switch (MaskedOpc) {
519 case ISD::MASKED_UDIV:
520 return ISD::UDIV;
521 case ISD::MASKED_SDIV:
522 return ISD::SDIV;
523 case ISD::MASKED_UREM:
524 return ISD::UREM;
525 case ISD::MASKED_SREM:
526 return ISD::SREM;
527 default:
528 llvm_unreachable("Expected masked binop opcode");
529 }
530}
531
532bool ISD::isVPOpcode(unsigned Opcode) {
533 switch (Opcode) {
534 default:
535 return false;
536#define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) \
537 case ISD::VPSD: \
538 return true;
539#include "llvm/IR/VPIntrinsics.def"
540 }
541}
542
543bool ISD::isVPBinaryOp(unsigned Opcode) {
544 switch (Opcode) {
545 default:
546 break;
547#define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) case ISD::VPSD:
548#define VP_PROPERTY_BINARYOP return true;
549#define END_REGISTER_VP_SDNODE(VPSD) break;
550#include "llvm/IR/VPIntrinsics.def"
551 }
552 return false;
553}
554
555bool ISD::isVPReduction(unsigned Opcode) {
556 switch (Opcode) {
557 default:
558 return false;
559 case ISD::VP_REDUCE_ADD:
560 case ISD::VP_REDUCE_MUL:
561 case ISD::VP_REDUCE_AND:
562 case ISD::VP_REDUCE_OR:
563 case ISD::VP_REDUCE_XOR:
564 case ISD::VP_REDUCE_SMAX:
565 case ISD::VP_REDUCE_SMIN:
566 case ISD::VP_REDUCE_UMAX:
567 case ISD::VP_REDUCE_UMIN:
568 case ISD::VP_REDUCE_FMAX:
569 case ISD::VP_REDUCE_FMIN:
570 case ISD::VP_REDUCE_FMAXIMUM:
571 case ISD::VP_REDUCE_FMINIMUM:
572 case ISD::VP_REDUCE_FADD:
573 case ISD::VP_REDUCE_FMUL:
574 case ISD::VP_REDUCE_SEQ_FADD:
575 case ISD::VP_REDUCE_SEQ_FMUL:
576 return true;
577 }
578}
579
580/// The operand position of the vector mask.
581std::optional<unsigned> ISD::getVPMaskIdx(unsigned Opcode) {
582 switch (Opcode) {
583 default:
584 return std::nullopt;
585#define BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, TDNAME, MASKPOS, ...) \
586 case ISD::VPSD: \
587 return MASKPOS;
588#include "llvm/IR/VPIntrinsics.def"
589 }
590}
591
592/// The operand position of the explicit vector length parameter.
593std::optional<unsigned> ISD::getVPExplicitVectorLengthIdx(unsigned Opcode) {
594 switch (Opcode) {
595 default:
596 return std::nullopt;
597#define BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, TDNAME, MASKPOS, EVLPOS) \
598 case ISD::VPSD: \
599 return EVLPOS;
600#include "llvm/IR/VPIntrinsics.def"
601 }
602}
603
604std::optional<unsigned> ISD::getBaseOpcodeForVP(unsigned VPOpcode,
605 bool hasFPExcept) {
606 // FIXME: Return strict opcodes in case of fp exceptions.
607 switch (VPOpcode) {
608 default:
609 return std::nullopt;
610#define BEGIN_REGISTER_VP_SDNODE(VPOPC, ...) case ISD::VPOPC:
611#define VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) return ISD::SDOPC;
612#define END_REGISTER_VP_SDNODE(VPOPC) break;
613#include "llvm/IR/VPIntrinsics.def"
614 }
615 return std::nullopt;
616}
617
618std::optional<unsigned> ISD::getVPForBaseOpcode(unsigned Opcode) {
619 switch (Opcode) {
620 default:
621 return std::nullopt;
622#define BEGIN_REGISTER_VP_SDNODE(VPOPC, ...) break;
623#define VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) case ISD::SDOPC:
624#define END_REGISTER_VP_SDNODE(VPOPC) return ISD::VPOPC;
625#include "llvm/IR/VPIntrinsics.def"
626 }
627}
628
629ISD::NodeType ISD::getExtForLoadExtType(bool IsFP, ISD::LoadExtType ExtType) {
630 switch (ExtType) {
631 case ISD::EXTLOAD:
632 return IsFP ? ISD::FP_EXTEND : ISD::ANY_EXTEND;
633 case ISD::SEXTLOAD:
634 return ISD::SIGN_EXTEND;
635 case ISD::ZEXTLOAD:
636 return ISD::ZERO_EXTEND;
637 default:
638 break;
639 }
640
641 llvm_unreachable("Invalid LoadExtType");
642}
643
644ISD::CondCode ISD::getSetCCSwappedOperands(ISD::CondCode Operation) {
645 // To perform this operation, we just need to swap the L and G bits of the
646 // operation.
647 unsigned OldL = (Operation >> 2) & 1;
648 unsigned OldG = (Operation >> 1) & 1;
649 return ISD::CondCode((Operation & ~6) | // Keep the N, U, E bits
650 (OldL << 1) | // New G bit
651 (OldG << 2)); // New L bit.
652}
653
654static ISD::CondCode getSetCCInverseImpl(ISD::CondCode Op, bool isIntegerLike) {
655 unsigned Operation = Op;
656 if (isIntegerLike)
657 Operation ^= 7; // Flip L, G, E bits, but not U.
658 else
659 Operation ^= 15; // Flip all of the condition bits.
660
661 if (Operation > ISD::SETTRUE2)
662 Operation &= ~8; // Don't let N and U bits get set.
663
664 return ISD::CondCode(Operation);
665}
666
667ISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, EVT Type) {
668 return getSetCCInverseImpl(Op, isIntegerLike: Type.isInteger());
669}
670
671ISD::CondCode ISD::GlobalISel::getSetCCInverse(ISD::CondCode Op,
672 bool isIntegerLike) {
673 return getSetCCInverseImpl(Op, isIntegerLike);
674}
675
676/// For an integer comparison, return 1 if the comparison is a signed operation
677/// and 2 if the result is an unsigned comparison. Return zero if the operation
678/// does not depend on the sign of the input (setne and seteq).
679static int isSignedOp(ISD::CondCode Opcode) {
680 switch (Opcode) {
681 default: llvm_unreachable("Illegal integer setcc operation!");
682 case ISD::SETEQ:
683 case ISD::SETNE: return 0;
684 case ISD::SETLT:
685 case ISD::SETLE:
686 case ISD::SETGT:
687 case ISD::SETGE: return 1;
688 case ISD::SETULT:
689 case ISD::SETULE:
690 case ISD::SETUGT:
691 case ISD::SETUGE: return 2;
692 }
693}
694
695ISD::CondCode ISD::getSetCCOrOperation(ISD::CondCode Op1, ISD::CondCode Op2,
696 EVT Type) {
697 bool IsInteger = Type.isInteger();
698 if (IsInteger && (isSignedOp(Opcode: Op1) | isSignedOp(Opcode: Op2)) == 3)
699 // Cannot fold a signed integer setcc with an unsigned integer setcc.
700 return ISD::SETCC_INVALID;
701
702 unsigned Op = Op1 | Op2; // Combine all of the condition bits.
703
704 // If the N and U bits get set, then the resultant comparison DOES suddenly
705 // care about orderedness, and it is true when ordered.
706 if (Op > ISD::SETTRUE2)
707 Op &= ~16; // Clear the U bit if the N bit is set.
708
709 // Canonicalize illegal integer setcc's.
710 if (IsInteger && Op == ISD::SETUNE) // e.g. SETUGT | SETULT
711 Op = ISD::SETNE;
712
713 return ISD::CondCode(Op);
714}
715
716ISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2,
717 EVT Type) {
718 bool IsInteger = Type.isInteger();
719 if (IsInteger && (isSignedOp(Opcode: Op1) | isSignedOp(Opcode: Op2)) == 3)
720 // Cannot fold a signed setcc with an unsigned setcc.
721 return ISD::SETCC_INVALID;
722
723 // Combine all of the condition bits.
724 ISD::CondCode Result = ISD::CondCode(Op1 & Op2);
725
726 // Canonicalize illegal integer setcc's.
727 if (IsInteger) {
728 switch (Result) {
729 default: break;
730 case ISD::SETUO : Result = ISD::SETFALSE; break; // SETUGT & SETULT
731 case ISD::SETOEQ: // SETEQ & SETU[LG]E
732 case ISD::SETUEQ: Result = ISD::SETEQ ; break; // SETUGE & SETULE
733 case ISD::SETOLT: Result = ISD::SETULT ; break; // SETULT & SETNE
734 case ISD::SETOGT: Result = ISD::SETUGT ; break; // SETUGT & SETNE
735 }
736 }
737
738 return Result;
739}
740
741//===----------------------------------------------------------------------===//
742// SDNode Key Support
743//===----------------------------------------------------------------------===//
744
745/// If this is an SDNode with special info, add this info to the NodeID data.
746/// MorphNodeTo passes the opcode \p N is becoming, so \p Opc may differ from
747/// N->getOpcode() and a case may only cast to a class \p N already is.
748static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N,
749 unsigned Opc) {
750 // Machine nodes are uniqued by opcode, value types and operands alone, as in
751 // getMachineNode; MorphNodeTo's clearMemRefs overwrites the fields below.
752 if (SDNode::isMachineOpcode(Opc))
753 return;
754
755 switch (Opc) {
756 case ISD::TargetExternalSymbol:
757 case ISD::ExternalSymbol:
758 case ISD::MCSymbol:
759 llvm_unreachable("Should only be used on nodes with operands");
760 default: break; // Normal nodes don't need extra info.
761 case ISD::TargetConstant:
762 case ISD::Constant: {
763 const ConstantSDNode *C = cast<ConstantSDNode>(Val: N);
764 ID.AddPointer(Ptr: C->getConstantIntValue());
765 ID.AddBoolean(B: C->isOpaque());
766 break;
767 }
768 case ISD::TargetConstantFP:
769 case ISD::ConstantFP:
770 ID.AddPointer(Ptr: cast<ConstantFPSDNode>(Val: N)->getConstantFPValue());
771 break;
772 case ISD::TargetGlobalAddress:
773 case ISD::GlobalAddress:
774 case ISD::TargetGlobalTLSAddress:
775 case ISD::GlobalTLSAddress: {
776 const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Val: N);
777 ID.AddPointer(Ptr: GA->getGlobal());
778 ID.AddInteger(I: GA->getOffset());
779 ID.AddInteger(I: GA->getTargetFlags());
780 break;
781 }
782 case ISD::BasicBlock:
783 ID.AddPointer(Ptr: cast<BasicBlockSDNode>(Val: N)->getBasicBlock());
784 break;
785 case ISD::EH_LABEL:
786 case ISD::ANNOTATION_LABEL:
787 ID.AddPointer(Ptr: cast<LabelSDNode>(Val: N)->getLabel());
788 break;
789 case ISD::DEACTIVATION_SYMBOL:
790 ID.AddPointer(Ptr: cast<DeactivationSymbolSDNode>(Val: N)->getGlobal());
791 break;
792 case ISD::Register:
793 ID.AddInteger(I: cast<RegisterSDNode>(Val: N)->getReg().id());
794 break;
795 case ISD::RegisterMask:
796 ID.AddPointer(Ptr: cast<RegisterMaskSDNode>(Val: N)->getRegMask());
797 break;
798 case ISD::SRCVALUE:
799 ID.AddPointer(Ptr: cast<SrcValueSDNode>(Val: N)->getValue());
800 break;
801 case ISD::FrameIndex:
802 case ISD::TargetFrameIndex:
803 ID.AddInteger(I: cast<FrameIndexSDNode>(Val: N)->getIndex());
804 break;
805 case ISD::PSEUDO_PROBE:
806 ID.AddInteger(I: cast<PseudoProbeSDNode>(Val: N)->getGuid());
807 ID.AddInteger(I: cast<PseudoProbeSDNode>(Val: N)->getIndex());
808 ID.AddInteger(I: cast<PseudoProbeSDNode>(Val: N)->getAttributes());
809 break;
810 case ISD::JumpTable:
811 case ISD::TargetJumpTable:
812 ID.AddInteger(I: cast<JumpTableSDNode>(Val: N)->getIndex());
813 ID.AddInteger(I: cast<JumpTableSDNode>(Val: N)->getTargetFlags());
814 break;
815 case ISD::ConstantPool:
816 case ISD::TargetConstantPool: {
817 const ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Val: N);
818 ID.AddInteger(I: CP->getAlign().value());
819 ID.AddInteger(I: CP->getOffset());
820 if (CP->isMachineConstantPoolEntry())
821 CP->getMachineCPVal()->addSelectionDAGCSEId(ID);
822 else
823 ID.AddPointer(Ptr: CP->getConstVal());
824 ID.AddInteger(I: CP->getTargetFlags());
825 break;
826 }
827 case ISD::TargetIndex: {
828 const TargetIndexSDNode *TI = cast<TargetIndexSDNode>(Val: N);
829 ID.AddInteger(I: TI->getIndex());
830 ID.AddInteger(I: TI->getOffset());
831 ID.AddInteger(I: TI->getTargetFlags());
832 break;
833 }
834 case ISD::LOAD: {
835 const LoadSDNode *LD = cast<LoadSDNode>(Val: N);
836 ID.AddInteger(I: LD->getMemoryVT().getRawBits());
837 ID.AddInteger(I: LD->getRawSubclassData());
838 ID.AddInteger(I: LD->getPointerInfo().getAddrSpace());
839 ID.AddInteger(I: LD->getMemOperand()->getFlags());
840 break;
841 }
842 case ISD::STORE: {
843 const StoreSDNode *ST = cast<StoreSDNode>(Val: N);
844 ID.AddInteger(I: ST->getMemoryVT().getRawBits());
845 ID.AddInteger(I: ST->getRawSubclassData());
846 ID.AddInteger(I: ST->getPointerInfo().getAddrSpace());
847 ID.AddInteger(I: ST->getMemOperand()->getFlags());
848 break;
849 }
850 case ISD::VP_LOAD: {
851 const VPLoadSDNode *ELD = cast<VPLoadSDNode>(Val: N);
852 ID.AddInteger(I: ELD->getMemoryVT().getRawBits());
853 ID.AddInteger(I: ELD->getRawSubclassData());
854 ID.AddInteger(I: ELD->getPointerInfo().getAddrSpace());
855 ID.AddInteger(I: ELD->getMemOperand()->getFlags());
856 break;
857 }
858 case ISD::VP_LOAD_FF: {
859 const auto *LD = cast<VPLoadFFSDNode>(Val: N);
860 ID.AddInteger(I: LD->getMemoryVT().getRawBits());
861 ID.AddInteger(I: LD->getRawSubclassData());
862 ID.AddInteger(I: LD->getPointerInfo().getAddrSpace());
863 ID.AddInteger(I: LD->getMemOperand()->getFlags());
864 break;
865 }
866 case ISD::VP_STORE: {
867 const VPStoreSDNode *EST = cast<VPStoreSDNode>(Val: N);
868 ID.AddInteger(I: EST->getMemoryVT().getRawBits());
869 ID.AddInteger(I: EST->getRawSubclassData());
870 ID.AddInteger(I: EST->getPointerInfo().getAddrSpace());
871 ID.AddInteger(I: EST->getMemOperand()->getFlags());
872 break;
873 }
874 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD: {
875 const VPStridedLoadSDNode *SLD = cast<VPStridedLoadSDNode>(Val: N);
876 ID.AddInteger(I: SLD->getMemoryVT().getRawBits());
877 ID.AddInteger(I: SLD->getRawSubclassData());
878 ID.AddInteger(I: SLD->getPointerInfo().getAddrSpace());
879 break;
880 }
881 case ISD::EXPERIMENTAL_VP_STRIDED_STORE: {
882 const VPStridedStoreSDNode *SST = cast<VPStridedStoreSDNode>(Val: N);
883 ID.AddInteger(I: SST->getMemoryVT().getRawBits());
884 ID.AddInteger(I: SST->getRawSubclassData());
885 ID.AddInteger(I: SST->getPointerInfo().getAddrSpace());
886 break;
887 }
888 case ISD::VP_GATHER: {
889 const VPGatherSDNode *EG = cast<VPGatherSDNode>(Val: N);
890 ID.AddInteger(I: EG->getMemoryVT().getRawBits());
891 ID.AddInteger(I: EG->getRawSubclassData());
892 ID.AddInteger(I: EG->getPointerInfo().getAddrSpace());
893 ID.AddInteger(I: EG->getMemOperand()->getFlags());
894 break;
895 }
896 case ISD::VP_SCATTER: {
897 const VPScatterSDNode *ES = cast<VPScatterSDNode>(Val: N);
898 ID.AddInteger(I: ES->getMemoryVT().getRawBits());
899 ID.AddInteger(I: ES->getRawSubclassData());
900 ID.AddInteger(I: ES->getPointerInfo().getAddrSpace());
901 ID.AddInteger(I: ES->getMemOperand()->getFlags());
902 break;
903 }
904 case ISD::MLOAD: {
905 const MaskedLoadSDNode *MLD = cast<MaskedLoadSDNode>(Val: N);
906 ID.AddInteger(I: MLD->getMemoryVT().getRawBits());
907 ID.AddInteger(I: MLD->getRawSubclassData());
908 ID.AddInteger(I: MLD->getPointerInfo().getAddrSpace());
909 ID.AddInteger(I: MLD->getMemOperand()->getFlags());
910 break;
911 }
912 case ISD::MSTORE: {
913 const MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(Val: N);
914 ID.AddInteger(I: MST->getMemoryVT().getRawBits());
915 ID.AddInteger(I: MST->getRawSubclassData());
916 ID.AddInteger(I: MST->getPointerInfo().getAddrSpace());
917 ID.AddInteger(I: MST->getMemOperand()->getFlags());
918 break;
919 }
920 case ISD::MGATHER: {
921 const MaskedGatherSDNode *MG = cast<MaskedGatherSDNode>(Val: N);
922 ID.AddInteger(I: MG->getMemoryVT().getRawBits());
923 ID.AddInteger(I: MG->getRawSubclassData());
924 ID.AddInteger(I: MG->getPointerInfo().getAddrSpace());
925 ID.AddInteger(I: MG->getMemOperand()->getFlags());
926 break;
927 }
928 case ISD::MSCATTER:
929 case ISD::GET_FPENV_MEM:
930 case ISD::SET_FPENV_MEM:
931 case ISD::EXPERIMENTAL_VECTOR_HISTOGRAM: {
932 const MemSDNode *M = cast<MemSDNode>(Val: N);
933 ID.AddInteger(I: M->getMemoryVT().getRawBits());
934 ID.AddInteger(I: M->getRawSubclassData());
935 ID.AddInteger(I: M->getPointerInfo().getAddrSpace());
936 ID.AddInteger(I: M->getMemOperand()->getFlags());
937 break;
938 }
939 case ISD::VECTOR_SHUFFLE: {
940 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Val: N)->getMask();
941 for (int M : Mask)
942 ID.AddInteger(I: M);
943 break;
944 }
945 case ISD::ADDRSPACECAST: {
946 const AddrSpaceCastSDNode *ASC = cast<AddrSpaceCastSDNode>(Val: N);
947 ID.AddInteger(I: ASC->getSrcAddressSpace());
948 ID.AddInteger(I: ASC->getDestAddressSpace());
949 break;
950 }
951 case ISD::TargetBlockAddress:
952 case ISD::BlockAddress: {
953 const BlockAddressSDNode *BA = cast<BlockAddressSDNode>(Val: N);
954 ID.AddPointer(Ptr: BA->getBlockAddress());
955 ID.AddInteger(I: BA->getOffset());
956 ID.AddInteger(I: BA->getTargetFlags());
957 break;
958 }
959 case ISD::AssertAlign:
960 ID.AddInteger(I: cast<AssertAlignSDNode>(Val: N)->getAlign().value());
961 break;
962 case ISD::PREFETCH:
963 case ISD::INTRINSIC_VOID:
964 case ISD::INTRINSIC_W_CHAIN:
965 // Handled by MemIntrinsicSDNode check after the switch.
966 break;
967 case ISD::MDNODE_SDNODE:
968 ID.AddPointer(Ptr: cast<MDNodeSDNode>(Val: N)->getMD());
969 break;
970 } // end switch (Opc)
971
972 if (auto *AT = dyn_cast<AtomicSDNode>(Val: N)) {
973 ID.AddInteger(I: AT->getMemoryVT().getRawBits());
974 ID.AddInteger(I: AT->getRawSubclassData());
975 ID.AddInteger(I: AT->getPointerInfo().getAddrSpace());
976 ID.AddInteger(I: AT->getMemOperand()->getFlags());
977 }
978
979 // MemIntrinsic nodes could also have subclass data, address spaces, and flags
980 // to check.
981 if (auto *MN = dyn_cast<MemIntrinsicSDNode>(Val: N)) {
982 ID.AddInteger(I: MN->getRawSubclassData());
983 ID.AddInteger(I: MN->getMemoryVT().getRawBits());
984 for (const MachineMemOperand *MMO : MN->memoperands()) {
985 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
986 ID.AddInteger(I: MMO->getFlags());
987 }
988 }
989}
990
991SDNodeKey::SDNodeKey(const SDNode &N)
992 : Opcode(N.getOpcode()), VTs(N.getVTList().VTs),
993 OpStorage(N.op_begin(), N.op_end()) {
994 Ops = OpStorage;
995 AddNodeIDCustom(ID&: Tail, N: &N, Opc: N.getOpcode());
996}
997
998#ifndef NDEBUG
999/// Serialize a node the way SDNodes were uniqued before SDNodeKey, to
1000/// cross-check the typed comparison against. Deliberately reads the node
1001/// rather than going through SDNodeKey, so the key constructor is checked too
1002/// and not used as its own oracle.
1003static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) {
1004 ID.AddInteger(N->getOpcode());
1005 ID.AddPointer(N->getVTList().VTs);
1006 for (const SDUse &Op : N->ops()) {
1007 ID.AddPointer(Op.getNode());
1008 ID.AddInteger(Op.getResNo());
1009 }
1010 AddNodeIDCustom(ID, N, N->getOpcode());
1011}
1012
1013static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNodeKey &Key) {
1014 ID.AddInteger(Key.Opcode);
1015 ID.AddPointer(Key.VTs);
1016 for (const SDValue &Op : Key.Ops) {
1017 ID.AddPointer(Op.getNode());
1018 ID.AddInteger(Op.getResNo());
1019 }
1020 ID.AddNodeID(Key.Tail);
1021}
1022#endif
1023
1024static bool keyMatches(const SDNodeKey &Key, const SDNode &N) {
1025 if (N.getOpcode() != Key.Opcode || N.getVTList().VTs != Key.VTs)
1026 return false;
1027 if (!llvm::equal(LRange: N.ops(), RRange: Key.Ops))
1028 return false;
1029 // Build only what AddNodeIDCustom would have added, which is nothing for
1030 // most opcodes, rather than the whole profile.
1031 FoldingSetNodeID Tail;
1032 AddNodeIDCustom(ID&: Tail, N: &N, Opc: N.getOpcode());
1033 return Tail == Key.Tail;
1034}
1035
1036bool SDNodeKeyInfo::isEqual(const SDNodeKey &Key, const SDNode &N) {
1037 bool Result = keyMatches(Key, N);
1038
1039#ifndef NDEBUG
1040 FoldingSetNodeID NodeProfile, KeyProfile;
1041 AddNodeIDNode(NodeProfile, &N);
1042 AddNodeIDNode(KeyProfile, Key);
1043 assert((NodeProfile == KeyProfile) == Result &&
1044 "SDNodeKey equality disagrees with profile");
1045#endif
1046 return Result;
1047}
1048
1049//===----------------------------------------------------------------------===//
1050// SelectionDAG Class
1051//===----------------------------------------------------------------------===//
1052
1053/// doNotCSE - Return true if CSE should not be performed for this node.
1054static bool doNotCSE(SDNode *N) {
1055 if (N->getValueType(ResNo: 0) == MVT::Glue)
1056 return true; // Never CSE anything that produces a glue result.
1057
1058 switch (N->getOpcode()) {
1059 default: break;
1060 case ISD::HANDLENODE:
1061 case ISD::EH_LABEL:
1062 return true; // Never CSE these nodes.
1063 }
1064
1065 // Check that remaining values produced are not flags.
1066 for (unsigned i = 1, e = N->getNumValues(); i != e; ++i)
1067 if (N->getValueType(ResNo: i) == MVT::Glue)
1068 return true; // Never CSE anything that produces a glue result.
1069
1070 return false;
1071}
1072
1073/// Construct a DemandedElts mask which demands all elements of \p V.
1074/// If \p V is not a fixed-length vector, then this will return a single bit.
1075static APInt getDemandAllEltsMask(SDValue V) {
1076 EVT VT = V.getValueType();
1077 // Since the number of lanes in a scalable vector is unknown at compile time,
1078 // we track one bit which is implicitly broadcast to all lanes. This means
1079 // that all lanes in a scalable vector are considered demanded.
1080 return VT.isFixedLengthVector() ? APInt::getAllOnes(numBits: VT.getVectorNumElements())
1081 : APInt(1, 1);
1082}
1083
1084/// RemoveDeadNodes - This method deletes all unreachable nodes in the
1085/// SelectionDAG.
1086void SelectionDAG::RemoveDeadNodes() {
1087 // Create a dummy node (which is not added to allnodes), that adds a reference
1088 // to the root node, preventing it from being deleted.
1089 HandleSDNode Dummy(getRoot());
1090
1091 SmallVector<SDNode*, 128> DeadNodes;
1092
1093 // Add all obviously-dead nodes to the DeadNodes worklist.
1094 for (SDNode &Node : allnodes())
1095 if (Node.use_empty())
1096 DeadNodes.push_back(Elt: &Node);
1097
1098 RemoveDeadNodes(DeadNodes);
1099
1100 // If the root changed (e.g. it was a dead load, update the root).
1101 setRoot(Dummy.getValue());
1102}
1103
1104/// RemoveDeadNodes - This method deletes the unreachable nodes in the
1105/// given list, and any nodes that become unreachable as a result.
1106void SelectionDAG::RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes) {
1107
1108 // Process the worklist, deleting the nodes and adding their uses to the
1109 // worklist.
1110 while (!DeadNodes.empty()) {
1111 SDNode *N = DeadNodes.pop_back_val();
1112 // Skip to next node if we've already managed to delete the node. This could
1113 // happen if replacing a node causes a node previously added to the node to
1114 // be deleted.
1115 if (N->getOpcode() == ISD::DELETED_NODE)
1116 continue;
1117
1118 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1119 DUL->NodeDeleted(N, nullptr);
1120
1121 // Take the node out of the appropriate CSE map.
1122 RemoveNodeFromCSEMaps(N);
1123
1124 // Next, brutally remove the operand list. This is safe to do, as there are
1125 // no cycles in the graph.
1126 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
1127 SDUse &Use = *I++;
1128 SDNode *Operand = Use.getNode();
1129 Use.set(SDValue());
1130
1131 // Now that we removed this operand, see if there are no uses of it left.
1132 if (Operand->use_empty())
1133 DeadNodes.push_back(Elt: Operand);
1134 }
1135
1136 DeallocateNode(N);
1137 }
1138}
1139
1140void SelectionDAG::RemoveDeadNode(SDNode *N){
1141 SmallVector<SDNode*, 16> DeadNodes(1, N);
1142
1143 // Create a dummy node that adds a reference to the root node, preventing
1144 // it from being deleted. (This matters if the root is an operand of the
1145 // dead node.)
1146 HandleSDNode Dummy(getRoot());
1147
1148 RemoveDeadNodes(DeadNodes);
1149}
1150
1151void SelectionDAG::DeleteNode(SDNode *N) {
1152 // First take this out of the appropriate CSE map.
1153 RemoveNodeFromCSEMaps(N);
1154
1155 // Finally, remove uses due to operands of this node, remove from the
1156 // AllNodes list, and delete the node.
1157 DeleteNodeNotInCSEMaps(N);
1158}
1159
1160void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) {
1161 assert(N->getIterator() != AllNodes.begin() &&
1162 "Cannot delete the entry node!");
1163 assert(N->use_empty() && "Cannot delete a node that is not dead!");
1164
1165 // Drop all of the operands and decrement used node's use counts.
1166 N->DropOperands();
1167
1168 DeallocateNode(N);
1169}
1170
1171void SDDbgInfo::add(SDDbgValue *V, bool isParameter) {
1172 assert(!(V->isVariadic() && isParameter));
1173 if (isParameter)
1174 ByvalParmDbgValues.push_back(Elt: V);
1175 else
1176 DbgValues.push_back(Elt: V);
1177 for (const SDNode *Node : V->getSDNodes())
1178 if (Node)
1179 DbgValMap[Node].push_back(Elt: V);
1180}
1181
1182void SDDbgInfo::erase(const SDNode *Node) {
1183 DbgValMapType::iterator I = DbgValMap.find(Val: Node);
1184 if (I == DbgValMap.end())
1185 return;
1186 for (auto &Val: I->second)
1187 Val->setIsInvalidated();
1188 DbgValMap.erase(I);
1189}
1190
1191void SelectionDAG::DeallocateNode(SDNode *N) {
1192 // If we have operands, deallocate them.
1193 removeOperands(Node: N);
1194
1195 NodeAllocator.Deallocate(E: AllNodes.remove(IT: N));
1196
1197 // Set the opcode to DELETED_NODE to help catch bugs when node
1198 // memory is reallocated.
1199 // FIXME: There are places in SDag that have grown a dependency on the opcode
1200 // value in the released node.
1201 __asan_unpoison_memory_region(&N->NodeType, sizeof(N->NodeType));
1202 N->NodeType = ISD::DELETED_NODE;
1203
1204 // If any of the SDDbgValue nodes refer to this SDNode, invalidate
1205 // them and forget about that node.
1206 DbgInfo->erase(Node: N);
1207
1208 // Invalidate extra info.
1209 SDEI.erase(Val: N);
1210}
1211
1212#ifndef NDEBUG
1213/// VerifySDNode - Check the given SDNode. Aborts if it is invalid.
1214void SelectionDAG::verifyNode(SDNode *N) const {
1215 switch (N->getOpcode()) {
1216 default:
1217 if (N->isTargetOpcode())
1218 getSelectionDAGInfo().verifyTargetNode(*this, N);
1219 break;
1220 case ISD::BUILD_PAIR: {
1221 EVT VT = N->getValueType(0);
1222 assert(N->getNumValues() == 1 && "Too many results!");
1223 assert(!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) &&
1224 "Wrong return type!");
1225 assert(N->getNumOperands() == 2 && "Wrong number of operands!");
1226 assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() &&
1227 "Mismatched operand types!");
1228 assert(N->getOperand(0).getValueType().isInteger() == VT.isInteger() &&
1229 "Wrong operand type!");
1230 assert(VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() &&
1231 "Wrong return type size");
1232 break;
1233 }
1234 case ISD::BUILD_VECTOR: {
1235 assert(N->getNumValues() == 1 && "Too many results!");
1236 assert(N->getValueType(0).isVector() && "Wrong return type!");
1237 assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() &&
1238 "Wrong number of operands!");
1239 EVT EltVT = N->getValueType(0).getVectorElementType();
1240 for (const SDUse &Op : N->ops()) {
1241 assert((Op.getValueType() == EltVT ||
1242 (EltVT.isInteger() && Op.getValueType().isInteger() &&
1243 EltVT.bitsLE(Op.getValueType()))) &&
1244 "Wrong operand type!");
1245 assert(Op.getValueType() == N->getOperand(0).getValueType() &&
1246 "Operands must all have the same type");
1247 }
1248 break;
1249 }
1250 case ISD::SADDO:
1251 case ISD::UADDO:
1252 case ISD::SSUBO:
1253 case ISD::USUBO:
1254 assert(N->getNumValues() == 2 && "Wrong number of results!");
1255 assert(N->getVTList().NumVTs == 2 && N->getNumOperands() == 2 &&
1256 "Invalid add/sub overflow op!");
1257 assert(N->getVTList().VTs[0].isInteger() &&
1258 N->getVTList().VTs[1].isInteger() &&
1259 N->getOperand(0).getValueType() == N->getOperand(1).getValueType() &&
1260 N->getOperand(0).getValueType() == N->getVTList().VTs[0] &&
1261 "Binary operator types must match!");
1262 break;
1263 }
1264}
1265#endif // NDEBUG
1266
1267/// Insert a newly allocated node into the DAG.
1268///
1269/// Handles insertion into the all nodes list and CSE map, as well as
1270/// verification and other common operations when a new node is allocated.
1271void SelectionDAG::InsertNode(SDNode *N) {
1272 AllNodes.push_back(val: N);
1273#ifndef NDEBUG
1274 N->PersistentId = NextPersistentId++;
1275 verifyNode(N);
1276#endif
1277 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1278 DUL->NodeInserted(N);
1279}
1280
1281/// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that
1282/// correspond to it. This is useful when we're about to delete or repurpose
1283/// the node. We don't want future request for structurally identical nodes
1284/// to return N anymore.
1285bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) {
1286 bool Erased = false;
1287 switch (N->getOpcode()) {
1288 case ISD::HANDLENODE: return false; // noop.
1289 case ISD::CONDCODE:
1290 assert(CondCodeNodes[cast<CondCodeSDNode>(N)->get()] &&
1291 "Cond code doesn't exist!");
1292 Erased = CondCodeNodes[cast<CondCodeSDNode>(Val: N)->get()] != nullptr;
1293 CondCodeNodes[cast<CondCodeSDNode>(Val: N)->get()] = nullptr;
1294 break;
1295 case ISD::ExternalSymbol:
1296 Erased = ExternalSymbols.erase(Key: cast<ExternalSymbolSDNode>(Val: N)->getSymbol());
1297 break;
1298 case ISD::TargetExternalSymbol: {
1299 ExternalSymbolSDNode *ESN = cast<ExternalSymbolSDNode>(Val: N);
1300 Erased = TargetExternalSymbols.erase(x: std::pair<std::string, unsigned>(
1301 ESN->getSymbol(), ESN->getTargetFlags()));
1302 break;
1303 }
1304 case ISD::MCSymbol: {
1305 auto *MCSN = cast<MCSymbolSDNode>(Val: N);
1306 Erased = MCSymbols.erase(Val: MCSN->getMCSymbol());
1307 break;
1308 }
1309 case ISD::VALUETYPE: {
1310 EVT VT = cast<VTSDNode>(Val: N)->getVT();
1311 if (VT.isExtended()) {
1312 Erased = ExtendedValueTypeNodes.erase(x: VT);
1313 } else {
1314 Erased = ValueTypeNodes[VT.getSimpleVT().SimpleTy] != nullptr;
1315 ValueTypeNodes[VT.getSimpleVT().SimpleTy] = nullptr;
1316 }
1317 break;
1318 }
1319 default:
1320 // Remove it from the CSE Map.
1321 assert(N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!");
1322 assert(N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!");
1323 Erased = CSEMap.erase(N);
1324 break;
1325 }
1326#ifndef NDEBUG
1327 // Verify that the node was actually in one of the CSE maps, unless it has a
1328 // glue result (which cannot be CSE'd) or is one of the special cases that are
1329 // not subject to CSE.
1330 if (!Erased && N->getValueType(N->getNumValues()-1) != MVT::Glue &&
1331 !N->isMachineOpcode() && !doNotCSE(N)) {
1332 N->dump(this);
1333 dbgs() << "\n";
1334 llvm_unreachable("Node is not in map!");
1335 }
1336#endif
1337 return Erased;
1338}
1339
1340/// AddModifiedNodeToCSEMaps - The specified node has been removed from the CSE
1341/// maps and modified in place. Add it back to the CSE maps, unless an identical
1342/// node already exists, in which case transfer all its users to the existing
1343/// node. This transfer can potentially trigger recursive merging.
1344void
1345SelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N) {
1346 // For node types that aren't CSE'd, just act as if no identical node
1347 // already exists.
1348 if (!doNotCSE(N)) {
1349 SDNode *Existing = CSEMap.getOrInsert(N);
1350 if (Existing != N) {
1351 // If there was already an existing matching node, use ReplaceAllUsesWith
1352 // to replace the dead one with the existing one. This can cause
1353 // recursive merging of other unrelated nodes down the line.
1354 Existing->intersectFlagsWith(Flags: N->getFlags());
1355 if (auto *MemNode = dyn_cast<MemSDNode>(Val: Existing)) {
1356 ArrayRef<MachineMemOperand *> NewMMOs =
1357 cast<MemSDNode>(Val: N)->memoperands();
1358 // Range and cache hint metadata are not part of the DAG CSE key because
1359 // we prefer to CSE even when metadata does not match. Merge potentially
1360 // differing metadata conservatively.
1361 MemNode->refineMMOMetadata(NewMMOs);
1362 }
1363 ReplaceAllUsesWith(From: N, To: Existing);
1364
1365 // N is now dead. Inform the listeners and delete it.
1366 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1367 DUL->NodeDeleted(N, Existing);
1368 DeleteNodeNotInCSEMaps(N);
1369 return;
1370 }
1371 }
1372
1373 // If the node doesn't already exist, we updated it. Inform listeners.
1374 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1375 DUL->NodeUpdated(N);
1376}
1377
1378/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1379/// were replaced with those specified. If this node is never memoized,
1380/// return null, otherwise return a pointer to the slot it would take. If a
1381/// node already exists with these operands, the slot will be non-null.
1382SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op,
1383 FoldingSetInsertToken &InsertToken) {
1384 if (doNotCSE(N))
1385 return nullptr;
1386
1387 SDValue Ops[] = { Op };
1388 SDNodeKey ID(N->getOpcode(), N->getVTList(), Ops);
1389 AddNodeIDCustom(ID&: ID.Tail, N, Opc: N->getOpcode());
1390 SDNode *Node = lookupNode(Key: ID, DL: SDLoc(N), InsertToken);
1391 if (Node)
1392 Node->intersectFlagsWith(Flags: N->getFlags());
1393 return Node;
1394}
1395
1396/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1397/// were replaced with those specified. If this node is never memoized,
1398/// return null, otherwise return a pointer to the slot it would take. If a
1399/// node already exists with these operands, the slot will be non-null.
1400SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op1, SDValue Op2,
1401 FoldingSetInsertToken &InsertToken) {
1402 if (doNotCSE(N))
1403 return nullptr;
1404
1405 SDValue Ops[] = { Op1, Op2 };
1406 SDNodeKey ID(N->getOpcode(), N->getVTList(), Ops);
1407 AddNodeIDCustom(ID&: ID.Tail, N, Opc: N->getOpcode());
1408 SDNode *Node = lookupNode(Key: ID, DL: SDLoc(N), InsertToken);
1409 if (Node)
1410 Node->intersectFlagsWith(Flags: N->getFlags());
1411 return Node;
1412}
1413
1414/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1415/// were replaced with those specified. If this node is never memoized,
1416/// return null, otherwise return a pointer to the slot it would take. If a
1417/// node already exists with these operands, the slot will be non-null.
1418SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, ArrayRef<SDValue> Ops,
1419 FoldingSetInsertToken &InsertToken) {
1420 if (doNotCSE(N))
1421 return nullptr;
1422
1423 SDNodeKey ID(N->getOpcode(), N->getVTList(), Ops);
1424 AddNodeIDCustom(ID&: ID.Tail, N, Opc: N->getOpcode());
1425 SDNode *Node = lookupNode(Key: ID, DL: SDLoc(N), InsertToken);
1426 if (Node)
1427 Node->intersectFlagsWith(Flags: N->getFlags());
1428 return Node;
1429}
1430
1431Align SelectionDAG::getEVTAlign(EVT VT) const {
1432 Type *Ty = VT == MVT::iPTR ? PointerType::get(C&: *getContext(), AddressSpace: 0)
1433 : VT.getTypeForEVT(Context&: *getContext());
1434
1435 return getDataLayout().getABITypeAlign(Ty);
1436}
1437
1438// EntryNode could meaningfully have debug info if we can find it...
1439SelectionDAG::SelectionDAG(const TargetMachine &tm, CodeGenOptLevel OL)
1440 : TM(tm), OptLevel(OL), EntryNode(ISD::EntryToken, 0, DebugLoc(),
1441 getVTList(VT1: MVT::Other, VT2: MVT::Glue)),
1442 Root(getEntryNode()) {
1443 InsertNode(N: &EntryNode);
1444 DbgInfo = new SDDbgInfo();
1445}
1446
1447void SelectionDAG::init(MachineFunction &NewMF,
1448 const TargetLibraryInfo *LibraryInfo,
1449 const LibcallLoweringInfo *LibcallsInfo,
1450 UniformityInfo *NewUA, ProfileSummaryInfo *PSIin,
1451 BlockFrequencyInfo *BFIin,
1452 FunctionVarLocs const *VarLocs) {
1453 MF = &NewMF;
1454 TLI = getSubtarget().getTargetLowering();
1455 TSI = getSubtarget().getSelectionDAGInfo();
1456 LibInfo = LibraryInfo;
1457 Libcalls = LibcallsInfo;
1458 Context = &MF->getFunction().getContext();
1459 DAGOpts = &Context->getLibraryOptions().get<SelectionDAGOptions>();
1460 UA = NewUA;
1461 PSI = PSIin;
1462 BFI = BFIin;
1463 FnVarLocs = VarLocs;
1464}
1465
1466SelectionDAG::~SelectionDAG() {
1467 assert(!UpdateListeners && "Dangling registered DAGUpdateListeners");
1468 allnodes_clear();
1469 OperandRecycler.clear(OperandAllocator);
1470 delete DbgInfo;
1471}
1472
1473bool SelectionDAG::shouldOptForSize() const {
1474 return llvm::shouldOptimizeForSize(BB: FLI->MBB->getBasicBlock(), PSI, BFI);
1475}
1476
1477const BasicBlock *SelectionDAG::getBasicBlock() const {
1478 return FLI->MBB->getBasicBlock();
1479}
1480
1481void SelectionDAG::allnodes_clear() {
1482 assert(&*AllNodes.begin() == &EntryNode);
1483 AllNodes.remove(IT: AllNodes.begin());
1484 while (!AllNodes.empty())
1485 DeallocateNode(N: &AllNodes.front());
1486#ifndef NDEBUG
1487 NextPersistentId = 0;
1488#endif
1489}
1490
1491SDNode *SelectionDAG::lookupNode(const SDNodeKey &Key,
1492 FoldingSetInsertToken &InsertToken) {
1493 SDNode *N = CSEMap.lookup(Key, Token&: InsertToken);
1494 if (N) {
1495 switch (N->getOpcode()) {
1496 default: break;
1497 case ISD::Constant:
1498 case ISD::ConstantFP:
1499 llvm_unreachable("Querying for Constant and ConstantFP nodes requires "
1500 "debug location. Use another overload.");
1501 }
1502 }
1503 return N;
1504}
1505
1506SDNode *SelectionDAG::lookupNode(const SDNodeKey &Key, const SDLoc &DL,
1507 FoldingSetInsertToken &InsertToken) {
1508 SDNode *N = CSEMap.lookup(Key, Token&: InsertToken);
1509 if (N) {
1510 switch (N->getOpcode()) {
1511 case ISD::Constant:
1512 case ISD::ConstantFP:
1513 // Erase debug location from the node if the node is used at several
1514 // different places. Do not propagate one location to all uses as it
1515 // will cause a worse single stepping debugging experience.
1516 if (N->getDebugLoc() != DL.getDebugLoc())
1517 N->setDebugLoc(DebugLoc());
1518 break;
1519 default:
1520 // When the node's point of use is located earlier in the instruction
1521 // sequence than its prior point of use, update its debug info to the
1522 // earlier location.
1523 if (DL.getIROrder() && DL.getIROrder() < N->getIROrder())
1524 N->setDebugLoc(DL.getDebugLoc());
1525 break;
1526 }
1527 }
1528 return N;
1529}
1530
1531void SelectionDAG::clear() {
1532 allnodes_clear();
1533 OperandRecycler.clear(OperandAllocator);
1534 OperandAllocator.Reset();
1535 CSEMap.clear();
1536
1537 ExtendedValueTypeNodes.clear();
1538 ExternalSymbols.clear();
1539 TargetExternalSymbols.clear();
1540 MCSymbols.clear();
1541 SDEI.clear();
1542 llvm::fill(Range&: CondCodeNodes, Value: nullptr);
1543 llvm::fill(Range&: ValueTypeNodes, Value: nullptr);
1544
1545 EntryNode.UseList = nullptr;
1546 InsertNode(N: &EntryNode);
1547 Root = getEntryNode();
1548 DbgInfo->clear();
1549}
1550
1551SDValue SelectionDAG::getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT) {
1552 return VT.bitsGT(VT: Op.getValueType())
1553 ? getNode(Opcode: ISD::FP_EXTEND, DL, VT, Operand: Op)
1554 : getNode(Opcode: ISD::FP_ROUND, DL, VT, N1: Op,
1555 N2: getIntPtrConstant(Val: 0, DL, /*isTarget=*/true));
1556}
1557
1558std::pair<SDValue, SDValue>
1559SelectionDAG::getStrictFPExtendOrRound(SDValue Op, SDValue Chain,
1560 const SDLoc &DL, EVT VT) {
1561 assert(!VT.bitsEq(Op.getValueType()) &&
1562 "Strict no-op FP extend/round not allowed.");
1563 SDValue Res =
1564 VT.bitsGT(VT: Op.getValueType())
1565 ? getNode(Opcode: ISD::STRICT_FP_EXTEND, DL, ResultTys: {VT, MVT::Other}, Ops: {Chain, Op})
1566 : getNode(Opcode: ISD::STRICT_FP_ROUND, DL, ResultTys: {VT, MVT::Other},
1567 Ops: {Chain, Op, getIntPtrConstant(Val: 0, DL, /*isTarget=*/true)});
1568
1569 return std::pair<SDValue, SDValue>(Res, SDValue(Res.getNode(), 1));
1570}
1571
1572SDValue SelectionDAG::getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1573 return VT.bitsGT(VT: Op.getValueType()) ?
1574 getNode(Opcode: ISD::ANY_EXTEND, DL, VT, Operand: Op) :
1575 getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Op);
1576}
1577
1578SDValue SelectionDAG::getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1579 return VT.bitsGT(VT: Op.getValueType()) ?
1580 getNode(Opcode: ISD::SIGN_EXTEND, DL, VT, Operand: Op) :
1581 getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Op);
1582}
1583
1584SDValue SelectionDAG::getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1585 return VT.bitsGT(VT: Op.getValueType()) ?
1586 getNode(Opcode: ISD::ZERO_EXTEND, DL, VT, Operand: Op) :
1587 getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Op);
1588}
1589
1590SDValue SelectionDAG::getBitcastedAnyExtOrTrunc(SDValue Op, const SDLoc &DL,
1591 EVT VT) {
1592 assert(!VT.isVector());
1593 auto Type = Op.getValueType();
1594 SDValue DestOp;
1595 if (Type == VT)
1596 return Op;
1597 auto Size = Op.getValueSizeInBits();
1598 DestOp = getBitcast(VT: EVT::getIntegerVT(Context&: *Context, BitWidth: Size), V: Op);
1599 if (DestOp.getValueType() == VT)
1600 return DestOp;
1601
1602 return getAnyExtOrTrunc(Op: DestOp, DL, VT);
1603}
1604
1605SDValue SelectionDAG::getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT,
1606 EVT OpVT) {
1607 if (VT.bitsLE(VT: Op.getValueType()))
1608 return getNode(Opcode: ISD::TRUNCATE, DL: SL, VT, Operand: Op);
1609
1610 TargetLowering::BooleanContent BType = TLI->getBooleanContents(Type: OpVT);
1611 return getNode(Opcode: TLI->getExtendForContent(Content: BType), DL: SL, VT, Operand: Op);
1612}
1613
1614SDValue SelectionDAG::getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) {
1615 EVT OpVT = Op.getValueType();
1616 assert(VT.isInteger() && OpVT.isInteger() &&
1617 "Cannot getZeroExtendInReg FP types");
1618 assert(VT.isVector() == OpVT.isVector() &&
1619 "getZeroExtendInReg type should be vector iff the operand "
1620 "type is vector!");
1621 assert((!VT.isVector() ||
1622 VT.getVectorElementCount() == OpVT.getVectorElementCount()) &&
1623 "Vector element counts must match in getZeroExtendInReg");
1624 assert(VT.getScalarType().bitsLE(OpVT.getScalarType()) && "Not extending!");
1625 if (OpVT == VT)
1626 return Op;
1627 // TODO: Use computeKnownBits instead of AssertZext.
1628 if (Op.getOpcode() == ISD::AssertZext && cast<VTSDNode>(Val: Op.getOperand(i: 1))
1629 ->getVT()
1630 .getScalarType()
1631 .bitsLE(VT: VT.getScalarType()))
1632 return Op;
1633 APInt Imm = APInt::getLowBitsSet(numBits: OpVT.getScalarSizeInBits(),
1634 loBitsSet: VT.getScalarSizeInBits());
1635 return getNode(Opcode: ISD::AND, DL, VT: OpVT, N1: Op, N2: getConstant(Val: Imm, DL, VT: OpVT));
1636}
1637
1638SDValue SelectionDAG::getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1639 // Only unsigned pointer semantics are supported right now. In the future this
1640 // might delegate to TLI to check pointer signedness.
1641 return getZExtOrTrunc(Op, DL, VT);
1642}
1643
1644SDValue SelectionDAG::getPtrExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) {
1645 // Only unsigned pointer semantics are supported right now. In the future this
1646 // might delegate to TLI to check pointer signedness.
1647 return getZeroExtendInReg(Op, DL, VT);
1648}
1649
1650SDValue SelectionDAG::getNegative(SDValue Val, const SDLoc &DL, EVT VT) {
1651 return getNode(Opcode: ISD::SUB, DL, VT, N1: getConstant(Val: 0, DL, VT), N2: Val);
1652}
1653
1654/// getNOT - Create a bitwise NOT operation as (XOR Val, -1).
1655SDValue SelectionDAG::getNOT(const SDLoc &DL, SDValue Val, EVT VT) {
1656 return getNode(Opcode: ISD::XOR, DL, VT, N1: Val, N2: getAllOnesConstant(DL, VT));
1657}
1658
1659SDValue SelectionDAG::getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT) {
1660 SDValue TrueValue = getBoolConstant(V: true, DL, VT, OpVT: VT);
1661 return getNode(Opcode: ISD::XOR, DL, VT, N1: Val, N2: TrueValue);
1662}
1663
1664SDValue SelectionDAG::getBoolConstant(bool V, const SDLoc &DL, EVT VT,
1665 EVT OpVT) {
1666 if (!V)
1667 return getConstant(Val: 0, DL, VT);
1668
1669 switch (TLI->getBooleanContents(Type: OpVT)) {
1670 case TargetLowering::ZeroOrOneBooleanContent:
1671 case TargetLowering::UndefinedBooleanContent:
1672 return getConstant(Val: 1, DL, VT);
1673 case TargetLowering::ZeroOrNegativeOneBooleanContent:
1674 return getAllOnesConstant(DL, VT);
1675 }
1676 llvm_unreachable("Unexpected boolean content enum!");
1677}
1678
1679SDValue SelectionDAG::getConstant(uint64_t Val, const SDLoc &DL, EVT VT,
1680 bool isT, bool isO) {
1681 return getConstant(Val: APInt(VT.getScalarSizeInBits(), Val, /*isSigned=*/false),
1682 DL, VT, isTarget: isT, isOpaque: isO);
1683}
1684
1685SDValue SelectionDAG::getConstant(const APInt &Val, const SDLoc &DL, EVT VT,
1686 bool isT, bool isO) {
1687 return getConstant(Val: *ConstantInt::get(Context&: *Context, V: Val), DL, VT, isTarget: isT, isOpaque: isO);
1688}
1689
1690SDValue SelectionDAG::getConstant(const ConstantInt &Val, const SDLoc &DL,
1691 EVT VT, bool isT, bool isO) {
1692 assert(VT.isInteger() && "Cannot create FP integer constant!");
1693
1694 EVT EltVT = VT.getScalarType();
1695 const ConstantInt *Elt = &Val;
1696
1697 // Vector splats are explicit within the DAG, with ConstantSDNode holding the
1698 // to-be-splatted scalar ConstantInt.
1699 if (isa<VectorType>(Val: Elt->getType()))
1700 Elt = ConstantInt::get(Context&: *getContext(), V: Elt->getValue());
1701
1702 // In some cases the vector type is legal but the element type is illegal and
1703 // needs to be promoted, for example v8i8 on ARM. In this case, promote the
1704 // inserted value (the type does not need to match the vector element type).
1705 // Any extra bits introduced will be truncated away.
1706 if (VT.isVector() && TLI->getTypeAction(Context&: *getContext(), VT: EltVT) ==
1707 TargetLowering::TypePromoteInteger) {
1708 EltVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT: EltVT);
1709 APInt NewVal;
1710 if (TLI->isSExtCheaperThanZExt(FromTy: VT.getScalarType(), ToTy: EltVT))
1711 NewVal = Elt->getValue().sextOrTrunc(width: EltVT.getSizeInBits());
1712 else
1713 NewVal = Elt->getValue().zextOrTrunc(width: EltVT.getSizeInBits());
1714 Elt = ConstantInt::get(Context&: *getContext(), V: NewVal);
1715 }
1716 // In other cases the element type is illegal and needs to be expanded, for
1717 // example v2i64 on MIPS32. In this case, find the nearest legal type, split
1718 // the value into n parts and use a vector type with n-times the elements.
1719 // Then bitcast to the type requested.
1720 // Legalizing constants too early makes the DAGCombiner's job harder so we
1721 // only legalize if the DAG tells us we must produce legal types.
1722 else if (NewNodesMustHaveLegalTypes && VT.isVector() &&
1723 TLI->getTypeAction(Context&: *getContext(), VT: EltVT) ==
1724 TargetLowering::TypeExpandInteger) {
1725 const APInt &NewVal = Elt->getValue();
1726 EVT ViaEltVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT: EltVT);
1727 unsigned ViaEltSizeInBits = ViaEltVT.getSizeInBits();
1728
1729 // For scalable vectors, try to use a SPLAT_VECTOR_PARTS node.
1730 if (VT.isScalableVector() ||
1731 TLI->isOperationLegal(Op: ISD::SPLAT_VECTOR, VT)) {
1732 assert(EltVT.getSizeInBits() % ViaEltSizeInBits == 0 &&
1733 "Can only handle an even split!");
1734 unsigned Parts = EltVT.getSizeInBits() / ViaEltSizeInBits;
1735
1736 SmallVector<SDValue, 2> ScalarParts;
1737 for (unsigned i = 0; i != Parts; ++i)
1738 ScalarParts.push_back(Elt: getConstant(
1739 Val: NewVal.extractBits(numBits: ViaEltSizeInBits, bitPosition: i * ViaEltSizeInBits), DL,
1740 VT: ViaEltVT, isT, isO));
1741
1742 return getNode(Opcode: ISD::SPLAT_VECTOR_PARTS, DL, VT, Ops: ScalarParts);
1743 }
1744
1745 unsigned ViaVecNumElts = VT.getSizeInBits() / ViaEltSizeInBits;
1746 EVT ViaVecVT = EVT::getVectorVT(Context&: *getContext(), VT: ViaEltVT, NumElements: ViaVecNumElts);
1747
1748 // Check the temporary vector is the correct size. If this fails then
1749 // getTypeToTransformTo() probably returned a type whose size (in bits)
1750 // isn't a power-of-2 factor of the requested type size.
1751 assert(ViaVecVT.getSizeInBits() == VT.getSizeInBits());
1752
1753 SmallVector<SDValue, 2> EltParts;
1754 for (unsigned i = 0; i < ViaVecNumElts / VT.getVectorNumElements(); ++i)
1755 EltParts.push_back(Elt: getConstant(
1756 Val: NewVal.extractBits(numBits: ViaEltSizeInBits, bitPosition: i * ViaEltSizeInBits), DL,
1757 VT: ViaEltVT, isT, isO));
1758
1759 // EltParts is currently in little endian order. If we actually want
1760 // big-endian order then reverse it now.
1761 if (getDataLayout().isBigEndian())
1762 std::reverse(first: EltParts.begin(), last: EltParts.end());
1763
1764 // The elements must be reversed when the element order is different
1765 // to the endianness of the elements (because the BITCAST is itself a
1766 // vector shuffle in this situation). However, we do not need any code to
1767 // perform this reversal because getConstant() is producing a vector
1768 // splat.
1769 // This situation occurs in MIPS MSA.
1770
1771 SmallVector<SDValue, 8> Ops;
1772 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i)
1773 llvm::append_range(C&: Ops, R&: EltParts);
1774
1775 SDValue V =
1776 getNode(Opcode: ISD::BITCAST, DL, VT, Operand: getBuildVector(VT: ViaVecVT, DL, Ops));
1777 return V;
1778 }
1779
1780 assert(Elt->getBitWidth() == EltVT.getSizeInBits() &&
1781 "APInt size does not match type size!");
1782 unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant;
1783 SDVTList VTs = getVTList(VT: EltVT);
1784 SDNodeKey ID(Opc, VTs, {});
1785 ID.AddPointer(P: Elt);
1786 ID.AddBoolean(B: isO);
1787 FoldingSetInsertToken InsertToken;
1788 SDNode *N = nullptr;
1789 if ((N = lookupNode(Key: ID, DL, InsertToken)))
1790 if (!VT.isVector())
1791 return SDValue(N, 0);
1792
1793 if (!N) {
1794 N = newSDNode<ConstantSDNode>(Args&: isT, Args&: isO, Args&: Elt, Args&: VTs);
1795 if (!isT)
1796 N->setDebugLoc(DL.getDebugLoc());
1797 CSEMap.insert(N, Token: InsertToken);
1798 InsertNode(N);
1799 NewSDValueDbgMsg(V: SDValue(N, 0), Msg: "Creating constant: ", G: this);
1800 }
1801
1802 SDValue Result(N, 0);
1803 if (VT.isVector())
1804 Result = getSplat(VT, DL, Op: Result);
1805 return Result;
1806}
1807
1808SDValue SelectionDAG::getSignedConstant(int64_t Val, const SDLoc &DL, EVT VT,
1809 bool isT, bool isO) {
1810 unsigned Size = VT.getScalarSizeInBits();
1811 return getConstant(Val: APInt(Size, Val, /*isSigned=*/true), DL, VT, isT, isO);
1812}
1813
1814SDValue SelectionDAG::getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget,
1815 bool IsOpaque) {
1816 return getConstant(Val: APInt::getAllOnes(numBits: VT.getScalarSizeInBits()), DL, VT,
1817 isT: IsTarget, isO: IsOpaque);
1818}
1819
1820SDValue SelectionDAG::getIntPtrConstant(uint64_t Val, const SDLoc &DL,
1821 bool isTarget) {
1822 return getConstant(Val, DL, VT: TLI->getPointerTy(DL: getDataLayout()), isT: isTarget);
1823}
1824
1825SDValue SelectionDAG::getShiftAmountConstant(uint64_t Val, EVT VT,
1826 const SDLoc &DL) {
1827 assert(VT.isInteger() && "Shift amount is not an integer type!");
1828 EVT ShiftVT = TLI->getShiftAmountTy(LHSTy: VT, DL: getDataLayout());
1829 return getConstant(Val, DL, VT: ShiftVT);
1830}
1831
1832SDValue SelectionDAG::getShiftAmountConstant(const APInt &Val, EVT VT,
1833 const SDLoc &DL) {
1834 assert(Val.ult(VT.getScalarSizeInBits()) && "Out of range shift");
1835 return getShiftAmountConstant(Val: Val.getZExtValue(), VT, DL);
1836}
1837
1838SDValue SelectionDAG::getVectorIdxConstant(uint64_t Val, const SDLoc &DL,
1839 bool isTarget) {
1840 return getConstant(Val, DL, VT: TLI->getVectorIdxTy(DL: getDataLayout()), isT: isTarget);
1841}
1842
1843SDValue SelectionDAG::getConstantFP(const APFloat &V, const SDLoc &DL, EVT VT,
1844 bool isTarget) {
1845 return getConstantFP(V: *ConstantFP::get(Context&: *getContext(), V), DL, VT, isTarget);
1846}
1847
1848SDValue SelectionDAG::getConstantFP(const ConstantFP &V, const SDLoc &DL,
1849 EVT VT, bool isTarget) {
1850 assert(VT.isFloatingPoint() && "Cannot create integer FP constant!");
1851
1852 EVT EltVT = VT.getScalarType();
1853 const ConstantFP *Elt = &V;
1854
1855 // Vector splats are explicit within the DAG, with ConstantFPSDNode holding
1856 // the to-be-splatted scalar ConstantFP.
1857 if (isa<VectorType>(Val: Elt->getType()))
1858 Elt = ConstantFP::get(Context&: *getContext(), V: Elt->getValue());
1859
1860 // Do the map lookup using the actual bit pattern for the floating point
1861 // value, so that we don't have problems with 0.0 comparing equal to -0.0, and
1862 // we don't have issues with SNANs.
1863 unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP;
1864 SDVTList VTs = getVTList(VT: EltVT);
1865 SDNodeKey ID(Opc, VTs, {});
1866 ID.AddPointer(P: Elt);
1867 FoldingSetInsertToken InsertToken;
1868 SDNode *N = nullptr;
1869 if ((N = lookupNode(Key: ID, DL, InsertToken)))
1870 if (!VT.isVector())
1871 return SDValue(N, 0);
1872
1873 if (!N) {
1874 N = newSDNode<ConstantFPSDNode>(Args&: isTarget, Args&: Elt, Args&: VTs);
1875 CSEMap.insert(N, Token: InsertToken);
1876 InsertNode(N);
1877 }
1878
1879 SDValue Result(N, 0);
1880 if (VT.isVector())
1881 Result = getSplat(VT, DL, Op: Result);
1882 NewSDValueDbgMsg(V: Result, Msg: "Creating fp constant: ", G: this);
1883 return Result;
1884}
1885
1886SDValue SelectionDAG::getConstantFP(double Val, const SDLoc &DL, EVT VT,
1887 bool isTarget) {
1888 EVT EltVT = VT.getScalarType();
1889 if (EltVT == MVT::f32)
1890 return getConstantFP(V: APFloat((float)Val), DL, VT, isTarget);
1891 if (EltVT == MVT::f64)
1892 return getConstantFP(V: APFloat(Val), DL, VT, isTarget);
1893 if (EltVT == MVT::f80 || EltVT == MVT::f128 || EltVT == MVT::ppcf128 ||
1894 EltVT == MVT::f16 || EltVT == MVT::bf16) {
1895 bool Ignored;
1896 APFloat APF = APFloat(Val);
1897 APF.convert(ToSemantics: EltVT.getFltSemantics(), RM: APFloat::rmNearestTiesToEven,
1898 losesInfo: &Ignored);
1899 return getConstantFP(V: APF, DL, VT, isTarget);
1900 }
1901 llvm_unreachable("Unsupported type in getConstantFP");
1902}
1903
1904SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, const SDLoc &DL,
1905 EVT VT, int64_t Offset, bool isTargetGA,
1906 unsigned TargetFlags) {
1907 assert((TargetFlags == 0 || isTargetGA) &&
1908 "Cannot set target flags on target-independent globals");
1909
1910 // Truncate (with sign-extension) the offset value to the pointer size.
1911 unsigned BitWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType());
1912 if (BitWidth < 64)
1913 Offset = SignExtend64(X: Offset, B: BitWidth);
1914
1915 unsigned Opc;
1916 if (GV->isThreadLocal())
1917 Opc = isTargetGA ? ISD::TargetGlobalTLSAddress : ISD::GlobalTLSAddress;
1918 else
1919 Opc = isTargetGA ? ISD::TargetGlobalAddress : ISD::GlobalAddress;
1920
1921 SDVTList VTs = getVTList(VT);
1922 SDNodeKey ID(Opc, VTs, {});
1923 ID.AddPointer(P: GV);
1924 ID.AddInteger(I: Offset);
1925 ID.AddInteger(I: TargetFlags);
1926 FoldingSetInsertToken InsertToken;
1927 if (SDNode *E = lookupNode(Key: ID, DL, InsertToken))
1928 return SDValue(E, 0);
1929
1930 auto *N = newSDNode<GlobalAddressSDNode>(
1931 Args&: Opc, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: GV, Args&: VTs, Args&: Offset, Args&: TargetFlags);
1932 CSEMap.insert(N, Token: InsertToken);
1933 InsertNode(N);
1934 return SDValue(N, 0);
1935}
1936
1937SDValue SelectionDAG::getDeactivationSymbol(const GlobalValue *GV) {
1938 SDVTList VTs = getVTList(VT: MVT::Untyped);
1939 SDNodeKey ID(ISD::DEACTIVATION_SYMBOL, VTs, {});
1940 ID.AddPointer(P: GV);
1941 FoldingSetInsertToken InsertToken;
1942 if (SDNode *E = lookupNode(Key: ID, DL: SDLoc(), InsertToken))
1943 return SDValue(E, 0);
1944
1945 auto *N = newSDNode<DeactivationSymbolSDNode>(Args&: GV, Args&: VTs);
1946 CSEMap.insert(N, Token: InsertToken);
1947 InsertNode(N);
1948 return SDValue(N, 0);
1949}
1950
1951SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) {
1952 unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex;
1953 SDVTList VTs = getVTList(VT);
1954 SDNodeKey ID(Opc, VTs, {});
1955 ID.AddInteger(I: FI);
1956 FoldingSetInsertToken InsertToken;
1957 if (SDNode *E = lookupNode(Key: ID, InsertToken))
1958 return SDValue(E, 0);
1959
1960 auto *N = newSDNode<FrameIndexSDNode>(Args&: FI, Args&: VTs, Args&: isTarget);
1961 CSEMap.insert(N, Token: InsertToken);
1962 InsertNode(N);
1963 return SDValue(N, 0);
1964}
1965
1966SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget,
1967 unsigned TargetFlags) {
1968 assert((TargetFlags == 0 || isTarget) &&
1969 "Cannot set target flags on target-independent jump tables");
1970 unsigned Opc = isTarget ? ISD::TargetJumpTable : ISD::JumpTable;
1971 SDVTList VTs = getVTList(VT);
1972 SDNodeKey ID(Opc, VTs, {});
1973 ID.AddInteger(I: JTI);
1974 ID.AddInteger(I: TargetFlags);
1975 FoldingSetInsertToken InsertToken;
1976 if (SDNode *E = lookupNode(Key: ID, InsertToken))
1977 return SDValue(E, 0);
1978
1979 auto *N = newSDNode<JumpTableSDNode>(Args&: JTI, Args&: VTs, Args&: isTarget, Args&: TargetFlags);
1980 CSEMap.insert(N, Token: InsertToken);
1981 InsertNode(N);
1982 return SDValue(N, 0);
1983}
1984
1985SDValue SelectionDAG::getJumpTableDebugInfo(int JTI, SDValue Chain,
1986 const SDLoc &DL) {
1987 EVT PTy = getTargetLoweringInfo().getPointerTy(DL: getDataLayout());
1988 return getNode(Opcode: ISD::JUMP_TABLE_DEBUG_INFO, DL, VT: MVT::Other, N1: Chain,
1989 N2: getTargetConstant(Val: static_cast<uint64_t>(JTI), DL, VT: PTy, isOpaque: true));
1990}
1991
1992SDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT,
1993 MaybeAlign Alignment, int Offset,
1994 bool isTarget, unsigned TargetFlags) {
1995 assert((TargetFlags == 0 || isTarget) &&
1996 "Cannot set target flags on target-independent globals");
1997 if (!Alignment)
1998 Alignment = shouldOptForSize()
1999 ? getDataLayout().getABITypeAlign(Ty: C->getType())
2000 : getDataLayout().getPrefTypeAlign(Ty: C->getType());
2001 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
2002 SDVTList VTs = getVTList(VT);
2003 SDNodeKey ID(Opc, VTs, {});
2004 ID.AddInteger(I: Alignment->value());
2005 ID.AddInteger(I: Offset);
2006 ID.AddPointer(P: C);
2007 ID.AddInteger(I: TargetFlags);
2008 FoldingSetInsertToken InsertToken;
2009 if (SDNode *E = lookupNode(Key: ID, InsertToken))
2010 return SDValue(E, 0);
2011
2012 auto *N = newSDNode<ConstantPoolSDNode>(Args&: isTarget, Args&: C, Args&: VTs, Args&: Offset, Args&: *Alignment,
2013 Args&: TargetFlags);
2014 CSEMap.insert(N, Token: InsertToken);
2015 InsertNode(N);
2016 SDValue V = SDValue(N, 0);
2017 NewSDValueDbgMsg(V, Msg: "Creating new constant pool: ", G: this);
2018 return V;
2019}
2020
2021SDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT,
2022 MaybeAlign Alignment, int Offset,
2023 bool isTarget, unsigned TargetFlags) {
2024 assert((TargetFlags == 0 || isTarget) &&
2025 "Cannot set target flags on target-independent globals");
2026 if (!Alignment)
2027 Alignment = getDataLayout().getPrefTypeAlign(Ty: C->getType());
2028 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
2029 SDVTList VTs = getVTList(VT);
2030 SDNodeKey ID(Opc, VTs, {});
2031 ID.AddInteger(I: Alignment->value());
2032 ID.AddInteger(I: Offset);
2033 C->addSelectionDAGCSEId(ID&: ID.Tail);
2034 ID.AddInteger(I: TargetFlags);
2035 FoldingSetInsertToken InsertToken;
2036 if (SDNode *E = lookupNode(Key: ID, InsertToken))
2037 return SDValue(E, 0);
2038
2039 auto *N = newSDNode<ConstantPoolSDNode>(Args&: isTarget, Args&: C, Args&: VTs, Args&: Offset, Args&: *Alignment,
2040 Args&: TargetFlags);
2041 CSEMap.insert(N, Token: InsertToken);
2042 InsertNode(N);
2043 return SDValue(N, 0);
2044}
2045
2046SDValue SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) {
2047 SDNodeKey ID(ISD::BasicBlock, getVTList(VT: MVT::Other), {});
2048 ID.AddPointer(P: MBB);
2049 FoldingSetInsertToken InsertToken;
2050 if (SDNode *E = lookupNode(Key: ID, InsertToken))
2051 return SDValue(E, 0);
2052
2053 auto *N = newSDNode<BasicBlockSDNode>(Args&: MBB);
2054 CSEMap.insert(N, Token: InsertToken);
2055 InsertNode(N);
2056 return SDValue(N, 0);
2057}
2058
2059SDValue SelectionDAG::getValueType(EVT VT) {
2060 if (VT.isSimple() && (unsigned)VT.getSimpleVT().SimpleTy >=
2061 ValueTypeNodes.size())
2062 ValueTypeNodes.resize(new_size: VT.getSimpleVT().SimpleTy+1);
2063
2064 SDNode *&N = VT.isExtended() ?
2065 ExtendedValueTypeNodes[VT] : ValueTypeNodes[VT.getSimpleVT().SimpleTy];
2066
2067 if (N) return SDValue(N, 0);
2068 N = newSDNode<VTSDNode>(Args&: VT);
2069 InsertNode(N);
2070 return SDValue(N, 0);
2071}
2072
2073SDValue SelectionDAG::getExternalSymbol(const char *Sym, EVT VT) {
2074 SDNode *&N = ExternalSymbols[Sym];
2075 if (N) return SDValue(N, 0);
2076 N = newSDNode<ExternalSymbolSDNode>(Args: false, Args&: Sym, Args: 0, Args: getVTList(VT));
2077 InsertNode(N);
2078 return SDValue(N, 0);
2079}
2080
2081SDValue SelectionDAG::getExternalSymbol(RTLIB::LibcallImpl Libcall, EVT VT) {
2082 StringRef SymName = RTLIB::RuntimeLibcallsInfo::getLibcallImplName(CallImpl: Libcall);
2083 return getExternalSymbol(Sym: SymName.data(), VT);
2084}
2085
2086SDValue SelectionDAG::getMCSymbol(MCSymbol *Sym, EVT VT) {
2087 SDNode *&N = MCSymbols[Sym];
2088 if (N)
2089 return SDValue(N, 0);
2090 N = newSDNode<MCSymbolSDNode>(Args&: Sym, Args: getVTList(VT));
2091 InsertNode(N);
2092 return SDValue(N, 0);
2093}
2094
2095SDValue SelectionDAG::getTargetExternalSymbol(const char *Sym, EVT VT,
2096 unsigned TargetFlags) {
2097 SDNode *&N =
2098 TargetExternalSymbols[std::pair<std::string, unsigned>(Sym, TargetFlags)];
2099 if (N) return SDValue(N, 0);
2100 N = newSDNode<ExternalSymbolSDNode>(Args: true, Args&: Sym, Args&: TargetFlags, Args: getVTList(VT));
2101 InsertNode(N);
2102 return SDValue(N, 0);
2103}
2104
2105SDValue SelectionDAG::getTargetExternalSymbol(RTLIB::LibcallImpl Libcall,
2106 EVT VT, unsigned TargetFlags) {
2107 StringRef SymName = RTLIB::RuntimeLibcallsInfo::getLibcallImplName(CallImpl: Libcall);
2108 return getTargetExternalSymbol(Sym: SymName.data(), VT, TargetFlags);
2109}
2110
2111SDValue SelectionDAG::getCondCode(ISD::CondCode Cond) {
2112 if ((unsigned)Cond >= CondCodeNodes.size())
2113 CondCodeNodes.resize(new_size: Cond+1);
2114
2115 if (!CondCodeNodes[Cond]) {
2116 auto *N = newSDNode<CondCodeSDNode>(Args&: Cond);
2117 CondCodeNodes[Cond] = N;
2118 InsertNode(N);
2119 }
2120
2121 return SDValue(CondCodeNodes[Cond], 0);
2122}
2123
2124SDValue SelectionDAG::getVScale(const SDLoc &DL, EVT VT, APInt MulImm) {
2125 assert(MulImm.getBitWidth() == VT.getSizeInBits() &&
2126 "APInt size does not match type size!");
2127
2128 if (MulImm == 0)
2129 return getConstant(Val: 0, DL, VT);
2130
2131 const MachineFunction &MF = getMachineFunction();
2132 const Function &F = MF.getFunction();
2133 ConstantRange CR = getVScaleRange(F: &F, BitWidth: 64);
2134 if (const APInt *C = CR.getSingleElement())
2135 return getConstant(Val: MulImm * C->getZExtValue(), DL, VT);
2136
2137 return getNode(Opcode: ISD::VSCALE, DL, VT, Operand: getConstant(Val: MulImm, DL, VT));
2138}
2139
2140/// \returns a value of type \p VT that represents the runtime value of \p
2141/// Quantity, i.e. scaled by vscale if it's scalable, or a fixed constant
2142/// otherwise. Quantity should be a FixedOrScalableQuantity, i.e. ElementCount
2143/// or TypeSize.
2144template <typename Ty>
2145static SDValue getFixedOrScalableQuantity(SelectionDAG &DAG, const SDLoc &DL,
2146 EVT VT, Ty Quantity) {
2147 if (Quantity.isScalable())
2148 return DAG.getVScale(
2149 DL, VT, MulImm: APInt(VT.getSizeInBits(), Quantity.getKnownMinValue()));
2150
2151 return DAG.getConstant(Quantity.getKnownMinValue(), DL, VT);
2152}
2153
2154SDValue SelectionDAG::getElementCount(const SDLoc &DL, EVT VT,
2155 ElementCount EC) {
2156 return getFixedOrScalableQuantity(DAG&: *this, DL, VT, Quantity: EC);
2157}
2158
2159SDValue SelectionDAG::getTypeSize(const SDLoc &DL, EVT VT, TypeSize TS) {
2160 return getFixedOrScalableQuantity(DAG&: *this, DL, VT, Quantity: TS);
2161}
2162
2163SDValue SelectionDAG::getMaskFromElementCount(const SDLoc &DL, EVT DataVT,
2164 ElementCount EC) {
2165 EVT IdxVT = TLI->getVectorIdxTy(DL: getDataLayout());
2166 EVT MaskVT = TLI->getSetCCResultType(DL: getDataLayout(), Context&: *getContext(), VT: DataVT);
2167 return getNode(Opcode: ISD::GET_ACTIVE_LANE_MASK, DL, VT: MaskVT,
2168 N1: getConstant(Val: 0, DL, VT: IdxVT), N2: getElementCount(DL, VT: IdxVT, EC));
2169}
2170
2171SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT) {
2172 APInt One(ResVT.getScalarSizeInBits(), 1);
2173 return getStepVector(DL, ResVT, StepVal: One);
2174}
2175
2176SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT,
2177 const APInt &StepVal) {
2178 assert(ResVT.getScalarSizeInBits() == StepVal.getBitWidth());
2179 if (ResVT.isScalableVector())
2180 return getNode(
2181 Opcode: ISD::STEP_VECTOR, DL, VT: ResVT,
2182 Operand: getTargetConstant(Val: StepVal, DL, VT: ResVT.getVectorElementType()));
2183
2184 SmallVector<SDValue, 16> OpsStepConstants;
2185 for (uint64_t i = 0; i < ResVT.getVectorNumElements(); i++)
2186 OpsStepConstants.push_back(
2187 Elt: getConstant(Val: StepVal * i, DL, VT: ResVT.getVectorElementType()));
2188 return getBuildVector(VT: ResVT, DL, Ops: OpsStepConstants);
2189}
2190
2191/// Swaps the values of N1 and N2. Swaps all indices in the shuffle mask M that
2192/// point at N1 to point at N2 and indices that point at N2 to point at N1.
2193static void commuteShuffle(SDValue &N1, SDValue &N2, MutableArrayRef<int> M) {
2194 std::swap(a&: N1, b&: N2);
2195 ShuffleVectorSDNode::commuteMask(Mask: M);
2196}
2197
2198SDValue SelectionDAG::getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1,
2199 SDValue N2, ArrayRef<int> Mask) {
2200 assert(VT.getVectorNumElements() == Mask.size() &&
2201 "Must have the same number of vector elements as mask elements!");
2202 assert(VT == N1.getValueType() && VT == N2.getValueType() &&
2203 "Invalid VECTOR_SHUFFLE");
2204
2205 // Canonicalize shuffle undef, undef -> undef
2206 if (N1.isUndef() && N2.isUndef()) {
2207 if (N1.getOpcode() == ISD::POISON && N2.getOpcode() == ISD::POISON)
2208 return getPOISON(VT);
2209 return getUNDEF(VT);
2210 }
2211
2212 // Validate that all indices in Mask are within the range of the elements
2213 // input to the shuffle.
2214 int NElts = Mask.size();
2215 assert(llvm::all_of(Mask,
2216 [&](int M) { return M < (NElts * 2) && M >= -1; }) &&
2217 "Index out of range");
2218
2219 // Copy the mask so we can do any needed cleanup.
2220 SmallVector<int, 8> MaskVec(Mask);
2221
2222 // Canonicalize shuffle v, v -> v, poison
2223 if (N1 == N2) {
2224 N2 = getPOISON(VT);
2225 for (int i = 0; i != NElts; ++i)
2226 if (MaskVec[i] >= NElts) MaskVec[i] -= NElts;
2227 }
2228
2229 // Canonicalize shuffle undef, v -> v, undef. Commute the shuffle mask.
2230 if (N1.isUndef())
2231 commuteShuffle(N1, N2, M: MaskVec);
2232
2233 if (TLI->hasVectorBlend()) {
2234 // If shuffling a splat, try to blend the splat instead. We do this here so
2235 // that even when this arises during lowering we don't have to re-handle it.
2236 auto BlendSplat = [&](BuildVectorSDNode *BV, int Offset) {
2237 BitVector UndefElements;
2238 SDValue Splat = BV->getSplatValue(UndefElements: &UndefElements);
2239 if (!Splat)
2240 return;
2241
2242 for (int i = 0; i < NElts; ++i) {
2243 if (MaskVec[i] < Offset || MaskVec[i] >= (Offset + NElts))
2244 continue;
2245
2246 // If this input comes from undef, mark it as such.
2247 if (UndefElements[MaskVec[i] - Offset]) {
2248 MaskVec[i] = -1;
2249 continue;
2250 }
2251
2252 // If we can blend a non-undef lane, use that instead.
2253 if (!UndefElements[i])
2254 MaskVec[i] = i + Offset;
2255 }
2256 };
2257 if (auto *N1BV = dyn_cast<BuildVectorSDNode>(Val&: N1))
2258 BlendSplat(N1BV, 0);
2259 if (auto *N2BV = dyn_cast<BuildVectorSDNode>(Val&: N2))
2260 BlendSplat(N2BV, NElts);
2261 }
2262
2263 // Canonicalize all index into lhs, -> shuffle lhs, poison
2264 // Canonicalize all index into rhs, -> shuffle rhs, poison
2265 bool AllLHS = true, AllRHS = true;
2266 bool N2Undef = N2.isUndef();
2267 for (int i = 0; i != NElts; ++i) {
2268 if (MaskVec[i] >= NElts) {
2269 if (N2Undef)
2270 MaskVec[i] = -1;
2271 else
2272 AllLHS = false;
2273 } else if (MaskVec[i] >= 0) {
2274 AllRHS = false;
2275 }
2276 }
2277 if (AllLHS && AllRHS)
2278 return getPOISON(VT);
2279 if (AllLHS && !N2Undef)
2280 N2 = getPOISON(VT);
2281 if (AllRHS) {
2282 N1 = getPOISON(VT);
2283 commuteShuffle(N1, N2, M: MaskVec);
2284 }
2285 // Reset our undef status after accounting for the mask.
2286 N2Undef = N2.isUndef();
2287 // Re-check whether both sides ended up undef.
2288 if (N1.isUndef() && N2Undef) {
2289 if (N1.getOpcode() == ISD::POISON && N2.getOpcode() == ISD::POISON)
2290 return getPOISON(VT);
2291 return getUNDEF(VT);
2292 }
2293
2294 // If Identity shuffle return that node.
2295 bool Identity = true, AllSame = true;
2296 for (int i = 0; i != NElts; ++i) {
2297 if (MaskVec[i] >= 0 && MaskVec[i] != i) Identity = false;
2298 if (MaskVec[i] != MaskVec[0]) AllSame = false;
2299 }
2300 if (Identity && NElts)
2301 return N1;
2302
2303 // Shuffling a constant splat doesn't change the result.
2304 if (N2Undef) {
2305 SDValue V = N1;
2306
2307 // Look through any bitcasts. We check that these don't change the number
2308 // (and size) of elements and just changes their types.
2309 while (V.getOpcode() == ISD::BITCAST)
2310 V = V->getOperand(Num: 0);
2311
2312 // A splat should always show up as a build vector node.
2313 if (auto *BV = dyn_cast<BuildVectorSDNode>(Val&: V)) {
2314 BitVector UndefElements;
2315 SDValue Splat = BV->getSplatValue(UndefElements: &UndefElements);
2316 // If this is a splat of an undef, shuffling it is also undef.
2317 if (Splat && Splat.isUndef())
2318 return Splat.getOpcode() == ISD::POISON ? getPOISON(VT) : getUNDEF(VT);
2319
2320 bool SameNumElts =
2321 V.getValueType().getVectorNumElements() == VT.getVectorNumElements();
2322
2323 // We only have a splat which can skip shuffles if there is a splatted
2324 // value and no undef lanes rearranged by the shuffle.
2325 if (Splat && UndefElements.none()) {
2326 // Splat of <x, x, ..., x>, return <x, x, ..., x>, provided that the
2327 // number of elements match or the value splatted is a zero constant.
2328 if (SameNumElts || isNullConstant(V: Splat))
2329 return N1;
2330 }
2331
2332 // If the shuffle itself creates a splat, build the vector directly.
2333 if (AllSame && SameNumElts) {
2334 EVT BuildVT = BV->getValueType(ResNo: 0);
2335 const SDValue &Splatted = BV->getOperand(Num: MaskVec[0]);
2336 SDValue NewBV = getSplatBuildVector(VT: BuildVT, DL: dl, Op: Splatted);
2337
2338 // We may have jumped through bitcasts, so the type of the
2339 // BUILD_VECTOR may not match the type of the shuffle.
2340 if (BuildVT != VT)
2341 NewBV = getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: NewBV);
2342 return NewBV;
2343 }
2344 }
2345 }
2346
2347 SDVTList VTs = getVTList(VT);
2348 SDValue Ops[2] = { N1, N2 };
2349 SDNodeKey ID(ISD::VECTOR_SHUFFLE, VTs, Ops);
2350 for (int i = 0; i != NElts; ++i)
2351 ID.AddInteger(I: MaskVec[i]);
2352
2353 FoldingSetInsertToken InsertToken;
2354 if (SDNode *E = lookupNode(Key: ID, DL: dl, InsertToken))
2355 return SDValue(E, 0);
2356
2357 // Allocate the mask array for the node out of the BumpPtrAllocator, since
2358 // SDNode doesn't have access to it. This memory will be "leaked" when
2359 // the node is deallocated, but recovered when the NodeAllocator is released.
2360 int *MaskAlloc = OperandAllocator.Allocate<int>(Num: NElts);
2361 llvm::copy(Range&: MaskVec, Out: MaskAlloc);
2362
2363 auto *N = newSDNode<ShuffleVectorSDNode>(Args&: VTs, Args: dl.getIROrder(),
2364 Args: dl.getDebugLoc(), Args&: MaskAlloc);
2365 createOperands(Node: N, Vals: Ops);
2366
2367 CSEMap.insert(N, Token: InsertToken);
2368 InsertNode(N);
2369 SDValue V = SDValue(N, 0);
2370 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
2371 return V;
2372}
2373
2374SDValue SelectionDAG::getCommutedVectorShuffle(const ShuffleVectorSDNode &SV) {
2375 EVT VT = SV.getValueType(ResNo: 0);
2376 SmallVector<int, 8> MaskVec(SV.getMask());
2377 ShuffleVectorSDNode::commuteMask(Mask: MaskVec);
2378
2379 SDValue Op0 = SV.getOperand(Num: 0);
2380 SDValue Op1 = SV.getOperand(Num: 1);
2381 return getVectorShuffle(VT, dl: SDLoc(&SV), N1: Op1, N2: Op0, Mask: MaskVec);
2382}
2383
2384SDValue SelectionDAG::getRegister(Register Reg, EVT VT) {
2385 SDVTList VTs = getVTList(VT);
2386 SDNodeKey ID(ISD::Register, VTs, {});
2387 ID.AddInteger(I: Reg.id());
2388 FoldingSetInsertToken InsertToken;
2389 if (SDNode *E = lookupNode(Key: ID, InsertToken))
2390 return SDValue(E, 0);
2391
2392 auto *N = newSDNode<RegisterSDNode>(Args&: Reg, Args&: VTs);
2393 N->SDNodeBits.IsDivergent = TLI->isSDNodeSourceOfDivergence(N, FLI, UA);
2394 CSEMap.insert(N, Token: InsertToken);
2395 InsertNode(N);
2396 return SDValue(N, 0);
2397}
2398
2399SDValue SelectionDAG::getRegisterMask(const uint32_t *RegMask) {
2400 SDNodeKey ID(ISD::RegisterMask, getVTList(VT: MVT::Untyped), {});
2401 ID.AddPointer(P: RegMask);
2402 FoldingSetInsertToken InsertToken;
2403 if (SDNode *E = lookupNode(Key: ID, InsertToken))
2404 return SDValue(E, 0);
2405
2406 auto *N = newSDNode<RegisterMaskSDNode>(Args&: RegMask);
2407 CSEMap.insert(N, Token: InsertToken);
2408 InsertNode(N);
2409 return SDValue(N, 0);
2410}
2411
2412SDValue SelectionDAG::getEHLabel(const SDLoc &dl, SDValue Root,
2413 MCSymbol *Label) {
2414 return getLabelNode(Opcode: ISD::EH_LABEL, dl, Root, Label);
2415}
2416
2417SDValue SelectionDAG::getLabelNode(unsigned Opcode, const SDLoc &dl,
2418 SDValue Root, MCSymbol *Label) {
2419 SDValue Ops[] = { Root };
2420 SDNodeKey ID(Opcode, getVTList(VT: MVT::Other), Ops);
2421 ID.AddPointer(P: Label);
2422 FoldingSetInsertToken InsertToken;
2423 if (SDNode *E = lookupNode(Key: ID, InsertToken))
2424 return SDValue(E, 0);
2425
2426 auto *N =
2427 newSDNode<LabelSDNode>(Args&: Opcode, Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: Label);
2428 createOperands(Node: N, Vals: Ops);
2429
2430 CSEMap.insert(N, Token: InsertToken);
2431 InsertNode(N);
2432 return SDValue(N, 0);
2433}
2434
2435SDValue SelectionDAG::getBlockAddress(const BlockAddress *BA, EVT VT,
2436 int64_t Offset, bool isTarget,
2437 unsigned TargetFlags) {
2438 unsigned Opc = isTarget ? ISD::TargetBlockAddress : ISD::BlockAddress;
2439 SDVTList VTs = getVTList(VT);
2440
2441 SDNodeKey ID(Opc, VTs, {});
2442 ID.AddPointer(P: BA);
2443 ID.AddInteger(I: Offset);
2444 ID.AddInteger(I: TargetFlags);
2445 FoldingSetInsertToken InsertToken;
2446 if (SDNode *E = lookupNode(Key: ID, InsertToken))
2447 return SDValue(E, 0);
2448
2449 auto *N = newSDNode<BlockAddressSDNode>(Args&: Opc, Args&: VTs, Args&: BA, Args&: Offset, Args&: TargetFlags);
2450 CSEMap.insert(N, Token: InsertToken);
2451 InsertNode(N);
2452 return SDValue(N, 0);
2453}
2454
2455SDValue SelectionDAG::getSrcValue(const Value *V) {
2456 SDNodeKey ID(ISD::SRCVALUE, getVTList(VT: MVT::Other), {});
2457 ID.AddPointer(P: V);
2458
2459 FoldingSetInsertToken InsertToken;
2460 if (SDNode *E = lookupNode(Key: ID, InsertToken))
2461 return SDValue(E, 0);
2462
2463 auto *N = newSDNode<SrcValueSDNode>(Args&: V);
2464 CSEMap.insert(N, Token: InsertToken);
2465 InsertNode(N);
2466 return SDValue(N, 0);
2467}
2468
2469SDValue SelectionDAG::getMDNode(const MDNode *MD) {
2470 SDNodeKey ID(ISD::MDNODE_SDNODE, getVTList(VT: MVT::Other), {});
2471 ID.AddPointer(P: MD);
2472
2473 FoldingSetInsertToken InsertToken;
2474 if (SDNode *E = lookupNode(Key: ID, InsertToken))
2475 return SDValue(E, 0);
2476
2477 auto *N = newSDNode<MDNodeSDNode>(Args&: MD);
2478 CSEMap.insert(N, Token: InsertToken);
2479 InsertNode(N);
2480 return SDValue(N, 0);
2481}
2482
2483SDValue SelectionDAG::getBitcast(EVT VT, SDValue V) {
2484 if (VT == V.getValueType())
2485 return V;
2486
2487 return getNode(Opcode: ISD::BITCAST, DL: SDLoc(V), VT, Operand: V);
2488}
2489
2490SDValue SelectionDAG::getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr,
2491 unsigned SrcAS, unsigned DestAS,
2492 const SDNodeFlags Flags) {
2493 SDVTList VTs = getVTList(VT);
2494 SDValue Ops[] = {Ptr};
2495 SDNodeKey ID(ISD::ADDRSPACECAST, VTs, Ops);
2496 ID.AddInteger(I: SrcAS);
2497 ID.AddInteger(I: DestAS);
2498
2499 FoldingSetInsertToken InsertToken;
2500 if (SDNode *E = lookupNode(Key: ID, DL: dl, InsertToken)) {
2501 E->intersectFlagsWith(Flags);
2502 return SDValue(E, 0);
2503 }
2504
2505 auto *N = newSDNode<AddrSpaceCastSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(),
2506 Args&: VTs, Args&: SrcAS, Args&: DestAS);
2507 N->setFlags(Flags);
2508 createOperands(Node: N, Vals: Ops);
2509
2510 CSEMap.insert(N, Token: InsertToken);
2511 InsertNode(N);
2512 return SDValue(N, 0);
2513}
2514
2515SDValue SelectionDAG::getFreeze(SDValue V) {
2516 return getNode(Opcode: ISD::FREEZE, DL: SDLoc(V), VT: V.getValueType(), Operand: V);
2517}
2518
2519SDValue SelectionDAG::getFreeze(SDValue V, const APInt &DemandedElts,
2520 UndefPoisonKind Kind) {
2521 if (isGuaranteedNotToBeUndefOrPoison(Op: V, DemandedElts, Kind))
2522 return V;
2523 return getFreeze(V);
2524}
2525
2526/// getShiftAmountOperand - Return the specified value casted to
2527/// the target's desired shift amount type.
2528SDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) {
2529 EVT OpTy = Op.getValueType();
2530 EVT ShTy = TLI->getShiftAmountTy(LHSTy, DL: getDataLayout());
2531 if (OpTy == ShTy || OpTy.isVector()) return Op;
2532
2533 return getZExtOrTrunc(Op, DL: SDLoc(Op), VT: ShTy);
2534}
2535
2536SDValue SelectionDAG::expandVAArg(SDNode *Node) {
2537 SDLoc dl(Node);
2538 const TargetLowering &TLI = getTargetLoweringInfo();
2539 const Value *V = cast<SrcValueSDNode>(Val: Node->getOperand(Num: 2))->getValue();
2540 EVT VT = Node->getValueType(ResNo: 0);
2541 SDValue Tmp1 = Node->getOperand(Num: 0);
2542 SDValue Tmp2 = Node->getOperand(Num: 1);
2543 const MaybeAlign MA(Node->getConstantOperandVal(Num: 3));
2544
2545 SDValue VAListLoad = getLoad(VT: TLI.getPointerTy(DL: getDataLayout()), dl, Chain: Tmp1,
2546 Ptr: Tmp2, PtrInfo: MachinePointerInfo(V));
2547 SDValue VAList = VAListLoad;
2548
2549 if (MA && *MA > TLI.getMinStackArgumentAlignment()) {
2550 VAList = getNode(Opcode: ISD::ADD, DL: dl, VT: VAList.getValueType(), N1: VAList,
2551 N2: getConstant(Val: MA->value() - 1, DL: dl, VT: VAList.getValueType()));
2552
2553 VAList = getNode(
2554 Opcode: ISD::AND, DL: dl, VT: VAList.getValueType(), N1: VAList,
2555 N2: getSignedConstant(Val: -(int64_t)MA->value(), DL: dl, VT: VAList.getValueType()));
2556 }
2557
2558 // Increment the pointer, VAList, to the next vaarg
2559 Tmp1 = getNode(Opcode: ISD::ADD, DL: dl, VT: VAList.getValueType(), N1: VAList,
2560 N2: getConstant(Val: getDataLayout().getTypeAllocSize(
2561 Ty: VT.getTypeForEVT(Context&: *getContext())),
2562 DL: dl, VT: VAList.getValueType()));
2563 // Store the incremented VAList to the legalized pointer
2564 Tmp1 =
2565 getStore(Chain: VAListLoad.getValue(R: 1), dl, Val: Tmp1, Ptr: Tmp2, PtrInfo: MachinePointerInfo(V));
2566 // Load the actual argument out of the pointer VAList
2567 return getLoad(VT, dl, Chain: Tmp1, Ptr: VAList, PtrInfo: MachinePointerInfo());
2568}
2569
2570SDValue SelectionDAG::expandVACopy(SDNode *Node) {
2571 SDLoc dl(Node);
2572 const TargetLowering &TLI = getTargetLoweringInfo();
2573 // This defaults to loading a pointer from the input and storing it to the
2574 // output, returning the chain.
2575 const Value *VD = cast<SrcValueSDNode>(Val: Node->getOperand(Num: 3))->getValue();
2576 const Value *VS = cast<SrcValueSDNode>(Val: Node->getOperand(Num: 4))->getValue();
2577 SDValue Tmp1 =
2578 getLoad(VT: TLI.getPointerTy(DL: getDataLayout()), dl, Chain: Node->getOperand(Num: 0),
2579 Ptr: Node->getOperand(Num: 2), PtrInfo: MachinePointerInfo(VS));
2580 return getStore(Chain: Tmp1.getValue(R: 1), dl, Val: Tmp1, Ptr: Node->getOperand(Num: 1),
2581 PtrInfo: MachinePointerInfo(VD));
2582}
2583
2584Align SelectionDAG::getReducedAlign(EVT VT, bool UseABI) {
2585 const DataLayout &DL = getDataLayout();
2586 Type *Ty = VT.getTypeForEVT(Context&: *getContext());
2587 Align RedAlign = UseABI ? DL.getABITypeAlign(Ty) : DL.getPrefTypeAlign(Ty);
2588
2589 if (TLI->isTypeLegal(VT) || !VT.isVector())
2590 return RedAlign;
2591
2592 const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering();
2593 const Align StackAlign = TFI->getStackAlign();
2594
2595 // See if we can choose a smaller ABI alignment in cases where it's an
2596 // illegal vector type that will get broken down.
2597 if (RedAlign > StackAlign) {
2598 EVT IntermediateVT;
2599 MVT RegisterVT;
2600 unsigned NumIntermediates;
2601 TLI->getVectorTypeBreakdown(Context&: *getContext(), VT, IntermediateVT,
2602 NumIntermediates, RegisterVT);
2603 Ty = IntermediateVT.getTypeForEVT(Context&: *getContext());
2604 Align RedAlign2 = UseABI ? DL.getABITypeAlign(Ty) : DL.getPrefTypeAlign(Ty);
2605 if (RedAlign2 < RedAlign)
2606 RedAlign = RedAlign2;
2607
2608 if (!getMachineFunction().getFrameInfo().isStackRealignable())
2609 // If the stack is not realignable, the alignment should be limited to the
2610 // StackAlignment
2611 RedAlign = std::min(a: RedAlign, b: StackAlign);
2612 }
2613
2614 return RedAlign;
2615}
2616
2617SDValue SelectionDAG::CreateStackTemporary(TypeSize Bytes, Align Alignment) {
2618 MachineFrameInfo &MFI = MF->getFrameInfo();
2619 const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering();
2620 int StackID = 0;
2621 if (Bytes.isScalable())
2622 StackID = TFI->getStackIDForScalableVectors();
2623 // The stack id gives an indication of whether the object is scalable or
2624 // not, so it's safe to pass in the minimum size here.
2625 int FrameIdx = MFI.CreateStackObject(Size: Bytes.getKnownMinValue(), Alignment,
2626 isSpillSlot: false, Alloca: nullptr, ID: StackID);
2627 return getFrameIndex(FI: FrameIdx, VT: TLI->getFrameIndexTy(DL: getDataLayout()));
2628}
2629
2630SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) {
2631 Type *Ty = VT.getTypeForEVT(Context&: *getContext());
2632 Align StackAlign =
2633 std::max(a: getDataLayout().getPrefTypeAlign(Ty), b: Align(minAlign));
2634 return CreateStackTemporary(Bytes: VT.getStoreSize(), Alignment: StackAlign);
2635}
2636
2637SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) {
2638 TypeSize VT1Size = VT1.getStoreSize();
2639 TypeSize VT2Size = VT2.getStoreSize();
2640 assert(VT1Size.isScalable() == VT2Size.isScalable() &&
2641 "Don't know how to choose the maximum size when creating a stack "
2642 "temporary");
2643 TypeSize Bytes = VT1Size.getKnownMinValue() > VT2Size.getKnownMinValue()
2644 ? VT1Size
2645 : VT2Size;
2646
2647 Type *Ty1 = VT1.getTypeForEVT(Context&: *getContext());
2648 Type *Ty2 = VT2.getTypeForEVT(Context&: *getContext());
2649 const DataLayout &DL = getDataLayout();
2650 Align Align = std::max(a: DL.getPrefTypeAlign(Ty: Ty1), b: DL.getPrefTypeAlign(Ty: Ty2));
2651 return CreateStackTemporary(Bytes, Alignment: Align);
2652}
2653
2654SDValue SelectionDAG::emitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
2655 const SDLoc &DL, SDValue Chain) {
2656 EVT SrcVT = SrcOp.getValueType();
2657 Type *DestType = DestVT.getTypeForEVT(Context&: *getContext());
2658 Align DestAlign = getDataLayout().getPrefTypeAlign(Ty: DestType);
2659
2660 // Create the stack frame object.
2661 Align SrcAlign =
2662 getDataLayout().getPrefTypeAlign(Ty: SrcVT.getTypeForEVT(Context&: *getContext()));
2663 SDValue FIPtr = CreateStackTemporary(Bytes: SlotVT.getStoreSize(), Alignment: SrcAlign);
2664
2665 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(Val&: FIPtr);
2666 int SPFI = StackPtrFI->getIndex();
2667 MachinePointerInfo PtrInfo =
2668 MachinePointerInfo::getFixedStack(MF&: getMachineFunction(), FI: SPFI);
2669
2670 // Emit a store to the stack slot. Use a truncstore if the input value is
2671 // later than DestVT.
2672 SDValue Store;
2673
2674 if (SrcVT.bitsGT(VT: SlotVT))
2675 Store = getTruncStore(Chain, dl: DL, Val: SrcOp, Ptr: FIPtr, PtrInfo, SVT: SlotVT, Alignment: SrcAlign);
2676 else {
2677 assert(SrcVT.bitsEq(SlotVT) && "Invalid store");
2678 Store = getStore(Chain, dl: DL, Val: SrcOp, Ptr: FIPtr, PtrInfo, Alignment: SrcAlign);
2679 }
2680
2681 // Result is a load from the stack slot.
2682 if (SlotVT.bitsEq(VT: DestVT))
2683 return getLoad(VT: DestVT, dl: DL, Chain: Store, Ptr: FIPtr, PtrInfo, Alignment: DestAlign);
2684
2685 assert(SlotVT.bitsLT(DestVT) && "Unknown extension!");
2686 return getExtLoad(ExtType: ISD::EXTLOAD, dl: DL, VT: DestVT, Chain: Store, Ptr: FIPtr, PtrInfo, MemVT: SlotVT,
2687 Alignment: DestAlign);
2688}
2689
2690SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1, SDValue N2,
2691 ISD::CondCode Cond, const SDLoc &dl,
2692 SDNodeFlags Flags) {
2693 EVT OpVT = N1.getValueType();
2694
2695 auto GetUndefBooleanConstant = [&]() {
2696 if (VT.getScalarType() == MVT::i1 ||
2697 TLI->getBooleanContents(Type: OpVT) ==
2698 TargetLowering::UndefinedBooleanContent)
2699 return getUNDEF(VT);
2700 // ZeroOrOne / ZeroOrNegative require specific values for the high bits,
2701 // so we cannot use getUNDEF(). Return zero instead.
2702 return getConstant(Val: 0, DL: dl, VT);
2703 };
2704
2705 // These setcc operations always fold.
2706 switch (Cond) {
2707 default: break;
2708 case ISD::SETFALSE:
2709 case ISD::SETFALSE2: return getBoolConstant(V: false, DL: dl, VT, OpVT);
2710 case ISD::SETTRUE:
2711 case ISD::SETTRUE2: return getBoolConstant(V: true, DL: dl, VT, OpVT);
2712
2713 case ISD::SETOEQ:
2714 case ISD::SETOGT:
2715 case ISD::SETOGE:
2716 case ISD::SETOLT:
2717 case ISD::SETOLE:
2718 case ISD::SETONE:
2719 case ISD::SETO:
2720 case ISD::SETUO:
2721 case ISD::SETUEQ:
2722 case ISD::SETUNE:
2723 assert(!OpVT.isInteger() && "Illegal setcc for integer!");
2724 break;
2725 }
2726
2727 if (OpVT.isInteger()) {
2728 // For EQ and NE, we can always pick a value for the undef to make the
2729 // predicate pass or fail, so we can return undef.
2730 // Matches behavior in llvm::ConstantFoldCompareInstruction.
2731 // icmp eq/ne X, undef -> undef.
2732 if ((N1.isUndef() || N2.isUndef()) &&
2733 (Cond == ISD::SETEQ || Cond == ISD::SETNE))
2734 return GetUndefBooleanConstant();
2735
2736 // If both operands are undef, we can return undef for int comparison.
2737 // icmp undef, undef -> undef.
2738 if (N1.isUndef() && N2.isUndef())
2739 return GetUndefBooleanConstant();
2740
2741 // icmp X, X -> true/false
2742 // icmp X, undef -> true/false because undef could be X.
2743 if (N1.isUndef() || N2.isUndef() || N1 == N2)
2744 return getBoolConstant(V: ISD::isTrueWhenEqual(Cond), DL: dl, VT, OpVT);
2745 }
2746
2747 if (ConstantSDNode *N2C = isConstOrConstSplat(N: N2)) {
2748 const APInt &C2 = N2C->getAPIntValue();
2749 if (ConstantSDNode *N1C = isConstOrConstSplat(N: N1)) {
2750 const APInt &C1 = N1C->getAPIntValue();
2751
2752 return getBoolConstant(V: ICmpInst::compare(LHS: C1, RHS: C2, Pred: getICmpCondCode(Pred: Cond)),
2753 DL: dl, VT, OpVT);
2754 }
2755 }
2756
2757 auto *N1CFP = dyn_cast<ConstantFPSDNode>(Val&: N1);
2758 auto *N2CFP = dyn_cast<ConstantFPSDNode>(Val&: N2);
2759
2760 if (N1CFP && N2CFP) {
2761 APFloat::cmpResult R = N1CFP->getValueAPF().compare(RHS: N2CFP->getValueAPF());
2762 switch (Cond) {
2763 default: break;
2764 case ISD::SETEQ: if (R==APFloat::cmpUnordered)
2765 return GetUndefBooleanConstant();
2766 [[fallthrough]];
2767 case ISD::SETOEQ: return getBoolConstant(V: R==APFloat::cmpEqual, DL: dl, VT,
2768 OpVT);
2769 case ISD::SETNE: if (R==APFloat::cmpUnordered)
2770 return GetUndefBooleanConstant();
2771 [[fallthrough]];
2772 case ISD::SETONE: return getBoolConstant(V: R==APFloat::cmpGreaterThan ||
2773 R==APFloat::cmpLessThan, DL: dl, VT,
2774 OpVT);
2775 case ISD::SETLT: if (R==APFloat::cmpUnordered)
2776 return GetUndefBooleanConstant();
2777 [[fallthrough]];
2778 case ISD::SETOLT: return getBoolConstant(V: R==APFloat::cmpLessThan, DL: dl, VT,
2779 OpVT);
2780 case ISD::SETGT: if (R==APFloat::cmpUnordered)
2781 return GetUndefBooleanConstant();
2782 [[fallthrough]];
2783 case ISD::SETOGT: return getBoolConstant(V: R==APFloat::cmpGreaterThan, DL: dl,
2784 VT, OpVT);
2785 case ISD::SETLE: if (R==APFloat::cmpUnordered)
2786 return GetUndefBooleanConstant();
2787 [[fallthrough]];
2788 case ISD::SETOLE: return getBoolConstant(V: R==APFloat::cmpLessThan ||
2789 R==APFloat::cmpEqual, DL: dl, VT,
2790 OpVT);
2791 case ISD::SETGE: if (R==APFloat::cmpUnordered)
2792 return GetUndefBooleanConstant();
2793 [[fallthrough]];
2794 case ISD::SETOGE: return getBoolConstant(V: R==APFloat::cmpGreaterThan ||
2795 R==APFloat::cmpEqual, DL: dl, VT, OpVT);
2796 case ISD::SETO: return getBoolConstant(V: R!=APFloat::cmpUnordered, DL: dl, VT,
2797 OpVT);
2798 case ISD::SETUO: return getBoolConstant(V: R==APFloat::cmpUnordered, DL: dl, VT,
2799 OpVT);
2800 case ISD::SETUEQ: return getBoolConstant(V: R==APFloat::cmpUnordered ||
2801 R==APFloat::cmpEqual, DL: dl, VT,
2802 OpVT);
2803 case ISD::SETUNE: return getBoolConstant(V: R!=APFloat::cmpEqual, DL: dl, VT,
2804 OpVT);
2805 case ISD::SETULT: return getBoolConstant(V: R==APFloat::cmpUnordered ||
2806 R==APFloat::cmpLessThan, DL: dl, VT,
2807 OpVT);
2808 case ISD::SETUGT: return getBoolConstant(V: R==APFloat::cmpGreaterThan ||
2809 R==APFloat::cmpUnordered, DL: dl, VT,
2810 OpVT);
2811 case ISD::SETULE: return getBoolConstant(V: R!=APFloat::cmpGreaterThan, DL: dl,
2812 VT, OpVT);
2813 case ISD::SETUGE: return getBoolConstant(V: R!=APFloat::cmpLessThan, DL: dl, VT,
2814 OpVT);
2815 }
2816 } else if (N1CFP && OpVT.isSimple() && !N2.isUndef()) {
2817 // Ensure that the constant occurs on the RHS.
2818 ISD::CondCode SwappedCond = ISD::getSetCCSwappedOperands(Operation: Cond);
2819 if (!TLI->isCondCodeLegal(CC: SwappedCond, VT: OpVT.getSimpleVT()))
2820 return SDValue();
2821 return getSetCC(DL: dl, VT, LHS: N2, RHS: N1, Cond: SwappedCond, /*Chain=*/{},
2822 /*IsSignaling=*/false, Flags);
2823 } else if ((N2CFP && N2CFP->getValueAPF().isNaN()) ||
2824 (OpVT.isFloatingPoint() && (N1.isUndef() || N2.isUndef()))) {
2825 // If an operand is known to be a nan (or undef that could be a nan), we can
2826 // fold it.
2827 // Choosing NaN for the undef will always make unordered comparison succeed
2828 // and ordered comparison fails.
2829 // Matches behavior in llvm::ConstantFoldCompareInstruction.
2830 switch (ISD::getUnorderedFlavor(Cond)) {
2831 default:
2832 llvm_unreachable("Unknown flavor!");
2833 case 0: // Known false.
2834 return getBoolConstant(V: false, DL: dl, VT, OpVT);
2835 case 1: // Known true.
2836 return getBoolConstant(V: true, DL: dl, VT, OpVT);
2837 case 2: // Undefined.
2838 return GetUndefBooleanConstant();
2839 }
2840 }
2841
2842 // Could not fold it.
2843 return SDValue();
2844}
2845
2846/// SignBitIsZero - Return true if the sign bit of Op is known to be zero. We
2847/// use this predicate to simplify operations downstream.
2848bool SelectionDAG::SignBitIsZero(SDValue Op, unsigned Depth) const {
2849 unsigned BitWidth = Op.getScalarValueSizeInBits();
2850 return MaskedValueIsZero(Op, Mask: APInt::getSignMask(BitWidth), Depth);
2851}
2852
2853// TODO: Should have argument to specify if sign bit of nan is ignorable.
2854bool SelectionDAG::SignBitIsZeroFP(SDValue Op, unsigned Depth) const {
2855 if (Depth >= MaxRecursionDepth)
2856 return false; // Limit search depth.
2857
2858 unsigned Opc = Op.getOpcode();
2859 switch (Opc) {
2860 case ISD::FABS:
2861 return true;
2862 case ISD::AssertNoFPClass: {
2863 FPClassTest NoFPClass =
2864 static_cast<FPClassTest>(Op.getConstantOperandVal(i: 1));
2865
2866 const FPClassTest TestMask = fcNan | fcNegative;
2867 return (NoFPClass & TestMask) == TestMask;
2868 }
2869 case ISD::ARITH_FENCE:
2870 return SignBitIsZeroFP(Op: Op.getOperand(i: 0), Depth: Depth + 1);
2871 case ISD::FEXP:
2872 case ISD::FEXP2:
2873 case ISD::FEXP10:
2874 return Op->getFlags().hasNoNaNs();
2875 case ISD::FMINNUM:
2876 case ISD::FMINNUM_IEEE:
2877 case ISD::FMINIMUM:
2878 case ISD::FMINIMUMNUM:
2879 return SignBitIsZeroFP(Op: Op.getOperand(i: 1), Depth: Depth + 1) &&
2880 SignBitIsZeroFP(Op: Op.getOperand(i: 0), Depth: Depth + 1);
2881 case ISD::FMAXNUM:
2882 case ISD::FMAXNUM_IEEE:
2883 case ISD::FMAXIMUM:
2884 case ISD::FMAXIMUMNUM:
2885 // TODO: If we can ignore the sign bit of nans, only one side being known 0
2886 // is sufficient.
2887 return SignBitIsZeroFP(Op: Op.getOperand(i: 1), Depth: Depth + 1) &&
2888 SignBitIsZeroFP(Op: Op.getOperand(i: 0), Depth: Depth + 1);
2889 default:
2890 return false;
2891 }
2892
2893 llvm_unreachable("covered opcode switch");
2894}
2895
2896/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero. We use
2897/// this predicate to simplify operations downstream. Mask is known to be zero
2898/// for bits that V cannot have.
2899bool SelectionDAG::MaskedValueIsZero(SDValue V, const APInt &Mask,
2900 unsigned Depth) const {
2901 return Mask.isSubsetOf(RHS: computeKnownBits(Op: V, Depth).Zero);
2902}
2903
2904/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero in
2905/// DemandedElts. We use this predicate to simplify operations downstream.
2906/// Mask is known to be zero for bits that V cannot have.
2907bool SelectionDAG::MaskedValueIsZero(SDValue V, const APInt &Mask,
2908 const APInt &DemandedElts,
2909 unsigned Depth) const {
2910 return Mask.isSubsetOf(RHS: computeKnownBits(Op: V, DemandedElts, Depth).Zero);
2911}
2912
2913/// MaskedVectorIsZero - Return true if 'Op' is known to be zero in
2914/// DemandedElts. We use this predicate to simplify operations downstream.
2915bool SelectionDAG::MaskedVectorIsZero(SDValue V, const APInt &DemandedElts,
2916 unsigned Depth /* = 0 */) const {
2917 return computeKnownBits(Op: V, DemandedElts, Depth).isZero();
2918}
2919
2920/// MaskedValueIsAllOnes - Return true if '(Op & Mask) == Mask'.
2921bool SelectionDAG::MaskedValueIsAllOnes(SDValue V, const APInt &Mask,
2922 unsigned Depth) const {
2923 return Mask.isSubsetOf(RHS: computeKnownBits(Op: V, Depth).One);
2924}
2925
2926APInt SelectionDAG::computeVectorKnownZeroElements(SDValue Op,
2927 const APInt &DemandedElts,
2928 unsigned Depth) const {
2929 EVT VT = Op.getValueType();
2930 assert(VT.isVector() && !VT.isScalableVector() && "Only for fixed vectors!");
2931
2932 unsigned NumElts = VT.getVectorNumElements();
2933 assert(DemandedElts.getBitWidth() == NumElts && "Unexpected demanded mask.");
2934
2935 APInt KnownZeroElements = APInt::getZero(numBits: NumElts);
2936 for (unsigned EltIdx = 0; EltIdx != NumElts; ++EltIdx) {
2937 if (!DemandedElts[EltIdx])
2938 continue; // Don't query elements that are not demanded.
2939 APInt Mask = APInt::getOneBitSet(numBits: NumElts, BitNo: EltIdx);
2940 if (MaskedVectorIsZero(V: Op, DemandedElts: Mask, Depth))
2941 KnownZeroElements.setBit(EltIdx);
2942 }
2943 return KnownZeroElements;
2944}
2945
2946/// isSplatValue - Return true if the vector V has the same value
2947/// across all DemandedElts. For scalable vectors, we don't know the
2948/// number of lanes at compile time. Instead, we use a 1 bit APInt
2949/// to represent a conservative value for all lanes; that is, that
2950/// one bit value is implicitly splatted across all lanes.
2951bool SelectionDAG::isSplatValue(SDValue V, const APInt &DemandedElts,
2952 APInt &UndefElts, unsigned Depth) const {
2953 unsigned Opcode = V.getOpcode();
2954 EVT VT = V.getValueType();
2955 assert(VT.isVector() && "Vector type expected");
2956 assert((!VT.isScalableVector() || DemandedElts.getBitWidth() == 1) &&
2957 "scalable demanded bits are ignored");
2958
2959 if (!DemandedElts)
2960 return false; // No demanded elts, better to assume we don't know anything.
2961
2962 if (Depth >= MaxRecursionDepth)
2963 return false; // Limit search depth.
2964
2965 // Deal with some common cases here that work for both fixed and scalable
2966 // vector types.
2967 switch (Opcode) {
2968 case ISD::SPLAT_VECTOR:
2969 UndefElts = V.getOperand(i: 0).isUndef()
2970 ? APInt::getAllOnes(numBits: DemandedElts.getBitWidth())
2971 : APInt(DemandedElts.getBitWidth(), 0);
2972 return true;
2973 case ISD::ADD:
2974 case ISD::SUB:
2975 case ISD::AND:
2976 case ISD::XOR:
2977 case ISD::OR: {
2978 APInt UndefLHS, UndefRHS;
2979 SDValue LHS = V.getOperand(i: 0);
2980 SDValue RHS = V.getOperand(i: 1);
2981 // Only recognize splats with the same demanded undef elements for both
2982 // operands, otherwise we might fail to handle binop-specific undef
2983 // handling.
2984 // e.g. (and undef, 0) -> 0 etc.
2985 if (isSplatValue(V: LHS, DemandedElts, UndefElts&: UndefLHS, Depth: Depth + 1) &&
2986 isSplatValue(V: RHS, DemandedElts, UndefElts&: UndefRHS, Depth: Depth + 1) &&
2987 (DemandedElts & UndefLHS) == (DemandedElts & UndefRHS)) {
2988 UndefElts = UndefLHS | UndefRHS;
2989 return true;
2990 }
2991 return false;
2992 }
2993 case ISD::ABS:
2994 case ISD::ABS_MIN_POISON:
2995 case ISD::TRUNCATE:
2996 case ISD::SIGN_EXTEND:
2997 case ISD::ZERO_EXTEND:
2998 return isSplatValue(V: V.getOperand(i: 0), DemandedElts, UndefElts, Depth: Depth + 1);
2999 default:
3000 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
3001 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)
3002 return TLI->isSplatValueForTargetNode(Op: V, DemandedElts, UndefElts, DAG: *this,
3003 Depth);
3004 break;
3005 }
3006
3007 // We don't support other cases than those above for scalable vectors at
3008 // the moment.
3009 if (VT.isScalableVector())
3010 return false;
3011
3012 unsigned NumElts = VT.getVectorNumElements();
3013 assert(NumElts == DemandedElts.getBitWidth() && "Vector size mismatch");
3014 UndefElts = APInt::getZero(numBits: NumElts);
3015
3016 switch (Opcode) {
3017 case ISD::BUILD_VECTOR: {
3018 SDValue Scl;
3019 for (unsigned i = 0; i != NumElts; ++i) {
3020 SDValue Op = V.getOperand(i);
3021 if (Op.isUndef()) {
3022 UndefElts.setBit(i);
3023 continue;
3024 }
3025 if (!DemandedElts[i])
3026 continue;
3027 if (Scl && Scl != Op)
3028 return false;
3029 Scl = Op;
3030 }
3031 return true;
3032 }
3033 case ISD::VECTOR_SHUFFLE: {
3034 // Check if this is a shuffle node doing a splat or a shuffle of a splat.
3035 APInt DemandedLHS = APInt::getZero(numBits: NumElts);
3036 APInt DemandedRHS = APInt::getZero(numBits: NumElts);
3037 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Val&: V)->getMask();
3038 for (int i = 0; i != (int)NumElts; ++i) {
3039 int M = Mask[i];
3040 if (M < 0) {
3041 UndefElts.setBit(i);
3042 continue;
3043 }
3044 if (!DemandedElts[i])
3045 continue;
3046 if (M < (int)NumElts)
3047 DemandedLHS.setBit(M);
3048 else
3049 DemandedRHS.setBit(M - NumElts);
3050 }
3051
3052 // If we aren't demanding either op, assume there's no splat.
3053 // If we are demanding both ops, assume there's no splat.
3054 if ((DemandedLHS.isZero() && DemandedRHS.isZero()) ||
3055 (!DemandedLHS.isZero() && !DemandedRHS.isZero()))
3056 return false;
3057
3058 // See if the demanded elts of the source op is a splat or we only demand
3059 // one element, which should always be a splat.
3060 // TODO: Handle source ops splats with undefs.
3061 auto CheckSplatSrc = [&](SDValue Src, const APInt &SrcElts) {
3062 APInt SrcUndefs;
3063 return (SrcElts.popcount() == 1) ||
3064 (isSplatValue(V: Src, DemandedElts: SrcElts, UndefElts&: SrcUndefs, Depth: Depth + 1) &&
3065 (SrcElts & SrcUndefs).isZero());
3066 };
3067 if (!DemandedLHS.isZero())
3068 return CheckSplatSrc(V.getOperand(i: 0), DemandedLHS);
3069 return CheckSplatSrc(V.getOperand(i: 1), DemandedRHS);
3070 }
3071 case ISD::EXTRACT_SUBVECTOR: {
3072 // Offset the demanded elts by the subvector index.
3073 SDValue Src = V.getOperand(i: 0);
3074 // We don't support scalable vectors at the moment.
3075 if (Src.getValueType().isScalableVector())
3076 return false;
3077 uint64_t Idx = V.getConstantOperandVal(i: 1);
3078 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3079 APInt UndefSrcElts;
3080 APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
3081 if (isSplatValue(V: Src, DemandedElts: DemandedSrcElts, UndefElts&: UndefSrcElts, Depth: Depth + 1)) {
3082 UndefElts = UndefSrcElts.extractBits(numBits: NumElts, bitPosition: Idx);
3083 return true;
3084 }
3085 break;
3086 }
3087 case ISD::ANY_EXTEND_VECTOR_INREG:
3088 case ISD::SIGN_EXTEND_VECTOR_INREG:
3089 case ISD::ZERO_EXTEND_VECTOR_INREG: {
3090 // Widen the demanded elts by the src element count.
3091 SDValue Src = V.getOperand(i: 0);
3092 // We don't support scalable vectors at the moment.
3093 if (Src.getValueType().isScalableVector())
3094 return false;
3095 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3096 APInt UndefSrcElts;
3097 APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts);
3098 if (isSplatValue(V: Src, DemandedElts: DemandedSrcElts, UndefElts&: UndefSrcElts, Depth: Depth + 1)) {
3099 UndefElts = UndefSrcElts.trunc(width: NumElts);
3100 return true;
3101 }
3102 break;
3103 }
3104 case ISD::BITCAST: {
3105 SDValue Src = V.getOperand(i: 0);
3106 EVT SrcVT = Src.getValueType();
3107 unsigned SrcBitWidth = SrcVT.getScalarSizeInBits();
3108 unsigned BitWidth = VT.getScalarSizeInBits();
3109
3110 // Ignore bitcasts from unsupported types.
3111 // TODO: Add fp support?
3112 if (!SrcVT.isVector() || !SrcVT.isInteger() || !VT.isInteger())
3113 break;
3114
3115 // Bitcast 'small element' vector to 'large element' vector.
3116 if ((BitWidth % SrcBitWidth) == 0) {
3117 // See if each sub element is a splat.
3118 unsigned Scale = BitWidth / SrcBitWidth;
3119 unsigned NumSrcElts = SrcVT.getVectorNumElements();
3120 APInt ScaledDemandedElts =
3121 APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumSrcElts);
3122 for (unsigned I = 0; I != Scale; ++I) {
3123 APInt SubUndefElts;
3124 APInt SubDemandedElt = APInt::getOneBitSet(numBits: Scale, BitNo: I);
3125 APInt SubDemandedElts = APInt::getSplat(NewLen: NumSrcElts, V: SubDemandedElt);
3126 SubDemandedElts &= ScaledDemandedElts;
3127 if (!isSplatValue(V: Src, DemandedElts: SubDemandedElts, UndefElts&: SubUndefElts, Depth: Depth + 1))
3128 return false;
3129 // TODO: Add support for merging sub undef elements.
3130 if (!SubUndefElts.isZero())
3131 return false;
3132 }
3133 return true;
3134 }
3135 break;
3136 }
3137 }
3138
3139 return false;
3140}
3141
3142/// Helper wrapper to main isSplatValue function.
3143bool SelectionDAG::isSplatValue(SDValue V, bool AllowUndefs) const {
3144 EVT VT = V.getValueType();
3145 assert(VT.isVector() && "Vector type expected");
3146
3147 APInt UndefElts;
3148 // Since the number of lanes in a scalable vector is unknown at compile time,
3149 // we track one bit which is implicitly broadcast to all lanes. This means
3150 // that all lanes in a scalable vector are considered demanded.
3151 APInt DemandedElts
3152 = APInt::getAllOnes(numBits: VT.isScalableVector() ? 1 : VT.getVectorNumElements());
3153 return isSplatValue(V, DemandedElts, UndefElts) &&
3154 (AllowUndefs || !UndefElts);
3155}
3156
3157SDValue SelectionDAG::getSplatSourceVector(SDValue V, int &SplatIdx) {
3158 V = peekThroughExtractSubvectors(V);
3159
3160 EVT VT = V.getValueType();
3161 unsigned Opcode = V.getOpcode();
3162 switch (Opcode) {
3163 default: {
3164 APInt UndefElts;
3165 // Since the number of lanes in a scalable vector is unknown at compile time,
3166 // we track one bit which is implicitly broadcast to all lanes. This means
3167 // that all lanes in a scalable vector are considered demanded.
3168 APInt DemandedElts
3169 = APInt::getAllOnes(numBits: VT.isScalableVector() ? 1 : VT.getVectorNumElements());
3170
3171 if (isSplatValue(V, DemandedElts, UndefElts)) {
3172 if (VT.isScalableVector()) {
3173 // DemandedElts and UndefElts are ignored for scalable vectors, since
3174 // the only supported cases are SPLAT_VECTOR nodes.
3175 SplatIdx = 0;
3176 } else {
3177 // Handle case where all demanded elements are UNDEF.
3178 if (DemandedElts.isSubsetOf(RHS: UndefElts)) {
3179 SplatIdx = 0;
3180 return getUNDEF(VT);
3181 }
3182 SplatIdx = (UndefElts & DemandedElts).countr_one();
3183 }
3184 return V;
3185 }
3186 break;
3187 }
3188 case ISD::SPLAT_VECTOR:
3189 SplatIdx = 0;
3190 return V;
3191 case ISD::VECTOR_SHUFFLE: {
3192 assert(!VT.isScalableVector());
3193 // Check if this is a shuffle node doing a splat.
3194 // TODO - remove this and rely purely on SelectionDAG::isSplatValue,
3195 // getTargetVShiftNode currently struggles without the splat source.
3196 auto *SVN = cast<ShuffleVectorSDNode>(Val&: V);
3197 if (!SVN->isSplat())
3198 break;
3199 int Idx = SVN->getSplatIndex();
3200 int NumElts = V.getValueType().getVectorNumElements();
3201 SplatIdx = Idx % NumElts;
3202 return V.getOperand(i: Idx / NumElts);
3203 }
3204 }
3205
3206 return SDValue();
3207}
3208
3209SDValue SelectionDAG::getSplatValue(SDValue V, bool LegalTypes) {
3210 int SplatIdx;
3211 if (SDValue SrcVector = getSplatSourceVector(V, SplatIdx)) {
3212 EVT SVT = SrcVector.getValueType().getScalarType();
3213 EVT LegalSVT = SVT;
3214 if (LegalTypes && !TLI->isTypeLegal(VT: SVT)) {
3215 if (!SVT.isInteger())
3216 return SDValue();
3217 LegalSVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT: LegalSVT);
3218 if (LegalSVT.bitsLT(VT: SVT))
3219 return SDValue();
3220 }
3221 return getExtractVectorElt(DL: SDLoc(V), VT: LegalSVT, Vec: SrcVector, Idx: SplatIdx);
3222 }
3223 return SDValue();
3224}
3225
3226std::optional<ConstantRange>
3227SelectionDAG::getValidShiftAmountRange(SDValue V, const APInt &DemandedElts,
3228 unsigned Depth) const {
3229 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
3230 V.getOpcode() == ISD::SRA) &&
3231 "Unknown shift node");
3232 // Shifting more than the bitwidth is not valid.
3233 unsigned BitWidth = V.getScalarValueSizeInBits();
3234
3235 if (auto *Cst = dyn_cast<ConstantSDNode>(Val: V.getOperand(i: 1))) {
3236 const APInt &ShAmt = Cst->getAPIntValue();
3237 if (ShAmt.uge(RHS: BitWidth))
3238 return std::nullopt;
3239 return ConstantRange(ShAmt);
3240 }
3241
3242 if (auto *BV = dyn_cast<BuildVectorSDNode>(Val: V.getOperand(i: 1))) {
3243 const APInt *MinAmt = nullptr, *MaxAmt = nullptr;
3244 for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) {
3245 if (!DemandedElts[i])
3246 continue;
3247 auto *SA = dyn_cast<ConstantSDNode>(Val: BV->getOperand(Num: i));
3248 if (!SA) {
3249 MinAmt = MaxAmt = nullptr;
3250 break;
3251 }
3252 const APInt &ShAmt = SA->getAPIntValue();
3253 if (ShAmt.uge(RHS: BitWidth))
3254 return std::nullopt;
3255 if (!MinAmt || MinAmt->ugt(RHS: ShAmt))
3256 MinAmt = &ShAmt;
3257 if (!MaxAmt || MaxAmt->ult(RHS: ShAmt))
3258 MaxAmt = &ShAmt;
3259 }
3260 assert(((!MinAmt && !MaxAmt) || (MinAmt && MaxAmt)) &&
3261 "Failed to find matching min/max shift amounts");
3262 if (MinAmt && MaxAmt)
3263 return ConstantRange(*MinAmt, *MaxAmt + 1);
3264 }
3265
3266 // Use computeKnownBits to find a hidden constant/knownbits (usually type
3267 // legalized). e.g. Hidden behind multiple bitcasts/build_vector/casts etc.
3268 KnownBits KnownAmt = computeKnownBits(Op: V.getOperand(i: 1), DemandedElts, Depth);
3269 if (KnownAmt.getMaxValue().ult(RHS: BitWidth))
3270 return ConstantRange::fromKnownBits(Known: KnownAmt, /*IsSigned=*/false);
3271
3272 return std::nullopt;
3273}
3274
3275std::optional<unsigned>
3276SelectionDAG::getValidShiftAmount(SDValue V, const APInt &DemandedElts,
3277 unsigned Depth) const {
3278 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
3279 V.getOpcode() == ISD::SRA) &&
3280 "Unknown shift node");
3281 if (std::optional<ConstantRange> AmtRange =
3282 getValidShiftAmountRange(V, DemandedElts, Depth))
3283 if (const APInt *ShAmt = AmtRange->getSingleElement())
3284 return ShAmt->getZExtValue();
3285 return std::nullopt;
3286}
3287
3288std::optional<unsigned>
3289SelectionDAG::getValidShiftAmount(SDValue V, unsigned Depth) const {
3290 APInt DemandedElts = getDemandAllEltsMask(V);
3291 return getValidShiftAmount(V, DemandedElts, Depth);
3292}
3293
3294std::optional<unsigned>
3295SelectionDAG::getValidMinimumShiftAmount(SDValue V, const APInt &DemandedElts,
3296 unsigned Depth) const {
3297 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
3298 V.getOpcode() == ISD::SRA) &&
3299 "Unknown shift node");
3300 if (std::optional<ConstantRange> AmtRange =
3301 getValidShiftAmountRange(V, DemandedElts, Depth))
3302 return AmtRange->getUnsignedMin().getZExtValue();
3303 return std::nullopt;
3304}
3305
3306std::optional<unsigned>
3307SelectionDAG::getValidMinimumShiftAmount(SDValue V, unsigned Depth) const {
3308 APInt DemandedElts = getDemandAllEltsMask(V);
3309 return getValidMinimumShiftAmount(V, DemandedElts, Depth);
3310}
3311
3312std::optional<unsigned>
3313SelectionDAG::getValidMaximumShiftAmount(SDValue V, const APInt &DemandedElts,
3314 unsigned Depth) const {
3315 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
3316 V.getOpcode() == ISD::SRA) &&
3317 "Unknown shift node");
3318 if (std::optional<ConstantRange> AmtRange =
3319 getValidShiftAmountRange(V, DemandedElts, Depth))
3320 return AmtRange->getUnsignedMax().getZExtValue();
3321 return std::nullopt;
3322}
3323
3324std::optional<unsigned>
3325SelectionDAG::getValidMaximumShiftAmount(SDValue V, unsigned Depth) const {
3326 APInt DemandedElts = getDemandAllEltsMask(V);
3327 return getValidMaximumShiftAmount(V, DemandedElts, Depth);
3328}
3329
3330/// Determine which bits of Op are known to be either zero or one and return
3331/// them in Known. For vectors, the known bits are those that are shared by
3332/// every vector element.
3333KnownBits SelectionDAG::computeKnownBits(SDValue Op, unsigned Depth) const {
3334 APInt DemandedElts = getDemandAllEltsMask(V: Op);
3335 return computeKnownBits(Op, DemandedElts, Depth);
3336}
3337
3338/// Determine which bits of Op are known to be either zero or one and return
3339/// them in Known. The DemandedElts argument allows us to only collect the known
3340/// bits that are shared by the requested vector elements.
3341KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts,
3342 unsigned Depth) const {
3343 unsigned BitWidth = Op.getScalarValueSizeInBits();
3344
3345 KnownBits Known(BitWidth); // Don't know anything.
3346
3347 if (auto OptAPInt = Op->bitcastToAPInt()) {
3348 // We know all of the bits for a constant!
3349 APInt V = *std::move(OptAPInt);
3350
3351 // Swap the low-order and high-order double of a ppc_fp128 when casting to
3352 // i128, see #44482.
3353 //
3354 // A ppc_fp128 is two doubles, with the high-order double stored at the
3355 // lower address. Reading that as an integer therefore puts the high-order
3356 // double in the high 64 bits on big-endian targets and in the low 64 bits
3357 // on little-endian targets.
3358 //
3359 // But APFloat::bitcastToAPInt is endianness-agnostic and always places the
3360 // high-order double in the low 64 bits. Hence the two doubles must be
3361 // flipped on big-endian targets.
3362 if (getDataLayout().isBigEndian() && Op.getValueType() == MVT::ppcf128)
3363 V = V.rotl(rotateAmt: 64);
3364
3365 return KnownBits::makeConstant(C: V);
3366 }
3367
3368 if (Depth >= MaxRecursionDepth)
3369 return Known; // Limit search depth.
3370
3371 KnownBits Known2;
3372 unsigned NumElts = DemandedElts.getBitWidth();
3373 assert((!Op.getValueType().isScalableVector() || NumElts == 1) &&
3374 "DemandedElts for scalable vectors must be 1 to represent all lanes");
3375 assert((!Op.getValueType().isFixedLengthVector() ||
3376 NumElts == Op.getValueType().getVectorNumElements()) &&
3377 "Unexpected vector size");
3378
3379 if (!DemandedElts)
3380 return Known; // No demanded elts, better to assume we don't know anything.
3381
3382 unsigned Opcode = Op.getOpcode();
3383 switch (Opcode) {
3384 case ISD::FREEZE: {
3385 if (isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 0), DemandedElts,
3386 Kind: UndefPoisonKind::UndefOrPoison))
3387 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3388 break;
3389 }
3390 case ISD::MERGE_VALUES:
3391 return computeKnownBits(Op: Op.getOperand(i: Op.getResNo()), DemandedElts,
3392 Depth: Depth + 1);
3393 case ISD::SPLAT_VECTOR: {
3394 SDValue SrcOp = Op.getOperand(i: 0);
3395 assert(SrcOp.getValueSizeInBits() >= BitWidth &&
3396 "Expected SPLAT_VECTOR implicit truncation");
3397 // Implicitly truncate the bits to match the official semantics of
3398 // SPLAT_VECTOR.
3399 Known = computeKnownBits(Op: SrcOp, Depth: Depth + 1).trunc(BitWidth);
3400 break;
3401 }
3402 case ISD::SPLAT_VECTOR_PARTS: {
3403 unsigned ScalarSize = Op.getOperand(i: 0).getScalarValueSizeInBits();
3404 assert(ScalarSize * Op.getNumOperands() == BitWidth &&
3405 "Expected SPLAT_VECTOR_PARTS scalars to cover element width");
3406 for (auto [I, SrcOp] : enumerate(First: Op->ops())) {
3407 Known.insertBits(SubBits: computeKnownBits(Op: SrcOp, Depth: Depth + 1), BitPosition: ScalarSize * I);
3408 }
3409 break;
3410 }
3411 case ISD::STEP_VECTOR: {
3412 const APInt &Step = Op.getConstantOperandAPInt(i: 0);
3413
3414 if (Step.isPowerOf2())
3415 Known.Zero.setLowBits(Step.logBase2());
3416
3417 const Function &F = getMachineFunction().getFunction();
3418
3419 if (!isUIntN(N: BitWidth, x: Op.getValueType().getVectorMinNumElements()))
3420 break;
3421 const APInt MinNumElts =
3422 APInt(BitWidth, Op.getValueType().getVectorMinNumElements());
3423
3424 bool Overflow;
3425 const APInt MaxNumElts = getVScaleRange(F: &F, BitWidth)
3426 .getUnsignedMax()
3427 .umul_ov(RHS: MinNumElts, Overflow);
3428 if (Overflow)
3429 break;
3430
3431 const APInt MaxValue = (MaxNumElts - 1).umul_ov(RHS: Step, Overflow);
3432 if (Overflow)
3433 break;
3434
3435 Known.Zero.setHighBits(MaxValue.countl_zero());
3436 break;
3437 }
3438 case ISD::BUILD_VECTOR:
3439 assert(!Op.getValueType().isScalableVector());
3440 // Collect the known bits that are shared by every demanded vector element.
3441 Known.setAllConflict();
3442 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
3443 if (!DemandedElts[i])
3444 continue;
3445
3446 SDValue SrcOp = Op.getOperand(i);
3447 if (SrcOp.getOpcode() == ISD::POISON)
3448 continue;
3449
3450 Known2 = computeKnownBits(Op: SrcOp, Depth: Depth + 1);
3451
3452 // BUILD_VECTOR can implicitly truncate sources, we must handle this.
3453 if (SrcOp.getValueSizeInBits() != BitWidth) {
3454 assert(SrcOp.getValueSizeInBits() > BitWidth &&
3455 "Expected BUILD_VECTOR implicit truncation");
3456 Known2 = Known2.trunc(BitWidth);
3457 }
3458
3459 // Known bits are the values that are shared by every demanded element.
3460 Known = Known.intersectWith(RHS: Known2);
3461
3462 // If we don't know any bits, early out.
3463 if (Known.isUnknown())
3464 break;
3465 }
3466
3467 // If every demanded element was poison, we know nothing.
3468 if (Known.hasConflict())
3469 Known.resetAll();
3470 break;
3471 case ISD::VECTOR_COMPRESS: {
3472 SDValue Vec = Op.getOperand(i: 0);
3473 SDValue PassThru = Op.getOperand(i: 2);
3474 Known = computeKnownBits(Op: PassThru, DemandedElts, Depth: Depth + 1);
3475 // If we don't know any bits, early out.
3476 if (Known.isUnknown())
3477 break;
3478 Known2 = computeKnownBits(Op: Vec, Depth: Depth + 1);
3479 Known = Known.intersectWith(RHS: Known2);
3480 break;
3481 }
3482 case ISD::VECTOR_SHUFFLE: {
3483 assert(!Op.getValueType().isScalableVector());
3484 // Collect the known bits that are shared by every vector element referenced
3485 // by the shuffle.
3486 APInt DemandedLHS, DemandedRHS;
3487 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
3488 assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
3489 if (!getShuffleDemandedElts(SrcWidth: NumElts, Mask: SVN->getMask(), DemandedElts,
3490 DemandedLHS, DemandedRHS))
3491 break;
3492
3493 // Known bits are the values that are shared by every demanded element.
3494 Known.setAllConflict();
3495 if (!!DemandedLHS) {
3496 SDValue LHS = Op.getOperand(i: 0);
3497 Known2 = computeKnownBits(Op: LHS, DemandedElts: DemandedLHS, Depth: Depth + 1);
3498 Known = Known.intersectWith(RHS: Known2);
3499 }
3500 // If we don't know any bits, early out.
3501 if (Known.isUnknown())
3502 break;
3503 if (!!DemandedRHS) {
3504 SDValue RHS = Op.getOperand(i: 1);
3505 Known2 = computeKnownBits(Op: RHS, DemandedElts: DemandedRHS, Depth: Depth + 1);
3506 Known = Known.intersectWith(RHS: Known2);
3507 }
3508 break;
3509 }
3510 case ISD::VSCALE: {
3511 const Function &F = getMachineFunction().getFunction();
3512 const APInt &Multiplier = Op.getConstantOperandAPInt(i: 0);
3513 Known = getVScaleRange(F: &F, BitWidth).multiply(Other: Multiplier).toKnownBits();
3514 break;
3515 }
3516 case ISD::CONCAT_VECTORS: {
3517 if (Op.getValueType().isScalableVector())
3518 break;
3519 // Split DemandedElts and test each of the demanded subvectors.
3520 Known.setAllConflict();
3521 EVT SubVectorVT = Op.getOperand(i: 0).getValueType();
3522 unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements();
3523 unsigned NumSubVectors = Op.getNumOperands();
3524 for (unsigned i = 0; i != NumSubVectors; ++i) {
3525 APInt DemandedSub =
3526 DemandedElts.extractBits(numBits: NumSubVectorElts, bitPosition: i * NumSubVectorElts);
3527 if (!!DemandedSub) {
3528 SDValue Sub = Op.getOperand(i);
3529 Known2 = computeKnownBits(Op: Sub, DemandedElts: DemandedSub, Depth: Depth + 1);
3530 Known = Known.intersectWith(RHS: Known2);
3531 }
3532 // If we don't know any bits, early out.
3533 if (Known.isUnknown())
3534 break;
3535 }
3536 break;
3537 }
3538 case ISD::INSERT_SUBVECTOR: {
3539 if (Op.getValueType().isScalableVector())
3540 break;
3541 // Demand any elements from the subvector and the remainder from the src its
3542 // inserted into.
3543 SDValue Src = Op.getOperand(i: 0);
3544 SDValue Sub = Op.getOperand(i: 1);
3545 uint64_t Idx = Op.getConstantOperandVal(i: 2);
3546 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
3547 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
3548 APInt DemandedSrcElts = DemandedElts;
3549 DemandedSrcElts.clearBits(LoBit: Idx, HiBit: Idx + NumSubElts);
3550
3551 Known.setAllConflict();
3552 if (!!DemandedSubElts) {
3553 Known = computeKnownBits(Op: Sub, DemandedElts: DemandedSubElts, Depth: Depth + 1);
3554 if (Known.isUnknown())
3555 break; // early-out.
3556 }
3557 if (!!DemandedSrcElts) {
3558 Known2 = computeKnownBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
3559 Known = Known.intersectWith(RHS: Known2);
3560 }
3561 break;
3562 }
3563 case ISD::EXTRACT_SUBVECTOR: {
3564 // Offset the demanded elts by the subvector index.
3565 SDValue Src = Op.getOperand(i: 0);
3566
3567 APInt DemandedSrcElts;
3568 if (Src.getValueType().isScalableVector())
3569 DemandedSrcElts = APInt(1, 1); // <=> 'demand all elements'
3570 else {
3571 uint64_t Idx = Op.getConstantOperandVal(i: 1);
3572 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3573 DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
3574 }
3575 Known = computeKnownBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
3576 break;
3577 }
3578 case ISD::SCALAR_TO_VECTOR: {
3579 if (Op.getValueType().isScalableVector())
3580 break;
3581 // We know about scalar_to_vector as much as we know about it source,
3582 // which becomes the first element of otherwise unknown vector.
3583 if (DemandedElts != 1)
3584 break;
3585
3586 SDValue N0 = Op.getOperand(i: 0);
3587 Known = computeKnownBits(Op: N0, Depth: Depth + 1);
3588 if (N0.getValueSizeInBits() != BitWidth)
3589 Known = Known.trunc(BitWidth);
3590
3591 break;
3592 }
3593 case ISD::BITCAST: {
3594 if (Op.getValueType().isScalableVector())
3595 break;
3596
3597 SDValue N0 = Op.getOperand(i: 0);
3598 EVT SubVT = N0.getValueType();
3599 unsigned SubBitWidth = SubVT.getScalarSizeInBits();
3600
3601 // Ignore bitcasts from unsupported types.
3602 if (!(SubVT.isInteger() || SubVT.isFloatingPoint()))
3603 break;
3604
3605 // Fast handling of 'identity' bitcasts.
3606 if (BitWidth == SubBitWidth) {
3607 Known = computeKnownBits(Op: N0, DemandedElts, Depth: Depth + 1);
3608 break;
3609 }
3610
3611 bool IsLE = getDataLayout().isLittleEndian();
3612
3613 // Bitcast 'small element' vector to 'large element' scalar/vector.
3614 if ((BitWidth % SubBitWidth) == 0) {
3615 assert(N0.getValueType().isVector() && "Expected bitcast from vector");
3616
3617 // Collect known bits for the (larger) output by collecting the known
3618 // bits from each set of sub elements and shift these into place.
3619 // We need to separately call computeKnownBits for each set of
3620 // sub elements as the knownbits for each is likely to be different.
3621 unsigned SubScale = BitWidth / SubBitWidth;
3622 APInt SubDemandedElts(NumElts * SubScale, 0);
3623 for (unsigned i = 0; i != NumElts; ++i)
3624 if (DemandedElts[i])
3625 SubDemandedElts.setBit(i * SubScale);
3626
3627 for (unsigned i = 0; i != SubScale; ++i) {
3628 Known2 = computeKnownBits(Op: N0, DemandedElts: SubDemandedElts.shl(shiftAmt: i),
3629 Depth: Depth + 1);
3630 unsigned Shifts = IsLE ? i : SubScale - 1 - i;
3631 Known.insertBits(SubBits: Known2, BitPosition: SubBitWidth * Shifts);
3632 }
3633 }
3634
3635 // Bitcast 'large element' scalar/vector to 'small element' vector.
3636 if ((SubBitWidth % BitWidth) == 0) {
3637 assert(Op.getValueType().isVector() && "Expected bitcast to vector");
3638
3639 // Collect known bits for the (smaller) output by collecting the known
3640 // bits from the overlapping larger input elements and extracting the
3641 // sub sections we actually care about.
3642 unsigned SubScale = SubBitWidth / BitWidth;
3643 APInt SubDemandedElts =
3644 APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumElts / SubScale);
3645 Known2 = computeKnownBits(Op: N0, DemandedElts: SubDemandedElts, Depth: Depth + 1);
3646
3647 Known.setAllConflict();
3648 for (unsigned i = 0; i != NumElts; ++i)
3649 if (DemandedElts[i]) {
3650 unsigned Shifts = IsLE ? i : NumElts - 1 - i;
3651 unsigned Offset = (Shifts % SubScale) * BitWidth;
3652 Known = Known.intersectWith(RHS: Known2.extractBits(NumBits: BitWidth, BitPosition: Offset));
3653 // If we don't know any bits, early out.
3654 if (Known.isUnknown())
3655 break;
3656 }
3657 }
3658 break;
3659 }
3660 case ISD::AND:
3661 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3662 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3663
3664 Known &= Known2;
3665 break;
3666 case ISD::OR:
3667 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3668 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3669
3670 Known |= Known2;
3671 break;
3672 case ISD::XOR:
3673 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3674 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3675
3676 Known ^= Known2;
3677 break;
3678 case ISD::MUL: {
3679 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3680 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3681 bool SelfMultiply = Op.getOperand(i: 0) == Op.getOperand(i: 1);
3682 // TODO: SelfMultiply can be poison, but not undef.
3683 if (SelfMultiply)
3684 SelfMultiply &= isGuaranteedNotToBeUndefOrPoison(
3685 Op: Op.getOperand(i: 0), DemandedElts, Kind: UndefPoisonKind::UndefOrPoison,
3686 Depth: Depth + 1);
3687 Known = KnownBits::mul(LHS: Known, RHS: Known2, NoUndefSelfMultiply: SelfMultiply);
3688
3689 // If the multiplication is known not to overflow, the product of a number
3690 // with itself is non-negative. Only do this if we didn't already computed
3691 // the opposite value for the sign bit.
3692 if (Op->getFlags().hasNoSignedWrap() &&
3693 Op.getOperand(i: 0) == Op.getOperand(i: 1) &&
3694 !Known.isNegative())
3695 Known.makeNonNegative();
3696 break;
3697 }
3698 case ISD::MULHU: {
3699 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3700 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3701 Known = KnownBits::mulhu(LHS: Known, RHS: Known2);
3702 break;
3703 }
3704 case ISD::MULHS: {
3705 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3706 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3707 Known = KnownBits::mulhs(LHS: Known, RHS: Known2);
3708 break;
3709 }
3710 case ISD::ABDU: {
3711 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3712 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3713 Known = KnownBits::abdu(LHS: Known, RHS: Known2);
3714 break;
3715 }
3716 case ISD::ABDS: {
3717 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3718 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3719 Known = KnownBits::abds(LHS: Known, RHS: Known2);
3720 unsigned SignBits1 =
3721 ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3722 if (SignBits1 == 1)
3723 break;
3724 unsigned SignBits0 =
3725 ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3726 Known.Zero.setHighBits(std::min(a: SignBits0, b: SignBits1) - 1);
3727 break;
3728 }
3729 case ISD::UMUL_LOHI: {
3730 assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result");
3731 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3732 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3733 bool SelfMultiply = Op.getOperand(i: 0) == Op.getOperand(i: 1);
3734 if (Op.getResNo() == 0)
3735 Known = KnownBits::mul(LHS: Known, RHS: Known2, NoUndefSelfMultiply: SelfMultiply);
3736 else
3737 Known = KnownBits::mulhu(LHS: Known, RHS: Known2);
3738 break;
3739 }
3740 case ISD::SMUL_LOHI: {
3741 assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result");
3742 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3743 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3744 bool SelfMultiply = Op.getOperand(i: 0) == Op.getOperand(i: 1);
3745 if (Op.getResNo() == 0)
3746 Known = KnownBits::mul(LHS: Known, RHS: Known2, NoUndefSelfMultiply: SelfMultiply);
3747 else
3748 Known = KnownBits::mulhs(LHS: Known, RHS: Known2);
3749 break;
3750 }
3751 case ISD::AVGFLOORU: {
3752 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3753 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3754 Known = KnownBits::avgFloorU(LHS: Known, RHS: Known2);
3755 break;
3756 }
3757 case ISD::AVGCEILU: {
3758 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3759 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3760 Known = KnownBits::avgCeilU(LHS: Known, RHS: Known2);
3761 break;
3762 }
3763 case ISD::AVGFLOORS: {
3764 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3765 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3766 Known = KnownBits::avgFloorS(LHS: Known, RHS: Known2);
3767 break;
3768 }
3769 case ISD::AVGCEILS: {
3770 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3771 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3772 Known = KnownBits::avgCeilS(LHS: Known, RHS: Known2);
3773 break;
3774 }
3775 case ISD::AVGSUBFLOORU:
3776 case ISD::AVGSUBFLOORS: {
3777 bool IsSigned = Op.getOpcode() == ISD::AVGSUBFLOORS;
3778 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3779 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3780 // Subtract in i[N+1] so the difference can't overflow, then halve.
3781 Known = IsSigned ? Known.sext(BitWidth: BitWidth + 1) : Known.zext(BitWidth: BitWidth + 1);
3782 Known2 = IsSigned ? Known2.sext(BitWidth: BitWidth + 1) : Known2.zext(BitWidth: BitWidth + 1);
3783 Known = KnownBits::sub(LHS: Known, RHS: Known2);
3784 // The shifted in top bit is unknown, but it is truncated away.
3785 Known >>= 1;
3786 Known = Known.trunc(BitWidth);
3787 break;
3788 }
3789 case ISD::SELECT:
3790 case ISD::VSELECT:
3791 Known = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth+1);
3792 // If we don't know any bits, early out.
3793 if (Known.isUnknown())
3794 break;
3795 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth+1);
3796
3797 // Only known if known in both the LHS and RHS.
3798 Known = Known.intersectWith(RHS: Known2);
3799 break;
3800 case ISD::SELECT_CC:
3801 Known = computeKnownBits(Op: Op.getOperand(i: 3), DemandedElts, Depth: Depth+1);
3802 // If we don't know any bits, early out.
3803 if (Known.isUnknown())
3804 break;
3805 Known2 = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth+1);
3806
3807 // Only known if known in both the LHS and RHS.
3808 Known = Known.intersectWith(RHS: Known2);
3809 break;
3810 case ISD::SMULO:
3811 case ISD::UMULO:
3812 if (Op.getResNo() != 1)
3813 break;
3814 // The boolean result conforms to getBooleanContents.
3815 // If we know the result of a setcc has the top bits zero, use this info.
3816 // We know that we have an integer-based boolean since these operations
3817 // are only available for integer.
3818 if (TLI->getBooleanContents(isVec: Op.getValueType().isVector(), isFloat: false) ==
3819 TargetLowering::ZeroOrOneBooleanContent &&
3820 BitWidth > 1)
3821 Known.Zero.setBitsFrom(1);
3822 break;
3823 case ISD::SETCC:
3824 case ISD::SETCCCARRY:
3825 case ISD::STRICT_FSETCC:
3826 case ISD::STRICT_FSETCCS: {
3827 unsigned OpNo = Op->isStrictFPOpcode() ? 1 : 0;
3828 // If we know the result of a setcc has the top bits zero, use this info.
3829 if (TLI->getBooleanContents(Type: Op.getOperand(i: OpNo).getValueType()) ==
3830 TargetLowering::ZeroOrOneBooleanContent &&
3831 BitWidth > 1)
3832 Known.Zero.setBitsFrom(1);
3833 break;
3834 }
3835 case ISD::SHL: {
3836 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3837 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3838
3839 bool NUW = Op->getFlags().hasNoUnsignedWrap();
3840 bool NSW = Op->getFlags().hasNoSignedWrap();
3841
3842 bool ShAmtNonZero = Known2.isNonZero();
3843
3844 Known = KnownBits::shl(LHS: Known, RHS: Known2, NUW, NSW, ShAmtNonZero);
3845
3846 // Minimum shift low bits are known zero.
3847 if (std::optional<unsigned> ShMinAmt =
3848 getValidMinimumShiftAmount(V: Op, DemandedElts, Depth: Depth + 1))
3849 Known.Zero.setLowBits(*ShMinAmt);
3850 break;
3851 }
3852 case ISD::SRL:
3853 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3854 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3855 Known = KnownBits::lshr(LHS: Known, RHS: Known2, /*ShAmtNonZero=*/false,
3856 Exact: Op->getFlags().hasExact());
3857
3858 // Minimum shift high bits are known zero.
3859 if (std::optional<unsigned> ShMinAmt =
3860 getValidMinimumShiftAmount(V: Op, DemandedElts, Depth: Depth + 1))
3861 Known.Zero.setHighBits(*ShMinAmt);
3862 break;
3863 case ISD::SRA:
3864 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3865 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3866 Known = KnownBits::ashr(LHS: Known, RHS: Known2, /*ShAmtNonZero=*/false,
3867 Exact: Op->getFlags().hasExact());
3868 break;
3869 case ISD::ROTL:
3870 case ISD::ROTR:
3871 if (ConstantSDNode *C =
3872 isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts)) {
3873 unsigned Amt = C->getAPIntValue().urem(RHS: BitWidth);
3874
3875 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3876
3877 // Canonicalize to ROTR.
3878 if (Opcode == ISD::ROTL && Amt != 0)
3879 Amt = BitWidth - Amt;
3880
3881 Known.Zero = Known.Zero.rotr(rotateAmt: Amt);
3882 Known.One = Known.One.rotr(rotateAmt: Amt);
3883 }
3884 break;
3885 case ISD::FSHL:
3886 case ISD::FSHR:
3887 if (ConstantSDNode *C = isConstOrConstSplat(N: Op.getOperand(i: 2), DemandedElts)) {
3888 unsigned Amt = C->getAPIntValue().urem(RHS: BitWidth);
3889
3890 // For fshl, 0-shift returns the 1st arg.
3891 // For fshr, 0-shift returns the 2nd arg.
3892 if (Amt == 0) {
3893 Known = computeKnownBits(Op: Op.getOperand(i: Opcode == ISD::FSHL ? 0 : 1),
3894 DemandedElts, Depth: Depth + 1);
3895 break;
3896 }
3897
3898 // fshl: (X << (Z % BW)) | (Y >> (BW - (Z % BW)))
3899 // fshr: (X << (BW - (Z % BW))) | (Y >> (Z % BW))
3900 const APInt ShAmt(BitWidth, Amt);
3901 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3902 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3903 Known = Opcode == ISD::FSHL ? KnownBits::fshl(LHS: Known, RHS: Known2, Amt: ShAmt)
3904 : KnownBits::fshr(LHS: Known, RHS: Known2, Amt: ShAmt);
3905 }
3906 break;
3907 case ISD::SHL_PARTS:
3908 case ISD::SRA_PARTS:
3909 case ISD::SRL_PARTS: {
3910 assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result");
3911
3912 // Collect lo/hi source values and concatenate.
3913 unsigned LoBits = Op.getOperand(i: 0).getScalarValueSizeInBits();
3914 unsigned HiBits = Op.getOperand(i: 1).getScalarValueSizeInBits();
3915 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3916 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3917 Known = Known2.concat(Lo: Known);
3918
3919 // Collect shift amount.
3920 Known2 = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth + 1);
3921
3922 if (Opcode == ISD::SHL_PARTS)
3923 Known = KnownBits::shl(LHS: Known, RHS: Known2);
3924 else if (Opcode == ISD::SRA_PARTS)
3925 Known = KnownBits::ashr(LHS: Known, RHS: Known2);
3926 else // if (Opcode == ISD::SRL_PARTS)
3927 Known = KnownBits::lshr(LHS: Known, RHS: Known2);
3928
3929 // TODO: Minimum shift low/high bits are known zero.
3930
3931 if (Op.getResNo() == 0)
3932 Known = Known.extractBits(NumBits: LoBits, BitPosition: 0);
3933 else
3934 Known = Known.extractBits(NumBits: HiBits, BitPosition: LoBits);
3935 break;
3936 }
3937 case ISD::SIGN_EXTEND_INREG: {
3938 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3939 EVT EVT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
3940 Known = Known.sextInReg(SrcBitWidth: EVT.getScalarSizeInBits());
3941 break;
3942 }
3943 case ISD::CTTZ:
3944 case ISD::CTTZ_ZERO_POISON: {
3945 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3946 // If we have a known 1, its position is our upper bound.
3947 unsigned PossibleTZ = Known2.countMaxTrailingZeros();
3948 unsigned LowBits = llvm::bit_width(Value: PossibleTZ);
3949 Known.Zero.setBitsFrom(LowBits);
3950 break;
3951 }
3952 case ISD::CTLZ:
3953 case ISD::CTLZ_ZERO_POISON: {
3954 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3955 // If we have a known 1, its position is our upper bound.
3956 unsigned PossibleLZ = Known2.countMaxLeadingZeros();
3957 unsigned LowBits = llvm::bit_width(Value: PossibleLZ);
3958 Known.Zero.setBitsFrom(LowBits);
3959 break;
3960 }
3961 case ISD::CTLS: {
3962 unsigned MinRedundantSignBits =
3963 ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1) - 1;
3964 ConstantRange Range(APInt(BitWidth, MinRedundantSignBits),
3965 APInt(BitWidth, BitWidth));
3966 Known = Range.toKnownBits();
3967 break;
3968 }
3969 case ISD::CTPOP: {
3970 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3971 // If we know some of the bits are zero, they can't be one.
3972 unsigned PossibleOnes = Known2.countMaxPopulation();
3973 Known.Zero.setBitsFrom(llvm::bit_width(Value: PossibleOnes));
3974 break;
3975 }
3976 case ISD::PARITY: {
3977 // Parity returns 0 everywhere but the LSB.
3978 Known.Zero.setBitsFrom(1);
3979 break;
3980 }
3981 case ISD::PDEP: {
3982 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3983 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3984 Known = KnownBits::pdep(Val: Known2, Mask: Known);
3985 break;
3986 }
3987 case ISD::PEXT: {
3988 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3989 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3990 Known = KnownBits::pext(Val: Known2, Mask: Known);
3991 break;
3992 }
3993 case ISD::CLMUL: {
3994 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3995 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3996 Known = KnownBits::clmul(LHS: Known, RHS: Known2);
3997 break;
3998 }
3999 case ISD::MGATHER:
4000 case ISD::MLOAD: {
4001 ISD::LoadExtType ETy =
4002 (Opcode == ISD::MGATHER)
4003 ? cast<MaskedGatherSDNode>(Val&: Op)->getExtensionType()
4004 : cast<MaskedLoadSDNode>(Val&: Op)->getExtensionType();
4005 if (ETy == ISD::ZEXTLOAD) {
4006 EVT MemVT = cast<MemSDNode>(Val&: Op)->getMemoryVT();
4007 KnownBits Known0(MemVT.getScalarSizeInBits());
4008 return Known0.zext(BitWidth);
4009 }
4010 break;
4011 }
4012 case ISD::LOAD: {
4013 LoadSDNode *LD = cast<LoadSDNode>(Val&: Op);
4014 const Constant *Cst = TLI->getTargetConstantFromLoad(LD);
4015 if (ISD::isNON_EXTLoad(N: LD) && Cst) {
4016 // Determine any common known bits from the loaded constant pool value.
4017 Type *CstTy = Cst->getType();
4018 if ((NumElts * BitWidth) == CstTy->getPrimitiveSizeInBits() &&
4019 !Op.getValueType().isScalableVector()) {
4020 // If its a vector splat, then we can (quickly) reuse the scalar path.
4021 // NOTE: We assume all elements match and none are UNDEF.
4022 if (CstTy->isVectorTy()) {
4023 if (const Constant *Splat = Cst->getSplatValue()) {
4024 Cst = Splat;
4025 CstTy = Cst->getType();
4026 }
4027 }
4028 // TODO - do we need to handle different bitwidths?
4029 if (CstTy->isVectorTy() && BitWidth == CstTy->getScalarSizeInBits()) {
4030 // Iterate across all vector elements finding common known bits.
4031 Known.setAllConflict();
4032 for (unsigned i = 0; i != NumElts; ++i) {
4033 if (!DemandedElts[i])
4034 continue;
4035 if (Constant *Elt = Cst->getAggregateElement(Elt: i)) {
4036 if (auto *CInt = dyn_cast<ConstantInt>(Val: Elt)) {
4037 const APInt &Value = CInt->getValue();
4038 Known.One &= Value;
4039 Known.Zero &= ~Value;
4040 continue;
4041 }
4042 if (auto *CFP = dyn_cast<ConstantFP>(Val: Elt)) {
4043 APInt Value = CFP->getValueAPF().bitcastToAPInt();
4044 Known.One &= Value;
4045 Known.Zero &= ~Value;
4046 continue;
4047 }
4048 }
4049 Known.One.clearAllBits();
4050 Known.Zero.clearAllBits();
4051 break;
4052 }
4053 } else if (BitWidth == CstTy->getPrimitiveSizeInBits()) {
4054 if (auto *CInt = dyn_cast<ConstantInt>(Val: Cst)) {
4055 Known = KnownBits::makeConstant(C: CInt->getValue());
4056 } else if (auto *CFP = dyn_cast<ConstantFP>(Val: Cst)) {
4057 Known =
4058 KnownBits::makeConstant(C: CFP->getValueAPF().bitcastToAPInt());
4059 }
4060 }
4061 }
4062 } else if (Op.getResNo() == 0) {
4063 unsigned ScalarMemorySize = LD->getMemoryVT().getScalarSizeInBits();
4064 KnownBits KnownScalarMemory(ScalarMemorySize);
4065 if (const MDNode *MD = LD->getRanges())
4066 computeKnownBitsFromRangeMetadata(Ranges: *MD, Known&: KnownScalarMemory);
4067
4068 // Extend the Known bits from memory to the size of the scalar result.
4069 if (ISD::isZEXTLoad(N: Op.getNode()))
4070 Known = KnownScalarMemory.zext(BitWidth);
4071 else if (ISD::isSEXTLoad(N: Op.getNode()))
4072 Known = KnownScalarMemory.sext(BitWidth);
4073 else if (ISD::isEXTLoad(N: Op.getNode()))
4074 Known = KnownScalarMemory.anyext(BitWidth);
4075 else
4076 Known = KnownScalarMemory;
4077 assert(Known.getBitWidth() == BitWidth);
4078 return Known;
4079 }
4080 break;
4081 }
4082 case ISD::ZERO_EXTEND_VECTOR_INREG: {
4083 if (Op.getValueType().isScalableVector())
4084 break;
4085 EVT InVT = Op.getOperand(i: 0).getValueType();
4086 APInt InDemandedElts = DemandedElts.zext(width: InVT.getVectorNumElements());
4087 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts: InDemandedElts, Depth: Depth + 1);
4088 Known = Known.zext(BitWidth);
4089 break;
4090 }
4091 case ISD::ZERO_EXTEND: {
4092 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4093 Known = Known.zext(BitWidth);
4094 break;
4095 }
4096 case ISD::SIGN_EXTEND_VECTOR_INREG: {
4097 if (Op.getValueType().isScalableVector())
4098 break;
4099 EVT InVT = Op.getOperand(i: 0).getValueType();
4100 APInt InDemandedElts = DemandedElts.zext(width: InVT.getVectorNumElements());
4101 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts: InDemandedElts, Depth: Depth + 1);
4102 // If the sign bit is known to be zero or one, then sext will extend
4103 // it to the top bits, else it will just zext.
4104 Known = Known.sext(BitWidth);
4105 break;
4106 }
4107 case ISD::SIGN_EXTEND: {
4108 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4109 // If the sign bit is known to be zero or one, then sext will extend
4110 // it to the top bits, else it will just zext.
4111 Known = Known.sext(BitWidth);
4112 break;
4113 }
4114 case ISD::ANY_EXTEND_VECTOR_INREG: {
4115 if (Op.getValueType().isScalableVector())
4116 break;
4117 EVT InVT = Op.getOperand(i: 0).getValueType();
4118 APInt InDemandedElts = DemandedElts.zext(width: InVT.getVectorNumElements());
4119 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts: InDemandedElts, Depth: Depth + 1);
4120 Known = Known.anyext(BitWidth);
4121 break;
4122 }
4123 case ISD::ANY_EXTEND: {
4124 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4125 Known = Known.anyext(BitWidth);
4126 break;
4127 }
4128 case ISD::TRUNCATE: {
4129 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4130 Known = Known.trunc(BitWidth);
4131 break;
4132 }
4133 case ISD::TRUNCATE_SSAT_S: {
4134 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4135 Known = Known.truncSSat(BitWidth);
4136 break;
4137 }
4138 case ISD::TRUNCATE_SSAT_U: {
4139 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4140 Known = Known.truncSSatU(BitWidth);
4141 break;
4142 }
4143 case ISD::TRUNCATE_USAT_U: {
4144 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4145 Known = Known.truncUSat(BitWidth);
4146 break;
4147 }
4148 case ISD::AssertZext: {
4149 EVT VT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
4150 APInt InMask = APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: VT.getSizeInBits());
4151 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4152 Known.Zero |= (~InMask);
4153 Known.One &= (~Known.Zero);
4154 break;
4155 }
4156 case ISD::AssertAlign: {
4157 unsigned LogOfAlign = Log2(A: cast<AssertAlignSDNode>(Val&: Op)->getAlign());
4158 assert(LogOfAlign != 0);
4159
4160 // TODO: Should use maximum with source
4161 // If a node is guaranteed to be aligned, set low zero bits accordingly as
4162 // well as clearing one bits.
4163 Known.Zero.setLowBits(LogOfAlign);
4164 Known.One.clearLowBits(loBits: LogOfAlign);
4165 break;
4166 }
4167 case ISD::AssertNoFPClass: {
4168 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4169
4170 FPClassTest NoFPClass =
4171 static_cast<FPClassTest>(Op.getConstantOperandVal(i: 1));
4172 const FPClassTest NegativeTestMask = fcNan | fcNegative;
4173 if ((NoFPClass & NegativeTestMask) == NegativeTestMask) {
4174 // Cannot be negative.
4175 Known.makeNonNegative();
4176 }
4177
4178 const FPClassTest PositiveTestMask = fcNan | fcPositive;
4179 if ((NoFPClass & PositiveTestMask) == PositiveTestMask) {
4180 // Cannot be positive.
4181 Known.makeNegative();
4182 }
4183
4184 break;
4185 }
4186 case ISD::FABS:
4187 // fabs clears the sign bit
4188 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4189 Known.Zero.setSignBit();
4190 Known.One.clearSignBit();
4191 break;
4192 case ISD::FGETSIGN:
4193 // All bits are zero except the low bit.
4194 Known.Zero.setBitsFrom(1);
4195 break;
4196 case ISD::ADD: {
4197 SDNodeFlags Flags = Op.getNode()->getFlags();
4198 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4199 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4200 bool SelfAdd = Op.getOperand(i: 0) == Op.getOperand(i: 1) &&
4201 isGuaranteedNotToBeUndefOrPoison(
4202 Op: Op.getOperand(i: 0), DemandedElts,
4203 Kind: UndefPoisonKind::UndefOrPoison, Depth: Depth + 1);
4204 Known = KnownBits::add(LHS: Known, RHS: Known2, NSW: Flags.hasNoSignedWrap(),
4205 NUW: Flags.hasNoUnsignedWrap(), SelfAdd);
4206 break;
4207 }
4208 case ISD::SUB: {
4209 SDNodeFlags Flags = Op.getNode()->getFlags();
4210 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4211 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4212 Known = KnownBits::sub(LHS: Known, RHS: Known2, NSW: Flags.hasNoSignedWrap(),
4213 NUW: Flags.hasNoUnsignedWrap());
4214 break;
4215 }
4216 case ISD::USUBO:
4217 case ISD::SSUBO:
4218 case ISD::USUBO_CARRY:
4219 case ISD::SSUBO_CARRY:
4220 if (Op.getResNo() == 1) {
4221 // If we know the result of a setcc has the top bits zero, use this info.
4222 if (TLI->getBooleanContents(Type: Op.getOperand(i: 0).getValueType()) ==
4223 TargetLowering::ZeroOrOneBooleanContent &&
4224 BitWidth > 1)
4225 Known.Zero.setBitsFrom(1);
4226 break;
4227 }
4228 [[fallthrough]];
4229 case ISD::SUBC: {
4230 assert(Op.getResNo() == 0 &&
4231 "We only compute knownbits for the difference here.");
4232
4233 // With USUBO_CARRY and SSUBO_CARRY a borrow bit may be added in.
4234 KnownBits Borrow(1);
4235 if (Opcode == ISD::USUBO_CARRY || Opcode == ISD::SSUBO_CARRY) {
4236 Borrow = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth + 1);
4237 // Borrow has bit width 1
4238 Borrow = Borrow.trunc(BitWidth: 1);
4239 } else {
4240 Borrow.setAllZero();
4241 }
4242
4243 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4244 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4245 Known = KnownBits::computeForSubBorrow(LHS: Known, RHS: Known2, Borrow);
4246 break;
4247 }
4248 case ISD::UADDO:
4249 case ISD::SADDO:
4250 case ISD::UADDO_CARRY:
4251 case ISD::SADDO_CARRY:
4252 if (Op.getResNo() == 1) {
4253 // If we know the result of a setcc has the top bits zero, use this info.
4254 if (TLI->getBooleanContents(Type: Op.getOperand(i: 0).getValueType()) ==
4255 TargetLowering::ZeroOrOneBooleanContent &&
4256 BitWidth > 1)
4257 Known.Zero.setBitsFrom(1);
4258 break;
4259 }
4260 [[fallthrough]];
4261 case ISD::ADDC:
4262 case ISD::ADDE: {
4263 assert(Op.getResNo() == 0 && "We only compute knownbits for the sum here.");
4264
4265 // With ADDE and UADDO_CARRY, a carry bit may be added in.
4266 KnownBits Carry(1);
4267 if (Opcode == ISD::ADDE)
4268 // Can't track carry from glue, set carry to unknown.
4269 Carry.resetAll();
4270 else if (Opcode == ISD::UADDO_CARRY || Opcode == ISD::SADDO_CARRY) {
4271 Carry = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth + 1);
4272 // Carry has bit width 1
4273 Carry = Carry.trunc(BitWidth: 1);
4274 } else {
4275 Carry.setAllZero();
4276 }
4277
4278 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4279 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4280 Known = KnownBits::computeForAddCarry(LHS: Known, RHS: Known2, Carry);
4281 break;
4282 }
4283 case ISD::UDIV: {
4284 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4285 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4286 Known = KnownBits::udiv(LHS: Known, RHS: Known2, Exact: Op->getFlags().hasExact());
4287 break;
4288 }
4289 case ISD::SDIV: {
4290 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4291 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4292 Known = KnownBits::sdiv(LHS: Known, RHS: Known2, Exact: Op->getFlags().hasExact());
4293 break;
4294 }
4295 case ISD::SREM: {
4296 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4297 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4298 Known = KnownBits::srem(LHS: Known, RHS: Known2);
4299 break;
4300 }
4301 case ISD::UREM: {
4302 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4303 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4304 Known = KnownBits::urem(LHS: Known, RHS: Known2);
4305 break;
4306 }
4307 case ISD::EXTRACT_ELEMENT: {
4308 Known = computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth+1);
4309 const unsigned Index = Op.getConstantOperandVal(i: 1);
4310 const unsigned EltBitWidth = Op.getValueSizeInBits();
4311
4312 Known = Known.extractBits(NumBits: EltBitWidth, BitPosition: Index * EltBitWidth);
4313 break;
4314 }
4315 case ISD::EXTRACT_VECTOR_ELT: {
4316 SDValue InVec = Op.getOperand(i: 0);
4317 SDValue EltNo = Op.getOperand(i: 1);
4318 EVT VecVT = InVec.getValueType();
4319 // computeKnownBits not yet implemented for scalable vectors.
4320 if (VecVT.isScalableVector())
4321 break;
4322 const unsigned EltBitWidth = VecVT.getScalarSizeInBits();
4323 const unsigned NumSrcElts = VecVT.getVectorNumElements();
4324
4325 // If BitWidth > EltBitWidth the value is anyext:ed. So we do not know
4326 // anything about the extended bits.
4327 if (BitWidth > EltBitWidth)
4328 Known = Known.trunc(BitWidth: EltBitWidth);
4329
4330 // If we know the element index, just demand that vector element, else for
4331 // an unknown element index, ignore DemandedElts and demand them all.
4332 APInt DemandedSrcElts = APInt::getAllOnes(numBits: NumSrcElts);
4333 auto *ConstEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
4334 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(RHS: NumSrcElts))
4335 DemandedSrcElts =
4336 APInt::getOneBitSet(numBits: NumSrcElts, BitNo: ConstEltNo->getZExtValue());
4337
4338 Known = computeKnownBits(Op: InVec, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
4339 if (BitWidth > EltBitWidth)
4340 Known = Known.anyext(BitWidth);
4341 break;
4342 }
4343 case ISD::BUILD_PAIR: {
4344 // Operand 0 is the low half and operand 1 the high half,
4345 // KnownBits::concat places its argument in the low bits.
4346 Known = computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
4347 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), Depth: Depth + 1);
4348 Known = Known2.concat(Lo: Known);
4349 break;
4350 }
4351 case ISD::INSERT_VECTOR_ELT: {
4352 if (Op.getValueType().isScalableVector())
4353 break;
4354
4355 // If we know the element index, split the demand between the
4356 // source vector and the inserted element, otherwise assume we need
4357 // the original demanded vector elements and the value.
4358 SDValue InVec = Op.getOperand(i: 0);
4359 SDValue InVal = Op.getOperand(i: 1);
4360 SDValue EltNo = Op.getOperand(i: 2);
4361 bool DemandedVal = true;
4362 APInt DemandedVecElts = DemandedElts;
4363 auto *CEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
4364 if (CEltNo && CEltNo->getAPIntValue().ult(RHS: NumElts)) {
4365 unsigned EltIdx = CEltNo->getZExtValue();
4366 DemandedVal = !!DemandedElts[EltIdx];
4367 DemandedVecElts.clearBit(BitPosition: EltIdx);
4368 }
4369 Known.setAllConflict();
4370 if (DemandedVal) {
4371 Known2 = computeKnownBits(Op: InVal, Depth: Depth + 1);
4372 Known = Known.intersectWith(RHS: Known2.zextOrTrunc(BitWidth));
4373 }
4374 if (!!DemandedVecElts) {
4375 Known2 = computeKnownBits(Op: InVec, DemandedElts: DemandedVecElts, Depth: Depth + 1);
4376 Known = Known.intersectWith(RHS: Known2);
4377 }
4378 break;
4379 }
4380 case ISD::BITREVERSE: {
4381 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4382 Known = Known2.reverseBits();
4383 break;
4384 }
4385 case ISD::BSWAP: {
4386 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4387 Known = Known2.byteSwap();
4388 break;
4389 }
4390 case ISD::ABS:
4391 case ISD::ABS_MIN_POISON: {
4392 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4393 Known = Known2.abs();
4394 Known.Zero.setHighBits(
4395 ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1) - 1);
4396 break;
4397 }
4398 case ISD::USUBSAT: {
4399 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4400 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4401 Known = KnownBits::usub_sat(LHS: Known, RHS: Known2);
4402 break;
4403 }
4404 case ISD::UMIN: {
4405 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4406 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4407 Known = KnownBits::umin(LHS: Known, RHS: Known2);
4408 break;
4409 }
4410 case ISD::UMAX: {
4411 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4412 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4413 Known = KnownBits::umax(LHS: Known, RHS: Known2);
4414 break;
4415 }
4416 case ISD::SMIN:
4417 case ISD::SMAX: {
4418 // If we have a clamp pattern, we know that the number of sign bits will be
4419 // the minimum of the clamp min/max range.
4420 bool IsMax = (Opcode == ISD::SMAX);
4421 ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr;
4422 if ((CstLow = isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts)))
4423 if (Op.getOperand(i: 0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX))
4424 CstHigh =
4425 isConstOrConstSplat(N: Op.getOperand(i: 0).getOperand(i: 1), DemandedElts);
4426 if (CstLow && CstHigh) {
4427 if (!IsMax)
4428 std::swap(a&: CstLow, b&: CstHigh);
4429
4430 const APInt &ValueLow = CstLow->getAPIntValue();
4431 const APInt &ValueHigh = CstHigh->getAPIntValue();
4432 if (ValueLow.sle(RHS: ValueHigh)) {
4433 unsigned LowSignBits = ValueLow.getNumSignBits();
4434 unsigned HighSignBits = ValueHigh.getNumSignBits();
4435 unsigned MinSignBits = std::min(a: LowSignBits, b: HighSignBits);
4436 if (ValueLow.isNegative() && ValueHigh.isNegative()) {
4437 Known.One.setHighBits(MinSignBits);
4438 break;
4439 }
4440 if (ValueLow.isNonNegative() && ValueHigh.isNonNegative()) {
4441 Known.Zero.setHighBits(MinSignBits);
4442 break;
4443 }
4444 }
4445 }
4446
4447 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4448 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4449 if (IsMax)
4450 Known = KnownBits::smax(LHS: Known, RHS: Known2);
4451 else
4452 Known = KnownBits::smin(LHS: Known, RHS: Known2);
4453
4454 // For SMAX, if CstLow is non-negative we know the result will be
4455 // non-negative and thus all sign bits are 0.
4456 // TODO: There's an equivalent of this for smin with negative constant for
4457 // known ones.
4458 if (IsMax && CstLow) {
4459 const APInt &ValueLow = CstLow->getAPIntValue();
4460 if (ValueLow.isNonNegative()) {
4461 unsigned SignBits = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
4462 Known.Zero.setHighBits(std::min(a: SignBits, b: ValueLow.getNumSignBits()));
4463 }
4464 }
4465
4466 break;
4467 }
4468 case ISD::UINT_TO_FP: {
4469 Known.makeNonNegative();
4470 break;
4471 }
4472 case ISD::SINT_TO_FP: {
4473 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4474 if (Known2.isNonNegative())
4475 Known.makeNonNegative();
4476 else if (Known2.isNegative())
4477 Known.makeNegative();
4478 break;
4479 }
4480 case ISD::FP_TO_UINT_SAT: {
4481 // FP_TO_UINT_SAT produces an unsigned value that fits in the saturating VT.
4482 EVT VT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
4483 Known.Zero |= APInt::getBitsSetFrom(numBits: BitWidth, loBit: VT.getScalarSizeInBits());
4484 break;
4485 }
4486 case ISD::ATOMIC_LOAD: {
4487 // If we are looking at the loaded value.
4488 if (Op.getResNo() == 0) {
4489 auto *AT = cast<AtomicSDNode>(Val&: Op);
4490 unsigned ScalarMemorySize = AT->getMemoryVT().getScalarSizeInBits();
4491 KnownBits KnownScalarMemory(ScalarMemorySize);
4492 if (const MDNode *MD = AT->getRanges())
4493 computeKnownBitsFromRangeMetadata(Ranges: *MD, Known&: KnownScalarMemory);
4494
4495 switch (AT->getExtensionType()) {
4496 case ISD::ZEXTLOAD:
4497 Known = KnownScalarMemory.zext(BitWidth);
4498 break;
4499 case ISD::SEXTLOAD:
4500 Known = KnownScalarMemory.sext(BitWidth);
4501 break;
4502 case ISD::EXTLOAD:
4503 switch (TLI->getExtendForAtomicOps()) {
4504 case ISD::ZERO_EXTEND:
4505 Known = KnownScalarMemory.zext(BitWidth);
4506 break;
4507 case ISD::SIGN_EXTEND:
4508 Known = KnownScalarMemory.sext(BitWidth);
4509 break;
4510 default:
4511 Known = KnownScalarMemory.anyext(BitWidth);
4512 break;
4513 }
4514 break;
4515 case ISD::NON_EXTLOAD:
4516 Known = KnownScalarMemory;
4517 break;
4518 }
4519 assert(Known.getBitWidth() == BitWidth);
4520 }
4521 break;
4522 }
4523 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
4524 if (Op.getResNo() == 1) {
4525 // The boolean result conforms to getBooleanContents.
4526 // If we know the result of a setcc has the top bits zero, use this info.
4527 // We know that we have an integer-based boolean since these operations
4528 // are only available for integer.
4529 if (TLI->getBooleanContents(isVec: Op.getValueType().isVector(), isFloat: false) ==
4530 TargetLowering::ZeroOrOneBooleanContent &&
4531 BitWidth > 1)
4532 Known.Zero.setBitsFrom(1);
4533 break;
4534 }
4535 [[fallthrough]];
4536 case ISD::ATOMIC_CMP_SWAP:
4537 case ISD::ATOMIC_SWAP:
4538 case ISD::ATOMIC_LOAD_ADD:
4539 case ISD::ATOMIC_LOAD_SUB:
4540 case ISD::ATOMIC_LOAD_AND:
4541 case ISD::ATOMIC_LOAD_CLR:
4542 case ISD::ATOMIC_LOAD_OR:
4543 case ISD::ATOMIC_LOAD_XOR:
4544 case ISD::ATOMIC_LOAD_NAND:
4545 case ISD::ATOMIC_LOAD_MIN:
4546 case ISD::ATOMIC_LOAD_MAX:
4547 case ISD::ATOMIC_LOAD_UMIN:
4548 case ISD::ATOMIC_LOAD_UMAX: {
4549 // If we are looking at the loaded value.
4550 if (Op.getResNo() == 0) {
4551 auto *AT = cast<AtomicSDNode>(Val&: Op);
4552 unsigned MemBits = AT->getMemoryVT().getScalarSizeInBits();
4553
4554 if (TLI->getExtendForAtomicOps() == ISD::ZERO_EXTEND)
4555 Known.Zero.setBitsFrom(MemBits);
4556 }
4557 break;
4558 }
4559 case ISD::FrameIndex:
4560 case ISD::TargetFrameIndex: {
4561 const MachineFunction &MF = getMachineFunction();
4562 int FrameIdx = cast<FrameIndexSDNode>(Val&: Op)->getIndex();
4563 TLI->computeKnownBitsForStackObjectPointer(
4564 Known, MF, Alignment: MF.getFrameInfo().getObjectAlign(ObjectIdx: FrameIdx));
4565 break;
4566 }
4567 case ISD::VP_LOAD_FF: {
4568 if (Op.getResNo() != 1)
4569 break;
4570 // The second result of vp.load.ff is an unsigned value that is less than or
4571 // equal to the EVL operand.
4572 KnownBits VLKB =
4573 computeKnownBits(Op: Op.getOperand(i: 3), DemandedElts, Depth: Depth + 1);
4574 // The new VL is also bounded by the largest vector length.
4575 EVT ResVT = Op->getValueType(ResNo: 0);
4576 auto ResKB = KnownBits::makeConstant(
4577 C: APInt(BitWidth, ResVT.getVectorMinNumElements()));
4578 if (ResVT.isScalableVector()) {
4579 const Function &F = getMachineFunction().getFunction();
4580 ResKB = KnownBits::mul(LHS: getVScaleRange(F: &F, BitWidth).toKnownBits(), RHS: ResKB);
4581 }
4582 Known.Zero.setHighBits(KnownBits::umin(LHS: VLKB, RHS: ResKB).countMinLeadingZeros());
4583 break;
4584 }
4585
4586 default:
4587 if (Opcode < ISD::BUILTIN_OP_END)
4588 break;
4589 [[fallthrough]];
4590 case ISD::INTRINSIC_WO_CHAIN:
4591 case ISD::INTRINSIC_W_CHAIN:
4592 case ISD::INTRINSIC_VOID:
4593 // Allow the target to implement this method for its nodes.
4594 TLI->computeKnownBitsForTargetNode(Op, Known, DemandedElts, DAG: *this, Depth);
4595 break;
4596 }
4597
4598 return Known;
4599}
4600
4601/// Convert ConstantRange OverflowResult into SelectionDAG::OverflowKind.
4602static SelectionDAG::OverflowKind mapOverflowResult(ConstantRange::OverflowResult OR) {
4603 switch (OR) {
4604 case ConstantRange::OverflowResult::MayOverflow:
4605 return SelectionDAG::OFK_Sometime;
4606 case ConstantRange::OverflowResult::AlwaysOverflowsLow:
4607 case ConstantRange::OverflowResult::AlwaysOverflowsHigh:
4608 return SelectionDAG::OFK_Always;
4609 case ConstantRange::OverflowResult::NeverOverflows:
4610 return SelectionDAG::OFK_Never;
4611 }
4612 llvm_unreachable("Unknown OverflowResult");
4613}
4614
4615SelectionDAG::OverflowKind
4616SelectionDAG::computeOverflowForSignedAdd(SDValue N0, SDValue N1) const {
4617 // X + 0 never overflow
4618 if (isNullConstant(V: N1))
4619 return OFK_Never;
4620
4621 // If both operands each have at least two sign bits, the addition
4622 // cannot overflow.
4623 if (ComputeNumSignBits(Op: N0) > 1 && ComputeNumSignBits(Op: N1) > 1)
4624 return OFK_Never;
4625
4626 // TODO: Add ConstantRange::signedAddMayOverflow handling.
4627 return OFK_Sometime;
4628}
4629
4630SelectionDAG::OverflowKind
4631SelectionDAG::computeOverflowForUnsignedAdd(SDValue N0, SDValue N1) const {
4632 // X + 0 never overflow
4633 if (isNullConstant(V: N1))
4634 return OFK_Never;
4635
4636 // mulhi + 1 never overflow
4637 KnownBits N1Known = computeKnownBits(Op: N1);
4638 if (N0.getOpcode() == ISD::UMUL_LOHI && N0.getResNo() == 1 &&
4639 N1Known.getMaxValue().ult(RHS: 2))
4640 return OFK_Never;
4641
4642 KnownBits N0Known = computeKnownBits(Op: N0);
4643 if (N1.getOpcode() == ISD::UMUL_LOHI && N1.getResNo() == 1 &&
4644 N0Known.getMaxValue().ult(RHS: 2))
4645 return OFK_Never;
4646
4647 // Fallback to ConstantRange::unsignedAddMayOverflow handling.
4648 ConstantRange N0Range = ConstantRange::fromKnownBits(Known: N0Known, IsSigned: false);
4649 ConstantRange N1Range = ConstantRange::fromKnownBits(Known: N1Known, IsSigned: false);
4650 return mapOverflowResult(OR: N0Range.unsignedAddMayOverflow(Other: N1Range));
4651}
4652
4653SelectionDAG::OverflowKind
4654SelectionDAG::computeOverflowForSignedSub(SDValue N0, SDValue N1) const {
4655 // X - 0 never overflow
4656 if (isNullConstant(V: N1))
4657 return OFK_Never;
4658
4659 // If both operands each have at least two sign bits, the subtraction
4660 // cannot overflow.
4661 if (ComputeNumSignBits(Op: N0) > 1 && ComputeNumSignBits(Op: N1) > 1)
4662 return OFK_Never;
4663
4664 KnownBits N0Known = computeKnownBits(Op: N0);
4665 KnownBits N1Known = computeKnownBits(Op: N1);
4666 ConstantRange N0Range = ConstantRange::fromKnownBits(Known: N0Known, IsSigned: true);
4667 ConstantRange N1Range = ConstantRange::fromKnownBits(Known: N1Known, IsSigned: true);
4668 return mapOverflowResult(OR: N0Range.signedSubMayOverflow(Other: N1Range));
4669}
4670
4671SelectionDAG::OverflowKind
4672SelectionDAG::computeOverflowForUnsignedSub(SDValue N0, SDValue N1) const {
4673 // X - 0 never overflow
4674 if (isNullConstant(V: N1))
4675 return OFK_Never;
4676
4677 ConstantRange N0Range =
4678 computeConstantRangeIncludingKnownBits(Op: N0, /*ForSigned=*/false);
4679 ConstantRange N1Range =
4680 computeConstantRangeIncludingKnownBits(Op: N1, /*ForSigned=*/false);
4681 return mapOverflowResult(OR: N0Range.unsignedSubMayOverflow(Other: N1Range));
4682}
4683
4684SelectionDAG::OverflowKind
4685SelectionDAG::computeOverflowForUnsignedMul(SDValue N0, SDValue N1) const {
4686 // X * 0 and X * 1 never overflow.
4687 if (isNullConstant(V: N1) || isOneConstant(V: N1))
4688 return OFK_Never;
4689
4690 ConstantRange N0Range = computeConstantRangeIncludingKnownBits(Op: N0, ForSigned: false);
4691 ConstantRange N1Range = computeConstantRangeIncludingKnownBits(Op: N1, ForSigned: false);
4692 return mapOverflowResult(OR: N0Range.unsignedMulMayOverflow(Other: N1Range));
4693}
4694
4695SelectionDAG::OverflowKind
4696SelectionDAG::computeOverflowForSignedMul(SDValue N0, SDValue N1) const {
4697 // X * 0 and X * 1 never overflow.
4698 if (isNullConstant(V: N1) || isOneConstant(V: N1))
4699 return OFK_Never;
4700
4701 // Get the size of the result.
4702 unsigned BitWidth = N0.getScalarValueSizeInBits();
4703
4704 // Sum of the sign bits.
4705 unsigned SignBits = ComputeNumSignBits(Op: N0) + ComputeNumSignBits(Op: N1);
4706
4707 // If we have enough sign bits, then there's no overflow.
4708 if (SignBits > BitWidth + 1)
4709 return OFK_Never;
4710
4711 if (SignBits == BitWidth + 1) {
4712 // The overflow occurs when the true multiplication of the
4713 // the operands is the minimum negative number.
4714 KnownBits N0Known = computeKnownBits(Op: N0);
4715 KnownBits N1Known = computeKnownBits(Op: N1);
4716 // If one of the operands is non-negative, then there's no
4717 // overflow.
4718 if (N0Known.isNonNegative() || N1Known.isNonNegative())
4719 return OFK_Never;
4720 }
4721
4722 return OFK_Sometime;
4723}
4724
4725ConstantRange SelectionDAG::computeConstantRange(SDValue Op, bool ForSigned,
4726 unsigned Depth) const {
4727 APInt DemandedElts = getDemandAllEltsMask(V: Op);
4728 return computeConstantRange(Op, DemandedElts, ForSigned, Depth);
4729}
4730
4731ConstantRange SelectionDAG::computeConstantRange(SDValue Op,
4732 const APInt &DemandedElts,
4733 bool ForSigned,
4734 unsigned Depth) const {
4735 EVT VT = Op.getValueType();
4736 unsigned BitWidth = VT.getScalarSizeInBits();
4737
4738 if (Depth >= MaxRecursionDepth)
4739 return ConstantRange::getFull(BitWidth);
4740
4741 if (ConstantSDNode *C = isConstOrConstSplat(N: Op, DemandedElts))
4742 return ConstantRange(C->getAPIntValue());
4743
4744 unsigned Opcode = Op.getOpcode();
4745 switch (Opcode) {
4746 case ISD::VSCALE: {
4747 const Function &F = getMachineFunction().getFunction();
4748 const APInt &Multiplier = Op.getConstantOperandAPInt(i: 0);
4749 return getVScaleRange(F: &F, BitWidth).multiply(Other: Multiplier);
4750 }
4751 default:
4752 break;
4753 }
4754
4755 return ConstantRange::getFull(BitWidth);
4756}
4757
4758ConstantRange
4759SelectionDAG::computeConstantRangeIncludingKnownBits(SDValue Op, bool ForSigned,
4760 unsigned Depth) const {
4761 APInt DemandedElts = getDemandAllEltsMask(V: Op);
4762 return computeConstantRangeIncludingKnownBits(Op, DemandedElts, ForSigned,
4763 Depth);
4764}
4765
4766ConstantRange SelectionDAG::computeConstantRangeIncludingKnownBits(
4767 SDValue Op, const APInt &DemandedElts, bool ForSigned,
4768 unsigned Depth) const {
4769 KnownBits Known = computeKnownBits(Op, DemandedElts, Depth);
4770 ConstantRange CR1 = ConstantRange::fromKnownBits(Known, IsSigned: ForSigned);
4771 ConstantRange CR2 = computeConstantRange(Op, DemandedElts, ForSigned, Depth);
4772 ConstantRange::PreferredRangeType RangeType =
4773 ForSigned ? ConstantRange::Signed : ConstantRange::Unsigned;
4774 return CR1.intersectWith(CR: CR2, Type: RangeType);
4775}
4776
4777bool SelectionDAG::isKnownToBeAPowerOfTwo(SDValue Val, bool OrZero,
4778 unsigned Depth) const {
4779 APInt DemandedElts = getDemandAllEltsMask(V: Val);
4780 return isKnownToBeAPowerOfTwo(Val, DemandedElts, OrZero, Depth);
4781}
4782
4783bool SelectionDAG::isKnownToBeAPowerOfTwo(SDValue Val,
4784 const APInt &DemandedElts,
4785 bool OrZero, unsigned Depth) const {
4786 if (Depth >= MaxRecursionDepth)
4787 return false; // Limit search depth.
4788
4789 EVT OpVT = Val.getValueType();
4790 unsigned BitWidth = OpVT.getScalarSizeInBits();
4791 [[maybe_unused]] unsigned NumElts = DemandedElts.getBitWidth();
4792 assert((!OpVT.isScalableVector() || NumElts == 1) &&
4793 "DemandedElts for scalable vectors must be 1 to represent all lanes");
4794 assert(
4795 (!OpVT.isFixedLengthVector() || NumElts == OpVT.getVectorNumElements()) &&
4796 "Unexpected vector size");
4797
4798 auto IsPowerOfTwoOrZero = [BitWidth, OrZero](const ConstantSDNode *C) {
4799 APInt V = C->getAPIntValue().zextOrTrunc(width: BitWidth);
4800 return (OrZero && V.isZero()) || V.isPowerOf2();
4801 };
4802
4803 // Is the constant a known power of 2 or zero?
4804 if (ISD::matchUnaryPredicate(Op: Val, DemandedElts, Match: IsPowerOfTwoOrZero,
4805 /*AllowUndefs=*/false, /*AllowTruncation=*/true))
4806 return true;
4807
4808 switch (Val.getOpcode()) {
4809 case ISD::EXTRACT_VECTOR_ELT: {
4810 SDValue InVec = Val.getOperand(i: 0);
4811 SDValue EltNo = Val.getOperand(i: 1);
4812 EVT VecVT = InVec.getValueType();
4813
4814 // Skip scalable vectors or implicit extensions.
4815 if (VecVT.isScalableVector() ||
4816 OpVT.getScalarSizeInBits() != VecVT.getScalarSizeInBits())
4817 break;
4818
4819 // If we know the element index, just demand that vector element, else for
4820 // an unknown element index, ignore DemandedElts and demand them all.
4821 const unsigned NumSrcElts = VecVT.getVectorNumElements();
4822 auto *ConstEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
4823 APInt DemandedSrcElts =
4824 ConstEltNo && ConstEltNo->getAPIntValue().ult(RHS: NumSrcElts)
4825 ? APInt::getOneBitSet(numBits: NumSrcElts, BitNo: ConstEltNo->getZExtValue())
4826 : APInt::getAllOnes(numBits: NumSrcElts);
4827 return isKnownToBeAPowerOfTwo(Val: InVec, DemandedElts: DemandedSrcElts, OrZero, Depth: Depth + 1);
4828 }
4829
4830 case ISD::AND: {
4831 // Looking for `x & -x` pattern:
4832 // If x == 0:
4833 // x & -x -> 0
4834 // If x != 0:
4835 // x & -x -> non-zero pow2
4836 // so if we find the pattern return whether we know `x` is non-zero.
4837 SDValue X, Z;
4838 if (sd_match(N: Val, P: m_And(L: m_Value(N&: X), R: m_Neg(V: m_Deferred(V&: X)))) ||
4839 (sd_match(N: Val, P: m_And(L: m_Value(N&: X), R: m_Sub(L: m_Value(N&: Z), R: m_Deferred(V&: X)))) &&
4840 MaskedVectorIsZero(V: Z, DemandedElts, Depth: Depth + 1)))
4841 return OrZero || isKnownNeverZero(Op: X, DemandedElts, Depth);
4842 break;
4843 }
4844
4845 case ISD::SHL: {
4846 // A left-shift of a constant one will have exactly one bit set because
4847 // shifting the bit off the end is undefined.
4848 auto *C = isConstOrConstSplat(N: Val.getOperand(i: 0), DemandedElts);
4849 if (C && C->getAPIntValue() == 1)
4850 return true;
4851 return (OrZero || isKnownNeverZero(Op: Val, DemandedElts, Depth)) &&
4852 isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), DemandedElts, OrZero,
4853 Depth: Depth + 1);
4854 }
4855
4856 case ISD::SRL: {
4857 // A logical right-shift of a constant sign-bit will have exactly
4858 // one bit set.
4859 auto *C = isConstOrConstSplat(N: Val.getOperand(i: 0), DemandedElts);
4860 if (C && C->getAPIntValue().isSignMask())
4861 return true;
4862 return (OrZero || isKnownNeverZero(Op: Val, DemandedElts, Depth)) &&
4863 isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), DemandedElts, OrZero,
4864 Depth: Depth + 1);
4865 }
4866
4867 case ISD::TRUNCATE:
4868 return (OrZero || isKnownNeverZero(Op: Val, DemandedElts, Depth)) &&
4869 isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), DemandedElts, OrZero,
4870 Depth: Depth + 1);
4871
4872 case ISD::ROTL:
4873 case ISD::ROTR:
4874 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), DemandedElts, OrZero,
4875 Depth: Depth + 1);
4876 case ISD::BSWAP:
4877 case ISD::BITREVERSE:
4878 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), DemandedElts, OrZero,
4879 Depth: Depth + 1);
4880
4881 case ISD::SMIN:
4882 case ISD::SMAX:
4883 case ISD::UMIN:
4884 case ISD::UMAX:
4885 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 1), DemandedElts, OrZero,
4886 Depth: Depth + 1) &&
4887 isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), DemandedElts, OrZero,
4888 Depth: Depth + 1);
4889
4890 case ISD::SELECT:
4891 case ISD::VSELECT:
4892 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 2), DemandedElts, OrZero,
4893 Depth: Depth + 1) &&
4894 isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 1), DemandedElts, OrZero,
4895 Depth: Depth + 1);
4896
4897 case ISD::ZERO_EXTEND:
4898 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), DemandedElts, OrZero,
4899 Depth: Depth + 1);
4900
4901 case ISD::VSCALE:
4902 // vscale(power-of-two) is a power-of-two
4903 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), /*OrZero=*/false,
4904 Depth: Depth + 1);
4905
4906 case ISD::VECTOR_SHUFFLE: {
4907 assert(!Val.getValueType().isScalableVector());
4908 // Demanded elements with undef shuffle mask elements are unknown
4909 // - we cannot guarantee they are a power of two, so return false.
4910 APInt DemandedLHS, DemandedRHS;
4911 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val);
4912 assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
4913 if (!getShuffleDemandedElts(SrcWidth: NumElts, Mask: SVN->getMask(), DemandedElts,
4914 DemandedLHS, DemandedRHS))
4915 return false;
4916
4917 // All demanded elements from LHS must be known power of two.
4918 if (!!DemandedLHS && !isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), DemandedElts: DemandedLHS,
4919 OrZero, Depth: Depth + 1))
4920 return false;
4921
4922 // All demanded elements from RHS must be known power of two.
4923 if (!!DemandedRHS && !isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 1), DemandedElts: DemandedRHS,
4924 OrZero, Depth: Depth + 1))
4925 return false;
4926
4927 return true;
4928 }
4929 }
4930
4931 // More could be done here, though the above checks are enough
4932 // to handle some common cases.
4933 return false;
4934}
4935
4936bool SelectionDAG::isKnownToBeAPowerOfTwoFP(SDValue Val, unsigned Depth) const {
4937 if (ConstantFPSDNode *C1 = isConstOrConstSplatFP(N: Val, AllowUndefs: true))
4938 return C1->getValueAPF().getExactLog2Abs() >= 0;
4939
4940 if (Val.getOpcode() == ISD::UINT_TO_FP || Val.getOpcode() == ISD::SINT_TO_FP)
4941 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), OrZero: Depth + 1);
4942
4943 return false;
4944}
4945
4946unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const {
4947 APInt DemandedElts = getDemandAllEltsMask(V: Op);
4948 return ComputeNumSignBits(Op, DemandedElts, Depth);
4949}
4950
4951unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, const APInt &DemandedElts,
4952 unsigned Depth) const {
4953 EVT VT = Op.getValueType();
4954 assert((VT.isInteger() || VT.isFloatingPoint()) && "Invalid VT!");
4955 unsigned VTBits = VT.getScalarSizeInBits();
4956 unsigned NumElts = DemandedElts.getBitWidth();
4957 unsigned Tmp, Tmp2;
4958 unsigned FirstAnswer = 1;
4959
4960 assert((!VT.isScalableVector() || NumElts == 1) &&
4961 "DemandedElts for scalable vectors must be 1 to represent all lanes");
4962
4963 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op)) {
4964 const APInt &Val = C->getAPIntValue();
4965 return Val.getNumSignBits();
4966 }
4967
4968 if (Depth >= MaxRecursionDepth)
4969 return 1; // Limit search depth.
4970
4971 if (!DemandedElts)
4972 return 1; // No demanded elts, better to assume we don't know anything.
4973
4974 unsigned Opcode = Op.getOpcode();
4975 switch (Opcode) {
4976 default: break;
4977 case ISD::AssertSext:
4978 Tmp = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT().getSizeInBits();
4979 return VTBits-Tmp+1;
4980 case ISD::AssertZext:
4981 Tmp = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT().getSizeInBits();
4982 return VTBits-Tmp;
4983 case ISD::FREEZE:
4984 if (isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 0), DemandedElts,
4985 Kind: UndefPoisonKind::UndefOrPoison))
4986 return ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4987 break;
4988 case ISD::MERGE_VALUES:
4989 return ComputeNumSignBits(Op: Op.getOperand(i: Op.getResNo()), DemandedElts,
4990 Depth: Depth + 1);
4991 case ISD::SPLAT_VECTOR: {
4992 // Check if the sign bits of source go down as far as the truncated value.
4993 unsigned NumSrcBits = Op.getOperand(i: 0).getValueSizeInBits();
4994 unsigned NumSrcSignBits = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
4995 if (NumSrcSignBits > (NumSrcBits - VTBits))
4996 return NumSrcSignBits - (NumSrcBits - VTBits);
4997 break;
4998 }
4999 case ISD::BUILD_VECTOR:
5000 assert(!VT.isScalableVector());
5001 Tmp = VTBits;
5002 for (unsigned i = 0, e = Op.getNumOperands(); (i < e) && (Tmp > 1); ++i) {
5003 if (!DemandedElts[i])
5004 continue;
5005
5006 SDValue SrcOp = Op.getOperand(i);
5007 // BUILD_VECTOR can implicitly truncate sources, we handle this specially
5008 // for constant nodes to ensure we only look at the sign bits.
5009 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: SrcOp)) {
5010 APInt T = C->getAPIntValue().trunc(width: VTBits);
5011 Tmp2 = T.getNumSignBits();
5012 } else if (SrcOp.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
5013 SrcOp.getOperand(i: 0).getScalarValueSizeInBits() >= VTBits) {
5014 // EXTRACT_VECTOR_ELT can extend the value with high bits undefined. If
5015 // this BUILD_VECTOR truncates those undefined bits we can just look
5016 // through the SrcOp and query the vector directly.
5017 SDValue InVec = SrcOp.getOperand(i: 0);
5018 EVT InVecVT = InVec.getValueType();
5019
5020 APInt DemandedSrcElts;
5021 if (InVecVT.isScalableVector())
5022 // Demand all elements.
5023 DemandedSrcElts = APInt(1, 1);
5024 else {
5025 unsigned NumSrcElts = InVecVT.getVectorNumElements();
5026 auto *ConstEltNo = dyn_cast<ConstantSDNode>(Val: SrcOp.getOperand(i: 1));
5027 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(RHS: NumSrcElts))
5028 DemandedSrcElts =
5029 APInt::getOneBitSet(numBits: NumSrcElts, BitNo: ConstEltNo->getZExtValue());
5030 else
5031 DemandedSrcElts = APInt::getAllOnes(numBits: NumSrcElts);
5032 }
5033
5034 Tmp2 = ComputeNumSignBits(Op: InVec, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
5035 unsigned ExtraBits = InVec.getScalarValueSizeInBits() - VTBits;
5036 if (ExtraBits)
5037 Tmp2 = (Tmp2 > ExtraBits ? Tmp2 - ExtraBits : 1);
5038 } else {
5039 Tmp2 = ComputeNumSignBits(Op: SrcOp, Depth: Depth + 1);
5040
5041 if (SrcOp.getValueSizeInBits() != VTBits) {
5042 assert(SrcOp.getValueSizeInBits() > VTBits &&
5043 "Expected BUILD_VECTOR implicit truncation");
5044 unsigned ExtraBits = SrcOp.getValueSizeInBits() - VTBits;
5045 Tmp2 = (Tmp2 > ExtraBits ? Tmp2 - ExtraBits : 1);
5046 }
5047 }
5048 Tmp = std::min(a: Tmp, b: Tmp2);
5049 }
5050 return Tmp;
5051
5052 case ISD::VECTOR_COMPRESS: {
5053 SDValue Vec = Op.getOperand(i: 0);
5054 SDValue PassThru = Op.getOperand(i: 2);
5055 Tmp = ComputeNumSignBits(Op: PassThru, DemandedElts, Depth: Depth + 1);
5056 if (Tmp == 1)
5057 return 1;
5058 Tmp2 = ComputeNumSignBits(Op: Vec, Depth: Depth + 1);
5059 Tmp = std::min(a: Tmp, b: Tmp2);
5060 return Tmp;
5061 }
5062
5063 case ISD::VECTOR_SHUFFLE: {
5064 // Collect the minimum number of sign bits that are shared by every vector
5065 // element referenced by the shuffle.
5066 APInt DemandedLHS, DemandedRHS;
5067 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
5068 assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
5069 if (!getShuffleDemandedElts(SrcWidth: NumElts, Mask: SVN->getMask(), DemandedElts,
5070 DemandedLHS, DemandedRHS))
5071 return 1;
5072
5073 Tmp = std::numeric_limits<unsigned>::max();
5074 if (!!DemandedLHS)
5075 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts: DemandedLHS, Depth: Depth + 1);
5076 if (!!DemandedRHS) {
5077 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts: DemandedRHS, Depth: Depth + 1);
5078 Tmp = std::min(a: Tmp, b: Tmp2);
5079 }
5080 // If we don't know anything, early out and try computeKnownBits fall-back.
5081 if (Tmp == 1)
5082 break;
5083 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
5084 return Tmp;
5085 }
5086
5087 case ISD::BITCAST: {
5088 if (VT.isScalableVector())
5089 break;
5090 SDValue N0 = Op.getOperand(i: 0);
5091 EVT SrcVT = N0.getValueType();
5092 unsigned SrcBits = SrcVT.getScalarSizeInBits();
5093
5094 // Ignore bitcasts from unsupported types..
5095 if (!(SrcVT.isInteger() || SrcVT.isFloatingPoint()))
5096 break;
5097
5098 // Fast handling of 'identity' bitcasts.
5099 if (VTBits == SrcBits)
5100 return ComputeNumSignBits(Op: N0, DemandedElts, Depth: Depth + 1);
5101
5102 bool IsLE = getDataLayout().isLittleEndian();
5103
5104 // Bitcast 'large element' scalar/vector to 'small element' vector.
5105 if ((SrcBits % VTBits) == 0) {
5106 assert(VT.isVector() && "Expected bitcast to vector");
5107
5108 unsigned Scale = SrcBits / VTBits;
5109 APInt SrcDemandedElts =
5110 APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumElts / Scale);
5111
5112 // Fast case - sign splat can be simply split across the small elements.
5113 Tmp = ComputeNumSignBits(Op: N0, DemandedElts: SrcDemandedElts, Depth: Depth + 1);
5114 if (Tmp == SrcBits)
5115 return VTBits;
5116
5117 // Slow case - determine how far the sign extends into each sub-element.
5118 Tmp2 = VTBits;
5119 for (unsigned i = 0; i != NumElts; ++i)
5120 if (DemandedElts[i]) {
5121 unsigned SubOffset = i % Scale;
5122 SubOffset = (IsLE ? ((Scale - 1) - SubOffset) : SubOffset);
5123 SubOffset = SubOffset * VTBits;
5124 if (Tmp <= SubOffset)
5125 return 1;
5126 Tmp2 = std::min(a: Tmp2, b: Tmp - SubOffset);
5127 }
5128 return Tmp2;
5129 }
5130 break;
5131 }
5132
5133 case ISD::FP_TO_SINT_SAT:
5134 // FP_TO_SINT_SAT produces a signed value that fits in the saturating VT.
5135 Tmp = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT().getScalarSizeInBits();
5136 return VTBits - Tmp + 1;
5137 case ISD::SIGN_EXTEND:
5138 Tmp = VTBits - Op.getOperand(i: 0).getScalarValueSizeInBits();
5139 return ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth+1) + Tmp;
5140 case ISD::SIGN_EXTEND_INREG:
5141 // Max of the input and what this extends.
5142 Tmp = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT().getScalarSizeInBits();
5143 Tmp = VTBits-Tmp+1;
5144 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth+1);
5145 return std::max(a: Tmp, b: Tmp2);
5146 case ISD::SIGN_EXTEND_VECTOR_INREG: {
5147 if (VT.isScalableVector())
5148 break;
5149 SDValue Src = Op.getOperand(i: 0);
5150 EVT SrcVT = Src.getValueType();
5151 APInt DemandedSrcElts = DemandedElts.zext(width: SrcVT.getVectorNumElements());
5152 Tmp = VTBits - SrcVT.getScalarSizeInBits();
5153 return ComputeNumSignBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth+1) + Tmp;
5154 }
5155 case ISD::SRA:
5156 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5157 // SRA X, C -> adds C sign bits.
5158 if (std::optional<unsigned> ShAmt =
5159 getValidMinimumShiftAmount(V: Op, DemandedElts, Depth: Depth + 1))
5160 Tmp = std::min(a: Tmp + *ShAmt, b: VTBits);
5161 return Tmp;
5162 case ISD::SHL:
5163 if (std::optional<ConstantRange> ShAmtRange =
5164 getValidShiftAmountRange(V: Op, DemandedElts, Depth: Depth + 1)) {
5165 unsigned MaxShAmt = ShAmtRange->getUnsignedMax().getZExtValue();
5166 unsigned MinShAmt = ShAmtRange->getUnsignedMin().getZExtValue();
5167 // Try to look through ZERO/SIGN/ANY_EXTEND. If all extended bits are
5168 // shifted out, then we can compute the number of sign bits for the
5169 // operand being extended. A future improvement could be to pass along the
5170 // "shifted left by" information in the recursive calls to
5171 // ComputeKnownSignBits. Allowing us to handle this more generically.
5172 if (ISD::isExtOpcode(Opcode: Op.getOperand(i: 0).getOpcode())) {
5173 SDValue Ext = Op.getOperand(i: 0);
5174 EVT ExtVT = Ext.getValueType();
5175 SDValue Extendee = Ext.getOperand(i: 0);
5176 EVT ExtendeeVT = Extendee.getValueType();
5177 unsigned SizeDifference =
5178 ExtVT.getScalarSizeInBits() - ExtendeeVT.getScalarSizeInBits();
5179 if (SizeDifference <= MinShAmt) {
5180 Tmp = SizeDifference +
5181 ComputeNumSignBits(Op: Extendee, DemandedElts, Depth: Depth + 1);
5182 if (MaxShAmt < Tmp)
5183 return Tmp - MaxShAmt;
5184 }
5185 }
5186 // shl destroys sign bits, ensure it doesn't shift out all sign bits.
5187 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5188 if (MaxShAmt < Tmp)
5189 return Tmp - MaxShAmt;
5190 }
5191 break;
5192 case ISD::AND:
5193 case ISD::OR:
5194 case ISD::XOR: // NOT is handled here.
5195 // Logical binary ops preserve the number of sign bits at the worst.
5196 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth+1);
5197 if (Tmp != 1) {
5198 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth+1);
5199 FirstAnswer = std::min(a: Tmp, b: Tmp2);
5200 // We computed what we know about the sign bits as our first
5201 // answer. Now proceed to the generic code that uses
5202 // computeKnownBits, and pick whichever answer is better.
5203 }
5204 break;
5205
5206 case ISD::SELECT:
5207 case ISD::VSELECT:
5208 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth+1);
5209 if (Tmp == 1) return 1; // Early out.
5210 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth+1);
5211 return std::min(a: Tmp, b: Tmp2);
5212 case ISD::SELECT_CC:
5213 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth+1);
5214 if (Tmp == 1) return 1; // Early out.
5215 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 3), DemandedElts, Depth: Depth+1);
5216 return std::min(a: Tmp, b: Tmp2);
5217
5218 case ISD::SMIN:
5219 case ISD::SMAX: {
5220 // If we have a clamp pattern, we know that the number of sign bits will be
5221 // the minimum of the clamp min/max range.
5222 bool IsMax = (Opcode == ISD::SMAX);
5223 ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr;
5224 if ((CstLow = isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts)))
5225 if (Op.getOperand(i: 0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX))
5226 CstHigh =
5227 isConstOrConstSplat(N: Op.getOperand(i: 0).getOperand(i: 1), DemandedElts);
5228 if (CstLow && CstHigh) {
5229 if (!IsMax)
5230 std::swap(a&: CstLow, b&: CstHigh);
5231 if (CstLow->getAPIntValue().sle(RHS: CstHigh->getAPIntValue())) {
5232 Tmp = CstLow->getAPIntValue().getNumSignBits();
5233 Tmp2 = CstHigh->getAPIntValue().getNumSignBits();
5234 return std::min(a: Tmp, b: Tmp2);
5235 }
5236 }
5237
5238 // Fallback - just get the minimum number of sign bits of the operands.
5239 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5240 if (Tmp == 1)
5241 return 1; // Early out.
5242 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5243 return std::min(a: Tmp, b: Tmp2);
5244 }
5245 case ISD::UMIN:
5246 case ISD::UMAX:
5247 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5248 if (Tmp == 1)
5249 return 1; // Early out.
5250 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5251 return std::min(a: Tmp, b: Tmp2);
5252 case ISD::SSUBO_CARRY:
5253 case ISD::USUBO_CARRY:
5254 // sub_carry(x,x,c) -> 0/-1 (sext carry)
5255 if (Op.getResNo() == 0 && Op.getOperand(i: 0) == Op.getOperand(i: 1))
5256 return VTBits;
5257 [[fallthrough]];
5258 case ISD::SADDO:
5259 case ISD::UADDO:
5260 case ISD::SADDO_CARRY:
5261 case ISD::UADDO_CARRY:
5262 case ISD::SSUBO:
5263 case ISD::USUBO:
5264 case ISD::SMULO:
5265 case ISD::UMULO:
5266 if (Op.getResNo() != 1)
5267 break;
5268 // The boolean result conforms to getBooleanContents. Fall through.
5269 // If setcc returns 0/-1, all bits are sign bits.
5270 // We know that we have an integer-based boolean since these operations
5271 // are only available for integer.
5272 if (TLI->getBooleanContents(isVec: VT.isVector(), isFloat: false) ==
5273 TargetLowering::ZeroOrNegativeOneBooleanContent)
5274 return VTBits;
5275 break;
5276 case ISD::SETCC:
5277 case ISD::SETCCCARRY:
5278 case ISD::STRICT_FSETCC:
5279 case ISD::STRICT_FSETCCS: {
5280 unsigned OpNo = Op->isStrictFPOpcode() ? 1 : 0;
5281 // If setcc returns 0/-1, all bits are sign bits.
5282 if (TLI->getBooleanContents(Type: Op.getOperand(i: OpNo).getValueType()) ==
5283 TargetLowering::ZeroOrNegativeOneBooleanContent)
5284 return VTBits;
5285 break;
5286 }
5287 case ISD::GET_ACTIVE_LANE_MASK:
5288 case ISD::MASK_BEFOREFIRST:
5289 // Semantically similar to icmp ult.
5290 if (TLI->getBooleanContents(isVec: VT.isVector(), /*isFloat=*/false) ==
5291 TargetLowering::ZeroOrNegativeOneBooleanContent)
5292 return VTBits;
5293 break;
5294 case ISD::ROTL:
5295 case ISD::ROTR: {
5296 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5297 ConstantSDNode *C = isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts);
5298 FirstAnswer = SignBitsOps::rot(
5299 SrcSignBits: Tmp, BitWidth: VTBits, RotAmt: C ? std::optional(C->getAPIntValue()) : std::nullopt,
5300 IsRotateRight: Opcode == ISD::ROTR);
5301 break;
5302 }
5303 case ISD::ADD:
5304 case ISD::ADDC:
5305 // TODO: Move Operand 1 check before Operand 0 check
5306 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5307 if (Tmp == 1) return 1; // Early out.
5308
5309 // Special case decrementing a value (ADD X, -1):
5310 if (ConstantSDNode *CRHS =
5311 isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts))
5312 if (CRHS->isAllOnes()) {
5313 KnownBits Known =
5314 computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5315
5316 // If the input is known to be 0 or 1, the output is 0/-1, which is all
5317 // sign bits set.
5318 if ((Known.Zero | 1).isAllOnes())
5319 return VTBits;
5320
5321 // If we are subtracting one from a positive number, there is no carry
5322 // out of the result.
5323 if (Known.isNonNegative())
5324 return Tmp;
5325 }
5326
5327 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5328 if (Tmp2 == 1) return 1; // Early out.
5329
5330 // Add can have at most one carry bit. Thus we know that the output
5331 // is, at worst, one more bit than the inputs.
5332 return std::min(a: Tmp, b: Tmp2) - 1;
5333 case ISD::SUB:
5334 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5335 if (Tmp2 == 1) return 1; // Early out.
5336
5337 // Handle NEG.
5338 if (ConstantSDNode *CLHS =
5339 isConstOrConstSplat(N: Op.getOperand(i: 0), DemandedElts))
5340 if (CLHS->isZero()) {
5341 KnownBits Known =
5342 computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5343 // If the input is known to be 0 or 1, the output is 0/-1, which is all
5344 // sign bits set.
5345 if ((Known.Zero | 1).isAllOnes())
5346 return VTBits;
5347
5348 // If the input is known to be positive (the sign bit is known clear),
5349 // the output of the NEG has the same number of sign bits as the input.
5350 if (Known.isNonNegative())
5351 return Tmp2;
5352
5353 // Otherwise, we treat this like a SUB.
5354 }
5355
5356 // Sub can have at most one carry bit. Thus we know that the output
5357 // is, at worst, one more bit than the inputs.
5358 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5359 if (Tmp == 1) return 1; // Early out.
5360 return std::min(a: Tmp, b: Tmp2) - 1;
5361 case ISD::MUL: {
5362 // The output of the Mul can be at most twice the valid bits in the inputs.
5363 unsigned SignBitsOp0 =
5364 ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5365 if (SignBitsOp0 == 1)
5366 break;
5367 unsigned SignBitsOp1 =
5368 ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5369 if (SignBitsOp1 == 1)
5370 break;
5371 unsigned OutValidBits =
5372 (VTBits - SignBitsOp0 + 1) + (VTBits - SignBitsOp1 + 1);
5373 return OutValidBits > VTBits ? 1 : VTBits - OutValidBits + 1;
5374 }
5375 case ISD::AVGCEILS:
5376 case ISD::AVGFLOORS:
5377 case ISD::AVGSUBFLOORS:
5378 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5379 if (Tmp == 1)
5380 return 1; // Early out.
5381 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5382 return std::min(a: Tmp, b: Tmp2);
5383 case ISD::SREM:
5384 // The sign bit is the LHS's sign bit, except when the result of the
5385 // remainder is zero. The magnitude of the result should be less than or
5386 // equal to the magnitude of the LHS. Therefore, the result should have
5387 // at least as many sign bits as the left hand side.
5388 return ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5389 case ISD::TRUNCATE: {
5390 // Check if the sign bits of source go down as far as the truncated value.
5391 unsigned NumSrcBits = Op.getOperand(i: 0).getScalarValueSizeInBits();
5392 unsigned NumSrcSignBits =
5393 ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5394 if (NumSrcSignBits > (NumSrcBits - VTBits))
5395 return NumSrcSignBits - (NumSrcBits - VTBits);
5396 break;
5397 }
5398 case ISD::EXTRACT_ELEMENT: {
5399 if (VT.isScalableVector())
5400 break;
5401 const int KnownSign = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth+1);
5402 const int BitWidth = Op.getValueSizeInBits();
5403 const int Items = Op.getOperand(i: 0).getValueSizeInBits() / BitWidth;
5404
5405 // Get reverse index (starting from 1), Op1 value indexes elements from
5406 // little end. Sign starts at big end.
5407 const int rIndex = Items - 1 - Op.getConstantOperandVal(i: 1);
5408
5409 // If the sign portion ends in our element the subtraction gives correct
5410 // result. Otherwise it gives either negative or > bitwidth result
5411 return std::clamp(val: KnownSign - rIndex * BitWidth, lo: 1, hi: BitWidth);
5412 }
5413 case ISD::INSERT_VECTOR_ELT: {
5414 if (VT.isScalableVector())
5415 break;
5416 // If we know the element index, split the demand between the
5417 // source vector and the inserted element, otherwise assume we need
5418 // the original demanded vector elements and the value.
5419 SDValue InVec = Op.getOperand(i: 0);
5420 SDValue InVal = Op.getOperand(i: 1);
5421 SDValue EltNo = Op.getOperand(i: 2);
5422 bool DemandedVal = true;
5423 APInt DemandedVecElts = DemandedElts;
5424 auto *CEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
5425 if (CEltNo && CEltNo->getAPIntValue().ult(RHS: NumElts)) {
5426 unsigned EltIdx = CEltNo->getZExtValue();
5427 DemandedVal = !!DemandedElts[EltIdx];
5428 DemandedVecElts.clearBit(BitPosition: EltIdx);
5429 }
5430 Tmp = std::numeric_limits<unsigned>::max();
5431 if (DemandedVal) {
5432 // TODO - handle implicit truncation of inserted elements.
5433 if (InVal.getScalarValueSizeInBits() != VTBits)
5434 break;
5435 Tmp2 = ComputeNumSignBits(Op: InVal, Depth: Depth + 1);
5436 Tmp = std::min(a: Tmp, b: Tmp2);
5437 }
5438 if (!!DemandedVecElts) {
5439 Tmp2 = ComputeNumSignBits(Op: InVec, DemandedElts: DemandedVecElts, Depth: Depth + 1);
5440 Tmp = std::min(a: Tmp, b: Tmp2);
5441 }
5442 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
5443 return Tmp;
5444 }
5445 case ISD::EXTRACT_VECTOR_ELT: {
5446 SDValue InVec = Op.getOperand(i: 0);
5447 SDValue EltNo = Op.getOperand(i: 1);
5448 EVT VecVT = InVec.getValueType();
5449 // ComputeNumSignBits not yet implemented for scalable vectors.
5450 if (VecVT.isScalableVector())
5451 break;
5452 const unsigned BitWidth = Op.getValueSizeInBits();
5453 const unsigned EltBitWidth = Op.getOperand(i: 0).getScalarValueSizeInBits();
5454 const unsigned NumSrcElts = VecVT.getVectorNumElements();
5455
5456 // If BitWidth > EltBitWidth the value is anyext:ed, and we do not know
5457 // anything about sign bits. But if the sizes match we can derive knowledge
5458 // about sign bits from the vector operand.
5459 if (BitWidth != EltBitWidth)
5460 break;
5461
5462 // If we know the element index, just demand that vector element, else for
5463 // an unknown element index, ignore DemandedElts and demand them all.
5464 APInt DemandedSrcElts = APInt::getAllOnes(numBits: NumSrcElts);
5465 auto *ConstEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
5466 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(RHS: NumSrcElts))
5467 DemandedSrcElts =
5468 APInt::getOneBitSet(numBits: NumSrcElts, BitNo: ConstEltNo->getZExtValue());
5469
5470 return ComputeNumSignBits(Op: InVec, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
5471 }
5472 case ISD::EXTRACT_SUBVECTOR: {
5473 // Offset the demanded elts by the subvector index.
5474 SDValue Src = Op.getOperand(i: 0);
5475
5476 APInt DemandedSrcElts;
5477 if (Src.getValueType().isScalableVector())
5478 DemandedSrcElts = APInt(1, 1);
5479 else {
5480 uint64_t Idx = Op.getConstantOperandVal(i: 1);
5481 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
5482 DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
5483 }
5484 return ComputeNumSignBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
5485 }
5486 case ISD::CONCAT_VECTORS: {
5487 if (VT.isScalableVector())
5488 break;
5489 // Determine the minimum number of sign bits across all demanded
5490 // elts of the input vectors. Early out if the result is already 1.
5491 Tmp = std::numeric_limits<unsigned>::max();
5492 EVT SubVectorVT = Op.getOperand(i: 0).getValueType();
5493 unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements();
5494 unsigned NumSubVectors = Op.getNumOperands();
5495 for (unsigned i = 0; (i < NumSubVectors) && (Tmp > 1); ++i) {
5496 APInt DemandedSub =
5497 DemandedElts.extractBits(numBits: NumSubVectorElts, bitPosition: i * NumSubVectorElts);
5498 if (!DemandedSub)
5499 continue;
5500 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i), DemandedElts: DemandedSub, Depth: Depth + 1);
5501 Tmp = std::min(a: Tmp, b: Tmp2);
5502 }
5503 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
5504 return Tmp;
5505 }
5506 case ISD::INSERT_SUBVECTOR: {
5507 SDValue Src = Op.getOperand(i: 0);
5508 SDValue Sub = Op.getOperand(i: 1);
5509 if (VT.isScalableVector()) {
5510 Tmp = ComputeNumSignBits(Op: Sub, Depth: Depth + 1);
5511 Tmp = std::min(a: Tmp, b: ComputeNumSignBits(Op: Src, Depth: Depth + 1));
5512 return Tmp;
5513 }
5514 // Demand any elements from the subvector and the remainder from the src its
5515 // inserted into.
5516 uint64_t Idx = Op.getConstantOperandVal(i: 2);
5517 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
5518 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
5519 APInt DemandedSrcElts = DemandedElts;
5520 DemandedSrcElts.clearBits(LoBit: Idx, HiBit: Idx + NumSubElts);
5521
5522 Tmp = std::numeric_limits<unsigned>::max();
5523 if (!!DemandedSubElts) {
5524 Tmp = ComputeNumSignBits(Op: Sub, DemandedElts: DemandedSubElts, Depth: Depth + 1);
5525 if (Tmp == 1)
5526 return 1; // early-out
5527 }
5528 if (!!DemandedSrcElts) {
5529 Tmp2 = ComputeNumSignBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
5530 Tmp = std::min(a: Tmp, b: Tmp2);
5531 }
5532 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
5533 return Tmp;
5534 }
5535 case ISD::LOAD: {
5536 // If we are looking at the loaded value of the SDNode.
5537 if (Op.getResNo() != 0)
5538 break;
5539
5540 LoadSDNode *LD = cast<LoadSDNode>(Val&: Op);
5541 if (const MDNode *Ranges = LD->getRanges()) {
5542 if (DemandedElts != 1)
5543 break;
5544
5545 ConstantRange CR = getConstantRangeFromMetadata(RangeMD: *Ranges);
5546 if (VTBits > CR.getBitWidth()) {
5547 switch (LD->getExtensionType()) {
5548 case ISD::SEXTLOAD:
5549 CR = CR.signExtend(BitWidth: VTBits);
5550 break;
5551 case ISD::ZEXTLOAD:
5552 CR = CR.zeroExtend(BitWidth: VTBits);
5553 break;
5554 default:
5555 break;
5556 }
5557 }
5558
5559 if (VTBits != CR.getBitWidth())
5560 break;
5561 return std::min(a: CR.getSignedMin().getNumSignBits(),
5562 b: CR.getSignedMax().getNumSignBits());
5563 }
5564
5565 unsigned ExtType = LD->getExtensionType();
5566 switch (ExtType) {
5567 default:
5568 break;
5569 case ISD::SEXTLOAD: // e.g. i16->i32 = '17' bits known.
5570 Tmp = LD->getMemoryVT().getScalarSizeInBits();
5571 return VTBits - Tmp + 1;
5572 case ISD::ZEXTLOAD: // e.g. i16->i32 = '16' bits known.
5573 Tmp = LD->getMemoryVT().getScalarSizeInBits();
5574 return VTBits - Tmp;
5575 case ISD::NON_EXTLOAD:
5576 if (const Constant *Cst = TLI->getTargetConstantFromLoad(LD)) {
5577 // We only need to handle vectors - computeKnownBits should handle
5578 // scalar cases.
5579 Type *CstTy = Cst->getType();
5580 if (CstTy->isVectorTy() && !VT.isScalableVector() &&
5581 (NumElts * VTBits) == CstTy->getPrimitiveSizeInBits() &&
5582 VTBits == CstTy->getScalarSizeInBits()) {
5583 Tmp = VTBits;
5584 for (unsigned i = 0; i != NumElts; ++i) {
5585 if (!DemandedElts[i])
5586 continue;
5587 if (Constant *Elt = Cst->getAggregateElement(Elt: i)) {
5588 if (auto *CInt = dyn_cast<ConstantInt>(Val: Elt)) {
5589 const APInt &Value = CInt->getValue();
5590 Tmp = std::min(a: Tmp, b: Value.getNumSignBits());
5591 continue;
5592 }
5593 if (auto *CFP = dyn_cast<ConstantFP>(Val: Elt)) {
5594 APInt Value = CFP->getValueAPF().bitcastToAPInt();
5595 Tmp = std::min(a: Tmp, b: Value.getNumSignBits());
5596 continue;
5597 }
5598 }
5599 // Unknown type. Conservatively assume no bits match sign bit.
5600 return 1;
5601 }
5602 return Tmp;
5603 }
5604 }
5605 break;
5606 }
5607
5608 break;
5609 }
5610 case ISD::ATOMIC_CMP_SWAP:
5611 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
5612 case ISD::ATOMIC_SWAP:
5613 case ISD::ATOMIC_LOAD_ADD:
5614 case ISD::ATOMIC_LOAD_SUB:
5615 case ISD::ATOMIC_LOAD_AND:
5616 case ISD::ATOMIC_LOAD_CLR:
5617 case ISD::ATOMIC_LOAD_OR:
5618 case ISD::ATOMIC_LOAD_XOR:
5619 case ISD::ATOMIC_LOAD_NAND:
5620 case ISD::ATOMIC_LOAD_MIN:
5621 case ISD::ATOMIC_LOAD_MAX:
5622 case ISD::ATOMIC_LOAD_UMIN:
5623 case ISD::ATOMIC_LOAD_UMAX:
5624 case ISD::ATOMIC_LOAD: {
5625 auto *AT = cast<AtomicSDNode>(Val&: Op);
5626 // If we are looking at the loaded value.
5627 if (Op.getResNo() == 0) {
5628 Tmp = AT->getMemoryVT().getScalarSizeInBits();
5629 if (Tmp == VTBits)
5630 return 1; // early-out
5631
5632 // For atomic_load, prefer to use the extension type.
5633 if (Op->getOpcode() == ISD::ATOMIC_LOAD) {
5634 switch (AT->getExtensionType()) {
5635 default:
5636 break;
5637 case ISD::SEXTLOAD:
5638 return VTBits - Tmp + 1;
5639 case ISD::ZEXTLOAD:
5640 return VTBits - Tmp;
5641 }
5642 }
5643
5644 if (TLI->getExtendForAtomicOps() == ISD::SIGN_EXTEND)
5645 return VTBits - Tmp + 1;
5646 if (TLI->getExtendForAtomicOps() == ISD::ZERO_EXTEND)
5647 return VTBits - Tmp;
5648 }
5649 break;
5650 }
5651 }
5652
5653 // Allow the target to implement this method for its nodes.
5654 if (Opcode >= ISD::BUILTIN_OP_END ||
5655 Opcode == ISD::INTRINSIC_WO_CHAIN ||
5656 Opcode == ISD::INTRINSIC_W_CHAIN ||
5657 Opcode == ISD::INTRINSIC_VOID) {
5658 // TODO: This can probably be removed once target code is audited. This
5659 // is here purely to reduce patch size and review complexity.
5660 if (!VT.isScalableVector()) {
5661 unsigned NumBits =
5662 TLI->ComputeNumSignBitsForTargetNode(Op, DemandedElts, DAG: *this, Depth);
5663 if (NumBits > 1)
5664 FirstAnswer = std::max(a: FirstAnswer, b: NumBits);
5665 }
5666 }
5667
5668 // Finally, if we can prove that the top bits of the result are 0's or 1's,
5669 // use this information.
5670 KnownBits Known = computeKnownBits(Op, DemandedElts, Depth);
5671 return std::max(a: FirstAnswer, b: Known.countMinSignBits());
5672}
5673
5674unsigned SelectionDAG::ComputeMaxSignificantBits(SDValue Op,
5675 unsigned Depth) const {
5676 unsigned SignBits = ComputeNumSignBits(Op, Depth);
5677 return Op.getScalarValueSizeInBits() - SignBits + 1;
5678}
5679
5680unsigned SelectionDAG::ComputeMaxSignificantBits(SDValue Op,
5681 const APInt &DemandedElts,
5682 unsigned Depth) const {
5683 unsigned SignBits = ComputeNumSignBits(Op, DemandedElts, Depth);
5684 return Op.getScalarValueSizeInBits() - SignBits + 1;
5685}
5686
5687bool SelectionDAG::isGuaranteedNotToBeUndefOrPoison(SDValue Op,
5688 UndefPoisonKind Kind,
5689 unsigned Depth) const {
5690 // Early out for FREEZE.
5691 if (Op.getOpcode() == ISD::FREEZE)
5692 return true;
5693
5694 APInt DemandedElts = getDemandAllEltsMask(V: Op);
5695 return isGuaranteedNotToBeUndefOrPoison(Op, DemandedElts, Kind, Depth);
5696}
5697
5698bool SelectionDAG::isGuaranteedNotToBeUndefOrPoison(SDValue Op,
5699 const APInt &DemandedElts,
5700 UndefPoisonKind Kind,
5701 unsigned Depth) const {
5702 unsigned Opcode = Op.getOpcode();
5703 unsigned NumElts = DemandedElts.getBitWidth();
5704
5705 // Early out for FREEZE.
5706 if (Opcode == ISD::FREEZE)
5707 return true;
5708
5709 if (Depth >= MaxRecursionDepth)
5710 return false; // Limit search depth.
5711
5712 if (isIntOrFPConstant(V: Op))
5713 return true;
5714
5715 switch (Opcode) {
5716 case ISD::CONDCODE:
5717 case ISD::VALUETYPE:
5718 case ISD::FrameIndex:
5719 case ISD::TargetFrameIndex:
5720 case ISD::CopyFromReg:
5721 return true;
5722
5723 case ISD::POISON:
5724 return !includesPoison(Kind);
5725
5726 case ISD::UNDEF:
5727 return !includesUndef(Kind);
5728
5729 case ISD::BITCAST: {
5730 SDValue Src = Op.getOperand(i: 0);
5731 EVT SrcVT = Src.getValueType();
5732 EVT DstVT = Op.getValueType();
5733
5734 if (!SrcVT.isVector() || !DstVT.isVector())
5735 return isGuaranteedNotToBeUndefOrPoison(Op: Src, Kind, Depth: Depth + 1);
5736
5737 unsigned SrcEltBits = SrcVT.getScalarSizeInBits();
5738 unsigned DstEltBits = DstVT.getScalarSizeInBits();
5739 ElementCount NumSrcElts = SrcVT.getVectorElementCount();
5740 [[maybe_unused]] ElementCount NumDstElts = DstVT.getVectorElementCount();
5741
5742 if (SrcEltBits == DstEltBits)
5743 return isGuaranteedNotToBeUndefOrPoison(Op: Src, DemandedElts, Kind,
5744 Depth: Depth + 1);
5745
5746 if (SrcEltBits < DstEltBits) {
5747 if (DstEltBits % SrcEltBits != 0)
5748 return isGuaranteedNotToBeUndefOrPoison(Op: Src, Kind, Depth: Depth + 1);
5749
5750 assert(NumSrcElts == NumDstElts * (DstEltBits / SrcEltBits) &&
5751 "Unexpected vector bitcast");
5752 APInt DemandedSrcElts =
5753 APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumSrcElts.getKnownMinValue());
5754 return isGuaranteedNotToBeUndefOrPoison(Op: Src, DemandedElts: DemandedSrcElts, Kind,
5755 Depth: Depth + 1);
5756 }
5757
5758 if (SrcEltBits % DstEltBits != 0)
5759 return isGuaranteedNotToBeUndefOrPoison(Op: Src, Kind, Depth: Depth + 1);
5760
5761 assert(NumDstElts == NumSrcElts * (SrcEltBits / DstEltBits) &&
5762 "Unexpected vector bitcast");
5763 APInt DemandedSrcElts =
5764 APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumSrcElts.getKnownMinValue());
5765 return isGuaranteedNotToBeUndefOrPoison(Op: Src, DemandedElts: DemandedSrcElts, Kind,
5766 Depth: Depth + 1);
5767 }
5768
5769 case ISD::BUILD_VECTOR:
5770 // NOTE: BUILD_VECTOR has implicit truncation of wider scalar elements -
5771 // this shouldn't affect the result.
5772 for (unsigned i = 0, e = Op.getNumOperands(); i < e; ++i) {
5773 if (!DemandedElts[i])
5774 continue;
5775 if (!isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i), Kind, Depth: Depth + 1))
5776 return false;
5777 }
5778 return true;
5779
5780 case ISD::CONCAT_VECTORS: {
5781 EVT VT = Op.getValueType();
5782 if (!VT.isFixedLengthVector())
5783 break;
5784
5785 EVT SubVT = Op.getOperand(i: 0).getValueType();
5786 unsigned NumSubElts = SubVT.getVectorNumElements();
5787 for (unsigned I = 0, E = Op.getNumOperands(); I != E; ++I) {
5788 APInt DemandedSubElts =
5789 DemandedElts.extractBits(numBits: NumSubElts, bitPosition: I * NumSubElts);
5790 if (!!DemandedSubElts &&
5791 !isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: I), DemandedElts: DemandedSubElts,
5792 Kind, Depth: Depth + 1))
5793 return false;
5794 }
5795 return true;
5796 }
5797
5798 case ISD::EXTRACT_SUBVECTOR: {
5799 SDValue Src = Op.getOperand(i: 0);
5800 if (Src.getValueType().isScalableVector())
5801 break;
5802 uint64_t Idx = Op.getConstantOperandVal(i: 1);
5803 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
5804 APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
5805 return isGuaranteedNotToBeUndefOrPoison(Op: Src, DemandedElts: DemandedSrcElts, Kind,
5806 Depth: Depth + 1);
5807 }
5808
5809 case ISD::INSERT_SUBVECTOR: {
5810 if (Op.getValueType().isScalableVector())
5811 break;
5812 SDValue Src = Op.getOperand(i: 0);
5813 SDValue Sub = Op.getOperand(i: 1);
5814 uint64_t Idx = Op.getConstantOperandVal(i: 2);
5815 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
5816 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
5817 APInt DemandedSrcElts = DemandedElts;
5818 DemandedSrcElts.clearBits(LoBit: Idx, HiBit: Idx + NumSubElts);
5819
5820 if (!!DemandedSubElts && !isGuaranteedNotToBeUndefOrPoison(
5821 Op: Sub, DemandedElts: DemandedSubElts, Kind, Depth: Depth + 1))
5822 return false;
5823 if (!!DemandedSrcElts && !isGuaranteedNotToBeUndefOrPoison(
5824 Op: Src, DemandedElts: DemandedSrcElts, Kind, Depth: Depth + 1))
5825 return false;
5826 return true;
5827 }
5828
5829 case ISD::EXTRACT_VECTOR_ELT: {
5830 SDValue Src = Op.getOperand(i: 0);
5831 auto *IndexC = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1));
5832 EVT SrcVT = Src.getValueType();
5833 if (SrcVT.isFixedLengthVector() && IndexC &&
5834 IndexC->getAPIntValue().ult(RHS: SrcVT.getVectorNumElements())) {
5835 APInt DemandedSrcElts = APInt::getOneBitSet(numBits: SrcVT.getVectorNumElements(),
5836 BitNo: IndexC->getZExtValue());
5837 return isGuaranteedNotToBeUndefOrPoison(Op: Src, DemandedElts: DemandedSrcElts, Kind,
5838 Depth: Depth + 1);
5839 }
5840 break;
5841 }
5842
5843 case ISD::INSERT_VECTOR_ELT: {
5844 SDValue InVec = Op.getOperand(i: 0);
5845 SDValue InVal = Op.getOperand(i: 1);
5846 SDValue EltNo = Op.getOperand(i: 2);
5847 EVT VT = InVec.getValueType();
5848 auto *IndexC = dyn_cast<ConstantSDNode>(Val&: EltNo);
5849 if (IndexC && VT.isFixedLengthVector() &&
5850 IndexC->getAPIntValue().ult(RHS: VT.getVectorNumElements())) {
5851 if (DemandedElts[IndexC->getZExtValue()] &&
5852 !isGuaranteedNotToBeUndefOrPoison(Op: InVal, Kind, Depth: Depth + 1))
5853 return false;
5854 APInt InVecDemandedElts = DemandedElts;
5855 InVecDemandedElts.clearBit(BitPosition: IndexC->getZExtValue());
5856 if (!!InVecDemandedElts &&
5857 !isGuaranteedNotToBeUndefOrPoison(
5858 Op: peekThroughInsertVectorElt(V: InVec, DemandedElts: InVecDemandedElts),
5859 DemandedElts: InVecDemandedElts, Kind, Depth: Depth + 1))
5860 return false;
5861 return true;
5862 }
5863 break;
5864 }
5865
5866 case ISD::SCALAR_TO_VECTOR:
5867 // Check upper (known poison) elements.
5868 if (DemandedElts.ugt(RHS: 1) && includesPoison(Kind))
5869 return false;
5870 // Check element zero.
5871 if (DemandedElts[0] &&
5872 !isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 0), Kind, Depth: Depth + 1))
5873 return false;
5874 return true;
5875
5876 case ISD::SPLAT_VECTOR:
5877 return isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 0), Kind, Depth: Depth + 1);
5878
5879 case ISD::SELECT: {
5880 return !canCreateUndefOrPoison(Op, DemandedElts, Kind,
5881 /*ConsiderFlags*/ true, Depth) &&
5882 isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 0), Kind,
5883 Depth: Depth + 1) &&
5884 isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 1), DemandedElts,
5885 Kind, Depth: Depth + 1) &&
5886 isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 2), DemandedElts,
5887 Kind, Depth: Depth + 1);
5888 }
5889
5890 case ISD::VECTOR_SHUFFLE: {
5891 APInt DemandedLHS, DemandedRHS;
5892 auto *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
5893 if (!getShuffleDemandedElts(SrcWidth: NumElts, Mask: SVN->getMask(), DemandedElts,
5894 DemandedLHS, DemandedRHS,
5895 /*AllowUndefElts=*/false))
5896 return false;
5897 if (!DemandedLHS.isZero() &&
5898 !isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 0), DemandedElts: DemandedLHS, Kind,
5899 Depth: Depth + 1))
5900 return false;
5901 if (!DemandedRHS.isZero() &&
5902 !isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 1), DemandedElts: DemandedRHS, Kind,
5903 Depth: Depth + 1))
5904 return false;
5905 return true;
5906 }
5907
5908 case ISD::SHL:
5909 case ISD::SRL:
5910 case ISD::SRA:
5911 // Shift amount operand is checked by canCreateUndefOrPoison. So it is
5912 // enough to check operand 0 if Op can't create undef/poison.
5913 return !canCreateUndefOrPoison(Op, DemandedElts, Kind,
5914 /*ConsiderFlags*/ true, Depth) &&
5915 isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 0), DemandedElts,
5916 Kind, Depth: Depth + 1);
5917
5918 case ISD::BSWAP:
5919 case ISD::CTPOP:
5920 case ISD::BITREVERSE:
5921 case ISD::AND:
5922 case ISD::OR:
5923 case ISD::XOR:
5924 case ISD::ADD:
5925 case ISD::SUB:
5926 case ISD::MUL:
5927 case ISD::SADDSAT:
5928 case ISD::UADDSAT:
5929 case ISD::SSUBSAT:
5930 case ISD::USUBSAT:
5931 case ISD::SSHLSAT:
5932 case ISD::USHLSAT:
5933 case ISD::SMIN:
5934 case ISD::SMAX:
5935 case ISD::UMIN:
5936 case ISD::UMAX:
5937 case ISD::ZERO_EXTEND:
5938 case ISD::SIGN_EXTEND:
5939 case ISD::ANY_EXTEND:
5940 case ISD::TRUNCATE:
5941 case ISD::VSELECT: {
5942 // If Op can't create undef/poison and none of its operands are undef/poison
5943 // then Op is never undef/poison. A difference from the more common check
5944 // below, outside the switch, is that we handle elementwise operations for
5945 // which the DemandedElts mask is valid for all operands here.
5946 return !canCreateUndefOrPoison(Op, DemandedElts, Kind,
5947 /*ConsiderFlags*/ true, Depth) &&
5948 all_of(Range: Op->ops(), P: [&](SDValue V) {
5949 return isGuaranteedNotToBeUndefOrPoison(Op: V, DemandedElts, Kind,
5950 Depth: Depth + 1);
5951 });
5952 }
5953
5954 case ISD::LOAD: {
5955 // If we're loading an unindexed constant pool, check if the constant
5956 // contains poison/undef - skip ANYEXT loads.
5957 // TODO: type size mismatch handling.
5958 auto *LD = cast<LoadSDNode>(Val&: Op);
5959 if (ISD::isUNINDEXEDLoad(N: LD) && !ISD::isEXTLoad(N: LD)) {
5960 if (const Constant *Cst = TLI->getTargetConstantFromLoad(LD)) {
5961 EVT VT = Op.getValueType();
5962 Type *CstTy = Cst->getType();
5963
5964 auto IsSafe = [Kind](const Constant *C) -> bool {
5965 // TODO: expose isGuaranteedNotToBeUndefOrPoison(UndefPoisonKind)
5966 if (includesPoison(Kind) && includesUndef(Kind))
5967 return ::isGuaranteedNotToBeUndefOrPoison(V: C);
5968 if (includesPoison(Kind))
5969 return ::isGuaranteedNotToBePoison(V: C);
5970 assert(includesUndef(Kind) && "UndefOnly expected");
5971 return ::isGuaranteedNotToBeUndef(V: C);
5972 };
5973
5974 // If Cst matches VT vector dimensions, then check per-element.
5975 if (CstTy->isVectorTy() && VT.isFixedLengthVector() &&
5976 VT.getSizeInBits() == CstTy->getPrimitiveSizeInBits() &&
5977 VT.getScalarSizeInBits() == CstTy->getScalarSizeInBits()) {
5978 for (unsigned I = 0; I != NumElts; ++I) {
5979 if (!DemandedElts[I])
5980 continue;
5981 if (Constant *Elt = Cst->getAggregateElement(Elt: I)) {
5982 if (IsSafe(Elt))
5983 continue;
5984 }
5985 return false;
5986 }
5987 } else {
5988 return IsSafe(Cst);
5989 }
5990 }
5991 }
5992 break;
5993 }
5994
5995 // TODO: Search for noundef attributes from library functions.
5996
5997 // TODO: Pointers dereferenced by ISD::LOAD/STORE ops are noundef.
5998
5999 default:
6000 // Allow the target to implement this method for its nodes.
6001 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
6002 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)
6003 return TLI->isGuaranteedNotToBeUndefOrPoisonForTargetNode(
6004 Op, DemandedElts, DAG: *this, Kind, Depth);
6005 break;
6006 }
6007
6008 // If Op can't create undef/poison and none of its operands are undef/poison
6009 // then Op is never undef/poison.
6010 // NOTE: TargetNodes can handle this in themselves in
6011 // isGuaranteedNotToBeUndefOrPoisonForTargetNode or let
6012 // TargetLowering::isGuaranteedNotToBeUndefOrPoisonForTargetNode handle it.
6013 return !canCreateUndefOrPoison(Op, Kind, /*ConsiderFlags*/ true, Depth) &&
6014 all_of(Range: Op->ops(), P: [&](SDValue V) {
6015 return isGuaranteedNotToBeUndefOrPoison(Op: V, Kind, Depth: Depth + 1);
6016 });
6017}
6018
6019bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, UndefPoisonKind Kind,
6020 bool ConsiderFlags,
6021 unsigned Depth) const {
6022 APInt DemandedElts = getDemandAllEltsMask(V: Op);
6023 return canCreateUndefOrPoison(Op, DemandedElts, Kind, ConsiderFlags, Depth);
6024}
6025
6026bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, const APInt &DemandedElts,
6027 UndefPoisonKind Kind,
6028 bool ConsiderFlags,
6029 unsigned Depth) const {
6030 if (ConsiderFlags && includesPoison(Kind) && Op->hasPoisonGeneratingFlags())
6031 return true;
6032
6033 unsigned Opcode = Op.getOpcode();
6034 switch (Opcode) {
6035 case ISD::AssertSext:
6036 case ISD::AssertZext:
6037 case ISD::AssertAlign:
6038 case ISD::AssertNoFPClass:
6039 // Assertion nodes can create poison if the assertion fails.
6040 return includesPoison(Kind);
6041
6042 case ISD::FREEZE:
6043 case ISD::CONCAT_VECTORS:
6044 case ISD::INSERT_SUBVECTOR:
6045 case ISD::EXTRACT_SUBVECTOR:
6046 case ISD::SADDSAT:
6047 case ISD::UADDSAT:
6048 case ISD::SSUBSAT:
6049 case ISD::USUBSAT:
6050 case ISD::MULHU:
6051 case ISD::MULHS:
6052 case ISD::AVGFLOORS:
6053 case ISD::AVGFLOORU:
6054 case ISD::AVGCEILS:
6055 case ISD::AVGCEILU:
6056 case ISD::AVGSUBFLOORS:
6057 case ISD::AVGSUBFLOORU:
6058 case ISD::ABDU:
6059 case ISD::ABDS:
6060 case ISD::SMIN:
6061 case ISD::SMAX:
6062 case ISD::SCMP:
6063 case ISD::UMIN:
6064 case ISD::UMAX:
6065 case ISD::UCMP:
6066 case ISD::AND:
6067 case ISD::XOR:
6068 case ISD::ROTL:
6069 case ISD::ROTR:
6070 case ISD::FSHL:
6071 case ISD::FSHR:
6072 case ISD::BSWAP:
6073 case ISD::CTTZ:
6074 case ISD::CTLZ:
6075 case ISD::CTLS:
6076 case ISD::CTPOP:
6077 case ISD::BITREVERSE:
6078 case ISD::PARITY:
6079 case ISD::SIGN_EXTEND:
6080 case ISD::TRUNCATE:
6081 case ISD::SIGN_EXTEND_INREG:
6082 case ISD::SIGN_EXTEND_VECTOR_INREG:
6083 case ISD::ZERO_EXTEND_VECTOR_INREG:
6084 case ISD::BITCAST:
6085 case ISD::BUILD_VECTOR:
6086 case ISD::BUILD_PAIR:
6087 case ISD::SPLAT_VECTOR:
6088 case ISD::FABS:
6089 case ISD::FCEIL:
6090 case ISD::FFLOOR:
6091 case ISD::FTRUNC:
6092 case ISD::FRINT:
6093 case ISD::FNEARBYINT:
6094 case ISD::FROUND:
6095 case ISD::FROUNDEVEN:
6096 return false;
6097
6098 case ISD::ABS:
6099 // ISD::ABS defines abs(INT_MIN) -> INT_MIN and never generates poison.
6100 // Different to Intrinsic::abs.
6101 return false;
6102 case ISD::ABS_MIN_POISON:
6103 // ABS_MIN_POISON may produce poison if the input is INT_MIN.
6104 return ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1) <= 1;
6105
6106 case ISD::ADDC:
6107 case ISD::SUBC:
6108 case ISD::ADDE:
6109 case ISD::SUBE:
6110 case ISD::SADDO:
6111 case ISD::SSUBO:
6112 case ISD::SMULO:
6113 case ISD::SADDO_CARRY:
6114 case ISD::SSUBO_CARRY:
6115 case ISD::UADDO:
6116 case ISD::USUBO:
6117 case ISD::UMULO:
6118 case ISD::UADDO_CARRY:
6119 case ISD::USUBO_CARRY:
6120 // No poison on result or overflow flags.
6121 return false;
6122
6123 case ISD::SELECT_CC:
6124 case ISD::SETCC: {
6125 // Integer setcc cannot create undef or poison.
6126 if (Op.getOperand(i: 0).getValueType().isInteger())
6127 return false;
6128
6129 // FP compares are more complicated. They can create poison for nan/infinity
6130 // based on options and flags. The options and flags also cause special
6131 // nonan condition codes to be used. Those condition codes may be preserved
6132 // even if the nonan flag is dropped somewhere.
6133 unsigned CCOp = Opcode == ISD::SETCC ? 2 : 4;
6134 ISD::CondCode CCCode = Op.getOperandCondCode(i: CCOp);
6135 return (unsigned)CCCode & 0x10U;
6136 }
6137
6138 case ISD::OR:
6139 case ISD::ZERO_EXTEND:
6140 case ISD::SELECT:
6141 case ISD::VSELECT:
6142 case ISD::ADD:
6143 case ISD::SUB:
6144 case ISD::MUL:
6145 case ISD::FNEG:
6146 case ISD::FADD:
6147 case ISD::FSUB:
6148 case ISD::FMUL:
6149 case ISD::FDIV:
6150 case ISD::FREM:
6151 case ISD::FCOPYSIGN:
6152 case ISD::FMA:
6153 case ISD::FMAD:
6154 case ISD::FMULADD:
6155 case ISD::FP_EXTEND:
6156 case ISD::FMINNUM:
6157 case ISD::FMAXNUM:
6158 case ISD::FMINNUM_IEEE:
6159 case ISD::FMAXNUM_IEEE:
6160 case ISD::FMINIMUM:
6161 case ISD::FMAXIMUM:
6162 case ISD::FMINIMUMNUM:
6163 case ISD::FMAXIMUMNUM:
6164 case ISD::FP_TO_SINT_SAT:
6165 case ISD::FP_TO_UINT_SAT:
6166 case ISD::TRUNCATE_SSAT_S:
6167 case ISD::TRUNCATE_SSAT_U:
6168 case ISD::TRUNCATE_USAT_U:
6169 // No poison except from flags (which is handled above)
6170 return false;
6171
6172 case ISD::SHL:
6173 case ISD::SRL:
6174 case ISD::SRA:
6175 // If the max shift amount isn't in range, then the shift can
6176 // create poison.
6177 return includesPoison(Kind) &&
6178 !getValidMaximumShiftAmount(V: Op, DemandedElts, Depth: Depth + 1);
6179
6180 case ISD::CTTZ_ZERO_POISON:
6181 case ISD::CTLZ_ZERO_POISON:
6182 // If the amount is zero then the result will be poison.
6183 // TODO: Add isKnownNeverZero DemandedElts handling.
6184 return includesPoison(Kind) &&
6185 !isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6186
6187 case ISD::SCALAR_TO_VECTOR:
6188 // Check if we demand any upper (poison) elements.
6189 return includesPoison(Kind) && DemandedElts.ugt(RHS: 1);
6190
6191 case ISD::INSERT_VECTOR_ELT:
6192 case ISD::EXTRACT_VECTOR_ELT: {
6193 // Ensure that the element index is in bounds.
6194 if (includesPoison(Kind)) {
6195 EVT VecVT = Op.getOperand(i: 0).getValueType();
6196 SDValue Idx = Op.getOperand(i: Opcode == ISD::INSERT_VECTOR_ELT ? 2 : 1);
6197 KnownBits KnownIdx = computeKnownBits(Op: Idx, Depth: Depth + 1);
6198 return KnownIdx.getMaxValue().uge(RHS: VecVT.getVectorMinNumElements());
6199 }
6200 return false;
6201 }
6202
6203 case ISD::VECTOR_SHUFFLE: {
6204 // Check for any demanded shuffle element that is undef.
6205 auto *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
6206 for (auto [Idx, Elt] : enumerate(First: SVN->getMask()))
6207 if (Elt < 0 && DemandedElts[Idx])
6208 return true;
6209 return false;
6210 }
6211
6212 case ISD::VECTOR_COMPRESS:
6213 return false;
6214
6215 default:
6216 // Allow the target to implement this method for its nodes.
6217 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
6218 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)
6219 return TLI->canCreateUndefOrPoisonForTargetNode(
6220 Op, DemandedElts, DAG: *this, Kind, ConsiderFlags, Depth);
6221 break;
6222 }
6223
6224 // Be conservative and return true.
6225 return true;
6226}
6227
6228bool SelectionDAG::isADDLike(SDValue Op, bool NoWrap) const {
6229 unsigned Opcode = Op.getOpcode();
6230 if (Opcode == ISD::OR)
6231 return Op->getFlags().hasDisjoint() ||
6232 haveNoCommonBitsSet(A: Op.getOperand(i: 0), B: Op.getOperand(i: 1));
6233 if (Opcode == ISD::XOR)
6234 return !NoWrap && isMinSignedConstant(V: Op.getOperand(i: 1));
6235 return false;
6236}
6237
6238bool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const {
6239 return Op.getNumOperands() == 2 && isa<ConstantSDNode>(Val: Op.getOperand(i: 1)) &&
6240 (Op.isAnyAdd() || isADDLike(Op));
6241}
6242
6243KnownFPClass SelectionDAG::computeKnownFPClass(SDValue Op,
6244 FPClassTest InterestedClasses,
6245 unsigned Depth) const {
6246 APInt DemandedElts = getDemandAllEltsMask(V: Op);
6247 return computeKnownFPClass(Op, DemandedElts, InterestedClasses, Depth);
6248}
6249
6250KnownFPClass SelectionDAG::computeKnownFPClass(SDValue Op,
6251 const APInt &DemandedElts,
6252 FPClassTest InterestedClasses,
6253 unsigned Depth) const {
6254 KnownFPClass Known;
6255
6256 if (const auto *CFP = dyn_cast<ConstantFPSDNode>(Val&: Op))
6257 return KnownFPClass(CFP->getValueAPF());
6258
6259 if (Depth >= MaxRecursionDepth)
6260 return Known;
6261
6262 if (Op.getOpcode() == ISD::UNDEF)
6263 return Known;
6264
6265 EVT VT = Op.getValueType();
6266 assert(VT.isFloatingPoint() && "Computing KnownFPClass on non-FP op!");
6267 assert((!VT.isFixedLengthVector() ||
6268 DemandedElts.getBitWidth() == VT.getVectorNumElements()) &&
6269 "Unexpected vector size");
6270
6271 if (!DemandedElts)
6272 return Known;
6273
6274 unsigned Opcode = Op.getOpcode();
6275 switch (Opcode) {
6276 case ISD::POISON: {
6277 Known = KnownFPClass::getEmpty();
6278 break;
6279 }
6280 case ISD::FNEG: {
6281 Known = computeKnownFPClass(Op: Op.getOperand(i: 0), DemandedElts,
6282 InterestedClasses, Depth: Depth + 1);
6283 Known.fneg();
6284 break;
6285 }
6286 case ISD::BUILD_VECTOR: {
6287 assert(!VT.isScalableVector());
6288 bool First = true;
6289 for (unsigned I = 0, E = Op.getNumOperands(); I != E; ++I) {
6290 if (!DemandedElts[I])
6291 continue;
6292
6293 if (First) {
6294 Known =
6295 computeKnownFPClass(Op: Op.getOperand(i: I), InterestedClasses, Depth: Depth + 1);
6296 First = false;
6297 } else {
6298 Known |=
6299 computeKnownFPClass(Op: Op.getOperand(i: I), InterestedClasses, Depth: Depth + 1);
6300 }
6301
6302 if (Known.isUnknown())
6303 break;
6304 }
6305 break;
6306 }
6307 case ISD::EXTRACT_VECTOR_ELT: {
6308 SDValue Src = Op.getOperand(i: 0);
6309 auto *CIdx = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1));
6310 EVT SrcVT = Src.getValueType();
6311 if (SrcVT.isFixedLengthVector() && CIdx) {
6312 if (CIdx->getAPIntValue().ult(RHS: SrcVT.getVectorNumElements())) {
6313 APInt DemandedSrcElts = APInt::getOneBitSet(
6314 numBits: SrcVT.getVectorNumElements(), BitNo: CIdx->getZExtValue());
6315 Known = computeKnownFPClass(Op: Src, DemandedElts: DemandedSrcElts, InterestedClasses,
6316 Depth: Depth + 1);
6317 } else {
6318 // Out of bounds index is poison.
6319 Known = KnownFPClass::getEmpty();
6320 }
6321 } else {
6322 Known = computeKnownFPClass(Op: Src, InterestedClasses, Depth: Depth + 1);
6323 }
6324 break;
6325 }
6326 case ISD::SPLAT_VECTOR: {
6327 Known = computeKnownFPClass(Op: Op.getOperand(i: 0), InterestedClasses, Depth: Depth + 1);
6328 break;
6329 }
6330 case ISD::BITCAST: {
6331 // FIXME: It should not be necessary to check for an elementwise bitcast.
6332 // If a bitcast is not elementwise between vector / scalar types,
6333 // computeKnownBits already splices the known bits of the source elements
6334 // appropriately so as to line up with the bits of the result's demanded
6335 // elements.
6336 EVT SrcVT = Op.getOperand(i: 0).getValueType();
6337 if (VT.isScalableVector() || SrcVT.isScalableVector())
6338 break;
6339 unsigned VTNumElts = VT.isVector() ? VT.getVectorNumElements() : 1;
6340 unsigned SrcVTNumElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
6341 if (VTNumElts != SrcVTNumElts)
6342 break;
6343
6344 KnownBits Bits = computeKnownBits(Op, DemandedElts, Depth: Depth + 1);
6345 Known = KnownFPClass::bitcast(FltSemantics: VT.getFltSemantics(), Bits);
6346 break;
6347 }
6348 case ISD::FABS: {
6349 Known = computeKnownFPClass(Op: Op.getOperand(i: 0), DemandedElts,
6350 InterestedClasses, Depth: Depth + 1);
6351 Known.fabs();
6352 break;
6353 }
6354 case ISD::FCOPYSIGN: {
6355 Known = computeKnownFPClass(Op: Op.getOperand(i: 0), DemandedElts,
6356 InterestedClasses, Depth: Depth + 1);
6357 KnownFPClass KnownSign = computeKnownFPClass(Op: Op.getOperand(i: 1), DemandedElts,
6358 InterestedClasses, Depth: Depth + 1);
6359 Known.copysign(Sign: KnownSign);
6360 break;
6361 }
6362 case ISD::AssertNoFPClass: {
6363 Known = computeKnownFPClass(Op: Op.getOperand(i: 0), DemandedElts,
6364 InterestedClasses, Depth: Depth + 1);
6365 FPClassTest AssertedClasses =
6366 static_cast<FPClassTest>(Op->getConstantOperandVal(Num: 1));
6367 Known.setKnownFPClasses(Known.getKnownFPClasses() & ~AssertedClasses);
6368 break;
6369 }
6370 case ISD::EXTRACT_SUBVECTOR: {
6371 SDValue Src = Op.getOperand(i: 0);
6372 EVT SrcVT = Src.getValueType();
6373 if (SrcVT.isFixedLengthVector()) {
6374 unsigned Idx = Op.getConstantOperandVal(i: 1);
6375 unsigned NumSrcElts = SrcVT.getVectorNumElements();
6376
6377 APInt DemandedSrcElts = DemandedElts.zextOrTrunc(width: NumSrcElts).shl(shiftAmt: Idx);
6378 Known = computeKnownFPClass(Op: Src, DemandedElts: DemandedSrcElts, InterestedClasses,
6379 Depth: Depth + 1);
6380 } else {
6381 Known = computeKnownFPClass(Op: Src, InterestedClasses, Depth: Depth + 1);
6382 }
6383 break;
6384 }
6385 case ISD::INSERT_SUBVECTOR: {
6386 SDValue BaseVector = Op.getOperand(i: 0);
6387 SDValue SubVector = Op.getOperand(i: 1);
6388 EVT BaseVT = BaseVector.getValueType();
6389 if (BaseVT.isFixedLengthVector()) {
6390 unsigned Idx = Op.getConstantOperandVal(i: 2);
6391 unsigned NumBaseElts = BaseVT.getVectorNumElements();
6392 unsigned NumSubElts = SubVector.getValueType().getVectorNumElements();
6393
6394 APInt DemandedMask =
6395 APInt::getBitsSet(numBits: NumBaseElts, loBit: Idx, hiBit: Idx + NumSubElts);
6396 APInt DemandedSrcElts = DemandedElts & ~DemandedMask;
6397 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
6398
6399 if (!DemandedSrcElts.isZero())
6400 Known = computeKnownFPClass(Op: BaseVector, DemandedElts: DemandedSrcElts,
6401 InterestedClasses, Depth: Depth + 1);
6402 if (!DemandedSubElts.isZero()) {
6403 KnownFPClass SubKnown = computeKnownFPClass(
6404 Op: SubVector, DemandedElts: DemandedSubElts, InterestedClasses, Depth: Depth + 1);
6405 Known = DemandedSrcElts.isZero() ? SubKnown : (Known | SubKnown);
6406 }
6407 } else {
6408 Known = computeKnownFPClass(Op: SubVector, InterestedClasses, Depth: Depth + 1);
6409 if (!Known.isUnknown())
6410 Known |= computeKnownFPClass(Op: BaseVector, InterestedClasses, Depth: Depth + 1);
6411 }
6412 break;
6413 }
6414 case ISD::SELECT:
6415 case ISD::VSELECT: {
6416 // TODO: Add adjustKnownFPClassForSelectArm clamp recognition as in
6417 // IR-level ValueTracking.
6418 KnownFPClass KnownFalseClass = computeKnownFPClass(
6419 Op: Op.getOperand(i: 2), DemandedElts, InterestedClasses, Depth: Depth + 1);
6420 if (KnownFalseClass.isUnknown())
6421 break;
6422 KnownFPClass KnownTrueClass = computeKnownFPClass(
6423 Op: Op.getOperand(i: 1), DemandedElts, InterestedClasses, Depth: Depth + 1);
6424 Known = KnownTrueClass.intersectWith(RHS: KnownFalseClass);
6425 break;
6426 }
6427 default:
6428 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
6429 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID) {
6430 TLI->computeKnownFPClassForTargetNode(Op, Known, DemandedElts, DAG: *this,
6431 Depth);
6432 }
6433 break;
6434 }
6435
6436 return Known;
6437}
6438
6439bool SelectionDAG::isKnownNeverNaN(SDValue Op, bool SNaN,
6440 unsigned Depth) const {
6441 APInt DemandedElts = getDemandAllEltsMask(V: Op);
6442 return isKnownNeverNaN(Op, DemandedElts, SNaN, Depth);
6443}
6444
6445bool SelectionDAG::isKnownNeverNaN(SDValue Op, const APInt &DemandedElts,
6446 bool SNaN, unsigned Depth) const {
6447 assert(!DemandedElts.isZero() && "No demanded elements");
6448
6449 // If we're told that NaNs won't happen, assume they won't.
6450 if (Op->getFlags().hasNoNaNs())
6451 return true;
6452
6453 if (Depth >= MaxRecursionDepth)
6454 return false; // Limit search depth.
6455
6456 unsigned Opcode = Op.getOpcode();
6457 switch (Opcode) {
6458 case ISD::FADD:
6459 case ISD::FSUB:
6460 case ISD::FMUL:
6461 case ISD::FDIV:
6462 case ISD::FREM:
6463 case ISD::FSIN:
6464 case ISD::FCOS:
6465 case ISD::FTAN:
6466 case ISD::FASIN:
6467 case ISD::FACOS:
6468 case ISD::FATAN:
6469 case ISD::FATAN2:
6470 case ISD::FSINH:
6471 case ISD::FCOSH:
6472 case ISD::FTANH:
6473 case ISD::FMA:
6474 case ISD::FMULADD:
6475 case ISD::FMAD: {
6476 if (SNaN)
6477 return true;
6478 // TODO: Need isKnownNeverInfinity
6479 return false;
6480 }
6481 case ISD::FCANONICALIZE:
6482 case ISD::FEXP:
6483 case ISD::FEXP2:
6484 case ISD::FEXP10:
6485 case ISD::FTRUNC:
6486 case ISD::FFLOOR:
6487 case ISD::FCEIL:
6488 case ISD::FROUND:
6489 case ISD::FROUNDEVEN:
6490 case ISD::LROUND:
6491 case ISD::LLROUND:
6492 case ISD::FRINT:
6493 case ISD::LRINT:
6494 case ISD::LLRINT:
6495 case ISD::FNEARBYINT:
6496 case ISD::FLDEXP: {
6497 if (SNaN)
6498 return true;
6499 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1);
6500 }
6501 case ISD::FABS:
6502 case ISD::FNEG:
6503 case ISD::FCOPYSIGN: {
6504 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1);
6505 }
6506 case ISD::SELECT:
6507 return isKnownNeverNaN(Op: Op.getOperand(i: 1), DemandedElts, SNaN, Depth: Depth + 1) &&
6508 isKnownNeverNaN(Op: Op.getOperand(i: 2), DemandedElts, SNaN, Depth: Depth + 1);
6509 case ISD::FP_EXTEND:
6510 case ISD::FP_ROUND: {
6511 if (SNaN)
6512 return true;
6513 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1);
6514 }
6515 case ISD::SINT_TO_FP:
6516 case ISD::UINT_TO_FP:
6517 return true;
6518 case ISD::FSQRT: // Need is known positive
6519 case ISD::FLOG:
6520 case ISD::FLOG2:
6521 case ISD::FLOG10:
6522 case ISD::FPOWI:
6523 case ISD::FPOW: {
6524 if (SNaN)
6525 return true;
6526 // TODO: Refine on operand
6527 return false;
6528 }
6529 case ISD::FMINNUM:
6530 case ISD::FMAXNUM:
6531 case ISD::FMINIMUMNUM:
6532 case ISD::FMAXIMUMNUM: {
6533 // Only one needs to be known not-nan, since it will be returned if the
6534 // other ends up being one.
6535 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1) ||
6536 isKnownNeverNaN(Op: Op.getOperand(i: 1), DemandedElts, SNaN, Depth: Depth + 1);
6537 }
6538 case ISD::FMINNUM_IEEE:
6539 case ISD::FMAXNUM_IEEE: {
6540 if (SNaN)
6541 return true;
6542 // This can return a NaN if either operand is an sNaN, or if both operands
6543 // are NaN.
6544 return (isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN: false, Depth: Depth + 1) &&
6545 isKnownNeverSNaN(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1)) ||
6546 (isKnownNeverNaN(Op: Op.getOperand(i: 1), DemandedElts, SNaN: false, Depth: Depth + 1) &&
6547 isKnownNeverSNaN(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1));
6548 }
6549 case ISD::FMINIMUM:
6550 case ISD::FMAXIMUM: {
6551 // TODO: Does this quiet or return the origina NaN as-is?
6552 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1) &&
6553 isKnownNeverNaN(Op: Op.getOperand(i: 1), DemandedElts, SNaN, Depth: Depth + 1);
6554 }
6555 case ISD::EXTRACT_VECTOR_ELT: {
6556 SDValue Src = Op.getOperand(i: 0);
6557 auto *Idx = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1));
6558 EVT SrcVT = Src.getValueType();
6559 if (SrcVT.isFixedLengthVector() && Idx &&
6560 Idx->getAPIntValue().ult(RHS: SrcVT.getVectorNumElements())) {
6561 APInt DemandedSrcElts = APInt::getOneBitSet(numBits: SrcVT.getVectorNumElements(),
6562 BitNo: Idx->getZExtValue());
6563 return isKnownNeverNaN(Op: Src, DemandedElts: DemandedSrcElts, SNaN, Depth: Depth + 1);
6564 }
6565 return isKnownNeverNaN(Op: Src, SNaN, Depth: Depth + 1);
6566 }
6567 case ISD::EXTRACT_SUBVECTOR: {
6568 SDValue Src = Op.getOperand(i: 0);
6569 if (Src.getValueType().isFixedLengthVector()) {
6570 unsigned Idx = Op.getConstantOperandVal(i: 1);
6571 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
6572 APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
6573 return isKnownNeverNaN(Op: Src, DemandedElts: DemandedSrcElts, SNaN, Depth: Depth + 1);
6574 }
6575 return isKnownNeverNaN(Op: Src, SNaN, Depth: Depth + 1);
6576 }
6577 case ISD::INSERT_SUBVECTOR: {
6578 SDValue BaseVector = Op.getOperand(i: 0);
6579 SDValue SubVector = Op.getOperand(i: 1);
6580 EVT BaseVectorVT = BaseVector.getValueType();
6581 if (BaseVectorVT.isFixedLengthVector()) {
6582 unsigned Idx = Op.getConstantOperandVal(i: 2);
6583 unsigned NumBaseElts = BaseVectorVT.getVectorNumElements();
6584 unsigned NumSubElts = SubVector.getValueType().getVectorNumElements();
6585
6586 // Clear/Extract the bits at the position where the subvector will be
6587 // inserted.
6588 APInt DemandedMask =
6589 APInt::getBitsSet(numBits: NumBaseElts, loBit: Idx, hiBit: Idx + NumSubElts);
6590 APInt DemandedSrcElts = DemandedElts & ~DemandedMask;
6591 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
6592
6593 bool NeverNaN = true;
6594 if (!DemandedSrcElts.isZero())
6595 NeverNaN &=
6596 isKnownNeverNaN(Op: BaseVector, DemandedElts: DemandedSrcElts, SNaN, Depth: Depth + 1);
6597 if (NeverNaN && !DemandedSubElts.isZero())
6598 NeverNaN &=
6599 isKnownNeverNaN(Op: SubVector, DemandedElts: DemandedSubElts, SNaN, Depth: Depth + 1);
6600 return NeverNaN;
6601 }
6602 return isKnownNeverNaN(Op: BaseVector, SNaN, Depth: Depth + 1) &&
6603 isKnownNeverNaN(Op: SubVector, SNaN, Depth: Depth + 1);
6604 }
6605 case ISD::BUILD_VECTOR: {
6606 unsigned NumElts = Op.getNumOperands();
6607 for (unsigned I = 0; I != NumElts; ++I)
6608 if (DemandedElts[I] &&
6609 !isKnownNeverNaN(Op: Op.getOperand(i: I), SNaN, Depth: Depth + 1))
6610 return false;
6611 return true;
6612 }
6613 case ISD::SPLAT_VECTOR:
6614 return isKnownNeverNaN(Op: Op.getOperand(i: 0), SNaN, Depth: Depth + 1);
6615 case ISD::AssertNoFPClass: {
6616 FPClassTest NoFPClass =
6617 static_cast<FPClassTest>(Op.getConstantOperandVal(i: 1));
6618 if ((NoFPClass & fcNan) == fcNan)
6619 return true;
6620 if (SNaN && (NoFPClass & fcSNan) == fcSNan)
6621 return true;
6622 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1);
6623 }
6624 default:
6625 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
6626 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID) {
6627 return TLI->isKnownNeverNaNForTargetNode(Op, DemandedElts, DAG: *this, SNaN,
6628 Depth);
6629 }
6630 break;
6631 }
6632
6633 FPClassTest NanMask = SNaN ? fcSNan : fcNan;
6634 KnownFPClass Known = computeKnownFPClass(Op, DemandedElts, InterestedClasses: NanMask, Depth);
6635 return Known.isKnownNever(Mask: NanMask);
6636}
6637
6638bool SelectionDAG::isKnownNeverLogicalZero(SDValue Op, unsigned Depth) const {
6639 APInt DemandedElts = getDemandAllEltsMask(V: Op);
6640 return isKnownNeverLogicalZero(Op, DemandedElts, Depth);
6641}
6642
6643bool SelectionDAG::isKnownNeverLogicalZero(SDValue Op,
6644 const APInt &DemandedElts,
6645 unsigned Depth) const {
6646 assert(!DemandedElts.isZero() && "No demanded elements");
6647 EVT VT = Op.getValueType();
6648 KnownFPClass Known =
6649 computeKnownFPClass(Op, DemandedElts, InterestedClasses: fcZero | fcSubnormal, Depth);
6650 return Known.isKnownNeverLogicalZero(Mode: getDenormalMode(VT));
6651}
6652
6653bool SelectionDAG::isKnownNeverZero(SDValue Op, unsigned Depth) const {
6654 APInt DemandedElts = getDemandAllEltsMask(V: Op);
6655 return isKnownNeverZero(Op, DemandedElts, Depth);
6656}
6657
6658bool SelectionDAG::isKnownNeverZero(SDValue Op, const APInt &DemandedElts,
6659 unsigned Depth) const {
6660 if (Depth >= MaxRecursionDepth)
6661 return false; // Limit search depth.
6662
6663 EVT OpVT = Op.getValueType();
6664 unsigned BitWidth = OpVT.getScalarSizeInBits();
6665
6666 assert(!Op.getValueType().isFloatingPoint() &&
6667 "Floating point types unsupported - use isKnownNeverLogicalZero");
6668
6669 // If the value is a constant, we can obviously see if it is a zero or not.
6670 auto IsNeverZero = [BitWidth](const ConstantSDNode *C) {
6671 APInt V = C->getAPIntValue().zextOrTrunc(width: BitWidth);
6672 return !V.isZero();
6673 };
6674
6675 if (ISD::matchUnaryPredicate(Op, DemandedElts, Match: IsNeverZero,
6676 /*AllowUndefs=*/false, /*AllowTruncation=*/true))
6677 return true;
6678
6679 // TODO: Recognize more cases here. Most of the cases are also incomplete to
6680 // some degree.
6681 switch (Op.getOpcode()) {
6682 default:
6683 break;
6684
6685 case ISD::EXTRACT_VECTOR_ELT: {
6686 SDValue InVec = Op.getOperand(i: 0);
6687 SDValue EltNo = Op.getOperand(i: 1);
6688 EVT VecVT = InVec.getValueType();
6689
6690 // Skip scalable vectors or implicit extensions.
6691 if (VecVT.isScalableVector() ||
6692 OpVT.getScalarSizeInBits() != VecVT.getScalarSizeInBits())
6693 break;
6694
6695 // If we know the element index, just demand that vector element, else for
6696 // an unknown element index, ignore DemandedElts and demand them all.
6697 const unsigned NumSrcElts = VecVT.getVectorNumElements();
6698 APInt DemandedSrcElts = APInt::getAllOnes(numBits: NumSrcElts);
6699 auto *ConstEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
6700 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(RHS: NumSrcElts))
6701 DemandedSrcElts =
6702 APInt::getOneBitSet(numBits: NumSrcElts, BitNo: ConstEltNo->getZExtValue());
6703
6704 return isKnownNeverZero(Op: InVec, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
6705 }
6706
6707 case ISD::OR:
6708 return isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1) ||
6709 isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6710
6711 case ISD::VSELECT:
6712 case ISD::SELECT:
6713 return isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1) &&
6714 isKnownNeverZero(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth + 1);
6715
6716 case ISD::SHL: {
6717 if (Op->getFlags().hasNoSignedWrap() || Op->getFlags().hasNoUnsignedWrap())
6718 return isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6719 KnownBits ValKnown =
6720 computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6721 // 1 << X is never zero.
6722 if (ValKnown.One[0])
6723 return true;
6724 // If max shift cnt of known ones is non-zero, result is non-zero.
6725 APInt MaxCnt = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1)
6726 .getMaxValue();
6727 if (MaxCnt.ult(RHS: ValKnown.getBitWidth()) &&
6728 !ValKnown.One.shl(ShiftAmt: MaxCnt).isZero())
6729 return true;
6730 break;
6731 }
6732
6733 case ISD::VECTOR_SHUFFLE: {
6734 if (Op.getValueType().isScalableVector())
6735 return false;
6736
6737 unsigned NumElts = DemandedElts.getBitWidth();
6738
6739 // All demanded elements from LHS and RHS must be known non-zero.
6740 // Demanded elements with undef shuffle mask elements are unknown.
6741
6742 APInt DemandedLHS, DemandedRHS;
6743 auto *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
6744 assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
6745 if (!getShuffleDemandedElts(SrcWidth: NumElts, Mask: SVN->getMask(), DemandedElts,
6746 DemandedLHS, DemandedRHS))
6747 return false;
6748
6749 return (!DemandedLHS ||
6750 isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts: DemandedLHS, Depth: Depth + 1)) &&
6751 (!DemandedRHS ||
6752 isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts: DemandedRHS, Depth: Depth + 1));
6753 }
6754
6755 case ISD::UADDSAT:
6756 case ISD::UMAX:
6757 return isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1) ||
6758 isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6759
6760 case ISD::UMIN:
6761 return isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1) &&
6762 isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6763
6764 // For smin/smax: If either operand is known negative/positive
6765 // respectively we don't need the other to be known at all.
6766 case ISD::SMAX: {
6767 KnownBits Op1 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
6768 if (Op1.isStrictlyPositive())
6769 return true;
6770
6771 KnownBits Op0 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6772 if (Op0.isStrictlyPositive())
6773 return true;
6774
6775 if (Op1.isNonZero() && Op0.isNonZero())
6776 return true;
6777
6778 return isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1) &&
6779 isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6780 }
6781 case ISD::SMIN: {
6782 KnownBits Op1 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
6783 if (Op1.isNegative())
6784 return true;
6785
6786 KnownBits Op0 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6787 if (Op0.isNegative())
6788 return true;
6789
6790 if (Op1.isNonZero() && Op0.isNonZero())
6791 return true;
6792
6793 return isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1) &&
6794 isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6795 }
6796
6797 case ISD::ROTL:
6798 case ISD::ROTR:
6799 case ISD::BITREVERSE:
6800 case ISD::BSWAP:
6801 case ISD::CTPOP:
6802 case ISD::ABS:
6803 case ISD::ABS_MIN_POISON:
6804 return isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6805
6806 case ISD::SRA:
6807 case ISD::SRL: {
6808 if (Op->getFlags().hasExact())
6809 return isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6810 KnownBits ValKnown =
6811 computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6812 if (ValKnown.isNegative())
6813 return true;
6814 // If max shift cnt of known ones is non-zero, result is non-zero.
6815 APInt MaxCnt = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1)
6816 .getMaxValue();
6817 if (MaxCnt.ult(RHS: ValKnown.getBitWidth()) &&
6818 !ValKnown.One.lshr(ShiftAmt: MaxCnt).isZero())
6819 return true;
6820 break;
6821 }
6822 case ISD::UDIV:
6823 case ISD::SDIV:
6824 // div exact can only produce a zero if the dividend is zero.
6825 // TODO: For udiv this is also true if Op1 u<= Op0
6826 if (Op->getFlags().hasExact())
6827 return isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6828 break;
6829
6830 case ISD::ADD:
6831 if (Op->getFlags().hasNoUnsignedWrap())
6832 if (isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1) ||
6833 isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1))
6834 return true;
6835 // TODO: There are a lot more cases we can prove for add.
6836 break;
6837
6838 case ISD::SUB: {
6839 if (isNullConstant(V: Op.getOperand(i: 0)))
6840 return isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
6841
6842 std::optional<bool> ne = KnownBits::ne(
6843 LHS: computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1),
6844 RHS: computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1));
6845 return ne && *ne;
6846 }
6847
6848 case ISD::MUL:
6849 if (Op->getFlags().hasNoSignedWrap() || Op->getFlags().hasNoUnsignedWrap())
6850 if (isKnownNeverZero(Op: Op.getOperand(i: 1), Depth: Depth + 1) &&
6851 isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1))
6852 return true;
6853 break;
6854
6855 case ISD::ZERO_EXTEND:
6856 case ISD::SIGN_EXTEND:
6857 return isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6858 case ISD::VSCALE: {
6859 const Function &F = getMachineFunction().getFunction();
6860 const APInt &Multiplier = Op.getConstantOperandAPInt(i: 0);
6861 ConstantRange CR =
6862 getVScaleRange(F: &F, BitWidth: Op.getScalarValueSizeInBits()).multiply(Other: Multiplier);
6863 if (!CR.contains(Val: APInt(CR.getBitWidth(), 0)))
6864 return true;
6865 break;
6866 }
6867 }
6868
6869 return computeKnownBits(Op, DemandedElts, Depth).isNonZero();
6870}
6871
6872bool SelectionDAG::cannotBeOrderedNegativeFP(SDValue Op) const {
6873 if (ConstantFPSDNode *C1 = isConstOrConstSplatFP(N: Op, AllowUndefs: true))
6874 return !C1->isNegative();
6875
6876 switch (Op.getOpcode()) {
6877 case ISD::FABS:
6878 case ISD::FEXP:
6879 case ISD::FEXP2:
6880 case ISD::FEXP10:
6881 return true;
6882 default:
6883 return false;
6884 }
6885
6886 llvm_unreachable("covered opcode switch");
6887}
6888
6889/// Return true if the standard nsz semantics may not apply for this opcode.
6890/// Opcode is assumed to take float inputs.
6891/// FIXME: Target min/max nodes (e.g. X86ISD::FMINC) aren't included.
6892static bool hasSpecialNszSemantics(unsigned Opcode) {
6893 switch (Opcode) {
6894 case ISD::FMINNUM:
6895 case ISD::FMAXNUM:
6896 case ISD::FMINNUM_IEEE:
6897 case ISD::FMAXNUM_IEEE:
6898 case ISD::FMINIMUM:
6899 case ISD::FMAXIMUM:
6900 case ISD::FMINIMUMNUM:
6901 case ISD::FMAXIMUMNUM:
6902 case ISD::STRICT_FMINNUM:
6903 case ISD::STRICT_FMAXNUM:
6904 case ISD::STRICT_FMINIMUM:
6905 case ISD::STRICT_FMAXIMUM:
6906 case ISD::PSEUDO_FMIN:
6907 case ISD::PSEUDO_FMAX:
6908 case ISD::STRICT_PSEUDO_FMIN:
6909 case ISD::STRICT_PSEUDO_FMAX:
6910 case ISD::VECREDUCE_FMIN:
6911 case ISD::VECREDUCE_FMAX:
6912 case ISD::VECREDUCE_FMINIMUM:
6913 case ISD::VECREDUCE_FMAXIMUM:
6914 case ISD::VECREDUCE_FMINIMUMNUM:
6915 case ISD::VECREDUCE_FMAXIMUMNUM:
6916 case ISD::VP_REDUCE_FMIN:
6917 case ISD::VP_REDUCE_FMAX:
6918 case ISD::VP_REDUCE_FMINIMUM:
6919 case ISD::VP_REDUCE_FMAXIMUM:
6920 return true;
6921 default:
6922 return false;
6923 }
6924}
6925
6926bool SelectionDAG::canIgnoreSignBitOfZero(const SDUse &Use) const {
6927 assert(Use.getValueType().isFloatingPoint());
6928 const SDNode *User = Use.getUser();
6929 unsigned Opcode = User->getOpcode();
6930 if (User->getFlags().hasNoSignedZeros() && !hasSpecialNszSemantics(Opcode))
6931 return true;
6932
6933 unsigned OperandNo = Use.getOperandNo();
6934 // Check if this use is insensitive to the sign of zero
6935 switch (Opcode) {
6936 case ISD::SETCC:
6937 // Comparisons: IEEE-754 specifies +0.0 == -0.0.
6938 case ISD::FABS:
6939 // fabs always produces +0.0.
6940 return true;
6941 case ISD::FCOPYSIGN:
6942 // copysign overwrites the sign bit of the first operand.
6943 return OperandNo == 0;
6944 case ISD::FADD:
6945 case ISD::FSUB: {
6946 // Arithmetic with non-zero constants fixes the uncertainty around the
6947 // sign bit.
6948 SDValue Other = User->getOperand(Num: 1 - OperandNo);
6949 return isKnownNeverLogicalZero(Op: Other);
6950 }
6951 case ISD::FP_TO_SINT:
6952 case ISD::FP_TO_UINT:
6953 // fp-to-int conversions normalize signed zeros.
6954 return true;
6955 case ISD::FMINNUM:
6956 case ISD::FMINNUM_IEEE:
6957 case ISD::FMINIMUM:
6958 case ISD::FMINIMUMNUM:
6959 case ISD::FMAXNUM:
6960 case ISD::FMAXNUM_IEEE:
6961 case ISD::FMAXIMUM:
6962 case ISD::FMAXIMUMNUM:
6963 case ISD::PSEUDO_FMIN:
6964 case ISD::PSEUDO_FMAX: {
6965 // If other value always beats zero, the zero can be ignored.
6966 bool IsMin = Opcode == ISD::FMINNUM || Opcode == ISD::FMINNUM_IEEE ||
6967 Opcode == ISD::FMINIMUM || Opcode == ISD::FMINIMUMNUM ||
6968 Opcode == ISD::PSEUDO_FMIN;
6969 FPClassTest AlwaysBeatsZero =
6970 IsMin ? fcNegNormal | fcNegInf : fcPosNormal | fcPosInf;
6971 SDValue Other = User->getOperand(Num: 1 - OperandNo);
6972 KnownFPClass Known = computeKnownFPClass(Op: Other, InterestedClasses: ~AlwaysBeatsZero);
6973 return Known.isKnownNever(Mask: ~AlwaysBeatsZero);
6974 }
6975 default:
6976 return false;
6977 }
6978}
6979
6980bool SelectionDAG::canIgnoreSignBitOfZeroResult(SDValue Op) const {
6981 assert(Op->getNumValues() == 1);
6982 // FIXME: Limit the amount of checked uses to not introduce a compile-time
6983 // regression. Ideally, this should be implemented as a demanded-bits
6984 // optimization that stems from the users.
6985 if (Op->use_size() > 2)
6986 return false;
6987 return all_of(Range: Op->uses(),
6988 P: [&](const SDUse &Use) { return canIgnoreSignBitOfZero(Use); });
6989}
6990
6991bool SelectionDAG::isEqualTo(SDValue A, SDValue B) const {
6992 // Check the obvious case.
6993 if (A == B) return true;
6994
6995 // For negative and positive zero.
6996 if (const ConstantFPSDNode *CA = dyn_cast<ConstantFPSDNode>(Val&: A))
6997 if (const ConstantFPSDNode *CB = dyn_cast<ConstantFPSDNode>(Val&: B))
6998 if (CA->isZero() && CB->isZero()) return true;
6999
7000 // Otherwise they may not be equal.
7001 return false;
7002}
7003
7004// Only bits set in Mask must be negated, other bits may be arbitrary.
7005SDValue llvm::getBitwiseNotOperand(SDValue V, SDValue Mask, bool AllowUndefs) {
7006 if (isBitwiseNot(V, AllowUndefs))
7007 return V.getOperand(i: 0);
7008
7009 // Handle any_extend (not (truncate X)) pattern, where Mask only sets
7010 // bits in the non-extended part.
7011 ConstantSDNode *MaskC = isConstOrConstSplat(N: Mask);
7012 if (!MaskC || V.getOpcode() != ISD::ANY_EXTEND)
7013 return SDValue();
7014 SDValue ExtArg = V.getOperand(i: 0);
7015 if (ExtArg.getScalarValueSizeInBits() >=
7016 MaskC->getAPIntValue().getActiveBits() &&
7017 isBitwiseNot(V: ExtArg, AllowUndefs) &&
7018 ExtArg.getOperand(i: 0).getOpcode() == ISD::TRUNCATE &&
7019 ExtArg.getOperand(i: 0).getOperand(i: 0).getValueType() == V.getValueType())
7020 return ExtArg.getOperand(i: 0).getOperand(i: 0);
7021 return SDValue();
7022}
7023
7024static bool haveNoCommonBitsSetCommutative(SDValue A, SDValue B) {
7025 // Match masked merge pattern (X & ~M) op (Y & M)
7026 // Including degenerate case (X & ~M) op M
7027 auto MatchNoCommonBitsPattern = [&](SDValue Not, SDValue Mask,
7028 SDValue Other) {
7029 if (SDValue NotOperand =
7030 getBitwiseNotOperand(V: Not, Mask, /* AllowUndefs */ true)) {
7031 if (NotOperand->getOpcode() == ISD::ZERO_EXTEND ||
7032 NotOperand->getOpcode() == ISD::TRUNCATE)
7033 NotOperand = NotOperand->getOperand(Num: 0);
7034
7035 if (Other == NotOperand)
7036 return true;
7037 if (Other->getOpcode() == ISD::AND)
7038 return NotOperand == Other->getOperand(Num: 0) ||
7039 NotOperand == Other->getOperand(Num: 1);
7040 }
7041 return false;
7042 };
7043
7044 if (A->getOpcode() == ISD::ZERO_EXTEND || A->getOpcode() == ISD::TRUNCATE)
7045 A = A->getOperand(Num: 0);
7046
7047 if (B->getOpcode() == ISD::ZERO_EXTEND || B->getOpcode() == ISD::TRUNCATE)
7048 B = B->getOperand(Num: 0);
7049
7050 if (A->getOpcode() == ISD::AND)
7051 return MatchNoCommonBitsPattern(A->getOperand(Num: 0), A->getOperand(Num: 1), B) ||
7052 MatchNoCommonBitsPattern(A->getOperand(Num: 1), A->getOperand(Num: 0), B);
7053 return false;
7054}
7055
7056// FIXME: unify with llvm::haveNoCommonBitsSet.
7057bool SelectionDAG::haveNoCommonBitsSet(SDValue A, SDValue B) const {
7058 assert(A.getValueType() == B.getValueType() &&
7059 "Values must have the same type");
7060 if (haveNoCommonBitsSetCommutative(A, B) ||
7061 haveNoCommonBitsSetCommutative(A: B, B: A))
7062 return true;
7063 return KnownBits::haveNoCommonBitsSet(LHS: computeKnownBits(Op: A),
7064 RHS: computeKnownBits(Op: B));
7065}
7066
7067static SDValue FoldSTEP_VECTOR(const SDLoc &DL, EVT VT, SDValue Step,
7068 SelectionDAG &DAG) {
7069 if (cast<ConstantSDNode>(Val&: Step)->isZero())
7070 return DAG.getConstant(Val: 0, DL, VT);
7071
7072 return SDValue();
7073}
7074
7075static SDValue FoldBUILD_VECTOR(const SDLoc &DL, EVT VT,
7076 ArrayRef<SDValue> Ops,
7077 SelectionDAG &DAG) {
7078 int NumOps = Ops.size();
7079 assert(NumOps != 0 && "Can't build an empty vector!");
7080 assert(!VT.isScalableVector() &&
7081 "BUILD_VECTOR cannot be used with scalable types");
7082 assert(VT.getVectorNumElements() == (unsigned)NumOps &&
7083 "Incorrect element count in BUILD_VECTOR!");
7084
7085 // BUILD_VECTOR of UNDEFs is UNDEF.
7086 bool AllPoison = true;
7087 if (llvm::all_of(Range&: Ops, P: [&AllPoison](SDValue Op) {
7088 AllPoison &= Op.getOpcode() == ISD::POISON;
7089 return Op.isUndef();
7090 }))
7091 return AllPoison ? DAG.getPOISON(VT) : DAG.getUNDEF(VT);
7092
7093 // BUILD_VECTOR of seq extract/insert from the same vector + type is Identity.
7094 SDValue IdentitySrc;
7095 bool IsIdentity = true;
7096 for (int i = 0; i != NumOps; ++i) {
7097 if (Ops[i].getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
7098 Ops[i].getOperand(i: 0).getValueType() != VT ||
7099 (IdentitySrc && Ops[i].getOperand(i: 0) != IdentitySrc) ||
7100 !isa<ConstantSDNode>(Val: Ops[i].getOperand(i: 1)) ||
7101 Ops[i].getConstantOperandAPInt(i: 1) != i) {
7102 IsIdentity = false;
7103 break;
7104 }
7105 IdentitySrc = Ops[i].getOperand(i: 0);
7106 }
7107 if (IsIdentity)
7108 return IdentitySrc;
7109
7110 return SDValue();
7111}
7112
7113/// Try to simplify vector concatenation to an input value, undef, or build
7114/// vector.
7115static SDValue foldCONCAT_VECTORS(const SDLoc &DL, EVT VT,
7116 ArrayRef<SDValue> Ops,
7117 SelectionDAG &DAG) {
7118 assert(!Ops.empty() && "Can't concatenate an empty list of vectors!");
7119 assert(llvm::all_of(Ops,
7120 [Ops](SDValue Op) {
7121 return Ops[0].getValueType() == Op.getValueType();
7122 }) &&
7123 "Concatenation of vectors with inconsistent value types!");
7124 assert((Ops[0].getValueType().getVectorElementCount() * Ops.size()) ==
7125 VT.getVectorElementCount() &&
7126 "Incorrect element count in vector concatenation!");
7127
7128 if (Ops.size() == 1)
7129 return Ops[0];
7130
7131 // Concat of UNDEFs is UNDEF.
7132 bool AllPoison = true;
7133 if (llvm::all_of(Range&: Ops, P: [&AllPoison](SDValue Op) {
7134 AllPoison &= Op.getOpcode() == ISD::POISON;
7135 return Op.isUndef();
7136 }))
7137 return AllPoison ? DAG.getPOISON(VT) : DAG.getUNDEF(VT);
7138
7139 // Scan the operands and look for extract operations from a single source
7140 // that correspond to insertion at the same location via this concatenation:
7141 // concat (extract X, 0*subvec_elts), (extract X, 1*subvec_elts), ...
7142 SDValue IdentitySrc;
7143 bool IsIdentity = true;
7144 for (unsigned i = 0, e = Ops.size(); i != e; ++i) {
7145 SDValue Op = Ops[i];
7146 unsigned IdentityIndex = i * Op.getValueType().getVectorMinNumElements();
7147 if (Op.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
7148 Op.getOperand(i: 0).getValueType() != VT ||
7149 (IdentitySrc && Op.getOperand(i: 0) != IdentitySrc) ||
7150 Op.getConstantOperandVal(i: 1) != IdentityIndex) {
7151 IsIdentity = false;
7152 break;
7153 }
7154 assert((!IdentitySrc || IdentitySrc == Op.getOperand(0)) &&
7155 "Unexpected identity source vector for concat of extracts");
7156 IdentitySrc = Op.getOperand(i: 0);
7157 }
7158 if (IsIdentity) {
7159 assert(IdentitySrc && "Failed to set source vector of extracts");
7160 return IdentitySrc;
7161 }
7162
7163 // The code below this point is only designed to work for fixed width
7164 // vectors, so we bail out for now.
7165 if (VT.isScalableVector())
7166 return SDValue();
7167
7168 // A CONCAT_VECTOR of scalar sources, such as UNDEF, BUILD_VECTOR and
7169 // single-element INSERT_VECTOR_ELT operands can be simplified to one big
7170 // BUILD_VECTOR.
7171 // FIXME: Add support for SCALAR_TO_VECTOR as well.
7172 EVT SVT = VT.getScalarType();
7173 SmallVector<SDValue, 16> Elts;
7174 for (SDValue Op : Ops) {
7175 EVT OpVT = Op.getValueType();
7176 if (Op.getOpcode() == ISD::POISON)
7177 Elts.append(NumInputs: OpVT.getVectorNumElements(), Elt: DAG.getPOISON(VT: SVT));
7178 else if (Op.getOpcode() == ISD::UNDEF)
7179 Elts.append(NumInputs: OpVT.getVectorNumElements(), Elt: DAG.getUNDEF(VT: SVT));
7180 else if (Op.getOpcode() == ISD::BUILD_VECTOR)
7181 Elts.append(in_start: Op->op_begin(), in_end: Op->op_end());
7182 else if (Op.getOpcode() == ISD::INSERT_VECTOR_ELT &&
7183 OpVT.getVectorNumElements() == 1 &&
7184 isNullConstant(V: Op.getOperand(i: 2)))
7185 Elts.push_back(Elt: Op.getOperand(i: 1));
7186 else
7187 return SDValue();
7188 }
7189
7190 // BUILD_VECTOR requires all inputs to be of the same type, find the
7191 // maximum type and extend them all.
7192 for (SDValue Op : Elts)
7193 SVT = (SVT.bitsLT(VT: Op.getValueType()) ? Op.getValueType() : SVT);
7194
7195 if (SVT.bitsGT(VT: VT.getScalarType())) {
7196 for (SDValue &Op : Elts) {
7197 if (Op.getOpcode() == ISD::POISON)
7198 Op = DAG.getPOISON(VT: SVT);
7199 else if (Op.getOpcode() == ISD::UNDEF)
7200 Op = DAG.getUNDEF(VT: SVT);
7201 else
7202 Op = DAG.getTargetLoweringInfo().isZExtFree(FromTy: Op.getValueType(), ToTy: SVT)
7203 ? DAG.getZExtOrTrunc(Op, DL, VT: SVT)
7204 : DAG.getSExtOrTrunc(Op, DL, VT: SVT);
7205 }
7206 }
7207
7208 SDValue V = DAG.getBuildVector(VT, DL, Ops: Elts);
7209 NewSDValueDbgMsg(V, Msg: "New node fold concat vectors: ", G: &DAG);
7210 return V;
7211}
7212
7213/// Gets or creates the specified node.
7214SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT) {
7215 SDVTList VTs = getVTList(VT);
7216 SDNodeKey ID(Opcode, VTs, {});
7217 FoldingSetInsertToken InsertToken;
7218 if (SDNode *E = lookupNode(Key: ID, DL, InsertToken))
7219 return SDValue(E, 0);
7220
7221 auto *N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
7222 CSEMap.insert(N, Token: InsertToken);
7223
7224 InsertNode(N);
7225 SDValue V = SDValue(N, 0);
7226 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
7227 return V;
7228}
7229
7230SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7231 SDValue N1) {
7232 SDNodeFlags Flags;
7233 if (Inserter)
7234 Flags = Inserter->getFlags();
7235 return getNode(Opcode, DL, VT, Operand: N1, Flags);
7236}
7237
7238SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7239 SDValue N1, const SDNodeFlags Flags) {
7240 assert(N1.getOpcode() != ISD::DELETED_NODE && "Operand is DELETED_NODE!");
7241
7242 // Constant fold unary operations with a vector integer or float operand.
7243 switch (Opcode) {
7244 default:
7245 // FIXME: Entirely reasonable to perform folding of other unary
7246 // operations here as the need arises.
7247 break;
7248 case ISD::FNEG:
7249 case ISD::FABS:
7250 case ISD::FCEIL:
7251 case ISD::FTRUNC:
7252 case ISD::FFLOOR:
7253 case ISD::FP_EXTEND:
7254 case ISD::FP_TO_SINT:
7255 case ISD::FP_TO_UINT:
7256 case ISD::FP_TO_FP16:
7257 case ISD::FP_TO_BF16:
7258 case ISD::TRUNCATE:
7259 case ISD::ANY_EXTEND:
7260 case ISD::ZERO_EXTEND:
7261 case ISD::SIGN_EXTEND:
7262 case ISD::UINT_TO_FP:
7263 case ISD::SINT_TO_FP:
7264 case ISD::FP16_TO_FP:
7265 case ISD::BF16_TO_FP:
7266 case ISD::BITCAST:
7267 case ISD::ABS:
7268 case ISD::ABS_MIN_POISON:
7269 case ISD::BITREVERSE:
7270 case ISD::BSWAP:
7271 case ISD::CTLZ:
7272 case ISD::CTLZ_ZERO_POISON:
7273 case ISD::CTTZ:
7274 case ISD::CTTZ_ZERO_POISON:
7275 case ISD::CTPOP:
7276 case ISD::PARITY:
7277 case ISD::CTLS:
7278 case ISD::VECREDUCE_ADD:
7279 case ISD::VECREDUCE_SMAX:
7280 case ISD::VECREDUCE_SMIN:
7281 case ISD::VECREDUCE_UMAX:
7282 case ISD::VECREDUCE_UMIN:
7283 case ISD::VECREDUCE_MUL:
7284 case ISD::VECREDUCE_AND:
7285 case ISD::VECREDUCE_OR:
7286 case ISD::VECREDUCE_XOR:
7287 case ISD::STEP_VECTOR: {
7288 SDValue Ops = {N1};
7289 if (SDValue Fold = FoldConstantArithmetic(Opcode, DL, VT, Ops, Flags))
7290 return Fold;
7291 }
7292 }
7293
7294 unsigned OpOpcode = N1.getNode()->getOpcode();
7295 switch (Opcode) {
7296 case ISD::STEP_VECTOR:
7297 assert(VT.isScalableVector() &&
7298 "STEP_VECTOR can only be used with scalable types");
7299 assert(OpOpcode == ISD::TargetConstant &&
7300 VT.getVectorElementType() == N1.getValueType() &&
7301 "Unexpected step operand");
7302 break;
7303 case ISD::FREEZE:
7304 assert(VT == N1.getValueType() && "Unexpected VT!");
7305 if (isGuaranteedNotToBeUndefOrPoison(Op: N1, Kind: UndefPoisonKind::UndefOrPoison))
7306 return N1;
7307 break;
7308 case ISD::TokenFactor:
7309 case ISD::MERGE_VALUES:
7310 case ISD::CONCAT_VECTORS:
7311 return N1; // Factor, merge or concat of one node? No need.
7312 case ISD::BUILD_VECTOR: {
7313 // Attempt to simplify BUILD_VECTOR.
7314 SDValue Ops[] = {N1};
7315 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, DAG&: *this))
7316 return V;
7317 break;
7318 }
7319 case ISD::FP_ROUND: llvm_unreachable("Invalid method to make FP_ROUND node");
7320 case ISD::FP_EXTEND:
7321 assert(VT.isFloatingPoint() && N1.getValueType().isFloatingPoint() &&
7322 "Invalid FP cast!");
7323 if (N1.getValueType() == VT) return N1; // noop conversion.
7324 assert((!VT.isVector() || VT.getVectorElementCount() ==
7325 N1.getValueType().getVectorElementCount()) &&
7326 "Vector element count mismatch!");
7327 assert(N1.getValueType().bitsLT(VT) && "Invalid fpext node, dst < src!");
7328 if (N1.isUndef())
7329 return getUNDEF(VT);
7330 break;
7331 case ISD::FP_TO_SINT:
7332 case ISD::FP_TO_UINT:
7333 if (N1.isUndef())
7334 return getUNDEF(VT);
7335 break;
7336 case ISD::SINT_TO_FP:
7337 case ISD::UINT_TO_FP:
7338 // [us]itofp(undef) = 0, because the result value is bounded.
7339 if (N1.isUndef())
7340 return getConstantFP(Val: 0.0, DL, VT);
7341 break;
7342 case ISD::SIGN_EXTEND:
7343 assert(VT.isInteger() && N1.getValueType().isInteger() &&
7344 "Invalid SIGN_EXTEND!");
7345 assert(VT.isVector() == N1.getValueType().isVector() &&
7346 "SIGN_EXTEND result type type should be vector iff the operand "
7347 "type is vector!");
7348 if (N1.getValueType() == VT) return N1; // noop extension
7349 assert((!VT.isVector() || VT.getVectorElementCount() ==
7350 N1.getValueType().getVectorElementCount()) &&
7351 "Vector element count mismatch!");
7352 assert(N1.getValueType().bitsLT(VT) && "Invalid sext node, dst < src!");
7353 if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND) {
7354 SDNodeFlags Flags;
7355 if (OpOpcode == ISD::ZERO_EXTEND)
7356 Flags.setNonNeg(N1->getFlags().hasNonNeg());
7357 SDValue NewVal = getNode(Opcode: OpOpcode, DL, VT, N1: N1.getOperand(i: 0), Flags);
7358 transferDbgValues(From: N1, To: NewVal);
7359 return NewVal;
7360 }
7361
7362 if (OpOpcode == ISD::POISON)
7363 return getPOISON(VT);
7364
7365 if (N1.isUndef())
7366 // sext(undef) = 0, because the top bits will all be the same.
7367 return getConstant(Val: 0, DL, VT);
7368
7369 // Skip unnecessary sext_inreg pattern:
7370 // (sext (trunc x)) -> x iff the upper bits are all signbits.
7371 if (OpOpcode == ISD::TRUNCATE) {
7372 SDValue OpOp = N1.getOperand(i: 0);
7373 if (OpOp.getValueType() == VT) {
7374 unsigned NumSignExtBits =
7375 VT.getScalarSizeInBits() - N1.getScalarValueSizeInBits();
7376 if (ComputeNumSignBits(Op: OpOp) > NumSignExtBits) {
7377 transferDbgValues(From: N1, To: OpOp);
7378 return OpOp;
7379 }
7380 }
7381 }
7382 break;
7383 case ISD::ZERO_EXTEND:
7384 assert(VT.isInteger() && N1.getValueType().isInteger() &&
7385 "Invalid ZERO_EXTEND!");
7386 assert(VT.isVector() == N1.getValueType().isVector() &&
7387 "ZERO_EXTEND result type type should be vector iff the operand "
7388 "type is vector!");
7389 if (N1.getValueType() == VT) return N1; // noop extension
7390 assert((!VT.isVector() || VT.getVectorElementCount() ==
7391 N1.getValueType().getVectorElementCount()) &&
7392 "Vector element count mismatch!");
7393 assert(N1.getValueType().bitsLT(VT) && "Invalid zext node, dst < src!");
7394 if (OpOpcode == ISD::ZERO_EXTEND) { // (zext (zext x)) -> (zext x)
7395 SDNodeFlags Flags;
7396 Flags.setNonNeg(N1->getFlags().hasNonNeg());
7397 SDValue NewVal =
7398 getNode(Opcode: ISD::ZERO_EXTEND, DL, VT, N1: N1.getOperand(i: 0), Flags);
7399 transferDbgValues(From: N1, To: NewVal);
7400 return NewVal;
7401 }
7402
7403 if (OpOpcode == ISD::POISON)
7404 return getPOISON(VT);
7405
7406 if (N1.isUndef())
7407 // zext(undef) = 0, because the top bits will be zero.
7408 return getConstant(Val: 0, DL, VT);
7409
7410 // Skip unnecessary zext_inreg pattern:
7411 // (zext (trunc x)) -> x iff the upper bits are known zero.
7412 // TODO: Remove (zext (trunc (and x, c))) exception which some targets
7413 // use to recognise zext_inreg patterns.
7414 if (OpOpcode == ISD::TRUNCATE) {
7415 SDValue OpOp = N1.getOperand(i: 0);
7416 if (OpOp.getValueType() == VT) {
7417 if (OpOp.getOpcode() != ISD::AND) {
7418 APInt HiBits = APInt::getBitsSetFrom(numBits: VT.getScalarSizeInBits(),
7419 loBit: N1.getScalarValueSizeInBits());
7420 if (MaskedValueIsZero(V: OpOp, Mask: HiBits)) {
7421 transferDbgValues(From: N1, To: OpOp);
7422 return OpOp;
7423 }
7424 }
7425 }
7426 }
7427 break;
7428 case ISD::ANY_EXTEND:
7429 assert(VT.isInteger() && N1.getValueType().isInteger() &&
7430 "Invalid ANY_EXTEND!");
7431 assert(VT.isVector() == N1.getValueType().isVector() &&
7432 "ANY_EXTEND result type type should be vector iff the operand "
7433 "type is vector!");
7434 if (N1.getValueType() == VT) return N1; // noop extension
7435 assert((!VT.isVector() || VT.getVectorElementCount() ==
7436 N1.getValueType().getVectorElementCount()) &&
7437 "Vector element count mismatch!");
7438 assert(N1.getValueType().bitsLT(VT) && "Invalid anyext node, dst < src!");
7439
7440 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
7441 OpOpcode == ISD::ANY_EXTEND) {
7442 SDNodeFlags Flags;
7443 if (OpOpcode == ISD::ZERO_EXTEND)
7444 Flags.setNonNeg(N1->getFlags().hasNonNeg());
7445 // (ext (zext x)) -> (zext x) and (ext (sext x)) -> (sext x)
7446 return getNode(Opcode: OpOpcode, DL, VT, N1: N1.getOperand(i: 0), Flags);
7447 }
7448 if (N1.isUndef())
7449 return getUNDEF(VT);
7450
7451 // (ext (trunc x)) -> x
7452 if (OpOpcode == ISD::TRUNCATE) {
7453 SDValue OpOp = N1.getOperand(i: 0);
7454 if (OpOp.getValueType() == VT) {
7455 transferDbgValues(From: N1, To: OpOp);
7456 return OpOp;
7457 }
7458 }
7459 break;
7460 case ISD::TRUNCATE:
7461 assert(VT.isInteger() && N1.getValueType().isInteger() &&
7462 "Invalid TRUNCATE!");
7463 assert(VT.isVector() == N1.getValueType().isVector() &&
7464 "TRUNCATE result type type should be vector iff the operand "
7465 "type is vector!");
7466 if (N1.getValueType() == VT) return N1; // noop truncate
7467 assert((!VT.isVector() || VT.getVectorElementCount() ==
7468 N1.getValueType().getVectorElementCount()) &&
7469 "Vector element count mismatch!");
7470 assert(N1.getValueType().bitsGT(VT) && "Invalid truncate node, src < dst!");
7471 if (OpOpcode == ISD::TRUNCATE)
7472 return getNode(Opcode: ISD::TRUNCATE, DL, VT, N1: N1.getOperand(i: 0));
7473 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
7474 OpOpcode == ISD::ANY_EXTEND) {
7475 // If the source is smaller than the dest, we still need an extend.
7476 if (N1.getOperand(i: 0).getValueType().getScalarType().bitsLT(
7477 VT: VT.getScalarType())) {
7478 SDNodeFlags Flags;
7479 if (OpOpcode == ISD::ZERO_EXTEND)
7480 Flags.setNonNeg(N1->getFlags().hasNonNeg());
7481 return getNode(Opcode: OpOpcode, DL, VT, N1: N1.getOperand(i: 0), Flags);
7482 }
7483 if (N1.getOperand(i: 0).getValueType().bitsGT(VT))
7484 return getNode(Opcode: ISD::TRUNCATE, DL, VT, N1: N1.getOperand(i: 0));
7485 return N1.getOperand(i: 0);
7486 }
7487 if (N1.isUndef())
7488 return getUNDEF(VT);
7489 if (OpOpcode == ISD::VSCALE && !NewNodesMustHaveLegalTypes)
7490 return getVScale(DL, VT,
7491 MulImm: N1.getConstantOperandAPInt(i: 0).trunc(width: VT.getSizeInBits()));
7492 break;
7493 case ISD::ANY_EXTEND_VECTOR_INREG:
7494 case ISD::ZERO_EXTEND_VECTOR_INREG:
7495 case ISD::SIGN_EXTEND_VECTOR_INREG:
7496 assert(VT.isVector() && "This DAG node is restricted to vector types.");
7497 assert(N1.getValueType().bitsLE(VT) &&
7498 "The input must be the same size or smaller than the result.");
7499 assert(VT.getVectorMinNumElements() <
7500 N1.getValueType().getVectorMinNumElements() &&
7501 "The destination vector type must have fewer lanes than the input.");
7502 break;
7503 case ISD::ABS:
7504 assert(VT.isInteger() && VT == N1.getValueType() && "Invalid ABS!");
7505 if (N1.isUndef())
7506 return getConstant(Val: 0, DL, VT);
7507 break;
7508 case ISD::ABS_MIN_POISON:
7509 assert(VT.isInteger() && VT == N1.getValueType() &&
7510 "Invalid ABS_MIN_POISON!");
7511 if (N1.isUndef())
7512 return getConstant(Val: 0, DL, VT);
7513 break;
7514 case ISD::BSWAP:
7515 assert(VT.isInteger() && VT == N1.getValueType() && "Invalid BSWAP!");
7516 assert((VT.getScalarSizeInBits() % 16 == 0) &&
7517 "BSWAP types must be a multiple of 16 bits!");
7518 if (N1.isUndef())
7519 return getUNDEF(VT);
7520 // bswap(bswap(X)) -> X.
7521 if (OpOpcode == ISD::BSWAP)
7522 return N1.getOperand(i: 0);
7523 break;
7524 case ISD::BITREVERSE:
7525 assert(VT.isInteger() && VT == N1.getValueType() && "Invalid BITREVERSE!");
7526 if (N1.isUndef())
7527 return getUNDEF(VT);
7528 break;
7529 case ISD::BITCAST:
7530 assert(VT.getSizeInBits() == N1.getValueSizeInBits() &&
7531 "Cannot BITCAST between types of different sizes!");
7532 if (VT == N1.getValueType()) return N1; // noop conversion.
7533 if (OpOpcode == ISD::BITCAST) // bitconv(bitconv(x)) -> bitconv(x)
7534 return getNode(Opcode: ISD::BITCAST, DL, VT, N1: N1.getOperand(i: 0));
7535 if (N1.isUndef())
7536 return getUNDEF(VT);
7537 break;
7538 case ISD::SCALAR_TO_VECTOR:
7539 assert(VT.isVector() && !N1.getValueType().isVector() &&
7540 (VT.getVectorElementType() == N1.getValueType() ||
7541 (VT.getVectorElementType().isInteger() &&
7542 N1.getValueType().isInteger() &&
7543 VT.getVectorElementType().bitsLE(N1.getValueType()))) &&
7544 "Illegal SCALAR_TO_VECTOR node!");
7545 if (N1.isUndef())
7546 return getUNDEF(VT);
7547 // scalar_to_vector(extract_vector_elt V, 0) -> V, top bits are undefined.
7548 if (OpOpcode == ISD::EXTRACT_VECTOR_ELT &&
7549 isa<ConstantSDNode>(Val: N1.getOperand(i: 1)) &&
7550 N1.getConstantOperandVal(i: 1) == 0 &&
7551 N1.getOperand(i: 0).getValueType() == VT)
7552 return N1.getOperand(i: 0);
7553 break;
7554 case ISD::FNEG:
7555 // Negation of an unknown bag of bits is still completely undefined.
7556 if (N1.isUndef())
7557 return getUNDEF(VT);
7558
7559 if (OpOpcode == ISD::FNEG) // --X -> X
7560 return N1.getOperand(i: 0);
7561 break;
7562 case ISD::FABS:
7563 if (OpOpcode == ISD::FNEG) // abs(-X) -> abs(X)
7564 return getNode(Opcode: ISD::FABS, DL, VT, N1: N1.getOperand(i: 0));
7565 break;
7566 case ISD::VSCALE:
7567 assert(VT == N1.getValueType() && "Unexpected VT!");
7568 break;
7569 case ISD::CTPOP:
7570 if (N1.getValueType().getScalarType() == MVT::i1)
7571 return N1;
7572 break;
7573 case ISD::CTLZ:
7574 case ISD::CTTZ:
7575 if (N1.getValueType().getScalarType() == MVT::i1)
7576 return getNOT(DL, Val: N1, VT: N1.getValueType());
7577 break;
7578 case ISD::CTLS:
7579 if (N1.getValueType().getScalarType() == MVT::i1)
7580 return getConstant(Val: 0, DL, VT);
7581 break;
7582 case ISD::VECREDUCE_ADD:
7583 if (N1.getValueType().getScalarType() == MVT::i1)
7584 return getNode(Opcode: ISD::VECREDUCE_XOR, DL, VT, N1);
7585 break;
7586 case ISD::VECREDUCE_SMIN:
7587 case ISD::VECREDUCE_UMAX:
7588 if (N1.getValueType().getScalarType() == MVT::i1)
7589 return getNode(Opcode: ISD::VECREDUCE_OR, DL, VT, N1);
7590 break;
7591 case ISD::VECREDUCE_SMAX:
7592 case ISD::VECREDUCE_UMIN:
7593 if (N1.getValueType().getScalarType() == MVT::i1)
7594 return getNode(Opcode: ISD::VECREDUCE_AND, DL, VT, N1);
7595 break;
7596 case ISD::MASK_BEFOREFIRST:
7597 // With a single lane, the result is true iff that lane is false.
7598 if (VT.getVectorElementCount().isScalar())
7599 return getLogicalNOT(DL, Val: N1, VT);
7600 break;
7601 case ISD::VECTOR_REPEAT:
7602 assert(N1.getValueType().isFixedLengthVector() &&
7603 "VECTOR_REPEAT requires a fixed-length vector operand");
7604 assert(VT.isScalableVector() &&
7605 "VECTOR_REPEAT requires a scalable vector result");
7606 assert(N1.getValueType().getVectorNumElements() ==
7607 VT.getVectorMinNumElements() &&
7608 "VECTOR_REPEAT operand and result element counts must match");
7609 if (VT.getVectorMinNumElements() == 1)
7610 return getSplatVector(
7611 VT, DL, Op: getExtractVectorElt(DL, VT: VT.getVectorElementType(), Vec: N1, Idx: 0));
7612 break;
7613 case ISD::SPLAT_VECTOR:
7614 assert(VT.isVector() && "Wrong return type!");
7615 // FIXME: Hexagon uses i32 scalar for a floating point zero vector so allow
7616 // that for now.
7617 assert((VT.getVectorElementType() == N1.getValueType() ||
7618 (VT.isFloatingPoint() && N1.getValueType() == MVT::i32) ||
7619 (VT.getVectorElementType().isInteger() &&
7620 N1.getValueType().isInteger() &&
7621 VT.getVectorElementType().bitsLE(N1.getValueType()))) &&
7622 "Wrong operand type!");
7623 break;
7624 }
7625
7626 SDNode *N;
7627 SDVTList VTs = getVTList(VT);
7628 SDValue Ops[] = {N1};
7629 if (VT != MVT::Glue) { // Don't CSE glue producing nodes
7630 SDNodeKey ID(Opcode, VTs, Ops);
7631 FoldingSetInsertToken InsertToken;
7632 if (SDNode *E = lookupNode(Key: ID, DL, InsertToken)) {
7633 E->intersectFlagsWith(Flags);
7634 return SDValue(E, 0);
7635 }
7636
7637 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
7638 N->setFlags(Flags);
7639 createOperands(Node: N, Vals: Ops);
7640 CSEMap.insert(N, Token: InsertToken);
7641 } else {
7642 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
7643 createOperands(Node: N, Vals: Ops);
7644 }
7645
7646 InsertNode(N);
7647 SDValue V = SDValue(N, 0);
7648 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
7649 return V;
7650}
7651
7652static APInt getIntegerIdentity(unsigned Opcode, unsigned BitWidth) {
7653 switch (Opcode) {
7654 default:
7655 llvm_unreachable("Unexpected integer identity opcode");
7656 case ISD::ADD:
7657 case ISD::OR:
7658 case ISD::XOR:
7659 case ISD::UMAX:
7660 return APInt::getZero(numBits: BitWidth);
7661 case ISD::MUL:
7662 return APInt(BitWidth, 1);
7663 case ISD::AND:
7664 case ISD::UMIN:
7665 return APInt::getAllOnes(numBits: BitWidth);
7666 case ISD::SMAX:
7667 return APInt::getSignedMinValue(numBits: BitWidth);
7668 case ISD::SMIN:
7669 return APInt::getSignedMaxValue(numBits: BitWidth);
7670 }
7671}
7672
7673static std::optional<APInt> FoldValue(unsigned Opcode, const APInt &C1,
7674 const APInt &C2) {
7675 switch (Opcode) {
7676 case ISD::ADD: return C1 + C2;
7677 case ISD::SUB: return C1 - C2;
7678 case ISD::MUL: return C1 * C2;
7679 case ISD::AND: return C1 & C2;
7680 case ISD::OR: return C1 | C2;
7681 case ISD::XOR: return C1 ^ C2;
7682 case ISD::SHL: return C1 << C2;
7683 case ISD::SRL: return C1.lshr(ShiftAmt: C2);
7684 case ISD::SRA: return C1.ashr(ShiftAmt: C2);
7685 case ISD::ROTL: return C1.rotl(rotateAmt: C2);
7686 case ISD::ROTR: return C1.rotr(rotateAmt: C2);
7687 case ISD::SMIN: return C1.sle(RHS: C2) ? C1 : C2;
7688 case ISD::SMAX: return C1.sge(RHS: C2) ? C1 : C2;
7689 case ISD::UMIN: return C1.ule(RHS: C2) ? C1 : C2;
7690 case ISD::UMAX: return C1.uge(RHS: C2) ? C1 : C2;
7691 case ISD::SADDSAT: return C1.sadd_sat(RHS: C2);
7692 case ISD::UADDSAT: return C1.uadd_sat(RHS: C2);
7693 case ISD::SSUBSAT: return C1.ssub_sat(RHS: C2);
7694 case ISD::USUBSAT: return C1.usub_sat(RHS: C2);
7695 case ISD::SSHLSAT: return C1.sshl_sat(RHS: C2);
7696 case ISD::USHLSAT: return C1.ushl_sat(RHS: C2);
7697 case ISD::UDIV:
7698 if (!C2.getBoolValue())
7699 break;
7700 return C1.udiv(RHS: C2);
7701 case ISD::UREM:
7702 if (!C2.getBoolValue())
7703 break;
7704 return C1.urem(RHS: C2);
7705 case ISD::SDIV:
7706 if (!C2.getBoolValue())
7707 break;
7708 return C1.sdiv(RHS: C2);
7709 case ISD::SREM:
7710 if (!C2.getBoolValue())
7711 break;
7712 return C1.srem(RHS: C2);
7713 case ISD::AVGFLOORS:
7714 return APIntOps::avgFloorS(C1, C2);
7715 case ISD::AVGFLOORU:
7716 return APIntOps::avgFloorU(C1, C2);
7717 case ISD::AVGCEILS:
7718 return APIntOps::avgCeilS(C1, C2);
7719 case ISD::AVGCEILU:
7720 return APIntOps::avgCeilU(C1, C2);
7721 case ISD::AVGSUBFLOORS:
7722 return APIntOps::avgSubFloorS(C1, C2);
7723 case ISD::AVGSUBFLOORU:
7724 return APIntOps::avgSubFloorU(C1, C2);
7725 case ISD::ABDS:
7726 return APIntOps::abds(A: C1, B: C2);
7727 case ISD::ABDU:
7728 return APIntOps::abdu(A: C1, B: C2);
7729 case ISD::MULHS:
7730 return APIntOps::mulhs(C1, C2);
7731 case ISD::MULHU:
7732 return APIntOps::mulhu(C1, C2);
7733 case ISD::CLMUL:
7734 return APIntOps::clmul(LHS: C1, RHS: C2);
7735 case ISD::CLMULR:
7736 return APIntOps::clmulr(LHS: C1, RHS: C2);
7737 case ISD::CLMULH:
7738 return APIntOps::clmulh(LHS: C1, RHS: C2);
7739 case ISD::PEXT:
7740 return APIntOps::pext(Val: C1, Mask: C2);
7741 case ISD::PDEP:
7742 return APIntOps::pdep(Val: C1, Mask: C2);
7743 }
7744 return std::nullopt;
7745}
7746// Handle constant folding with UNDEF.
7747// TODO: Handle more cases.
7748static std::optional<APInt> FoldValueWithUndef(unsigned Opcode, const APInt &C1,
7749 bool IsUndef1, const APInt &C2,
7750 bool IsUndef2) {
7751 if (!(IsUndef1 || IsUndef2))
7752 return FoldValue(Opcode, C1, C2);
7753
7754 // Fold and(x, undef) -> 0
7755 // Fold mul(x, undef) -> 0
7756 if (Opcode == ISD::AND || Opcode == ISD::MUL)
7757 return APInt::getZero(numBits: C1.getBitWidth());
7758
7759 return std::nullopt;
7760}
7761
7762SDValue SelectionDAG::FoldSymbolOffset(unsigned Opcode, EVT VT,
7763 const GlobalAddressSDNode *GA,
7764 const SDNode *N2) {
7765 if (GA->getOpcode() != ISD::GlobalAddress)
7766 return SDValue();
7767 if (!TLI->isOffsetFoldingLegal(GA))
7768 return SDValue();
7769 auto *C2 = dyn_cast<ConstantSDNode>(Val: N2);
7770 if (!C2)
7771 return SDValue();
7772 int64_t Offset = C2->getSExtValue();
7773 switch (Opcode) {
7774 case ISD::ADD:
7775 case ISD::PTRADD:
7776 break;
7777 case ISD::SUB: Offset = -uint64_t(Offset); break;
7778 default: return SDValue();
7779 }
7780 return getGlobalAddress(GV: GA->getGlobal(), DL: SDLoc(C2), VT,
7781 Offset: GA->getOffset() + uint64_t(Offset));
7782}
7783
7784bool SelectionDAG::isUndef(unsigned Opcode, ArrayRef<SDValue> Ops) {
7785 switch (Opcode) {
7786 case ISD::SDIV:
7787 case ISD::UDIV:
7788 case ISD::SREM:
7789 case ISD::UREM: {
7790 // If a divisor is zero/undef or any element of a divisor vector is
7791 // zero/undef, the whole op is undef.
7792 assert(Ops.size() == 2 && "Div/rem should have 2 operands");
7793 SDValue Divisor = Ops[1];
7794 if (Divisor.isUndef() || isNullConstant(V: Divisor))
7795 return true;
7796
7797 return ISD::isBuildVectorOfConstantSDNodes(V: Divisor) &&
7798 llvm::any_of(Range: Divisor->op_values(), P: [](SDValue V) {
7799 return V.isUndef() || isNullConstant(V);
7800 });
7801 // TODO: Handle signed overflow.
7802 }
7803 // TODO: Handle oversized shifts.
7804 default:
7805 return false;
7806 }
7807}
7808
7809SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL,
7810 EVT VT, ArrayRef<SDValue> Ops,
7811 SDNodeFlags Flags) {
7812 // If the opcode is a target-specific ISD node, there's nothing we can
7813 // do here and the operand rules may not line up with the below, so
7814 // bail early.
7815 // We can't create a scalar CONCAT_VECTORS so skip it. It will break
7816 // for concats involving SPLAT_VECTOR. Concats of BUILD_VECTORS are handled by
7817 // foldCONCAT_VECTORS in getNode before this is called.
7818 // VECTOR_SHUFFLE_VAR is not elementwise.
7819 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::CONCAT_VECTORS ||
7820 Opcode == ISD::VECTOR_SHUFFLE_VAR)
7821 return SDValue();
7822
7823 unsigned NumOps = Ops.size();
7824 if (NumOps == 0)
7825 return SDValue();
7826
7827 if (isUndef(Opcode, Ops))
7828 return getUNDEF(VT);
7829
7830 // Handle unary special cases.
7831 if (NumOps == 1) {
7832 SDValue N1 = Ops[0];
7833
7834 // Constant fold unary operations with an integer constant operand. Even
7835 // opaque constant will be folded, because the folding of unary operations
7836 // doesn't create new constants with different values. Nevertheless, the
7837 // opaque flag is preserved during folding to prevent future folding with
7838 // other constants.
7839 if (auto *C = dyn_cast<ConstantSDNode>(Val&: N1)) {
7840 const APInt &Val = C->getAPIntValue();
7841 switch (Opcode) {
7842 case ISD::SIGN_EXTEND:
7843 return getConstant(Val: Val.sextOrTrunc(width: VT.getSizeInBits()), DL, VT,
7844 isT: C->isTargetOpcode(), isO: C->isOpaque());
7845 case ISD::TRUNCATE:
7846 if (C->isOpaque())
7847 break;
7848 [[fallthrough]];
7849 case ISD::ZERO_EXTEND:
7850 return getConstant(Val: Val.zextOrTrunc(width: VT.getSizeInBits()), DL, VT,
7851 isT: C->isTargetOpcode(), isO: C->isOpaque());
7852 case ISD::ANY_EXTEND:
7853 // Some targets like RISCV prefer to sign extend some types.
7854 if (TLI->isSExtCheaperThanZExt(FromTy: N1.getValueType(), ToTy: VT))
7855 return getConstant(Val: Val.sextOrTrunc(width: VT.getSizeInBits()), DL, VT,
7856 isT: C->isTargetOpcode(), isO: C->isOpaque());
7857 return getConstant(Val: Val.zextOrTrunc(width: VT.getSizeInBits()), DL, VT,
7858 isT: C->isTargetOpcode(), isO: C->isOpaque());
7859 case ISD::ABS:
7860 return getConstant(Val: Val.abs(), DL, VT, isT: C->isTargetOpcode(),
7861 isO: C->isOpaque());
7862 case ISD::ABS_MIN_POISON:
7863 if (Val.isMinSignedValue())
7864 return getPOISON(VT);
7865 return getConstant(Val: Val.abs(), DL, VT, isT: C->isTargetOpcode(),
7866 isO: C->isOpaque());
7867 case ISD::BITREVERSE:
7868 return getConstant(Val: Val.reverseBits(), DL, VT, isT: C->isTargetOpcode(),
7869 isO: C->isOpaque());
7870 case ISD::BSWAP:
7871 return getConstant(Val: Val.byteSwap(), DL, VT, isT: C->isTargetOpcode(),
7872 isO: C->isOpaque());
7873 case ISD::CTPOP:
7874 return getConstant(Val: Val.popcount(), DL, VT, isT: C->isTargetOpcode(),
7875 isO: C->isOpaque());
7876 case ISD::PARITY:
7877 return getConstant(Val: Val.popcount() & 1, DL, VT, isT: C->isTargetOpcode(),
7878 isO: C->isOpaque());
7879 case ISD::CTLZ:
7880 case ISD::CTLZ_ZERO_POISON:
7881 return getConstant(Val: Val.countl_zero(), DL, VT, isT: C->isTargetOpcode(),
7882 isO: C->isOpaque());
7883 case ISD::CTTZ:
7884 case ISD::CTTZ_ZERO_POISON:
7885 return getConstant(Val: Val.countr_zero(), DL, VT, isT: C->isTargetOpcode(),
7886 isO: C->isOpaque());
7887 case ISD::CTLS:
7888 // CTLS returns the number of extra sign bits so subtract one.
7889 return getConstant(Val: Val.getNumSignBits() - 1, DL, VT,
7890 isT: C->isTargetOpcode(), isO: C->isOpaque());
7891 case ISD::UINT_TO_FP:
7892 case ISD::SINT_TO_FP: {
7893 APFloat FPV(VT.getFltSemantics(), APInt::getZero(numBits: VT.getSizeInBits()));
7894 (void)FPV.convertFromAPInt(Input: Val, IsSigned: Opcode == ISD::SINT_TO_FP,
7895 RM: APFloat::rmNearestTiesToEven);
7896 return getConstantFP(V: FPV, DL, VT);
7897 }
7898 case ISD::FP16_TO_FP:
7899 case ISD::BF16_TO_FP: {
7900 bool Ignored;
7901 APFloat FPV(Opcode == ISD::FP16_TO_FP ? APFloat::IEEEhalf()
7902 : APFloat::BFloat(),
7903 (Val.getBitWidth() == 16) ? Val : Val.trunc(width: 16));
7904
7905 // This can return overflow, underflow, or inexact; we don't care.
7906 // FIXME need to be more flexible about rounding mode.
7907 (void)FPV.convert(ToSemantics: VT.getFltSemantics(), RM: APFloat::rmNearestTiesToEven,
7908 losesInfo: &Ignored);
7909 return getConstantFP(V: FPV, DL, VT);
7910 }
7911 case ISD::STEP_VECTOR:
7912 if (SDValue V = FoldSTEP_VECTOR(DL, VT, Step: N1, DAG&: *this))
7913 return V;
7914 break;
7915 case ISD::BITCAST:
7916 if (C->isOpaque())
7917 break;
7918 if (VT == MVT::f16 && C->getValueType(ResNo: 0) == MVT::i16)
7919 return getConstantFP(V: APFloat(APFloat::IEEEhalf(), Val), DL, VT);
7920 if (VT == MVT::f32 && C->getValueType(ResNo: 0) == MVT::i32)
7921 return getConstantFP(V: APFloat(APFloat::IEEEsingle(), Val), DL, VT);
7922 if (VT == MVT::f64 && C->getValueType(ResNo: 0) == MVT::i64)
7923 return getConstantFP(V: APFloat(APFloat::IEEEdouble(), Val), DL, VT);
7924 if (VT == MVT::f128 && C->getValueType(ResNo: 0) == MVT::i128)
7925 return getConstantFP(V: APFloat(APFloat::IEEEquad(), Val), DL, VT);
7926 break;
7927 }
7928 }
7929
7930 // Constant fold unary operations with a floating point constant operand.
7931 if (auto *C = dyn_cast<ConstantFPSDNode>(Val&: N1)) {
7932 APFloat V = C->getValueAPF(); // make copy
7933 switch (Opcode) {
7934 case ISD::FNEG:
7935 V.changeSign();
7936 return getConstantFP(V, DL, VT);
7937 case ISD::FABS:
7938 V.clearSign();
7939 return getConstantFP(V, DL, VT);
7940 case ISD::FCEIL: {
7941 APFloat::opStatus fs = V.roundToIntegral(RM: APFloat::rmTowardPositive);
7942 if (fs == APFloat::opOK || fs == APFloat::opInexact)
7943 return getConstantFP(V, DL, VT);
7944 return SDValue();
7945 }
7946 case ISD::FTRUNC: {
7947 APFloat::opStatus fs = V.roundToIntegral(RM: APFloat::rmTowardZero);
7948 if (fs == APFloat::opOK || fs == APFloat::opInexact)
7949 return getConstantFP(V, DL, VT);
7950 return SDValue();
7951 }
7952 case ISD::FFLOOR: {
7953 APFloat::opStatus fs = V.roundToIntegral(RM: APFloat::rmTowardNegative);
7954 if (fs == APFloat::opOK || fs == APFloat::opInexact)
7955 return getConstantFP(V, DL, VT);
7956 return SDValue();
7957 }
7958 case ISD::FP_EXTEND: {
7959 bool ignored;
7960 // This can return overflow, underflow, or inexact; we don't care.
7961 // FIXME need to be more flexible about rounding mode.
7962 (void)V.convert(ToSemantics: VT.getFltSemantics(), RM: APFloat::rmNearestTiesToEven,
7963 losesInfo: &ignored);
7964 return getConstantFP(V, DL, VT);
7965 }
7966 case ISD::FP_TO_SINT:
7967 case ISD::FP_TO_UINT: {
7968 bool ignored;
7969 APSInt IntVal(VT.getSizeInBits(), Opcode == ISD::FP_TO_UINT);
7970 // FIXME need to be more flexible about rounding mode.
7971 APFloat::opStatus s =
7972 V.convertToInteger(Result&: IntVal, RM: APFloat::rmTowardZero, IsExact: &ignored);
7973 if (s == APFloat::opInvalidOp) // inexact is OK, in fact usual
7974 break;
7975 return getConstant(Val: IntVal, DL, VT);
7976 }
7977 case ISD::FP_TO_FP16:
7978 case ISD::FP_TO_BF16: {
7979 bool Ignored;
7980 // This can return overflow, underflow, or inexact; we don't care.
7981 // FIXME need to be more flexible about rounding mode.
7982 (void)V.convert(ToSemantics: Opcode == ISD::FP_TO_FP16 ? APFloat::IEEEhalf()
7983 : APFloat::BFloat(),
7984 RM: APFloat::rmNearestTiesToEven, losesInfo: &Ignored);
7985 return getConstant(Val: V.bitcastToAPInt().getZExtValue(), DL, VT);
7986 }
7987 case ISD::BITCAST:
7988 if (VT == MVT::i16 && C->getValueType(ResNo: 0) == MVT::f16)
7989 return getConstant(Val: (uint16_t)V.bitcastToAPInt().getZExtValue(), DL,
7990 VT);
7991 if (VT == MVT::i16 && C->getValueType(ResNo: 0) == MVT::bf16)
7992 return getConstant(Val: (uint16_t)V.bitcastToAPInt().getZExtValue(), DL,
7993 VT);
7994 if (VT == MVT::i32 && C->getValueType(ResNo: 0) == MVT::f32)
7995 return getConstant(Val: (uint32_t)V.bitcastToAPInt().getZExtValue(), DL,
7996 VT);
7997 if (VT == MVT::i64 && C->getValueType(ResNo: 0) == MVT::f64)
7998 return getConstant(Val: V.bitcastToAPInt().getZExtValue(), DL, VT);
7999 break;
8000 }
8001 }
8002
8003 // Early-out if we failed to constant fold a bitcast.
8004 if (Opcode == ISD::BITCAST)
8005 return SDValue();
8006
8007 // Constant fold integer vector reductions with constant BUILD_VECTORs.
8008 if ((Opcode == ISD::VECREDUCE_ADD || Opcode == ISD::VECREDUCE_SMAX ||
8009 Opcode == ISD::VECREDUCE_SMIN || Opcode == ISD::VECREDUCE_UMAX ||
8010 Opcode == ISD::VECREDUCE_UMIN || Opcode == ISD::VECREDUCE_MUL ||
8011 Opcode == ISD::VECREDUCE_OR || Opcode == ISD::VECREDUCE_XOR ||
8012 Opcode == ISD::VECREDUCE_AND) &&
8013 ISD::isBuildVectorOfConstantSDNodes(V: N1)) {
8014 unsigned EltBits = N1.getValueType().getScalarSizeInBits();
8015 unsigned BaseOpcode = ISD::getVecReduceBaseOpcode(VecReduceOpcode: Opcode);
8016 APInt Acc = getIntegerIdentity(Opcode: BaseOpcode, BitWidth: EltBits);
8017 for (SDValue Elt : N1->op_values()) {
8018 if (Elt.getOpcode() == ISD::POISON)
8019 return getPOISON(VT);
8020 if (Elt.isUndef() || cast<ConstantSDNode>(Val&: Elt)->isOpaque())
8021 return SDValue();
8022 APInt Value = cast<ConstantSDNode>(Val&: Elt)->getAPIntValue().trunc(width: EltBits);
8023 std::optional<APInt> Folded = FoldValue(Opcode: BaseOpcode, C1: Acc, C2: Value);
8024 assert(Folded &&
8025 "Expected vector reduction base opcode to be foldable");
8026 Acc = *Folded;
8027 }
8028 EVT EltVT = N1.getValueType().getScalarType();
8029 return getAnyExtOrTrunc(Op: getConstant(Val: Acc, DL, VT: EltVT), DL, VT);
8030 }
8031 }
8032
8033 // Handle binops special cases.
8034 if (NumOps == 2) {
8035 if (SDValue CFP = foldConstantFPMath(Opcode, DL, VT, Ops))
8036 return CFP;
8037
8038 if (auto *C1 = dyn_cast<ConstantSDNode>(Val: Ops[0])) {
8039 if (auto *C2 = dyn_cast<ConstantSDNode>(Val: Ops[1])) {
8040 if (C1->isOpaque() || C2->isOpaque())
8041 return SDValue();
8042
8043 std::optional<APInt> FoldAttempt =
8044 FoldValue(Opcode, C1: C1->getAPIntValue(), C2: C2->getAPIntValue());
8045 if (!FoldAttempt)
8046 return SDValue();
8047
8048 SDValue Folded = getConstant(Val: *FoldAttempt, DL, VT);
8049 assert((!Folded || !VT.isVector()) &&
8050 "Can't fold vectors ops with scalar operands");
8051 return Folded;
8052 }
8053 }
8054
8055 // fold (add Sym, c) -> Sym+c
8056 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Val: Ops[0]))
8057 return FoldSymbolOffset(Opcode, VT, GA, N2: Ops[1].getNode());
8058 if (TLI->isCommutativeBinOp(Opcode))
8059 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Val: Ops[1]))
8060 return FoldSymbolOffset(Opcode, VT, GA, N2: Ops[0].getNode());
8061
8062 // fold (sext_in_reg c1) -> c2
8063 if (Opcode == ISD::SIGN_EXTEND_INREG) {
8064 EVT EVT = cast<VTSDNode>(Val: Ops[1])->getVT();
8065
8066 auto SignExtendInReg = [&](APInt Val, llvm::EVT ConstantVT) {
8067 unsigned FromBits = EVT.getScalarSizeInBits();
8068 Val <<= Val.getBitWidth() - FromBits;
8069 Val.ashrInPlace(ShiftAmt: Val.getBitWidth() - FromBits);
8070 return getConstant(Val, DL, VT: ConstantVT);
8071 };
8072
8073 if (auto *C1 = dyn_cast<ConstantSDNode>(Val: Ops[0])) {
8074 const APInt &Val = C1->getAPIntValue();
8075 return SignExtendInReg(Val, VT);
8076 }
8077
8078 if (ISD::isBuildVectorOfConstantSDNodes(V: Ops[0])) {
8079 SmallVector<SDValue, 8> ScalarOps;
8080 llvm::EVT OpVT = Ops[0].getOperand(i: 0).getValueType();
8081 for (int I = 0, E = VT.getVectorNumElements(); I != E; ++I) {
8082 SDValue Op = Ops[0].getOperand(i: I);
8083 if (Op.isUndef()) {
8084 ScalarOps.push_back(Elt: getUNDEF(VT: OpVT));
8085 continue;
8086 }
8087 const APInt &Val = cast<ConstantSDNode>(Val&: Op)->getAPIntValue();
8088 ScalarOps.push_back(Elt: SignExtendInReg(Val, OpVT));
8089 }
8090 return getBuildVector(VT, DL, Ops: ScalarOps);
8091 }
8092
8093 if (Ops[0].getOpcode() == ISD::SPLAT_VECTOR &&
8094 isa<ConstantSDNode>(Val: Ops[0].getOperand(i: 0)))
8095 return getNode(Opcode: ISD::SPLAT_VECTOR, DL, VT,
8096 N1: SignExtendInReg(Ops[0].getConstantOperandAPInt(i: 0),
8097 Ops[0].getOperand(i: 0).getValueType()));
8098 }
8099 }
8100
8101 // Handle fshl/fshr special cases.
8102 if (Opcode == ISD::FSHL || Opcode == ISD::FSHR) {
8103 auto *C1 = dyn_cast<ConstantSDNode>(Val: Ops[0]);
8104 auto *C2 = dyn_cast<ConstantSDNode>(Val: Ops[1]);
8105 auto *C3 = dyn_cast<ConstantSDNode>(Val: Ops[2]);
8106
8107 if (C1 && C2 && C3) {
8108 if (C1->isOpaque() || C2->isOpaque() || C3->isOpaque())
8109 return SDValue();
8110 const APInt &V1 = C1->getAPIntValue(), &V2 = C2->getAPIntValue(),
8111 &V3 = C3->getAPIntValue();
8112
8113 APInt FoldedVal = Opcode == ISD::FSHL ? APIntOps::fshl(Hi: V1, Lo: V2, Shift: V3)
8114 : APIntOps::fshr(Hi: V1, Lo: V2, Shift: V3);
8115 return getConstant(Val: FoldedVal, DL, VT);
8116 }
8117 }
8118
8119 // Handle fma/fmad special cases.
8120 if (Opcode == ISD::FMA || Opcode == ISD::FMAD || Opcode == ISD::FMULADD) {
8121 assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
8122 assert(Ops[0].getValueType() == VT && Ops[1].getValueType() == VT &&
8123 Ops[2].getValueType() == VT && "FMA types must match!");
8124 ConstantFPSDNode *C1 = dyn_cast<ConstantFPSDNode>(Val: Ops[0]);
8125 ConstantFPSDNode *C2 = dyn_cast<ConstantFPSDNode>(Val: Ops[1]);
8126 ConstantFPSDNode *C3 = dyn_cast<ConstantFPSDNode>(Val: Ops[2]);
8127 if (C1 && C2 && C3) {
8128 APFloat V1 = C1->getValueAPF();
8129 const APFloat &V2 = C2->getValueAPF();
8130 const APFloat &V3 = C3->getValueAPF();
8131 if (Opcode == ISD::FMAD || Opcode == ISD::FMULADD) {
8132 V1.multiply(RHS: V2, RM: APFloat::rmNearestTiesToEven);
8133 V1.add(RHS: V3, RM: APFloat::rmNearestTiesToEven);
8134 } else
8135 V1.fusedMultiplyAdd(Multiplicand: V2, Addend: V3, RM: APFloat::rmNearestTiesToEven);
8136 return getConstantFP(V: V1, DL, VT);
8137 }
8138 }
8139
8140 // This is for vector folding only from here on.
8141 if (!VT.isVector())
8142 return SDValue();
8143
8144 // Constant fold integer partial reductions with constant BUILD_VECTOR
8145 // operands. The reduction order is deliberately unspecified. Use the same
8146 // subvector layout as TargetLowering::expandPartialReduceMLA(), where input
8147 // lane I contributes to accumulator lane I % NumAccElts.
8148 if (Opcode == ISD::PARTIAL_REDUCE_SMLA ||
8149 Opcode == ISD::PARTIAL_REDUCE_UMLA ||
8150 Opcode == ISD::PARTIAL_REDUCE_SUMLA) {
8151 // These nodes have no scalar form, so unsupported cases must not fall
8152 // through to generic per-lane vector folding.
8153 if (!llvm::all_of(Range&: Ops, P: [](SDValue Op) {
8154 return ISD::isBuildVectorOfConstantSDNodes(V: Op);
8155 }))
8156 return SDValue();
8157
8158 unsigned AccEltBits = VT.getScalarSizeInBits();
8159 unsigned InputEltBits = Ops[1].getScalarValueSizeInBits();
8160 unsigned NumAccElts = VT.getVectorNumElements();
8161 unsigned NumInputElts = Ops[1].getValueType().getVectorNumElements();
8162 SmallVector<APInt, 8> Results(NumAccElts, APInt::getZero(numBits: AccEltBits));
8163 BitVector PoisonElts(NumAccElts);
8164
8165 for (unsigned I = 0; I != NumAccElts; ++I) {
8166 SDValue Elt = Ops[0].getOperand(i: I);
8167 if (Elt.getOpcode() == ISD::POISON) {
8168 PoisonElts.set(I);
8169 continue;
8170 }
8171 auto *C = dyn_cast<ConstantSDNode>(Val&: Elt);
8172 if (!C || C->isOpaque())
8173 return SDValue();
8174 Results[I] = C->getAPIntValue().trunc(width: AccEltBits);
8175 }
8176
8177 bool IsLHSSigned = Opcode != ISD::PARTIAL_REDUCE_UMLA;
8178 bool IsRHSSigned = Opcode == ISD::PARTIAL_REDUCE_SMLA;
8179 for (unsigned I = 0; I != NumInputElts; ++I) {
8180 const unsigned AccIdx = I % NumAccElts;
8181 SDValue LHSElt = Ops[1].getOperand(i: I);
8182 SDValue RHSElt = Ops[2].getOperand(i: I);
8183 if (LHSElt.getOpcode() == ISD::POISON ||
8184 RHSElt.getOpcode() == ISD::POISON) {
8185 PoisonElts.set(AccIdx);
8186 continue;
8187 }
8188
8189 auto *LHS = dyn_cast<ConstantSDNode>(Val&: LHSElt);
8190 auto *RHS = dyn_cast<ConstantSDNode>(Val&: RHSElt);
8191 if (!LHS || !RHS || LHS->isOpaque() || RHS->isOpaque())
8192 return SDValue();
8193
8194 APInt LHSVal = LHS->getAPIntValue().trunc(width: InputEltBits);
8195 APInt RHSVal = RHS->getAPIntValue().trunc(width: InputEltBits);
8196 LHSVal = IsLHSSigned ? LHSVal.sext(width: AccEltBits) : LHSVal.zext(width: AccEltBits);
8197 RHSVal = IsRHSSigned ? RHSVal.sext(width: AccEltBits) : RHSVal.zext(width: AccEltBits);
8198 Results[AccIdx] += LHSVal * RHSVal;
8199 }
8200
8201 // After type legalization the vector element type may not be a legal
8202 // scalar type (e.g. i16 on AArch64). Create the folded constants in the
8203 // promoted legal scalar type instead, matching the generic per-lane path
8204 // below. Bail out if legalization would narrow the type, since the lane
8205 // value would not fit.
8206 EVT AccEltVT = VT.getVectorElementType();
8207 EVT LegalSVT = AccEltVT;
8208 if (NewNodesMustHaveLegalTypes && LegalSVT.isInteger()) {
8209 LegalSVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT: LegalSVT);
8210 if (LegalSVT.bitsLT(VT: AccEltVT))
8211 return SDValue();
8212 }
8213
8214 SmallVector<SDValue, 8> ResultOps;
8215 for (unsigned I = 0; I != NumAccElts; ++I)
8216 ResultOps.push_back(
8217 Elt: PoisonElts[I] ? getPOISON(VT: LegalSVT)
8218 : getConstant(Val: Results[I].sext(width: LegalSVT.getSizeInBits()),
8219 DL, VT: LegalSVT));
8220 return getBuildVector(VT, DL, Ops: ResultOps);
8221 }
8222
8223 ElementCount NumElts = VT.getVectorElementCount();
8224
8225 // See if we can fold through any bitcasted integer ops.
8226 if (NumOps == 2 && VT.isFixedLengthVector() && VT.isInteger() &&
8227 Ops[0].getValueType() == VT && Ops[1].getValueType() == VT &&
8228 (Ops[0].getOpcode() == ISD::BITCAST ||
8229 Ops[1].getOpcode() == ISD::BITCAST)) {
8230 SDValue N1 = peekThroughBitcasts(V: Ops[0]);
8231 SDValue N2 = peekThroughBitcasts(V: Ops[1]);
8232 auto *BV1 = dyn_cast<BuildVectorSDNode>(Val&: N1);
8233 auto *BV2 = dyn_cast<BuildVectorSDNode>(Val&: N2);
8234 if (BV1 && BV2 && N1.getValueType().isInteger() &&
8235 N2.getValueType().isInteger()) {
8236 bool IsLE = getDataLayout().isLittleEndian();
8237 unsigned EltBits = VT.getScalarSizeInBits();
8238 SmallVector<APInt> RawBits1, RawBits2;
8239 BitVector UndefElts1, UndefElts2;
8240 if (BV1->getConstantRawBits(IsLittleEndian: IsLE, DstEltSizeInBits: EltBits, RawBitElements&: RawBits1, UndefElements&: UndefElts1) &&
8241 BV2->getConstantRawBits(IsLittleEndian: IsLE, DstEltSizeInBits: EltBits, RawBitElements&: RawBits2, UndefElements&: UndefElts2)) {
8242 SmallVector<APInt> RawBits;
8243 for (unsigned I = 0, E = NumElts.getFixedValue(); I != E; ++I) {
8244 std::optional<APInt> Fold = FoldValueWithUndef(
8245 Opcode, C1: RawBits1[I], IsUndef1: UndefElts1[I], C2: RawBits2[I], IsUndef2: UndefElts2[I]);
8246 if (!Fold)
8247 break;
8248 RawBits.push_back(Elt: *Fold);
8249 }
8250 if (RawBits.size() == NumElts.getFixedValue()) {
8251 // We have constant folded, but we might need to cast this again back
8252 // to the original (possibly legalized) type.
8253 EVT BVVT, BVEltVT;
8254 if (N1.getValueType() == VT) {
8255 BVVT = N1.getValueType();
8256 BVEltVT = BV1->getOperand(Num: 0).getValueType();
8257 } else {
8258 BVVT = N2.getValueType();
8259 BVEltVT = BV2->getOperand(Num: 0).getValueType();
8260 }
8261 unsigned BVEltBits = BVEltVT.getSizeInBits();
8262 SmallVector<APInt> DstBits;
8263 BitVector DstUndefs;
8264 BuildVectorSDNode::recastRawBits(IsLittleEndian: IsLE, DstEltSizeInBits: BVVT.getScalarSizeInBits(),
8265 DstBitElements&: DstBits, SrcBitElements: RawBits, DstUndefElements&: DstUndefs,
8266 SrcUndefElements: BitVector(RawBits.size(), false));
8267 SmallVector<SDValue> Ops(DstBits.size(), getUNDEF(VT: BVEltVT));
8268 for (unsigned I = 0, E = DstBits.size(); I != E; ++I) {
8269 if (DstUndefs[I])
8270 continue;
8271 Ops[I] = getConstant(Val: DstBits[I].sext(width: BVEltBits), DL, VT: BVEltVT);
8272 }
8273 return getBitcast(VT, V: getBuildVector(VT: BVVT, DL, Ops));
8274 }
8275 }
8276 }
8277 // Logic ops can be folded from raw integer bits - mainly for AVX512 masks.
8278 if (ISD::isBitwiseLogicOp(Opcode) && isa<ConstantSDNode>(Val: N1) &&
8279 isa<ConstantSDNode>(Val: N2)) {
8280 if (SDValue Res = FoldConstantArithmetic(Opcode, DL, VT: N1.getValueType(),
8281 Ops: {N1, N2}, Flags))
8282 return getBitcast(VT, V: Res);
8283 }
8284 }
8285
8286 // Fold (mul step_vector(C0), C1) to (step_vector(C0 * C1)).
8287 // (shl step_vector(C0), C1) -> (step_vector(C0 << C1))
8288 if ((Opcode == ISD::MUL || Opcode == ISD::SHL) &&
8289 Ops[0].getOpcode() == ISD::STEP_VECTOR) {
8290 APInt RHSVal;
8291 if (ISD::isConstantSplatVector(V: Ops[1], SplatVal&: RHSVal)) {
8292 APInt NewStep = Opcode == ISD::MUL
8293 ? Ops[0].getConstantOperandAPInt(i: 0) * RHSVal
8294 : Ops[0].getConstantOperandAPInt(i: 0) << RHSVal;
8295 return getStepVector(DL, ResVT: VT, StepVal: NewStep);
8296 }
8297 }
8298
8299 auto IsScalarOrSameVectorSize = [NumElts](const SDValue &Op) {
8300 return !Op.getValueType().isVector() ||
8301 Op.getValueType().getVectorElementCount() == NumElts;
8302 };
8303
8304 auto IsBuildVectorSplatVectorOrUndef = [](const SDValue &Op) {
8305 return Op.isUndef() || Op.getOpcode() == ISD::CONDCODE ||
8306 Op.getOpcode() == ISD::BUILD_VECTOR ||
8307 Op.getOpcode() == ISD::SPLAT_VECTOR;
8308 };
8309
8310 // All operands must be vector types with the same number of elements as
8311 // the result type and must be either UNDEF or a build/splat vector
8312 // or UNDEF scalars.
8313 if (!llvm::all_of(Range&: Ops, P: IsBuildVectorSplatVectorOrUndef) ||
8314 !llvm::all_of(Range&: Ops, P: IsScalarOrSameVectorSize))
8315 return SDValue();
8316
8317 // If we are comparing vectors, then the result needs to be a i1 boolean that
8318 // is then extended back to the legal result type depending on how booleans
8319 // are represented.
8320 EVT SVT = (Opcode == ISD::SETCC ? MVT::i1 : VT.getScalarType());
8321 ISD::NodeType ExtendCode =
8322 (Opcode == ISD::SETCC && SVT != VT.getScalarType())
8323 ? TargetLowering::getExtendForContent(Content: TLI->getBooleanContents(Type: VT))
8324 : ISD::SIGN_EXTEND;
8325
8326 // Find legal integer scalar type for constant promotion and
8327 // ensure that its scalar size is at least as large as source.
8328 EVT LegalSVT = VT.getScalarType();
8329 if (NewNodesMustHaveLegalTypes && LegalSVT.isInteger()) {
8330 LegalSVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT: LegalSVT);
8331 if (LegalSVT.bitsLT(VT: VT.getScalarType()))
8332 return SDValue();
8333 }
8334
8335 // For scalable vector types we know we're dealing with SPLAT_VECTORs. We
8336 // only have one operand to check. For fixed-length vector types we may have
8337 // a combination of BUILD_VECTOR and SPLAT_VECTOR.
8338 unsigned NumVectorElts = NumElts.isScalable() ? 1 : NumElts.getFixedValue();
8339
8340 // Constant fold each scalar lane separately.
8341 SmallVector<SDValue, 4> ScalarResults;
8342 for (unsigned I = 0; I != NumVectorElts; I++) {
8343 SmallVector<SDValue, 4> ScalarOps;
8344 for (SDValue Op : Ops) {
8345 EVT InSVT = Op.getValueType().getScalarType();
8346 if (Op.getOpcode() != ISD::BUILD_VECTOR &&
8347 Op.getOpcode() != ISD::SPLAT_VECTOR) {
8348 if (Op.isUndef())
8349 ScalarOps.push_back(Elt: getUNDEF(VT: InSVT));
8350 else
8351 ScalarOps.push_back(Elt: Op);
8352 continue;
8353 }
8354
8355 SDValue ScalarOp =
8356 Op.getOperand(i: Op.getOpcode() == ISD::SPLAT_VECTOR ? 0 : I);
8357 EVT ScalarVT = ScalarOp.getValueType();
8358
8359 // Build vector (integer) scalar operands may need implicit
8360 // truncation - do this before constant folding.
8361 if (ScalarVT.isInteger() && ScalarVT.bitsGT(VT: InSVT)) {
8362 // Don't create illegally-typed nodes unless they're constants or undef
8363 // - if we fail to constant fold we can't guarantee the (dead) nodes
8364 // we're creating will be cleaned up before being visited for
8365 // legalization.
8366 if (NewNodesMustHaveLegalTypes && !ScalarOp.isUndef() &&
8367 !isa<ConstantSDNode>(Val: ScalarOp) &&
8368 TLI->getTypeAction(Context&: *getContext(), VT: InSVT) !=
8369 TargetLowering::TypeLegal)
8370 return SDValue();
8371 ScalarOp = getNode(Opcode: ISD::TRUNCATE, DL, VT: InSVT, N1: ScalarOp);
8372 }
8373
8374 ScalarOps.push_back(Elt: ScalarOp);
8375 }
8376
8377 // Constant fold the scalar operands.
8378 SDValue ScalarResult = getNode(Opcode, DL, VT: SVT, Ops: ScalarOps, Flags);
8379
8380 // Scalar folding only succeeded if the result is a constant or UNDEF.
8381 if (!ScalarResult.isUndef() && ScalarResult.getOpcode() != ISD::Constant &&
8382 ScalarResult.getOpcode() != ISD::ConstantFP)
8383 return SDValue();
8384
8385 // Legalize the (integer) scalar constant if necessary. We only do
8386 // this once we know the folding succeeded, since otherwise we would
8387 // get a node with illegal type which has a user.
8388 if (LegalSVT != SVT)
8389 ScalarResult = getNode(Opcode: ExtendCode, DL, VT: LegalSVT, N1: ScalarResult);
8390
8391 ScalarResults.push_back(Elt: ScalarResult);
8392 }
8393
8394 SDValue V = NumElts.isScalable() ? getSplatVector(VT, DL, Op: ScalarResults[0])
8395 : getBuildVector(VT, DL, Ops: ScalarResults);
8396 NewSDValueDbgMsg(V, Msg: "New node fold constant vector: ", G: this);
8397 return V;
8398}
8399
8400SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL,
8401 SDVTList VTList,
8402 ArrayRef<SDValue> Ops,
8403 SDNodeFlags Flags) {
8404 if (VTList.NumVTs == 1)
8405 return FoldConstantArithmetic(Opcode, DL, VT: VTList.VTs[0], Ops, Flags);
8406
8407 switch (Opcode) {
8408 case ISD::SADDO:
8409 case ISD::UADDO:
8410 case ISD::SSUBO:
8411 case ISD::USUBO:
8412 case ISD::SMULO:
8413 case ISD::UMULO: {
8414 // TODO: Support non-splat vector constants.
8415 ConstantSDNode *C1 = isConstOrConstSplat(N: Ops[0]);
8416 ConstantSDNode *C2 = isConstOrConstSplat(N: Ops[1]);
8417 if (!C1 || !C2 || C1->isOpaque() || C2->isOpaque())
8418 return SDValue();
8419
8420 const APInt &V1 = C1->getAPIntValue(), &V2 = C2->getAPIntValue();
8421 bool Overflow;
8422 APInt Result;
8423 switch (Opcode) {
8424 default:
8425 llvm_unreachable("Unexpected overflow opcode");
8426 case ISD::SADDO:
8427 Result = V1.sadd_ov(RHS: V2, Overflow);
8428 break;
8429 case ISD::UADDO:
8430 Result = V1.uadd_ov(RHS: V2, Overflow);
8431 break;
8432 case ISD::SSUBO:
8433 Result = V1.ssub_ov(RHS: V2, Overflow);
8434 break;
8435 case ISD::USUBO:
8436 Result = V1.usub_ov(RHS: V2, Overflow);
8437 break;
8438 case ISD::SMULO:
8439 Result = V1.smul_ov(RHS: V2, Overflow);
8440 break;
8441 case ISD::UMULO:
8442 Result = V1.umul_ov(RHS: V2, Overflow);
8443 break;
8444 }
8445
8446 EVT VT = VTList.VTs[0];
8447 return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList,
8448 Ops: {getConstant(Val: Result, DL, VT),
8449 getBoolConstant(V: Overflow, DL, VT: VTList.VTs[1], OpVT: VT)},
8450 Flags);
8451 }
8452 case ISD::SMUL_LOHI:
8453 case ISD::UMUL_LOHI:
8454 // TODO: Support non-splat vector constants.
8455 ConstantSDNode *C1 = isConstOrConstSplat(N: Ops[0]);
8456 ConstantSDNode *C2 = isConstOrConstSplat(N: Ops[1]);
8457 if (!C1 || !C2 || C1->isOpaque() || C2->isOpaque())
8458 return SDValue();
8459
8460 const APInt &V1 = C1->getAPIntValue(), &V2 = C2->getAPIntValue();
8461 APInt Lo = V1 * V2;
8462 APInt Hi = (Opcode == ISD::SMUL_LOHI) ? APIntOps::mulhs(C1: V1, C2: V2)
8463 : APIntOps::mulhu(C1: V1, C2: V2);
8464 return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList,
8465 Ops: {getConstant(Val: Lo, DL, VT: VTList.VTs[0]),
8466 getConstant(Val: Hi, DL, VT: VTList.VTs[1])},
8467 Flags);
8468 }
8469 return SDValue();
8470}
8471
8472SDValue SelectionDAG::foldConstantFPMath(unsigned Opcode, const SDLoc &DL,
8473 EVT VT, ArrayRef<SDValue> Ops) {
8474 // TODO: Add support for unary/ternary fp opcodes.
8475 if (Ops.size() != 2)
8476 return SDValue();
8477
8478 // TODO: We don't do any constant folding for strict FP opcodes here, but we
8479 // should. That will require dealing with a potentially non-default
8480 // rounding mode, checking the "opStatus" return value from the APFloat
8481 // math calculations, and possibly other variations.
8482 SDValue N1 = Ops[0];
8483 SDValue N2 = Ops[1];
8484 ConstantFPSDNode *N1CFP = isConstOrConstSplatFP(N: N1, /*AllowUndefs*/ false);
8485 ConstantFPSDNode *N2CFP = isConstOrConstSplatFP(N: N2, /*AllowUndefs*/ false);
8486 if (N1CFP && N2CFP) {
8487 APFloat C1 = N1CFP->getValueAPF(); // make copy
8488 const APFloat &C2 = N2CFP->getValueAPF();
8489 switch (Opcode) {
8490 case ISD::FADD:
8491 C1.add(RHS: C2, RM: APFloat::rmNearestTiesToEven);
8492 return getConstantFP(V: C1, DL, VT);
8493 case ISD::FSUB:
8494 C1.subtract(RHS: C2, RM: APFloat::rmNearestTiesToEven);
8495 return getConstantFP(V: C1, DL, VT);
8496 case ISD::FMUL:
8497 C1.multiply(RHS: C2, RM: APFloat::rmNearestTiesToEven);
8498 return getConstantFP(V: C1, DL, VT);
8499 case ISD::FDIV:
8500 C1.divide(RHS: C2, RM: APFloat::rmNearestTiesToEven);
8501 return getConstantFP(V: C1, DL, VT);
8502 case ISD::FREM:
8503 C1.mod(RHS: C2);
8504 return getConstantFP(V: C1, DL, VT);
8505 case ISD::FCOPYSIGN:
8506 C1.copySign(RHS: C2);
8507 return getConstantFP(V: C1, DL, VT);
8508 case ISD::FMINNUM:
8509 return getConstantFP(V: minnum(A: C1, B: C2), DL, VT);
8510 case ISD::FMAXNUM:
8511 return getConstantFP(V: maxnum(A: C1, B: C2), DL, VT);
8512 case ISD::FMINIMUM:
8513 return getConstantFP(V: minimum(A: C1, B: C2), DL, VT);
8514 case ISD::FMAXIMUM:
8515 return getConstantFP(V: maximum(A: C1, B: C2), DL, VT);
8516 case ISD::FMINIMUMNUM:
8517 return getConstantFP(V: minimumnum(A: C1, B: C2), DL, VT);
8518 case ISD::FMAXIMUMNUM:
8519 return getConstantFP(V: maximumnum(A: C1, B: C2), DL, VT);
8520 default: break;
8521 }
8522 }
8523 if (N1CFP && Opcode == ISD::FP_ROUND) {
8524 APFloat C1 = N1CFP->getValueAPF(); // make copy
8525 bool Unused;
8526 // This can return overflow, underflow, or inexact; we don't care.
8527 // FIXME need to be more flexible about rounding mode.
8528 (void)C1.convert(ToSemantics: VT.getFltSemantics(), RM: APFloat::rmNearestTiesToEven,
8529 losesInfo: &Unused);
8530 return getConstantFP(V: C1, DL, VT);
8531 }
8532
8533 switch (Opcode) {
8534 case ISD::FSUB:
8535 // -0.0 - undef --> undef (consistent with "fneg undef")
8536 if (ConstantFPSDNode *N1C = isConstOrConstSplatFP(N: N1, /*AllowUndefs*/ true))
8537 if (N1C && N1C->getValueAPF().isNegZero() && N2.isUndef())
8538 return getUNDEF(VT);
8539 [[fallthrough]];
8540
8541 case ISD::FADD:
8542 case ISD::FMUL:
8543 case ISD::FDIV:
8544 case ISD::FREM:
8545 // If both operands are undef, the result is undef. If 1 operand is undef,
8546 // the result is NaN. This should match the behavior of the IR optimizer.
8547 if (N1.isUndef() && N2.isUndef())
8548 return getUNDEF(VT);
8549 if (N1.isUndef() || N2.isUndef())
8550 return getConstantFP(V: APFloat::getNaN(Sem: VT.getFltSemantics()), DL, VT);
8551 }
8552 return SDValue();
8553}
8554
8555SDValue SelectionDAG::FoldConstantBuildVector(BuildVectorSDNode *BV,
8556 const SDLoc &DL, EVT DstEltVT) {
8557 EVT SrcEltVT = BV->getValueType(ResNo: 0).getVectorElementType();
8558
8559 // If this is already the right type, we're done.
8560 if (SrcEltVT == DstEltVT)
8561 return SDValue(BV, 0);
8562
8563 unsigned SrcBitSize = SrcEltVT.getSizeInBits();
8564 unsigned DstBitSize = DstEltVT.getSizeInBits();
8565
8566 // If this is a conversion of N elements of one type to N elements of another
8567 // type, convert each element. This handles FP<->INT cases.
8568 if (SrcBitSize == DstBitSize) {
8569 SmallVector<SDValue, 8> Ops;
8570 for (SDValue Op : BV->op_values()) {
8571 // If the vector element type is not legal, the BUILD_VECTOR operands
8572 // are promoted and implicitly truncated. Make that explicit here.
8573 if (Op.getValueType() != SrcEltVT)
8574 Op = getNode(Opcode: ISD::TRUNCATE, DL, VT: SrcEltVT, N1: Op);
8575 Ops.push_back(Elt: getBitcast(VT: DstEltVT, V: Op));
8576 }
8577 EVT VT = EVT::getVectorVT(Context&: *getContext(), VT: DstEltVT,
8578 NumElements: BV->getValueType(ResNo: 0).getVectorNumElements());
8579 return getBuildVector(VT, DL, Ops);
8580 }
8581
8582 // Otherwise, we're growing or shrinking the elements. To avoid having to
8583 // handle annoying details of growing/shrinking FP values, we convert them to
8584 // int first.
8585 if (SrcEltVT.isFloatingPoint()) {
8586 // Convert the input float vector to a int vector where the elements are the
8587 // same sizes.
8588 EVT IntEltVT = EVT::getIntegerVT(Context&: *getContext(), BitWidth: SrcEltVT.getSizeInBits());
8589 if (SDValue Tmp = FoldConstantBuildVector(BV, DL, DstEltVT: IntEltVT))
8590 return FoldConstantBuildVector(BV: cast<BuildVectorSDNode>(Val&: Tmp), DL,
8591 DstEltVT);
8592 return SDValue();
8593 }
8594
8595 // Now we know the input is an integer vector. If the output is a FP type,
8596 // convert to integer first, then to FP of the right size.
8597 if (DstEltVT.isFloatingPoint()) {
8598 EVT IntEltVT = EVT::getIntegerVT(Context&: *getContext(), BitWidth: DstEltVT.getSizeInBits());
8599 if (SDValue Tmp = FoldConstantBuildVector(BV, DL, DstEltVT: IntEltVT))
8600 return FoldConstantBuildVector(BV: cast<BuildVectorSDNode>(Val&: Tmp), DL,
8601 DstEltVT);
8602 return SDValue();
8603 }
8604
8605 // Okay, we know the src/dst types are both integers of differing types.
8606 assert(SrcEltVT.isInteger() && DstEltVT.isInteger());
8607
8608 // Extract the constant raw bit data.
8609 BitVector UndefElements;
8610 SmallVector<APInt> RawBits;
8611 bool IsLE = getDataLayout().isLittleEndian();
8612 if (!BV->getConstantRawBits(IsLittleEndian: IsLE, DstEltSizeInBits: DstBitSize, RawBitElements&: RawBits, UndefElements))
8613 return SDValue();
8614
8615 SmallVector<SDValue, 8> Ops;
8616 for (unsigned I = 0, E = RawBits.size(); I != E; ++I) {
8617 if (UndefElements[I])
8618 Ops.push_back(Elt: getUNDEF(VT: DstEltVT));
8619 else
8620 Ops.push_back(Elt: getConstant(Val: RawBits[I], DL, VT: DstEltVT));
8621 }
8622
8623 EVT VT = EVT::getVectorVT(Context&: *getContext(), VT: DstEltVT, NumElements: Ops.size());
8624 return getBuildVector(VT, DL, Ops);
8625}
8626
8627SDValue SelectionDAG::getAssertAlign(const SDLoc &DL, SDValue Val, Align A) {
8628 assert(Val.getValueType().isInteger() && "Invalid AssertAlign!");
8629
8630 // There's no need to assert on a byte-aligned pointer. All pointers are at
8631 // least byte aligned.
8632 if (A == Align(1))
8633 return Val;
8634
8635 SDVTList VTs = getVTList(VT: Val.getValueType());
8636 SDValue Ops[] = {Val};
8637 SDNodeKey ID(ISD::AssertAlign, VTs, Ops);
8638 ID.AddInteger(I: A.value());
8639
8640 FoldingSetInsertToken InsertToken;
8641 if (SDNode *E = lookupNode(Key: ID, DL, InsertToken))
8642 return SDValue(E, 0);
8643
8644 auto *N =
8645 newSDNode<AssertAlignSDNode>(Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs, Args&: A);
8646 createOperands(Node: N, Vals: {Val});
8647
8648 CSEMap.insert(N, Token: InsertToken);
8649 InsertNode(N);
8650
8651 SDValue V(N, 0);
8652 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
8653 return V;
8654}
8655
8656SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
8657 SDValue N1, SDValue N2) {
8658 SDNodeFlags Flags;
8659 if (Inserter)
8660 Flags = Inserter->getFlags();
8661 return getNode(Opcode, DL, VT, N1, N2, Flags);
8662}
8663
8664void SelectionDAG::canonicalizeCommutativeBinop(unsigned Opcode, SDValue &N1,
8665 SDValue &N2) const {
8666 if (!TLI->isCommutativeBinOp(Opcode))
8667 return;
8668
8669 // Canonicalize:
8670 // binop(const, nonconst) -> binop(nonconst, const)
8671 bool N1C = isConstantIntBuildVectorOrConstantInt(N: N1);
8672 bool N2C = isConstantIntBuildVectorOrConstantInt(N: N2);
8673 bool N1CFP = isConstantFPBuildVectorOrConstantFP(N: N1);
8674 bool N2CFP = isConstantFPBuildVectorOrConstantFP(N: N2);
8675 if ((N1C && !N2C) || (N1CFP && !N2CFP))
8676 std::swap(a&: N1, b&: N2);
8677
8678 // Canonicalize:
8679 // binop(splat(x), step_vector) -> binop(step_vector, splat(x))
8680 else if (N1.getOpcode() == ISD::SPLAT_VECTOR &&
8681 N2.getOpcode() == ISD::STEP_VECTOR)
8682 std::swap(a&: N1, b&: N2);
8683}
8684
8685SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
8686 SDValue N1, SDValue N2, const SDNodeFlags Flags) {
8687 assert(N1.getOpcode() != ISD::DELETED_NODE &&
8688 N2.getOpcode() != ISD::DELETED_NODE &&
8689 "Operand is DELETED_NODE!");
8690
8691 canonicalizeCommutativeBinop(Opcode, N1, N2);
8692
8693 auto *N1C = dyn_cast<ConstantSDNode>(Val&: N1);
8694 auto *N2C = dyn_cast<ConstantSDNode>(Val&: N2);
8695
8696 // Don't allow undefs in vector splats - we might be returning N2 when folding
8697 // to zero etc.
8698 ConstantSDNode *N2CV =
8699 isConstOrConstSplat(N: N2, /*AllowUndefs*/ false, /*AllowTruncation*/ true);
8700
8701 switch (Opcode) {
8702 default: break;
8703 case ISD::TokenFactor:
8704 assert(VT == MVT::Other && N1.getValueType() == MVT::Other &&
8705 N2.getValueType() == MVT::Other && "Invalid token factor!");
8706 // Fold trivial token factors.
8707 if (N1.getOpcode() == ISD::EntryToken) return N2;
8708 if (N2.getOpcode() == ISD::EntryToken) return N1;
8709 if (N1 == N2) return N1;
8710 break;
8711 case ISD::BUILD_VECTOR: {
8712 // Attempt to simplify BUILD_VECTOR.
8713 SDValue Ops[] = {N1, N2};
8714 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, DAG&: *this))
8715 return V;
8716 break;
8717 }
8718 case ISD::CONCAT_VECTORS: {
8719 SDValue Ops[] = {N1, N2};
8720 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, DAG&: *this))
8721 return V;
8722 break;
8723 }
8724 case ISD::AND:
8725 assert(VT.isInteger() && "This operator does not apply to FP types!");
8726 assert(N1.getValueType() == N2.getValueType() &&
8727 N1.getValueType() == VT && "Binary operator types must match!");
8728 // (X & 0) -> 0. This commonly occurs when legalizing i64 values, so it's
8729 // worth handling here.
8730 if (N2CV && N2CV->isZero())
8731 return N2;
8732 if (N2CV && N2CV->isAllOnes()) // X & -1 -> X
8733 return N1;
8734 break;
8735 case ISD::OR:
8736 case ISD::XOR:
8737 case ISD::ADD:
8738 case ISD::PTRADD:
8739 case ISD::SUB:
8740 assert(VT.isInteger() && "This operator does not apply to FP types!");
8741 assert(N1.getValueType() == N2.getValueType() &&
8742 N1.getValueType() == VT && "Binary operator types must match!");
8743 // The equal operand types requirement is unnecessarily strong for PTRADD.
8744 // However, the SelectionDAGBuilder does not generate PTRADDs with different
8745 // operand types, and we'd need to re-implement GEP's non-standard wrapping
8746 // logic everywhere where PTRADDs may be folded or combined to properly
8747 // support them. If/when we introduce pointer types to the SDAG, we will
8748 // need to relax this constraint.
8749
8750 // (X ^|+- 0) -> X. This commonly occurs when legalizing i64 values, so
8751 // it's worth handling here.
8752 if (N2CV && N2CV->isZero())
8753 return N1;
8754 if ((Opcode == ISD::ADD || Opcode == ISD::SUB) &&
8755 VT.getScalarType() == MVT::i1)
8756 return getNode(Opcode: ISD::XOR, DL, VT, N1, N2);
8757 // Fold (add (vscale * C0), (vscale * C1)) to (vscale * (C0 + C1)).
8758 if (Opcode == ISD::ADD && N1.getOpcode() == ISD::VSCALE &&
8759 N2.getOpcode() == ISD::VSCALE) {
8760 const APInt &C1 = N1->getConstantOperandAPInt(Num: 0);
8761 const APInt &C2 = N2->getConstantOperandAPInt(Num: 0);
8762 return getVScale(DL, VT, MulImm: C1 + C2);
8763 }
8764 break;
8765 case ISD::MUL:
8766 assert(VT.isInteger() && "This operator does not apply to FP types!");
8767 assert(N1.getValueType() == N2.getValueType() &&
8768 N1.getValueType() == VT && "Binary operator types must match!");
8769 if (VT.getScalarType() == MVT::i1)
8770 return getNode(Opcode: ISD::AND, DL, VT, N1, N2);
8771 if (N2CV && N2CV->isZero())
8772 return N2;
8773 if (N2C && (N1.getOpcode() == ISD::VSCALE) && Flags.hasNoSignedWrap()) {
8774 const APInt &MulImm = N1->getConstantOperandAPInt(Num: 0);
8775 const APInt &N2CImm = N2C->getAPIntValue();
8776 return getVScale(DL, VT, MulImm: MulImm * N2CImm);
8777 }
8778 break;
8779 case ISD::UDIV:
8780 case ISD::UREM:
8781 case ISD::MULHU:
8782 case ISD::MULHS:
8783 case ISD::SDIV:
8784 case ISD::SREM:
8785 case ISD::SADDSAT:
8786 case ISD::SSUBSAT:
8787 case ISD::UADDSAT:
8788 case ISD::USUBSAT:
8789 assert(VT.isInteger() && "This operator does not apply to FP types!");
8790 assert(N1.getValueType() == N2.getValueType() &&
8791 N1.getValueType() == VT && "Binary operator types must match!");
8792 if (VT.getScalarType() == MVT::i1) {
8793 // fold (add_sat x, y) -> (or x, y) for bool types.
8794 if (Opcode == ISD::SADDSAT || Opcode == ISD::UADDSAT)
8795 return getNode(Opcode: ISD::OR, DL, VT, N1, N2);
8796 // fold (sub_sat x, y) -> (and x, ~y) for bool types.
8797 if (Opcode == ISD::SSUBSAT || Opcode == ISD::USUBSAT)
8798 return getNode(Opcode: ISD::AND, DL, VT, N1, N2: getNOT(DL, Val: N2, VT));
8799 // mulhs/mulhu on i1 are always 0: the high bit of the 2-bit product of
8800 // two booleans is never set. The i1 constant 1 is also -1, so folding
8801 // (mulhs x, 1) to (sra x, 0) in DAGCombiner would be wrong here.
8802 if (Opcode == ISD::MULHS || Opcode == ISD::MULHU)
8803 return getConstant(Val: 0, DL, VT);
8804 }
8805 break;
8806 case ISD::SCMP:
8807 case ISD::UCMP:
8808 assert(N1.getValueType() == N2.getValueType() &&
8809 "Types of operands of UCMP/SCMP must match");
8810 assert(N1.getValueType().isVector() == VT.isVector() &&
8811 "Operands and return type of must both be scalars or vectors");
8812 if (VT.isVector())
8813 assert(VT.getVectorElementCount() ==
8814 N1.getValueType().getVectorElementCount() &&
8815 "Result and operands must have the same number of elements");
8816 break;
8817 case ISD::AVGFLOORS:
8818 case ISD::AVGFLOORU:
8819 case ISD::AVGCEILS:
8820 case ISD::AVGCEILU:
8821 case ISD::AVGSUBFLOORS:
8822 case ISD::AVGSUBFLOORU:
8823 assert(VT.isInteger() && "This operator does not apply to FP types!");
8824 assert(N1.getValueType() == N2.getValueType() &&
8825 N1.getValueType() == VT && "Binary operator types must match!");
8826 break;
8827 case ISD::ABDS:
8828 case ISD::ABDU:
8829 assert(VT.isInteger() && "This operator does not apply to FP types!");
8830 assert(N1.getValueType() == N2.getValueType() &&
8831 N1.getValueType() == VT && "Binary operator types must match!");
8832 if (VT.getScalarType() == MVT::i1)
8833 return getNode(Opcode: ISD::XOR, DL, VT, N1, N2);
8834 break;
8835 case ISD::SMIN:
8836 case ISD::UMAX:
8837 assert(VT.isInteger() && "This operator does not apply to FP types!");
8838 assert(N1.getValueType() == N2.getValueType() &&
8839 N1.getValueType() == VT && "Binary operator types must match!");
8840 if (VT.getScalarType() == MVT::i1)
8841 return getNode(Opcode: ISD::OR, DL, VT, N1, N2);
8842 break;
8843 case ISD::SMAX:
8844 case ISD::UMIN:
8845 assert(VT.isInteger() && "This operator does not apply to FP types!");
8846 assert(N1.getValueType() == N2.getValueType() &&
8847 N1.getValueType() == VT && "Binary operator types must match!");
8848 if (VT.getScalarType() == MVT::i1)
8849 return getNode(Opcode: ISD::AND, DL, VT, N1, N2);
8850 break;
8851 case ISD::FADD:
8852 case ISD::FSUB:
8853 case ISD::FMUL:
8854 case ISD::FDIV:
8855 case ISD::FREM:
8856 assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
8857 assert(N1.getValueType() == N2.getValueType() &&
8858 N1.getValueType() == VT && "Binary operator types must match!");
8859 if (SDValue V = simplifyFPBinop(Opcode, X: N1, Y: N2, Flags))
8860 return V;
8861 break;
8862 case ISD::FCOPYSIGN: // N1 and result must match. N1/N2 need not match.
8863 assert(N1.getValueType() == VT &&
8864 N1.getValueType().isFloatingPoint() &&
8865 N2.getValueType().isFloatingPoint() &&
8866 "Invalid FCOPYSIGN!");
8867 break;
8868 case ISD::SHL:
8869 if (N2C && (N1.getOpcode() == ISD::VSCALE) && Flags.hasNoSignedWrap()) {
8870 const APInt &MulImm = N1->getConstantOperandAPInt(Num: 0);
8871 const APInt &ShiftImm = N2C->getAPIntValue();
8872 return getVScale(DL, VT, MulImm: MulImm << ShiftImm);
8873 }
8874 [[fallthrough]];
8875 case ISD::SRA:
8876 case ISD::SRL:
8877 if (SDValue V = simplifyShift(X: N1, Y: N2))
8878 return V;
8879 [[fallthrough]];
8880 case ISD::ROTL:
8881 case ISD::ROTR:
8882 case ISD::SSHLSAT:
8883 case ISD::USHLSAT:
8884 assert(VT == N1.getValueType() &&
8885 "Shift operators return type must be the same as their first arg");
8886 assert(VT.isInteger() && N2.getValueType().isInteger() &&
8887 "Shifts only work on integers");
8888 assert((!VT.isVector() || VT == N2.getValueType()) &&
8889 "Vector shift amounts must be in the same as their first arg");
8890 // Verify that the shift amount VT is big enough to hold valid shift
8891 // amounts. This catches things like trying to shift an i1024 value by an
8892 // i8, which is easy to fall into in generic code that uses
8893 // TLI.getShiftAmount().
8894 assert(N2.getValueType().getScalarSizeInBits() >=
8895 Log2_32_Ceil(VT.getScalarSizeInBits()) &&
8896 "Invalid use of small shift amount with oversized value!");
8897
8898 // Always fold shifts of i1 values so the code generator doesn't need to
8899 // handle them. Since we know the size of the shift has to be less than the
8900 // size of the value, the shift/rotate count is guaranteed to be zero.
8901 if (VT == MVT::i1)
8902 return N1;
8903 if (N2CV && N2CV->isZero())
8904 return N1;
8905 break;
8906 case ISD::FP_ROUND:
8907 assert(VT.isFloatingPoint() && N1.getValueType().isFloatingPoint() &&
8908 VT.bitsLE(N1.getValueType()) && N2C &&
8909 (N2C->getZExtValue() == 0 || N2C->getZExtValue() == 1) &&
8910 N2.getOpcode() == ISD::TargetConstant && "Invalid FP_ROUND!");
8911 if (N1.getValueType() == VT) return N1; // noop conversion.
8912 break;
8913 case ISD::IS_FPCLASS: {
8914 assert(N1.getValueType().isFloatingPoint() &&
8915 "IS_FPCLASS is used for a non-floating type");
8916 assert(isa<ConstantSDNode>(N2) && "FPClassTest is not Constant");
8917 // is.fpclass(poison, mask) -> poison
8918 if (N1.getOpcode() == ISD::POISON)
8919 return getPOISON(VT);
8920 FPClassTest Mask = static_cast<FPClassTest>(N2->getAsZExtVal());
8921 // If all tests are made, it doesn't matter what the value is.
8922 if ((Mask & fcAllFlags) == fcAllFlags)
8923 return getBoolConstant(V: true, DL, VT, OpVT: N1.getValueType());
8924 if ((Mask & fcAllFlags) == 0)
8925 return getBoolConstant(V: false, DL, VT, OpVT: N1.getValueType());
8926 break;
8927 }
8928 case ISD::AssertNoFPClass: {
8929 assert(N1.getValueType().isFloatingPoint() &&
8930 "AssertNoFPClass is used for a non-floating type");
8931 assert(isa<ConstantSDNode>(N2) && "NoFPClass is not Constant");
8932 FPClassTest NoFPClass = static_cast<FPClassTest>(N2->getAsZExtVal());
8933 assert(llvm::to_underlying(NoFPClass) <=
8934 BitmaskEnumDetail::Mask<FPClassTest>() &&
8935 "FPClassTest value too large");
8936 (void)NoFPClass;
8937 break;
8938 }
8939 case ISD::AssertSext:
8940 case ISD::AssertZext: {
8941 EVT EVT = cast<VTSDNode>(Val&: N2)->getVT();
8942 assert(VT == N1.getValueType() && "Not an inreg extend!");
8943 assert(VT.isInteger() && EVT.isInteger() &&
8944 "Cannot *_EXTEND_INREG FP types");
8945 assert(!EVT.isVector() &&
8946 "AssertSExt/AssertZExt type should be the vector element type "
8947 "rather than the vector type!");
8948 assert(EVT.bitsLE(VT.getScalarType()) && "Not extending!");
8949 if (VT.getScalarType() == EVT) return N1; // noop assertion.
8950 break;
8951 }
8952 case ISD::SIGN_EXTEND_INREG: {
8953 EVT EVT = cast<VTSDNode>(Val&: N2)->getVT();
8954 assert(VT == N1.getValueType() && "Not an inreg extend!");
8955 assert(VT.isInteger() && EVT.isInteger() &&
8956 "Cannot *_EXTEND_INREG FP types");
8957 assert(EVT.isVector() == VT.isVector() &&
8958 "SIGN_EXTEND_INREG type should be vector iff the operand "
8959 "type is vector!");
8960 assert((!EVT.isVector() ||
8961 EVT.getVectorElementCount() == VT.getVectorElementCount()) &&
8962 "Vector element counts must match in SIGN_EXTEND_INREG");
8963 assert(EVT.getScalarType().bitsLE(VT.getScalarType()) && "Not extending!");
8964 if (EVT == VT) return N1; // Not actually extending
8965 break;
8966 }
8967 case ISD::FP_TO_SINT_SAT:
8968 case ISD::FP_TO_UINT_SAT: {
8969 assert(VT.isInteger() && cast<VTSDNode>(N2)->getVT().isInteger() &&
8970 N1.getValueType().isFloatingPoint() && "Invalid FP_TO_*INT_SAT");
8971 assert(N1.getValueType().isVector() == VT.isVector() &&
8972 "FP_TO_*INT_SAT type should be vector iff the operand type is "
8973 "vector!");
8974 assert((!VT.isVector() || VT.getVectorElementCount() ==
8975 N1.getValueType().getVectorElementCount()) &&
8976 "Vector element counts must match in FP_TO_*INT_SAT");
8977 assert(!cast<VTSDNode>(N2)->getVT().isVector() &&
8978 "Type to saturate to must be a scalar.");
8979 assert(cast<VTSDNode>(N2)->getVT().bitsLE(VT.getScalarType()) &&
8980 "Not extending!");
8981 break;
8982 }
8983 case ISD::EXTRACT_VECTOR_ELT:
8984 assert(VT.getSizeInBits() >= N1.getValueType().getScalarSizeInBits() &&
8985 "The result of EXTRACT_VECTOR_ELT must be at least as wide as the \
8986 element type of the vector.");
8987
8988 // Extract from an undefined value or using an undefined index is undefined.
8989 if (N1.isUndef() || N2.isUndef())
8990 return getUNDEF(VT);
8991
8992 // EXTRACT_VECTOR_ELT of out-of-bounds element is POISON for fixed length
8993 // vectors. For scalable vectors we will provide appropriate support for
8994 // dealing with arbitrary indices.
8995 if (N2C && N1.getValueType().isFixedLengthVector() &&
8996 N2C->getAPIntValue().uge(RHS: N1.getValueType().getVectorNumElements()))
8997 return getPOISON(VT);
8998
8999 // EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is
9000 // expanding copies of large vectors from registers. This only works for
9001 // fixed length vectors, since we need to know the exact number of
9002 // elements.
9003 if (N2C && N1.getOpcode() == ISD::CONCAT_VECTORS &&
9004 N1.getOperand(i: 0).getValueType().isFixedLengthVector()) {
9005 unsigned Factor = N1.getOperand(i: 0).getValueType().getVectorNumElements();
9006 return getExtractVectorElt(DL, VT,
9007 Vec: N1.getOperand(i: N2C->getZExtValue() / Factor),
9008 Idx: N2C->getZExtValue() % Factor);
9009 }
9010
9011 // EXTRACT_VECTOR_ELT of BUILD_VECTOR or SPLAT_VECTOR is often formed while
9012 // lowering is expanding large vector constants.
9013 if (N2C && (N1.getOpcode() == ISD::BUILD_VECTOR ||
9014 N1.getOpcode() == ISD::SPLAT_VECTOR)) {
9015 assert((N1.getOpcode() != ISD::BUILD_VECTOR ||
9016 N1.getValueType().isFixedLengthVector()) &&
9017 "BUILD_VECTOR used for scalable vectors");
9018 unsigned Index =
9019 N1.getOpcode() == ISD::BUILD_VECTOR ? N2C->getZExtValue() : 0;
9020 SDValue Elt = N1.getOperand(i: Index);
9021
9022 if (VT != Elt.getValueType())
9023 // If the vector element type is not legal, the BUILD_VECTOR operands
9024 // are promoted and implicitly truncated, and the result implicitly
9025 // extended. Make that explicit here.
9026 Elt = getAnyExtOrTrunc(Op: Elt, DL, VT);
9027
9028 return Elt;
9029 }
9030
9031 // EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector
9032 // operations are lowered to scalars.
9033 if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT) {
9034 // If the indices are the same, return the inserted element else
9035 // if the indices are known different, extract the element from
9036 // the original vector.
9037 SDValue N1Op2 = N1.getOperand(i: 2);
9038 ConstantSDNode *N1Op2C = dyn_cast<ConstantSDNode>(Val&: N1Op2);
9039
9040 if (N1Op2C && N2C) {
9041 if (N1Op2C->getZExtValue() == N2C->getZExtValue()) {
9042 if (VT == N1.getOperand(i: 1).getValueType())
9043 return N1.getOperand(i: 1);
9044 if (VT.isFloatingPoint()) {
9045 assert(VT.getSizeInBits() > N1.getOperand(1).getValueType().getSizeInBits());
9046 return getFPExtendOrRound(Op: N1.getOperand(i: 1), DL, VT);
9047 }
9048 return getSExtOrTrunc(Op: N1.getOperand(i: 1), DL, VT);
9049 }
9050 return getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT, N1: N1.getOperand(i: 0), N2);
9051 }
9052 }
9053
9054 // EXTRACT_VECTOR_ELT of v1iX EXTRACT_SUBVECTOR could be formed
9055 // when vector types are scalarized and v1iX is legal.
9056 // vextract (v1iX extract_subvector(vNiX, Idx)) -> vextract(vNiX,Idx).
9057 // Here we are completely ignoring the extract element index (N2),
9058 // which is fine for fixed width vectors, since any index other than 0
9059 // is undefined anyway. However, this cannot be ignored for scalable
9060 // vectors - in theory we could support this, but we don't want to do this
9061 // without a profitability check.
9062 if (N1.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
9063 N1.getValueType().isFixedLengthVector() &&
9064 N1.getValueType().getVectorNumElements() == 1) {
9065 return getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT, N1: N1.getOperand(i: 0),
9066 N2: N1.getOperand(i: 1));
9067 }
9068 break;
9069 case ISD::EXTRACT_ELEMENT:
9070 assert(N2C && (unsigned)N2C->getZExtValue() < 2 && "Bad EXTRACT_ELEMENT!");
9071 assert(!N1.getValueType().isVector() && !VT.isVector() &&
9072 (N1.getValueType().isInteger() == VT.isInteger()) &&
9073 N1.getValueType() != VT &&
9074 "Wrong types for EXTRACT_ELEMENT!");
9075
9076 // EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding
9077 // 64-bit integers into 32-bit parts. Instead of building the extract of
9078 // the BUILD_PAIR, only to have legalize rip it apart, just do it now.
9079 if (N1.getOpcode() == ISD::BUILD_PAIR)
9080 return N1.getOperand(i: N2C->getZExtValue());
9081
9082 // EXTRACT_ELEMENT of a constant int is also very common.
9083 if (N1C) {
9084 unsigned ElementSize = VT.getSizeInBits();
9085 unsigned Shift = ElementSize * N2C->getZExtValue();
9086 const APInt &Val = N1C->getAPIntValue();
9087 return getConstant(Val: Val.extractBits(numBits: ElementSize, bitPosition: Shift), DL, VT);
9088 }
9089 break;
9090 case ISD::VECTOR_SHUFFLE_VAR: {
9091 [[maybe_unused]] EVT MaskVT = N2.getValueType();
9092 assert(VT == N1.getValueType() &&
9093 "VECTOR_SHUFFLE_VAR result and source types don't match.");
9094 assert(VT.isVector() && MaskVT.isVector() &&
9095 "VECTOR_SHUFFLE_VAR operands must be vectors.");
9096 assert(MaskVT.getVectorElementType().isInteger() &&
9097 "VECTOR_SHUFFLE_VAR mask must be a vector of integers.");
9098 assert(VT.getVectorElementCount() == MaskVT.getVectorElementCount() &&
9099 "VECTOR_SHUFFLE_VAR mask has the wrong number of elements.");
9100 // The only in-range index is 0; any other index gives poison.
9101 if (VT.getVectorElementCount().isScalar())
9102 return N1;
9103 break;
9104 }
9105 case ISD::EXTRACT_SUBVECTOR: {
9106 EVT N1VT = N1.getValueType();
9107 assert(VT.isVector() && N1VT.isVector() &&
9108 "Extract subvector VTs must be vectors!");
9109 assert(VT.getVectorElementType() == N1VT.getVectorElementType() &&
9110 "Extract subvector VTs must have the same element type!");
9111 assert((VT.isFixedLengthVector() || N1VT.isScalableVector()) &&
9112 "Cannot extract a scalable vector from a fixed length vector!");
9113 assert((VT.isScalableVector() != N1VT.isScalableVector() ||
9114 VT.getVectorMinNumElements() <= N1VT.getVectorMinNumElements()) &&
9115 "Extract subvector must be from larger vector to smaller vector!");
9116 assert(N2C && "Extract subvector index must be a constant");
9117 assert((VT.isScalableVector() != N1VT.isScalableVector() ||
9118 (VT.getVectorMinNumElements() + N2C->getZExtValue()) <=
9119 N1VT.getVectorMinNumElements()) &&
9120 "Extract subvector overflow!");
9121 assert(N2C->getAPIntValue().getBitWidth() ==
9122 TLI->getVectorIdxWidth(getDataLayout()) &&
9123 "Constant index for EXTRACT_SUBVECTOR has an invalid size");
9124 assert(N2C->getZExtValue() % VT.getVectorMinNumElements() == 0 &&
9125 "Extract index is not a multiple of the output vector length");
9126
9127 // Trivial extraction.
9128 if (VT == N1VT)
9129 return N1;
9130
9131 // EXTRACT_SUBVECTOR of an UNDEF is an UNDEF.
9132 if (N1.isUndef())
9133 return getUNDEF(VT);
9134
9135 // EXTRACT_SUBVECTOR of CONCAT_VECTOR can be simplified if the pieces of
9136 // the concat have the same type as the extract.
9137 if (N1.getOpcode() == ISD::CONCAT_VECTORS &&
9138 VT == N1.getOperand(i: 0).getValueType()) {
9139 unsigned Factor = VT.getVectorMinNumElements();
9140 return N1.getOperand(i: N2C->getZExtValue() / Factor);
9141 }
9142
9143 // EXTRACT_SUBVECTOR of INSERT_SUBVECTOR is often created
9144 // during shuffle legalization.
9145 if (N1.getOpcode() == ISD::INSERT_SUBVECTOR && N2 == N1.getOperand(i: 2) &&
9146 VT == N1.getOperand(i: 1).getValueType())
9147 return N1.getOperand(i: 1);
9148 break;
9149 }
9150 }
9151
9152 if (N1.getOpcode() == ISD::POISON || N2.getOpcode() == ISD::POISON) {
9153 switch (Opcode) {
9154 case ISD::XOR:
9155 case ISD::ADD:
9156 case ISD::PTRADD:
9157 case ISD::SUB:
9158 case ISD::SIGN_EXTEND_INREG:
9159 case ISD::UDIV:
9160 case ISD::SDIV:
9161 case ISD::UREM:
9162 case ISD::SREM:
9163 case ISD::MUL:
9164 case ISD::AND:
9165 case ISD::SSUBSAT:
9166 case ISD::USUBSAT:
9167 case ISD::UMIN:
9168 case ISD::OR:
9169 case ISD::SADDSAT:
9170 case ISD::UADDSAT:
9171 case ISD::UMAX:
9172 case ISD::SMAX:
9173 case ISD::SMIN:
9174 // fold op(arg1, poison) -> poison, fold op(poison, arg2) -> poison.
9175 return N2.getOpcode() == ISD::POISON ? N2 : N1;
9176 }
9177 }
9178
9179 // Canonicalize an UNDEF to the RHS, even over a constant.
9180 if (N1.getOpcode() == ISD::UNDEF && N2.getOpcode() != ISD::UNDEF) {
9181 if (TLI->isCommutativeBinOp(Opcode)) {
9182 std::swap(a&: N1, b&: N2);
9183 } else {
9184 switch (Opcode) {
9185 case ISD::PTRADD:
9186 case ISD::SUB:
9187 // fold op(undef, non_undef_arg2) -> undef.
9188 return N1;
9189 case ISD::SIGN_EXTEND_INREG:
9190 case ISD::UDIV:
9191 case ISD::SDIV:
9192 case ISD::UREM:
9193 case ISD::SREM:
9194 case ISD::SSUBSAT:
9195 case ISD::USUBSAT:
9196 // fold op(undef, non_undef_arg2) -> 0.
9197 return getConstant(Val: 0, DL, VT);
9198 }
9199 }
9200 }
9201
9202 // Fold a bunch of operators when the RHS is undef.
9203 if (N2.getOpcode() == ISD::UNDEF) {
9204 switch (Opcode) {
9205 case ISD::XOR:
9206 if (N1.getOpcode() == ISD::UNDEF)
9207 // Handle undef ^ undef -> 0 special case. This is a common
9208 // idiom (misuse).
9209 return getConstant(Val: 0, DL, VT);
9210 [[fallthrough]];
9211 case ISD::ADD:
9212 case ISD::PTRADD:
9213 case ISD::SUB:
9214 // fold op(arg1, undef) -> undef.
9215 return N2;
9216 case ISD::UDIV:
9217 case ISD::SDIV:
9218 case ISD::UREM:
9219 case ISD::SREM:
9220 // fold op(arg1, undef) -> poison.
9221 return getPOISON(VT);
9222 case ISD::MUL:
9223 case ISD::AND:
9224 case ISD::SSUBSAT:
9225 case ISD::USUBSAT:
9226 case ISD::UMIN:
9227 // fold op(undef, undef) -> undef, fold op(arg1, undef) -> 0.
9228 return N1.getOpcode() == ISD::UNDEF ? N2 : getConstant(Val: 0, DL, VT);
9229 case ISD::OR:
9230 case ISD::SADDSAT:
9231 case ISD::UADDSAT:
9232 case ISD::UMAX:
9233 // fold op(undef, undef) -> undef, fold op(arg1, undef) -> -1.
9234 return N1.getOpcode() == ISD::UNDEF ? N2 : getAllOnesConstant(DL, VT);
9235 case ISD::SMAX:
9236 // fold op(undef, undef) -> undef, fold op(arg1, undef) -> MAX_INT.
9237 return N1.getOpcode() == ISD::UNDEF
9238 ? N2
9239 : getConstant(
9240 Val: APInt::getSignedMaxValue(numBits: VT.getScalarSizeInBits()), DL,
9241 VT);
9242 case ISD::SMIN:
9243 // fold op(undef, undef) -> undef, fold op(arg1, undef) -> MIN_INT.
9244 return N1.getOpcode() == ISD::UNDEF
9245 ? N2
9246 : getConstant(
9247 Val: APInt::getSignedMinValue(numBits: VT.getScalarSizeInBits()), DL,
9248 VT);
9249 }
9250 }
9251
9252 // Perform trivial constant folding.
9253 if (SDValue SV = FoldConstantArithmetic(Opcode, DL, VT, Ops: {N1, N2}, Flags))
9254 return SV;
9255
9256 // Memoize this node if possible.
9257 SDNode *N;
9258 SDVTList VTs = getVTList(VT);
9259 SDValue Ops[] = {N1, N2};
9260 if (VT != MVT::Glue) {
9261 SDNodeKey ID(Opcode, VTs, Ops);
9262 FoldingSetInsertToken InsertToken;
9263 if (SDNode *E = lookupNode(Key: ID, DL, InsertToken)) {
9264 E->intersectFlagsWith(Flags);
9265 return SDValue(E, 0);
9266 }
9267
9268 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
9269 N->setFlags(Flags);
9270 createOperands(Node: N, Vals: Ops);
9271 CSEMap.insert(N, Token: InsertToken);
9272 } else {
9273 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
9274 createOperands(Node: N, Vals: Ops);
9275 }
9276
9277 InsertNode(N);
9278 SDValue V = SDValue(N, 0);
9279 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
9280 return V;
9281}
9282
9283SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
9284 SDValue N1, SDValue N2, SDValue N3) {
9285 SDNodeFlags Flags;
9286 if (Inserter)
9287 Flags = Inserter->getFlags();
9288 return getNode(Opcode, DL, VT, N1, N2, N3, Flags);
9289}
9290
9291SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
9292 SDValue N1, SDValue N2, SDValue N3,
9293 const SDNodeFlags Flags) {
9294 assert(N1.getOpcode() != ISD::DELETED_NODE &&
9295 N2.getOpcode() != ISD::DELETED_NODE &&
9296 N3.getOpcode() != ISD::DELETED_NODE &&
9297 "Operand is DELETED_NODE!");
9298 // Perform various simplifications.
9299 switch (Opcode) {
9300 case ISD::BUILD_VECTOR: {
9301 // Attempt to simplify BUILD_VECTOR.
9302 SDValue Ops[] = {N1, N2, N3};
9303 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, DAG&: *this))
9304 return V;
9305 break;
9306 }
9307 case ISD::CONCAT_VECTORS: {
9308 SDValue Ops[] = {N1, N2, N3};
9309 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, DAG&: *this))
9310 return V;
9311 break;
9312 }
9313 case ISD::SETCC: {
9314 assert(VT.isInteger() && "SETCC result type must be an integer!");
9315 assert(N1.getValueType() == N2.getValueType() &&
9316 "SETCC operands must have the same type!");
9317 assert(VT.isVector() == N1.getValueType().isVector() &&
9318 "SETCC type should be vector iff the operand type is vector!");
9319 assert((!VT.isVector() || VT.getVectorElementCount() ==
9320 N1.getValueType().getVectorElementCount()) &&
9321 "SETCC vector element counts must match!");
9322 // Use FoldSetCC to simplify SETCC's.
9323 if (SDValue V =
9324 FoldSetCC(VT, N1, N2, Cond: cast<CondCodeSDNode>(Val&: N3)->get(), dl: DL, Flags))
9325 return V;
9326 break;
9327 }
9328 case ISD::SELECT:
9329 case ISD::VSELECT:
9330 if (SDValue V = simplifySelect(Cond: N1, TVal: N2, FVal: N3))
9331 return V;
9332 break;
9333 case ISD::VECTOR_SHUFFLE:
9334 llvm_unreachable("should use getVectorShuffle constructor!");
9335 case ISD::VECTOR_SPLICE_LEFT:
9336 if (isNullConstant(V: N3))
9337 return N1;
9338 break;
9339 case ISD::VECTOR_SPLICE_RIGHT:
9340 if (isNullConstant(V: N3))
9341 return N2;
9342 break;
9343 case ISD::INSERT_VECTOR_ELT: {
9344 assert(VT.isVector() && VT == N1.getValueType() &&
9345 "INSERT_VECTOR_ELT vector type mismatch");
9346 assert(VT.isFloatingPoint() == N2.getValueType().isFloatingPoint() &&
9347 "INSERT_VECTOR_ELT scalar fp/int mismatch");
9348 assert((!VT.isFloatingPoint() ||
9349 VT.getVectorElementType() == N2.getValueType()) &&
9350 "INSERT_VECTOR_ELT fp scalar type mismatch");
9351 assert((!VT.isInteger() ||
9352 VT.getScalarSizeInBits() <= N2.getScalarValueSizeInBits()) &&
9353 "INSERT_VECTOR_ELT int scalar size mismatch");
9354
9355 auto *N3C = dyn_cast<ConstantSDNode>(Val&: N3);
9356 // INSERT_VECTOR_ELT into out-of-bounds element is an UNDEF, except
9357 // for scalable vectors where we will generate appropriate code to
9358 // deal with out-of-bounds cases correctly.
9359 if (N3C && VT.isFixedLengthVector() &&
9360 N3C->getZExtValue() >= VT.getVectorNumElements())
9361 return getUNDEF(VT);
9362
9363 // Undefined index can be assumed out-of-bounds, so that's UNDEF too.
9364 if (N3.isUndef())
9365 return getUNDEF(VT);
9366
9367 // If inserting poison, just use the input vector.
9368 if (N2.getOpcode() == ISD::POISON)
9369 return N1;
9370
9371 // Inserting undef into undef/poison is still undef.
9372 if (N2.getOpcode() == ISD::UNDEF && N1.isUndef())
9373 return getUNDEF(VT);
9374
9375 // If the inserted element is an UNDEF, just use the input vector.
9376 // But not if skipping the insert could make the result more poisonous.
9377 if (N2.isUndef()) {
9378 if (N3C && VT.isFixedLengthVector()) {
9379 APInt EltMask =
9380 APInt::getOneBitSet(numBits: VT.getVectorNumElements(), BitNo: N3C->getZExtValue());
9381 if (isGuaranteedNotToBePoison(Op: N1, DemandedElts: EltMask))
9382 return N1;
9383 } else if (isGuaranteedNotToBePoison(Op: N1))
9384 return N1;
9385 }
9386 break;
9387 }
9388 case ISD::INSERT_SUBVECTOR: {
9389 // If inserting poison, just use the input vector,
9390 if (N2.getOpcode() == ISD::POISON)
9391 return N1;
9392
9393 // Inserting undef into undef/poison is still undef.
9394 if (N2.getOpcode() == ISD::UNDEF && N1.isUndef())
9395 return getUNDEF(VT);
9396
9397 EVT N2VT = N2.getValueType();
9398 assert(VT == N1.getValueType() &&
9399 "Dest and insert subvector source types must match!");
9400 assert(VT.isVector() && N2VT.isVector() &&
9401 "Insert subvector VTs must be vectors!");
9402 assert(VT.getVectorElementType() == N2VT.getVectorElementType() &&
9403 "Insert subvector VTs must have the same element type!");
9404 assert((VT.isScalableVector() || N2VT.isFixedLengthVector()) &&
9405 "Cannot insert a scalable vector into a fixed length vector!");
9406 assert((VT.isScalableVector() != N2VT.isScalableVector() ||
9407 VT.getVectorMinNumElements() >= N2VT.getVectorMinNumElements()) &&
9408 "Insert subvector must be from smaller vector to larger vector!");
9409 assert(isa<ConstantSDNode>(N3) &&
9410 "Insert subvector index must be constant");
9411 assert((VT.isScalableVector() != N2VT.isScalableVector() ||
9412 (N2VT.getVectorMinNumElements() + N3->getAsZExtVal()) <=
9413 VT.getVectorMinNumElements()) &&
9414 "Insert subvector overflow!");
9415 assert(N3->getAsAPIntVal().getBitWidth() ==
9416 TLI->getVectorIdxWidth(getDataLayout()) &&
9417 "Constant index for INSERT_SUBVECTOR has an invalid size");
9418
9419 // Trivial insertion.
9420 if (VT == N2VT)
9421 return N2;
9422
9423 // If this is an insert of an extracted vector into an undef/poison vector,
9424 // we can just use the input to the extract. But not if skipping the
9425 // extract+insert could make the result more poisonous.
9426 if (N1.isUndef() && N2.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
9427 N2.getOperand(i: 1) == N3 && N2.getOperand(i: 0).getValueType() == VT) {
9428 if (N1.getOpcode() == ISD::POISON)
9429 return N2.getOperand(i: 0);
9430 if (VT.isFixedLengthVector() && N2VT.isFixedLengthVector()) {
9431 unsigned LoBit = N3->getAsZExtVal();
9432 unsigned HiBit = LoBit + N2VT.getVectorNumElements();
9433 APInt EltMask =
9434 APInt::getBitsSet(numBits: VT.getVectorNumElements(), loBit: LoBit, hiBit: HiBit);
9435 if (isGuaranteedNotToBePoison(Op: N2.getOperand(i: 0), DemandedElts: ~EltMask))
9436 return N2.getOperand(i: 0);
9437 } else if (isGuaranteedNotToBePoison(Op: N2.getOperand(i: 0)))
9438 return N2.getOperand(i: 0);
9439 }
9440
9441 // If the inserted subvector is UNDEF, just use the input vector.
9442 // But not if skipping the insert could make the result more poisonous.
9443 if (N2.isUndef()) {
9444 if (VT.isFixedLengthVector()) {
9445 unsigned LoBit = N3->getAsZExtVal();
9446 unsigned HiBit = LoBit + N2VT.getVectorNumElements();
9447 APInt EltMask =
9448 APInt::getBitsSet(numBits: VT.getVectorNumElements(), loBit: LoBit, hiBit: HiBit);
9449 if (isGuaranteedNotToBePoison(Op: N1, DemandedElts: EltMask))
9450 return N1;
9451 } else if (isGuaranteedNotToBePoison(Op: N1))
9452 return N1;
9453 }
9454 break;
9455 }
9456 case ISD::BITCAST:
9457 // Fold bit_convert nodes from a type to themselves.
9458 if (N1.getValueType() == VT)
9459 return N1;
9460 break;
9461 case ISD::VECTOR_COMPRESS: {
9462 [[maybe_unused]] EVT VecVT = N1.getValueType();
9463 [[maybe_unused]] EVT MaskVT = N2.getValueType();
9464 [[maybe_unused]] EVT PassthruVT = N3.getValueType();
9465 assert(VT == VecVT && "Vector and result type don't match.");
9466 assert(VecVT.isVector() && MaskVT.isVector() && PassthruVT.isVector() &&
9467 "All inputs must be vectors.");
9468 assert(VecVT == PassthruVT && "Vector and passthru types don't match.");
9469 assert(VecVT.getVectorElementCount() == MaskVT.getVectorElementCount() &&
9470 "Vector and mask must have same number of elements.");
9471
9472 if (N1.isUndef() || N2.isUndef())
9473 return N3;
9474
9475 break;
9476 }
9477 case ISD::PARTIAL_REDUCE_UMLA:
9478 case ISD::PARTIAL_REDUCE_SMLA:
9479 case ISD::PARTIAL_REDUCE_SUMLA:
9480 case ISD::PARTIAL_REDUCE_FMLA: {
9481 [[maybe_unused]] EVT AccVT = N1.getValueType();
9482 [[maybe_unused]] EVT Input1VT = N2.getValueType();
9483 [[maybe_unused]] EVT Input2VT = N3.getValueType();
9484 assert(Input1VT.isVector() && Input1VT == Input2VT &&
9485 "Expected the second and third operands of the PARTIAL_REDUCE_MLA "
9486 "node to have the same type!");
9487 assert(VT.isVector() && VT == AccVT &&
9488 "Expected the first operand of the PARTIAL_REDUCE_MLA node to have "
9489 "the same type as its result!");
9490 assert(Input1VT.getVectorElementCount().hasKnownScalarFactor(
9491 AccVT.getVectorElementCount()) &&
9492 "Expected the element count of the second and third operands of the "
9493 "PARTIAL_REDUCE_MLA node to be a positive integer multiple of the "
9494 "element count of the first operand and the result!");
9495 assert(N2.getScalarValueSizeInBits() <= N1.getScalarValueSizeInBits() &&
9496 "Expected the second and third operands of the PARTIAL_REDUCE_MLA "
9497 "node to have an element type which is the same as or smaller than "
9498 "the element type of the first operand and result!");
9499 break;
9500 }
9501 }
9502
9503 // Perform trivial constant folding for arithmetic operators.
9504 switch (Opcode) {
9505 case ISD::PARTIAL_REDUCE_SMLA:
9506 case ISD::PARTIAL_REDUCE_UMLA:
9507 case ISD::PARTIAL_REDUCE_SUMLA:
9508 case ISD::FMA:
9509 case ISD::FMAD:
9510 case ISD::SETCC:
9511 case ISD::FSHL:
9512 case ISD::FSHR:
9513 if (SDValue SV =
9514 FoldConstantArithmetic(Opcode, DL, VT, Ops: {N1, N2, N3}, Flags))
9515 return SV;
9516 break;
9517 }
9518
9519 // Memoize node if it doesn't produce a glue result.
9520 SDNode *N;
9521 SDVTList VTs = getVTList(VT);
9522 SDValue Ops[] = {N1, N2, N3};
9523 if (VT != MVT::Glue) {
9524 SDNodeKey ID(Opcode, VTs, Ops);
9525 FoldingSetInsertToken InsertToken;
9526 if (SDNode *E = lookupNode(Key: ID, DL, InsertToken)) {
9527 E->intersectFlagsWith(Flags);
9528 return SDValue(E, 0);
9529 }
9530
9531 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
9532 N->setFlags(Flags);
9533 createOperands(Node: N, Vals: Ops);
9534 CSEMap.insert(N, Token: InsertToken);
9535 } else {
9536 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
9537 createOperands(Node: N, Vals: Ops);
9538 }
9539
9540 InsertNode(N);
9541 SDValue V = SDValue(N, 0);
9542 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
9543 return V;
9544}
9545
9546SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
9547 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
9548 const SDNodeFlags Flags) {
9549 SDValue Ops[] = { N1, N2, N3, N4 };
9550 return getNode(Opcode, DL, VT, Ops, Flags);
9551}
9552
9553SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
9554 SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
9555 SDNodeFlags Flags;
9556 if (Inserter)
9557 Flags = Inserter->getFlags();
9558 return getNode(Opcode, DL, VT, N1, N2, N3, N4, Flags);
9559}
9560
9561SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
9562 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
9563 SDValue N5, const SDNodeFlags Flags) {
9564 SDValue Ops[] = { N1, N2, N3, N4, N5 };
9565 return getNode(Opcode, DL, VT, Ops, Flags);
9566}
9567
9568SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
9569 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
9570 SDValue N5) {
9571 SDNodeFlags Flags;
9572 if (Inserter)
9573 Flags = Inserter->getFlags();
9574 return getNode(Opcode, DL, VT, N1, N2, N3, N4, N5, Flags);
9575}
9576
9577/// getStackArgumentTokenFactor - Compute a TokenFactor to force all
9578/// the incoming stack arguments to be loaded from the stack.
9579SDValue SelectionDAG::getStackArgumentTokenFactor(SDValue Chain) {
9580 SmallVector<SDValue, 8> ArgChains;
9581
9582 // Include the original chain at the beginning of the list. When this is
9583 // used by target LowerCall hooks, this helps legalize find the
9584 // CALLSEQ_BEGIN node.
9585 ArgChains.push_back(Elt: Chain);
9586
9587 // Add a chain value for each stack argument.
9588 for (SDNode *U : getEntryNode().getNode()->users())
9589 if (LoadSDNode *L = dyn_cast<LoadSDNode>(Val: U))
9590 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val: L->getBasePtr()))
9591 if (FI->getIndex() < 0)
9592 ArgChains.push_back(Elt: SDValue(L, 1));
9593
9594 // Build a tokenfactor for all the chains.
9595 return getNode(Opcode: ISD::TokenFactor, DL: SDLoc(Chain), VT: MVT::Other, Ops: ArgChains);
9596}
9597
9598/// getMemsetValue - Vectorized representation of the memset value
9599/// operand.
9600static SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG,
9601 const SDLoc &dl) {
9602 assert(!Value.isUndef());
9603
9604 unsigned NumBits = VT.getScalarSizeInBits();
9605 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: Value)) {
9606 assert(C->getAPIntValue().getBitWidth() == 8);
9607 APInt Val = APInt::getSplat(NewLen: NumBits, V: C->getAPIntValue());
9608 if (VT.isInteger()) {
9609 bool IsOpaque = VT.getSizeInBits() > 64 ||
9610 !DAG.getTargetLoweringInfo().isLegalStoreImmediate(Value: C->getSExtValue());
9611 return DAG.getConstant(Val, DL: dl, VT, isT: false, isO: IsOpaque);
9612 }
9613 return DAG.getConstantFP(V: APFloat(VT.getFltSemantics(), Val), DL: dl, VT);
9614 }
9615
9616 assert(Value.getValueType() == MVT::i8 && "memset with non-byte fill value?");
9617 EVT IntVT = VT.getScalarType();
9618 if (!IntVT.isInteger())
9619 IntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: IntVT.getSizeInBits());
9620
9621 Value = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: IntVT, N1: Value);
9622 if (NumBits > 8) {
9623 // Use a multiplication with 0x010101... to extend the input to the
9624 // required length.
9625 APInt Magic = APInt::getSplat(NewLen: NumBits, V: APInt(8, 0x01));
9626 Value = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: IntVT, N1: Value,
9627 N2: DAG.getConstant(Val: Magic, DL: dl, VT: IntVT));
9628 }
9629
9630 if (VT != Value.getValueType() && !VT.isInteger())
9631 Value = DAG.getBitcast(VT: VT.getScalarType(), V: Value);
9632 if (VT != Value.getValueType())
9633 Value = DAG.getSplatBuildVector(VT, DL: dl, Op: Value);
9634
9635 return Value;
9636}
9637
9638/// getMemsetStringVal - Similar to getMemsetValue. Except this is only
9639/// used when a memcpy is turned into a memset when the source is a constant
9640/// string ptr.
9641static SDValue getMemsetStringVal(EVT VT, const SDLoc &dl, SelectionDAG &DAG,
9642 const TargetLowering &TLI,
9643 const ConstantDataArraySlice &Slice) {
9644 // Handle vector with all elements zero.
9645 if (Slice.Array == nullptr) {
9646 if (VT.isInteger())
9647 return DAG.getConstant(Val: 0, DL: dl, VT);
9648 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT,
9649 N1: DAG.getConstant(Val: 0, DL: dl, VT: VT.changeTypeToInteger()));
9650 }
9651
9652 assert(!VT.isVector() && "Can't handle vector type here!");
9653 unsigned NumVTBits = VT.getSizeInBits();
9654 unsigned NumVTBytes = NumVTBits / 8;
9655 unsigned NumBytes = std::min(a: NumVTBytes, b: unsigned(Slice.Length));
9656
9657 APInt Val(NumVTBits, 0);
9658 if (DAG.getDataLayout().isLittleEndian()) {
9659 for (unsigned i = 0; i != NumBytes; ++i)
9660 Val |= (uint64_t)(unsigned char)Slice[i] << i*8;
9661 } else {
9662 for (unsigned i = 0; i != NumBytes; ++i)
9663 Val |= (uint64_t)(unsigned char)Slice[i] << (NumVTBytes-i-1)*8;
9664 }
9665
9666 // If the "cost" of materializing the integer immediate is less than the cost
9667 // of a load, then it is cost effective to turn the load into the immediate.
9668 Type *Ty = VT.getTypeForEVT(Context&: *DAG.getContext());
9669 if (TLI.shouldConvertConstantLoadToIntImm(Imm: Val, Ty))
9670 return DAG.getConstant(Val, DL: dl, VT);
9671 return SDValue();
9672}
9673
9674SDValue SelectionDAG::getMemBasePlusOffset(SDValue Base, TypeSize Offset,
9675 const SDLoc &DL,
9676 const SDNodeFlags Flags) {
9677 SDValue Index = getTypeSize(DL, VT: Base.getValueType(), TS: Offset);
9678 return getMemBasePlusOffset(Base, Offset: Index, DL, Flags);
9679}
9680
9681SDValue SelectionDAG::getMemBasePlusOffset(SDValue Ptr, SDValue Offset,
9682 const SDLoc &DL,
9683 const SDNodeFlags Flags) {
9684 assert(Offset.getValueType().isInteger());
9685 EVT BasePtrVT = Ptr.getValueType();
9686 if (TLI->shouldPreservePtrArith(F: this->getMachineFunction().getFunction(),
9687 PtrVT: BasePtrVT))
9688 return getNode(Opcode: ISD::PTRADD, DL, VT: BasePtrVT, N1: Ptr, N2: Offset, Flags);
9689 // InBounds only applies to PTRADD, don't set it if we generate ADD.
9690 SDNodeFlags AddFlags = Flags;
9691 AddFlags.setInBounds(false);
9692 return getNode(Opcode: ISD::ADD, DL, VT: BasePtrVT, N1: Ptr, N2: Offset, Flags: AddFlags);
9693}
9694
9695/// Returns true if memcpy source is constant data.
9696static bool isMemSrcFromConstant(SDValue Src, ConstantDataArraySlice &Slice) {
9697 uint64_t SrcDelta = 0;
9698 GlobalAddressSDNode *G = nullptr;
9699 if (Src.getOpcode() == ISD::GlobalAddress)
9700 G = cast<GlobalAddressSDNode>(Val&: Src);
9701 else if (Src->isAnyAdd() &&
9702 Src.getOperand(i: 0).getOpcode() == ISD::GlobalAddress &&
9703 Src.getOperand(i: 1).getOpcode() == ISD::Constant) {
9704 G = cast<GlobalAddressSDNode>(Val: Src.getOperand(i: 0));
9705 SrcDelta = Src.getConstantOperandVal(i: 1);
9706 }
9707 if (!G)
9708 return false;
9709
9710 return getConstantDataArrayInfo(V: G->getGlobal(), Slice, ElementSize: 8,
9711 Offset: SrcDelta + G->getOffset());
9712}
9713
9714static bool shouldLowerMemFuncForSize(const MachineFunction &MF,
9715 SelectionDAG &DAG) {
9716 // On Darwin, -Os means optimize for size without hurting performance, so
9717 // only really optimize for size when -Oz (MinSize) is used.
9718 if (MF.getTarget().getTargetTriple().isOSDarwin())
9719 return MF.getFunction().hasMinSize();
9720 return DAG.shouldOptForSize();
9721}
9722
9723static void chainLoadsAndStoresForMemcpy(SelectionDAG &DAG, const SDLoc &dl,
9724 SmallVector<SDValue, 32> &OutChains, unsigned From,
9725 unsigned To, SmallVector<SDValue, 16> &OutLoadChains,
9726 SmallVector<SDValue, 16> &OutStoreChains) {
9727 assert(OutLoadChains.size() && "Missing loads in memcpy inlining");
9728 assert(OutStoreChains.size() && "Missing stores in memcpy inlining");
9729 SmallVector<SDValue, 16> GluedLoadChains;
9730 for (unsigned i = From; i < To; ++i) {
9731 OutChains.push_back(Elt: OutLoadChains[i]);
9732 GluedLoadChains.push_back(Elt: OutLoadChains[i]);
9733 }
9734
9735 // Chain for all loads.
9736 SDValue LoadToken = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other,
9737 Ops: GluedLoadChains);
9738
9739 for (unsigned i = From; i < To; ++i) {
9740 StoreSDNode *ST = dyn_cast<StoreSDNode>(Val&: OutStoreChains[i]);
9741 SDValue NewStore = DAG.getTruncStore(Chain: LoadToken, dl, Val: ST->getValue(),
9742 Ptr: ST->getBasePtr(), SVT: ST->getMemoryVT(),
9743 MMO: ST->getMemOperand());
9744 OutChains.push_back(Elt: NewStore);
9745 }
9746}
9747
9748static SDValue
9749getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain,
9750 SDValue Dst, SDValue Src, uint64_t Size, Align DstAlign,
9751 Align SrcAlign, bool isVol, bool AlwaysInline,
9752 MachinePointerInfo DstPtrInfo,
9753 MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo,
9754 BatchAAResults *BatchAA, const MDNode *DstMemCacheHint,
9755 const MDNode *SrcMemCacheHint) {
9756 // Turn a memcpy of undef to nop.
9757 // FIXME: We need to honor volatile even is Src is undef.
9758 if (Src.isUndef())
9759 return Chain;
9760
9761 // Expand memcpy to a series of load and store ops if the size operand falls
9762 // below a certain threshold.
9763 // TODO: In the AlwaysInline case, if the size is big then generate a loop
9764 // rather than maybe a humongous number of loads and stores.
9765 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9766 const DataLayout &DL = DAG.getDataLayout();
9767 LLVMContext &C = *DAG.getContext();
9768 std::vector<EVT> MemOps;
9769 bool DstAlignCanChange = false;
9770 MachineFunction &MF = DAG.getMachineFunction();
9771 MachineFrameInfo &MFI = MF.getFrameInfo();
9772 bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
9773 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Dst);
9774 if (FI && !MFI.isFixedObjectIndex(ObjectIdx: FI->getIndex()))
9775 DstAlignCanChange = true;
9776 SrcAlign = std::max(a: SrcAlign, b: DAG.InferPtrAlign(Ptr: Src).valueOrOne());
9777 ConstantDataArraySlice Slice;
9778 // If marked as volatile, perform a copy even when marked as constant.
9779 bool CopyFromConstant = !isVol && isMemSrcFromConstant(Src, Slice);
9780 bool isZeroConstant = CopyFromConstant && Slice.Array == nullptr;
9781 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(OptSize);
9782 const MemOp Op = isZeroConstant
9783 ? MemOp::Set(Size, DstAlignCanChange, DstAlign,
9784 /*IsZeroMemset*/ true, IsVolatile: isVol)
9785 : MemOp::Copy(Size, DstAlignCanChange, DstAlign,
9786 SrcAlign, IsVolatile: isVol, MemcpyStrSrc: CopyFromConstant);
9787 if (!TLI.findOptimalMemOpLowering(
9788 Context&: C, MemOps, Limit, Op, DstAS: DstPtrInfo.getAddrSpace(),
9789 SrcAS: SrcPtrInfo.getAddrSpace(), FuncAttributes: MF.getFunction().getAttributes(), LargestVT: nullptr))
9790 return SDValue();
9791
9792 if (DstAlignCanChange) {
9793 Type *Ty = MemOps[0].getTypeForEVT(Context&: C);
9794 Align NewDstAlign = DL.getABITypeAlign(Ty);
9795
9796 // Don't promote to an alignment that would require dynamic stack
9797 // realignment which may conflict with optimizations such as tail call
9798 // optimization.
9799 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
9800 if (!TRI->hasStackRealignment(MF))
9801 if (MaybeAlign StackAlign = DL.getStackAlignment())
9802 NewDstAlign = std::min(a: NewDstAlign, b: *StackAlign);
9803
9804 if (NewDstAlign > DstAlign) {
9805 // Give the stack frame object a larger alignment if needed.
9806 if (MFI.getObjectAlign(ObjectIdx: FI->getIndex()) < NewDstAlign)
9807 MFI.setObjectAlignment(ObjectIdx: FI->getIndex(), Alignment: NewDstAlign);
9808 DstAlign = NewDstAlign;
9809 }
9810 }
9811
9812 // Prepare AAInfo for loads/stores after lowering this memcpy.
9813 AAMDNodes NewAAInfo = AAInfo;
9814 NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
9815
9816 const Value *SrcVal = dyn_cast_if_present<const Value *>(Val&: SrcPtrInfo.V);
9817 bool isConstant =
9818 BatchAA && SrcVal &&
9819 BatchAA->pointsToConstantMemory(Loc: MemoryLocation(SrcVal, Size, AAInfo));
9820
9821 MachineMemOperand::Flags MMOFlags =
9822 isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone;
9823 SmallVector<SDValue, 16> OutLoadChains;
9824 SmallVector<SDValue, 16> OutStoreChains;
9825 SmallVector<SDValue, 32> OutChains;
9826 unsigned NumMemOps = MemOps.size();
9827 uint64_t SrcOff = 0, DstOff = 0;
9828 for (unsigned i = 0; i != NumMemOps; ++i) {
9829 EVT VT = MemOps[i];
9830 unsigned VTSize = VT.getSizeInBits() / 8;
9831 SDValue Value, Store;
9832
9833 if (VTSize > Size) {
9834 // Issuing an unaligned load / store pair that overlaps with the previous
9835 // pair. Adjust the offset accordingly.
9836 assert(i == NumMemOps-1 && i != 0);
9837 SrcOff -= VTSize - Size;
9838 DstOff -= VTSize - Size;
9839 }
9840
9841 if (CopyFromConstant &&
9842 (isZeroConstant || (VT.isInteger() && !VT.isVector()))) {
9843 // It's unlikely a store of a vector immediate can be done in a single
9844 // instruction. It would require a load from a constantpool first.
9845 // We only handle zero vectors here.
9846 // FIXME: Handle other cases where store of vector immediate is done in
9847 // a single instruction.
9848 ConstantDataArraySlice SubSlice;
9849 if (SrcOff < Slice.Length) {
9850 SubSlice = Slice;
9851 SubSlice.move(Delta: SrcOff);
9852 } else {
9853 // This is an out-of-bounds access and hence UB. Pretend we read zero.
9854 SubSlice.Array = nullptr;
9855 SubSlice.Offset = 0;
9856 SubSlice.Length = VTSize;
9857 }
9858 Value = getMemsetStringVal(VT, dl, DAG, TLI, Slice: SubSlice);
9859 if (Value.getNode()) {
9860 Store = DAG.getStore(
9861 Chain, dl, Val: Value,
9862 Ptr: DAG.getObjectPtrOffset(SL: dl, Ptr: Dst, Offset: TypeSize::getFixed(ExactSize: DstOff)),
9863 PtrInfo: DstPtrInfo.getWithOffset(O: DstOff), Alignment: DstAlign, MMOFlags,
9864 Metadata: MMOMetadata(NewAAInfo, /*Ranges=*/nullptr, DstMemCacheHint));
9865 OutChains.push_back(Elt: Store);
9866 }
9867 }
9868
9869 if (!Store.getNode()) {
9870 // The type might not be legal for the target. This should only happen
9871 // if the type is smaller than a legal type, as on PPC, so the right
9872 // thing to do is generate a LoadExt/StoreTrunc pair. These simplify
9873 // to Load/Store if NVT==VT.
9874 // FIXME does the case above also need this?
9875 EVT NVT = TLI.getTypeToTransformTo(Context&: C, VT);
9876 assert(NVT.bitsGE(VT));
9877
9878 bool isDereferenceable =
9879 SrcPtrInfo.getWithOffset(O: SrcOff).isDereferenceable(Size: VTSize, C, DL);
9880 MachineMemOperand::Flags SrcMMOFlags = MMOFlags;
9881 if (isDereferenceable)
9882 SrcMMOFlags |= MachineMemOperand::MODereferenceable;
9883 if (isConstant)
9884 SrcMMOFlags |= MachineMemOperand::MOInvariant;
9885
9886 Value = DAG.getExtLoad(
9887 ExtType: ISD::EXTLOAD, dl, VT: NVT, Chain,
9888 Ptr: DAG.getObjectPtrOffset(SL: dl, Ptr: Src, Offset: TypeSize::getFixed(ExactSize: SrcOff)),
9889 PtrInfo: SrcPtrInfo.getWithOffset(O: SrcOff), MemVT: VT,
9890 Alignment: commonAlignment(A: SrcAlign, Offset: SrcOff), MMOFlags: SrcMMOFlags,
9891 Metadata: MMOMetadata(NewAAInfo, /*Ranges=*/nullptr, SrcMemCacheHint));
9892 OutLoadChains.push_back(Elt: Value.getValue(R: 1));
9893
9894 Store = DAG.getTruncStore(
9895 Chain, dl, Val: Value,
9896 Ptr: DAG.getObjectPtrOffset(SL: dl, Ptr: Dst, Offset: TypeSize::getFixed(ExactSize: DstOff)),
9897 PtrInfo: DstPtrInfo.getWithOffset(O: DstOff), SVT: VT, Alignment: DstAlign, MMOFlags,
9898 Metadata: MMOMetadata(NewAAInfo, /*Ranges=*/nullptr, DstMemCacheHint));
9899 OutStoreChains.push_back(Elt: Store);
9900 }
9901 SrcOff += VTSize;
9902 DstOff += VTSize;
9903 Size -= VTSize;
9904 }
9905
9906 unsigned MaxLdStGlue = DAG.getSelectionDAGOptions().ldstmemcpy_glue_max;
9907 unsigned GluedLdStLimit =
9908 MaxLdStGlue == 0 ? TLI.getMaxGluedStoresPerMemcpy() : MaxLdStGlue;
9909 unsigned NumLdStInMemcpy = OutStoreChains.size();
9910
9911 if (NumLdStInMemcpy) {
9912 // It may be that memcpy might be converted to memset if it's memcpy
9913 // of constants. In such a case, we won't have loads and stores, but
9914 // just stores. In the absence of loads, there is nothing to gang up.
9915 if ((GluedLdStLimit <= 1) ||
9916 !DAG.getSelectionDAGOptions().enable_memcpy_dag_opt) {
9917 // If target does not care, just leave as it.
9918 for (unsigned i = 0; i < NumLdStInMemcpy; ++i) {
9919 OutChains.push_back(Elt: OutLoadChains[i]);
9920 OutChains.push_back(Elt: OutStoreChains[i]);
9921 }
9922 } else {
9923 // Ld/St less than/equal limit set by target.
9924 if (NumLdStInMemcpy <= GluedLdStLimit) {
9925 chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, From: 0,
9926 To: NumLdStInMemcpy, OutLoadChains,
9927 OutStoreChains);
9928 } else {
9929 unsigned NumberLdChain = NumLdStInMemcpy / GluedLdStLimit;
9930 unsigned RemainingLdStInMemcpy = NumLdStInMemcpy % GluedLdStLimit;
9931 unsigned GlueIter = 0;
9932
9933 // Residual ld/st.
9934 if (RemainingLdStInMemcpy) {
9935 chainLoadsAndStoresForMemcpy(
9936 DAG, dl, OutChains, From: NumLdStInMemcpy - RemainingLdStInMemcpy,
9937 To: NumLdStInMemcpy, OutLoadChains, OutStoreChains);
9938 }
9939
9940 for (unsigned cnt = 0; cnt < NumberLdChain; ++cnt) {
9941 unsigned IndexFrom = NumLdStInMemcpy - RemainingLdStInMemcpy -
9942 GlueIter - GluedLdStLimit;
9943 unsigned IndexTo = NumLdStInMemcpy - RemainingLdStInMemcpy - GlueIter;
9944 chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, From: IndexFrom, To: IndexTo,
9945 OutLoadChains, OutStoreChains);
9946 GlueIter += GluedLdStLimit;
9947 }
9948 }
9949 }
9950 }
9951 return DAG.getTokenFactor(DL: dl, Vals&: OutChains);
9952}
9953
9954static SDValue getMemmoveLoadsAndStores(
9955 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
9956 uint64_t Size, Align DstAlign, Align SrcAlign, bool isVol,
9957 bool AlwaysInline, MachinePointerInfo DstPtrInfo,
9958 MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo) {
9959 // Turn a memmove of undef to nop.
9960 // FIXME: We need to honor volatile even is Src is undef.
9961 if (Src.isUndef())
9962 return Chain;
9963
9964 // Expand memmove to a series of load and store ops if the size operand falls
9965 // below a certain threshold.
9966 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9967 const DataLayout &DL = DAG.getDataLayout();
9968 LLVMContext &C = *DAG.getContext();
9969 std::vector<EVT> MemOps;
9970 bool DstAlignCanChange = false;
9971 MachineFunction &MF = DAG.getMachineFunction();
9972 MachineFrameInfo &MFI = MF.getFrameInfo();
9973 bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
9974 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Dst);
9975 if (FI && !MFI.isFixedObjectIndex(ObjectIdx: FI->getIndex()))
9976 DstAlignCanChange = true;
9977 SrcAlign = std::max(a: SrcAlign, b: DAG.InferPtrAlign(Ptr: Src).valueOrOne());
9978 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize);
9979 if (!TLI.findOptimalMemOpLowering(
9980 Context&: C, MemOps, Limit,
9981 Op: MemOp::Move(Size, DstAlignCanChange, DstAlign, SrcAlign, IsVolatile: isVol),
9982 DstAS: DstPtrInfo.getAddrSpace(), SrcAS: SrcPtrInfo.getAddrSpace(),
9983 FuncAttributes: MF.getFunction().getAttributes(), LargestVT: nullptr))
9984 return SDValue();
9985
9986 if (DstAlignCanChange) {
9987 Type *Ty = MemOps[0].getTypeForEVT(Context&: C);
9988 Align NewDstAlign = DL.getABITypeAlign(Ty);
9989
9990 // Don't promote to an alignment that would require dynamic stack
9991 // realignment which may conflict with optimizations such as tail call
9992 // optimization.
9993 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
9994 if (!TRI->hasStackRealignment(MF))
9995 if (MaybeAlign StackAlign = DL.getStackAlignment())
9996 NewDstAlign = std::min(a: NewDstAlign, b: *StackAlign);
9997
9998 if (NewDstAlign > DstAlign) {
9999 // Give the stack frame object a larger alignment if needed.
10000 if (MFI.getObjectAlign(ObjectIdx: FI->getIndex()) < NewDstAlign)
10001 MFI.setObjectAlignment(ObjectIdx: FI->getIndex(), Alignment: NewDstAlign);
10002 DstAlign = NewDstAlign;
10003 }
10004 }
10005
10006 // Prepare AAInfo for loads/stores after lowering this memmove.
10007 AAMDNodes NewAAInfo = AAInfo;
10008 NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
10009
10010 MachineMemOperand::Flags MMOFlags =
10011 isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone;
10012 uint64_t SrcOff = 0;
10013 SmallVector<SDValue, 8> LoadValues;
10014 SmallVector<SDValue, 8> LoadChains;
10015 SmallVector<SDValue, 8> OutChains;
10016 unsigned NumMemOps = MemOps.size();
10017 for (unsigned i = 0; i < NumMemOps; i++) {
10018 EVT VT = MemOps[i];
10019 unsigned VTSize = VT.getSizeInBits() / 8;
10020 SDValue Value;
10021 bool IsOverlapping = false;
10022
10023 if (i == NumMemOps - 1 && i != 0 && VTSize > Size - SrcOff) {
10024 // Issuing an unaligned load / store pair that overlaps with the previous
10025 // pair. Adjust the offset accordingly.
10026 SrcOff = Size - VTSize;
10027 IsOverlapping = true;
10028 }
10029
10030 // Calculate the actual alignment at the current offset. The alignment at
10031 // SrcOff may be lower than the base alignment, especially when using
10032 // overlapping loads.
10033 Align SrcAlignAtOffset = commonAlignment(A: SrcAlign, Offset: SrcOff);
10034 if (IsOverlapping) {
10035 // Verify that the target allows misaligned memory accesses at the
10036 // adjusted offset when using overlapping loads.
10037 unsigned Fast;
10038 if (!TLI.allowsMisalignedMemoryAccesses(VT, AddrSpace: SrcPtrInfo.getAddrSpace(),
10039 Alignment: SrcAlignAtOffset, Flags: MMOFlags,
10040 &Fast) ||
10041 !Fast) {
10042 // This should have been caught by findOptimalMemOpLowering, but verify
10043 // here for safety.
10044 return SDValue();
10045 }
10046 }
10047
10048 bool isDereferenceable =
10049 SrcPtrInfo.getWithOffset(O: SrcOff).isDereferenceable(Size: VTSize, C, DL);
10050 MachineMemOperand::Flags SrcMMOFlags = MMOFlags;
10051 if (isDereferenceable)
10052 SrcMMOFlags |= MachineMemOperand::MODereferenceable;
10053 Value =
10054 DAG.getLoad(VT, dl, Chain,
10055 Ptr: DAG.getObjectPtrOffset(SL: dl, Ptr: Src, Offset: TypeSize::getFixed(ExactSize: SrcOff)),
10056 PtrInfo: SrcPtrInfo.getWithOffset(O: SrcOff), Alignment: SrcAlignAtOffset,
10057 MMOFlags: SrcMMOFlags, Metadata: NewAAInfo);
10058 LoadValues.push_back(Elt: Value);
10059 LoadChains.push_back(Elt: Value.getValue(R: 1));
10060 SrcOff += VTSize;
10061 }
10062 Chain = DAG.getTokenFactor(DL: dl, Vals&: LoadChains);
10063 OutChains.clear();
10064 uint64_t DstOff = 0;
10065 for (unsigned i = 0; i < NumMemOps; i++) {
10066 EVT VT = MemOps[i];
10067 unsigned VTSize = VT.getSizeInBits() / 8;
10068 SDValue Store;
10069 bool IsOverlapping = false;
10070
10071 if (i == NumMemOps - 1 && i != 0 && VTSize > Size - DstOff) {
10072 // Issuing an unaligned load / store pair that overlaps with the previous
10073 // pair. Adjust the offset accordingly.
10074 DstOff = Size - VTSize;
10075 IsOverlapping = true;
10076 }
10077
10078 // Calculate the actual alignment at the current offset. The alignment at
10079 // DstOff may be lower than the base alignment, especially when using
10080 // overlapping stores.
10081 Align DstAlignAtOffset = commonAlignment(A: DstAlign, Offset: DstOff);
10082 if (IsOverlapping) {
10083 // Verify that the target allows misaligned memory accesses at the
10084 // adjusted offset when using overlapping stores.
10085 unsigned Fast;
10086 if (!TLI.allowsMisalignedMemoryAccesses(VT, AddrSpace: DstPtrInfo.getAddrSpace(),
10087 Alignment: DstAlignAtOffset, Flags: MMOFlags,
10088 &Fast) ||
10089 !Fast) {
10090 // This should have been caught by findOptimalMemOpLowering, but verify
10091 // here for safety.
10092 return SDValue();
10093 }
10094 }
10095 Store = DAG.getStore(
10096 Chain, dl, Val: LoadValues[i],
10097 Ptr: DAG.getObjectPtrOffset(SL: dl, Ptr: Dst, Offset: TypeSize::getFixed(ExactSize: DstOff)),
10098 PtrInfo: DstPtrInfo.getWithOffset(O: DstOff), Alignment: DstAlignAtOffset, MMOFlags,
10099 Metadata: NewAAInfo);
10100 OutChains.push_back(Elt: Store);
10101 DstOff += VTSize;
10102 }
10103
10104 return DAG.getTokenFactor(DL: dl, Vals&: OutChains);
10105}
10106
10107/// Lower the call to 'memset' intrinsic function into a series of store
10108/// operations.
10109///
10110/// \param DAG Selection DAG where lowered code is placed.
10111/// \param dl Link to corresponding IR location.
10112/// \param Chain Control flow dependency.
10113/// \param Dst Pointer to destination memory location.
10114/// \param Src Value of byte to write into the memory.
10115/// \param Size Number of bytes to write.
10116/// \param Alignment Alignment of the destination in bytes.
10117/// \param isVol True if destination is volatile.
10118/// \param AlwaysInline Makes sure no function call is generated.
10119/// \param DstPtrInfo IR information on the memory pointer.
10120/// \returns New head in the control flow, if lowering was successful, empty
10121/// SDValue otherwise.
10122///
10123/// The function tries to replace 'llvm.memset' intrinsic with several store
10124/// operations and value calculation code. This is usually profitable for small
10125/// memory size or when the semantic requires inlining.
10126static SDValue getMemsetStores(SelectionDAG &DAG, const SDLoc &dl,
10127 SDValue Chain, SDValue Dst, SDValue Src,
10128 uint64_t Size, Align Alignment, bool isVol,
10129 bool AlwaysInline, MachinePointerInfo DstPtrInfo,
10130 const AAMDNodes &AAInfo) {
10131 // Turn a memset of undef to nop.
10132 // FIXME: We need to honor volatile even is Src is undef.
10133 if (Src.isUndef())
10134 return Chain;
10135
10136 // Expand memset to a series of load/store ops if the size operand
10137 // falls below a certain threshold.
10138 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
10139 std::vector<EVT> MemOps;
10140 bool DstAlignCanChange = false;
10141 LLVMContext &C = *DAG.getContext();
10142 MachineFunction &MF = DAG.getMachineFunction();
10143 MachineFrameInfo &MFI = MF.getFrameInfo();
10144 bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
10145 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Dst);
10146 if (FI && !MFI.isFixedObjectIndex(ObjectIdx: FI->getIndex()))
10147 DstAlignCanChange = true;
10148 bool IsZeroVal = isNullConstant(V: Src);
10149 unsigned Limit = AlwaysInline ? ~0 : TLI.getMaxStoresPerMemset(OptSize);
10150
10151 EVT LargestVT;
10152 if (!TLI.findOptimalMemOpLowering(
10153 Context&: C, MemOps, Limit,
10154 Op: MemOp::Set(Size, DstAlignCanChange, DstAlign: Alignment, IsZeroMemset: IsZeroVal, IsVolatile: isVol),
10155 DstAS: DstPtrInfo.getAddrSpace(), SrcAS: ~0u, FuncAttributes: MF.getFunction().getAttributes(),
10156 LargestVT: &LargestVT))
10157 return SDValue();
10158
10159 if (DstAlignCanChange) {
10160 Type *Ty = MemOps[0].getTypeForEVT(Context&: *DAG.getContext());
10161 const DataLayout &DL = DAG.getDataLayout();
10162 Align NewAlign = DL.getABITypeAlign(Ty);
10163
10164 // Don't promote to an alignment that would require dynamic stack
10165 // realignment which may conflict with optimizations such as tail call
10166 // optimization.
10167 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
10168 if (!TRI->hasStackRealignment(MF))
10169 if (MaybeAlign StackAlign = DL.getStackAlignment())
10170 NewAlign = std::min(a: NewAlign, b: *StackAlign);
10171
10172 if (NewAlign > Alignment) {
10173 // Give the stack frame object a larger alignment if needed.
10174 if (MFI.getObjectAlign(ObjectIdx: FI->getIndex()) < NewAlign)
10175 MFI.setObjectAlignment(ObjectIdx: FI->getIndex(), Alignment: NewAlign);
10176 Alignment = NewAlign;
10177 }
10178 }
10179
10180 SmallVector<SDValue, 8> OutChains;
10181 uint64_t DstOff = 0;
10182 unsigned NumMemOps = MemOps.size();
10183
10184 // Find the largest store and generate the bit pattern for it.
10185 // If target didn't set LargestVT, compute it from MemOps.
10186 if (!LargestVT.isSimple()) {
10187 LargestVT = MemOps[0];
10188 for (unsigned i = 1; i < NumMemOps; i++)
10189 if (MemOps[i].bitsGT(VT: LargestVT))
10190 LargestVT = MemOps[i];
10191 }
10192 SDValue MemSetValue = getMemsetValue(Value: Src, VT: LargestVT, DAG, dl);
10193
10194 // Prepare AAInfo for loads/stores after lowering this memset.
10195 AAMDNodes NewAAInfo = AAInfo;
10196 NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
10197
10198 for (unsigned i = 0; i < NumMemOps; i++) {
10199 EVT VT = MemOps[i];
10200 unsigned VTSize = VT.getSizeInBits() / 8;
10201 // The target should specify store types that exactly cover the memset size
10202 // (with the last store potentially being oversized for overlapping stores).
10203 assert(Size > 0 && "Target specified more stores than needed in "
10204 "findOptimalMemOpLowering");
10205 if (VTSize > Size) {
10206 // Issuing an unaligned load / store pair that overlaps with the previous
10207 // pair. Adjust the offset accordingly.
10208 assert(i == NumMemOps-1 && i != 0);
10209 DstOff -= VTSize - Size;
10210 }
10211
10212 // If this store is smaller than the largest store see whether we can get
10213 // the smaller value for free with a truncate or extract vector element and
10214 // then store.
10215 SDValue Value = MemSetValue;
10216 if (VT.bitsLT(VT: LargestVT)) {
10217 unsigned Index;
10218 unsigned NElts = LargestVT.getSizeInBits() / VT.getSizeInBits();
10219 EVT SVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: VT.getScalarType(), NumElements: NElts);
10220 if (!LargestVT.isVector() && !VT.isVector() &&
10221 TLI.isTruncateFree(FromVT: LargestVT, ToVT: VT))
10222 Value = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, N1: MemSetValue);
10223 else if (LargestVT.isVector() && !VT.isVector() &&
10224 TLI.shallExtractConstSplatVectorElementToStore(
10225 VectorTy: LargestVT.getTypeForEVT(Context&: *DAG.getContext()),
10226 ElemSizeInBits: VT.getSizeInBits(), Index) &&
10227 TLI.isTypeLegal(VT: SVT) &&
10228 LargestVT.getSizeInBits() == SVT.getSizeInBits()) {
10229 // Target which can combine store(extractelement VectorTy, Idx) can get
10230 // the smaller value for free.
10231 SDValue TailValue = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: SVT, N1: MemSetValue);
10232 Value = DAG.getExtractVectorElt(DL: dl, VT, Vec: TailValue, Idx: Index);
10233 } else
10234 Value = getMemsetValue(Value: Src, VT, DAG, dl);
10235 }
10236 assert(Value.getValueType() == VT && "Value with wrong type.");
10237 SDValue Store = DAG.getStore(
10238 Chain, dl, Val: Value,
10239 Ptr: DAG.getObjectPtrOffset(SL: dl, Ptr: Dst, Offset: TypeSize::getFixed(ExactSize: DstOff)),
10240 PtrInfo: DstPtrInfo.getWithOffset(O: DstOff), Alignment,
10241 MMOFlags: isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone,
10242 Metadata: NewAAInfo);
10243 OutChains.push_back(Elt: Store);
10244 DstOff += VT.getSizeInBits() / 8;
10245 // For oversized overlapping stores, only subtract the remaining bytes.
10246 // For normal stores, subtract the full store size.
10247 if (VTSize > Size) {
10248 Size = 0;
10249 } else {
10250 Size -= VTSize;
10251 }
10252 }
10253
10254 // After processing all stores, Size should be exactly 0. Any remaining bytes
10255 // indicate a bug in the target's findOptimalMemOpLowering implementation.
10256 assert(Size == 0 && "Target's findOptimalMemOpLowering did not specify "
10257 "stores that exactly cover the memset size");
10258
10259 return DAG.getTokenFactor(DL: dl, Vals&: OutChains);
10260}
10261
10262static void checkAddrSpaceIsValidForLibcall(const TargetLowering *TLI,
10263 const DataLayout &DL, unsigned AS) {
10264 // Lowering memcpy / memset / memmove intrinsics to calls is only valid if all
10265 // pointer operands can be losslessly bitcasted to pointers of address space 0
10266 if (AS != 0 && !TLI->getTargetMachine().isNoopAddrSpaceCast(DL, SrcAS: AS, DestAS: 0)) {
10267 report_fatal_error(reason: "cannot lower memory intrinsic in address space " +
10268 Twine(AS));
10269 }
10270}
10271
10272/// The length of a memory intrinsic (e.g. number of bytes to copy) is unsigned
10273/// and may have any integer type. We zero-extend or truncate it to the pointer
10274/// type of the narrower address space being accessed.
10275static SDValue getMemIntrinsicSize(SelectionDAG &DAG, const SDLoc &dl,
10276 SDValue Size, unsigned DstAS,
10277 unsigned SrcAS) {
10278 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
10279 MVT DstVT = TLI.getPointerTy(DL: DAG.getDataLayout(), AS: DstAS);
10280 MVT SrcVT = TLI.getPointerTy(DL: DAG.getDataLayout(), AS: SrcAS);
10281 return DAG.getZExtOrTrunc(Op: Size, DL: dl, VT: DstVT.bitsLT(VT: SrcVT) ? DstVT : SrcVT);
10282}
10283
10284static bool isInTailCallPositionWrapper(const CallInst *CI,
10285 const SelectionDAG *SelDAG,
10286 bool AllowReturnsFirstArg) {
10287 if (!CI || !CI->isTailCall())
10288 return false;
10289 // TODO: Fix "returns-first-arg" determination so it doesn't depend on which
10290 // helper symbol we lower to.
10291 return isInTailCallPosition(Call: *CI, TM: SelDAG->getTarget(),
10292 ReturnsFirstArg: AllowReturnsFirstArg &&
10293 funcReturnsFirstArgOfCall(CI: *CI));
10294}
10295
10296static std::pair<SDValue, SDValue>
10297getRuntimeCallSDValueHelper(SDValue Chain, const SDLoc &dl,
10298 TargetLowering::ArgListTy &&Args,
10299 const CallInst *CI, RTLIB::Libcall Call,
10300 SelectionDAG *DAG, const TargetLowering *TLI) {
10301 RTLIB::LibcallImpl LCImpl = DAG->getLibcalls().getLibcallImpl(Call);
10302
10303 if (LCImpl == RTLIB::Unsupported)
10304 return {};
10305
10306 TargetLowering::CallLoweringInfo CLI(*DAG);
10307 bool IsTailCall =
10308 isInTailCallPositionWrapper(CI, SelDAG: DAG, /*AllowReturnsFirstArg=*/true) &&
10309 // Lowering doesn't support tail calling inside a function with
10310 // a swifterror argument yet.
10311 !DAG->hasSwiftErrorArg();
10312 SDValue Callee =
10313 DAG->getExternalSymbol(Libcall: LCImpl, VT: TLI->getPointerTy(DL: DAG->getDataLayout()));
10314
10315 CLI.setDebugLoc(dl)
10316 .setChain(Chain)
10317 .setLibCallee(CC: DAG->getLibcalls().getLibcallImplCallingConv(Call: LCImpl),
10318 ResultType: CI->getType(), Target: Callee, ArgsList: std::move(Args))
10319 .setTailCall(IsTailCall);
10320
10321 return TLI->LowerCallTo(CLI);
10322}
10323
10324std::pair<SDValue, SDValue> SelectionDAG::getStrcmp(SDValue Chain,
10325 const SDLoc &dl, SDValue S1,
10326 SDValue S2,
10327 const CallInst *CI) {
10328 PointerType *PT = PointerType::getUnqual(C&: *getContext());
10329 TargetLowering::ArgListTy Args = {{S1, PT}, {S2, PT}};
10330 return getRuntimeCallSDValueHelper(Chain, dl, Args: std::move(Args), CI,
10331 Call: RTLIB::STRCMP, DAG: this, TLI);
10332}
10333
10334std::pair<SDValue, SDValue> SelectionDAG::getStrstr(SDValue Chain,
10335 const SDLoc &dl, SDValue S1,
10336 SDValue S2,
10337 const CallInst *CI) {
10338 PointerType *PT = PointerType::getUnqual(C&: *getContext());
10339 TargetLowering::ArgListTy Args = {{S1, PT}, {S2, PT}};
10340 return getRuntimeCallSDValueHelper(Chain, dl, Args: std::move(Args), CI,
10341 Call: RTLIB::STRSTR, DAG: this, TLI);
10342}
10343
10344std::pair<SDValue, SDValue> SelectionDAG::getMemccpy(SDValue Chain,
10345 const SDLoc &dl,
10346 SDValue Dst, SDValue Src,
10347 SDValue C, SDValue Size,
10348 const CallInst *CI) {
10349 PointerType *PT = PointerType::getUnqual(C&: *getContext());
10350
10351 TargetLowering::ArgListTy Args = {
10352 {Dst, PT},
10353 {Src, PT},
10354 {C, Type::getInt32Ty(C&: *getContext())},
10355 {Size, getDataLayout().getIntPtrType(C&: *getContext())}};
10356 return getRuntimeCallSDValueHelper(Chain, dl, Args: std::move(Args), CI,
10357 Call: RTLIB::MEMCCPY, DAG: this, TLI);
10358}
10359
10360std::pair<SDValue, SDValue>
10361SelectionDAG::getMemcmp(SDValue Chain, const SDLoc &dl, SDValue Mem0,
10362 SDValue Mem1, SDValue Size, const CallInst *CI) {
10363 PointerType *PT = PointerType::getUnqual(C&: *getContext());
10364 TargetLowering::ArgListTy Args = {
10365 {Mem0, PT},
10366 {Mem1, PT},
10367 {Size, getDataLayout().getIntPtrType(C&: *getContext())}};
10368 return getRuntimeCallSDValueHelper(Chain, dl, Args: std::move(Args), CI,
10369 Call: RTLIB::MEMCMP, DAG: this, TLI);
10370}
10371
10372std::pair<SDValue, SDValue> SelectionDAG::getStrcpy(SDValue Chain,
10373 const SDLoc &dl,
10374 SDValue Dst, SDValue Src,
10375 const CallInst *CI) {
10376 PointerType *PT = PointerType::getUnqual(C&: *getContext());
10377 TargetLowering::ArgListTy Args = {{Dst, PT}, {Src, PT}};
10378 return getRuntimeCallSDValueHelper(Chain, dl, Args: std::move(Args), CI,
10379 Call: RTLIB::STRCPY, DAG: this, TLI);
10380}
10381
10382std::pair<SDValue, SDValue> SelectionDAG::getStrlen(SDValue Chain,
10383 const SDLoc &dl,
10384 SDValue Src,
10385 const CallInst *CI) {
10386 // Emit a library call.
10387 TargetLowering::ArgListTy Args = {
10388 {Src, PointerType::getUnqual(C&: *getContext())}};
10389 return getRuntimeCallSDValueHelper(Chain, dl, Args: std::move(Args), CI,
10390 Call: RTLIB::STRLEN, DAG: this, TLI);
10391}
10392
10393bool SelectionDAG::hasSwiftErrorArg() const {
10394 return TLI->supportSwiftError() &&
10395 MF->getFunction().getAttributes().hasAttrSomewhere(
10396 Kind: Attribute::SwiftError);
10397}
10398
10399SDValue SelectionDAG::getMemcpy(
10400 SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size,
10401 Align DstAlign, Align SrcAlign, bool isVol, bool AlwaysInline,
10402 const CallInst *CI, std::optional<bool> OverrideTailCall,
10403 MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo,
10404 const AAMDNodes &AAInfo, BatchAAResults *BatchAA) {
10405 Size = getMemIntrinsicSize(DAG&: *this, dl, Size, DstAS: DstPtrInfo.getAddrSpace(),
10406 SrcAS: SrcPtrInfo.getAddrSpace());
10407 // Check to see if we should lower the memcpy to loads and stores first.
10408 // For cases within the target-specified limits, this is the best choice.
10409 const MDNode *DstMemCacheHint =
10410 CI ? getMemCacheHint(I: *CI, /*OperandNo=*/0) : nullptr;
10411 const MDNode *SrcMemCacheHint =
10412 CI ? getMemCacheHint(I: *CI, /*OperandNo=*/1) : nullptr;
10413
10414 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Val&: Size);
10415 if (ConstantSize) {
10416 // Memcpy with size zero? Just return the original chain.
10417 if (ConstantSize->isZero())
10418 return Chain;
10419
10420 SDValue Result = getMemcpyLoadsAndStores(
10421 DAG&: *this, dl, Chain, Dst, Src, Size: ConstantSize->getZExtValue(), DstAlign,
10422 SrcAlign, isVol, AlwaysInline: false, DstPtrInfo, SrcPtrInfo, AAInfo, BatchAA,
10423 DstMemCacheHint, SrcMemCacheHint);
10424 if (Result.getNode())
10425 return Result;
10426 }
10427
10428 // Then check to see if we should lower the memcpy with target-specific
10429 // code. If the target chooses to do this, this is the next best.
10430 if (TSI) {
10431 bool IsTailCall = OverrideTailCall.value_or(u: isInTailCallPositionWrapper(
10432 CI, SelDAG: this, /*AllowReturnsFirstArg=*/false)) &&
10433 !hasSwiftErrorArg();
10434 [[maybe_unused]] SDValue OldRoot = getRoot();
10435 SDValue Result = TSI->EmitTargetCodeForMemcpy(
10436 DAG&: *this, dl, Chain, Op1: Dst, Op2: Src, Op3: Size, DstAlign, SrcAlign, isVolatile: isVol,
10437 AlwaysInline, IsTailCall, DstPtrInfo, SrcPtrInfo);
10438 assert((Result.getNode() || !IsTailCall || getRoot() != OldRoot) &&
10439 "target declined without clearing IsTailCall");
10440 if (Result.getNode() || IsTailCall)
10441 return Result;
10442 }
10443
10444 // If we really need inline code and the target declined to provide it,
10445 // use a (potentially long) sequence of loads and stores.
10446 if (AlwaysInline) {
10447 assert(ConstantSize && "AlwaysInline requires a constant size!");
10448 return getMemcpyLoadsAndStores(
10449 DAG&: *this, dl, Chain, Dst, Src, Size: ConstantSize->getZExtValue(), DstAlign,
10450 SrcAlign, isVol, AlwaysInline: true, DstPtrInfo, SrcPtrInfo, AAInfo, BatchAA,
10451 DstMemCacheHint, SrcMemCacheHint);
10452 }
10453
10454 checkAddrSpaceIsValidForLibcall(TLI, DL: getDataLayout(),
10455 AS: DstPtrInfo.getAddrSpace());
10456 checkAddrSpaceIsValidForLibcall(TLI, DL: getDataLayout(),
10457 AS: SrcPtrInfo.getAddrSpace());
10458
10459 // FIXME: If the memcpy is volatile (isVol), lowering it to a plain libc
10460 // memcpy is not guaranteed to be safe. libc memcpys aren't required to
10461 // respect volatile, so they may do things like read or write memory
10462 // beyond the given memory regions. But fixing this isn't easy, and most
10463 // people don't care.
10464
10465 // Emit a library call.
10466 TargetLowering::ArgListTy Args;
10467 Type *PtrTy = PointerType::getUnqual(C&: *getContext());
10468 Args.emplace_back(args&: Dst, args&: PtrTy);
10469 Args.emplace_back(args&: Src, args&: PtrTy);
10470 Args.emplace_back(args&: Size, args: getDataLayout().getIntPtrType(C&: *getContext()));
10471 // FIXME: pass in SDLoc
10472 TargetLowering::CallLoweringInfo CLI(*this);
10473 bool IsTailCall = false;
10474 RTLIB::LibcallImpl MemCpyImpl = Libcalls->getMemcpyImpl();
10475
10476 if (OverrideTailCall.has_value()) {
10477 IsTailCall = *OverrideTailCall;
10478 } else {
10479 bool LowersToMemcpy = MemCpyImpl == RTLIB::impl_memcpy;
10480 IsTailCall = isInTailCallPositionWrapper(CI, SelDAG: this, AllowReturnsFirstArg: LowersToMemcpy);
10481 }
10482 // Lowering doesn't support tail calling inside a function with a
10483 // swifterror argument yet.
10484 IsTailCall &= !hasSwiftErrorArg();
10485
10486 CLI.setDebugLoc(dl)
10487 .setChain(Chain)
10488 .setLibCallee(
10489 CC: Libcalls->getLibcallImplCallingConv(Call: MemCpyImpl),
10490 ResultType: Dst.getValueType().getTypeForEVT(Context&: *getContext()),
10491 Target: getExternalSymbol(Libcall: MemCpyImpl, VT: TLI->getPointerTy(DL: getDataLayout())),
10492 ArgsList: std::move(Args))
10493 .setDiscardResult()
10494 .setTailCall(IsTailCall);
10495
10496 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
10497 return CallResult.second;
10498}
10499
10500static TargetLoweringBase::ArgListEntry
10501getAtomicMemOpSizeArg(SelectionDAG *DAG, const SDLoc &dl, SDValue Size) {
10502 const Module &M = *DAG->getMachineFunction().getFunction().getParent();
10503 Type *SizeTTy = DAG->getLibInfo().getSizeTType(M);
10504 return TargetLoweringBase::ArgListEntry(
10505 DAG->getZExtOrTrunc(Op: Size, DL: dl, VT: MVT::getVT(Ty: SizeTTy)), SizeTTy);
10506}
10507
10508SDValue SelectionDAG::getAtomicMemcpy(SDValue Chain, const SDLoc &dl,
10509 SDValue Dst, SDValue Src, SDValue Size,
10510 Type *SizeTy, unsigned ElemSz,
10511 bool isTailCall,
10512 MachinePointerInfo DstPtrInfo,
10513 MachinePointerInfo SrcPtrInfo) {
10514 // Lowering doesn't support tail calling inside a function with a
10515 // swifterror argument yet.
10516 isTailCall &= !hasSwiftErrorArg();
10517
10518 // Emit a library call.
10519 TargetLowering::ArgListTy Args;
10520 Type *ArgTy = getDataLayout().getIntPtrType(C&: *getContext());
10521 Args.emplace_back(args&: Dst, args&: ArgTy);
10522 Args.emplace_back(args&: Src, args&: ArgTy);
10523 Args.emplace_back(args: getAtomicMemOpSizeArg(DAG: this, dl, Size));
10524
10525 RTLIB::Libcall LibraryCall =
10526 RTLIB::getMEMCPY_ELEMENT_UNORDERED_ATOMIC(ElementSize: ElemSz);
10527 RTLIB::LibcallImpl LibcallImpl = Libcalls->getLibcallImpl(Call: LibraryCall);
10528 if (LibcallImpl == RTLIB::Unsupported)
10529 report_fatal_error(reason: "Unsupported element size");
10530
10531 TargetLowering::CallLoweringInfo CLI(*this);
10532 CLI.setDebugLoc(dl)
10533 .setChain(Chain)
10534 .setLibCallee(
10535 CC: Libcalls->getLibcallImplCallingConv(Call: LibcallImpl),
10536 ResultType: Type::getVoidTy(C&: *getContext()),
10537 Target: getExternalSymbol(Libcall: LibcallImpl, VT: TLI->getPointerTy(DL: getDataLayout())),
10538 ArgsList: std::move(Args))
10539 .setDiscardResult()
10540 .setTailCall(isTailCall);
10541
10542 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
10543 return CallResult.second;
10544}
10545
10546SDValue SelectionDAG::getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst,
10547 SDValue Src, SDValue Size, Align DstAlign,
10548 Align SrcAlign, bool isVol, const CallInst *CI,
10549 std::optional<bool> OverrideTailCall,
10550 MachinePointerInfo DstPtrInfo,
10551 MachinePointerInfo SrcPtrInfo,
10552 const AAMDNodes &AAInfo,
10553 BatchAAResults *BatchAA) {
10554 Size = getMemIntrinsicSize(DAG&: *this, dl, Size, DstAS: DstPtrInfo.getAddrSpace(),
10555 SrcAS: SrcPtrInfo.getAddrSpace());
10556 // Check to see if we should lower the memmove to loads and stores first.
10557 // For cases within the target-specified limits, this is the best choice.
10558 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Val&: Size);
10559 if (ConstantSize) {
10560 // Memmove with size zero? Just return the original chain.
10561 if (ConstantSize->isZero())
10562 return Chain;
10563
10564 SDValue Result = getMemmoveLoadsAndStores(
10565 DAG&: *this, dl, Chain, Dst, Src, Size: ConstantSize->getZExtValue(), DstAlign,
10566 SrcAlign, isVol, AlwaysInline: false, DstPtrInfo, SrcPtrInfo, AAInfo);
10567 if (Result.getNode())
10568 return Result;
10569 }
10570
10571 // Then check to see if we should lower the memmove with target-specific
10572 // code. If the target chooses to do this, this is the next best.
10573 if (TSI) {
10574 bool IsTailCall = OverrideTailCall.value_or(u: isInTailCallPositionWrapper(
10575 CI, SelDAG: this, /*AllowReturnsFirstArg=*/false)) &&
10576 !hasSwiftErrorArg();
10577 [[maybe_unused]] SDValue OldRoot = getRoot();
10578 SDValue Result = TSI->EmitTargetCodeForMemmove(
10579 DAG&: *this, dl, Chain, Op1: Dst, Op2: Src, Op3: Size, DstAlign, SrcAlign, isVolatile: isVol, IsTailCall,
10580 DstPtrInfo, SrcPtrInfo);
10581 assert((Result.getNode() || !IsTailCall || getRoot() != OldRoot) &&
10582 "target declined without clearing IsTailCall");
10583 if (Result.getNode() || IsTailCall)
10584 return Result;
10585 }
10586
10587 checkAddrSpaceIsValidForLibcall(TLI, DL: getDataLayout(),
10588 AS: DstPtrInfo.getAddrSpace());
10589 checkAddrSpaceIsValidForLibcall(TLI, DL: getDataLayout(),
10590 AS: SrcPtrInfo.getAddrSpace());
10591
10592 // FIXME: If the memmove is volatile, lowering it to plain libc memmove may
10593 // not be safe. See memcpy above for more details.
10594
10595 // Emit a library call.
10596 TargetLowering::ArgListTy Args;
10597 Type *PtrTy = PointerType::getUnqual(C&: *getContext());
10598 Args.emplace_back(args&: Dst, args&: PtrTy);
10599 Args.emplace_back(args&: Src, args&: PtrTy);
10600 Args.emplace_back(args&: Size, args: getDataLayout().getIntPtrType(C&: *getContext()));
10601 // FIXME: pass in SDLoc
10602 TargetLowering::CallLoweringInfo CLI(*this);
10603
10604 RTLIB::LibcallImpl MemmoveImpl = Libcalls->getLibcallImpl(Call: RTLIB::MEMMOVE);
10605
10606 bool IsTailCall = false;
10607 if (OverrideTailCall.has_value()) {
10608 IsTailCall = *OverrideTailCall;
10609 } else {
10610 bool LowersToMemmove = MemmoveImpl == RTLIB::impl_memmove;
10611 IsTailCall = isInTailCallPositionWrapper(CI, SelDAG: this, AllowReturnsFirstArg: LowersToMemmove);
10612 }
10613 // Lowering doesn't support tail calling inside a function with a
10614 // swifterror argument yet.
10615 IsTailCall &= !hasSwiftErrorArg();
10616
10617 CLI.setDebugLoc(dl)
10618 .setChain(Chain)
10619 .setLibCallee(
10620 CC: Libcalls->getLibcallImplCallingConv(Call: MemmoveImpl),
10621 ResultType: Dst.getValueType().getTypeForEVT(Context&: *getContext()),
10622 Target: getExternalSymbol(Libcall: MemmoveImpl, VT: TLI->getPointerTy(DL: getDataLayout())),
10623 ArgsList: std::move(Args))
10624 .setDiscardResult()
10625 .setTailCall(IsTailCall);
10626
10627 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
10628 return CallResult.second;
10629}
10630
10631SDValue SelectionDAG::getAtomicMemmove(SDValue Chain, const SDLoc &dl,
10632 SDValue Dst, SDValue Src, SDValue Size,
10633 Type *SizeTy, unsigned ElemSz,
10634 bool isTailCall,
10635 MachinePointerInfo DstPtrInfo,
10636 MachinePointerInfo SrcPtrInfo) {
10637 // Lowering doesn't support tail calling inside a function with a
10638 // swifterror argument yet.
10639 isTailCall &= !hasSwiftErrorArg();
10640
10641 // Emit a library call.
10642 TargetLowering::ArgListTy Args;
10643 Type *IntPtrTy = getDataLayout().getIntPtrType(C&: *getContext());
10644 Args.emplace_back(args&: Dst, args&: IntPtrTy);
10645 Args.emplace_back(args&: Src, args&: IntPtrTy);
10646 Args.emplace_back(args: getAtomicMemOpSizeArg(DAG: this, dl, Size));
10647
10648 RTLIB::Libcall LibraryCall =
10649 RTLIB::getMEMMOVE_ELEMENT_UNORDERED_ATOMIC(ElementSize: ElemSz);
10650 RTLIB::LibcallImpl LibcallImpl = Libcalls->getLibcallImpl(Call: LibraryCall);
10651 if (LibcallImpl == RTLIB::Unsupported)
10652 report_fatal_error(reason: "Unsupported element size");
10653
10654 TargetLowering::CallLoweringInfo CLI(*this);
10655 CLI.setDebugLoc(dl)
10656 .setChain(Chain)
10657 .setLibCallee(
10658 CC: Libcalls->getLibcallImplCallingConv(Call: LibcallImpl),
10659 ResultType: Type::getVoidTy(C&: *getContext()),
10660 Target: getExternalSymbol(Libcall: LibcallImpl, VT: TLI->getPointerTy(DL: getDataLayout())),
10661 ArgsList: std::move(Args))
10662 .setDiscardResult()
10663 .setTailCall(isTailCall);
10664
10665 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
10666 return CallResult.second;
10667}
10668
10669SDValue SelectionDAG::getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst,
10670 SDValue Src, SDValue Size, Align Alignment,
10671 bool isVol, bool AlwaysInline,
10672 const CallInst *CI,
10673 MachinePointerInfo DstPtrInfo,
10674 const AAMDNodes &AAInfo) {
10675 Size = getMemIntrinsicSize(DAG&: *this, dl, Size, DstAS: DstPtrInfo.getAddrSpace(),
10676 SrcAS: DstPtrInfo.getAddrSpace());
10677 // Check to see if we should lower the memset to stores first.
10678 // For cases within the target-specified limits, this is the best choice.
10679 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Val&: Size);
10680 if (ConstantSize) {
10681 // Memset with size zero? Just return the original chain.
10682 if (ConstantSize->isZero())
10683 return Chain;
10684
10685 SDValue Result = getMemsetStores(DAG&: *this, dl, Chain, Dst, Src,
10686 Size: ConstantSize->getZExtValue(), Alignment,
10687 isVol, AlwaysInline: false, DstPtrInfo, AAInfo);
10688
10689 if (Result.getNode())
10690 return Result;
10691 }
10692
10693 // Then check to see if we should lower the memset with target-specific
10694 // code. If the target chooses to do this, this is the next best.
10695 if (TSI) {
10696 bool IsTailCall =
10697 isInTailCallPositionWrapper(CI, SelDAG: this, /*AllowReturnsFirstArg=*/false) &&
10698 !hasSwiftErrorArg();
10699 [[maybe_unused]] SDValue OldRoot = getRoot();
10700 SDValue Result = TSI->EmitTargetCodeForMemset(
10701 DAG&: *this, dl, Chain, Op1: Dst, Op2: Src, Op3: Size, Alignment, isVolatile: isVol, AlwaysInline,
10702 IsTailCall, DstPtrInfo);
10703 assert((Result.getNode() || !IsTailCall || getRoot() != OldRoot) &&
10704 "target declined without clearing IsTailCall");
10705 if (Result.getNode() || IsTailCall)
10706 return Result;
10707 }
10708
10709 // If we really need inline code and the target declined to provide it,
10710 // use a (potentially long) sequence of loads and stores.
10711 if (AlwaysInline) {
10712 assert(ConstantSize && "AlwaysInline requires a constant size!");
10713 SDValue Result = getMemsetStores(DAG&: *this, dl, Chain, Dst, Src,
10714 Size: ConstantSize->getZExtValue(), Alignment,
10715 isVol, AlwaysInline: true, DstPtrInfo, AAInfo);
10716 assert(Result &&
10717 "getMemsetStores must return a valid sequence when AlwaysInline");
10718 return Result;
10719 }
10720
10721 checkAddrSpaceIsValidForLibcall(TLI, DL: getDataLayout(),
10722 AS: DstPtrInfo.getAddrSpace());
10723
10724 // Emit a library call.
10725 auto &Ctx = *getContext();
10726 const auto& DL = getDataLayout();
10727
10728 TargetLowering::CallLoweringInfo CLI(*this);
10729 // FIXME: pass in SDLoc
10730 CLI.setDebugLoc(dl).setChain(Chain);
10731
10732 RTLIB::LibcallImpl BzeroImpl = Libcalls->getLibcallImpl(Call: RTLIB::BZERO);
10733 bool UseBZero = BzeroImpl != RTLIB::Unsupported && isNullConstant(V: Src);
10734
10735 // If zeroing out and bzero is present, use it.
10736 if (UseBZero) {
10737 TargetLowering::ArgListTy Args;
10738 Args.emplace_back(args&: Dst, args: PointerType::getUnqual(C&: Ctx));
10739 Args.emplace_back(args&: Size, args: DL.getIntPtrType(C&: Ctx));
10740 CLI.setLibCallee(
10741 CC: Libcalls->getLibcallImplCallingConv(Call: BzeroImpl), ResultType: Type::getVoidTy(C&: Ctx),
10742 Target: getExternalSymbol(Libcall: BzeroImpl, VT: TLI->getPointerTy(DL)), ArgsList: std::move(Args));
10743 } else {
10744 RTLIB::LibcallImpl MemsetImpl = Libcalls->getLibcallImpl(Call: RTLIB::MEMSET);
10745
10746 MVT IntVT = MVT::getIntegerVT(BitWidth: LibInfo->getIntSize());
10747 TargetLowering::ArgListTy Args;
10748 Args.emplace_back(args&: Dst, args: PointerType::getUnqual(C&: Ctx));
10749 Args.emplace_back(args: getAnyExtOrTrunc(Op: Src, DL: dl, VT: IntVT),
10750 args: Type::getIntNTy(C&: Ctx, N: LibInfo->getIntSize()),
10751 args: LibInfo->getExtAttrForI32Param());
10752 Args.emplace_back(args&: Size, args: DL.getIntPtrType(C&: Ctx));
10753
10754 CLI.setLibCallee(CC: Libcalls->getLibcallImplCallingConv(Call: MemsetImpl),
10755 ResultType: Dst.getValueType().getTypeForEVT(Context&: Ctx),
10756 Target: getExternalSymbol(Libcall: MemsetImpl, VT: TLI->getPointerTy(DL)),
10757 ArgsList: std::move(Args));
10758 }
10759
10760 RTLIB::LibcallImpl MemsetImpl = Libcalls->getLibcallImpl(Call: RTLIB::MEMSET);
10761 bool LowersToMemset = MemsetImpl == RTLIB::impl_memset;
10762
10763 // If we're going to use bzero, make sure not to tail call unless the
10764 // subsequent return doesn't need a value, as bzero doesn't return the first
10765 // arg unlike memset.
10766 bool ReturnsFirstArg = CI && funcReturnsFirstArgOfCall(CI: *CI) && !UseBZero;
10767 bool IsTailCall = CI && CI->isTailCall() &&
10768 isInTailCallPosition(Call: *CI, TM: getTarget(),
10769 ReturnsFirstArg: ReturnsFirstArg && LowersToMemset) &&
10770 // Lowering doesn't support tail calling inside a function
10771 // with a swifterror argument yet.
10772 !hasSwiftErrorArg();
10773 CLI.setDiscardResult().setTailCall(IsTailCall);
10774
10775 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
10776 return CallResult.second;
10777}
10778
10779SDValue SelectionDAG::getAtomicMemset(SDValue Chain, const SDLoc &dl,
10780 SDValue Dst, SDValue Value, SDValue Size,
10781 Type *SizeTy, unsigned ElemSz,
10782 bool isTailCall,
10783 MachinePointerInfo DstPtrInfo) {
10784 // Lowering doesn't support tail calling inside a function with a
10785 // swifterror argument yet.
10786 isTailCall &= !hasSwiftErrorArg();
10787
10788 // Emit a library call.
10789 MVT IntVT = MVT::getIntegerVT(BitWidth: LibInfo->getIntSize());
10790 TargetLowering::ArgListTy Args;
10791 Args.emplace_back(args&: Dst, args: getDataLayout().getIntPtrType(C&: *getContext()));
10792 Args.emplace_back(args: getAnyExtOrTrunc(Op: Value, DL: dl, VT: IntVT),
10793 args: Type::getIntNTy(C&: *getContext(), N: LibInfo->getIntSize()),
10794 args: LibInfo->getExtAttrForI32Param());
10795 Args.emplace_back(args: getAtomicMemOpSizeArg(DAG: this, dl, Size));
10796
10797 RTLIB::Libcall LibraryCall =
10798 RTLIB::getMEMSET_ELEMENT_UNORDERED_ATOMIC(ElementSize: ElemSz);
10799 RTLIB::LibcallImpl LibcallImpl = Libcalls->getLibcallImpl(Call: LibraryCall);
10800 if (LibcallImpl == RTLIB::Unsupported)
10801 report_fatal_error(reason: "Unsupported element size");
10802
10803 TargetLowering::CallLoweringInfo CLI(*this);
10804 CLI.setDebugLoc(dl)
10805 .setChain(Chain)
10806 .setLibCallee(
10807 CC: Libcalls->getLibcallImplCallingConv(Call: LibcallImpl),
10808 ResultType: Type::getVoidTy(C&: *getContext()),
10809 Target: getExternalSymbol(Libcall: LibcallImpl, VT: TLI->getPointerTy(DL: getDataLayout())),
10810 ArgsList: std::move(Args))
10811 .setDiscardResult()
10812 .setTailCall(isTailCall);
10813
10814 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
10815 return CallResult.second;
10816}
10817
10818SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
10819 SDVTList VTList, ArrayRef<SDValue> Ops,
10820 MachineMemOperand *MMO,
10821 ISD::LoadExtType ExtType) {
10822 SDNodeKey ID(Opcode, VTList, Ops);
10823 ID.AddInteger(I: MemVT.getRawBits());
10824 ID.AddInteger(I: getSyntheticNodeSubclassData<AtomicSDNode>(
10825 IROrder: dl.getIROrder(), Args&: Opcode, Args&: VTList, Args&: MemVT, Args&: MMO, Args&: ExtType));
10826 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10827 ID.AddInteger(I: MMO->getFlags());
10828 FoldingSetInsertToken InsertToken;
10829 if (auto *E = cast_or_null<AtomicSDNode>(Val: lookupNode(Key: ID, DL: dl, InsertToken))) {
10830 E->refineAlignment(NewMMO: MMO);
10831 E->refineMMOMetadata(NewMMO: MMO);
10832 return SDValue(E, 0);
10833 }
10834
10835 auto *N = newSDNode<AtomicSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: Opcode,
10836 Args&: VTList, Args&: MemVT, Args&: MMO, Args&: ExtType);
10837 createOperands(Node: N, Vals: Ops);
10838
10839 CSEMap.insert(N, Token: InsertToken);
10840 InsertNode(N);
10841 SDValue V(N, 0);
10842 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10843 return V;
10844}
10845
10846SDValue SelectionDAG::getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl,
10847 EVT MemVT, SDVTList VTs, SDValue Chain,
10848 SDValue Ptr, SDValue Cmp, SDValue Swp,
10849 MachineMemOperand *MMO) {
10850 assert(Opcode == ISD::ATOMIC_CMP_SWAP ||
10851 Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
10852 assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
10853
10854 SDValue Ops[] = {Chain, Ptr, Cmp, Swp};
10855 return getAtomic(Opcode, dl, MemVT, VTList: VTs, Ops, MMO);
10856}
10857
10858SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
10859 SDValue Chain, SDValue Ptr, SDValue Val,
10860 MachineMemOperand *MMO) {
10861 assert((Opcode == ISD::ATOMIC_LOAD_ADD || Opcode == ISD::ATOMIC_LOAD_SUB ||
10862 Opcode == ISD::ATOMIC_LOAD_AND || Opcode == ISD::ATOMIC_LOAD_CLR ||
10863 Opcode == ISD::ATOMIC_LOAD_OR || Opcode == ISD::ATOMIC_LOAD_XOR ||
10864 Opcode == ISD::ATOMIC_LOAD_NAND || Opcode == ISD::ATOMIC_LOAD_MIN ||
10865 Opcode == ISD::ATOMIC_LOAD_MAX || Opcode == ISD::ATOMIC_LOAD_UMIN ||
10866 Opcode == ISD::ATOMIC_LOAD_UMAX || Opcode == ISD::ATOMIC_LOAD_FADD ||
10867 Opcode == ISD::ATOMIC_LOAD_FSUB || Opcode == ISD::ATOMIC_LOAD_FMAX ||
10868 Opcode == ISD::ATOMIC_LOAD_FMIN ||
10869 Opcode == ISD::ATOMIC_LOAD_FMINIMUM ||
10870 Opcode == ISD::ATOMIC_LOAD_FMAXIMUM ||
10871 Opcode == ISD::ATOMIC_LOAD_FMINIMUMNUM ||
10872 Opcode == ISD::ATOMIC_LOAD_FMAXIMUMNUM ||
10873 Opcode == ISD::ATOMIC_LOAD_UINC_WRAP ||
10874 Opcode == ISD::ATOMIC_LOAD_UDEC_WRAP ||
10875 Opcode == ISD::ATOMIC_LOAD_USUB_COND ||
10876 Opcode == ISD::ATOMIC_LOAD_USUB_SAT || Opcode == ISD::ATOMIC_SWAP ||
10877 Opcode == ISD::ATOMIC_STORE) &&
10878 "Invalid Atomic Op");
10879
10880 EVT VT = Val.getValueType();
10881
10882 SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(VT: MVT::Other) :
10883 getVTList(VT1: VT, VT2: MVT::Other);
10884 SDValue Ops[] = {Chain, Ptr, Val};
10885 return getAtomic(Opcode, dl, MemVT, VTList: VTs, Ops, MMO);
10886}
10887
10888SDValue SelectionDAG::getAtomicLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
10889 EVT MemVT, EVT VT, SDValue Chain,
10890 SDValue Ptr, MachineMemOperand *MMO) {
10891 SDVTList VTs = getVTList(VT1: VT, VT2: MVT::Other);
10892 SDValue Ops[] = {Chain, Ptr};
10893 return getAtomic(Opcode: ISD::ATOMIC_LOAD, dl, MemVT, VTList: VTs, Ops, MMO, ExtType);
10894}
10895
10896/// getMergeValues - Create a MERGE_VALUES node from the given operands.
10897SDValue SelectionDAG::getMergeValues(ArrayRef<SDValue> Ops, const SDLoc &dl) {
10898 if (Ops.size() == 1)
10899 return Ops[0];
10900
10901 SmallVector<EVT, 4> VTs;
10902 VTs.reserve(N: Ops.size());
10903 for (const SDValue &Op : Ops)
10904 VTs.push_back(Elt: Op.getValueType());
10905 return getNode(Opcode: ISD::MERGE_VALUES, DL: dl, VTList: getVTList(VTs), Ops);
10906}
10907
10908SDValue SelectionDAG::getErrorMergeValues(ArrayRef<EVT> ResultTypes,
10909 SDValue Chain, const SDLoc &dl) {
10910 SmallVector<SDValue, 4> RetValues;
10911 RetValues.reserve(N: ResultTypes.size());
10912 for (EVT VT : ResultTypes)
10913 RetValues.push_back(Elt: VT == MVT::Other ? Chain : getPOISON(VT));
10914 return getMergeValues(Ops: RetValues, dl);
10915}
10916
10917SDValue SelectionDAG::getMemIntrinsicNode(
10918 unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef<SDValue> Ops,
10919 EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment,
10920 MachineMemOperand::Flags Flags, LocationSize Size,
10921 const AAMDNodes &AAInfo) {
10922 if (Size.hasValue() && !Size.getValue())
10923 Size = LocationSize::precise(Value: MemVT.getStoreSize());
10924
10925 MachineFunction &MF = getMachineFunction();
10926 MachineMemOperand *MMO =
10927 MF.getMachineMemOperand(PtrInfo, F: Flags, Size, BaseAlignment: Alignment, Metadata: AAInfo);
10928
10929 return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, MMO);
10930}
10931
10932SDValue SelectionDAG::getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl,
10933 SDVTList VTList,
10934 ArrayRef<SDValue> Ops, EVT MemVT,
10935 MachineMemOperand *MMO) {
10936 return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, MMOs: ArrayRef(MMO));
10937}
10938
10939SDValue SelectionDAG::getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl,
10940 SDVTList VTList,
10941 ArrayRef<SDValue> Ops, EVT MemVT,
10942 ArrayRef<MachineMemOperand *> MMOs) {
10943 assert(!MMOs.empty() && "Must have at least one MMO");
10944 assert(
10945 (Opcode == ISD::INTRINSIC_VOID || Opcode == ISD::INTRINSIC_W_CHAIN ||
10946 Opcode == ISD::PREFETCH ||
10947 (Opcode <= (unsigned)std::numeric_limits<int>::max() &&
10948 Opcode >= ISD::BUILTIN_OP_END && TSI->isTargetMemoryOpcode(Opcode))) &&
10949 "Opcode is not a memory-accessing opcode!");
10950
10951 PointerUnion<MachineMemOperand *, MachineMemOperand **> MemRefs;
10952 if (MMOs.size() == 1) {
10953 MemRefs = MMOs[0];
10954 } else {
10955 // Allocate: [size_t count][MMO*][MMO*]...
10956 size_t AllocSize =
10957 sizeof(size_t) + MMOs.size() * sizeof(MachineMemOperand *);
10958 void *Buffer = Allocator.Allocate(Size: AllocSize, Alignment: alignof(size_t));
10959 size_t *CountPtr = static_cast<size_t *>(Buffer);
10960 *CountPtr = MMOs.size();
10961 MachineMemOperand **Array =
10962 reinterpret_cast<MachineMemOperand **>(CountPtr + 1);
10963 llvm::copy(Range&: MMOs, Out: Array);
10964 MemRefs = Array;
10965 }
10966
10967 // Memoize the node unless it returns a glue result.
10968 MemIntrinsicSDNode *N;
10969 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
10970 SDNodeKey ID(Opcode, VTList, Ops);
10971 ID.AddInteger(I: getSyntheticNodeSubclassData<MemIntrinsicSDNode>(
10972 Opc: Opcode, Order: dl.getIROrder(), VTs: VTList, MemoryVT: MemVT, MemRefs));
10973 ID.AddInteger(I: MemVT.getRawBits());
10974 for (const MachineMemOperand *MMO : MMOs) {
10975 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10976 ID.AddInteger(I: MMO->getFlags());
10977 }
10978 FoldingSetInsertToken InsertToken;
10979 if (SDNode *E = lookupNode(Key: ID, DL: dl, InsertToken)) {
10980 cast<MemIntrinsicSDNode>(Val: E)->refineAlignment(NewMMOs: MMOs);
10981 return SDValue(E, 0);
10982 }
10983
10984 N = newSDNode<MemIntrinsicSDNode>(Args&: Opcode, Args: dl.getIROrder(), Args: dl.getDebugLoc(),
10985 Args&: VTList, Args&: MemVT, Args&: MemRefs);
10986 createOperands(Node: N, Vals: Ops);
10987 CSEMap.insert(N, Token: InsertToken);
10988 } else {
10989 N = newSDNode<MemIntrinsicSDNode>(Args&: Opcode, Args: dl.getIROrder(), Args: dl.getDebugLoc(),
10990 Args&: VTList, Args&: MemVT, Args&: MemRefs);
10991 createOperands(Node: N, Vals: Ops);
10992 }
10993 InsertNode(N);
10994 SDValue V(N, 0);
10995 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10996 return V;
10997}
10998
10999SDValue SelectionDAG::getLifetimeNode(bool IsStart, const SDLoc &dl,
11000 SDValue Chain, int FrameIndex) {
11001 const unsigned Opcode = IsStart ? ISD::LIFETIME_START : ISD::LIFETIME_END;
11002 const auto VTs = getVTList(VT: MVT::Other);
11003 SDValue Ops[2] = {
11004 Chain,
11005 getFrameIndex(FI: FrameIndex,
11006 VT: getTargetLoweringInfo().getFrameIndexTy(DL: getDataLayout()),
11007 isTarget: true)};
11008
11009 SDNodeKey ID(Opcode, VTs, Ops);
11010 FoldingSetInsertToken InsertToken;
11011 if (SDNode *E = lookupNode(Key: ID, DL: dl, InsertToken))
11012 return SDValue(E, 0);
11013
11014 LifetimeSDNode *N =
11015 newSDNode<LifetimeSDNode>(Args: Opcode, Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args: VTs);
11016 createOperands(Node: N, Vals: Ops);
11017 CSEMap.insert(N, Token: InsertToken);
11018 InsertNode(N);
11019 SDValue V(N, 0);
11020 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11021 return V;
11022}
11023
11024SDValue SelectionDAG::getPseudoProbeNode(const SDLoc &Dl, SDValue Chain,
11025 uint64_t Guid, uint64_t Index,
11026 uint32_t Attr) {
11027 const unsigned Opcode = ISD::PSEUDO_PROBE;
11028 const auto VTs = getVTList(VT: MVT::Other);
11029 SDValue Ops[] = {Chain};
11030 SDNodeKey ID(Opcode, VTs, Ops);
11031 ID.AddInteger(I: Guid);
11032 ID.AddInteger(I: Index);
11033 ID.AddInteger(I: Attr);
11034 FoldingSetInsertToken InsertToken;
11035 if (SDNode *E = lookupNode(Key: ID, DL: Dl, InsertToken))
11036 return SDValue(E, 0);
11037
11038 auto *N = newSDNode<PseudoProbeSDNode>(
11039 Args: Opcode, Args: Dl.getIROrder(), Args: Dl.getDebugLoc(), Args: VTs, Args&: Guid, Args&: Index, Args&: Attr);
11040 createOperands(Node: N, Vals: Ops);
11041 CSEMap.insert(N, Token: InsertToken);
11042 InsertNode(N);
11043 SDValue V(N, 0);
11044 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11045 return V;
11046}
11047
11048/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
11049/// MachinePointerInfo record from it. This is particularly useful because the
11050/// code generator has many cases where it doesn't bother passing in a
11051/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
11052static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info,
11053 SelectionDAG &DAG, SDValue Ptr,
11054 int64_t Offset = 0) {
11055 // If this is FI+Offset, we can model it.
11056 if (const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Ptr))
11057 return MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(),
11058 FI: FI->getIndex(), Offset);
11059
11060 // If this is (FI+Offset1)+Offset2, we can model it.
11061 if (Ptr.getOpcode() != ISD::ADD ||
11062 !isa<ConstantSDNode>(Val: Ptr.getOperand(i: 1)) ||
11063 !isa<FrameIndexSDNode>(Val: Ptr.getOperand(i: 0)))
11064 return Info;
11065
11066 int FI = cast<FrameIndexSDNode>(Val: Ptr.getOperand(i: 0))->getIndex();
11067 return MachinePointerInfo::getFixedStack(
11068 MF&: DAG.getMachineFunction(), FI,
11069 Offset: Offset + cast<ConstantSDNode>(Val: Ptr.getOperand(i: 1))->getSExtValue());
11070}
11071
11072/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
11073/// MachinePointerInfo record from it. This is particularly useful because the
11074/// code generator has many cases where it doesn't bother passing in a
11075/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
11076static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info,
11077 SelectionDAG &DAG, SDValue Ptr,
11078 SDValue OffsetOp) {
11079 // If the 'Offset' value isn't a constant, we can't handle this.
11080 if (ConstantSDNode *OffsetNode = dyn_cast<ConstantSDNode>(Val&: OffsetOp))
11081 return InferPointerInfo(Info, DAG, Ptr, Offset: OffsetNode->getSExtValue());
11082 if (OffsetOp.isUndef())
11083 return InferPointerInfo(Info, DAG, Ptr);
11084 return Info;
11085}
11086
11087SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
11088 EVT VT, const SDLoc &dl, SDValue Chain,
11089 SDValue Ptr, SDValue Offset,
11090 MachinePointerInfo PtrInfo, EVT MemVT,
11091 Align Alignment,
11092 MachineMemOperand::Flags MMOFlags,
11093 const MMOMetadata &Metadata) {
11094 assert(Chain.getValueType() == MVT::Other &&
11095 "Invalid chain type");
11096
11097 MMOFlags |= MachineMemOperand::MOLoad;
11098 assert((MMOFlags & MachineMemOperand::MOStore) == 0);
11099 // If we don't have a PtrInfo, infer the trivial frame index case to simplify
11100 // clients.
11101 if (PtrInfo.V.isNull())
11102 PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr, OffsetOp: Offset);
11103
11104 TypeSize Size = MemVT.getStoreSize();
11105 MachineFunction &MF = getMachineFunction();
11106 MachineMemOperand *MMO =
11107 MF.getMachineMemOperand(PtrInfo, F: MMOFlags, Size, BaseAlignment: Alignment, Metadata);
11108 return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO);
11109}
11110
11111SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
11112 EVT VT, const SDLoc &dl, SDValue Chain,
11113 SDValue Ptr, SDValue Offset, EVT MemVT,
11114 MachineMemOperand *MMO) {
11115 if (VT == MemVT) {
11116 ExtType = ISD::NON_EXTLOAD;
11117 } else if (ExtType == ISD::NON_EXTLOAD) {
11118 assert(VT == MemVT && "Non-extending load from different memory type!");
11119 } else {
11120 // Extending load.
11121 assert(MemVT.getScalarType().bitsLT(VT.getScalarType()) &&
11122 "Should only be an extending load, not truncating!");
11123 assert(VT.isInteger() == MemVT.isInteger() &&
11124 "Cannot convert from FP to Int or Int -> FP!");
11125 assert(VT.isVector() == MemVT.isVector() &&
11126 "Cannot use an ext load to convert to or from a vector!");
11127 assert((!VT.isVector() ||
11128 VT.getVectorElementCount() == MemVT.getVectorElementCount()) &&
11129 "Cannot use an ext load to change the number of vector elements!");
11130 }
11131
11132 assert((!MMO->getRanges() ||
11133 (mdconst::extract<ConstantInt>(MMO->getRanges()->getOperand(0))
11134 ->getBitWidth() == MemVT.getScalarSizeInBits() &&
11135 MemVT.isInteger())) &&
11136 "Range metadata and load type must match!");
11137
11138 bool Indexed = AM != ISD::UNINDEXED;
11139 assert((Indexed || Offset.getOpcode() == ISD::POISON) &&
11140 "Unindexed load with an offset!");
11141
11142 SDVTList VTs = Indexed ?
11143 getVTList(VT1: VT, VT2: Ptr.getValueType(), VT3: MVT::Other) : getVTList(VT1: VT, VT2: MVT::Other);
11144 SDValue Ops[] = { Chain, Ptr, Offset };
11145 SDNodeKey ID(ISD::LOAD, VTs, Ops);
11146 ID.AddInteger(I: MemVT.getRawBits());
11147 ID.AddInteger(I: getSyntheticNodeSubclassData<LoadSDNode>(
11148 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: ExtType, Args&: MemVT, Args&: MMO));
11149 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11150 ID.AddInteger(I: MMO->getFlags());
11151 FoldingSetInsertToken InsertToken;
11152 if (auto *E = cast_or_null<LoadSDNode>(Val: lookupNode(Key: ID, DL: dl, InsertToken))) {
11153 E->refineAlignment(NewMMO: MMO);
11154 E->refineMMOMetadata(NewMMO: MMO);
11155 return SDValue(E, 0);
11156 }
11157 auto *N = newSDNode<LoadSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM,
11158 Args&: ExtType, Args&: MemVT, Args&: MMO);
11159 createOperands(Node: N, Vals: Ops);
11160
11161 CSEMap.insert(N, Token: InsertToken);
11162 InsertNode(N);
11163 SDValue V(N, 0);
11164 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11165 return V;
11166}
11167
11168SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain,
11169 SDValue Ptr, MachinePointerInfo PtrInfo,
11170 MaybeAlign Alignment,
11171 MachineMemOperand::Flags MMOFlags,
11172 const MMOMetadata &Metadata) {
11173 SDValue Undef = getPOISON(VT: Ptr.getValueType());
11174 return getLoad(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Offset: Undef,
11175 PtrInfo, MemVT: VT, Alignment, MMOFlags, Metadata);
11176}
11177
11178SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain,
11179 SDValue Ptr, MachineMemOperand *MMO) {
11180 SDValue Undef = getPOISON(VT: Ptr.getValueType());
11181 return getLoad(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Offset: Undef,
11182 MemVT: VT, MMO);
11183}
11184
11185SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
11186 EVT VT, SDValue Chain, SDValue Ptr,
11187 MachinePointerInfo PtrInfo, EVT MemVT,
11188 MaybeAlign Alignment,
11189 MachineMemOperand::Flags MMOFlags,
11190 const MMOMetadata &Metadata) {
11191 SDValue Undef = getPOISON(VT: Ptr.getValueType());
11192 return getLoad(AM: ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Offset: Undef, PtrInfo,
11193 MemVT, Alignment, MMOFlags, Metadata);
11194}
11195
11196SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
11197 EVT VT, SDValue Chain, SDValue Ptr, EVT MemVT,
11198 MachineMemOperand *MMO) {
11199 SDValue Undef = getPOISON(VT: Ptr.getValueType());
11200 return getLoad(AM: ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Offset: Undef,
11201 MemVT, MMO);
11202}
11203
11204SDValue SelectionDAG::getIndexedLoad(SDValue OrigLoad, const SDLoc &dl,
11205 SDValue Base, SDValue Offset,
11206 ISD::MemIndexedMode AM) {
11207 LoadSDNode *LD = cast<LoadSDNode>(Val&: OrigLoad);
11208 assert(LD->getOffset().getOpcode() == ISD::POISON &&
11209 "Load is already a indexed load!");
11210 // Don't propagate the invariant or dereferenceable flags.
11211 auto MMOFlags =
11212 LD->getMemOperand()->getFlags() &
11213 ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
11214 return getLoad(
11215 AM, ExtType: LD->getExtensionType(), VT: OrigLoad.getValueType(), dl, Chain: LD->getChain(),
11216 Ptr: Base, Offset, PtrInfo: LD->getPointerInfo(), MemVT: LD->getMemoryVT(), Alignment: LD->getAlign(),
11217 MMOFlags,
11218 Metadata: MMOMetadata(LD->getAAInfo(), LD->getRanges(), LD->getMemCacheHint()));
11219}
11220
11221SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
11222 SDValue Ptr, MachinePointerInfo PtrInfo,
11223 Align Alignment,
11224 MachineMemOperand::Flags MMOFlags,
11225 const MMOMetadata &Metadata) {
11226 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
11227
11228 MMOFlags |= MachineMemOperand::MOStore;
11229 assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
11230 assert(!Metadata.Ranges && "range metadata is invalid for stores");
11231
11232 if (PtrInfo.V.isNull())
11233 PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr);
11234
11235 MachineFunction &MF = getMachineFunction();
11236 TypeSize Size = Val.getValueType().getStoreSize();
11237 MachineMemOperand *MMO =
11238 MF.getMachineMemOperand(PtrInfo, F: MMOFlags, Size, BaseAlignment: Alignment, Metadata);
11239 return getStore(Chain, dl, Val, Ptr, MMO);
11240}
11241
11242SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
11243 SDValue Ptr, MachineMemOperand *MMO) {
11244 SDValue Undef = getPOISON(VT: Ptr.getValueType());
11245 return getStore(Chain, dl, Val, Ptr, Offset: Undef, SVT: Val.getValueType(), MMO,
11246 AM: ISD::UNINDEXED);
11247}
11248
11249SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
11250 SDValue Ptr, SDValue Offset, EVT SVT,
11251 MachineMemOperand *MMO, ISD::MemIndexedMode AM,
11252 bool IsTruncating) {
11253 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
11254 EVT VT = Val.getValueType();
11255 if (VT == SVT) {
11256 IsTruncating = false;
11257 } else if (!IsTruncating) {
11258 assert(VT == SVT && "No-truncating store from different memory type!");
11259 } else {
11260 assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&
11261 "Should only be a truncating store, not extending!");
11262 assert(VT.isInteger() == SVT.isInteger() && "Can't do FP-INT conversion!");
11263 assert(VT.isVector() == SVT.isVector() &&
11264 "Cannot use trunc store to convert to or from a vector!");
11265 assert((!VT.isVector() ||
11266 VT.getVectorElementCount() == SVT.getVectorElementCount()) &&
11267 "Cannot use trunc store to change the number of vector elements!");
11268 }
11269
11270 bool Indexed = AM != ISD::UNINDEXED;
11271 assert((Indexed || Offset.getOpcode() == ISD::POISON) &&
11272 "Unindexed store with an offset!");
11273 SDVTList VTs = Indexed ? getVTList(VT1: Ptr.getValueType(), VT2: MVT::Other)
11274 : getVTList(VT: MVT::Other);
11275 SDValue Ops[] = {Chain, Val, Ptr, Offset};
11276 SDNodeKey ID(ISD::STORE, VTs, Ops);
11277 ID.AddInteger(I: SVT.getRawBits());
11278 ID.AddInteger(I: getSyntheticNodeSubclassData<StoreSDNode>(
11279 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: IsTruncating, Args&: SVT, Args&: MMO));
11280 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11281 ID.AddInteger(I: MMO->getFlags());
11282 FoldingSetInsertToken InsertToken;
11283 if (SDNode *E = lookupNode(Key: ID, DL: dl, InsertToken)) {
11284 cast<StoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11285 cast<StoreSDNode>(Val: E)->refineMMOMetadata(NewMMO: MMO);
11286 return SDValue(E, 0);
11287 }
11288 auto *N = newSDNode<StoreSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM,
11289 Args&: IsTruncating, Args&: SVT, Args&: MMO);
11290 createOperands(Node: N, Vals: Ops);
11291
11292 CSEMap.insert(N, Token: InsertToken);
11293 InsertNode(N);
11294 SDValue V(N, 0);
11295 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11296 return V;
11297}
11298
11299SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
11300 SDValue Ptr, SDValue Offset,
11301 MachinePointerInfo PtrInfo, EVT SVT,
11302 Align Alignment,
11303 MachineMemOperand::Flags MMOFlags,
11304 const MMOMetadata &Metadata) {
11305 assert(Chain.getValueType() == MVT::Other &&
11306 "Invalid chain type");
11307
11308 MMOFlags |= MachineMemOperand::MOStore;
11309 assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
11310 assert(!Metadata.Ranges && "range metadata is invalid for stores");
11311
11312 if (PtrInfo.V.isNull())
11313 PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr);
11314
11315 MachineFunction &MF = getMachineFunction();
11316 MachineMemOperand *MMO = MF.getMachineMemOperand(
11317 PtrInfo, F: MMOFlags, Size: SVT.getStoreSize(), BaseAlignment: Alignment, Metadata);
11318 return getTruncStore(Chain, dl, Val, Ptr, Offset, SVT, MMO);
11319}
11320
11321SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
11322 SDValue Ptr, MachinePointerInfo PtrInfo,
11323 EVT SVT, Align Alignment,
11324 MachineMemOperand::Flags MMOFlags,
11325 const MMOMetadata &Metadata) {
11326 return getTruncStore(Chain, dl, Val, Ptr, Offset: getPOISON(VT: Ptr.getValueType()),
11327 PtrInfo, SVT, Alignment, MMOFlags, Metadata);
11328}
11329
11330SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
11331 SDValue Ptr, SDValue Offset, EVT SVT,
11332 MachineMemOperand *MMO) {
11333 return getStore(Chain, dl, Val, Ptr, Offset, SVT, MMO, AM: ISD::UNINDEXED, IsTruncating: true);
11334}
11335
11336SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
11337 SDValue Ptr, EVT SVT,
11338 MachineMemOperand *MMO) {
11339 return getStore(Chain, dl, Val, Ptr, Offset: getPOISON(VT: Ptr.getValueType()), SVT, MMO,
11340 AM: ISD::UNINDEXED, IsTruncating: true);
11341}
11342
11343SDValue SelectionDAG::getIndexedStore(SDValue OrigStore, const SDLoc &dl,
11344 SDValue Base, SDValue Offset,
11345 ISD::MemIndexedMode AM) {
11346 StoreSDNode *ST = cast<StoreSDNode>(Val&: OrigStore);
11347 assert(ST->getOffset().getOpcode() == ISD::POISON &&
11348 "Store is already a indexed store!");
11349 return getStore(Chain: ST->getChain(), dl, Val: ST->getValue(), Ptr: Base, Offset,
11350 SVT: ST->getMemoryVT(), MMO: ST->getMemOperand(), AM,
11351 IsTruncating: ST->isTruncatingStore());
11352}
11353
11354SDValue SelectionDAG::getLoadVP(
11355 ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &dl,
11356 SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL,
11357 MachinePointerInfo PtrInfo, EVT MemVT, Align Alignment,
11358 MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo,
11359 const MDNode *Ranges, bool IsExpanding) {
11360 MMOFlags |= MachineMemOperand::MOLoad;
11361 assert((MMOFlags & MachineMemOperand::MOStore) == 0);
11362 // If we don't have a PtrInfo, infer the trivial frame index case to simplify
11363 // clients.
11364 if (PtrInfo.V.isNull())
11365 PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr, OffsetOp: Offset);
11366
11367 TypeSize Size = MemVT.getStoreSize();
11368 MachineFunction &MF = getMachineFunction();
11369 MachineMemOperand *MMO = MF.getMachineMemOperand(
11370 PtrInfo, F: MMOFlags, Size, BaseAlignment: Alignment, Metadata: MMOMetadata(AAInfo, Ranges));
11371 return getLoadVP(AM, ExtType, VT, dl, Chain, Ptr, Offset, Mask, EVL, MemVT,
11372 MMO, IsExpanding);
11373}
11374
11375SDValue SelectionDAG::getLoadVP(ISD::MemIndexedMode AM,
11376 ISD::LoadExtType ExtType, EVT VT,
11377 const SDLoc &dl, SDValue Chain, SDValue Ptr,
11378 SDValue Offset, SDValue Mask, SDValue EVL,
11379 EVT MemVT, MachineMemOperand *MMO,
11380 bool IsExpanding) {
11381 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
11382 assert(Mask.getValueType().getVectorElementCount() ==
11383 VT.getVectorElementCount() &&
11384 "Vector width mismatch between mask and data");
11385
11386 bool Indexed = AM != ISD::UNINDEXED;
11387 assert((Indexed || Offset.getOpcode() == ISD::POISON) &&
11388 "Unindexed load with an offset!");
11389
11390 SDVTList VTs = Indexed ? getVTList(VT1: VT, VT2: Ptr.getValueType(), VT3: MVT::Other)
11391 : getVTList(VT1: VT, VT2: MVT::Other);
11392 SDValue Ops[] = {Chain, Ptr, Offset, Mask, EVL};
11393 SDNodeKey ID(ISD::VP_LOAD, VTs, Ops);
11394 ID.AddInteger(I: MemVT.getRawBits());
11395 ID.AddInteger(I: getSyntheticNodeSubclassData<VPLoadSDNode>(
11396 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: ExtType, Args&: IsExpanding, Args&: MemVT, Args&: MMO));
11397 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11398 ID.AddInteger(I: MMO->getFlags());
11399 FoldingSetInsertToken InsertToken;
11400 if (auto *E = cast_or_null<VPLoadSDNode>(Val: lookupNode(Key: ID, DL: dl, InsertToken))) {
11401 E->refineAlignment(NewMMO: MMO);
11402 E->refineMMOMetadata(NewMMO: MMO);
11403 return SDValue(E, 0);
11404 }
11405 auto *N = newSDNode<VPLoadSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM,
11406 Args&: ExtType, Args&: IsExpanding, Args&: MemVT, Args&: MMO);
11407 createOperands(Node: N, Vals: Ops);
11408
11409 CSEMap.insert(N, Token: InsertToken);
11410 InsertNode(N);
11411 SDValue V(N, 0);
11412 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11413 return V;
11414}
11415
11416SDValue SelectionDAG::getLoadVP(EVT VT, const SDLoc &dl, SDValue Chain,
11417 SDValue Ptr, SDValue Mask, SDValue EVL,
11418 MachinePointerInfo PtrInfo,
11419 MaybeAlign Alignment,
11420 MachineMemOperand::Flags MMOFlags,
11421 const AAMDNodes &AAInfo, const MDNode *Ranges,
11422 bool IsExpanding) {
11423 SDValue Undef = getPOISON(VT: Ptr.getValueType());
11424 return getLoadVP(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Offset: Undef,
11425 Mask, EVL, PtrInfo, MemVT: VT, Alignment, MMOFlags, AAInfo, Ranges,
11426 IsExpanding);
11427}
11428
11429SDValue SelectionDAG::getLoadVP(EVT VT, const SDLoc &dl, SDValue Chain,
11430 SDValue Ptr, SDValue Mask, SDValue EVL,
11431 MachineMemOperand *MMO, bool IsExpanding) {
11432 SDValue Undef = getPOISON(VT: Ptr.getValueType());
11433 return getLoadVP(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Offset: Undef,
11434 Mask, EVL, MemVT: VT, MMO, IsExpanding);
11435}
11436
11437SDValue SelectionDAG::getExtLoadVP(ISD::LoadExtType ExtType, const SDLoc &dl,
11438 EVT VT, SDValue Chain, SDValue Ptr,
11439 SDValue Mask, SDValue EVL,
11440 MachinePointerInfo PtrInfo, EVT MemVT,
11441 MaybeAlign Alignment,
11442 MachineMemOperand::Flags MMOFlags,
11443 const AAMDNodes &AAInfo, bool IsExpanding) {
11444 SDValue Undef = getPOISON(VT: Ptr.getValueType());
11445 return getLoadVP(AM: ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Offset: Undef, Mask,
11446 EVL, PtrInfo, MemVT, Alignment, MMOFlags, AAInfo, Ranges: nullptr,
11447 IsExpanding);
11448}
11449
11450SDValue SelectionDAG::getExtLoadVP(ISD::LoadExtType ExtType, const SDLoc &dl,
11451 EVT VT, SDValue Chain, SDValue Ptr,
11452 SDValue Mask, SDValue EVL, EVT MemVT,
11453 MachineMemOperand *MMO, bool IsExpanding) {
11454 SDValue Undef = getPOISON(VT: Ptr.getValueType());
11455 return getLoadVP(AM: ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Offset: Undef, Mask,
11456 EVL, MemVT, MMO, IsExpanding);
11457}
11458
11459SDValue SelectionDAG::getStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val,
11460 SDValue Ptr, SDValue Offset, SDValue Mask,
11461 SDValue EVL, EVT MemVT, MachineMemOperand *MMO,
11462 ISD::MemIndexedMode AM, bool IsTruncating,
11463 bool IsCompressing) {
11464 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
11465 assert(Mask.getValueType().getVectorElementCount() ==
11466 Val.getValueType().getVectorElementCount() &&
11467 "Vector width mismatch between mask and data");
11468
11469 bool Indexed = AM != ISD::UNINDEXED;
11470 assert((Indexed || Offset.getOpcode() == ISD::POISON) &&
11471 "Unindexed vp_store with an offset!");
11472 SDVTList VTs = Indexed ? getVTList(VT1: Ptr.getValueType(), VT2: MVT::Other)
11473 : getVTList(VT: MVT::Other);
11474 SDValue Ops[] = {Chain, Val, Ptr, Offset, Mask, EVL};
11475 SDNodeKey ID(ISD::VP_STORE, VTs, Ops);
11476 ID.AddInteger(I: MemVT.getRawBits());
11477 ID.AddInteger(I: getSyntheticNodeSubclassData<VPStoreSDNode>(
11478 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO));
11479 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11480 ID.AddInteger(I: MMO->getFlags());
11481 FoldingSetInsertToken InsertToken;
11482 if (SDNode *E = lookupNode(Key: ID, DL: dl, InsertToken)) {
11483 cast<VPStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11484 return SDValue(E, 0);
11485 }
11486 auto *N = newSDNode<VPStoreSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM,
11487 Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO);
11488 createOperands(Node: N, Vals: Ops);
11489
11490 CSEMap.insert(N, Token: InsertToken);
11491 InsertNode(N);
11492 SDValue V(N, 0);
11493 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11494 return V;
11495}
11496
11497SDValue SelectionDAG::getTruncStoreVP(SDValue Chain, const SDLoc &dl,
11498 SDValue Val, SDValue Ptr, SDValue Mask,
11499 SDValue EVL, MachinePointerInfo PtrInfo,
11500 EVT SVT, Align Alignment,
11501 MachineMemOperand::Flags MMOFlags,
11502 const AAMDNodes &AAInfo,
11503 bool IsCompressing) {
11504 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
11505
11506 MMOFlags |= MachineMemOperand::MOStore;
11507 assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
11508
11509 if (PtrInfo.V.isNull())
11510 PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr);
11511
11512 MachineFunction &MF = getMachineFunction();
11513 MachineMemOperand *MMO = MF.getMachineMemOperand(
11514 PtrInfo, F: MMOFlags, Size: SVT.getStoreSize(), BaseAlignment: Alignment, Metadata: AAInfo);
11515 return getTruncStoreVP(Chain, dl, Val, Ptr, Mask, EVL, SVT, MMO,
11516 IsCompressing);
11517}
11518
11519SDValue SelectionDAG::getTruncStoreVP(SDValue Chain, const SDLoc &dl,
11520 SDValue Val, SDValue Ptr, SDValue Mask,
11521 SDValue EVL, EVT SVT,
11522 MachineMemOperand *MMO,
11523 bool IsCompressing) {
11524 EVT VT = Val.getValueType();
11525
11526 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
11527 if (VT == SVT)
11528 return getStoreVP(Chain, dl, Val, Ptr, Offset: getPOISON(VT: Ptr.getValueType()), Mask,
11529 EVL, MemVT: VT, MMO, AM: ISD::UNINDEXED,
11530 /*IsTruncating*/ false, IsCompressing);
11531
11532 assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&
11533 "Should only be a truncating store, not extending!");
11534 assert(VT.isInteger() == SVT.isInteger() && "Can't do FP-INT conversion!");
11535 assert(VT.isVector() == SVT.isVector() &&
11536 "Cannot use trunc store to convert to or from a vector!");
11537 assert((!VT.isVector() ||
11538 VT.getVectorElementCount() == SVT.getVectorElementCount()) &&
11539 "Cannot use trunc store to change the number of vector elements!");
11540
11541 SDVTList VTs = getVTList(VT: MVT::Other);
11542 SDValue Undef = getPOISON(VT: Ptr.getValueType());
11543 SDValue Ops[] = {Chain, Val, Ptr, Undef, Mask, EVL};
11544 SDNodeKey ID(ISD::VP_STORE, VTs, Ops);
11545 ID.AddInteger(I: SVT.getRawBits());
11546 ID.AddInteger(I: getSyntheticNodeSubclassData<VPStoreSDNode>(
11547 IROrder: dl.getIROrder(), Args&: VTs, Args: ISD::UNINDEXED, Args: true, Args&: IsCompressing, Args&: SVT, Args&: MMO));
11548 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11549 ID.AddInteger(I: MMO->getFlags());
11550 FoldingSetInsertToken InsertToken;
11551 if (SDNode *E = lookupNode(Key: ID, DL: dl, InsertToken)) {
11552 cast<VPStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11553 return SDValue(E, 0);
11554 }
11555 auto *N =
11556 newSDNode<VPStoreSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs,
11557 Args: ISD::UNINDEXED, Args: true, Args&: IsCompressing, Args&: SVT, Args&: MMO);
11558 createOperands(Node: N, Vals: Ops);
11559
11560 CSEMap.insert(N, Token: InsertToken);
11561 InsertNode(N);
11562 SDValue V(N, 0);
11563 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11564 return V;
11565}
11566
11567SDValue SelectionDAG::getStridedLoadVP(
11568 ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &DL,
11569 SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask,
11570 SDValue EVL, EVT MemVT, MachineMemOperand *MMO, bool IsExpanding) {
11571 bool Indexed = AM != ISD::UNINDEXED;
11572 assert((Indexed || Offset.getOpcode() == ISD::POISON) &&
11573 "Unindexed load with an offset!");
11574
11575 SDValue Ops[] = {Chain, Ptr, Offset, Stride, Mask, EVL};
11576 SDVTList VTs = Indexed ? getVTList(VT1: VT, VT2: Ptr.getValueType(), VT3: MVT::Other)
11577 : getVTList(VT1: VT, VT2: MVT::Other);
11578 SDNodeKey ID(ISD::EXPERIMENTAL_VP_STRIDED_LOAD, VTs, Ops);
11579 ID.AddInteger(I: MemVT.getRawBits());
11580 ID.AddInteger(I: getSyntheticNodeSubclassData<VPStridedLoadSDNode>(
11581 IROrder: DL.getIROrder(), Args&: VTs, Args&: AM, Args&: ExtType, Args&: IsExpanding, Args&: MemVT, Args&: MMO));
11582 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11583
11584 FoldingSetInsertToken InsertToken;
11585 if (SDNode *E = lookupNode(Key: ID, DL, InsertToken)) {
11586 cast<VPStridedLoadSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11587 return SDValue(E, 0);
11588 }
11589
11590 auto *N =
11591 newSDNode<VPStridedLoadSDNode>(Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs, Args&: AM,
11592 Args&: ExtType, Args&: IsExpanding, Args&: MemVT, Args&: MMO);
11593 createOperands(Node: N, Vals: Ops);
11594 CSEMap.insert(N, Token: InsertToken);
11595 InsertNode(N);
11596 SDValue V(N, 0);
11597 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11598 return V;
11599}
11600
11601SDValue SelectionDAG::getStridedLoadVP(EVT VT, const SDLoc &DL, SDValue Chain,
11602 SDValue Ptr, SDValue Stride,
11603 SDValue Mask, SDValue EVL,
11604 MachineMemOperand *MMO,
11605 bool IsExpanding) {
11606 SDValue Undef = getPOISON(VT: Ptr.getValueType());
11607 return getStridedLoadVP(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, DL, Chain, Ptr,
11608 Offset: Undef, Stride, Mask, EVL, MemVT: VT, MMO, IsExpanding);
11609}
11610
11611SDValue SelectionDAG::getExtStridedLoadVP(
11612 ISD::LoadExtType ExtType, const SDLoc &DL, EVT VT, SDValue Chain,
11613 SDValue Ptr, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT,
11614 MachineMemOperand *MMO, bool IsExpanding) {
11615 SDValue Undef = getPOISON(VT: Ptr.getValueType());
11616 return getStridedLoadVP(AM: ISD::UNINDEXED, ExtType, VT, DL, Chain, Ptr, Offset: Undef,
11617 Stride, Mask, EVL, MemVT, MMO, IsExpanding);
11618}
11619
11620SDValue SelectionDAG::getStridedStoreVP(SDValue Chain, const SDLoc &DL,
11621 SDValue Val, SDValue Ptr,
11622 SDValue Offset, SDValue Stride,
11623 SDValue Mask, SDValue EVL, EVT MemVT,
11624 MachineMemOperand *MMO,
11625 ISD::MemIndexedMode AM,
11626 bool IsTruncating, bool IsCompressing) {
11627 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
11628 bool Indexed = AM != ISD::UNINDEXED;
11629 assert((Indexed || Offset.getOpcode() == ISD::POISON) &&
11630 "Unindexed vp_store with an offset!");
11631 SDVTList VTs = Indexed ? getVTList(VT1: Ptr.getValueType(), VT2: MVT::Other)
11632 : getVTList(VT: MVT::Other);
11633 SDValue Ops[] = {Chain, Val, Ptr, Offset, Stride, Mask, EVL};
11634 SDNodeKey ID(ISD::EXPERIMENTAL_VP_STRIDED_STORE, VTs, Ops);
11635 ID.AddInteger(I: MemVT.getRawBits());
11636 ID.AddInteger(I: getSyntheticNodeSubclassData<VPStridedStoreSDNode>(
11637 IROrder: DL.getIROrder(), Args&: VTs, Args&: AM, Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO));
11638 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11639 FoldingSetInsertToken InsertToken;
11640 if (SDNode *E = lookupNode(Key: ID, DL, InsertToken)) {
11641 cast<VPStridedStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11642 return SDValue(E, 0);
11643 }
11644 auto *N = newSDNode<VPStridedStoreSDNode>(Args: DL.getIROrder(), Args: DL.getDebugLoc(),
11645 Args&: VTs, Args&: AM, Args&: IsTruncating,
11646 Args&: IsCompressing, Args&: MemVT, Args&: MMO);
11647 createOperands(Node: N, Vals: Ops);
11648
11649 CSEMap.insert(N, Token: InsertToken);
11650 InsertNode(N);
11651 SDValue V(N, 0);
11652 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11653 return V;
11654}
11655
11656SDValue SelectionDAG::getGatherVP(SDVTList VTs, EVT VT, const SDLoc &dl,
11657 ArrayRef<SDValue> Ops, MachineMemOperand *MMO,
11658 ISD::MemIndexType IndexType) {
11659 assert(Ops.size() == 6 && "Incompatible number of operands");
11660
11661 SDNodeKey ID(ISD::VP_GATHER, VTs, Ops);
11662 ID.AddInteger(I: VT.getRawBits());
11663 ID.AddInteger(I: getSyntheticNodeSubclassData<VPGatherSDNode>(
11664 IROrder: dl.getIROrder(), Args&: VTs, Args&: VT, Args&: MMO, Args&: IndexType));
11665 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11666 ID.AddInteger(I: MMO->getFlags());
11667 FoldingSetInsertToken InsertToken;
11668 if (SDNode *E = lookupNode(Key: ID, DL: dl, InsertToken)) {
11669 cast<VPGatherSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11670 return SDValue(E, 0);
11671 }
11672
11673 auto *N = newSDNode<VPGatherSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs,
11674 Args&: VT, Args&: MMO, Args&: IndexType);
11675 createOperands(Node: N, Vals: Ops);
11676
11677 assert(N->getMask().getValueType().getVectorElementCount() ==
11678 N->getValueType(0).getVectorElementCount() &&
11679 "Vector width mismatch between mask and data");
11680 assert(N->getIndex().getValueType().getVectorElementCount().isScalable() ==
11681 N->getValueType(0).getVectorElementCount().isScalable() &&
11682 "Scalable flags of index and data do not match");
11683 assert(ElementCount::isKnownGE(
11684 N->getIndex().getValueType().getVectorElementCount(),
11685 N->getValueType(0).getVectorElementCount()) &&
11686 "Vector width mismatch between index and data");
11687 assert(isa<ConstantSDNode>(N->getScale()) &&
11688 N->getScale()->getAsAPIntVal().isPowerOf2() &&
11689 "Scale should be a constant power of 2");
11690
11691 CSEMap.insert(N, Token: InsertToken);
11692 InsertNode(N);
11693 SDValue V(N, 0);
11694 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11695 return V;
11696}
11697
11698SDValue SelectionDAG::getScatterVP(SDVTList VTs, EVT VT, const SDLoc &dl,
11699 ArrayRef<SDValue> Ops,
11700 MachineMemOperand *MMO,
11701 ISD::MemIndexType IndexType) {
11702 assert(Ops.size() == 7 && "Incompatible number of operands");
11703
11704 SDNodeKey ID(ISD::VP_SCATTER, VTs, Ops);
11705 ID.AddInteger(I: VT.getRawBits());
11706 ID.AddInteger(I: getSyntheticNodeSubclassData<VPScatterSDNode>(
11707 IROrder: dl.getIROrder(), Args&: VTs, Args&: VT, Args&: MMO, Args&: IndexType));
11708 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11709 ID.AddInteger(I: MMO->getFlags());
11710 FoldingSetInsertToken InsertToken;
11711 if (SDNode *E = lookupNode(Key: ID, DL: dl, InsertToken)) {
11712 cast<VPScatterSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11713 return SDValue(E, 0);
11714 }
11715 auto *N = newSDNode<VPScatterSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs,
11716 Args&: VT, Args&: MMO, Args&: IndexType);
11717 createOperands(Node: N, Vals: Ops);
11718
11719 assert(N->getMask().getValueType().getVectorElementCount() ==
11720 N->getValue().getValueType().getVectorElementCount() &&
11721 "Vector width mismatch between mask and data");
11722 assert(
11723 N->getIndex().getValueType().getVectorElementCount().isScalable() ==
11724 N->getValue().getValueType().getVectorElementCount().isScalable() &&
11725 "Scalable flags of index and data do not match");
11726 assert(ElementCount::isKnownGE(
11727 N->getIndex().getValueType().getVectorElementCount(),
11728 N->getValue().getValueType().getVectorElementCount()) &&
11729 "Vector width mismatch between index and data");
11730 assert(isa<ConstantSDNode>(N->getScale()) &&
11731 N->getScale()->getAsAPIntVal().isPowerOf2() &&
11732 "Scale should be a constant power of 2");
11733
11734 CSEMap.insert(N, Token: InsertToken);
11735 InsertNode(N);
11736 SDValue V(N, 0);
11737 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11738 return V;
11739}
11740
11741SDValue SelectionDAG::getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain,
11742 SDValue Base, SDValue Offset, SDValue Mask,
11743 SDValue PassThru, EVT MemVT,
11744 MachineMemOperand *MMO,
11745 ISD::MemIndexedMode AM,
11746 ISD::LoadExtType ExtTy, bool isExpanding) {
11747 bool Indexed = AM != ISD::UNINDEXED;
11748 assert((Indexed || Offset.getOpcode() == ISD::POISON) &&
11749 "Unindexed masked load with an offset!");
11750 SDVTList VTs = Indexed ? getVTList(VT1: VT, VT2: Base.getValueType(), VT3: MVT::Other)
11751 : getVTList(VT1: VT, VT2: MVT::Other);
11752 SDValue Ops[] = {Chain, Base, Offset, Mask, PassThru};
11753 SDNodeKey ID(ISD::MLOAD, VTs, Ops);
11754 ID.AddInteger(I: MemVT.getRawBits());
11755 ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedLoadSDNode>(
11756 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: ExtTy, Args&: isExpanding, Args&: MemVT, Args&: MMO));
11757 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11758 ID.AddInteger(I: MMO->getFlags());
11759 FoldingSetInsertToken InsertToken;
11760 if (SDNode *E = lookupNode(Key: ID, DL: dl, InsertToken)) {
11761 cast<MaskedLoadSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11762 return SDValue(E, 0);
11763 }
11764 auto *N = newSDNode<MaskedLoadSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs,
11765 Args&: AM, Args&: ExtTy, Args&: isExpanding, Args&: MemVT, Args&: MMO);
11766 createOperands(Node: N, Vals: Ops);
11767
11768 CSEMap.insert(N, Token: InsertToken);
11769 InsertNode(N);
11770 SDValue V(N, 0);
11771 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11772 return V;
11773}
11774
11775SDValue SelectionDAG::getIndexedMaskedLoad(SDValue OrigLoad, const SDLoc &dl,
11776 SDValue Base, SDValue Offset,
11777 ISD::MemIndexedMode AM) {
11778 MaskedLoadSDNode *LD = cast<MaskedLoadSDNode>(Val&: OrigLoad);
11779 assert(LD->getOffset().getOpcode() == ISD::POISON &&
11780 "Masked load is already a indexed load!");
11781 return getMaskedLoad(VT: OrigLoad.getValueType(), dl, Chain: LD->getChain(), Base,
11782 Offset, Mask: LD->getMask(), PassThru: LD->getPassThru(),
11783 MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand(), AM,
11784 ExtTy: LD->getExtensionType(), isExpanding: LD->isExpandingLoad());
11785}
11786
11787SDValue SelectionDAG::getMaskedStore(SDValue Chain, const SDLoc &dl,
11788 SDValue Val, SDValue Base, SDValue Offset,
11789 SDValue Mask, EVT MemVT,
11790 MachineMemOperand *MMO,
11791 ISD::MemIndexedMode AM, bool IsTruncating,
11792 bool IsCompressing) {
11793 assert(Chain.getValueType() == MVT::Other &&
11794 "Invalid chain type");
11795 bool Indexed = AM != ISD::UNINDEXED;
11796 assert((Indexed || Offset.getOpcode() == ISD::POISON) &&
11797 "Unindexed masked store with an offset!");
11798 SDVTList VTs = Indexed ? getVTList(VT1: Base.getValueType(), VT2: MVT::Other)
11799 : getVTList(VT: MVT::Other);
11800 SDValue Ops[] = {Chain, Val, Base, Offset, Mask};
11801 SDNodeKey ID(ISD::MSTORE, VTs, Ops);
11802 ID.AddInteger(I: MemVT.getRawBits());
11803 ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedStoreSDNode>(
11804 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO));
11805 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11806 ID.AddInteger(I: MMO->getFlags());
11807 FoldingSetInsertToken InsertToken;
11808 if (SDNode *E = lookupNode(Key: ID, DL: dl, InsertToken)) {
11809 cast<MaskedStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11810 return SDValue(E, 0);
11811 }
11812 auto *N =
11813 newSDNode<MaskedStoreSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM,
11814 Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO);
11815 createOperands(Node: N, Vals: Ops);
11816
11817 CSEMap.insert(N, Token: InsertToken);
11818 InsertNode(N);
11819 SDValue V(N, 0);
11820 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11821 return V;
11822}
11823
11824SDValue SelectionDAG::getIndexedMaskedStore(SDValue OrigStore, const SDLoc &dl,
11825 SDValue Base, SDValue Offset,
11826 ISD::MemIndexedMode AM) {
11827 MaskedStoreSDNode *ST = cast<MaskedStoreSDNode>(Val&: OrigStore);
11828 assert(ST->getOffset().getOpcode() == ISD::POISON &&
11829 "Masked store is already a indexed store!");
11830 return getMaskedStore(Chain: ST->getChain(), dl, Val: ST->getValue(), Base, Offset,
11831 Mask: ST->getMask(), MemVT: ST->getMemoryVT(), MMO: ST->getMemOperand(),
11832 AM, IsTruncating: ST->isTruncatingStore(), IsCompressing: ST->isCompressingStore());
11833}
11834
11835SDValue SelectionDAG::getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl,
11836 ArrayRef<SDValue> Ops,
11837 MachineMemOperand *MMO,
11838 ISD::MemIndexType IndexType,
11839 ISD::LoadExtType ExtTy) {
11840 assert(Ops.size() == 6 && "Incompatible number of operands");
11841
11842 SDNodeKey ID(ISD::MGATHER, VTs, Ops);
11843 ID.AddInteger(I: MemVT.getRawBits());
11844 ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedGatherSDNode>(
11845 IROrder: dl.getIROrder(), Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType, Args&: ExtTy));
11846 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11847 ID.AddInteger(I: MMO->getFlags());
11848 FoldingSetInsertToken InsertToken;
11849 if (SDNode *E = lookupNode(Key: ID, DL: dl, InsertToken)) {
11850 cast<MaskedGatherSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11851 return SDValue(E, 0);
11852 }
11853
11854 auto *N = newSDNode<MaskedGatherSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(),
11855 Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType, Args&: ExtTy);
11856 createOperands(Node: N, Vals: Ops);
11857
11858 assert(N->getPassThru().getValueType() == N->getValueType(0) &&
11859 "Incompatible type of the PassThru value in MaskedGatherSDNode");
11860 assert(N->getMask().getValueType().getVectorElementCount() ==
11861 N->getValueType(0).getVectorElementCount() &&
11862 "Vector width mismatch between mask and data");
11863 assert(N->getIndex().getValueType().getVectorElementCount().isScalable() ==
11864 N->getValueType(0).getVectorElementCount().isScalable() &&
11865 "Scalable flags of index and data do not match");
11866 assert(ElementCount::isKnownGE(
11867 N->getIndex().getValueType().getVectorElementCount(),
11868 N->getValueType(0).getVectorElementCount()) &&
11869 "Vector width mismatch between index and data");
11870 assert(isa<ConstantSDNode>(N->getScale()) &&
11871 N->getScale()->getAsAPIntVal().isPowerOf2() &&
11872 "Scale should be a constant power of 2");
11873
11874 CSEMap.insert(N, Token: InsertToken);
11875 InsertNode(N);
11876 SDValue V(N, 0);
11877 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11878 return V;
11879}
11880
11881SDValue SelectionDAG::getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl,
11882 ArrayRef<SDValue> Ops,
11883 MachineMemOperand *MMO,
11884 ISD::MemIndexType IndexType,
11885 bool IsTrunc) {
11886 assert(Ops.size() == 6 && "Incompatible number of operands");
11887
11888 SDNodeKey ID(ISD::MSCATTER, VTs, Ops);
11889 ID.AddInteger(I: MemVT.getRawBits());
11890 ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedScatterSDNode>(
11891 IROrder: dl.getIROrder(), Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType, Args&: IsTrunc));
11892 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11893 ID.AddInteger(I: MMO->getFlags());
11894 FoldingSetInsertToken InsertToken;
11895 if (SDNode *E = lookupNode(Key: ID, DL: dl, InsertToken)) {
11896 cast<MaskedScatterSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11897 return SDValue(E, 0);
11898 }
11899
11900 auto *N = newSDNode<MaskedScatterSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(),
11901 Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType, Args&: IsTrunc);
11902 createOperands(Node: N, Vals: Ops);
11903
11904 assert(N->getMask().getValueType().getVectorElementCount() ==
11905 N->getValue().getValueType().getVectorElementCount() &&
11906 "Vector width mismatch between mask and data");
11907 assert(
11908 N->getIndex().getValueType().getVectorElementCount().isScalable() ==
11909 N->getValue().getValueType().getVectorElementCount().isScalable() &&
11910 "Scalable flags of index and data do not match");
11911 assert(ElementCount::isKnownGE(
11912 N->getIndex().getValueType().getVectorElementCount(),
11913 N->getValue().getValueType().getVectorElementCount()) &&
11914 "Vector width mismatch between index and data");
11915 assert(isa<ConstantSDNode>(N->getScale()) &&
11916 N->getScale()->getAsAPIntVal().isPowerOf2() &&
11917 "Scale should be a constant power of 2");
11918
11919 CSEMap.insert(N, Token: InsertToken);
11920 InsertNode(N);
11921 SDValue V(N, 0);
11922 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11923 return V;
11924}
11925
11926SDValue SelectionDAG::getMaskedHistogram(SDVTList VTs, EVT MemVT,
11927 const SDLoc &dl, ArrayRef<SDValue> Ops,
11928 MachineMemOperand *MMO,
11929 ISD::MemIndexType IndexType) {
11930 assert(Ops.size() == 7 && "Incompatible number of operands");
11931
11932 SDNodeKey ID(ISD::EXPERIMENTAL_VECTOR_HISTOGRAM, VTs, Ops);
11933 ID.AddInteger(I: MemVT.getRawBits());
11934 ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedHistogramSDNode>(
11935 IROrder: dl.getIROrder(), Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType));
11936 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11937 ID.AddInteger(I: MMO->getFlags());
11938 FoldingSetInsertToken InsertToken;
11939 if (SDNode *E = lookupNode(Key: ID, DL: dl, InsertToken)) {
11940 cast<MaskedGatherSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11941 return SDValue(E, 0);
11942 }
11943
11944 auto *N = newSDNode<MaskedHistogramSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(),
11945 Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType);
11946 createOperands(Node: N, Vals: Ops);
11947
11948 assert(N->getMask().getValueType().getVectorElementCount() ==
11949 N->getIndex().getValueType().getVectorElementCount() &&
11950 "Vector width mismatch between mask and data");
11951 assert(isa<ConstantSDNode>(N->getScale()) &&
11952 N->getScale()->getAsAPIntVal().isPowerOf2() &&
11953 "Scale should be a constant power of 2");
11954 assert(N->getInc().getValueType().isInteger() && "Non integer update value");
11955
11956 CSEMap.insert(N, Token: InsertToken);
11957 InsertNode(N);
11958 SDValue V(N, 0);
11959 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11960 return V;
11961}
11962
11963SDValue SelectionDAG::getLoadFFVP(EVT VT, const SDLoc &DL, SDValue Chain,
11964 SDValue Ptr, SDValue Mask, SDValue EVL,
11965 MachineMemOperand *MMO) {
11966 SDVTList VTs = getVTList(VT1: VT, VT2: EVL.getValueType(), VT3: MVT::Other);
11967 SDValue Ops[] = {Chain, Ptr, Mask, EVL};
11968 SDNodeKey ID(ISD::VP_LOAD_FF, VTs, Ops);
11969 ID.AddInteger(I: VT.getRawBits());
11970 ID.AddInteger(I: getSyntheticNodeSubclassData<VPLoadFFSDNode>(IROrder: DL.getIROrder(),
11971 Args&: VTs, Args&: VT, Args&: MMO));
11972 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11973 ID.AddInteger(I: MMO->getFlags());
11974 FoldingSetInsertToken InsertToken;
11975 if (SDNode *E = lookupNode(Key: ID, DL, InsertToken)) {
11976 cast<VPLoadFFSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11977 return SDValue(E, 0);
11978 }
11979 auto *N = newSDNode<VPLoadFFSDNode>(Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs,
11980 Args&: VT, Args&: MMO);
11981 createOperands(Node: N, Vals: Ops);
11982
11983 CSEMap.insert(N, Token: InsertToken);
11984 InsertNode(N);
11985 SDValue V(N, 0);
11986 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11987 return V;
11988}
11989
11990SDValue SelectionDAG::getGetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr,
11991 EVT MemVT, MachineMemOperand *MMO) {
11992 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
11993 SDVTList VTs = getVTList(VT: MVT::Other);
11994 SDValue Ops[] = {Chain, Ptr};
11995 SDNodeKey ID(ISD::GET_FPENV_MEM, VTs, Ops);
11996 ID.AddInteger(I: MemVT.getRawBits());
11997 ID.AddInteger(I: getSyntheticNodeSubclassData<FPStateAccessSDNode>(
11998 Opc: ISD::GET_FPENV_MEM, Order: dl.getIROrder(), VTs, MemoryVT: MemVT, MMO));
11999 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
12000 ID.AddInteger(I: MMO->getFlags());
12001 FoldingSetInsertToken InsertToken;
12002 if (SDNode *E = lookupNode(Key: ID, DL: dl, InsertToken))
12003 return SDValue(E, 0);
12004
12005 auto *N = newSDNode<FPStateAccessSDNode>(Args: ISD::GET_FPENV_MEM, Args: dl.getIROrder(),
12006 Args: dl.getDebugLoc(), Args&: VTs, Args&: MemVT, Args&: MMO);
12007 createOperands(Node: N, Vals: Ops);
12008
12009 CSEMap.insert(N, Token: InsertToken);
12010 InsertNode(N);
12011 SDValue V(N, 0);
12012 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
12013 return V;
12014}
12015
12016SDValue SelectionDAG::getSetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr,
12017 EVT MemVT, MachineMemOperand *MMO) {
12018 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
12019 SDVTList VTs = getVTList(VT: MVT::Other);
12020 SDValue Ops[] = {Chain, Ptr};
12021 SDNodeKey ID(ISD::SET_FPENV_MEM, VTs, Ops);
12022 ID.AddInteger(I: MemVT.getRawBits());
12023 ID.AddInteger(I: getSyntheticNodeSubclassData<FPStateAccessSDNode>(
12024 Opc: ISD::SET_FPENV_MEM, Order: dl.getIROrder(), VTs, MemoryVT: MemVT, MMO));
12025 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
12026 ID.AddInteger(I: MMO->getFlags());
12027 FoldingSetInsertToken InsertToken;
12028 if (SDNode *E = lookupNode(Key: ID, DL: dl, InsertToken))
12029 return SDValue(E, 0);
12030
12031 auto *N = newSDNode<FPStateAccessSDNode>(Args: ISD::SET_FPENV_MEM, Args: dl.getIROrder(),
12032 Args: dl.getDebugLoc(), Args&: VTs, Args&: MemVT, Args&: MMO);
12033 createOperands(Node: N, Vals: Ops);
12034
12035 CSEMap.insert(N, Token: InsertToken);
12036 InsertNode(N);
12037 SDValue V(N, 0);
12038 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
12039 return V;
12040}
12041
12042SDValue SelectionDAG::simplifySelect(SDValue Cond, SDValue T, SDValue F) {
12043 // select undef, T, F --> T (if T is a constant), otherwise F
12044 // select, ?, undef, F --> F
12045 // select, ?, T, undef --> T
12046 if (Cond.isUndef())
12047 return isConstantValueOfAnyType(N: T) ? T : F;
12048 if (T.isUndef())
12049 return isGuaranteedNotToBePoison(Op: F) ? F : getFreeze(V: F);
12050 if (F.isUndef())
12051 return isGuaranteedNotToBePoison(Op: T) ? T : getFreeze(V: T);
12052
12053 // select true, T, F --> T
12054 // select false, T, F --> F
12055 if (auto C = isBoolConstant(N: Cond))
12056 return *C ? T : F;
12057
12058 // select ?, T, T --> T
12059 if (T == F)
12060 return T;
12061
12062 return SDValue();
12063}
12064
12065SDValue SelectionDAG::simplifyShift(SDValue X, SDValue Y) {
12066 // shift undef, Y --> 0 (can always assume that the undef value is 0)
12067 if (X.isUndef())
12068 return getConstant(Val: 0, DL: SDLoc(X.getNode()), VT: X.getValueType());
12069 // shift X, undef --> undef (because it may shift by the bitwidth)
12070 if (Y.isUndef())
12071 return getUNDEF(VT: X.getValueType());
12072
12073 // shift 0, Y --> 0
12074 // shift X, 0 --> X
12075 if (isNullOrNullSplat(V: X) || isNullOrNullSplat(V: Y))
12076 return X;
12077
12078 // shift X, C >= bitwidth(X) --> undef
12079 // All vector elements must be too big (or undef) to avoid partial undefs.
12080 auto isShiftTooBig = [X](ConstantSDNode *Val) {
12081 return !Val || Val->getAPIntValue().uge(RHS: X.getScalarValueSizeInBits());
12082 };
12083 if (ISD::matchUnaryPredicate(Op: Y, Match: isShiftTooBig, AllowUndefs: true))
12084 return getUNDEF(VT: X.getValueType());
12085
12086 // shift i1/vXi1 X, Y --> X (any non-zero shift amount is undefined).
12087 if (X.getValueType().getScalarType() == MVT::i1)
12088 return X;
12089
12090 return SDValue();
12091}
12092
12093SDValue SelectionDAG::simplifyFPBinop(unsigned Opcode, SDValue X, SDValue Y,
12094 SDNodeFlags Flags) {
12095 // If this operation has 'nnan' or 'ninf' and at least 1 disallowed operand
12096 // (an undef operand can be chosen to be Nan/Inf), then the result of this
12097 // operation is poison. That result can be relaxed to undef.
12098 ConstantFPSDNode *XC = isConstOrConstSplatFP(N: X, /* AllowUndefs */ true);
12099 ConstantFPSDNode *YC = isConstOrConstSplatFP(N: Y, /* AllowUndefs */ true);
12100 bool HasNan = (XC && XC->getValueAPF().isNaN()) ||
12101 (YC && YC->getValueAPF().isNaN());
12102 bool HasInf = (XC && XC->getValueAPF().isInfinity()) ||
12103 (YC && YC->getValueAPF().isInfinity());
12104
12105 if (Flags.hasNoNaNs() && (HasNan || X.isUndef() || Y.isUndef()))
12106 return getUNDEF(VT: X.getValueType());
12107
12108 if (Flags.hasNoInfs() && (HasInf || X.isUndef() || Y.isUndef()))
12109 return getUNDEF(VT: X.getValueType());
12110
12111 if (!YC)
12112 return SDValue();
12113
12114 // X + -0.0 --> X
12115 if (Opcode == ISD::FADD)
12116 if (YC->getValueAPF().isNegZero())
12117 return X;
12118
12119 // X - +0.0 --> X
12120 if (Opcode == ISD::FSUB)
12121 if (YC->getValueAPF().isPosZero())
12122 return X;
12123
12124 // X * 1.0 --> X
12125 // X / 1.0 --> X
12126 if (Opcode == ISD::FMUL || Opcode == ISD::FDIV)
12127 if (YC->getValueAPF().isOne())
12128 return X;
12129
12130 // X * 0.0 --> 0.0
12131 if (Opcode == ISD::FMUL && Flags.hasNoNaNs() && Flags.hasNoSignedZeros())
12132 if (YC->getValueAPF().isZero())
12133 return getConstantFP(Val: 0.0, DL: SDLoc(Y), VT: Y.getValueType());
12134
12135 return SDValue();
12136}
12137
12138SDValue SelectionDAG::getVAArg(EVT VT, const SDLoc &dl, SDValue Chain,
12139 SDValue Ptr, SDValue SV, unsigned Align) {
12140 SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Val: Align, DL: dl, VT: MVT::i32) };
12141 return getNode(Opcode: ISD::VAARG, DL: dl, VTList: getVTList(VT1: VT, VT2: MVT::Other), Ops);
12142}
12143
12144SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
12145 ArrayRef<SDUse> Ops) {
12146 switch (Ops.size()) {
12147 case 0: return getNode(Opcode, DL, VT);
12148 case 1: return getNode(Opcode, DL, VT, N1: Ops[0].get());
12149 case 2: return getNode(Opcode, DL, VT, N1: Ops[0], N2: Ops[1]);
12150 case 3: return getNode(Opcode, DL, VT, N1: Ops[0], N2: Ops[1], N3: Ops[2]);
12151 default: break;
12152 }
12153
12154 // Copy from an SDUse array into an SDValue array for use with
12155 // the regular getNode logic.
12156 SmallVector<SDValue, 8> NewOps(Ops);
12157 return getNode(Opcode, DL, VT, Ops: NewOps);
12158}
12159
12160SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
12161 ArrayRef<SDValue> Ops) {
12162 SDNodeFlags Flags;
12163 if (Inserter)
12164 Flags = Inserter->getFlags();
12165 return getNode(Opcode, DL, VT, Ops, Flags);
12166}
12167
12168SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
12169 ArrayRef<SDValue> Ops, const SDNodeFlags Flags) {
12170 unsigned NumOps = Ops.size();
12171 switch (NumOps) {
12172 case 0: return getNode(Opcode, DL, VT);
12173 case 1: return getNode(Opcode, DL, VT, N1: Ops[0], Flags);
12174 case 2: return getNode(Opcode, DL, VT, N1: Ops[0], N2: Ops[1], Flags);
12175 case 3: return getNode(Opcode, DL, VT, N1: Ops[0], N2: Ops[1], N3: Ops[2], Flags);
12176 default: break;
12177 }
12178
12179#ifndef NDEBUG
12180 for (const auto &Op : Ops)
12181 assert(Op.getOpcode() != ISD::DELETED_NODE &&
12182 "Operand is DELETED_NODE!");
12183#endif
12184
12185 switch (Opcode) {
12186 default: break;
12187 case ISD::BUILD_VECTOR:
12188 // Attempt to simplify BUILD_VECTOR.
12189 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, DAG&: *this))
12190 return V;
12191 break;
12192 case ISD::CONCAT_VECTORS:
12193 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, DAG&: *this))
12194 return V;
12195 break;
12196 case ISD::SELECT_CC:
12197 assert(NumOps == 5 && "SELECT_CC takes 5 operands!");
12198 assert(Ops[0].getValueType() == Ops[1].getValueType() &&
12199 "LHS and RHS of condition must have same type!");
12200 assert(Ops[2].getValueType() == Ops[3].getValueType() &&
12201 "True and False arms of SelectCC must have same type!");
12202 assert(Ops[2].getValueType() == VT &&
12203 "select_cc node must be of same type as true and false value!");
12204 assert((!Ops[0].getValueType().isVector() ||
12205 Ops[0].getValueType().getVectorElementCount() ==
12206 VT.getVectorElementCount()) &&
12207 "Expected select_cc with vector result to have the same sized "
12208 "comparison type!");
12209 break;
12210 case ISD::BR_CC:
12211 assert(NumOps == 5 && "BR_CC takes 5 operands!");
12212 assert(Ops[2].getValueType() == Ops[3].getValueType() &&
12213 "LHS/RHS of comparison should match types!");
12214 break;
12215 case ISD::VP_REDUCE_MUL:
12216 // If it is VP_REDUCE_MUL mask operation then turn it to VP_REDUCE_AND
12217 if (VT == MVT::i1)
12218 Opcode = ISD::VP_REDUCE_AND;
12219 break;
12220 case ISD::VP_REDUCE_ADD:
12221 // If it is VP_REDUCE_ADD mask operation then turn it to VP_REDUCE_XOR
12222 if (VT == MVT::i1)
12223 Opcode = ISD::VP_REDUCE_XOR;
12224 break;
12225 case ISD::VP_REDUCE_SMAX:
12226 case ISD::VP_REDUCE_UMIN:
12227 // If it is VP_REDUCE_SMAX/VP_REDUCE_UMIN mask operation then turn it to
12228 // VP_REDUCE_AND.
12229 if (VT == MVT::i1)
12230 Opcode = ISD::VP_REDUCE_AND;
12231 break;
12232 case ISD::VP_REDUCE_SMIN:
12233 case ISD::VP_REDUCE_UMAX:
12234 // If it is VP_REDUCE_SMIN/VP_REDUCE_UMAX mask operation then turn it to
12235 // VP_REDUCE_OR.
12236 if (VT == MVT::i1)
12237 Opcode = ISD::VP_REDUCE_OR;
12238 break;
12239 }
12240
12241 // Memoize nodes.
12242 SDNode *N;
12243 SDVTList VTs = getVTList(VT);
12244
12245 if (VT != MVT::Glue) {
12246 SDNodeKey ID(Opcode, VTs, Ops);
12247 FoldingSetInsertToken InsertToken;
12248
12249 if (SDNode *E = lookupNode(Key: ID, DL, InsertToken)) {
12250 E->intersectFlagsWith(Flags);
12251 return SDValue(E, 0);
12252 }
12253
12254 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
12255 createOperands(Node: N, Vals: Ops);
12256
12257 CSEMap.insert(N, Token: InsertToken);
12258 } else {
12259 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
12260 createOperands(Node: N, Vals: Ops);
12261 }
12262
12263 N->setFlags(Flags);
12264 InsertNode(N);
12265 SDValue V(N, 0);
12266 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
12267 return V;
12268}
12269
12270SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
12271 ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops) {
12272 SDNodeFlags Flags;
12273 if (Inserter)
12274 Flags = Inserter->getFlags();
12275 return getNode(Opcode, DL, VTList: getVTList(VTs: ResultTys), Ops, Flags);
12276}
12277
12278SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
12279 ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops,
12280 const SDNodeFlags Flags) {
12281 return getNode(Opcode, DL, VTList: getVTList(VTs: ResultTys), Ops, Flags);
12282}
12283
12284SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
12285 ArrayRef<SDValue> Ops) {
12286 SDNodeFlags Flags;
12287 if (Inserter)
12288 Flags = Inserter->getFlags();
12289 return getNode(Opcode, DL, VTList, Ops, Flags);
12290}
12291
12292SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
12293 ArrayRef<SDValue> Ops, const SDNodeFlags Flags) {
12294 if (VTList.NumVTs == 1)
12295 return getNode(Opcode, DL, VT: VTList.VTs[0], Ops, Flags);
12296
12297#ifndef NDEBUG
12298 for (const auto &Op : Ops)
12299 assert(Op.getOpcode() != ISD::DELETED_NODE &&
12300 "Operand is DELETED_NODE!");
12301#endif
12302
12303 if (SDValue V = FoldConstantArithmetic(Opcode, DL, VTList, Ops, Flags))
12304 return V;
12305
12306 switch (Opcode) {
12307 case ISD::SADDO:
12308 case ISD::UADDO:
12309 case ISD::SSUBO:
12310 case ISD::USUBO: {
12311 assert(VTList.NumVTs == 2 && Ops.size() == 2 &&
12312 "Invalid add/sub overflow op!");
12313 assert(VTList.VTs[0].isInteger() && VTList.VTs[1].isInteger() &&
12314 Ops[0].getValueType() == Ops[1].getValueType() &&
12315 Ops[0].getValueType() == VTList.VTs[0] &&
12316 "Binary operator types must match!");
12317 SDValue N1 = Ops[0], N2 = Ops[1];
12318 canonicalizeCommutativeBinop(Opcode, N1, N2);
12319
12320 // (X +- 0) -> X with zero-overflow.
12321 ConstantSDNode *N2CV = isConstOrConstSplat(N: N2, /*AllowUndefs*/ false,
12322 /*AllowTruncation*/ true);
12323 if (N2CV && N2CV->isZero()) {
12324 SDValue ZeroOverFlow = getConstant(Val: 0, DL, VT: VTList.VTs[1]);
12325 return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList, Ops: {N1, ZeroOverFlow}, Flags);
12326 }
12327
12328 if (VTList.VTs[0].getScalarType() == MVT::i1 &&
12329 VTList.VTs[1].getScalarType() == MVT::i1) {
12330 SDValue F1 = getFreeze(V: N1);
12331 SDValue F2 = getFreeze(V: N2);
12332 // {vXi1,vXi1} (u/s)addo(vXi1 x, vXi1y) -> {xor(x,y),and(x,y)}
12333 if (Opcode == ISD::UADDO || Opcode == ISD::SADDO)
12334 return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList,
12335 Ops: {getNode(Opcode: ISD::XOR, DL, VT: VTList.VTs[0], N1: F1, N2: F2),
12336 getNode(Opcode: ISD::AND, DL, VT: VTList.VTs[1], N1: F1, N2: F2)},
12337 Flags);
12338 // {vXi1,vXi1} (u/s)subo(vXi1 x, vXi1y) -> {xor(x,y),and(~x,y)}
12339 if (Opcode == ISD::USUBO || Opcode == ISD::SSUBO) {
12340 SDValue NotF1 = getNOT(DL, Val: F1, VT: VTList.VTs[0]);
12341 return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList,
12342 Ops: {getNode(Opcode: ISD::XOR, DL, VT: VTList.VTs[0], N1: F1, N2: F2),
12343 getNode(Opcode: ISD::AND, DL, VT: VTList.VTs[1], N1: NotF1, N2: F2)},
12344 Flags);
12345 }
12346 }
12347 break;
12348 }
12349 case ISD::SADDO_CARRY:
12350 case ISD::UADDO_CARRY:
12351 case ISD::SSUBO_CARRY:
12352 case ISD::USUBO_CARRY:
12353 assert(VTList.NumVTs == 2 && Ops.size() == 3 &&
12354 "Invalid add/sub overflow op!");
12355 assert(VTList.VTs[0].isInteger() && VTList.VTs[1].isInteger() &&
12356 Ops[0].getValueType() == Ops[1].getValueType() &&
12357 Ops[0].getValueType() == VTList.VTs[0] &&
12358 Ops[2].getValueType() == VTList.VTs[1] &&
12359 "Binary operator types must match!");
12360 break;
12361 case ISD::SMUL_LOHI:
12362 case ISD::UMUL_LOHI: {
12363 assert(VTList.NumVTs == 2 && Ops.size() == 2 && "Invalid mul lo/hi op!");
12364 assert(VTList.VTs[0].isInteger() && VTList.VTs[0] == VTList.VTs[1] &&
12365 VTList.VTs[0] == Ops[0].getValueType() &&
12366 VTList.VTs[0] == Ops[1].getValueType() &&
12367 "Binary operator types must match!");
12368 break;
12369 }
12370 case ISD::FFREXP: {
12371 assert(VTList.NumVTs == 2 && Ops.size() == 1 && "Invalid ffrexp op!");
12372 assert(VTList.VTs[0].isFloatingPoint() && VTList.VTs[1].isInteger() &&
12373 VTList.VTs[0] == Ops[0].getValueType() && "frexp type mismatch");
12374
12375 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val: Ops[0])) {
12376 int FrexpExp;
12377 APFloat FrexpMant =
12378 frexp(X: C->getValueAPF(), Exp&: FrexpExp, RM: APFloat::rmNearestTiesToEven);
12379 SDValue Result0 = getConstantFP(V: FrexpMant, DL, VT: VTList.VTs[0]);
12380 SDValue Result1 = getSignedConstant(Val: FrexpMant.isFinite() ? FrexpExp : 0,
12381 DL, VT: VTList.VTs[1]);
12382 return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList, Ops: {Result0, Result1}, Flags);
12383 }
12384
12385 break;
12386 }
12387 case ISD::STRICT_FP_EXTEND:
12388 assert(VTList.NumVTs == 2 && Ops.size() == 2 &&
12389 "Invalid STRICT_FP_EXTEND!");
12390 assert(VTList.VTs[0].isFloatingPoint() &&
12391 Ops[1].getValueType().isFloatingPoint() && "Invalid FP cast!");
12392 assert(VTList.VTs[0].isVector() == Ops[1].getValueType().isVector() &&
12393 "STRICT_FP_EXTEND result type should be vector iff the operand "
12394 "type is vector!");
12395 assert((!VTList.VTs[0].isVector() ||
12396 VTList.VTs[0].getVectorElementCount() ==
12397 Ops[1].getValueType().getVectorElementCount()) &&
12398 "Vector element count mismatch!");
12399 assert(Ops[1].getValueType().bitsLT(VTList.VTs[0]) &&
12400 "Invalid fpext node, dst <= src!");
12401 break;
12402 case ISD::STRICT_FP_ROUND:
12403 assert(VTList.NumVTs == 2 && Ops.size() == 3 && "Invalid STRICT_FP_ROUND!");
12404 assert(VTList.VTs[0].isVector() == Ops[1].getValueType().isVector() &&
12405 "STRICT_FP_ROUND result type should be vector iff the operand "
12406 "type is vector!");
12407 assert((!VTList.VTs[0].isVector() ||
12408 VTList.VTs[0].getVectorElementCount() ==
12409 Ops[1].getValueType().getVectorElementCount()) &&
12410 "Vector element count mismatch!");
12411 assert(VTList.VTs[0].isFloatingPoint() &&
12412 Ops[1].getValueType().isFloatingPoint() &&
12413 VTList.VTs[0].bitsLT(Ops[1].getValueType()) &&
12414 Ops[2].getOpcode() == ISD::TargetConstant &&
12415 (Ops[2]->getAsZExtVal() == 0 || Ops[2]->getAsZExtVal() == 1) &&
12416 "Invalid STRICT_FP_ROUND!");
12417 break;
12418 }
12419
12420 // Memoize the node unless it returns a glue result.
12421 SDNode *N;
12422 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
12423 SDNodeKey ID(Opcode, VTList, Ops);
12424 FoldingSetInsertToken InsertToken;
12425 if (SDNode *E = lookupNode(Key: ID, DL, InsertToken)) {
12426 E->intersectFlagsWith(Flags);
12427 return SDValue(E, 0);
12428 }
12429
12430 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTList);
12431 createOperands(Node: N, Vals: Ops);
12432 CSEMap.insert(N, Token: InsertToken);
12433 } else {
12434 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTList);
12435 createOperands(Node: N, Vals: Ops);
12436 }
12437
12438 N->setFlags(Flags);
12439 InsertNode(N);
12440 SDValue V(N, 0);
12441 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
12442 return V;
12443}
12444
12445SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
12446 SDVTList VTList) {
12447 return getNode(Opcode, DL, VTList, Ops: ArrayRef<SDValue>());
12448}
12449
12450SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
12451 SDValue N1) {
12452 SDValue Ops[] = { N1 };
12453 return getNode(Opcode, DL, VTList, Ops);
12454}
12455
12456SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
12457 SDValue N1, SDValue N2) {
12458 SDValue Ops[] = { N1, N2 };
12459 return getNode(Opcode, DL, VTList, Ops);
12460}
12461
12462SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
12463 SDValue N1, SDValue N2, SDValue N3) {
12464 SDValue Ops[] = { N1, N2, N3 };
12465 return getNode(Opcode, DL, VTList, Ops);
12466}
12467
12468SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
12469 SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
12470 SDValue Ops[] = { N1, N2, N3, N4 };
12471 return getNode(Opcode, DL, VTList, Ops);
12472}
12473
12474SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
12475 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
12476 SDValue N5) {
12477 SDValue Ops[] = { N1, N2, N3, N4, N5 };
12478 return getNode(Opcode, DL, VTList, Ops);
12479}
12480
12481SDVTList SelectionDAG::getVTList(EVT VT) {
12482 if (!VT.isExtended())
12483 return makeVTList(VTs: SDNode::getValueTypeList(VT: VT.getSimpleVT()), NumVTs: 1);
12484
12485 EVT VTs[] = {VT};
12486 return getVTList(VTs);
12487}
12488
12489SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2) {
12490 EVT VTs[] = {VT1, VT2};
12491 return getVTList(VTs);
12492}
12493
12494SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3) {
12495 EVT VTs[] = {VT1, VT2, VT3};
12496 return getVTList(VTs);
12497}
12498
12499SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4) {
12500 EVT VTs[] = {VT1, VT2, VT3, VT4};
12501 return getVTList(VTs);
12502}
12503
12504SDVTList SelectionDAG::getVTList(ArrayRef<EVT> VTs) {
12505 auto It = VTLists.find(V: VTs);
12506 if (It == VTLists.end()) {
12507 EVT *Array = Allocator.Allocate<EVT>(Num: VTs.size());
12508 llvm::copy(Range&: VTs, Out: Array);
12509 It = VTLists.insert(V: ArrayRef(Array, VTs.size())).first;
12510 }
12511 return makeVTList(VTs: It->data(), NumVTs: It->size());
12512}
12513
12514/// UpdateNodeOperands - *Mutate* the specified node in-place to have the
12515/// specified operands. If the resultant node already exists in the DAG,
12516/// this does not modify the specified node, instead it returns the node that
12517/// already exists. If the resultant node does not exist in the DAG, the
12518/// input node is returned. As a degenerate case, if you specify the same
12519/// input operands as the node already has, the input node is returned.
12520SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op) {
12521 assert(N->getNumOperands() == 1 && "Update with wrong number of operands");
12522
12523 // Check to see if there is no change.
12524 if (Op == N->getOperand(Num: 0)) return N;
12525
12526 // See if the modified node already exists.
12527 FoldingSetInsertToken InsertToken;
12528 if (SDNode *Existing = FindModifiedNodeSlot(N, Op, InsertToken)) {
12529 Existing->intersectFlagsWith(Flags: N->getFlags());
12530 return Existing;
12531 }
12532
12533 // Nope it doesn't. Remove the node from its current place in the maps.
12534 if (InsertToken)
12535 if (!RemoveNodeFromCSEMaps(N))
12536 InsertToken = {};
12537
12538 // Now we update the operands.
12539 N->OperandList[0].set(Op);
12540
12541 updateDivergence(N);
12542 // If this gets put into a CSE map, add it.
12543 if (InsertToken)
12544 CSEMap.insert(N, Token: InsertToken);
12545 return N;
12546}
12547
12548SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2) {
12549 assert(N->getNumOperands() == 2 && "Update with wrong number of operands");
12550
12551 // Check to see if there is no change.
12552 if (Op1 == N->getOperand(Num: 0) && Op2 == N->getOperand(Num: 1))
12553 return N; // No operands changed, just return the input node.
12554
12555 // See if the modified node already exists.
12556 FoldingSetInsertToken InsertToken;
12557 if (SDNode *Existing = FindModifiedNodeSlot(N, Op1, Op2, InsertToken)) {
12558 Existing->intersectFlagsWith(Flags: N->getFlags());
12559 return Existing;
12560 }
12561
12562 // Nope it doesn't. Remove the node from its current place in the maps.
12563 if (InsertToken)
12564 if (!RemoveNodeFromCSEMaps(N))
12565 InsertToken = {};
12566
12567 // Now we update the operands.
12568 if (N->OperandList[0] != Op1)
12569 N->OperandList[0].set(Op1);
12570 if (N->OperandList[1] != Op2)
12571 N->OperandList[1].set(Op2);
12572
12573 updateDivergence(N);
12574 // If this gets put into a CSE map, add it.
12575 if (InsertToken)
12576 CSEMap.insert(N, Token: InsertToken);
12577 return N;
12578}
12579
12580SDNode *SelectionDAG::
12581UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, SDValue Op3) {
12582 SDValue Ops[] = { Op1, Op2, Op3 };
12583 return UpdateNodeOperands(N, Ops);
12584}
12585
12586SDNode *SelectionDAG::
12587UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
12588 SDValue Op3, SDValue Op4) {
12589 SDValue Ops[] = { Op1, Op2, Op3, Op4 };
12590 return UpdateNodeOperands(N, Ops);
12591}
12592
12593SDNode *SelectionDAG::
12594UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
12595 SDValue Op3, SDValue Op4, SDValue Op5) {
12596 SDValue Ops[] = { Op1, Op2, Op3, Op4, Op5 };
12597 return UpdateNodeOperands(N, Ops);
12598}
12599
12600SDNode *SelectionDAG::
12601UpdateNodeOperands(SDNode *N, ArrayRef<SDValue> Ops) {
12602 unsigned NumOps = Ops.size();
12603 assert(N->getNumOperands() == NumOps &&
12604 "Update with wrong number of operands");
12605
12606 // If no operands changed just return the input node.
12607 if (std::equal(first1: Ops.begin(), last1: Ops.end(), first2: N->op_begin()))
12608 return N;
12609
12610 // See if the modified node already exists.
12611 FoldingSetInsertToken InsertToken;
12612 if (SDNode *Existing = FindModifiedNodeSlot(N, Ops, InsertToken)) {
12613 Existing->intersectFlagsWith(Flags: N->getFlags());
12614 return Existing;
12615 }
12616
12617 // Nope it doesn't. Remove the node from its current place in the maps.
12618 if (InsertToken)
12619 if (!RemoveNodeFromCSEMaps(N))
12620 InsertToken = {};
12621
12622 // Now we update the operands.
12623 for (unsigned i = 0; i != NumOps; ++i)
12624 if (N->OperandList[i] != Ops[i])
12625 N->OperandList[i].set(Ops[i]);
12626
12627 updateDivergence(N);
12628 // If this gets put into a CSE map, add it.
12629 if (InsertToken)
12630 CSEMap.insert(N, Token: InsertToken);
12631 return N;
12632}
12633
12634/// DropOperands - Release the operands and set this node to have
12635/// zero operands.
12636void SDNode::DropOperands() {
12637 // Unlike the code in MorphNodeTo that does this, we don't need to
12638 // watch for dead nodes here.
12639 for (op_iterator I = op_begin(), E = op_end(); I != E; ) {
12640 SDUse &Use = *I++;
12641 Use.set(SDValue());
12642 }
12643}
12644
12645void SelectionDAG::setNodeMemRefs(MachineSDNode *N,
12646 ArrayRef<MachineMemOperand *> NewMemRefs) {
12647 if (NewMemRefs.empty()) {
12648 N->clearMemRefs();
12649 return;
12650 }
12651
12652 // Check if we can avoid allocating by storing a single reference directly.
12653 if (NewMemRefs.size() == 1) {
12654 N->MemRefs = NewMemRefs[0];
12655 N->NumMemRefs = 1;
12656 return;
12657 }
12658
12659 MachineMemOperand **MemRefsBuffer =
12660 Allocator.template Allocate<MachineMemOperand *>(Num: NewMemRefs.size());
12661 llvm::copy(Range&: NewMemRefs, Out: MemRefsBuffer);
12662 N->MemRefs = MemRefsBuffer;
12663 N->NumMemRefs = static_cast<int>(NewMemRefs.size());
12664}
12665
12666/// SelectNodeTo - These are wrappers around MorphNodeTo that accept a
12667/// machine opcode.
12668///
12669SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12670 EVT VT) {
12671 SDVTList VTs = getVTList(VT);
12672 return SelectNodeTo(N, MachineOpc, VTs, Ops: {});
12673}
12674
12675SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12676 EVT VT, SDValue Op1) {
12677 SDVTList VTs = getVTList(VT);
12678 SDValue Ops[] = { Op1 };
12679 return SelectNodeTo(N, MachineOpc, VTs, Ops);
12680}
12681
12682SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12683 EVT VT, SDValue Op1,
12684 SDValue Op2) {
12685 SDVTList VTs = getVTList(VT);
12686 SDValue Ops[] = { Op1, Op2 };
12687 return SelectNodeTo(N, MachineOpc, VTs, Ops);
12688}
12689
12690SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12691 EVT VT, SDValue Op1,
12692 SDValue Op2, SDValue Op3) {
12693 SDVTList VTs = getVTList(VT);
12694 SDValue Ops[] = { Op1, Op2, Op3 };
12695 return SelectNodeTo(N, MachineOpc, VTs, Ops);
12696}
12697
12698SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12699 EVT VT, ArrayRef<SDValue> Ops) {
12700 SDVTList VTs = getVTList(VT);
12701 return SelectNodeTo(N, MachineOpc, VTs, Ops);
12702}
12703
12704SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12705 EVT VT1, EVT VT2, ArrayRef<SDValue> Ops) {
12706 SDVTList VTs = getVTList(VT1, VT2);
12707 return SelectNodeTo(N, MachineOpc, VTs, Ops);
12708}
12709
12710SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12711 EVT VT1, EVT VT2) {
12712 SDVTList VTs = getVTList(VT1, VT2);
12713 return SelectNodeTo(N, MachineOpc, VTs, Ops: {});
12714}
12715
12716SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12717 EVT VT1, EVT VT2, EVT VT3,
12718 ArrayRef<SDValue> Ops) {
12719 SDVTList VTs = getVTList(VT1, VT2, VT3);
12720 return SelectNodeTo(N, MachineOpc, VTs, Ops);
12721}
12722
12723SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12724 EVT VT1, EVT VT2,
12725 SDValue Op1, SDValue Op2) {
12726 SDVTList VTs = getVTList(VT1, VT2);
12727 SDValue Ops[] = { Op1, Op2 };
12728 return SelectNodeTo(N, MachineOpc, VTs, Ops);
12729}
12730
12731SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12732 SDVTList VTs,ArrayRef<SDValue> Ops) {
12733 SDNode *New = MorphNodeTo(N, Opc: ~MachineOpc, VTs, Ops);
12734 // Reset the NodeID to -1.
12735 New->setNodeId(-1);
12736 if (New != N) {
12737 ReplaceAllUsesWith(From: N, To: New);
12738 RemoveDeadNode(N);
12739 }
12740 return New;
12741}
12742
12743/// UpdateSDLocOnMergeSDNode - If the opt level is -O0 then it throws away
12744/// the line number information on the merged node since it is not possible to
12745/// preserve the information that operation is associated with multiple lines.
12746/// This will make the debugger work better at -O0, where there is a higher
12747/// probability of having other instructions associated with that line.
12748///
12749/// For IROrder, we keep the smaller of the two
12750SDNode *SelectionDAG::UpdateSDLocOnMergeSDNode(SDNode *N, const SDLoc &OLoc) {
12751 DebugLoc NLoc = N->getDebugLoc();
12752 if (NLoc && OptLevel == CodeGenOptLevel::None && OLoc.getDebugLoc() != NLoc) {
12753 N->setDebugLoc(DebugLoc());
12754 }
12755 unsigned Order = std::min(a: N->getIROrder(), b: OLoc.getIROrder());
12756 N->setIROrder(Order);
12757 return N;
12758}
12759
12760/// MorphNodeTo - This *mutates* the specified node to have the specified
12761/// return type, opcode, and operands.
12762///
12763/// Note that MorphNodeTo returns the resultant node. If there is already a
12764/// node of the specified opcode and operands, it returns that node instead of
12765/// the current one. Note that the SDLoc need not be the same.
12766///
12767/// Using MorphNodeTo is faster than creating a new node and swapping it in
12768/// with ReplaceAllUsesWith both because it often avoids allocating a new
12769/// node, and because it doesn't require CSE recalculation for any of
12770/// the node's users.
12771///
12772/// However, note that MorphNodeTo recursively deletes dead nodes from the DAG.
12773/// As a consequence it isn't appropriate to use from within the DAG combiner or
12774/// the legalizer which maintain worklists that would need to be updated when
12775/// deleting things.
12776SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
12777 SDVTList VTs, ArrayRef<SDValue> Ops) {
12778 // If an identical node already exists, use it.
12779 FoldingSetInsertToken InsertToken;
12780 if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) {
12781 SDNodeKey ID(Opc, VTs, Ops);
12782 AddNodeIDCustom(ID&: ID.Tail, N, Opc);
12783 if (SDNode *ON = lookupNode(Key: ID, DL: SDLoc(N), InsertToken))
12784 return UpdateSDLocOnMergeSDNode(N: ON, OLoc: SDLoc(N));
12785 }
12786
12787 if (!RemoveNodeFromCSEMaps(N))
12788 InsertToken = {};
12789
12790 // Start the morphing.
12791 N->NodeType = Opc;
12792 N->ValueList = VTs.VTs;
12793 N->NumValues = VTs.NumVTs;
12794
12795 // Clear the operands list, updating used nodes to remove this from their
12796 // use list. Keep track of any operands that become dead as a result.
12797 SmallPtrSet<SDNode*, 16> DeadNodeSet;
12798 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
12799 SDUse &Use = *I++;
12800 SDNode *Used = Use.getNode();
12801 Use.set(SDValue());
12802 if (Used->use_empty())
12803 DeadNodeSet.insert(Ptr: Used);
12804 }
12805
12806 // For MachineNode, initialize the memory references information.
12807 if (MachineSDNode *MN = dyn_cast<MachineSDNode>(Val: N))
12808 MN->clearMemRefs();
12809
12810 // Swap for an appropriately sized array from the recycler.
12811 removeOperands(Node: N);
12812 createOperands(Node: N, Vals: Ops);
12813
12814 // Delete any nodes that are still dead after adding the uses for the
12815 // new operands.
12816 if (!DeadNodeSet.empty()) {
12817 SmallVector<SDNode *, 16> DeadNodes;
12818 for (SDNode *N : DeadNodeSet)
12819 if (N->use_empty())
12820 DeadNodes.push_back(Elt: N);
12821 RemoveDeadNodes(DeadNodes);
12822 }
12823
12824 if (InsertToken)
12825 CSEMap.insert(N, Token: InsertToken); // Memoize the new node.
12826 return N;
12827}
12828
12829SDNode* SelectionDAG::mutateStrictFPToFP(SDNode *Node) {
12830 unsigned OrigOpc = Node->getOpcode();
12831 unsigned NewOpc;
12832 switch (OrigOpc) {
12833 default:
12834 llvm_unreachable("mutateStrictFPToFP called with unexpected opcode!");
12835#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
12836 case ISD::STRICT_##DAGN: NewOpc = ISD::DAGN; break;
12837#define CMP_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
12838 case ISD::STRICT_##DAGN: NewOpc = ISD::SETCC; break;
12839#include "llvm/IR/ConstrainedOps.def"
12840 }
12841
12842 assert(Node->getNumValues() == 2 && "Unexpected number of results!");
12843
12844 // We're taking this node out of the chain, so we need to re-link things.
12845 SDValue InputChain = Node->getOperand(Num: 0);
12846 SDValue OutputChain = SDValue(Node, 1);
12847 ReplaceAllUsesOfValueWith(From: OutputChain, To: InputChain);
12848
12849 SmallVector<SDValue, 3> Ops;
12850 for (unsigned i = 1, e = Node->getNumOperands(); i != e; ++i)
12851 Ops.push_back(Elt: Node->getOperand(Num: i));
12852
12853 SDVTList VTs = getVTList(VT: Node->getValueType(ResNo: 0));
12854 SDNode *Res = MorphNodeTo(N: Node, Opc: NewOpc, VTs, Ops);
12855
12856 // MorphNodeTo can operate in two ways: if an existing node with the
12857 // specified operands exists, it can just return it. Otherwise, it
12858 // updates the node in place to have the requested operands.
12859 if (Res == Node) {
12860 // If we updated the node in place, reset the node ID. To the isel,
12861 // this should be just like a newly allocated machine node.
12862 Res->setNodeId(-1);
12863 } else {
12864 ReplaceAllUsesWith(From: Node, To: Res);
12865 RemoveDeadNode(N: Node);
12866 }
12867
12868 return Res;
12869}
12870
12871/// getMachineNode - These are used for target selectors to create a new node
12872/// with specified return type(s), MachineInstr opcode, and operands.
12873///
12874/// Note that getMachineNode returns the resultant node. If there is already a
12875/// node of the specified opcode and operands, it returns that node instead of
12876/// the current one.
12877MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12878 EVT VT) {
12879 SDVTList VTs = getVTList(VT);
12880 return getMachineNode(Opcode, dl, VTs, Ops: {});
12881}
12882
12883MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12884 EVT VT, SDValue Op1) {
12885 SDVTList VTs = getVTList(VT);
12886 SDValue Ops[] = { Op1 };
12887 return getMachineNode(Opcode, dl, VTs, Ops);
12888}
12889
12890MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12891 EVT VT, SDValue Op1, SDValue Op2) {
12892 SDVTList VTs = getVTList(VT);
12893 SDValue Ops[] = { Op1, Op2 };
12894 return getMachineNode(Opcode, dl, VTs, Ops);
12895}
12896
12897MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12898 EVT VT, SDValue Op1, SDValue Op2,
12899 SDValue Op3) {
12900 SDVTList VTs = getVTList(VT);
12901 SDValue Ops[] = { Op1, Op2, Op3 };
12902 return getMachineNode(Opcode, dl, VTs, Ops);
12903}
12904
12905MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12906 EVT VT, ArrayRef<SDValue> Ops) {
12907 SDVTList VTs = getVTList(VT);
12908 return getMachineNode(Opcode, dl, VTs, Ops);
12909}
12910
12911MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12912 EVT VT1, EVT VT2, SDValue Op1,
12913 SDValue Op2) {
12914 SDVTList VTs = getVTList(VT1, VT2);
12915 SDValue Ops[] = { Op1, Op2 };
12916 return getMachineNode(Opcode, dl, VTs, Ops);
12917}
12918
12919MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12920 EVT VT1, EVT VT2, SDValue Op1,
12921 SDValue Op2, SDValue Op3) {
12922 SDVTList VTs = getVTList(VT1, VT2);
12923 SDValue Ops[] = { Op1, Op2, Op3 };
12924 return getMachineNode(Opcode, dl, VTs, Ops);
12925}
12926
12927MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12928 EVT VT1, EVT VT2,
12929 ArrayRef<SDValue> Ops) {
12930 SDVTList VTs = getVTList(VT1, VT2);
12931 return getMachineNode(Opcode, dl, VTs, Ops);
12932}
12933
12934MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12935 EVT VT1, EVT VT2, EVT VT3,
12936 SDValue Op1, SDValue Op2) {
12937 SDVTList VTs = getVTList(VT1, VT2, VT3);
12938 SDValue Ops[] = { Op1, Op2 };
12939 return getMachineNode(Opcode, dl, VTs, Ops);
12940}
12941
12942MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12943 EVT VT1, EVT VT2, EVT VT3,
12944 SDValue Op1, SDValue Op2,
12945 SDValue Op3) {
12946 SDVTList VTs = getVTList(VT1, VT2, VT3);
12947 SDValue Ops[] = { Op1, Op2, Op3 };
12948 return getMachineNode(Opcode, dl, VTs, Ops);
12949}
12950
12951MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12952 EVT VT1, EVT VT2, EVT VT3,
12953 ArrayRef<SDValue> Ops) {
12954 SDVTList VTs = getVTList(VT1, VT2, VT3);
12955 return getMachineNode(Opcode, dl, VTs, Ops);
12956}
12957
12958MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12959 ArrayRef<EVT> ResultTys,
12960 ArrayRef<SDValue> Ops) {
12961 SDVTList VTs = getVTList(VTs: ResultTys);
12962 return getMachineNode(Opcode, dl, VTs, Ops);
12963}
12964
12965MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &DL,
12966 SDVTList VTs,
12967 ArrayRef<SDValue> Ops) {
12968 bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue;
12969 MachineSDNode *N;
12970 FoldingSetInsertToken InsertToken;
12971
12972 if (DoCSE) {
12973 SDNodeKey ID(~Opcode, VTs, Ops);
12974 if (SDNode *E = lookupNode(Key: ID, DL, InsertToken)) {
12975 return cast<MachineSDNode>(Val: UpdateSDLocOnMergeSDNode(N: E, OLoc: DL));
12976 }
12977 }
12978
12979 // Allocate a new MachineSDNode.
12980 N = newSDNode<MachineSDNode>(Args: ~Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
12981 createOperands(Node: N, Vals: Ops);
12982
12983 if (DoCSE)
12984 CSEMap.insert(N, Token: InsertToken);
12985
12986 InsertNode(N);
12987 NewSDValueDbgMsg(V: SDValue(N, 0), Msg: "Creating new machine node: ", G: this);
12988 return N;
12989}
12990
12991/// getTargetExtractSubreg - A convenience function for creating
12992/// TargetOpcode::EXTRACT_SUBREG nodes.
12993SDValue SelectionDAG::getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT,
12994 SDValue Operand) {
12995 SDValue SRIdxVal = getTargetConstant(Val: SRIdx, DL, VT: MVT::i32);
12996 SDNode *Subreg = getMachineNode(Opcode: TargetOpcode::EXTRACT_SUBREG, dl: DL,
12997 VT, Op1: Operand, Op2: SRIdxVal);
12998 return SDValue(Subreg, 0);
12999}
13000
13001/// getTargetInsertSubreg - A convenience function for creating
13002/// TargetOpcode::INSERT_SUBREG nodes.
13003SDValue SelectionDAG::getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT,
13004 SDValue Operand, SDValue Subreg) {
13005 SDValue SRIdxVal = getTargetConstant(Val: SRIdx, DL, VT: MVT::i32);
13006 SDNode *Result = getMachineNode(Opcode: TargetOpcode::INSERT_SUBREG, dl: DL,
13007 VT, Op1: Operand, Op2: Subreg, Op3: SRIdxVal);
13008 return SDValue(Result, 0);
13009}
13010
13011/// getNodeIfExists - Get the specified node if it's already available, or
13012/// else return NULL.
13013SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,
13014 ArrayRef<SDValue> Ops,
13015 bool AllowCommute) {
13016 SDNodeFlags Flags;
13017 if (Inserter)
13018 Flags = Inserter->getFlags();
13019 return getNodeIfExists(Opcode, VTList, Ops, Flags, AllowCommute);
13020}
13021
13022SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,
13023 ArrayRef<SDValue> Ops,
13024 const SDNodeFlags Flags,
13025 bool AllowCommute) {
13026 if (VTList.VTs[VTList.NumVTs - 1] == MVT::Glue)
13027 return nullptr;
13028
13029 auto Lookup = [&](ArrayRef<SDValue> LookupOps) -> SDNode * {
13030 SDNodeKey ID(Opcode, VTList, LookupOps);
13031 FoldingSetInsertToken InsertToken;
13032 if (SDNode *E = lookupNode(Key: ID, InsertToken)) {
13033 E->intersectFlagsWith(Flags);
13034 return E;
13035 }
13036 return nullptr;
13037 };
13038
13039 if (SDNode *Existing = Lookup(Ops))
13040 return Existing;
13041
13042 if (AllowCommute && TLI->isCommutativeBinOp(Opcode))
13043 return Lookup({Ops[1], Ops[0]});
13044
13045 return nullptr;
13046}
13047
13048/// doesNodeExist - Check if a node exists without modifying its flags.
13049bool SelectionDAG::doesNodeExist(unsigned Opcode, SDVTList VTList,
13050 ArrayRef<SDValue> Ops) {
13051 if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) {
13052 SDNodeKey ID(Opcode, VTList, Ops);
13053 FoldingSetInsertToken InsertToken;
13054 if (lookupNode(Key: ID, DL: SDLoc(), InsertToken))
13055 return true;
13056 }
13057 return false;
13058}
13059
13060/// getDbgValue - Creates a SDDbgValue node.
13061///
13062/// SDNode
13063SDDbgValue *SelectionDAG::getDbgValue(DIVariable *Var, DIExpression *Expr,
13064 SDNode *N, unsigned R, bool IsIndirect,
13065 const DebugLoc &DL, unsigned O) {
13066 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
13067 "Expected inlined-at fields to agree");
13068 return new (DbgInfo->getAlloc())
13069 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromNode(Node: N, ResNo: R),
13070 {}, IsIndirect, DL, O,
13071 /*IsVariadic=*/false);
13072}
13073
13074/// Constant
13075SDDbgValue *SelectionDAG::getConstantDbgValue(DIVariable *Var,
13076 DIExpression *Expr,
13077 const Value *C,
13078 const DebugLoc &DL, unsigned O) {
13079 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
13080 "Expected inlined-at fields to agree");
13081 return new (DbgInfo->getAlloc())
13082 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromConst(Const: C), {},
13083 /*IsIndirect=*/false, DL, O,
13084 /*IsVariadic=*/false);
13085}
13086
13087/// FrameIndex
13088SDDbgValue *SelectionDAG::getFrameIndexDbgValue(DIVariable *Var,
13089 DIExpression *Expr, unsigned FI,
13090 bool IsIndirect,
13091 const DebugLoc &DL,
13092 unsigned O) {
13093 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
13094 "Expected inlined-at fields to agree");
13095 return getFrameIndexDbgValue(Var, Expr, FI, Dependencies: {}, IsIndirect, DL, O);
13096}
13097
13098/// FrameIndex with dependencies
13099SDDbgValue *SelectionDAG::getFrameIndexDbgValue(DIVariable *Var,
13100 DIExpression *Expr, unsigned FI,
13101 ArrayRef<SDNode *> Dependencies,
13102 bool IsIndirect,
13103 const DebugLoc &DL,
13104 unsigned O) {
13105 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
13106 "Expected inlined-at fields to agree");
13107 return new (DbgInfo->getAlloc())
13108 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromFrameIdx(FrameIdx: FI),
13109 Dependencies, IsIndirect, DL, O,
13110 /*IsVariadic=*/false);
13111}
13112
13113/// VReg
13114SDDbgValue *SelectionDAG::getVRegDbgValue(DIVariable *Var, DIExpression *Expr,
13115 Register VReg, bool IsIndirect,
13116 const DebugLoc &DL, unsigned O) {
13117 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
13118 "Expected inlined-at fields to agree");
13119 return new (DbgInfo->getAlloc())
13120 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromVReg(VReg),
13121 {}, IsIndirect, DL, O,
13122 /*IsVariadic=*/false);
13123}
13124
13125SDDbgValue *SelectionDAG::getDbgValueList(DIVariable *Var, DIExpression *Expr,
13126 ArrayRef<SDDbgOperand> Locs,
13127 ArrayRef<SDNode *> Dependencies,
13128 bool IsIndirect, const DebugLoc &DL,
13129 unsigned O, bool IsVariadic) {
13130 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
13131 "Expected inlined-at fields to agree");
13132 return new (DbgInfo->getAlloc())
13133 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, Locs, Dependencies, IsIndirect,
13134 DL, O, IsVariadic);
13135}
13136
13137void SelectionDAG::transferDbgValues(SDValue From, SDValue To,
13138 unsigned OffsetInBits, unsigned SizeInBits,
13139 bool InvalidateDbg) {
13140 SDNode *FromNode = From.getNode();
13141 SDNode *ToNode = To.getNode();
13142 assert(FromNode && ToNode && "Can't modify dbg values");
13143
13144 // PR35338
13145 // TODO: assert(From != To && "Redundant dbg value transfer");
13146 // TODO: assert(FromNode != ToNode && "Intranode dbg value transfer");
13147 if (From == To || FromNode == ToNode)
13148 return;
13149
13150 if (!FromNode->getHasDebugValue())
13151 return;
13152
13153 SDDbgOperand FromLocOp =
13154 SDDbgOperand::fromNode(Node: From.getNode(), ResNo: From.getResNo());
13155 SDDbgOperand ToLocOp = SDDbgOperand::fromNode(Node: To.getNode(), ResNo: To.getResNo());
13156
13157 SmallVector<SDDbgValue *, 2> ClonedDVs;
13158 for (SDDbgValue *Dbg : GetDbgValues(SD: FromNode)) {
13159 if (Dbg->isInvalidated())
13160 continue;
13161
13162 // TODO: assert(!Dbg->isInvalidated() && "Transfer of invalid dbg value");
13163
13164 // Create a new location ops vector that is equal to the old vector, but
13165 // with each instance of FromLocOp replaced with ToLocOp.
13166 bool Changed = false;
13167 auto NewLocOps = Dbg->copyLocationOps();
13168 std::replace_if(
13169 first: NewLocOps.begin(), last: NewLocOps.end(),
13170 pred: [&Changed, FromLocOp](const SDDbgOperand &Op) {
13171 bool Match = Op == FromLocOp;
13172 Changed |= Match;
13173 return Match;
13174 },
13175 new_value: ToLocOp);
13176 // Ignore this SDDbgValue if we didn't find a matching location.
13177 if (!Changed)
13178 continue;
13179
13180 DIVariable *Var = Dbg->getVariable();
13181 auto *Expr = Dbg->getExpression();
13182 // If a fragment is requested, update the expression.
13183 if (SizeInBits) {
13184 // When splitting a larger (e.g., sign-extended) value whose
13185 // lower bits are described with an SDDbgValue, do not attempt
13186 // to transfer the SDDbgValue to the upper bits.
13187 if (auto FI = Expr->getFragmentInfo())
13188 if (OffsetInBits + SizeInBits > FI->SizeInBits)
13189 continue;
13190 auto Fragment = DIExpression::createFragmentExpression(Expr, OffsetInBits,
13191 SizeInBits);
13192 if (!Fragment)
13193 continue;
13194 Expr = *Fragment;
13195 }
13196
13197 auto AdditionalDependencies = Dbg->getAdditionalDependencies();
13198 // Clone the SDDbgValue and move it to To.
13199 SDDbgValue *Clone = getDbgValueList(
13200 Var, Expr, Locs: NewLocOps, Dependencies: AdditionalDependencies, IsIndirect: Dbg->isIndirect(),
13201 DL: Dbg->getDebugLoc(), O: std::max(a: ToNode->getIROrder(), b: Dbg->getOrder()),
13202 IsVariadic: Dbg->isVariadic());
13203 ClonedDVs.push_back(Elt: Clone);
13204
13205 if (InvalidateDbg) {
13206 // Invalidate value and indicate the SDDbgValue should not be emitted.
13207 Dbg->setIsInvalidated();
13208 Dbg->setIsEmitted();
13209 }
13210 }
13211
13212 for (SDDbgValue *Dbg : ClonedDVs) {
13213 assert(is_contained(Dbg->getSDNodes(), ToNode) &&
13214 "Transferred DbgValues should depend on the new SDNode");
13215 AddDbgValue(DB: Dbg, isParameter: false);
13216 }
13217}
13218
13219void SelectionDAG::salvageDebugInfo(SDNode &N) {
13220 if (!N.getHasDebugValue())
13221 return;
13222
13223 auto GetLocationOperand = [](SDNode *Node, unsigned ResNo) {
13224 if (auto *FISDN = dyn_cast<FrameIndexSDNode>(Val: Node))
13225 return SDDbgOperand::fromFrameIdx(FrameIdx: FISDN->getIndex());
13226 return SDDbgOperand::fromNode(Node, ResNo);
13227 };
13228
13229 SmallVector<SDDbgValue *, 2> ClonedDVs;
13230 for (auto *DV : GetDbgValues(SD: &N)) {
13231 if (DV->isInvalidated())
13232 continue;
13233 switch (N.getOpcode()) {
13234 default:
13235 break;
13236 case ISD::ADD: {
13237 SDValue N0 = N.getOperand(Num: 0);
13238 SDValue N1 = N.getOperand(Num: 1);
13239 if (!isa<ConstantSDNode>(Val: N0)) {
13240 bool RHSConstant = isa<ConstantSDNode>(Val: N1);
13241 uint64_t Offset;
13242 if (RHSConstant)
13243 Offset = N.getConstantOperandVal(Num: 1);
13244 // We are not allowed to turn indirect debug values variadic, so
13245 // don't salvage those.
13246 if (!RHSConstant && DV->isIndirect())
13247 continue;
13248
13249 // Rewrite an ADD constant node into a DIExpression. Since we are
13250 // performing arithmetic to compute the variable's *value* in the
13251 // DIExpression, we need to mark the expression with a
13252 // DW_OP_stack_value.
13253 auto *DIExpr = DV->getExpression();
13254 auto NewLocOps = DV->copyLocationOps();
13255 bool Changed = false;
13256 size_t OrigLocOpsSize = NewLocOps.size();
13257 for (size_t i = 0; i < OrigLocOpsSize; ++i) {
13258 // We're not given a ResNo to compare against because the whole
13259 // node is going away. We know that any ISD::ADD only has one
13260 // result, so we can assume any node match is using the result.
13261 if (NewLocOps[i].getKind() != SDDbgOperand::SDNODE ||
13262 NewLocOps[i].getSDNode() != &N)
13263 continue;
13264 NewLocOps[i] = GetLocationOperand(N0.getNode(), N0.getResNo());
13265 if (RHSConstant) {
13266 SmallVector<uint64_t, 3> ExprOps;
13267 DIExpression::appendOffset(Ops&: ExprOps, Offset);
13268 DIExpr = DIExpression::appendOpsToArg(Expr: DIExpr, Ops: ExprOps, ArgNo: i, StackValue: true);
13269 } else {
13270 // Convert to a variadic expression (if not already).
13271 // convertToVariadicExpression() returns a const pointer, so we use
13272 // a temporary const variable here.
13273 const auto *TmpDIExpr =
13274 DIExpression::convertToVariadicExpression(Expr: DIExpr);
13275 SmallVector<uint64_t, 3> ExprOps;
13276 ExprOps.push_back(Elt: dwarf::DW_OP_LLVM_arg);
13277 ExprOps.push_back(Elt: NewLocOps.size());
13278 ExprOps.push_back(Elt: dwarf::DW_OP_plus);
13279 SDDbgOperand RHS =
13280 SDDbgOperand::fromNode(Node: N1.getNode(), ResNo: N1.getResNo());
13281 NewLocOps.push_back(Elt: RHS);
13282 DIExpr = DIExpression::appendOpsToArg(Expr: TmpDIExpr, Ops: ExprOps, ArgNo: i, StackValue: true);
13283 }
13284 Changed = true;
13285 }
13286 (void)Changed;
13287 assert(Changed && "Salvage target doesn't use N");
13288
13289 bool IsVariadic =
13290 DV->isVariadic() || OrigLocOpsSize != NewLocOps.size();
13291
13292 auto AdditionalDependencies = DV->getAdditionalDependencies();
13293 SDDbgValue *Clone = getDbgValueList(
13294 Var: DV->getVariable(), Expr: DIExpr, Locs: NewLocOps, Dependencies: AdditionalDependencies,
13295 IsIndirect: DV->isIndirect(), DL: DV->getDebugLoc(), O: DV->getOrder(), IsVariadic);
13296 ClonedDVs.push_back(Elt: Clone);
13297 DV->setIsInvalidated();
13298 DV->setIsEmitted();
13299 LLVM_DEBUG(dbgs() << "SALVAGE: Rewriting";
13300 N0.getNode()->dumprFull(this);
13301 dbgs() << " into " << *DIExpr << '\n');
13302 }
13303 break;
13304 }
13305 case ISD::TRUNCATE: {
13306 SDValue N0 = N.getOperand(Num: 0);
13307 TypeSize FromSize = N0.getValueSizeInBits();
13308 TypeSize ToSize = N.getValueSizeInBits(ResNo: 0);
13309
13310 DIExpression *DbgExpression = DV->getExpression();
13311 auto ExtOps = DIExpression::getExtOps(FromSize, ToSize, Signed: false);
13312 auto NewLocOps = DV->copyLocationOps();
13313 bool Changed = false;
13314 for (size_t i = 0; i < NewLocOps.size(); ++i) {
13315 if (NewLocOps[i].getKind() != SDDbgOperand::SDNODE ||
13316 NewLocOps[i].getSDNode() != &N)
13317 continue;
13318
13319 NewLocOps[i] = GetLocationOperand(N0.getNode(), N0.getResNo());
13320 DbgExpression = DIExpression::appendOpsToArg(Expr: DbgExpression, Ops: ExtOps, ArgNo: i);
13321 Changed = true;
13322 }
13323 assert(Changed && "Salvage target doesn't use N");
13324 (void)Changed;
13325
13326 SDDbgValue *Clone =
13327 getDbgValueList(Var: DV->getVariable(), Expr: DbgExpression, Locs: NewLocOps,
13328 Dependencies: DV->getAdditionalDependencies(), IsIndirect: DV->isIndirect(),
13329 DL: DV->getDebugLoc(), O: DV->getOrder(), IsVariadic: DV->isVariadic());
13330
13331 ClonedDVs.push_back(Elt: Clone);
13332 DV->setIsInvalidated();
13333 DV->setIsEmitted();
13334 LLVM_DEBUG(dbgs() << "SALVAGE: Rewriting"; N0.getNode()->dumprFull(this);
13335 dbgs() << " into " << *DbgExpression << '\n');
13336 break;
13337 }
13338 }
13339 }
13340
13341 for (SDDbgValue *Dbg : ClonedDVs) {
13342 assert((!Dbg->getSDNodes().empty() ||
13343 llvm::any_of(Dbg->getLocationOps(),
13344 [&](const SDDbgOperand &Op) {
13345 return Op.getKind() == SDDbgOperand::FRAMEIX;
13346 })) &&
13347 "Salvaged DbgValue should depend on a new SDNode");
13348 AddDbgValue(DB: Dbg, isParameter: false);
13349 }
13350}
13351
13352/// Creates a SDDbgLabel node.
13353SDDbgLabel *SelectionDAG::getDbgLabel(DILabel *Label,
13354 const DebugLoc &DL, unsigned O) {
13355 assert(cast<DILabel>(Label)->isValidLocationForIntrinsic(DL) &&
13356 "Expected inlined-at fields to agree");
13357 return new (DbgInfo->getAlloc()) SDDbgLabel(Label, DL, O);
13358}
13359
13360namespace {
13361
13362/// RAUWUpdateListener - Helper for ReplaceAllUsesWith - When the node
13363/// pointed to by a use iterator is deleted, increment the use iterator
13364/// so that it doesn't dangle.
13365///
13366class RAUWUpdateListener : public SelectionDAG::DAGUpdateListener {
13367 SDNode::use_iterator &UI;
13368 SDNode::use_iterator &UE;
13369
13370 void NodeDeleted(SDNode *N, SDNode *E) override {
13371 // Increment the iterator as needed.
13372 while (UI != UE && N == UI->getUser())
13373 ++UI;
13374 }
13375
13376public:
13377 RAUWUpdateListener(SelectionDAG &d,
13378 SDNode::use_iterator &ui,
13379 SDNode::use_iterator &ue)
13380 : SelectionDAG::DAGUpdateListener(d), UI(ui), UE(ue) {}
13381};
13382
13383} // end anonymous namespace
13384
13385/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
13386/// This can cause recursive merging of nodes in the DAG.
13387///
13388/// This version assumes From has a single result value.
13389///
13390void SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To) {
13391 SDNode *From = FromN.getNode();
13392 assert(From->getNumValues() == 1 && FromN.getResNo() == 0 &&
13393 "Cannot replace with this method!");
13394 assert(From != To.getNode() && "Cannot replace uses of with self");
13395
13396 // Preserve Debug Values
13397 transferDbgValues(From: FromN, To);
13398 // Preserve extra info.
13399 copyExtraInfo(From, To: To.getNode());
13400
13401 // Iterate over all the existing uses of From. New uses will be added
13402 // to the beginning of the use list, which we avoid visiting.
13403 // This specifically avoids visiting uses of From that arise while the
13404 // replacement is happening, because any such uses would be the result
13405 // of CSE: If an existing node looks like From after one of its operands
13406 // is replaced by To, we don't want to replace of all its users with To
13407 // too. See PR3018 for more info.
13408 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
13409 RAUWUpdateListener Listener(*this, UI, UE);
13410 while (UI != UE) {
13411 SDNode *User = UI->getUser();
13412
13413 // This node is about to morph, remove its old self from the CSE maps.
13414 RemoveNodeFromCSEMaps(N: User);
13415
13416 // A user can appear in a use list multiple times, and when this
13417 // happens the uses are usually next to each other in the list.
13418 // To help reduce the number of CSE recomputations, process all
13419 // the uses of this user that we can find this way.
13420 do {
13421 SDUse &Use = *UI;
13422 ++UI;
13423 Use.set(To);
13424 if (To->isDivergent() != From->isDivergent())
13425 updateDivergence(N: User);
13426 } while (UI != UE && UI->getUser() == User);
13427 // Now that we have modified User, add it back to the CSE maps. If it
13428 // already exists there, recursively merge the results together.
13429 AddModifiedNodeToCSEMaps(N: User);
13430 }
13431
13432 // If we just RAUW'd the root, take note.
13433 if (FromN == getRoot())
13434 setRoot(To);
13435}
13436
13437/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
13438/// This can cause recursive merging of nodes in the DAG.
13439///
13440/// This version assumes that for each value of From, there is a
13441/// corresponding value in To in the same position with the same type.
13442///
13443void SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To) {
13444#ifndef NDEBUG
13445 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
13446 assert((!From->hasAnyUseOfValue(i) ||
13447 From->getValueType(i) == To->getValueType(i)) &&
13448 "Cannot use this version of ReplaceAllUsesWith!");
13449#endif
13450
13451 // Handle the trivial case.
13452 if (From == To)
13453 return;
13454
13455 // Preserve Debug Info. Only do this if there's a use.
13456 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
13457 if (From->hasAnyUseOfValue(Value: i)) {
13458 assert((i < To->getNumValues()) && "Invalid To location");
13459 transferDbgValues(From: SDValue(From, i), To: SDValue(To, i));
13460 }
13461 // Preserve extra info.
13462 copyExtraInfo(From, To);
13463
13464 // Iterate over just the existing users of From. See the comments in
13465 // the ReplaceAllUsesWith above.
13466 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
13467 RAUWUpdateListener Listener(*this, UI, UE);
13468 while (UI != UE) {
13469 SDNode *User = UI->getUser();
13470
13471 // This node is about to morph, remove its old self from the CSE maps.
13472 RemoveNodeFromCSEMaps(N: User);
13473
13474 // A user can appear in a use list multiple times, and when this
13475 // happens the uses are usually next to each other in the list.
13476 // To help reduce the number of CSE recomputations, process all
13477 // the uses of this user that we can find this way.
13478 do {
13479 SDUse &Use = *UI;
13480 ++UI;
13481 Use.setNode(To);
13482 if (To->isDivergent() != From->isDivergent())
13483 updateDivergence(N: User);
13484 } while (UI != UE && UI->getUser() == User);
13485
13486 // Now that we have modified User, add it back to the CSE maps. If it
13487 // already exists there, recursively merge the results together.
13488 AddModifiedNodeToCSEMaps(N: User);
13489 }
13490
13491 // If we just RAUW'd the root, take note.
13492 if (From == getRoot().getNode())
13493 setRoot(SDValue(To, getRoot().getResNo()));
13494}
13495
13496/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
13497/// This can cause recursive merging of nodes in the DAG.
13498///
13499/// This version can replace From with any result values. To must match the
13500/// number and types of values returned by From.
13501void SelectionDAG::ReplaceAllUsesWith(SDNode *From, const SDValue *To) {
13502 if (From->getNumValues() == 1) // Handle the simple case efficiently.
13503 return ReplaceAllUsesWith(FromN: SDValue(From, 0), To: To[0]);
13504
13505 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i) {
13506 // Preserve Debug Info.
13507 transferDbgValues(From: SDValue(From, i), To: To[i]);
13508 // Preserve extra info.
13509 copyExtraInfo(From, To: To[i].getNode());
13510 }
13511
13512 // Iterate over just the existing users of From. See the comments in
13513 // the ReplaceAllUsesWith above.
13514 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
13515 RAUWUpdateListener Listener(*this, UI, UE);
13516 while (UI != UE) {
13517 SDNode *User = UI->getUser();
13518
13519 // This node is about to morph, remove its old self from the CSE maps.
13520 RemoveNodeFromCSEMaps(N: User);
13521
13522 // A user can appear in a use list multiple times, and when this happens the
13523 // uses are usually next to each other in the list. To help reduce the
13524 // number of CSE and divergence recomputations, process all the uses of this
13525 // user that we can find this way.
13526 bool To_IsDivergent = false;
13527 do {
13528 SDUse &Use = *UI;
13529 const SDValue &ToOp = To[Use.getResNo()];
13530 ++UI;
13531 Use.set(ToOp);
13532 if (ToOp.getValueType() != MVT::Other)
13533 To_IsDivergent |= ToOp->isDivergent();
13534 } while (UI != UE && UI->getUser() == User);
13535
13536 if (To_IsDivergent != From->isDivergent())
13537 updateDivergence(N: User);
13538
13539 // Now that we have modified User, add it back to the CSE maps. If it
13540 // already exists there, recursively merge the results together.
13541 AddModifiedNodeToCSEMaps(N: User);
13542 }
13543
13544 // If we just RAUW'd the root, take note.
13545 if (From == getRoot().getNode())
13546 setRoot(SDValue(To[getRoot().getResNo()]));
13547}
13548
13549/// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving
13550/// uses of other values produced by From.getNode() alone. The Deleted
13551/// vector is handled the same way as for ReplaceAllUsesWith.
13552void SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To){
13553 // Handle the really simple, really trivial case efficiently.
13554 if (From == To) return;
13555
13556 // Handle the simple, trivial, case efficiently.
13557 if (From.getNode()->getNumValues() == 1) {
13558 ReplaceAllUsesWith(FromN: From, To);
13559 return;
13560 }
13561
13562 // Preserve Debug Info.
13563 transferDbgValues(From, To);
13564 copyExtraInfo(From: From.getNode(), To: To.getNode());
13565
13566 // Iterate over just the existing users of From. See the comments in
13567 // the ReplaceAllUsesWith above.
13568 SDNode::use_iterator UI = From.getNode()->use_begin(),
13569 UE = From.getNode()->use_end();
13570 RAUWUpdateListener Listener(*this, UI, UE);
13571 while (UI != UE) {
13572 SDNode *User = UI->getUser();
13573 bool UserRemovedFromCSEMaps = false;
13574
13575 // A user can appear in a use list multiple times, and when this
13576 // happens the uses are usually next to each other in the list.
13577 // To help reduce the number of CSE recomputations, process all
13578 // the uses of this user that we can find this way.
13579 do {
13580 SDUse &Use = *UI;
13581
13582 // Skip uses of different values from the same node.
13583 if (Use.getResNo() != From.getResNo()) {
13584 ++UI;
13585 continue;
13586 }
13587
13588 // If this node hasn't been modified yet, it's still in the CSE maps,
13589 // so remove its old self from the CSE maps.
13590 if (!UserRemovedFromCSEMaps) {
13591 RemoveNodeFromCSEMaps(N: User);
13592 UserRemovedFromCSEMaps = true;
13593 }
13594
13595 ++UI;
13596 Use.set(To);
13597 if (To->isDivergent() != From->isDivergent())
13598 updateDivergence(N: User);
13599 } while (UI != UE && UI->getUser() == User);
13600 // We are iterating over all uses of the From node, so if a use
13601 // doesn't use the specific value, no changes are made.
13602 if (!UserRemovedFromCSEMaps)
13603 continue;
13604
13605 // Now that we have modified User, add it back to the CSE maps. If it
13606 // already exists there, recursively merge the results together.
13607 AddModifiedNodeToCSEMaps(N: User);
13608 }
13609
13610 // If we just RAUW'd the root, take note.
13611 if (From == getRoot())
13612 setRoot(To);
13613}
13614
13615namespace {
13616
13617/// UseMemo - This class is used by SelectionDAG::ReplaceAllUsesOfValuesWith
13618/// to record information about a use.
13619struct UseMemo {
13620 SDNode *User;
13621 unsigned Index;
13622 SDUse *Use;
13623};
13624
13625/// operator< - Sort Memos by User.
13626bool operator<(const UseMemo &L, const UseMemo &R) {
13627 return (intptr_t)L.User < (intptr_t)R.User;
13628}
13629
13630/// RAUOVWUpdateListener - Helper for ReplaceAllUsesOfValuesWith - When the node
13631/// pointed to by a UseMemo is deleted, set the User to nullptr to indicate that
13632/// the node already has been taken care of recursively.
13633class RAUOVWUpdateListener : public SelectionDAG::DAGUpdateListener {
13634 SmallVectorImpl<UseMemo> &Uses;
13635
13636 void NodeDeleted(SDNode *N, SDNode *E) override {
13637 for (UseMemo &Memo : Uses)
13638 if (Memo.User == N)
13639 Memo.User = nullptr;
13640 }
13641
13642public:
13643 RAUOVWUpdateListener(SelectionDAG &d, SmallVectorImpl<UseMemo> &uses)
13644 : SelectionDAG::DAGUpdateListener(d), Uses(uses) {}
13645};
13646
13647} // end anonymous namespace
13648
13649/// Return true if a glue output should propagate divergence information.
13650static bool gluePropagatesDivergence(const SDNode *Node) {
13651 switch (Node->getOpcode()) {
13652 case ISD::CopyFromReg:
13653 case ISD::CopyToReg:
13654 return false;
13655 default:
13656 return true;
13657 }
13658
13659 llvm_unreachable("covered opcode switch");
13660}
13661
13662bool SelectionDAG::calculateDivergence(SDNode *N) {
13663 if (TLI->isSDNodeAlwaysUniform(N)) {
13664 assert(!TLI->isSDNodeSourceOfDivergence(N, FLI, UA) &&
13665 "Conflicting divergence information!");
13666 return false;
13667 }
13668 if (TLI->isSDNodeSourceOfDivergence(N, FLI, UA))
13669 return true;
13670 for (const auto &Op : N->ops()) {
13671 EVT VT = Op.getValueType();
13672
13673 // Skip Chain. It does not carry divergence.
13674 if (VT != MVT::Other && Op.getNode()->isDivergent() &&
13675 (VT != MVT::Glue || gluePropagatesDivergence(Node: Op.getNode())))
13676 return true;
13677 }
13678 return false;
13679}
13680
13681void SelectionDAG::updateDivergence(SDNode *N) {
13682 SmallVector<SDNode *, 16> Worklist(1, N);
13683 do {
13684 N = Worklist.pop_back_val();
13685 bool IsDivergent = calculateDivergence(N);
13686 if (N->SDNodeBits.IsDivergent != IsDivergent) {
13687 N->SDNodeBits.IsDivergent = IsDivergent;
13688 llvm::append_range(C&: Worklist, R: N->users());
13689 }
13690 } while (!Worklist.empty());
13691}
13692
13693void SelectionDAG::CreateTopologicalOrder(std::vector<SDNode *> &Order) {
13694 DenseMap<SDNode *, unsigned> Degree;
13695 Order.reserve(n: AllNodes.size());
13696 for (auto &N : allnodes()) {
13697 unsigned NOps = N.getNumOperands();
13698 Degree[&N] = NOps;
13699 if (0 == NOps)
13700 Order.push_back(x: &N);
13701 }
13702 for (size_t I = 0; I != Order.size(); ++I) {
13703 SDNode *N = Order[I];
13704 for (auto *U : N->users()) {
13705 unsigned &UnsortedOps = Degree[U];
13706 if (0 == --UnsortedOps)
13707 Order.push_back(x: U);
13708 }
13709 }
13710}
13711
13712#if !defined(NDEBUG) && LLVM_ENABLE_ABI_BREAKING_CHECKS
13713void SelectionDAG::VerifyDAGDivergence() {
13714 std::vector<SDNode *> TopoOrder;
13715 CreateTopologicalOrder(TopoOrder);
13716 for (auto *N : TopoOrder) {
13717 assert(calculateDivergence(N) == N->isDivergent() &&
13718 "Divergence bit inconsistency detected");
13719 }
13720}
13721#endif
13722
13723/// ReplaceAllUsesOfValuesWith - Replace any uses of From with To, leaving
13724/// uses of other values produced by From.getNode() alone. The same value
13725/// may appear in both the From and To list. The Deleted vector is
13726/// handled the same way as for ReplaceAllUsesWith.
13727void SelectionDAG::ReplaceAllUsesOfValuesWith(const SDValue *From,
13728 const SDValue *To,
13729 unsigned Num){
13730 // Handle the simple, trivial case efficiently.
13731 if (Num == 1)
13732 return ReplaceAllUsesOfValueWith(From: *From, To: *To);
13733
13734 transferDbgValues(From: *From, To: *To);
13735 copyExtraInfo(From: From->getNode(), To: To->getNode());
13736
13737 // Read up all the uses and make records of them. This helps
13738 // processing new uses that are introduced during the
13739 // replacement process.
13740 SmallVector<UseMemo, 4> Uses;
13741 for (unsigned i = 0; i != Num; ++i) {
13742 unsigned FromResNo = From[i].getResNo();
13743 SDNode *FromNode = From[i].getNode();
13744 for (SDUse &Use : FromNode->uses()) {
13745 if (Use.getResNo() == FromResNo) {
13746 UseMemo Memo = {.User: Use.getUser(), .Index: i, .Use: &Use};
13747 Uses.push_back(Elt: Memo);
13748 }
13749 }
13750 }
13751
13752 // Sort the uses, so that all the uses from a given User are together.
13753 llvm::sort(C&: Uses);
13754 RAUOVWUpdateListener Listener(*this, Uses);
13755
13756 for (unsigned UseIndex = 0, UseIndexEnd = Uses.size();
13757 UseIndex != UseIndexEnd; ) {
13758 // We know that this user uses some value of From. If it is the right
13759 // value, update it.
13760 SDNode *User = Uses[UseIndex].User;
13761 // If the node has been deleted by recursive CSE updates when updating
13762 // another node, then just skip this entry.
13763 if (User == nullptr) {
13764 ++UseIndex;
13765 continue;
13766 }
13767
13768 // This node is about to morph, remove its old self from the CSE maps.
13769 RemoveNodeFromCSEMaps(N: User);
13770
13771 // The Uses array is sorted, so all the uses for a given User
13772 // are next to each other in the list.
13773 // To help reduce the number of CSE recomputations, process all
13774 // the uses of this user that we can find this way.
13775 do {
13776 unsigned i = Uses[UseIndex].Index;
13777 SDUse &Use = *Uses[UseIndex].Use;
13778 ++UseIndex;
13779
13780 Use.set(To[i]);
13781 } while (UseIndex != UseIndexEnd && Uses[UseIndex].User == User);
13782
13783 // Now that we have modified User, add it back to the CSE maps. If it
13784 // already exists there, recursively merge the results together.
13785 AddModifiedNodeToCSEMaps(N: User);
13786 }
13787}
13788
13789/// AssignTopologicalOrder - Assign a unique node id for each node in the DAG
13790/// based on their topological order. It returns the maximum id and a vector
13791/// of the SDNodes* in assigned order by reference.
13792unsigned SelectionDAG::AssignTopologicalOrder() {
13793 unsigned DAGSize = 0;
13794
13795 // SortedPos tracks the progress of the algorithm. Nodes before it are
13796 // sorted, nodes after it are unsorted. When the algorithm completes
13797 // it is at the end of the list.
13798 allnodes_iterator SortedPos = allnodes_begin();
13799
13800 // Visit all the nodes. Move nodes with no operands to the front of
13801 // the list immediately. Annotate nodes that do have operands with their
13802 // operand count. Before we do this, the Node Id fields of the nodes
13803 // may contain arbitrary values. After, the Node Id fields for nodes
13804 // before SortedPos will contain the topological sort index, and the
13805 // Node Id fields for nodes At SortedPos and after will contain the
13806 // count of outstanding operands.
13807 for (SDNode &N : llvm::make_early_inc_range(Range: allnodes())) {
13808 checkForCycles(N: &N, DAG: this);
13809 unsigned Degree = N.getNumOperands();
13810 if (Degree == 0) {
13811 // A node with no uses, add it to the result array immediately.
13812 N.setNodeId(DAGSize++);
13813 allnodes_iterator Q(&N);
13814 if (Q != SortedPos)
13815 SortedPos = AllNodes.insert(where: SortedPos, New: AllNodes.remove(IT&: Q));
13816 assert(SortedPos != AllNodes.end() && "Overran node list");
13817 ++SortedPos;
13818 } else {
13819 // Temporarily use the Node Id as scratch space for the degree count.
13820 N.setNodeId(Degree);
13821 }
13822 }
13823
13824 // Visit all the nodes. As we iterate, move nodes into sorted order,
13825 // such that by the time the end is reached all nodes will be sorted.
13826 for (SDNode &Node : allnodes()) {
13827 SDNode *N = &Node;
13828 checkForCycles(N, DAG: this);
13829 // N is in sorted position, so all its uses have one less operand
13830 // that needs to be sorted.
13831 for (SDNode *P : N->users()) {
13832 unsigned Degree = P->getNodeId();
13833 assert(Degree != 0 && "Invalid node degree");
13834 --Degree;
13835 if (Degree == 0) {
13836 // All of P's operands are sorted, so P may sorted now.
13837 P->setNodeId(DAGSize++);
13838 if (P->getIterator() != SortedPos)
13839 SortedPos = AllNodes.insert(where: SortedPos, New: AllNodes.remove(IT: P));
13840 assert(SortedPos != AllNodes.end() && "Overran node list");
13841 ++SortedPos;
13842 } else {
13843 // Update P's outstanding operand count.
13844 P->setNodeId(Degree);
13845 }
13846 }
13847 if (Node.getIterator() == SortedPos) {
13848#ifndef NDEBUG
13849 allnodes_iterator I(N);
13850 SDNode *S = &*++I;
13851 dbgs() << "Overran sorted position:\n";
13852 S->dumprFull(this); dbgs() << "\n";
13853 dbgs() << "Checking if this is due to cycles\n";
13854 checkForCycles(this, true);
13855#endif
13856 llvm_unreachable(nullptr);
13857 }
13858 }
13859
13860 assert(SortedPos == AllNodes.end() &&
13861 "Topological sort incomplete!");
13862 assert(AllNodes.front().getOpcode() == ISD::EntryToken &&
13863 "First node in topological sort is not the entry token!");
13864 assert(AllNodes.front().getNodeId() == 0 &&
13865 "First node in topological sort has non-zero id!");
13866 assert(AllNodes.front().getNumOperands() == 0 &&
13867 "First node in topological sort has operands!");
13868 assert(AllNodes.back().getNodeId() == (int)DAGSize-1 &&
13869 "Last node in topologic sort has unexpected id!");
13870 assert(AllNodes.back().use_empty() &&
13871 "Last node in topologic sort has users!");
13872 assert(DAGSize == allnodes_size() && "Node count mismatch!");
13873 return DAGSize;
13874}
13875
13876void SelectionDAG::getTopologicallyOrderedNodes(
13877 SmallVectorImpl<const SDNode *> &SortedNodes) const {
13878 SortedNodes.clear();
13879 // Node -> remaining number of outstanding operands.
13880 DenseMap<const SDNode *, unsigned> RemainingOperands;
13881
13882 // Put nodes without any operands into SortedNodes first.
13883 for (const SDNode &N : allnodes()) {
13884 checkForCycles(N: &N, DAG: this);
13885 unsigned NumOperands = N.getNumOperands();
13886 if (NumOperands == 0)
13887 SortedNodes.push_back(Elt: &N);
13888 else
13889 // Record their total number of outstanding operands.
13890 RemainingOperands[&N] = NumOperands;
13891 }
13892
13893 // A node is pushed into SortedNodes when all of its operands (predecessors in
13894 // the graph) are also in SortedNodes.
13895 for (unsigned i = 0U; i < SortedNodes.size(); ++i) {
13896 const SDNode *N = SortedNodes[i];
13897 for (const SDNode *U : N->users()) {
13898 // HandleSDNode is never part of a DAG and therefore has no entry in
13899 // RemainingOperands.
13900 if (U->getOpcode() == ISD::HANDLENODE)
13901 continue;
13902 unsigned &NumRemOperands = RemainingOperands[U];
13903 assert(NumRemOperands && "Invalid number of remaining operands");
13904 --NumRemOperands;
13905 if (!NumRemOperands)
13906 SortedNodes.push_back(Elt: U);
13907 }
13908 }
13909
13910 assert(SortedNodes.size() == AllNodes.size() && "Node count mismatch");
13911 assert(SortedNodes.front()->getOpcode() == ISD::EntryToken &&
13912 "First node in topological sort is not the entry token");
13913 assert(SortedNodes.front()->getNumOperands() == 0 &&
13914 "First node in topological sort has operands");
13915}
13916
13917/// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the
13918/// value is produced by SD.
13919void SelectionDAG::AddDbgValue(SDDbgValue *DB, bool isParameter) {
13920 for (SDNode *SD : DB->getSDNodes()) {
13921 if (!SD)
13922 continue;
13923 assert(DbgInfo->getSDDbgValues(SD).empty() || SD->getHasDebugValue());
13924 SD->setHasDebugValue(true);
13925 }
13926 DbgInfo->add(V: DB, isParameter);
13927}
13928
13929void SelectionDAG::AddDbgLabel(SDDbgLabel *DB) { DbgInfo->add(L: DB); }
13930
13931SDValue SelectionDAG::makeEquivalentMemoryOrdering(SDValue OldChain,
13932 SDValue NewMemOpChain) {
13933 assert(isa<MemSDNode>(NewMemOpChain) && "Expected a memop node");
13934 assert(NewMemOpChain.getValueType() == MVT::Other && "Expected a token VT");
13935 // The new memory operation must have the same position as the old load in
13936 // terms of memory dependency. Create a TokenFactor for the old load and new
13937 // memory operation and update uses of the old load's output chain to use that
13938 // TokenFactor.
13939 if (OldChain == NewMemOpChain || OldChain.use_empty())
13940 return NewMemOpChain;
13941
13942 SDValue TokenFactor = getNode(Opcode: ISD::TokenFactor, DL: SDLoc(OldChain), VT: MVT::Other,
13943 N1: OldChain, N2: NewMemOpChain);
13944 ReplaceAllUsesOfValueWith(From: OldChain, To: TokenFactor);
13945 UpdateNodeOperands(N: TokenFactor.getNode(), Op1: OldChain, Op2: NewMemOpChain);
13946 return TokenFactor;
13947}
13948
13949SDValue SelectionDAG::makeEquivalentMemoryOrdering(LoadSDNode *OldLoad,
13950 SDValue NewMemOp) {
13951 assert(isa<MemSDNode>(NewMemOp.getNode()) && "Expected a memop node");
13952 SDValue OldChain = SDValue(OldLoad, 1);
13953 SDValue NewMemOpChain = NewMemOp.getValue(R: 1);
13954 return makeEquivalentMemoryOrdering(OldChain, NewMemOpChain);
13955}
13956
13957SDValue SelectionDAG::getSymbolFunctionGlobalAddress(SDValue Op,
13958 Function **OutFunction) {
13959 assert(isa<ExternalSymbolSDNode>(Op) && "Node should be an ExternalSymbol");
13960
13961 auto *Symbol = cast<ExternalSymbolSDNode>(Val&: Op)->getSymbol();
13962 auto *Module = MF->getFunction().getParent();
13963 auto *Function = Module->getFunction(Name: Symbol);
13964
13965 if (OutFunction != nullptr)
13966 *OutFunction = Function;
13967
13968 if (Function != nullptr) {
13969 auto PtrTy = TLI->getPointerTy(DL: getDataLayout(), AS: Function->getAddressSpace());
13970 return getGlobalAddress(GV: Function, DL: SDLoc(Op), VT: PtrTy);
13971 }
13972
13973 std::string ErrorStr;
13974 raw_string_ostream ErrorFormatter(ErrorStr);
13975 ErrorFormatter << "Undefined external symbol ";
13976 ErrorFormatter << '"' << Symbol << '"';
13977 report_fatal_error(reason: Twine(ErrorStr));
13978}
13979
13980//===----------------------------------------------------------------------===//
13981// SDNode Class
13982//===----------------------------------------------------------------------===//
13983
13984bool llvm::isNullConstant(SDValue V) {
13985 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val&: V);
13986 return Const != nullptr && Const->isZero();
13987}
13988
13989bool llvm::isNullConstantOrUndef(SDValue V) {
13990 return V.isUndef() || isNullConstant(V);
13991}
13992
13993bool llvm::isNullFPConstant(SDValue V) {
13994 ConstantFPSDNode *Const = dyn_cast<ConstantFPSDNode>(Val&: V);
13995 return Const != nullptr && Const->isZero() && !Const->isNegative();
13996}
13997
13998bool llvm::isAllOnesConstant(SDValue V) {
13999 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val&: V);
14000 return Const != nullptr && Const->isAllOnes();
14001}
14002
14003bool llvm::isOneConstant(SDValue V) {
14004 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val&: V);
14005 return Const != nullptr && Const->isOne();
14006}
14007
14008bool llvm::isMinSignedConstant(SDValue V) {
14009 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val&: V);
14010 return Const != nullptr && Const->isMinSignedValue();
14011}
14012
14013bool SelectionDAG::isIdentityElement(unsigned Opcode, SDNodeFlags Flags,
14014 SDValue V, unsigned OperandNo,
14015 unsigned Depth) const {
14016 APInt DemandedElts = getDemandAllEltsMask(V);
14017 return isIdentityElement(Opc: Opcode, Flags, V, DemandedElts, OperandNo, Depth);
14018}
14019
14020bool SelectionDAG::isIdentityElement(unsigned Opcode, SDNodeFlags Flags,
14021 SDValue V, const APInt &DemandedElts,
14022 unsigned OperandNo, unsigned Depth) const {
14023 // NOTE: The cases should match with IR's ConstantExpr::getBinOpIdentity().
14024 // TODO: Target-specific opcodes could be added.
14025 if (V.getValueType().isInteger()) {
14026 KnownBits Known = computeKnownBits(Op: V, DemandedElts, Depth);
14027 if (Known.isConstant()) {
14028 const APInt &Const = Known.getConstant();
14029 switch (Opcode) {
14030 case ISD::ADD:
14031 case ISD::OR:
14032 case ISD::XOR:
14033 case ISD::UMAX:
14034 return Const.isZero();
14035 case ISD::MUL:
14036 return Const.isOne();
14037 case ISD::AND:
14038 case ISD::UMIN:
14039 return Const.isAllOnes();
14040 case ISD::SMAX:
14041 return Const.isMinSignedValue();
14042 case ISD::SMIN:
14043 return Const.isMaxSignedValue();
14044 case ISD::SUB:
14045 case ISD::SHL:
14046 case ISD::SRA:
14047 case ISD::SRL:
14048 return OperandNo == 1 && Const.isZero();
14049 case ISD::UDIV:
14050 case ISD::SDIV:
14051 return OperandNo == 1 && Const.isOne();
14052 }
14053 }
14054 } else if (auto *ConstFP = isConstOrConstSplatFP(N: V, DemandedElts)) {
14055 switch (Opcode) {
14056 case ISD::FADD:
14057 return ConstFP->isZero() &&
14058 (Flags.hasNoSignedZeros() || ConstFP->isNegative());
14059 case ISD::FSUB:
14060 return OperandNo == 1 && ConstFP->isZero() &&
14061 (Flags.hasNoSignedZeros() || !ConstFP->isNegative());
14062 case ISD::FMUL:
14063 return ConstFP->isOne();
14064 case ISD::FDIV:
14065 return OperandNo == 1 && ConstFP->isOne();
14066 case ISD::FMINNUM:
14067 case ISD::FMAXNUM:
14068 case ISD::FMINIMUMNUM:
14069 case ISD::FMAXIMUMNUM: {
14070 // Neutral element for fminnum/fminimumnum is NaN, Inf or FLT_MAX,
14071 // depending on fast-math flags (FMF).
14072 EVT VT = V.getValueType();
14073 const fltSemantics &Semantics = VT.getFltSemantics();
14074 APFloat NeutralAF = !Flags.hasNoNaNs() ? APFloat::getQNaN(Sem: Semantics)
14075 : !Flags.hasNoInfs() ? APFloat::getInf(Sem: Semantics)
14076 : APFloat::getLargest(Sem: Semantics);
14077 if (Opcode == ISD::FMAXNUM || Opcode == ISD::FMAXIMUMNUM)
14078 NeutralAF.changeSign();
14079
14080 return ConstFP->isExactlyValue(V: NeutralAF);
14081 }
14082 case ISD::FMINIMUM:
14083 case ISD::FMAXIMUM: {
14084 // Neutral element for fminimum is Inf or FLT_MAX, depending on FMF.
14085 const APFloat &VAPF = ConstFP->getValueAPF();
14086 bool NeutralNegative = (Opcode == ISD::FMAXIMUM);
14087 if (Flags.hasNoInfs())
14088 return VAPF.isLargest() && VAPF.isNegative() == NeutralNegative;
14089 return VAPF.isInfinity() && VAPF.isNegative() == NeutralNegative;
14090 }
14091 }
14092 }
14093 return false;
14094}
14095
14096SDValue llvm::peekThroughBitcasts(SDValue V) {
14097 while (V.getOpcode() == ISD::BITCAST)
14098 V = V.getOperand(i: 0);
14099 return V;
14100}
14101
14102SDValue llvm::peekThroughOneUseBitcasts(SDValue V) {
14103 while (V.getOpcode() == ISD::BITCAST && V.getOperand(i: 0).hasOneUse())
14104 V = V.getOperand(i: 0);
14105 return V;
14106}
14107
14108SDValue llvm::peekThroughExtractSubvectors(SDValue V) {
14109 while (V.getOpcode() == ISD::EXTRACT_SUBVECTOR)
14110 V = V.getOperand(i: 0);
14111 return V;
14112}
14113
14114SDValue llvm::peekThroughInsertVectorElt(SDValue V, const APInt &DemandedElts) {
14115 while (V.getOpcode() == ISD::INSERT_VECTOR_ELT) {
14116 SDValue InVec = V.getOperand(i: 0);
14117 SDValue EltNo = V.getOperand(i: 2);
14118 EVT VT = InVec.getValueType();
14119 auto *IndexC = dyn_cast<ConstantSDNode>(Val&: EltNo);
14120 if (IndexC && VT.isFixedLengthVector() &&
14121 IndexC->getAPIntValue().ult(RHS: VT.getVectorNumElements()) &&
14122 !DemandedElts[IndexC->getZExtValue()]) {
14123 V = InVec;
14124 continue;
14125 }
14126 break;
14127 }
14128 return V;
14129}
14130
14131SDValue llvm::peekThroughTruncates(SDValue V) {
14132 while (V.getOpcode() == ISD::TRUNCATE)
14133 V = V.getOperand(i: 0);
14134 return V;
14135}
14136
14137bool llvm::isBitwiseNot(SDValue V, bool AllowUndefs) {
14138 if (V.getOpcode() != ISD::XOR)
14139 return false;
14140 V = peekThroughBitcasts(V: V.getOperand(i: 1));
14141 unsigned NumBits = V.getScalarValueSizeInBits();
14142 ConstantSDNode *C =
14143 isConstOrConstSplat(N: V, AllowUndefs, /*AllowTruncation*/ true);
14144 return C && (C->getAPIntValue().countr_one() >= NumBits);
14145}
14146
14147ConstantSDNode *llvm::isConstOrConstSplat(SDValue N, bool AllowUndefs,
14148 bool AllowTruncation) {
14149 APInt DemandedElts = getDemandAllEltsMask(V: N);
14150 return isConstOrConstSplat(N, DemandedElts, AllowUndefs, AllowTruncation);
14151}
14152
14153ConstantSDNode *llvm::isConstOrConstSplat(SDValue N, const APInt &DemandedElts,
14154 bool AllowUndefs,
14155 bool AllowTruncation) {
14156 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val&: N))
14157 return CN;
14158
14159 // SplatVectors can truncate their operands. Ignore that case here unless
14160 // AllowTruncation is set.
14161 if (N->getOpcode() == ISD::SPLAT_VECTOR) {
14162 EVT VecEltVT = N->getValueType(ResNo: 0).getVectorElementType();
14163 if (auto *CN = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 0))) {
14164 EVT CVT = CN->getValueType(ResNo: 0);
14165 assert(CVT.bitsGE(VecEltVT) && "Illegal splat_vector element extension");
14166 if (AllowTruncation || CVT == VecEltVT)
14167 return CN;
14168 }
14169 }
14170
14171 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Val&: N)) {
14172 BitVector UndefElements;
14173 ConstantSDNode *CN = BV->getConstantSplatNode(DemandedElts, UndefElements: &UndefElements);
14174
14175 // BuildVectors can truncate their operands. Ignore that case here unless
14176 // AllowTruncation is set.
14177 // TODO: Look into whether we should allow UndefElements in non-DemandedElts
14178 if (CN && (UndefElements.none() || AllowUndefs)) {
14179 EVT CVT = CN->getValueType(ResNo: 0);
14180 EVT NSVT = N.getValueType().getScalarType();
14181 assert(CVT.bitsGE(NSVT) && "Illegal build vector element extension");
14182 if (AllowTruncation || (CVT == NSVT))
14183 return CN;
14184 }
14185 }
14186
14187 if (N.getOpcode() == ISD::MERGE_VALUES)
14188 return isConstOrConstSplat(N: N.getOperand(i: N.getResNo()), DemandedElts,
14189 AllowUndefs, AllowTruncation);
14190
14191 return nullptr;
14192}
14193
14194ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N, bool AllowUndefs) {
14195 APInt DemandedElts = getDemandAllEltsMask(V: N);
14196 return isConstOrConstSplatFP(N, DemandedElts, AllowUndefs);
14197}
14198
14199ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N,
14200 const APInt &DemandedElts,
14201 bool AllowUndefs) {
14202 if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(Val&: N))
14203 return CN;
14204
14205 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Val&: N)) {
14206 BitVector UndefElements;
14207 ConstantFPSDNode *CN =
14208 BV->getConstantFPSplatNode(DemandedElts, UndefElements: &UndefElements);
14209 // TODO: Look into whether we should allow UndefElements in non-DemandedElts
14210 if (CN && (UndefElements.none() || AllowUndefs))
14211 return CN;
14212 }
14213
14214 if (N.getOpcode() == ISD::SPLAT_VECTOR)
14215 if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(Val: N.getOperand(i: 0)))
14216 return CN;
14217
14218 return nullptr;
14219}
14220
14221bool llvm::isNullOrNullSplat(SDValue N, bool AllowUndefs) {
14222 // TODO: may want to use peekThroughBitcast() here.
14223 ConstantSDNode *C =
14224 isConstOrConstSplat(N, AllowUndefs, /*AllowTruncation=*/true);
14225 return C && C->isZero();
14226}
14227
14228bool llvm::isOneOrOneSplat(SDValue N, bool AllowUndefs) {
14229 ConstantSDNode *C =
14230 isConstOrConstSplat(N, AllowUndefs, /*AllowTruncation*/ true);
14231 return C && C->isOne();
14232}
14233
14234bool llvm::isOneOrOneSplatFP(SDValue N, bool AllowUndefs) {
14235 ConstantFPSDNode *C = isConstOrConstSplatFP(N, AllowUndefs);
14236 return C && C->isOne();
14237}
14238
14239bool llvm::isAllOnesOrAllOnesSplat(SDValue N, bool AllowUndefs) {
14240 N = peekThroughBitcasts(V: N);
14241 unsigned BitWidth = N.getScalarValueSizeInBits();
14242 ConstantSDNode *C =
14243 isConstOrConstSplat(N, AllowUndefs, /*AllowTruncation=*/true);
14244 return C && C->getAPIntValue().countTrailingOnes() >= BitWidth;
14245}
14246
14247bool llvm::isZeroOrZeroSplat(SDValue N, bool AllowUndefs) {
14248 N = peekThroughBitcasts(V: N);
14249 ConstantSDNode *C = isConstOrConstSplat(N, AllowUndefs, AllowTruncation: true);
14250 return C && C->isZero();
14251}
14252
14253bool llvm::isZeroOrZeroSplatFP(SDValue N, bool AllowUndefs) {
14254 ConstantFPSDNode *C = isConstOrConstSplatFP(N, AllowUndefs);
14255 return C && C->isZero();
14256}
14257
14258HandleSDNode::~HandleSDNode() {
14259 DropOperands();
14260}
14261
14262MemSDNode::MemSDNode(
14263 unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTs, EVT memvt,
14264 PointerUnion<MachineMemOperand *, MachineMemOperand **> memrefs)
14265 : SDNode(Opc, Order, dl, VTs), MemoryVT(memvt), MemRefs(memrefs) {
14266 bool IsVolatile = false;
14267 bool IsNonTemporal = false;
14268 bool IsDereferenceable = true;
14269 bool IsInvariant = true;
14270 for (const MachineMemOperand *MMO : memoperands()) {
14271 IsVolatile |= MMO->isVolatile();
14272 IsNonTemporal |= MMO->isNonTemporal();
14273 IsDereferenceable &= MMO->isDereferenceable();
14274 IsInvariant &= MMO->isInvariant();
14275 }
14276 MemSDNodeBits.IsVolatile = IsVolatile;
14277 MemSDNodeBits.IsNonTemporal = IsNonTemporal;
14278 MemSDNodeBits.IsDereferenceable = IsDereferenceable;
14279 MemSDNodeBits.IsInvariant = IsInvariant;
14280
14281 // For the single-MMO case, we check here that the size of the memory operand
14282 // fits within the size of the MMO. This is because the MMO might indicate
14283 // only a possible address range instead of specifying the affected memory
14284 // addresses precisely.
14285 assert((getNumMemOperands() != 1 || !getMemOperand()->getType().isValid() ||
14286 TypeSize::isKnownLE(memvt.getStoreSize(),
14287 getMemOperand()->getSize().getValue())) &&
14288 "Size mismatch!");
14289}
14290
14291namespace {
14292
14293 struct EVTArray {
14294 std::vector<EVT> VTs;
14295
14296 EVTArray() {
14297 VTs.reserve(n: MVT::VALUETYPE_SIZE);
14298 for (unsigned i = 0; i < MVT::VALUETYPE_SIZE; ++i)
14299 VTs.push_back(x: MVT((MVT::SimpleValueType)i));
14300 }
14301 };
14302
14303} // end anonymous namespace
14304
14305/// getValueTypeList - Return a pointer to the specified value type.
14306///
14307const EVT *SDNode::getValueTypeList(MVT VT) {
14308 static EVTArray SimpleVTArray;
14309
14310 assert(VT < MVT::VALUETYPE_SIZE && "Value type out of range!");
14311 return &SimpleVTArray.VTs[VT.SimpleTy];
14312}
14313
14314/// hasAnyUseOfValue - Return true if there are any use of the indicated
14315/// value. This method ignores uses of other values defined by this operation.
14316bool SDNode::hasAnyUseOfValue(unsigned Value) const {
14317 assert(Value < getNumValues() && "Bad value!");
14318
14319 for (SDUse &U : uses())
14320 if (U.getResNo() == Value)
14321 return true;
14322
14323 return false;
14324}
14325
14326/// isOnlyUserOf - Return true if this node is the only use of N.
14327bool SDNode::isOnlyUserOf(const SDNode *N) const {
14328 bool Seen = false;
14329 for (const SDNode *User : N->users()) {
14330 if (User == this)
14331 Seen = true;
14332 else
14333 return false;
14334 }
14335
14336 return Seen;
14337}
14338
14339/// Return true if the only users of N are contained in Nodes.
14340bool SDNode::areOnlyUsersOf(ArrayRef<const SDNode *> Nodes, const SDNode *N) {
14341 bool Seen = false;
14342 for (const SDNode *User : N->users()) {
14343 if (llvm::is_contained(Range&: Nodes, Element: User))
14344 Seen = true;
14345 else
14346 return false;
14347 }
14348
14349 return Seen;
14350}
14351
14352/// Return true if the referenced return value is an operand of N.
14353bool SDValue::isOperandOf(const SDNode *N) const {
14354 return is_contained(Range: N->op_values(), Element: *this);
14355}
14356
14357bool SDNode::isOperandOf(const SDNode *N) const {
14358 return any_of(Range: N->op_values(),
14359 P: [this](SDValue Op) { return this == Op.getNode(); });
14360}
14361
14362/// reachesChainWithoutSideEffects - Return true if this operand (which must
14363/// be a chain) reaches the specified operand without crossing any
14364/// side-effecting instructions on any chain path. In practice, this looks
14365/// through token factors and non-volatile loads. In order to remain efficient,
14366/// this only looks a couple of nodes in, it does not do an exhaustive search.
14367///
14368/// Note that we only need to examine chains when we're searching for
14369/// side-effects; SelectionDAG requires that all side-effects are represented
14370/// by chains, even if another operand would force a specific ordering. This
14371/// constraint is necessary to allow transformations like splitting loads.
14372bool SDValue::reachesChainWithoutSideEffects(SDValue Dest,
14373 unsigned Depth) const {
14374 if (*this == Dest) return true;
14375
14376 // Don't search too deeply, we just want to be able to see through
14377 // TokenFactor's etc.
14378 if (Depth == 0) return false;
14379
14380 // If this is a token factor, all inputs to the TF happen in parallel.
14381 if (getOpcode() == ISD::TokenFactor) {
14382 // First, try a shallow search.
14383 if (is_contained(Range: (*this)->ops(), Element: Dest)) {
14384 // We found the chain we want as an operand of this TokenFactor.
14385 // Essentially, we reach the chain without side-effects if we could
14386 // serialize the TokenFactor into a simple chain of operations with
14387 // Dest as the last operation. This is automatically true if the
14388 // chain has one use: there are no other ordering constraints.
14389 // If the chain has more than one use, we give up: some other
14390 // use of Dest might force a side-effect between Dest and the current
14391 // node.
14392 if (Dest.hasOneUse())
14393 return true;
14394 }
14395 // Next, try a deep search: check whether every operand of the TokenFactor
14396 // reaches Dest.
14397 return llvm::all_of(Range: (*this)->ops(), P: [=](SDValue Op) {
14398 return Op.reachesChainWithoutSideEffects(Dest, Depth: Depth - 1);
14399 });
14400 }
14401
14402 // Loads don't have side effects, look through them.
14403 if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(Val: *this)) {
14404 if (Ld->isUnordered())
14405 return Ld->getChain().reachesChainWithoutSideEffects(Dest, Depth: Depth-1);
14406 }
14407 return false;
14408}
14409
14410bool SDNode::hasPredecessor(const SDNode *N) const {
14411 SmallPtrSet<const SDNode *, 32> Visited;
14412 SmallVector<const SDNode *, 16> Worklist;
14413 Worklist.push_back(Elt: this);
14414 return hasPredecessorHelper(N, Visited, Worklist);
14415}
14416
14417void SDNode::intersectFlagsWith(const SDNodeFlags Flags) {
14418 this->Flags &= Flags;
14419}
14420
14421SDValue
14422SelectionDAG::matchBinOpReduction(SDNode *Extract, ISD::NodeType &BinOp,
14423 ArrayRef<ISD::NodeType> CandidateBinOps,
14424 bool AllowPartials) {
14425 // The pattern must end in an extract from index 0.
14426 if (Extract->getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
14427 !isNullConstant(V: Extract->getOperand(Num: 1)))
14428 return SDValue();
14429
14430 // Match against one of the candidate binary ops.
14431 SDValue Op = Extract->getOperand(Num: 0);
14432 if (llvm::none_of(Range&: CandidateBinOps, P: [Op](ISD::NodeType BinOp) {
14433 return Op.getOpcode() == unsigned(BinOp);
14434 }))
14435 return SDValue();
14436
14437 // Floating-point reductions may require relaxed constraints on the final step
14438 // of the reduction because they may reorder intermediate operations.
14439 unsigned CandidateBinOp = Op.getOpcode();
14440 if (Op.getValueType().isFloatingPoint()) {
14441 SDNodeFlags Flags = Op->getFlags();
14442 switch (CandidateBinOp) {
14443 case ISD::FADD:
14444 if (!Flags.hasNoSignedZeros() || !Flags.hasAllowReassociation())
14445 return SDValue();
14446 break;
14447 default:
14448 llvm_unreachable("Unhandled FP opcode for binop reduction");
14449 }
14450 }
14451
14452 // Matching failed - attempt to see if we did enough stages that a partial
14453 // reduction from a subvector is possible.
14454 auto PartialReduction = [&](SDValue Op, unsigned NumSubElts) {
14455 if (!AllowPartials || !Op)
14456 return SDValue();
14457 EVT OpVT = Op.getValueType();
14458 EVT OpSVT = OpVT.getScalarType();
14459 EVT SubVT = EVT::getVectorVT(Context&: *getContext(), VT: OpSVT, NumElements: NumSubElts);
14460 if (TLI->getExtractSubvectorCost(ResVT: SubVT, SrcVT: OpVT, Index: 0) >
14461 TargetLowering::ExtractSubvectorCost::Cheap)
14462 return SDValue();
14463 BinOp = (ISD::NodeType)CandidateBinOp;
14464 return getExtractSubvector(DL: SDLoc(Op), VT: SubVT, Vec: Op, Idx: 0);
14465 };
14466
14467 // At each stage, we're looking for something that looks like:
14468 // %s = shufflevector <8 x i32> %op, <8 x i32> undef,
14469 // <8 x i32> <i32 2, i32 3, i32 undef, i32 undef,
14470 // i32 undef, i32 undef, i32 undef, i32 undef>
14471 // %a = binop <8 x i32> %op, %s
14472 // Where the mask changes according to the stage. E.g. for a 3-stage pyramid,
14473 // we expect something like:
14474 // <4,5,6,7,u,u,u,u>
14475 // <2,3,u,u,u,u,u,u>
14476 // <1,u,u,u,u,u,u,u>
14477 // While a partial reduction match would be:
14478 // <2,3,u,u,u,u,u,u>
14479 // <1,u,u,u,u,u,u,u>
14480 unsigned Stages = Log2_32(Value: Op.getValueType().getVectorNumElements());
14481 SDValue PrevOp;
14482 for (unsigned i = 0; i < Stages; ++i) {
14483 unsigned MaskEnd = (1 << i);
14484
14485 if (Op.getOpcode() != CandidateBinOp)
14486 return PartialReduction(PrevOp, MaskEnd);
14487
14488 SDValue Op0 = Op.getOperand(i: 0);
14489 SDValue Op1 = Op.getOperand(i: 1);
14490
14491 ShuffleVectorSDNode *Shuffle = dyn_cast<ShuffleVectorSDNode>(Val&: Op0);
14492 if (Shuffle) {
14493 Op = Op1;
14494 } else {
14495 Shuffle = dyn_cast<ShuffleVectorSDNode>(Val&: Op1);
14496 Op = Op0;
14497 }
14498
14499 // The first operand of the shuffle should be the same as the other operand
14500 // of the binop.
14501 if (!Shuffle || Shuffle->getOperand(Num: 0) != Op)
14502 return PartialReduction(PrevOp, MaskEnd);
14503
14504 // Verify the shuffle has the expected (at this stage of the pyramid) mask.
14505 for (int Index = 0; Index < (int)MaskEnd; ++Index)
14506 if (Shuffle->getMaskElt(Idx: Index) != (int)(MaskEnd + Index))
14507 return PartialReduction(PrevOp, MaskEnd);
14508
14509 PrevOp = Op;
14510 }
14511
14512 // Handle subvector reductions, which tend to appear after the shuffle
14513 // reduction stages.
14514 while (Op.getOpcode() == CandidateBinOp) {
14515 unsigned NumElts = Op.getValueType().getVectorNumElements();
14516 SDValue Op0 = Op.getOperand(i: 0);
14517 SDValue Op1 = Op.getOperand(i: 1);
14518 if (Op0.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
14519 Op1.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
14520 Op0.getOperand(i: 0) != Op1.getOperand(i: 0))
14521 break;
14522 SDValue Src = Op0.getOperand(i: 0);
14523 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
14524 if (NumSrcElts != (2 * NumElts))
14525 break;
14526 if (!(Op0.getConstantOperandAPInt(i: 1) == 0 &&
14527 Op1.getConstantOperandAPInt(i: 1) == NumElts) &&
14528 !(Op1.getConstantOperandAPInt(i: 1) == 0 &&
14529 Op0.getConstantOperandAPInt(i: 1) == NumElts))
14530 break;
14531 Op = Src;
14532 }
14533
14534 BinOp = (ISD::NodeType)CandidateBinOp;
14535 return Op;
14536}
14537
14538SDValue SelectionDAG::UnrollVectorOp(SDNode *N, unsigned ResNE) {
14539 EVT VT = N->getValueType(ResNo: 0);
14540 EVT EltVT = VT.getVectorElementType();
14541 unsigned NE = getMaxRuntimeNumElements(VT);
14542
14543 if (VT.isScalableVector() && (NE == 0 || ResNE != 0))
14544 reportFatalUsageError(reason: "Cannot unroll scalable vector!");
14545
14546 assert(NE && "Nothing to unroll!");
14547 SDLoc dl(N);
14548
14549 // If ResNE is 0, fully unroll the vector op.
14550 if (ResNE == 0)
14551 ResNE = NE;
14552 else if (NE > ResNE)
14553 NE = ResNE;
14554
14555 if (N->getNumValues() == 2) {
14556 SmallVector<SDValue, 8> Scalars0, Scalars1;
14557 SmallVector<SDValue, 4> Operands(N->getNumOperands());
14558 EVT VT1 = N->getValueType(ResNo: 1);
14559 EVT EltVT1 = VT1.getVectorElementType();
14560
14561 unsigned i;
14562 for (i = 0; i != NE; ++i) {
14563 for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) {
14564 SDValue Operand = N->getOperand(Num: j);
14565 EVT OperandVT = Operand.getValueType();
14566
14567 // A vector operand; extract a single element.
14568 EVT OperandEltVT = OperandVT.getVectorElementType();
14569 Operands[j] = getExtractVectorElt(DL: dl, VT: OperandEltVT, Vec: Operand, Idx: i);
14570 }
14571
14572 SDValue EltOp = getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {EltVT, EltVT1}, Ops: Operands);
14573 Scalars0.push_back(Elt: EltOp);
14574 Scalars1.push_back(Elt: EltOp.getValue(R: 1));
14575 }
14576
14577 for (; i < ResNE; ++i) {
14578 Scalars0.push_back(Elt: getUNDEF(VT: EltVT));
14579 Scalars1.push_back(Elt: getUNDEF(VT: EltVT1));
14580 }
14581
14582 ElementCount ResEC = VT.isScalableVector() ? VT.getVectorElementCount()
14583 : ElementCount::getFixed(MinVal: ResNE);
14584 EVT VecVT = EVT::getVectorVT(Context&: *getContext(), VT: EltVT, EC: ResEC);
14585 EVT VecVT1 = EVT::getVectorVT(Context&: *getContext(), VT: EltVT1, EC: ResEC);
14586 SDValue Vec0 = buildVectorFromUnrolledParts(VT: VecVT, DL: dl, Scalars: Scalars0);
14587 SDValue Vec1 = buildVectorFromUnrolledParts(VT: VecVT1, DL: dl, Scalars: Scalars1);
14588 return getMergeValues(Ops: {Vec0, Vec1}, dl);
14589 }
14590
14591 assert(N->getNumValues() == 1 &&
14592 "Can't unroll a vector with multiple results!");
14593
14594 SmallVector<SDValue, 8> Scalars;
14595 SmallVector<SDValue, 4> Operands(N->getNumOperands());
14596
14597 unsigned i;
14598 for (i= 0; i != NE; ++i) {
14599 for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) {
14600 SDValue Operand = N->getOperand(Num: j);
14601 EVT OperandVT = Operand.getValueType();
14602 if (OperandVT.isVector()) {
14603 // A vector operand; extract a single element.
14604 EVT OperandEltVT = OperandVT.getVectorElementType();
14605 Operands[j] = getExtractVectorElt(DL: dl, VT: OperandEltVT, Vec: Operand, Idx: i);
14606 } else {
14607 // A scalar operand; just use it as is.
14608 Operands[j] = Operand;
14609 }
14610 }
14611
14612 switch (N->getOpcode()) {
14613 default: {
14614 Scalars.push_back(Elt: getNode(Opcode: N->getOpcode(), DL: dl, VT: EltVT, Ops: Operands,
14615 Flags: N->getFlags()));
14616 break;
14617 }
14618 case ISD::VSELECT: {
14619 SDValue Cond = Operands[0];
14620 EVT CondVT = Cond.getValueType();
14621 const TargetLowering &TLI = getTargetLoweringInfo();
14622 if (CondVT != MVT::i1 &&
14623 TLI.getBooleanContents(Type: N->getOperand(Num: 1).getValueType()) !=
14624 TLI.getBooleanContents(Type: CondVT)) {
14625 EVT SetCCVT =
14626 TLI.getSetCCResultType(DL: getDataLayout(), Context&: *getContext(), VT: CondVT);
14627 Operands[0] =
14628 getSetCC(DL: dl, VT: SetCCVT, LHS: Cond, RHS: getConstant(Val: 0, DL: dl, VT: CondVT), Cond: ISD::SETNE);
14629 }
14630 Scalars.push_back(
14631 Elt: getNode(Opcode: ISD::SELECT, DL: dl, VT: EltVT, Ops: Operands, Flags: N->getFlags()));
14632 break;
14633 }
14634 case ISD::SHL:
14635 case ISD::SRA:
14636 case ISD::SRL:
14637 case ISD::ROTL:
14638 case ISD::ROTR:
14639 Scalars.push_back(Elt: getNode(Opcode: N->getOpcode(), DL: dl, VT: EltVT, N1: Operands[0],
14640 N2: getShiftAmountOperand(LHSTy: Operands[0].getValueType(),
14641 Op: Operands[1])));
14642 break;
14643 case ISD::SIGN_EXTEND_INREG: {
14644 EVT ExtVT = cast<VTSDNode>(Val&: Operands[1])->getVT().getVectorElementType();
14645 Scalars.push_back(Elt: getNode(Opcode: N->getOpcode(), DL: dl, VT: EltVT,
14646 N1: Operands[0],
14647 N2: getValueType(VT: ExtVT)));
14648 break;
14649 }
14650 case ISD::ADDRSPACECAST: {
14651 const auto *ASC = cast<AddrSpaceCastSDNode>(Val: N);
14652 Scalars.push_back(
14653 Elt: getAddrSpaceCast(dl, VT: EltVT, Ptr: Operands[0], SrcAS: ASC->getSrcAddressSpace(),
14654 DestAS: ASC->getDestAddressSpace(), Flags: ASC->getFlags()));
14655 break;
14656 }
14657 }
14658 }
14659
14660 for (; i < ResNE; ++i)
14661 Scalars.push_back(Elt: getUNDEF(VT: EltVT));
14662
14663 EVT VecVT = VT.isScalableVector()
14664 ? VT
14665 : EVT::getVectorVT(Context&: *getContext(), VT: EltVT, NumElements: ResNE);
14666 return buildVectorFromUnrolledParts(VT: VecVT, DL: dl, Scalars);
14667}
14668
14669std::pair<SDValue, SDValue> SelectionDAG::UnrollVectorOverflowOp(
14670 SDNode *N, unsigned ResNE) {
14671 unsigned Opcode = N->getOpcode();
14672 assert((Opcode == ISD::UADDO || Opcode == ISD::SADDO ||
14673 Opcode == ISD::USUBO || Opcode == ISD::SSUBO ||
14674 Opcode == ISD::UMULO || Opcode == ISD::SMULO) &&
14675 "Expected an overflow opcode");
14676
14677 EVT ResVT = N->getValueType(ResNo: 0);
14678 EVT OvVT = N->getValueType(ResNo: 1);
14679 EVT ResEltVT = ResVT.getVectorElementType();
14680 EVT OvEltVT = OvVT.getVectorElementType();
14681 SDLoc dl(N);
14682
14683 // If ResNE is 0, fully unroll the vector op.
14684 unsigned NE = ResVT.getVectorNumElements();
14685 if (ResNE == 0)
14686 ResNE = NE;
14687 else if (NE > ResNE)
14688 NE = ResNE;
14689
14690 SmallVector<SDValue, 8> LHSScalars;
14691 SmallVector<SDValue, 8> RHSScalars;
14692 ExtractVectorElements(Op: N->getOperand(Num: 0), Args&: LHSScalars, Start: 0, Count: NE);
14693 ExtractVectorElements(Op: N->getOperand(Num: 1), Args&: RHSScalars, Start: 0, Count: NE);
14694
14695 EVT SVT = TLI->getSetCCResultType(DL: getDataLayout(), Context&: *getContext(), VT: ResEltVT);
14696 SDVTList VTs = getVTList(VT1: ResEltVT, VT2: SVT);
14697 SmallVector<SDValue, 8> ResScalars;
14698 SmallVector<SDValue, 8> OvScalars;
14699 for (unsigned i = 0; i < NE; ++i) {
14700 SDValue Res = getNode(Opcode, DL: dl, VTList: VTs, N1: LHSScalars[i], N2: RHSScalars[i]);
14701 SDValue Ov =
14702 getSelect(DL: dl, VT: OvEltVT, Cond: Res.getValue(R: 1),
14703 LHS: getBoolConstant(V: true, DL: dl, VT: OvEltVT, OpVT: ResVT),
14704 RHS: getConstant(Val: 0, DL: dl, VT: OvEltVT));
14705
14706 ResScalars.push_back(Elt: Res);
14707 OvScalars.push_back(Elt: Ov);
14708 }
14709
14710 ResScalars.append(NumInputs: ResNE - NE, Elt: getUNDEF(VT: ResEltVT));
14711 OvScalars.append(NumInputs: ResNE - NE, Elt: getUNDEF(VT: OvEltVT));
14712
14713 EVT NewResVT = EVT::getVectorVT(Context&: *getContext(), VT: ResEltVT, NumElements: ResNE);
14714 EVT NewOvVT = EVT::getVectorVT(Context&: *getContext(), VT: OvEltVT, NumElements: ResNE);
14715 return std::make_pair(x: getBuildVector(VT: NewResVT, DL: dl, Ops: ResScalars),
14716 y: getBuildVector(VT: NewOvVT, DL: dl, Ops: OvScalars));
14717}
14718
14719static bool areNonVolatileConsecutiveLoadsOrStores(LSBaseSDNode *LS,
14720 LSBaseSDNode *Base,
14721 TypeSize Bytes, int Dist,
14722 const SelectionDAG &DAG) {
14723 if (LS->isVolatile() || Base->isVolatile())
14724 return false;
14725 // TODO: probably too restrictive for atomics, revisit
14726 if (!LS->isSimple())
14727 return false;
14728 if (LS->isIndexed() || Base->isIndexed())
14729 return false;
14730 if (LS->getChain() != Base->getChain())
14731 return false;
14732 EVT VT = LS->getMemoryVT();
14733 if (!VT.getScalarType().isByteSized())
14734 return false;
14735 if (VT.getStoreSize() != Bytes)
14736 return false;
14737
14738 auto BaseLocDecomp = BaseIndexOffset::match(N: Base, DAG);
14739 auto LocDecomp = BaseIndexOffset::match(N: LS, DAG);
14740 StackOffset Offset;
14741 if (!BaseLocDecomp.equalBaseIndex(Other: LocDecomp, DAG, Off&: Offset))
14742 return false;
14743
14744 int64_t ExpectedOffset = Dist * (int64_t)Bytes.getKnownMinValue();
14745 if (Bytes.isFixed())
14746 return Offset == StackOffset::getFixed(Fixed: ExpectedOffset);
14747 return Offset == StackOffset::getScalable(Scalable: ExpectedOffset);
14748}
14749
14750bool SelectionDAG::areNonVolatileConsecutiveLoads(LoadSDNode *LD,
14751 LoadSDNode *Base,
14752 TypeSize Bytes,
14753 int Dist) const {
14754 return areNonVolatileConsecutiveLoadsOrStores(LS: LD, Base, Bytes, Dist, DAG: *this);
14755}
14756
14757bool SelectionDAG::areNonVolatileConsecutiveStores(StoreSDNode *ST,
14758 StoreSDNode *Base,
14759 TypeSize Bytes,
14760 int Dist) const {
14761 return areNonVolatileConsecutiveLoadsOrStores(LS: ST, Base, Bytes, Dist, DAG: *this);
14762}
14763
14764/// InferPtrAlignment - Infer alignment of a load / store address. Return
14765/// std::nullopt if it cannot be inferred.
14766MaybeAlign SelectionDAG::InferPtrAlign(SDValue Ptr) const {
14767 // If this is a GlobalAddress + cst, return the alignment.
14768 const GlobalValue *GV = nullptr;
14769 int64_t GVOffset = 0;
14770 if (TLI->isGAPlusOffset(N: Ptr.getNode(), GA&: GV, Offset&: GVOffset)) {
14771 unsigned PtrWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType());
14772 KnownBits Known(PtrWidth);
14773 llvm::computeKnownBits(V: GV, Known, DL: getDataLayout());
14774 unsigned AlignBits = Known.countMinTrailingZeros();
14775 if (AlignBits)
14776 return commonAlignment(A: Align::fromLog2(Log2Value: std::min(a: 31U, b: AlignBits)),
14777 Offset: GVOffset);
14778 }
14779
14780 // If this is a direct reference to a stack slot, use information about the
14781 // stack slot's alignment.
14782 int FrameIdx = INT_MIN;
14783 int64_t FrameOffset = 0;
14784 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Ptr)) {
14785 FrameIdx = FI->getIndex();
14786 } else if (isBaseWithConstantOffset(Op: Ptr) &&
14787 isa<FrameIndexSDNode>(Val: Ptr.getOperand(i: 0))) {
14788 // Handle FI+Cst
14789 FrameIdx = cast<FrameIndexSDNode>(Val: Ptr.getOperand(i: 0))->getIndex();
14790 FrameOffset = Ptr.getConstantOperandVal(i: 1);
14791 }
14792
14793 if (FrameIdx != INT_MIN) {
14794 const MachineFrameInfo &MFI = getMachineFunction().getFrameInfo();
14795 return commonAlignment(A: MFI.getObjectAlign(ObjectIdx: FrameIdx), Offset: FrameOffset);
14796 }
14797
14798 return std::nullopt;
14799}
14800
14801/// Split the scalar node with EXTRACT_ELEMENT using the provided
14802/// VTs and return the low/high part.
14803std::pair<SDValue, SDValue> SelectionDAG::SplitScalar(const SDValue &N,
14804 const SDLoc &DL,
14805 const EVT &LoVT,
14806 const EVT &HiVT) {
14807 assert(!LoVT.isVector() && !HiVT.isVector() && !N.getValueType().isVector() &&
14808 "Split node must be a scalar type");
14809 SDValue Lo =
14810 getNode(Opcode: ISD::EXTRACT_ELEMENT, DL, VT: LoVT, N1: N, N2: getIntPtrConstant(Val: 0, DL));
14811 SDValue Hi =
14812 getNode(Opcode: ISD::EXTRACT_ELEMENT, DL, VT: HiVT, N1: N, N2: getIntPtrConstant(Val: 1, DL));
14813 return std::make_pair(x&: Lo, y&: Hi);
14814}
14815
14816/// GetSplitDestVTs - Compute the VTs needed for the low/hi parts of a type
14817/// which is split (or expanded) into two not necessarily identical pieces.
14818std::pair<EVT, EVT> SelectionDAG::GetSplitDestVTs(const EVT &VT) const {
14819 // Currently all types are split in half.
14820 EVT LoVT, HiVT;
14821 if (!VT.isVector())
14822 LoVT = HiVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT);
14823 else
14824 LoVT = HiVT = VT.getHalfNumVectorElementsVT(Context&: *getContext());
14825
14826 return std::make_pair(x&: LoVT, y&: HiVT);
14827}
14828
14829/// GetDependentSplitDestVTs - Compute the VTs needed for the low/hi parts of a
14830/// type, dependent on an enveloping VT that has been split into two identical
14831/// pieces. Sets the HiIsEmpty flag when hi type has zero storage size.
14832std::pair<EVT, EVT>
14833SelectionDAG::GetDependentSplitDestVTs(const EVT &VT, const EVT &EnvVT,
14834 bool *HiIsEmpty) const {
14835 EVT EltTp = VT.getVectorElementType();
14836 // Examples:
14837 // custom VL=8 with enveloping VL=8/8 yields 8/0 (hi empty)
14838 // custom VL=9 with enveloping VL=8/8 yields 8/1
14839 // custom VL=10 with enveloping VL=8/8 yields 8/2
14840 // etc.
14841 ElementCount VTNumElts = VT.getVectorElementCount();
14842 ElementCount EnvNumElts = EnvVT.getVectorElementCount();
14843 assert(VTNumElts.isScalable() == EnvNumElts.isScalable() &&
14844 "Mixing fixed width and scalable vectors when enveloping a type");
14845 EVT LoVT, HiVT;
14846 if (VTNumElts.getKnownMinValue() > EnvNumElts.getKnownMinValue()) {
14847 LoVT = EVT::getVectorVT(Context&: *getContext(), VT: EltTp, EC: EnvNumElts);
14848 HiVT = EVT::getVectorVT(Context&: *getContext(), VT: EltTp, EC: VTNumElts - EnvNumElts);
14849 *HiIsEmpty = false;
14850 } else {
14851 // Flag that hi type has zero storage size, but return split envelop type
14852 // (this would be easier if vector types with zero elements were allowed).
14853 LoVT = EVT::getVectorVT(Context&: *getContext(), VT: EltTp, EC: VTNumElts);
14854 HiVT = EVT::getVectorVT(Context&: *getContext(), VT: EltTp, EC: EnvNumElts);
14855 *HiIsEmpty = true;
14856 }
14857 return std::make_pair(x&: LoVT, y&: HiVT);
14858}
14859
14860/// SplitVector - Split the vector with EXTRACT_SUBVECTOR and return the
14861/// low/high part.
14862std::pair<SDValue, SDValue>
14863SelectionDAG::SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT,
14864 const EVT &HiVT) {
14865 assert(LoVT.isScalableVector() == HiVT.isScalableVector() &&
14866 LoVT.isScalableVector() == N.getValueType().isScalableVector() &&
14867 "Splitting vector with an invalid mixture of fixed and scalable "
14868 "vector types");
14869 assert(LoVT.getVectorMinNumElements() + HiVT.getVectorMinNumElements() <=
14870 N.getValueType().getVectorMinNumElements() &&
14871 "More vector elements requested than available!");
14872 SDValue Lo, Hi;
14873 Lo = getExtractSubvector(DL, VT: LoVT, Vec: N, Idx: 0);
14874 // For scalable vectors it is safe to use LoVT.getVectorMinNumElements()
14875 // (rather than having to use ElementCount), because EXTRACT_SUBVECTOR scales
14876 // IDX with the runtime scaling factor of the result vector type. For
14877 // fixed-width result vectors, that runtime scaling factor is 1.
14878 Hi = getExtractSubvector(DL, VT: HiVT, Vec: N, Idx: LoVT.getVectorMinNumElements());
14879 return std::make_pair(x&: Lo, y&: Hi);
14880}
14881
14882std::pair<SDValue, SDValue> SelectionDAG::SplitEVL(SDValue N, EVT VecVT,
14883 const SDLoc &DL) {
14884 // Split the vector length parameter.
14885 // %evl -> umin(%evl, %halfnumelts) and usubsat(%evl - %halfnumelts).
14886 EVT VT = N.getValueType();
14887 assert(VecVT.getVectorElementCount().isKnownEven() &&
14888 "Expecting the mask to be an evenly-sized vector");
14889 SDValue HalfNumElts = getElementCount(
14890 DL, VT, EC: VecVT.getVectorElementCount().divideCoefficientBy(RHS: 2));
14891 SDValue Lo = getNode(Opcode: ISD::UMIN, DL, VT, N1: N, N2: HalfNumElts);
14892 SDValue Hi = getNode(Opcode: ISD::USUBSAT, DL, VT, N1: N, N2: HalfNumElts);
14893 return std::make_pair(x&: Lo, y&: Hi);
14894}
14895
14896/// Widen the vector up to the next power of two using INSERT_SUBVECTOR.
14897SDValue SelectionDAG::WidenVector(const SDValue &N, const SDLoc &DL) {
14898 EVT VT = N.getValueType();
14899 EVT WideVT = VT.changeElementCount(
14900 Context&: *getContext(),
14901 EC: ElementCount::getFixed(MinVal: NextPowerOf2(A: VT.getVectorNumElements())));
14902 return getInsertSubvector(DL, Vec: getPOISON(VT: WideVT), SubVec: N, Idx: 0);
14903}
14904
14905void SelectionDAG::ExtractVectorElements(SDValue Op,
14906 SmallVectorImpl<SDValue> &Args,
14907 unsigned Start, unsigned Count,
14908 EVT EltVT) {
14909 EVT VT = Op.getValueType();
14910 if (Count == 0)
14911 Count = VT.getVectorNumElements();
14912 if (EltVT == EVT())
14913 EltVT = VT.getVectorElementType();
14914 SDLoc SL(Op);
14915 for (unsigned i = Start, e = Start + Count; i != e; ++i) {
14916 Args.push_back(Elt: getExtractVectorElt(DL: SL, VT: EltVT, Vec: Op, Idx: i));
14917 }
14918}
14919
14920// getAddressSpace - Return the address space this GlobalAddress belongs to.
14921unsigned GlobalAddressSDNode::getAddressSpace() const {
14922 return getGlobal()->getType()->getAddressSpace();
14923}
14924
14925Type *ConstantPoolSDNode::getType() const {
14926 if (isMachineConstantPoolEntry())
14927 return Val.MachineCPVal->getType();
14928 return Val.ConstVal->getType();
14929}
14930
14931bool BuildVectorSDNode::isConstantSplat(APInt &SplatValue, APInt &SplatUndef,
14932 unsigned &SplatBitSize,
14933 bool &HasAnyUndefs,
14934 unsigned MinSplatBits,
14935 bool IsBigEndian) const {
14936 EVT VT = getValueType(ResNo: 0);
14937 assert(VT.isVector() && "Expected a vector type");
14938 unsigned VecWidth = VT.getSizeInBits();
14939 if (MinSplatBits > VecWidth)
14940 return false;
14941
14942 // FIXME: The widths are based on this node's type, but build vectors can
14943 // truncate their operands.
14944 SplatValue = APInt(VecWidth, 0);
14945 SplatUndef = APInt(VecWidth, 0);
14946
14947 // Get the bits. Bits with undefined values (when the corresponding element
14948 // of the vector is an ISD::UNDEF value) are set in SplatUndef and cleared
14949 // in SplatValue. If any of the values are not constant, give up and return
14950 // false.
14951 unsigned int NumOps = getNumOperands();
14952 assert(NumOps > 0 && "isConstantSplat has 0-size build vector");
14953 unsigned EltWidth = VT.getScalarSizeInBits();
14954
14955 for (unsigned j = 0; j < NumOps; ++j) {
14956 unsigned i = IsBigEndian ? NumOps - 1 - j : j;
14957 SDValue OpVal = getOperand(Num: i);
14958 unsigned BitPos = j * EltWidth;
14959
14960 if (OpVal.isUndef())
14961 SplatUndef.setBits(loBit: BitPos, hiBit: BitPos + EltWidth);
14962 else if (auto *CN = dyn_cast<ConstantSDNode>(Val&: OpVal))
14963 SplatValue.insertBits(SubBits: CN->getAPIntValue().zextOrTrunc(width: EltWidth), bitPosition: BitPos);
14964 else if (auto *CN = dyn_cast<ConstantFPSDNode>(Val&: OpVal))
14965 SplatValue.insertBits(SubBits: CN->getValueAPF().bitcastToAPInt(), bitPosition: BitPos);
14966 else
14967 return false;
14968 }
14969
14970 // The build_vector is all constants or undefs. Find the smallest element
14971 // size that splats the vector.
14972 HasAnyUndefs = (SplatUndef != 0);
14973
14974 // FIXME: This does not work for vectors with elements less than 8 bits.
14975 while (VecWidth > 8) {
14976 // If we can't split in half, stop here.
14977 if (VecWidth & 1)
14978 break;
14979
14980 unsigned HalfSize = VecWidth / 2;
14981 APInt HighValue = SplatValue.extractBits(numBits: HalfSize, bitPosition: HalfSize);
14982 APInt LowValue = SplatValue.extractBits(numBits: HalfSize, bitPosition: 0);
14983 APInt HighUndef = SplatUndef.extractBits(numBits: HalfSize, bitPosition: HalfSize);
14984 APInt LowUndef = SplatUndef.extractBits(numBits: HalfSize, bitPosition: 0);
14985
14986 // If the two halves do not match (ignoring undef bits), stop here.
14987 if ((HighValue & ~LowUndef) != (LowValue & ~HighUndef) ||
14988 MinSplatBits > HalfSize)
14989 break;
14990
14991 SplatValue = HighValue | LowValue;
14992 SplatUndef = HighUndef & LowUndef;
14993
14994 VecWidth = HalfSize;
14995 }
14996
14997 // FIXME: The loop above only tries to split in halves. But if the input
14998 // vector for example is <3 x i16> it wouldn't be able to detect a
14999 // SplatBitSize of 16. No idea if that is a design flaw currently limiting
15000 // optimizations. I guess that back in the days when this helper was created
15001 // vectors normally was power-of-2 sized.
15002
15003 SplatBitSize = VecWidth;
15004 return true;
15005}
15006
15007SDValue BuildVectorSDNode::getSplatValue(const APInt &DemandedElts,
15008 BitVector *UndefElements) const {
15009 unsigned NumOps = getNumOperands();
15010 if (UndefElements) {
15011 UndefElements->clear();
15012 UndefElements->resize(N: NumOps);
15013 }
15014 assert(NumOps == DemandedElts.getBitWidth() && "Unexpected vector size");
15015 if (!DemandedElts)
15016 return SDValue();
15017 SDValue Splatted;
15018 for (unsigned i = 0; i != NumOps; ++i) {
15019 if (!DemandedElts[i])
15020 continue;
15021 SDValue Op = getOperand(Num: i);
15022 if (Op.isUndef()) {
15023 if (UndefElements)
15024 (*UndefElements)[i] = true;
15025 } else if (!Splatted) {
15026 Splatted = Op;
15027 } else if (Splatted != Op) {
15028 return SDValue();
15029 }
15030 }
15031
15032 if (!Splatted) {
15033 unsigned FirstDemandedIdx = DemandedElts.countr_zero();
15034 assert(getOperand(FirstDemandedIdx).isUndef() &&
15035 "Can only have a splat without a constant for all undefs.");
15036 return getOperand(Num: FirstDemandedIdx);
15037 }
15038
15039 return Splatted;
15040}
15041
15042SDValue BuildVectorSDNode::getSplatValue(BitVector *UndefElements) const {
15043 APInt DemandedElts = APInt::getAllOnes(numBits: getNumOperands());
15044 return getSplatValue(DemandedElts, UndefElements);
15045}
15046
15047bool BuildVectorSDNode::getRepeatedSequence(const APInt &DemandedElts,
15048 SmallVectorImpl<SDValue> &Sequence,
15049 BitVector *UndefElements) const {
15050 unsigned NumOps = getNumOperands();
15051 Sequence.clear();
15052 if (UndefElements) {
15053 UndefElements->clear();
15054 UndefElements->resize(N: NumOps);
15055 }
15056 assert(NumOps == DemandedElts.getBitWidth() && "Unexpected vector size");
15057 if (!DemandedElts || NumOps < 2 || !isPowerOf2_32(Value: NumOps))
15058 return false;
15059
15060 // Set the undefs even if we don't find a sequence (like getSplatValue).
15061 if (UndefElements)
15062 for (unsigned I = 0; I != NumOps; ++I)
15063 if (DemandedElts[I] && getOperand(Num: I).isUndef())
15064 (*UndefElements)[I] = true;
15065
15066 // Iteratively widen the sequence length looking for repetitions.
15067 for (unsigned SeqLen = 1; SeqLen < NumOps; SeqLen *= 2) {
15068 Sequence.append(NumInputs: SeqLen, Elt: SDValue());
15069 for (unsigned I = 0; I != NumOps; ++I) {
15070 if (!DemandedElts[I])
15071 continue;
15072 SDValue &SeqOp = Sequence[I % SeqLen];
15073 SDValue Op = getOperand(Num: I);
15074 if (Op.isUndef()) {
15075 if (!SeqOp)
15076 SeqOp = Op;
15077 continue;
15078 }
15079 if (SeqOp && !SeqOp.isUndef() && SeqOp != Op) {
15080 Sequence.clear();
15081 break;
15082 }
15083 SeqOp = Op;
15084 }
15085 if (!Sequence.empty())
15086 return true;
15087 }
15088
15089 assert(Sequence.empty() && "Failed to empty non-repeating sequence pattern");
15090 return false;
15091}
15092
15093bool BuildVectorSDNode::getRepeatedSequence(SmallVectorImpl<SDValue> &Sequence,
15094 BitVector *UndefElements) const {
15095 APInt DemandedElts = APInt::getAllOnes(numBits: getNumOperands());
15096 return getRepeatedSequence(DemandedElts, Sequence, UndefElements);
15097}
15098
15099ConstantSDNode *
15100BuildVectorSDNode::getConstantSplatNode(const APInt &DemandedElts,
15101 BitVector *UndefElements) const {
15102 return dyn_cast_or_null<ConstantSDNode>(
15103 Val: getSplatValue(DemandedElts, UndefElements));
15104}
15105
15106ConstantSDNode *
15107BuildVectorSDNode::getConstantSplatNode(BitVector *UndefElements) const {
15108 return dyn_cast_or_null<ConstantSDNode>(Val: getSplatValue(UndefElements));
15109}
15110
15111ConstantFPSDNode *
15112BuildVectorSDNode::getConstantFPSplatNode(const APInt &DemandedElts,
15113 BitVector *UndefElements) const {
15114 return dyn_cast_or_null<ConstantFPSDNode>(
15115 Val: getSplatValue(DemandedElts, UndefElements));
15116}
15117
15118ConstantFPSDNode *
15119BuildVectorSDNode::getConstantFPSplatNode(BitVector *UndefElements) const {
15120 return dyn_cast_or_null<ConstantFPSDNode>(Val: getSplatValue(UndefElements));
15121}
15122
15123int32_t
15124BuildVectorSDNode::getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements,
15125 uint32_t BitWidth) const {
15126 if (ConstantFPSDNode *CN =
15127 dyn_cast_or_null<ConstantFPSDNode>(Val: getSplatValue(UndefElements))) {
15128 bool IsExact;
15129 APSInt IntVal(BitWidth);
15130 const APFloat &APF = CN->getValueAPF();
15131 if (APF.convertToInteger(Result&: IntVal, RM: APFloat::rmTowardZero, IsExact: &IsExact) !=
15132 APFloat::opOK ||
15133 !IsExact)
15134 return -1;
15135
15136 return IntVal.exactLogBase2();
15137 }
15138 return -1;
15139}
15140
15141bool BuildVectorSDNode::getConstantRawBits(
15142 bool IsLittleEndian, unsigned DstEltSizeInBits,
15143 SmallVectorImpl<APInt> &RawBitElements, BitVector &UndefElements) const {
15144 // Early-out if this contains anything but Undef/Constant/ConstantFP.
15145 if (!isConstant())
15146 return false;
15147
15148 unsigned NumSrcOps = getNumOperands();
15149 unsigned SrcEltSizeInBits = getValueType(ResNo: 0).getScalarSizeInBits();
15150 assert(((NumSrcOps * SrcEltSizeInBits) % DstEltSizeInBits) == 0 &&
15151 "Invalid bitcast scale");
15152
15153 // If the elements don't pack cleanly, don't attempt to recast.
15154 // TODO: Add handling (e.g. bitcast <33 x i8> --> <8 x i33>)
15155 if ((SrcEltSizeInBits % DstEltSizeInBits) != 0 &&
15156 (DstEltSizeInBits % SrcEltSizeInBits) != 0)
15157 return false;
15158
15159 // Extract raw src bits.
15160 SmallVector<APInt> SrcBitElements(NumSrcOps,
15161 APInt::getZero(numBits: SrcEltSizeInBits));
15162 BitVector SrcUndeElements(NumSrcOps, false);
15163
15164 for (unsigned I = 0; I != NumSrcOps; ++I) {
15165 SDValue Op = getOperand(Num: I);
15166 if (Op.isUndef()) {
15167 SrcUndeElements.set(I);
15168 continue;
15169 }
15170 auto *CInt = dyn_cast<ConstantSDNode>(Val&: Op);
15171 auto *CFP = dyn_cast<ConstantFPSDNode>(Val&: Op);
15172 assert((CInt || CFP) && "Unknown constant");
15173 SrcBitElements[I] = CInt ? CInt->getAPIntValue().trunc(width: SrcEltSizeInBits)
15174 : CFP->getValueAPF().bitcastToAPInt();
15175 }
15176
15177 // Recast to dst width.
15178 recastRawBits(IsLittleEndian, DstEltSizeInBits, DstBitElements&: RawBitElements,
15179 SrcBitElements, DstUndefElements&: UndefElements, SrcUndefElements: SrcUndeElements);
15180 return true;
15181}
15182
15183void BuildVectorSDNode::recastRawBits(bool IsLittleEndian,
15184 unsigned DstEltSizeInBits,
15185 SmallVectorImpl<APInt> &DstBitElements,
15186 ArrayRef<APInt> SrcBitElements,
15187 BitVector &DstUndefElements,
15188 const BitVector &SrcUndefElements) {
15189 unsigned NumSrcOps = SrcBitElements.size();
15190 unsigned SrcEltSizeInBits = SrcBitElements[0].getBitWidth();
15191 assert(((NumSrcOps * SrcEltSizeInBits) % DstEltSizeInBits) == 0 &&
15192 "Invalid bitcast scale");
15193 assert(((SrcEltSizeInBits % DstEltSizeInBits) == 0 ||
15194 (DstEltSizeInBits % SrcEltSizeInBits) == 0) &&
15195 "Unsupported bitcast scale");
15196 assert(NumSrcOps == SrcUndefElements.size() && "Vector size mismatch");
15197
15198 unsigned NumDstOps = (NumSrcOps * SrcEltSizeInBits) / DstEltSizeInBits;
15199 DstUndefElements.clear();
15200 DstUndefElements.resize(N: NumDstOps, t: false);
15201 DstBitElements.assign(NumElts: NumDstOps, Elt: APInt::getZero(numBits: DstEltSizeInBits));
15202
15203 // Concatenate src elements constant bits together into dst element.
15204 if (SrcEltSizeInBits <= DstEltSizeInBits) {
15205 unsigned Scale = DstEltSizeInBits / SrcEltSizeInBits;
15206 for (unsigned I = 0; I != NumDstOps; ++I) {
15207 DstUndefElements.set(I);
15208 APInt &DstBits = DstBitElements[I];
15209 for (unsigned J = 0; J != Scale; ++J) {
15210 unsigned Idx = (I * Scale) + (IsLittleEndian ? J : (Scale - J - 1));
15211 if (SrcUndefElements[Idx])
15212 continue;
15213 DstUndefElements.reset(Idx: I);
15214 const APInt &SrcBits = SrcBitElements[Idx];
15215 assert(SrcBits.getBitWidth() == SrcEltSizeInBits &&
15216 "Illegal constant bitwidths");
15217 DstBits.insertBits(SubBits: SrcBits, bitPosition: J * SrcEltSizeInBits);
15218 }
15219 }
15220 return;
15221 }
15222
15223 // Split src element constant bits into dst elements.
15224 unsigned Scale = SrcEltSizeInBits / DstEltSizeInBits;
15225 for (unsigned I = 0; I != NumSrcOps; ++I) {
15226 if (SrcUndefElements[I]) {
15227 DstUndefElements.set(I: I * Scale, E: (I + 1) * Scale);
15228 continue;
15229 }
15230 const APInt &SrcBits = SrcBitElements[I];
15231 for (unsigned J = 0; J != Scale; ++J) {
15232 unsigned Idx = (I * Scale) + (IsLittleEndian ? J : (Scale - J - 1));
15233 APInt &DstBits = DstBitElements[Idx];
15234 DstBits = SrcBits.extractBits(numBits: DstEltSizeInBits, bitPosition: J * DstEltSizeInBits);
15235 }
15236 }
15237}
15238
15239bool BuildVectorSDNode::isConstant() const {
15240 for (const SDValue &Op : op_values()) {
15241 unsigned Opc = Op.getOpcode();
15242 if (!Op.isUndef() && Opc != ISD::Constant && Opc != ISD::ConstantFP)
15243 return false;
15244 }
15245 return true;
15246}
15247
15248std::optional<std::pair<APInt, APInt>>
15249BuildVectorSDNode::isArithmeticSequence() const {
15250 unsigned NumOps = getNumOperands();
15251 if (NumOps < 2)
15252 return std::nullopt;
15253
15254 unsigned EltSize = getValueType(ResNo: 0).getScalarSizeInBits();
15255 APInt Start, Stride;
15256 int FirstIdx = -1, SecondIdx = -1;
15257
15258 // Find the first two non-undef constant elements to determine Start and
15259 // Stride, then verify all remaining elements match the sequence.
15260 for (unsigned I = 0; I < NumOps; ++I) {
15261 SDValue Op = getOperand(Num: I);
15262 if (Op->isUndef())
15263 continue;
15264 if (!isa<ConstantSDNode>(Val: Op))
15265 return std::nullopt;
15266
15267 APInt Val = getConstantOperandAPInt(Num: I).trunc(width: EltSize);
15268 if (FirstIdx < 0) {
15269 FirstIdx = I;
15270 Start = Val;
15271 } else if (SecondIdx < 0) {
15272 SecondIdx = I;
15273 // Compute stride using modular arithmetic. Simple division would handle
15274 // common strides (1, 2, -1, etc.), but modular inverse maximizes matches.
15275 // Example: <0, poison, poison, 0xFF> has stride 0x55 since 3*0x55 = 0xFF
15276 // Note that modular arithmetic is agnostic to signed/unsigned.
15277 unsigned IdxDiff = I - FirstIdx;
15278 APInt ValDiff = Val - Start;
15279
15280 // Step 1: Factor out common powers of 2 from IdxDiff and ValDiff.
15281 unsigned CommonPow2Bits = llvm::countr_zero(Val: IdxDiff);
15282 if (ValDiff.countr_zero() < CommonPow2Bits)
15283 return std::nullopt; // ValDiff not divisible by 2^CommonPow2Bits
15284 IdxDiff >>= CommonPow2Bits;
15285 ValDiff.lshrInPlace(ShiftAmt: CommonPow2Bits);
15286
15287 // Step 2: IdxDiff is now odd, so its inverse mod 2^EltSize exists.
15288 // TODO: There are 2^CommonPow2Bits valid strides; currently we only try
15289 // one, but we could try all candidates to handle more cases.
15290 Stride = ValDiff * APInt(EltSize, IdxDiff).multiplicativeInverse();
15291 if (Stride.isZero())
15292 return std::nullopt;
15293
15294 // Step 3: Adjust Start based on the first defined element's index.
15295 Start -= Stride * FirstIdx;
15296 } else {
15297 // Verify this element matches the sequence.
15298 if (Val != Start + Stride * I)
15299 return std::nullopt;
15300 }
15301 }
15302
15303 // Need at least two defined elements.
15304 if (SecondIdx < 0)
15305 return std::nullopt;
15306
15307 return std::make_pair(x&: Start, y&: Stride);
15308}
15309
15310bool ShuffleVectorSDNode::isSplatMask(ArrayRef<int> Mask) {
15311 // Find the first non-undef value in the shuffle mask.
15312 unsigned i, e;
15313 for (i = 0, e = Mask.size(); i != e && Mask[i] < 0; ++i)
15314 /* search */;
15315
15316 // If all elements are undefined, this shuffle can be considered a splat
15317 // (although it should eventually get simplified away completely).
15318 if (i == e)
15319 return true;
15320
15321 // Make sure all remaining elements are either undef or the same as the first
15322 // non-undef value.
15323 for (int Idx = Mask[i]; i != e; ++i)
15324 if (Mask[i] >= 0 && Mask[i] != Idx)
15325 return false;
15326 return true;
15327}
15328
15329// Returns true if it is a constant integer BuildVector or constant integer,
15330// possibly hidden by a bitcast.
15331bool SelectionDAG::isConstantIntBuildVectorOrConstantInt(
15332 SDValue N, bool AllowOpaques) const {
15333 N = peekThroughBitcasts(V: N);
15334
15335 if (auto *C = dyn_cast<ConstantSDNode>(Val&: N))
15336 return AllowOpaques || !C->isOpaque();
15337
15338 if (ISD::isBuildVectorOfConstantSDNodes(V: N))
15339 return true;
15340
15341 // Treat a GlobalAddress supporting constant offset folding as a
15342 // constant integer.
15343 if (auto *GA = dyn_cast<GlobalAddressSDNode>(Val&: N))
15344 if (GA->getOpcode() == ISD::GlobalAddress &&
15345 TLI->isOffsetFoldingLegal(GA))
15346 return true;
15347
15348 if ((N.getOpcode() == ISD::SPLAT_VECTOR) &&
15349 isa<ConstantSDNode>(Val: N.getOperand(i: 0)))
15350 return true;
15351 return false;
15352}
15353
15354// Returns true if it is a constant float BuildVector or constant float.
15355bool SelectionDAG::isConstantFPBuildVectorOrConstantFP(SDValue N) const {
15356 if (isa<ConstantFPSDNode>(Val: N))
15357 return true;
15358
15359 if (ISD::isBuildVectorOfConstantFPSDNodes(V: N))
15360 return true;
15361
15362 if ((N.getOpcode() == ISD::SPLAT_VECTOR) &&
15363 isa<ConstantFPSDNode>(Val: N.getOperand(i: 0)))
15364 return true;
15365
15366 return false;
15367}
15368
15369std::optional<bool> SelectionDAG::isBoolConstant(SDValue N) const {
15370 ConstantSDNode *Const =
15371 isConstOrConstSplat(N, AllowUndefs: false, /*AllowTruncation=*/true);
15372 if (!Const)
15373 return std::nullopt;
15374
15375 EVT VT = N.getValueType();
15376 const APInt CVal = Const->getAPIntValue().trunc(width: VT.getScalarSizeInBits());
15377 switch (TLI->getBooleanContents(Type: VT)) {
15378 case TargetLowering::ZeroOrOneBooleanContent:
15379 if (CVal.isOne())
15380 return true;
15381 if (CVal.isZero())
15382 return false;
15383 return std::nullopt;
15384 case TargetLowering::ZeroOrNegativeOneBooleanContent:
15385 if (CVal.isAllOnes())
15386 return true;
15387 if (CVal.isZero())
15388 return false;
15389 return std::nullopt;
15390 case TargetLowering::UndefinedBooleanContent:
15391 return CVal[0];
15392 }
15393 llvm_unreachable("Unknown BooleanContent enum");
15394}
15395
15396void SelectionDAG::createOperands(SDNode *Node, ArrayRef<SDValue> Vals) {
15397 assert(!Node->OperandList && "Node already has operands");
15398 assert(SDNode::getMaxNumOperands() >= Vals.size() &&
15399 "too many operands to fit into SDNode");
15400 SDUse *Ops = OperandRecycler.allocate(
15401 Cap: ArrayRecycler<SDUse>::Capacity::get(N: Vals.size()), Allocator&: OperandAllocator);
15402
15403 bool IsDivergent = false;
15404 for (unsigned I = 0; I != Vals.size(); ++I) {
15405 Ops[I].setUser(Node);
15406 Ops[I].setInitial(Vals[I]);
15407 EVT VT = Ops[I].getValueType();
15408
15409 // Skip Chain. It does not carry divergence.
15410 if (VT != MVT::Other &&
15411 (VT != MVT::Glue || gluePropagatesDivergence(Node: Ops[I].getNode())) &&
15412 Ops[I].getNode()->isDivergent()) {
15413 IsDivergent = true;
15414 }
15415 }
15416 Node->NumOperands = Vals.size();
15417 Node->OperandList = Ops;
15418 if (!TLI->isSDNodeAlwaysUniform(N: Node)) {
15419 IsDivergent |= TLI->isSDNodeSourceOfDivergence(N: Node, FLI, UA);
15420 Node->SDNodeBits.IsDivergent = IsDivergent;
15421 }
15422 checkForCycles(N: Node);
15423}
15424
15425SDValue SelectionDAG::getTokenFactor(const SDLoc &DL,
15426 SmallVectorImpl<SDValue> &Vals) {
15427 size_t Limit = SDNode::getMaxNumOperands();
15428 while (Vals.size() > Limit) {
15429 unsigned SliceIdx = Vals.size() - Limit;
15430 auto ExtractedTFs = ArrayRef<SDValue>(Vals).slice(N: SliceIdx, M: Limit);
15431 SDValue NewTF = getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: ExtractedTFs);
15432 Vals.erase(CS: Vals.begin() + SliceIdx, CE: Vals.end());
15433 Vals.emplace_back(Args&: NewTF);
15434 }
15435 return getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: Vals);
15436}
15437
15438SDValue SelectionDAG::getIdentityElement(unsigned Opcode, const SDLoc &DL,
15439 EVT VT, SDNodeFlags Flags) {
15440 switch (Opcode) {
15441 default:
15442 return SDValue();
15443 case ISD::ADD:
15444 case ISD::OR:
15445 case ISD::XOR:
15446 case ISD::UMAX:
15447 case ISD::MUL:
15448 case ISD::AND:
15449 case ISD::UMIN:
15450 case ISD::SMAX:
15451 case ISD::SMIN:
15452 return getConstant(Val: getIntegerIdentity(Opcode, BitWidth: VT.getScalarSizeInBits()), DL,
15453 VT);
15454 case ISD::FADD:
15455 // If flags allow, prefer positive zero since it's generally cheaper
15456 // to materialize on most targets.
15457 return getConstantFP(Val: Flags.hasNoSignedZeros() ? 0.0 : -0.0, DL, VT);
15458 case ISD::FMUL:
15459 return getConstantFP(Val: 1.0, DL, VT);
15460 case ISD::FMINNUM:
15461 case ISD::FMAXNUM:
15462 case ISD::FMINIMUMNUM:
15463 case ISD::FMAXIMUMNUM: {
15464 // Neutral element for fminnum/fminimumnum is NaN, Inf or FLT_MAX,
15465 // depending on fast-math flags (FMF).
15466 const fltSemantics &Semantics = VT.getFltSemantics();
15467 APFloat NeutralAF = !Flags.hasNoNaNs() ? APFloat::getQNaN(Sem: Semantics) :
15468 !Flags.hasNoInfs() ? APFloat::getInf(Sem: Semantics) :
15469 APFloat::getLargest(Sem: Semantics);
15470 if (Opcode == ISD::FMAXNUM || Opcode == ISD::FMAXIMUMNUM)
15471 NeutralAF.changeSign();
15472
15473 return getConstantFP(V: NeutralAF, DL, VT);
15474 }
15475 case ISD::FMINIMUM:
15476 case ISD::FMAXIMUM: {
15477 // Neutral element for fminimum is Inf or FLT_MAX, depending on FMF.
15478 const fltSemantics &Semantics = VT.getFltSemantics();
15479 APFloat NeutralAF = !Flags.hasNoInfs() ? APFloat::getInf(Sem: Semantics)
15480 : APFloat::getLargest(Sem: Semantics);
15481 if (Opcode == ISD::FMAXIMUM)
15482 NeutralAF.changeSign();
15483
15484 return getConstantFP(V: NeutralAF, DL, VT);
15485 }
15486
15487 }
15488}
15489
15490SDValue SelectionDAG::getPartialReduceMLS(unsigned Opc, const SDLoc &DL,
15491 SDValue Acc, SDValue LHS,
15492 SDValue RHS) {
15493 EVT AccVT = Acc.getValueType();
15494 if (AccVT.isFloatingPoint()) {
15495 assert(Opc == ISD::PARTIAL_REDUCE_FMLA && "Unexpected opcode");
15496 SDValue NegRHS = getNode(Opcode: ISD::FNEG, DL, VT: RHS.getValueType(), N1: RHS);
15497 return getNode(Opcode: Opc, DL, VT: AccVT, N1: Acc, N2: LHS, N3: NegRHS);
15498 }
15499 assert((Opc == ISD::PARTIAL_REDUCE_UMLA || Opc == ISD::PARTIAL_REDUCE_SMLA ||
15500 Opc == ISD::PARTIAL_REDUCE_SUMLA) &&
15501 "Unexpected opcode");
15502 SDValue NegAcc = getNegative(Val: Acc, DL, VT: AccVT);
15503 SDValue MLA = getNode(Opcode: Opc, DL, VT: AccVT, N1: NegAcc, N2: LHS, N3: RHS);
15504 return getNegative(Val: MLA, DL, VT: AccVT);
15505}
15506
15507/// Helper used to make a call to a library function that has one argument of
15508/// pointer type.
15509///
15510/// Such functions include 'fegetmode', 'fesetenv' and some others, which are
15511/// used to get or set floating-point state. They have one argument of pointer
15512/// type, which points to the memory region containing bits of the
15513/// floating-point state. The value returned by such function is ignored in the
15514/// created call.
15515///
15516/// \param LibFunc Reference to library function (value of RTLIB::Libcall).
15517/// \param Ptr Pointer used to save/load state.
15518/// \param InChain Ingoing token chain.
15519/// \param Node Node being legalized
15520/// \returns Outgoing chain token.
15521SDValue SelectionDAG::makeStateFunctionCall(unsigned LibFunc, SDValue Ptr,
15522 SDValue InChain, SDNode *Node) {
15523 assert(InChain.getValueType() == MVT::Other && "Expected token chain");
15524 RTLIB::LibcallImpl LibcallImpl =
15525 Libcalls->getLibcallImpl(Call: static_cast<RTLIB::Libcall>(LibFunc));
15526 if (LibcallImpl == RTLIB::Unsupported) {
15527 getContext()->emitError(ErrorStr: Twine("no libcall available for ") +
15528 Node->getOperationName(G: this));
15529 return InChain;
15530 }
15531
15532 TargetLowering::ArgListTy Args;
15533 Args.emplace_back(args&: Ptr, args: Ptr.getValueType().getTypeForEVT(Context&: *getContext()));
15534 SDValue Callee =
15535 getExternalSymbol(Libcall: LibcallImpl, VT: TLI->getPointerTy(DL: getDataLayout()));
15536 TargetLowering::CallLoweringInfo CLI(*this);
15537 CLI.setDebugLoc(SDLoc(Node))
15538 .setChain(InChain)
15539 .setLibCallee(CC: Libcalls->getLibcallImplCallingConv(Call: LibcallImpl),
15540 ResultType: Type::getVoidTy(C&: *getContext()), Target: Callee, ArgsList: std::move(Args));
15541 return TLI->LowerCallTo(CLI).second;
15542}
15543
15544void SelectionDAG::copyExtraInfo(SDNode *From, SDNode *To) {
15545 assert(From && To && "Invalid SDNode; empty source SDValue?");
15546 auto I = SDEI.find(Val: From);
15547 if (I == SDEI.end())
15548 return;
15549
15550 // Use of operator[] on the DenseMap may cause an insertion, which invalidates
15551 // the iterator, hence the need to make a copy to prevent a use-after-free.
15552 NodeExtraInfo NEI = I->second;
15553 if (LLVM_LIKELY(!NEI.PCSections)) {
15554 // No deep copy required for the types of extra info set.
15555 //
15556 // FIXME: Investigate if other types of extra info also need deep copy. This
15557 // depends on the types of nodes they can be attached to: if some extra info
15558 // is only ever attached to nodes where a replacement To node is always the
15559 // node where later use and propagation of the extra info has the intended
15560 // semantics, no deep copy is required.
15561 SDEI[To] = std::move(NEI);
15562 return;
15563 }
15564
15565 const SDNode *EntrySDN = getEntryNode().getNode();
15566
15567 // We need to copy NodeExtraInfo to all _new_ nodes that are being introduced
15568 // through the replacement of From with To. Otherwise, replacements of a node
15569 // (From) with more complex nodes (To and its operands) may result in lost
15570 // extra info where the root node (To) is insignificant in further propagating
15571 // and using extra info when further lowering to MIR.
15572 //
15573 // In the first step pre-populate the visited set with the nodes reachable
15574 // from the old From node. This avoids copying NodeExtraInfo to parts of the
15575 // DAG that is not new and should be left untouched.
15576 SmallVector<const SDNode *> Leafs{From}; // Leafs reachable with VisitFrom.
15577 DenseSet<const SDNode *> FromReach; // The set of nodes reachable from From.
15578 auto VisitFrom = [&](auto &&Self, const SDNode *N, int MaxDepth) {
15579 if (MaxDepth == 0) {
15580 // Remember this node in case we need to increase MaxDepth and continue
15581 // populating FromReach from this node.
15582 Leafs.emplace_back(Args&: N);
15583 return;
15584 }
15585 if (!FromReach.insert(V: N).second)
15586 return;
15587 for (const SDValue &Op : N->op_values())
15588 Self(Self, Op.getNode(), MaxDepth - 1);
15589 };
15590
15591 // Copy extra info to To and all its transitive operands (that are new).
15592 SmallPtrSet<const SDNode *, 8> Visited;
15593 auto DeepCopyTo = [&](auto &&Self, const SDNode *N) {
15594 if (FromReach.contains(V: N))
15595 return true;
15596 if (!Visited.insert(Ptr: N).second)
15597 return true;
15598 if (EntrySDN == N)
15599 return false;
15600 for (const SDValue &Op : N->op_values()) {
15601 if (N == To && Op.getNode() == EntrySDN) {
15602 // Special case: New node's operand is the entry node; just need to
15603 // copy extra info to new node.
15604 break;
15605 }
15606 if (!Self(Self, Op.getNode()))
15607 return false;
15608 }
15609 // Copy only if entry node was not reached.
15610 SDEI[N] = std::move(NEI);
15611 return true;
15612 };
15613
15614 // We first try with a lower MaxDepth, assuming that the path to common
15615 // operands between From and To is relatively short. This significantly
15616 // improves performance in the common case. The initial MaxDepth is big
15617 // enough to avoid retry in the common case; the last MaxDepth is large
15618 // enough to avoid having to use the fallback below (and protects from
15619 // potential stack exhaustion from recursion).
15620 for (int PrevDepth = 0, MaxDepth = 16; MaxDepth <= 1024;
15621 PrevDepth = MaxDepth, MaxDepth *= 2, Visited.clear()) {
15622 // StartFrom is the previous (or initial) set of leafs reachable at the
15623 // previous maximum depth.
15624 SmallVector<const SDNode *> StartFrom;
15625 std::swap(LHS&: StartFrom, RHS&: Leafs);
15626 for (const SDNode *N : StartFrom)
15627 VisitFrom(VisitFrom, N, MaxDepth - PrevDepth);
15628 if (LLVM_LIKELY(DeepCopyTo(DeepCopyTo, To)))
15629 return;
15630 // This should happen very rarely (reached the entry node).
15631 LLVM_DEBUG(dbgs() << __func__ << ": MaxDepth=" << MaxDepth << " too low\n");
15632 assert(!Leafs.empty());
15633 }
15634
15635 // This should not happen - but if it did, that means the subgraph reachable
15636 // from From has depth greater or equal to maximum MaxDepth, and VisitFrom()
15637 // could not visit all reachable common operands. Consequently, we were able
15638 // to reach the entry node.
15639 errs() << "warning: incomplete propagation of SelectionDAG::NodeExtraInfo\n";
15640 assert(false && "From subgraph too complex - increase max. MaxDepth?");
15641 // Best-effort fallback if assertions disabled.
15642 SDEI[To] = std::move(NEI);
15643}
15644
15645unsigned SelectionDAG::getMaxRuntimeNumElements(EVT VT) const {
15646 assert(VT.isVector() && "Can only unroll vector types!");
15647 if (VT.isFixedLengthVector())
15648 return VT.getVectorNumElements();
15649
15650 const MachineFunction &MF = getMachineFunction();
15651 const Function &F = MF.getFunction();
15652
15653 APInt MaxVScale = getVScaleRange(F: &F, BitWidth: sizeof(unsigned) * 8).getUnsignedMax();
15654 if (MaxVScale.ugt(RHS: getSelectionDAGOptions().vscale_unroll_limit))
15655 return 0;
15656
15657 bool Overflow;
15658 APInt MinNElts(sizeof(unsigned) * 8, VT.getVectorMinNumElements());
15659 APInt MaxNElts = MinNElts.umul_ov(RHS: MaxVScale, Overflow);
15660 if (Overflow)
15661 return 0;
15662
15663 return MaxNElts.getZExtValue();
15664}
15665
15666SDValue SelectionDAG::buildVectorFromUnrolledParts(EVT VT, const SDLoc &DL,
15667 ArrayRef<SDValue> Scalars) {
15668 assert(Scalars.size() == getMaxRuntimeNumElements(VT) &&
15669 "Element count mismatch!");
15670 if (VT.isFixedLengthVector())
15671 return getBuildVector(VT, DL, Ops: Scalars);
15672
15673 SDValue Vec = getPOISON(VT);
15674 // Iterate in reverse so result remains poison until we encounter a lane that
15675 // exists, after which all lower-numbered lanes must also exist.
15676 for (unsigned IdxVal : reverse(C: seq(Size: Scalars.size())))
15677 Vec = getInsertVectorElt(DL, Vec, Elt: Scalars[IdxVal], Idx: IdxVal);
15678
15679 return Vec;
15680}
15681
15682#ifndef NDEBUG
15683static void checkForCyclesHelper(const SDNode *N,
15684 SmallPtrSetImpl<const SDNode*> &Visited,
15685 SmallPtrSetImpl<const SDNode*> &Checked,
15686 const llvm::SelectionDAG *DAG) {
15687 // If this node has already been checked, don't check it again.
15688 if (Checked.count(N))
15689 return;
15690
15691 // If a node has already been visited on this depth-first walk, reject it as
15692 // a cycle.
15693 if (!Visited.insert(N).second) {
15694 errs() << "Detected cycle in SelectionDAG\n";
15695 dbgs() << "Offending node:\n";
15696 N->dumprFull(DAG); dbgs() << "\n";
15697 abort();
15698 }
15699
15700 for (const SDValue &Op : N->op_values())
15701 checkForCyclesHelper(Op.getNode(), Visited, Checked, DAG);
15702
15703 Checked.insert(N);
15704 Visited.erase(N);
15705}
15706#endif
15707
15708void llvm::checkForCycles(const llvm::SDNode *N,
15709 const llvm::SelectionDAG *DAG,
15710 bool force) {
15711#ifndef NDEBUG
15712 bool check = force;
15713#ifdef EXPENSIVE_CHECKS
15714 check = true;
15715#endif // EXPENSIVE_CHECKS
15716 if (check) {
15717 assert(N && "Checking nonexistent SDNode");
15718 SmallPtrSet<const SDNode*, 32> visited;
15719 SmallPtrSet<const SDNode*, 32> checked;
15720 checkForCyclesHelper(N, visited, checked, DAG);
15721 }
15722#endif // !NDEBUG
15723}
15724
15725void llvm::checkForCycles(const llvm::SelectionDAG *DAG, bool force) {
15726 checkForCycles(N: DAG->getRoot().getNode(), DAG, force);
15727}
15728