1//===- SelectionDAG.cpp - Implement the SelectionDAG data structures ------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements the SelectionDAG class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/CodeGen/SelectionDAG.h"
14#include "SDNodeDbgValue.h"
15#include "llvm/ADT/APFloat.h"
16#include "llvm/ADT/APInt.h"
17#include "llvm/ADT/APSInt.h"
18#include "llvm/ADT/ArrayRef.h"
19#include "llvm/ADT/BitVector.h"
20#include "llvm/ADT/DenseSet.h"
21#include "llvm/ADT/FoldingSet.h"
22#include "llvm/ADT/STLExtras.h"
23#include "llvm/ADT/SmallPtrSet.h"
24#include "llvm/ADT/SmallVector.h"
25#include "llvm/ADT/Twine.h"
26#include "llvm/Analysis/AliasAnalysis.h"
27#include "llvm/Analysis/MemoryLocation.h"
28#include "llvm/Analysis/TargetLibraryInfo.h"
29#include "llvm/Analysis/ValueTracking.h"
30#include "llvm/Analysis/VectorUtils.h"
31#include "llvm/BinaryFormat/Dwarf.h"
32#include "llvm/CodeGen/Analysis.h"
33#include "llvm/CodeGen/CodeGenCommonISel.h"
34#include "llvm/CodeGen/FunctionLoweringInfo.h"
35#include "llvm/CodeGen/ISDOpcodes.h"
36#include "llvm/CodeGen/MachineBasicBlock.h"
37#include "llvm/CodeGen/MachineConstantPool.h"
38#include "llvm/CodeGen/MachineFrameInfo.h"
39#include "llvm/CodeGen/MachineFunction.h"
40#include "llvm/CodeGen/MachineMemOperand.h"
41#include "llvm/CodeGen/RuntimeLibcallUtil.h"
42#include "llvm/CodeGen/SDPatternMatch.h"
43#include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
44#include "llvm/CodeGen/SelectionDAGNodes.h"
45#include "llvm/CodeGen/SelectionDAGTargetInfo.h"
46#include "llvm/CodeGen/TargetFrameLowering.h"
47#include "llvm/CodeGen/TargetLowering.h"
48#include "llvm/CodeGen/TargetRegisterInfo.h"
49#include "llvm/CodeGen/TargetSubtargetInfo.h"
50#include "llvm/CodeGen/ValueTypes.h"
51#include "llvm/CodeGenTypes/MachineValueType.h"
52#include "llvm/IR/Constant.h"
53#include "llvm/IR/Constants.h"
54#include "llvm/IR/DataLayout.h"
55#include "llvm/IR/DebugInfoMetadata.h"
56#include "llvm/IR/DebugLoc.h"
57#include "llvm/IR/DerivedTypes.h"
58#include "llvm/IR/Function.h"
59#include "llvm/IR/GlobalValue.h"
60#include "llvm/IR/Metadata.h"
61#include "llvm/IR/Type.h"
62#include "llvm/Support/Casting.h"
63#include "llvm/Support/CodeGen.h"
64#include "llvm/Support/Compiler.h"
65#include "llvm/Support/Debug.h"
66#include "llvm/Support/ErrorHandling.h"
67#include "llvm/Support/KnownBits.h"
68#include "llvm/Support/KnownFPClass.h"
69#include "llvm/Support/MathExtras.h"
70#include "llvm/Support/raw_ostream.h"
71#include "llvm/Target/TargetMachine.h"
72#include "llvm/Target/TargetOptions.h"
73#include "llvm/TargetParser/Triple.h"
74#include "llvm/Transforms/Utils/SizeOpts.h"
75#include <algorithm>
76#include <cassert>
77#include <cstdint>
78#include <cstdlib>
79#include <limits>
80#include <optional>
81#include <string>
82#include <utility>
83#include <vector>
84
85using namespace llvm;
86using namespace llvm::SDPatternMatch;
87
88/// makeVTList - Return an instance of the SDVTList struct initialized with the
89/// specified members.
90static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) {
91 SDVTList Res = {.VTs: VTs, .NumVTs: NumVTs};
92 return Res;
93}
94
95// Default null implementations of the callbacks.
96void SelectionDAG::DAGUpdateListener::NodeDeleted(SDNode*, SDNode*) {}
97void SelectionDAG::DAGUpdateListener::NodeUpdated(SDNode*) {}
98void SelectionDAG::DAGUpdateListener::NodeInserted(SDNode *) {}
99
100void SelectionDAG::DAGNodeDeletedListener::anchor() {}
101void SelectionDAG::DAGNodeInsertedListener::anchor() {}
102
103#define DEBUG_TYPE "selectiondag"
104
105static cl::opt<bool> EnableMemCpyDAGOpt("enable-memcpy-dag-opt",
106 cl::Hidden, cl::init(Val: true),
107 cl::desc("Gang up loads and stores generated by inlining of memcpy"));
108
109static cl::opt<int> MaxLdStGlue("ldstmemcpy-glue-max",
110 cl::desc("Number limit for gluing ld/st of memcpy."),
111 cl::Hidden, cl::init(Val: 0));
112
113static cl::opt<unsigned>
114 MaxSteps("has-predecessor-max-steps", cl::Hidden, cl::init(Val: 8192),
115 cl::desc("DAG combiner limit number of steps when searching DAG "
116 "for predecessor nodes"));
117
118static cl::opt<int> VScaleUnrollLimit(
119 "vscale-unroll-limit",
120 cl::desc("Maximum vscale for which vector unrolling is allowed."),
121 cl::Hidden, cl::init(Val: 64));
122
123static void NewSDValueDbgMsg(SDValue V, StringRef Msg, SelectionDAG *G) {
124 LLVM_DEBUG(dbgs() << Msg; V.getNode()->dump(G););
125}
126
127unsigned SelectionDAG::getHasPredecessorMaxSteps() { return MaxSteps; }
128
129//===----------------------------------------------------------------------===//
130// ConstantFPSDNode Class
131//===----------------------------------------------------------------------===//
132
133/// isExactlyValue - We don't rely on operator== working on double values, as
134/// it returns true for things that are clearly not equal, like -0.0 and 0.0.
135/// As such, this method can be used to do an exact bit-for-bit comparison of
136/// two floating point values.
137bool ConstantFPSDNode::isExactlyValue(const APFloat& V) const {
138 return getValueAPF().bitwiseIsEqual(RHS: V);
139}
140
141bool ConstantFPSDNode::isValueValidForType(EVT VT,
142 const APFloat& Val) {
143 assert(VT.isFloatingPoint() && "Can only convert between FP types");
144
145 // convert modifies in place, so make a copy.
146 APFloat Val2 = APFloat(Val);
147 bool losesInfo;
148 (void)Val2.convert(ToSemantics: VT.getFltSemantics(), RM: APFloat::rmNearestTiesToEven,
149 losesInfo: &losesInfo);
150 return !losesInfo;
151}
152
153//===----------------------------------------------------------------------===//
154// ISD Namespace
155//===----------------------------------------------------------------------===//
156
157bool ISD::isConstantSplatVector(const SDNode *N, APInt &SplatVal) {
158 if (N->getOpcode() == ISD::SPLAT_VECTOR) {
159 if (auto OptAPInt = N->getOperand(Num: 0)->bitcastToAPInt()) {
160 unsigned EltSize =
161 N->getValueType(ResNo: 0).getVectorElementType().getSizeInBits();
162 SplatVal = OptAPInt->trunc(width: EltSize);
163 return true;
164 }
165 }
166
167 auto *BV = dyn_cast<BuildVectorSDNode>(Val: N);
168 if (!BV)
169 return false;
170
171 APInt SplatUndef;
172 unsigned SplatBitSize;
173 bool HasUndefs;
174 unsigned EltSize = N->getValueType(ResNo: 0).getVectorElementType().getSizeInBits();
175 // Endianness does not matter here. We are checking for a splat given the
176 // element size of the vector, and if we find such a splat for little endian
177 // layout, then that should be valid also for big endian (as the full vector
178 // size is known to be a multiple of the element size).
179 const bool IsBigEndian = false;
180 return BV->isConstantSplat(SplatValue&: SplatVal, SplatUndef, SplatBitSize, HasAnyUndefs&: HasUndefs,
181 MinSplatBits: EltSize, isBigEndian: IsBigEndian) &&
182 EltSize == SplatBitSize;
183}
184
185// FIXME: AllOnes and AllZeros duplicate a lot of code. Could these be
186// specializations of the more general isConstantSplatVector()?
187
188bool ISD::isConstantSplatVectorAllOnes(const SDNode *N, bool BuildVectorOnly) {
189 // Look through a bit convert.
190 while (N->getOpcode() == ISD::BITCAST)
191 N = N->getOperand(Num: 0).getNode();
192
193 if (!BuildVectorOnly && N->getOpcode() == ISD::SPLAT_VECTOR) {
194 APInt SplatVal;
195 return isConstantSplatVector(N, SplatVal) && SplatVal.isAllOnes();
196 }
197
198 if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
199
200 unsigned i = 0, e = N->getNumOperands();
201
202 // Skip over all of the undef values.
203 while (i != e && N->getOperand(Num: i).isUndef())
204 ++i;
205
206 // Do not accept an all-undef vector.
207 if (i == e) return false;
208
209 // Do not accept build_vectors that aren't all constants or which have non-~0
210 // elements. We have to be a bit careful here, as the type of the constant
211 // may not be the same as the type of the vector elements due to type
212 // legalization (the elements are promoted to a legal type for the target and
213 // a vector of a type may be legal when the base element type is not).
214 // We only want to check enough bits to cover the vector elements, because
215 // we care if the resultant vector is all ones, not whether the individual
216 // constants are.
217 SDValue NotZero = N->getOperand(Num: i);
218 if (auto OptAPInt = NotZero->bitcastToAPInt()) {
219 unsigned EltSize = N->getValueType(ResNo: 0).getScalarSizeInBits();
220 if (OptAPInt->countr_one() < EltSize)
221 return false;
222 } else
223 return false;
224
225 // Okay, we have at least one ~0 value, check to see if the rest match or are
226 // undefs. Even with the above element type twiddling, this should be OK, as
227 // the same type legalization should have applied to all the elements.
228 for (++i; i != e; ++i)
229 if (N->getOperand(Num: i) != NotZero && !N->getOperand(Num: i).isUndef())
230 return false;
231 return true;
232}
233
234bool ISD::isConstantSplatVectorAllZeros(const SDNode *N, bool BuildVectorOnly) {
235 // Look through a bit convert.
236 while (N->getOpcode() == ISD::BITCAST)
237 N = N->getOperand(Num: 0).getNode();
238
239 if (!BuildVectorOnly && N->getOpcode() == ISD::SPLAT_VECTOR) {
240 APInt SplatVal;
241 return isConstantSplatVector(N, SplatVal) && SplatVal.isZero();
242 }
243
244 if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
245
246 bool IsAllUndef = true;
247 for (const SDValue &Op : N->op_values()) {
248 if (Op.isUndef())
249 continue;
250 IsAllUndef = false;
251 // Do not accept build_vectors that aren't all constants or which have non-0
252 // elements. We have to be a bit careful here, as the type of the constant
253 // may not be the same as the type of the vector elements due to type
254 // legalization (the elements are promoted to a legal type for the target
255 // and a vector of a type may be legal when the base element type is not).
256 // We only want to check enough bits to cover the vector elements, because
257 // we care if the resultant vector is all zeros, not whether the individual
258 // constants are.
259 if (auto OptAPInt = Op->bitcastToAPInt()) {
260 unsigned EltSize = N->getValueType(ResNo: 0).getScalarSizeInBits();
261 if (OptAPInt->countr_zero() < EltSize)
262 return false;
263 } else
264 return false;
265 }
266
267 // Do not accept an all-undef vector.
268 if (IsAllUndef)
269 return false;
270 return true;
271}
272
273bool ISD::isBuildVectorAllOnes(const SDNode *N) {
274 return isConstantSplatVectorAllOnes(N, /*BuildVectorOnly*/ true);
275}
276
277bool ISD::isBuildVectorAllZeros(const SDNode *N) {
278 return isConstantSplatVectorAllZeros(N, /*BuildVectorOnly*/ true);
279}
280
281bool ISD::isBuildVectorOfConstantSDNodes(const SDNode *N) {
282 if (N->getOpcode() != ISD::BUILD_VECTOR)
283 return false;
284
285 for (const SDValue &Op : N->op_values()) {
286 if (Op.isUndef())
287 continue;
288 if (!isa<ConstantSDNode>(Val: Op))
289 return false;
290 }
291 return true;
292}
293
294bool ISD::isBuildVectorOfConstantFPSDNodes(const SDNode *N) {
295 if (N->getOpcode() != ISD::BUILD_VECTOR)
296 return false;
297
298 for (const SDValue &Op : N->op_values()) {
299 if (Op.isUndef())
300 continue;
301 if (!isa<ConstantFPSDNode>(Val: Op))
302 return false;
303 }
304 return true;
305}
306
307bool ISD::isVectorShrinkable(const SDNode *N, unsigned NewEltSize,
308 bool Signed) {
309 assert(N->getValueType(0).isVector() && "Expected a vector!");
310
311 unsigned EltSize = N->getValueType(ResNo: 0).getScalarSizeInBits();
312 if (EltSize <= NewEltSize)
313 return false;
314
315 if (N->getOpcode() == ISD::ZERO_EXTEND) {
316 return (N->getOperand(Num: 0).getValueType().getScalarSizeInBits() <=
317 NewEltSize) &&
318 !Signed;
319 }
320 if (N->getOpcode() == ISD::SIGN_EXTEND) {
321 return (N->getOperand(Num: 0).getValueType().getScalarSizeInBits() <=
322 NewEltSize) &&
323 Signed;
324 }
325 if (N->getOpcode() != ISD::BUILD_VECTOR)
326 return false;
327
328 for (const SDValue &Op : N->op_values()) {
329 if (Op.isUndef())
330 continue;
331 if (!isa<ConstantSDNode>(Val: Op))
332 return false;
333
334 APInt C = Op->getAsAPIntVal().trunc(width: EltSize);
335 if (Signed && C.trunc(width: NewEltSize).sext(width: EltSize) != C)
336 return false;
337 if (!Signed && C.trunc(width: NewEltSize).zext(width: EltSize) != C)
338 return false;
339 }
340
341 return true;
342}
343
344bool ISD::allOperandsUndef(const SDNode *N) {
345 // Return false if the node has no operands.
346 // This is "logically inconsistent" with the definition of "all" but
347 // is probably the desired behavior.
348 if (N->getNumOperands() == 0)
349 return false;
350 return all_of(Range: N->op_values(), P: [](SDValue Op) { return Op.isUndef(); });
351}
352
353bool ISD::isFreezeUndef(const SDNode *N) {
354 return N->getOpcode() == ISD::FREEZE && N->getOperand(Num: 0).isUndef();
355}
356
357template <typename ConstNodeType>
358bool ISD::matchUnaryPredicateImpl(SDValue Op, const APInt &DemandedElts,
359 std::function<bool(ConstNodeType *)> Match,
360 bool AllowUndefs, bool AllowTruncation) {
361 // FIXME: Add support for scalar UNDEF cases?
362 if (auto *C = dyn_cast<ConstNodeType>(Op))
363 return Match(C);
364
365 // FIXME: Add support for vector UNDEF cases?
366 if (ISD::BUILD_VECTOR != Op.getOpcode() &&
367 ISD::SPLAT_VECTOR != Op.getOpcode())
368 return false;
369
370 if (ISD::SPLAT_VECTOR == Op.getOpcode() && !DemandedElts)
371 return true;
372
373 EVT SVT = Op.getValueType().getScalarType();
374 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
375 if (ISD::SPLAT_VECTOR != Op.getOpcode() && !DemandedElts[i])
376 continue;
377
378 if (AllowUndefs && Op.getOperand(i).isUndef()) {
379 if (!Match(nullptr))
380 return false;
381 continue;
382 }
383
384 auto *Cst = dyn_cast<ConstNodeType>(Op.getOperand(i));
385 if (!Cst || (!AllowTruncation && Cst->getValueType(0) != SVT) ||
386 !Match(Cst))
387 return false;
388 }
389 return true;
390}
391// Build used template types.
392template bool ISD::matchUnaryPredicateImpl<ConstantSDNode>(
393 SDValue, const APInt &, std::function<bool(ConstantSDNode *)>, bool, bool);
394template bool ISD::matchUnaryPredicateImpl<ConstantFPSDNode>(
395 SDValue, const APInt &, std::function<bool(ConstantFPSDNode *)>, bool,
396 bool);
397
398bool ISD::matchBinaryPredicate(
399 SDValue LHS, SDValue RHS, const APInt &DemandedElts,
400 std::function<bool(ConstantSDNode *, ConstantSDNode *)> Match,
401 bool AllowUndefs, bool AllowTypeMismatch) {
402 if (!AllowTypeMismatch && LHS.getValueType() != RHS.getValueType())
403 return false;
404
405 // TODO: Add support for scalar UNDEF cases?
406 if (auto *LHSCst = dyn_cast<ConstantSDNode>(Val&: LHS))
407 if (auto *RHSCst = dyn_cast<ConstantSDNode>(Val&: RHS))
408 return Match(LHSCst, RHSCst);
409
410 // TODO: Add support for vector UNDEF cases?
411 if (LHS.getOpcode() != RHS.getOpcode() ||
412 (LHS.getOpcode() != ISD::BUILD_VECTOR &&
413 LHS.getOpcode() != ISD::SPLAT_VECTOR))
414 return false;
415
416 if (ISD::SPLAT_VECTOR == LHS.getOpcode() && !DemandedElts)
417 return true;
418
419 EVT SVT = LHS.getValueType().getScalarType();
420 for (unsigned i = 0, e = LHS.getNumOperands(); i != e; ++i) {
421 if (ISD::SPLAT_VECTOR != LHS.getOpcode() && !DemandedElts[i])
422 continue;
423 SDValue LHSOp = LHS.getOperand(i);
424 SDValue RHSOp = RHS.getOperand(i);
425 bool LHSUndef = AllowUndefs && LHSOp.isUndef();
426 bool RHSUndef = AllowUndefs && RHSOp.isUndef();
427 auto *LHSCst = dyn_cast<ConstantSDNode>(Val&: LHSOp);
428 auto *RHSCst = dyn_cast<ConstantSDNode>(Val&: RHSOp);
429 if ((!LHSCst && !LHSUndef) || (!RHSCst && !RHSUndef))
430 return false;
431 if (!AllowTypeMismatch && (LHSOp.getValueType() != SVT ||
432 LHSOp.getValueType() != RHSOp.getValueType()))
433 return false;
434 if (!Match(LHSCst, RHSCst))
435 return false;
436 }
437 return true;
438}
439
440ISD::NodeType ISD::getInverseMinMaxOpcode(unsigned MinMaxOpc) {
441 switch (MinMaxOpc) {
442 default:
443 llvm_unreachable("unrecognized opcode");
444 case ISD::UMIN:
445 return ISD::UMAX;
446 case ISD::UMAX:
447 return ISD::UMIN;
448 case ISD::SMIN:
449 return ISD::SMAX;
450 case ISD::SMAX:
451 return ISD::SMIN;
452 }
453}
454
455ISD::NodeType ISD::getOppositeSignednessMinMaxOpcode(unsigned MinMaxOpc) {
456 switch (MinMaxOpc) {
457 default:
458 llvm_unreachable("unrecognized min/max opcode");
459 case ISD::SMIN:
460 return ISD::UMIN;
461 case ISD::SMAX:
462 return ISD::UMAX;
463 case ISD::UMIN:
464 return ISD::SMIN;
465 case ISD::UMAX:
466 return ISD::SMAX;
467 }
468}
469
470ISD::NodeType ISD::getVecReduceBaseOpcode(unsigned VecReduceOpcode) {
471 switch (VecReduceOpcode) {
472 default:
473 llvm_unreachable("Expected VECREDUCE opcode");
474 case ISD::VECREDUCE_FADD:
475 case ISD::VECREDUCE_SEQ_FADD:
476 case ISD::VP_REDUCE_FADD:
477 case ISD::VP_REDUCE_SEQ_FADD:
478 return ISD::FADD;
479 case ISD::VECREDUCE_FMUL:
480 case ISD::VECREDUCE_SEQ_FMUL:
481 case ISD::VP_REDUCE_FMUL:
482 case ISD::VP_REDUCE_SEQ_FMUL:
483 return ISD::FMUL;
484 case ISD::VECREDUCE_ADD:
485 case ISD::VP_REDUCE_ADD:
486 return ISD::ADD;
487 case ISD::VECREDUCE_MUL:
488 case ISD::VP_REDUCE_MUL:
489 return ISD::MUL;
490 case ISD::VECREDUCE_AND:
491 case ISD::VP_REDUCE_AND:
492 return ISD::AND;
493 case ISD::VECREDUCE_OR:
494 case ISD::VP_REDUCE_OR:
495 return ISD::OR;
496 case ISD::VECREDUCE_XOR:
497 case ISD::VP_REDUCE_XOR:
498 return ISD::XOR;
499 case ISD::VECREDUCE_SMAX:
500 case ISD::VP_REDUCE_SMAX:
501 return ISD::SMAX;
502 case ISD::VECREDUCE_SMIN:
503 case ISD::VP_REDUCE_SMIN:
504 return ISD::SMIN;
505 case ISD::VECREDUCE_UMAX:
506 case ISD::VP_REDUCE_UMAX:
507 return ISD::UMAX;
508 case ISD::VECREDUCE_UMIN:
509 case ISD::VP_REDUCE_UMIN:
510 return ISD::UMIN;
511 case ISD::VECREDUCE_FMAX:
512 case ISD::VP_REDUCE_FMAX:
513 return ISD::FMAXNUM;
514 case ISD::VECREDUCE_FMIN:
515 case ISD::VP_REDUCE_FMIN:
516 return ISD::FMINNUM;
517 case ISD::VECREDUCE_FMAXIMUM:
518 case ISD::VP_REDUCE_FMAXIMUM:
519 return ISD::FMAXIMUM;
520 case ISD::VECREDUCE_FMINIMUM:
521 case ISD::VP_REDUCE_FMINIMUM:
522 return ISD::FMINIMUM;
523 case ISD::VECREDUCE_FMAXIMUMNUM:
524 return ISD::FMAXIMUMNUM;
525 case ISD::VECREDUCE_FMINIMUMNUM:
526 return ISD::FMINIMUMNUM;
527 }
528}
529
530ISD::NodeType ISD::getUnmaskedBinOpOpcode(unsigned MaskedOpc) {
531 switch (MaskedOpc) {
532 case ISD::MASKED_UDIV:
533 return ISD::UDIV;
534 case ISD::MASKED_SDIV:
535 return ISD::SDIV;
536 case ISD::MASKED_UREM:
537 return ISD::UREM;
538 case ISD::MASKED_SREM:
539 return ISD::SREM;
540 default:
541 llvm_unreachable("Expected masked binop opcode");
542 }
543}
544
545bool ISD::isVPOpcode(unsigned Opcode) {
546 switch (Opcode) {
547 default:
548 return false;
549#define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) \
550 case ISD::VPSD: \
551 return true;
552#include "llvm/IR/VPIntrinsics.def"
553 }
554}
555
556bool ISD::isVPBinaryOp(unsigned Opcode) {
557 switch (Opcode) {
558 default:
559 break;
560#define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) case ISD::VPSD:
561#define VP_PROPERTY_BINARYOP return true;
562#define END_REGISTER_VP_SDNODE(VPSD) break;
563#include "llvm/IR/VPIntrinsics.def"
564 }
565 return false;
566}
567
568bool ISD::isVPReduction(unsigned Opcode) {
569 switch (Opcode) {
570 default:
571 return false;
572 case ISD::VP_REDUCE_ADD:
573 case ISD::VP_REDUCE_MUL:
574 case ISD::VP_REDUCE_AND:
575 case ISD::VP_REDUCE_OR:
576 case ISD::VP_REDUCE_XOR:
577 case ISD::VP_REDUCE_SMAX:
578 case ISD::VP_REDUCE_SMIN:
579 case ISD::VP_REDUCE_UMAX:
580 case ISD::VP_REDUCE_UMIN:
581 case ISD::VP_REDUCE_FMAX:
582 case ISD::VP_REDUCE_FMIN:
583 case ISD::VP_REDUCE_FMAXIMUM:
584 case ISD::VP_REDUCE_FMINIMUM:
585 case ISD::VP_REDUCE_FADD:
586 case ISD::VP_REDUCE_FMUL:
587 case ISD::VP_REDUCE_SEQ_FADD:
588 case ISD::VP_REDUCE_SEQ_FMUL:
589 return true;
590 }
591}
592
593/// The operand position of the vector mask.
594std::optional<unsigned> ISD::getVPMaskIdx(unsigned Opcode) {
595 switch (Opcode) {
596 default:
597 return std::nullopt;
598#define BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, TDNAME, MASKPOS, ...) \
599 case ISD::VPSD: \
600 return MASKPOS;
601#include "llvm/IR/VPIntrinsics.def"
602 }
603}
604
605/// The operand position of the explicit vector length parameter.
606std::optional<unsigned> ISD::getVPExplicitVectorLengthIdx(unsigned Opcode) {
607 switch (Opcode) {
608 default:
609 return std::nullopt;
610#define BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, TDNAME, MASKPOS, EVLPOS) \
611 case ISD::VPSD: \
612 return EVLPOS;
613#include "llvm/IR/VPIntrinsics.def"
614 }
615}
616
617std::optional<unsigned> ISD::getBaseOpcodeForVP(unsigned VPOpcode,
618 bool hasFPExcept) {
619 // FIXME: Return strict opcodes in case of fp exceptions.
620 switch (VPOpcode) {
621 default:
622 return std::nullopt;
623#define BEGIN_REGISTER_VP_SDNODE(VPOPC, ...) case ISD::VPOPC:
624#define VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) return ISD::SDOPC;
625#define END_REGISTER_VP_SDNODE(VPOPC) break;
626#include "llvm/IR/VPIntrinsics.def"
627 }
628 return std::nullopt;
629}
630
631std::optional<unsigned> ISD::getVPForBaseOpcode(unsigned Opcode) {
632 switch (Opcode) {
633 default:
634 return std::nullopt;
635#define BEGIN_REGISTER_VP_SDNODE(VPOPC, ...) break;
636#define VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) case ISD::SDOPC:
637#define END_REGISTER_VP_SDNODE(VPOPC) return ISD::VPOPC;
638#include "llvm/IR/VPIntrinsics.def"
639 }
640}
641
642ISD::NodeType ISD::getExtForLoadExtType(bool IsFP, ISD::LoadExtType ExtType) {
643 switch (ExtType) {
644 case ISD::EXTLOAD:
645 return IsFP ? ISD::FP_EXTEND : ISD::ANY_EXTEND;
646 case ISD::SEXTLOAD:
647 return ISD::SIGN_EXTEND;
648 case ISD::ZEXTLOAD:
649 return ISD::ZERO_EXTEND;
650 default:
651 break;
652 }
653
654 llvm_unreachable("Invalid LoadExtType");
655}
656
657ISD::CondCode ISD::getSetCCSwappedOperands(ISD::CondCode Operation) {
658 // To perform this operation, we just need to swap the L and G bits of the
659 // operation.
660 unsigned OldL = (Operation >> 2) & 1;
661 unsigned OldG = (Operation >> 1) & 1;
662 return ISD::CondCode((Operation & ~6) | // Keep the N, U, E bits
663 (OldL << 1) | // New G bit
664 (OldG << 2)); // New L bit.
665}
666
667static ISD::CondCode getSetCCInverseImpl(ISD::CondCode Op, bool isIntegerLike) {
668 unsigned Operation = Op;
669 if (isIntegerLike)
670 Operation ^= 7; // Flip L, G, E bits, but not U.
671 else
672 Operation ^= 15; // Flip all of the condition bits.
673
674 if (Operation > ISD::SETTRUE2)
675 Operation &= ~8; // Don't let N and U bits get set.
676
677 return ISD::CondCode(Operation);
678}
679
680ISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, EVT Type) {
681 return getSetCCInverseImpl(Op, isIntegerLike: Type.isInteger());
682}
683
684ISD::CondCode ISD::GlobalISel::getSetCCInverse(ISD::CondCode Op,
685 bool isIntegerLike) {
686 return getSetCCInverseImpl(Op, isIntegerLike);
687}
688
689/// For an integer comparison, return 1 if the comparison is a signed operation
690/// and 2 if the result is an unsigned comparison. Return zero if the operation
691/// does not depend on the sign of the input (setne and seteq).
692static int isSignedOp(ISD::CondCode Opcode) {
693 switch (Opcode) {
694 default: llvm_unreachable("Illegal integer setcc operation!");
695 case ISD::SETEQ:
696 case ISD::SETNE: return 0;
697 case ISD::SETLT:
698 case ISD::SETLE:
699 case ISD::SETGT:
700 case ISD::SETGE: return 1;
701 case ISD::SETULT:
702 case ISD::SETULE:
703 case ISD::SETUGT:
704 case ISD::SETUGE: return 2;
705 }
706}
707
708ISD::CondCode ISD::getSetCCOrOperation(ISD::CondCode Op1, ISD::CondCode Op2,
709 EVT Type) {
710 bool IsInteger = Type.isInteger();
711 if (IsInteger && (isSignedOp(Opcode: Op1) | isSignedOp(Opcode: Op2)) == 3)
712 // Cannot fold a signed integer setcc with an unsigned integer setcc.
713 return ISD::SETCC_INVALID;
714
715 unsigned Op = Op1 | Op2; // Combine all of the condition bits.
716
717 // If the N and U bits get set, then the resultant comparison DOES suddenly
718 // care about orderedness, and it is true when ordered.
719 if (Op > ISD::SETTRUE2)
720 Op &= ~16; // Clear the U bit if the N bit is set.
721
722 // Canonicalize illegal integer setcc's.
723 if (IsInteger && Op == ISD::SETUNE) // e.g. SETUGT | SETULT
724 Op = ISD::SETNE;
725
726 return ISD::CondCode(Op);
727}
728
729ISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2,
730 EVT Type) {
731 bool IsInteger = Type.isInteger();
732 if (IsInteger && (isSignedOp(Opcode: Op1) | isSignedOp(Opcode: Op2)) == 3)
733 // Cannot fold a signed setcc with an unsigned setcc.
734 return ISD::SETCC_INVALID;
735
736 // Combine all of the condition bits.
737 ISD::CondCode Result = ISD::CondCode(Op1 & Op2);
738
739 // Canonicalize illegal integer setcc's.
740 if (IsInteger) {
741 switch (Result) {
742 default: break;
743 case ISD::SETUO : Result = ISD::SETFALSE; break; // SETUGT & SETULT
744 case ISD::SETOEQ: // SETEQ & SETU[LG]E
745 case ISD::SETUEQ: Result = ISD::SETEQ ; break; // SETUGE & SETULE
746 case ISD::SETOLT: Result = ISD::SETULT ; break; // SETULT & SETNE
747 case ISD::SETOGT: Result = ISD::SETUGT ; break; // SETUGT & SETNE
748 }
749 }
750
751 return Result;
752}
753
754//===----------------------------------------------------------------------===//
755// SDNode Key Support
756//===----------------------------------------------------------------------===//
757
758/// If this is an SDNode with special info, add this info to the NodeID data.
759/// MorphNodeTo passes the opcode \p N is becoming, so \p Opc may differ from
760/// N->getOpcode() and a case may only cast to a class \p N already is.
761static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N,
762 unsigned Opc) {
763 // Machine nodes are uniqued by opcode, value types and operands alone, as in
764 // getMachineNode; MorphNodeTo's clearMemRefs overwrites the fields below.
765 if (SDNode::isMachineOpcode(Opc))
766 return;
767
768 switch (Opc) {
769 case ISD::TargetExternalSymbol:
770 case ISD::ExternalSymbol:
771 case ISD::MCSymbol:
772 llvm_unreachable("Should only be used on nodes with operands");
773 default: break; // Normal nodes don't need extra info.
774 case ISD::TargetConstant:
775 case ISD::Constant: {
776 const ConstantSDNode *C = cast<ConstantSDNode>(Val: N);
777 ID.AddPointer(Ptr: C->getConstantIntValue());
778 ID.AddBoolean(B: C->isOpaque());
779 break;
780 }
781 case ISD::TargetConstantFP:
782 case ISD::ConstantFP:
783 ID.AddPointer(Ptr: cast<ConstantFPSDNode>(Val: N)->getConstantFPValue());
784 break;
785 case ISD::TargetGlobalAddress:
786 case ISD::GlobalAddress:
787 case ISD::TargetGlobalTLSAddress:
788 case ISD::GlobalTLSAddress: {
789 const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Val: N);
790 ID.AddPointer(Ptr: GA->getGlobal());
791 ID.AddInteger(I: GA->getOffset());
792 ID.AddInteger(I: GA->getTargetFlags());
793 break;
794 }
795 case ISD::BasicBlock:
796 ID.AddPointer(Ptr: cast<BasicBlockSDNode>(Val: N)->getBasicBlock());
797 break;
798 case ISD::EH_LABEL:
799 case ISD::ANNOTATION_LABEL:
800 ID.AddPointer(Ptr: cast<LabelSDNode>(Val: N)->getLabel());
801 break;
802 case ISD::DEACTIVATION_SYMBOL:
803 ID.AddPointer(Ptr: cast<DeactivationSymbolSDNode>(Val: N)->getGlobal());
804 break;
805 case ISD::Register:
806 ID.AddInteger(I: cast<RegisterSDNode>(Val: N)->getReg().id());
807 break;
808 case ISD::RegisterMask:
809 ID.AddPointer(Ptr: cast<RegisterMaskSDNode>(Val: N)->getRegMask());
810 break;
811 case ISD::SRCVALUE:
812 ID.AddPointer(Ptr: cast<SrcValueSDNode>(Val: N)->getValue());
813 break;
814 case ISD::FrameIndex:
815 case ISD::TargetFrameIndex:
816 ID.AddInteger(I: cast<FrameIndexSDNode>(Val: N)->getIndex());
817 break;
818 case ISD::PSEUDO_PROBE:
819 ID.AddInteger(I: cast<PseudoProbeSDNode>(Val: N)->getGuid());
820 ID.AddInteger(I: cast<PseudoProbeSDNode>(Val: N)->getIndex());
821 ID.AddInteger(I: cast<PseudoProbeSDNode>(Val: N)->getAttributes());
822 break;
823 case ISD::JumpTable:
824 case ISD::TargetJumpTable:
825 ID.AddInteger(I: cast<JumpTableSDNode>(Val: N)->getIndex());
826 ID.AddInteger(I: cast<JumpTableSDNode>(Val: N)->getTargetFlags());
827 break;
828 case ISD::ConstantPool:
829 case ISD::TargetConstantPool: {
830 const ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Val: N);
831 ID.AddInteger(I: CP->getAlign().value());
832 ID.AddInteger(I: CP->getOffset());
833 if (CP->isMachineConstantPoolEntry())
834 CP->getMachineCPVal()->addSelectionDAGCSEId(ID);
835 else
836 ID.AddPointer(Ptr: CP->getConstVal());
837 ID.AddInteger(I: CP->getTargetFlags());
838 break;
839 }
840 case ISD::TargetIndex: {
841 const TargetIndexSDNode *TI = cast<TargetIndexSDNode>(Val: N);
842 ID.AddInteger(I: TI->getIndex());
843 ID.AddInteger(I: TI->getOffset());
844 ID.AddInteger(I: TI->getTargetFlags());
845 break;
846 }
847 case ISD::LOAD: {
848 const LoadSDNode *LD = cast<LoadSDNode>(Val: N);
849 ID.AddInteger(I: LD->getMemoryVT().getRawBits());
850 ID.AddInteger(I: LD->getRawSubclassData());
851 ID.AddInteger(I: LD->getPointerInfo().getAddrSpace());
852 ID.AddInteger(I: LD->getMemOperand()->getFlags());
853 break;
854 }
855 case ISD::STORE: {
856 const StoreSDNode *ST = cast<StoreSDNode>(Val: N);
857 ID.AddInteger(I: ST->getMemoryVT().getRawBits());
858 ID.AddInteger(I: ST->getRawSubclassData());
859 ID.AddInteger(I: ST->getPointerInfo().getAddrSpace());
860 ID.AddInteger(I: ST->getMemOperand()->getFlags());
861 break;
862 }
863 case ISD::VP_LOAD: {
864 const VPLoadSDNode *ELD = cast<VPLoadSDNode>(Val: N);
865 ID.AddInteger(I: ELD->getMemoryVT().getRawBits());
866 ID.AddInteger(I: ELD->getRawSubclassData());
867 ID.AddInteger(I: ELD->getPointerInfo().getAddrSpace());
868 ID.AddInteger(I: ELD->getMemOperand()->getFlags());
869 break;
870 }
871 case ISD::VP_LOAD_FF: {
872 const auto *LD = cast<VPLoadFFSDNode>(Val: N);
873 ID.AddInteger(I: LD->getMemoryVT().getRawBits());
874 ID.AddInteger(I: LD->getRawSubclassData());
875 ID.AddInteger(I: LD->getPointerInfo().getAddrSpace());
876 ID.AddInteger(I: LD->getMemOperand()->getFlags());
877 break;
878 }
879 case ISD::VP_STORE: {
880 const VPStoreSDNode *EST = cast<VPStoreSDNode>(Val: N);
881 ID.AddInteger(I: EST->getMemoryVT().getRawBits());
882 ID.AddInteger(I: EST->getRawSubclassData());
883 ID.AddInteger(I: EST->getPointerInfo().getAddrSpace());
884 ID.AddInteger(I: EST->getMemOperand()->getFlags());
885 break;
886 }
887 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD: {
888 const VPStridedLoadSDNode *SLD = cast<VPStridedLoadSDNode>(Val: N);
889 ID.AddInteger(I: SLD->getMemoryVT().getRawBits());
890 ID.AddInteger(I: SLD->getRawSubclassData());
891 ID.AddInteger(I: SLD->getPointerInfo().getAddrSpace());
892 break;
893 }
894 case ISD::EXPERIMENTAL_VP_STRIDED_STORE: {
895 const VPStridedStoreSDNode *SST = cast<VPStridedStoreSDNode>(Val: N);
896 ID.AddInteger(I: SST->getMemoryVT().getRawBits());
897 ID.AddInteger(I: SST->getRawSubclassData());
898 ID.AddInteger(I: SST->getPointerInfo().getAddrSpace());
899 break;
900 }
901 case ISD::VP_GATHER: {
902 const VPGatherSDNode *EG = cast<VPGatherSDNode>(Val: N);
903 ID.AddInteger(I: EG->getMemoryVT().getRawBits());
904 ID.AddInteger(I: EG->getRawSubclassData());
905 ID.AddInteger(I: EG->getPointerInfo().getAddrSpace());
906 ID.AddInteger(I: EG->getMemOperand()->getFlags());
907 break;
908 }
909 case ISD::VP_SCATTER: {
910 const VPScatterSDNode *ES = cast<VPScatterSDNode>(Val: N);
911 ID.AddInteger(I: ES->getMemoryVT().getRawBits());
912 ID.AddInteger(I: ES->getRawSubclassData());
913 ID.AddInteger(I: ES->getPointerInfo().getAddrSpace());
914 ID.AddInteger(I: ES->getMemOperand()->getFlags());
915 break;
916 }
917 case ISD::MLOAD: {
918 const MaskedLoadSDNode *MLD = cast<MaskedLoadSDNode>(Val: N);
919 ID.AddInteger(I: MLD->getMemoryVT().getRawBits());
920 ID.AddInteger(I: MLD->getRawSubclassData());
921 ID.AddInteger(I: MLD->getPointerInfo().getAddrSpace());
922 ID.AddInteger(I: MLD->getMemOperand()->getFlags());
923 break;
924 }
925 case ISD::MSTORE: {
926 const MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(Val: N);
927 ID.AddInteger(I: MST->getMemoryVT().getRawBits());
928 ID.AddInteger(I: MST->getRawSubclassData());
929 ID.AddInteger(I: MST->getPointerInfo().getAddrSpace());
930 ID.AddInteger(I: MST->getMemOperand()->getFlags());
931 break;
932 }
933 case ISD::MGATHER: {
934 const MaskedGatherSDNode *MG = cast<MaskedGatherSDNode>(Val: N);
935 ID.AddInteger(I: MG->getMemoryVT().getRawBits());
936 ID.AddInteger(I: MG->getRawSubclassData());
937 ID.AddInteger(I: MG->getPointerInfo().getAddrSpace());
938 ID.AddInteger(I: MG->getMemOperand()->getFlags());
939 break;
940 }
941 case ISD::MSCATTER:
942 case ISD::GET_FPENV_MEM:
943 case ISD::SET_FPENV_MEM:
944 case ISD::EXPERIMENTAL_VECTOR_HISTOGRAM: {
945 const MemSDNode *M = cast<MemSDNode>(Val: N);
946 ID.AddInteger(I: M->getMemoryVT().getRawBits());
947 ID.AddInteger(I: M->getRawSubclassData());
948 ID.AddInteger(I: M->getPointerInfo().getAddrSpace());
949 ID.AddInteger(I: M->getMemOperand()->getFlags());
950 break;
951 }
952 case ISD::VECTOR_SHUFFLE: {
953 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Val: N)->getMask();
954 for (int M : Mask)
955 ID.AddInteger(I: M);
956 break;
957 }
958 case ISD::ADDRSPACECAST: {
959 const AddrSpaceCastSDNode *ASC = cast<AddrSpaceCastSDNode>(Val: N);
960 ID.AddInteger(I: ASC->getSrcAddressSpace());
961 ID.AddInteger(I: ASC->getDestAddressSpace());
962 break;
963 }
964 case ISD::TargetBlockAddress:
965 case ISD::BlockAddress: {
966 const BlockAddressSDNode *BA = cast<BlockAddressSDNode>(Val: N);
967 ID.AddPointer(Ptr: BA->getBlockAddress());
968 ID.AddInteger(I: BA->getOffset());
969 ID.AddInteger(I: BA->getTargetFlags());
970 break;
971 }
972 case ISD::AssertAlign:
973 ID.AddInteger(I: cast<AssertAlignSDNode>(Val: N)->getAlign().value());
974 break;
975 case ISD::PREFETCH:
976 case ISD::INTRINSIC_VOID:
977 case ISD::INTRINSIC_W_CHAIN:
978 // Handled by MemIntrinsicSDNode check after the switch.
979 break;
980 case ISD::MDNODE_SDNODE:
981 ID.AddPointer(Ptr: cast<MDNodeSDNode>(Val: N)->getMD());
982 break;
983 } // end switch (Opc)
984
985 if (auto *AT = dyn_cast<AtomicSDNode>(Val: N)) {
986 ID.AddInteger(I: AT->getMemoryVT().getRawBits());
987 ID.AddInteger(I: AT->getRawSubclassData());
988 ID.AddInteger(I: AT->getPointerInfo().getAddrSpace());
989 ID.AddInteger(I: AT->getMemOperand()->getFlags());
990 }
991
992 // MemIntrinsic nodes could also have subclass data, address spaces, and flags
993 // to check.
994 if (auto *MN = dyn_cast<MemIntrinsicSDNode>(Val: N)) {
995 ID.AddInteger(I: MN->getRawSubclassData());
996 ID.AddInteger(I: MN->getMemoryVT().getRawBits());
997 for (const MachineMemOperand *MMO : MN->memoperands()) {
998 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
999 ID.AddInteger(I: MMO->getFlags());
1000 }
1001 }
1002}
1003
1004SDNodeKey::SDNodeKey(const SDNode &N)
1005 : Opcode(N.getOpcode()), VTs(N.getVTList().VTs),
1006 OpStorage(N.op_begin(), N.op_end()) {
1007 Ops = OpStorage;
1008 AddNodeIDCustom(ID&: Tail, N: &N, Opc: N.getOpcode());
1009}
1010
1011#ifndef NDEBUG
1012/// Serialize a node the way SDNodes were uniqued before SDNodeKey, to
1013/// cross-check the typed comparison against. Deliberately reads the node
1014/// rather than going through SDNodeKey, so the key constructor is checked too
1015/// and not used as its own oracle.
1016static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) {
1017 ID.AddInteger(N->getOpcode());
1018 ID.AddPointer(N->getVTList().VTs);
1019 for (const SDUse &Op : N->ops()) {
1020 ID.AddPointer(Op.getNode());
1021 ID.AddInteger(Op.getResNo());
1022 }
1023 AddNodeIDCustom(ID, N, N->getOpcode());
1024}
1025
1026static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNodeKey &Key) {
1027 ID.AddInteger(Key.Opcode);
1028 ID.AddPointer(Key.VTs);
1029 for (const SDValue &Op : Key.Ops) {
1030 ID.AddPointer(Op.getNode());
1031 ID.AddInteger(Op.getResNo());
1032 }
1033 ID.AddNodeID(Key.Tail);
1034}
1035#endif
1036
1037static bool keyMatches(const SDNodeKey &Key, const SDNode &N) {
1038 if (N.getOpcode() != Key.Opcode || N.getVTList().VTs != Key.VTs)
1039 return false;
1040 if (!llvm::equal(LRange: N.ops(), RRange: Key.Ops))
1041 return false;
1042 // Build only what AddNodeIDCustom would have added, which is nothing for
1043 // most opcodes, rather than the whole profile.
1044 FoldingSetNodeID Tail;
1045 AddNodeIDCustom(ID&: Tail, N: &N, Opc: N.getOpcode());
1046 return Tail == Key.Tail;
1047}
1048
1049bool SDNodeKeyInfo::isEqual(const SDNodeKey &Key, const SDNode &N) {
1050 bool Result = keyMatches(Key, N);
1051
1052#ifndef NDEBUG
1053 FoldingSetNodeID NodeProfile, KeyProfile;
1054 AddNodeIDNode(NodeProfile, &N);
1055 AddNodeIDNode(KeyProfile, Key);
1056 assert((NodeProfile == KeyProfile) == Result &&
1057 "SDNodeKey equality disagrees with profile");
1058#endif
1059 return Result;
1060}
1061
1062//===----------------------------------------------------------------------===//
1063// SelectionDAG Class
1064//===----------------------------------------------------------------------===//
1065
1066/// doNotCSE - Return true if CSE should not be performed for this node.
1067static bool doNotCSE(SDNode *N) {
1068 if (N->getValueType(ResNo: 0) == MVT::Glue)
1069 return true; // Never CSE anything that produces a glue result.
1070
1071 switch (N->getOpcode()) {
1072 default: break;
1073 case ISD::HANDLENODE:
1074 case ISD::EH_LABEL:
1075 return true; // Never CSE these nodes.
1076 }
1077
1078 // Check that remaining values produced are not flags.
1079 for (unsigned i = 1, e = N->getNumValues(); i != e; ++i)
1080 if (N->getValueType(ResNo: i) == MVT::Glue)
1081 return true; // Never CSE anything that produces a glue result.
1082
1083 return false;
1084}
1085
1086/// Construct a DemandedElts mask which demands all elements of \p V.
1087/// If \p V is not a fixed-length vector, then this will return a single bit.
1088static APInt getDemandAllEltsMask(SDValue V) {
1089 EVT VT = V.getValueType();
1090 // Since the number of lanes in a scalable vector is unknown at compile time,
1091 // we track one bit which is implicitly broadcast to all lanes. This means
1092 // that all lanes in a scalable vector are considered demanded.
1093 return VT.isFixedLengthVector() ? APInt::getAllOnes(numBits: VT.getVectorNumElements())
1094 : APInt(1, 1);
1095}
1096
1097/// RemoveDeadNodes - This method deletes all unreachable nodes in the
1098/// SelectionDAG.
1099void SelectionDAG::RemoveDeadNodes() {
1100 // Create a dummy node (which is not added to allnodes), that adds a reference
1101 // to the root node, preventing it from being deleted.
1102 HandleSDNode Dummy(getRoot());
1103
1104 SmallVector<SDNode*, 128> DeadNodes;
1105
1106 // Add all obviously-dead nodes to the DeadNodes worklist.
1107 for (SDNode &Node : allnodes())
1108 if (Node.use_empty())
1109 DeadNodes.push_back(Elt: &Node);
1110
1111 RemoveDeadNodes(DeadNodes);
1112
1113 // If the root changed (e.g. it was a dead load, update the root).
1114 setRoot(Dummy.getValue());
1115}
1116
1117/// RemoveDeadNodes - This method deletes the unreachable nodes in the
1118/// given list, and any nodes that become unreachable as a result.
1119void SelectionDAG::RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes) {
1120
1121 // Process the worklist, deleting the nodes and adding their uses to the
1122 // worklist.
1123 while (!DeadNodes.empty()) {
1124 SDNode *N = DeadNodes.pop_back_val();
1125 // Skip to next node if we've already managed to delete the node. This could
1126 // happen if replacing a node causes a node previously added to the node to
1127 // be deleted.
1128 if (N->getOpcode() == ISD::DELETED_NODE)
1129 continue;
1130
1131 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1132 DUL->NodeDeleted(N, nullptr);
1133
1134 // Take the node out of the appropriate CSE map.
1135 RemoveNodeFromCSEMaps(N);
1136
1137 // Next, brutally remove the operand list. This is safe to do, as there are
1138 // no cycles in the graph.
1139 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
1140 SDUse &Use = *I++;
1141 SDNode *Operand = Use.getNode();
1142 Use.set(SDValue());
1143
1144 // Now that we removed this operand, see if there are no uses of it left.
1145 if (Operand->use_empty())
1146 DeadNodes.push_back(Elt: Operand);
1147 }
1148
1149 DeallocateNode(N);
1150 }
1151}
1152
1153void SelectionDAG::RemoveDeadNode(SDNode *N){
1154 SmallVector<SDNode*, 16> DeadNodes(1, N);
1155
1156 // Create a dummy node that adds a reference to the root node, preventing
1157 // it from being deleted. (This matters if the root is an operand of the
1158 // dead node.)
1159 HandleSDNode Dummy(getRoot());
1160
1161 RemoveDeadNodes(DeadNodes);
1162}
1163
1164void SelectionDAG::DeleteNode(SDNode *N) {
1165 // First take this out of the appropriate CSE map.
1166 RemoveNodeFromCSEMaps(N);
1167
1168 // Finally, remove uses due to operands of this node, remove from the
1169 // AllNodes list, and delete the node.
1170 DeleteNodeNotInCSEMaps(N);
1171}
1172
1173void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) {
1174 assert(N->getIterator() != AllNodes.begin() &&
1175 "Cannot delete the entry node!");
1176 assert(N->use_empty() && "Cannot delete a node that is not dead!");
1177
1178 // Drop all of the operands and decrement used node's use counts.
1179 N->DropOperands();
1180
1181 DeallocateNode(N);
1182}
1183
1184void SDDbgInfo::add(SDDbgValue *V, bool isParameter) {
1185 assert(!(V->isVariadic() && isParameter));
1186 if (isParameter)
1187 ByvalParmDbgValues.push_back(Elt: V);
1188 else
1189 DbgValues.push_back(Elt: V);
1190 for (const SDNode *Node : V->getSDNodes())
1191 if (Node)
1192 DbgValMap[Node].push_back(Elt: V);
1193}
1194
1195void SDDbgInfo::erase(const SDNode *Node) {
1196 DbgValMapType::iterator I = DbgValMap.find(Val: Node);
1197 if (I == DbgValMap.end())
1198 return;
1199 for (auto &Val: I->second)
1200 Val->setIsInvalidated();
1201 DbgValMap.erase(I);
1202}
1203
1204void SelectionDAG::DeallocateNode(SDNode *N) {
1205 // If we have operands, deallocate them.
1206 removeOperands(Node: N);
1207
1208 NodeAllocator.Deallocate(E: AllNodes.remove(IT: N));
1209
1210 // Set the opcode to DELETED_NODE to help catch bugs when node
1211 // memory is reallocated.
1212 // FIXME: There are places in SDag that have grown a dependency on the opcode
1213 // value in the released node.
1214 __asan_unpoison_memory_region(&N->NodeType, sizeof(N->NodeType));
1215 N->NodeType = ISD::DELETED_NODE;
1216
1217 // If any of the SDDbgValue nodes refer to this SDNode, invalidate
1218 // them and forget about that node.
1219 DbgInfo->erase(Node: N);
1220
1221 // Invalidate extra info.
1222 SDEI.erase(Val: N);
1223}
1224
1225#ifndef NDEBUG
1226/// VerifySDNode - Check the given SDNode. Aborts if it is invalid.
1227void SelectionDAG::verifyNode(SDNode *N) const {
1228 switch (N->getOpcode()) {
1229 default:
1230 if (N->isTargetOpcode())
1231 getSelectionDAGInfo().verifyTargetNode(*this, N);
1232 break;
1233 case ISD::BUILD_PAIR: {
1234 EVT VT = N->getValueType(0);
1235 assert(N->getNumValues() == 1 && "Too many results!");
1236 assert(!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) &&
1237 "Wrong return type!");
1238 assert(N->getNumOperands() == 2 && "Wrong number of operands!");
1239 assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() &&
1240 "Mismatched operand types!");
1241 assert(N->getOperand(0).getValueType().isInteger() == VT.isInteger() &&
1242 "Wrong operand type!");
1243 assert(VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() &&
1244 "Wrong return type size");
1245 break;
1246 }
1247 case ISD::BUILD_VECTOR: {
1248 assert(N->getNumValues() == 1 && "Too many results!");
1249 assert(N->getValueType(0).isVector() && "Wrong return type!");
1250 assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() &&
1251 "Wrong number of operands!");
1252 EVT EltVT = N->getValueType(0).getVectorElementType();
1253 for (const SDUse &Op : N->ops()) {
1254 assert((Op.getValueType() == EltVT ||
1255 (EltVT.isInteger() && Op.getValueType().isInteger() &&
1256 EltVT.bitsLE(Op.getValueType()))) &&
1257 "Wrong operand type!");
1258 assert(Op.getValueType() == N->getOperand(0).getValueType() &&
1259 "Operands must all have the same type");
1260 }
1261 break;
1262 }
1263 case ISD::SADDO:
1264 case ISD::UADDO:
1265 case ISD::SSUBO:
1266 case ISD::USUBO:
1267 assert(N->getNumValues() == 2 && "Wrong number of results!");
1268 assert(N->getVTList().NumVTs == 2 && N->getNumOperands() == 2 &&
1269 "Invalid add/sub overflow op!");
1270 assert(N->getVTList().VTs[0].isInteger() &&
1271 N->getVTList().VTs[1].isInteger() &&
1272 N->getOperand(0).getValueType() == N->getOperand(1).getValueType() &&
1273 N->getOperand(0).getValueType() == N->getVTList().VTs[0] &&
1274 "Binary operator types must match!");
1275 break;
1276 }
1277}
1278#endif // NDEBUG
1279
1280/// Insert a newly allocated node into the DAG.
1281///
1282/// Handles insertion into the all nodes list and CSE map, as well as
1283/// verification and other common operations when a new node is allocated.
1284void SelectionDAG::InsertNode(SDNode *N) {
1285 AllNodes.push_back(val: N);
1286#ifndef NDEBUG
1287 N->PersistentId = NextPersistentId++;
1288 verifyNode(N);
1289#endif
1290 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1291 DUL->NodeInserted(N);
1292}
1293
1294/// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that
1295/// correspond to it. This is useful when we're about to delete or repurpose
1296/// the node. We don't want future request for structurally identical nodes
1297/// to return N anymore.
1298bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) {
1299 bool Erased = false;
1300 switch (N->getOpcode()) {
1301 case ISD::HANDLENODE: return false; // noop.
1302 case ISD::CONDCODE:
1303 assert(CondCodeNodes[cast<CondCodeSDNode>(N)->get()] &&
1304 "Cond code doesn't exist!");
1305 Erased = CondCodeNodes[cast<CondCodeSDNode>(Val: N)->get()] != nullptr;
1306 CondCodeNodes[cast<CondCodeSDNode>(Val: N)->get()] = nullptr;
1307 break;
1308 case ISD::ExternalSymbol:
1309 Erased = ExternalSymbols.erase(Key: cast<ExternalSymbolSDNode>(Val: N)->getSymbol());
1310 break;
1311 case ISD::TargetExternalSymbol: {
1312 ExternalSymbolSDNode *ESN = cast<ExternalSymbolSDNode>(Val: N);
1313 Erased = TargetExternalSymbols.erase(x: std::pair<std::string, unsigned>(
1314 ESN->getSymbol(), ESN->getTargetFlags()));
1315 break;
1316 }
1317 case ISD::MCSymbol: {
1318 auto *MCSN = cast<MCSymbolSDNode>(Val: N);
1319 Erased = MCSymbols.erase(Val: MCSN->getMCSymbol());
1320 break;
1321 }
1322 case ISD::VALUETYPE: {
1323 EVT VT = cast<VTSDNode>(Val: N)->getVT();
1324 if (VT.isExtended()) {
1325 Erased = ExtendedValueTypeNodes.erase(x: VT);
1326 } else {
1327 Erased = ValueTypeNodes[VT.getSimpleVT().SimpleTy] != nullptr;
1328 ValueTypeNodes[VT.getSimpleVT().SimpleTy] = nullptr;
1329 }
1330 break;
1331 }
1332 default:
1333 // Remove it from the CSE Map.
1334 assert(N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!");
1335 assert(N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!");
1336 Erased = CSEMap.erase(N);
1337 break;
1338 }
1339#ifndef NDEBUG
1340 // Verify that the node was actually in one of the CSE maps, unless it has a
1341 // glue result (which cannot be CSE'd) or is one of the special cases that are
1342 // not subject to CSE.
1343 if (!Erased && N->getValueType(N->getNumValues()-1) != MVT::Glue &&
1344 !N->isMachineOpcode() && !doNotCSE(N)) {
1345 N->dump(this);
1346 dbgs() << "\n";
1347 llvm_unreachable("Node is not in map!");
1348 }
1349#endif
1350 return Erased;
1351}
1352
1353/// AddModifiedNodeToCSEMaps - The specified node has been removed from the CSE
1354/// maps and modified in place. Add it back to the CSE maps, unless an identical
1355/// node already exists, in which case transfer all its users to the existing
1356/// node. This transfer can potentially trigger recursive merging.
1357void
1358SelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N) {
1359 // For node types that aren't CSE'd, just act as if no identical node
1360 // already exists.
1361 if (!doNotCSE(N)) {
1362 SDNode *Existing = CSEMap.getOrInsert(N);
1363 if (Existing != N) {
1364 // If there was already an existing matching node, use ReplaceAllUsesWith
1365 // to replace the dead one with the existing one. This can cause
1366 // recursive merging of other unrelated nodes down the line.
1367 Existing->intersectFlagsWith(Flags: N->getFlags());
1368 if (auto *MemNode = dyn_cast<MemSDNode>(Val: Existing)) {
1369 ArrayRef<MachineMemOperand *> NewMMOs =
1370 cast<MemSDNode>(Val: N)->memoperands();
1371 // Range and cache hint metadata are not part of the DAG CSE key because
1372 // we prefer to CSE even when metadata does not match. Merge potentially
1373 // differing metadata conservatively.
1374 MemNode->refineMMOMetadata(NewMMOs);
1375 }
1376 ReplaceAllUsesWith(From: N, To: Existing);
1377
1378 // N is now dead. Inform the listeners and delete it.
1379 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1380 DUL->NodeDeleted(N, Existing);
1381 DeleteNodeNotInCSEMaps(N);
1382 return;
1383 }
1384 }
1385
1386 // If the node doesn't already exist, we updated it. Inform listeners.
1387 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1388 DUL->NodeUpdated(N);
1389}
1390
1391/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1392/// were replaced with those specified. If this node is never memoized,
1393/// return null, otherwise return a pointer to the slot it would take. If a
1394/// node already exists with these operands, the slot will be non-null.
1395SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op,
1396 FoldingSetInsertToken &InsertToken) {
1397 if (doNotCSE(N))
1398 return nullptr;
1399
1400 SDValue Ops[] = { Op };
1401 SDNodeKey ID(N->getOpcode(), N->getVTList(), Ops);
1402 AddNodeIDCustom(ID&: ID.Tail, N, Opc: N->getOpcode());
1403 SDNode *Node = lookupNode(Key: ID, DL: SDLoc(N), InsertToken);
1404 if (Node)
1405 Node->intersectFlagsWith(Flags: N->getFlags());
1406 return Node;
1407}
1408
1409/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1410/// were replaced with those specified. If this node is never memoized,
1411/// return null, otherwise return a pointer to the slot it would take. If a
1412/// node already exists with these operands, the slot will be non-null.
1413SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op1, SDValue Op2,
1414 FoldingSetInsertToken &InsertToken) {
1415 if (doNotCSE(N))
1416 return nullptr;
1417
1418 SDValue Ops[] = { Op1, Op2 };
1419 SDNodeKey ID(N->getOpcode(), N->getVTList(), Ops);
1420 AddNodeIDCustom(ID&: ID.Tail, N, Opc: N->getOpcode());
1421 SDNode *Node = lookupNode(Key: ID, DL: SDLoc(N), InsertToken);
1422 if (Node)
1423 Node->intersectFlagsWith(Flags: N->getFlags());
1424 return Node;
1425}
1426
1427/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1428/// were replaced with those specified. If this node is never memoized,
1429/// return null, otherwise return a pointer to the slot it would take. If a
1430/// node already exists with these operands, the slot will be non-null.
1431SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, ArrayRef<SDValue> Ops,
1432 FoldingSetInsertToken &InsertToken) {
1433 if (doNotCSE(N))
1434 return nullptr;
1435
1436 SDNodeKey ID(N->getOpcode(), N->getVTList(), Ops);
1437 AddNodeIDCustom(ID&: ID.Tail, N, Opc: N->getOpcode());
1438 SDNode *Node = lookupNode(Key: ID, DL: SDLoc(N), InsertToken);
1439 if (Node)
1440 Node->intersectFlagsWith(Flags: N->getFlags());
1441 return Node;
1442}
1443
1444Align SelectionDAG::getEVTAlign(EVT VT) const {
1445 Type *Ty = VT == MVT::iPTR ? PointerType::get(C&: *getContext(), AddressSpace: 0)
1446 : VT.getTypeForEVT(Context&: *getContext());
1447
1448 return getDataLayout().getABITypeAlign(Ty);
1449}
1450
1451// EntryNode could meaningfully have debug info if we can find it...
1452SelectionDAG::SelectionDAG(const TargetMachine &tm, CodeGenOptLevel OL)
1453 : TM(tm), OptLevel(OL), EntryNode(ISD::EntryToken, 0, DebugLoc(),
1454 getVTList(VT1: MVT::Other, VT2: MVT::Glue)),
1455 Root(getEntryNode()) {
1456 InsertNode(N: &EntryNode);
1457 DbgInfo = new SDDbgInfo();
1458}
1459
1460void SelectionDAG::init(MachineFunction &NewMF,
1461 const TargetLibraryInfo *LibraryInfo,
1462 const LibcallLoweringInfo *LibcallsInfo,
1463 UniformityInfo *NewUA, ProfileSummaryInfo *PSIin,
1464 BlockFrequencyInfo *BFIin,
1465 FunctionVarLocs const *VarLocs) {
1466 MF = &NewMF;
1467 TLI = getSubtarget().getTargetLowering();
1468 TSI = getSubtarget().getSelectionDAGInfo();
1469 LibInfo = LibraryInfo;
1470 Libcalls = LibcallsInfo;
1471 Context = &MF->getFunction().getContext();
1472 UA = NewUA;
1473 PSI = PSIin;
1474 BFI = BFIin;
1475 FnVarLocs = VarLocs;
1476}
1477
1478SelectionDAG::~SelectionDAG() {
1479 assert(!UpdateListeners && "Dangling registered DAGUpdateListeners");
1480 allnodes_clear();
1481 OperandRecycler.clear(OperandAllocator);
1482 delete DbgInfo;
1483}
1484
1485bool SelectionDAG::shouldOptForSize() const {
1486 return llvm::shouldOptimizeForSize(BB: FLI->MBB->getBasicBlock(), PSI, BFI);
1487}
1488
1489void SelectionDAG::allnodes_clear() {
1490 assert(&*AllNodes.begin() == &EntryNode);
1491 AllNodes.remove(IT: AllNodes.begin());
1492 while (!AllNodes.empty())
1493 DeallocateNode(N: &AllNodes.front());
1494#ifndef NDEBUG
1495 NextPersistentId = 0;
1496#endif
1497}
1498
1499SDNode *SelectionDAG::lookupNode(const SDNodeKey &Key,
1500 FoldingSetInsertToken &InsertToken) {
1501 SDNode *N = CSEMap.lookup(Key, Token&: InsertToken);
1502 if (N) {
1503 switch (N->getOpcode()) {
1504 default: break;
1505 case ISD::Constant:
1506 case ISD::ConstantFP:
1507 llvm_unreachable("Querying for Constant and ConstantFP nodes requires "
1508 "debug location. Use another overload.");
1509 }
1510 }
1511 return N;
1512}
1513
1514SDNode *SelectionDAG::lookupNode(const SDNodeKey &Key, const SDLoc &DL,
1515 FoldingSetInsertToken &InsertToken) {
1516 SDNode *N = CSEMap.lookup(Key, Token&: InsertToken);
1517 if (N) {
1518 switch (N->getOpcode()) {
1519 case ISD::Constant:
1520 case ISD::ConstantFP:
1521 // Erase debug location from the node if the node is used at several
1522 // different places. Do not propagate one location to all uses as it
1523 // will cause a worse single stepping debugging experience.
1524 if (N->getDebugLoc() != DL.getDebugLoc())
1525 N->setDebugLoc(DebugLoc());
1526 break;
1527 default:
1528 // When the node's point of use is located earlier in the instruction
1529 // sequence than its prior point of use, update its debug info to the
1530 // earlier location.
1531 if (DL.getIROrder() && DL.getIROrder() < N->getIROrder())
1532 N->setDebugLoc(DL.getDebugLoc());
1533 break;
1534 }
1535 }
1536 return N;
1537}
1538
1539void SelectionDAG::clear() {
1540 allnodes_clear();
1541 OperandRecycler.clear(OperandAllocator);
1542 OperandAllocator.Reset();
1543 CSEMap.clear();
1544
1545 ExtendedValueTypeNodes.clear();
1546 ExternalSymbols.clear();
1547 TargetExternalSymbols.clear();
1548 MCSymbols.clear();
1549 SDEI.clear();
1550 llvm::fill(Range&: CondCodeNodes, Value: nullptr);
1551 llvm::fill(Range&: ValueTypeNodes, Value: nullptr);
1552
1553 EntryNode.UseList = nullptr;
1554 InsertNode(N: &EntryNode);
1555 Root = getEntryNode();
1556 DbgInfo->clear();
1557}
1558
1559SDValue SelectionDAG::getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT) {
1560 return VT.bitsGT(VT: Op.getValueType())
1561 ? getNode(Opcode: ISD::FP_EXTEND, DL, VT, Operand: Op)
1562 : getNode(Opcode: ISD::FP_ROUND, DL, VT, N1: Op,
1563 N2: getIntPtrConstant(Val: 0, DL, /*isTarget=*/true));
1564}
1565
1566std::pair<SDValue, SDValue>
1567SelectionDAG::getStrictFPExtendOrRound(SDValue Op, SDValue Chain,
1568 const SDLoc &DL, EVT VT) {
1569 assert(!VT.bitsEq(Op.getValueType()) &&
1570 "Strict no-op FP extend/round not allowed.");
1571 SDValue Res =
1572 VT.bitsGT(VT: Op.getValueType())
1573 ? getNode(Opcode: ISD::STRICT_FP_EXTEND, DL, ResultTys: {VT, MVT::Other}, Ops: {Chain, Op})
1574 : getNode(Opcode: ISD::STRICT_FP_ROUND, DL, ResultTys: {VT, MVT::Other},
1575 Ops: {Chain, Op, getIntPtrConstant(Val: 0, DL, /*isTarget=*/true)});
1576
1577 return std::pair<SDValue, SDValue>(Res, SDValue(Res.getNode(), 1));
1578}
1579
1580SDValue SelectionDAG::getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1581 return VT.bitsGT(VT: Op.getValueType()) ?
1582 getNode(Opcode: ISD::ANY_EXTEND, DL, VT, Operand: Op) :
1583 getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Op);
1584}
1585
1586SDValue SelectionDAG::getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1587 return VT.bitsGT(VT: Op.getValueType()) ?
1588 getNode(Opcode: ISD::SIGN_EXTEND, DL, VT, Operand: Op) :
1589 getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Op);
1590}
1591
1592SDValue SelectionDAG::getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1593 return VT.bitsGT(VT: Op.getValueType()) ?
1594 getNode(Opcode: ISD::ZERO_EXTEND, DL, VT, Operand: Op) :
1595 getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Op);
1596}
1597
1598SDValue SelectionDAG::getBitcastedAnyExtOrTrunc(SDValue Op, const SDLoc &DL,
1599 EVT VT) {
1600 assert(!VT.isVector());
1601 auto Type = Op.getValueType();
1602 SDValue DestOp;
1603 if (Type == VT)
1604 return Op;
1605 auto Size = Op.getValueSizeInBits();
1606 DestOp = getBitcast(VT: EVT::getIntegerVT(Context&: *Context, BitWidth: Size), V: Op);
1607 if (DestOp.getValueType() == VT)
1608 return DestOp;
1609
1610 return getAnyExtOrTrunc(Op: DestOp, DL, VT);
1611}
1612
1613SDValue SelectionDAG::getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT,
1614 EVT OpVT) {
1615 if (VT.bitsLE(VT: Op.getValueType()))
1616 return getNode(Opcode: ISD::TRUNCATE, DL: SL, VT, Operand: Op);
1617
1618 TargetLowering::BooleanContent BType = TLI->getBooleanContents(Type: OpVT);
1619 return getNode(Opcode: TLI->getExtendForContent(Content: BType), DL: SL, VT, Operand: Op);
1620}
1621
1622SDValue SelectionDAG::getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) {
1623 EVT OpVT = Op.getValueType();
1624 assert(VT.isInteger() && OpVT.isInteger() &&
1625 "Cannot getZeroExtendInReg FP types");
1626 assert(VT.isVector() == OpVT.isVector() &&
1627 "getZeroExtendInReg type should be vector iff the operand "
1628 "type is vector!");
1629 assert((!VT.isVector() ||
1630 VT.getVectorElementCount() == OpVT.getVectorElementCount()) &&
1631 "Vector element counts must match in getZeroExtendInReg");
1632 assert(VT.getScalarType().bitsLE(OpVT.getScalarType()) && "Not extending!");
1633 if (OpVT == VT)
1634 return Op;
1635 // TODO: Use computeKnownBits instead of AssertZext.
1636 if (Op.getOpcode() == ISD::AssertZext && cast<VTSDNode>(Val: Op.getOperand(i: 1))
1637 ->getVT()
1638 .getScalarType()
1639 .bitsLE(VT: VT.getScalarType()))
1640 return Op;
1641 APInt Imm = APInt::getLowBitsSet(numBits: OpVT.getScalarSizeInBits(),
1642 loBitsSet: VT.getScalarSizeInBits());
1643 return getNode(Opcode: ISD::AND, DL, VT: OpVT, N1: Op, N2: getConstant(Val: Imm, DL, VT: OpVT));
1644}
1645
1646SDValue SelectionDAG::getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1647 // Only unsigned pointer semantics are supported right now. In the future this
1648 // might delegate to TLI to check pointer signedness.
1649 return getZExtOrTrunc(Op, DL, VT);
1650}
1651
1652SDValue SelectionDAG::getPtrExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) {
1653 // Only unsigned pointer semantics are supported right now. In the future this
1654 // might delegate to TLI to check pointer signedness.
1655 return getZeroExtendInReg(Op, DL, VT);
1656}
1657
1658SDValue SelectionDAG::getNegative(SDValue Val, const SDLoc &DL, EVT VT) {
1659 return getNode(Opcode: ISD::SUB, DL, VT, N1: getConstant(Val: 0, DL, VT), N2: Val);
1660}
1661
1662/// getNOT - Create a bitwise NOT operation as (XOR Val, -1).
1663SDValue SelectionDAG::getNOT(const SDLoc &DL, SDValue Val, EVT VT) {
1664 return getNode(Opcode: ISD::XOR, DL, VT, N1: Val, N2: getAllOnesConstant(DL, VT));
1665}
1666
1667SDValue SelectionDAG::getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT) {
1668 SDValue TrueValue = getBoolConstant(V: true, DL, VT, OpVT: VT);
1669 return getNode(Opcode: ISD::XOR, DL, VT, N1: Val, N2: TrueValue);
1670}
1671
1672SDValue SelectionDAG::getBoolConstant(bool V, const SDLoc &DL, EVT VT,
1673 EVT OpVT) {
1674 if (!V)
1675 return getConstant(Val: 0, DL, VT);
1676
1677 switch (TLI->getBooleanContents(Type: OpVT)) {
1678 case TargetLowering::ZeroOrOneBooleanContent:
1679 case TargetLowering::UndefinedBooleanContent:
1680 return getConstant(Val: 1, DL, VT);
1681 case TargetLowering::ZeroOrNegativeOneBooleanContent:
1682 return getAllOnesConstant(DL, VT);
1683 }
1684 llvm_unreachable("Unexpected boolean content enum!");
1685}
1686
1687SDValue SelectionDAG::getConstant(uint64_t Val, const SDLoc &DL, EVT VT,
1688 bool isT, bool isO) {
1689 return getConstant(Val: APInt(VT.getScalarSizeInBits(), Val, /*isSigned=*/false),
1690 DL, VT, isTarget: isT, isOpaque: isO);
1691}
1692
1693SDValue SelectionDAG::getConstant(const APInt &Val, const SDLoc &DL, EVT VT,
1694 bool isT, bool isO) {
1695 return getConstant(Val: *ConstantInt::get(Context&: *Context, V: Val), DL, VT, isTarget: isT, isOpaque: isO);
1696}
1697
1698SDValue SelectionDAG::getConstant(const ConstantInt &Val, const SDLoc &DL,
1699 EVT VT, bool isT, bool isO) {
1700 assert(VT.isInteger() && "Cannot create FP integer constant!");
1701
1702 EVT EltVT = VT.getScalarType();
1703 const ConstantInt *Elt = &Val;
1704
1705 // Vector splats are explicit within the DAG, with ConstantSDNode holding the
1706 // to-be-splatted scalar ConstantInt.
1707 if (isa<VectorType>(Val: Elt->getType()))
1708 Elt = ConstantInt::get(Context&: *getContext(), V: Elt->getValue());
1709
1710 // In some cases the vector type is legal but the element type is illegal and
1711 // needs to be promoted, for example v8i8 on ARM. In this case, promote the
1712 // inserted value (the type does not need to match the vector element type).
1713 // Any extra bits introduced will be truncated away.
1714 if (VT.isVector() && TLI->getTypeAction(Context&: *getContext(), VT: EltVT) ==
1715 TargetLowering::TypePromoteInteger) {
1716 EltVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT: EltVT);
1717 APInt NewVal;
1718 if (TLI->isSExtCheaperThanZExt(FromTy: VT.getScalarType(), ToTy: EltVT))
1719 NewVal = Elt->getValue().sextOrTrunc(width: EltVT.getSizeInBits());
1720 else
1721 NewVal = Elt->getValue().zextOrTrunc(width: EltVT.getSizeInBits());
1722 Elt = ConstantInt::get(Context&: *getContext(), V: NewVal);
1723 }
1724 // In other cases the element type is illegal and needs to be expanded, for
1725 // example v2i64 on MIPS32. In this case, find the nearest legal type, split
1726 // the value into n parts and use a vector type with n-times the elements.
1727 // Then bitcast to the type requested.
1728 // Legalizing constants too early makes the DAGCombiner's job harder so we
1729 // only legalize if the DAG tells us we must produce legal types.
1730 else if (NewNodesMustHaveLegalTypes && VT.isVector() &&
1731 TLI->getTypeAction(Context&: *getContext(), VT: EltVT) ==
1732 TargetLowering::TypeExpandInteger) {
1733 const APInt &NewVal = Elt->getValue();
1734 EVT ViaEltVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT: EltVT);
1735 unsigned ViaEltSizeInBits = ViaEltVT.getSizeInBits();
1736
1737 // For scalable vectors, try to use a SPLAT_VECTOR_PARTS node.
1738 if (VT.isScalableVector() ||
1739 TLI->isOperationLegal(Op: ISD::SPLAT_VECTOR, VT)) {
1740 assert(EltVT.getSizeInBits() % ViaEltSizeInBits == 0 &&
1741 "Can only handle an even split!");
1742 unsigned Parts = EltVT.getSizeInBits() / ViaEltSizeInBits;
1743
1744 SmallVector<SDValue, 2> ScalarParts;
1745 for (unsigned i = 0; i != Parts; ++i)
1746 ScalarParts.push_back(Elt: getConstant(
1747 Val: NewVal.extractBits(numBits: ViaEltSizeInBits, bitPosition: i * ViaEltSizeInBits), DL,
1748 VT: ViaEltVT, isT, isO));
1749
1750 return getNode(Opcode: ISD::SPLAT_VECTOR_PARTS, DL, VT, Ops: ScalarParts);
1751 }
1752
1753 unsigned ViaVecNumElts = VT.getSizeInBits() / ViaEltSizeInBits;
1754 EVT ViaVecVT = EVT::getVectorVT(Context&: *getContext(), VT: ViaEltVT, NumElements: ViaVecNumElts);
1755
1756 // Check the temporary vector is the correct size. If this fails then
1757 // getTypeToTransformTo() probably returned a type whose size (in bits)
1758 // isn't a power-of-2 factor of the requested type size.
1759 assert(ViaVecVT.getSizeInBits() == VT.getSizeInBits());
1760
1761 SmallVector<SDValue, 2> EltParts;
1762 for (unsigned i = 0; i < ViaVecNumElts / VT.getVectorNumElements(); ++i)
1763 EltParts.push_back(Elt: getConstant(
1764 Val: NewVal.extractBits(numBits: ViaEltSizeInBits, bitPosition: i * ViaEltSizeInBits), DL,
1765 VT: ViaEltVT, isT, isO));
1766
1767 // EltParts is currently in little endian order. If we actually want
1768 // big-endian order then reverse it now.
1769 if (getDataLayout().isBigEndian())
1770 std::reverse(first: EltParts.begin(), last: EltParts.end());
1771
1772 // The elements must be reversed when the element order is different
1773 // to the endianness of the elements (because the BITCAST is itself a
1774 // vector shuffle in this situation). However, we do not need any code to
1775 // perform this reversal because getConstant() is producing a vector
1776 // splat.
1777 // This situation occurs in MIPS MSA.
1778
1779 SmallVector<SDValue, 8> Ops;
1780 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i)
1781 llvm::append_range(C&: Ops, R&: EltParts);
1782
1783 SDValue V =
1784 getNode(Opcode: ISD::BITCAST, DL, VT, Operand: getBuildVector(VT: ViaVecVT, DL, Ops));
1785 return V;
1786 }
1787
1788 assert(Elt->getBitWidth() == EltVT.getSizeInBits() &&
1789 "APInt size does not match type size!");
1790 unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant;
1791 SDVTList VTs = getVTList(VT: EltVT);
1792 SDNodeKey ID(Opc, VTs, {});
1793 ID.AddPointer(P: Elt);
1794 ID.AddBoolean(B: isO);
1795 FoldingSetInsertToken InsertToken;
1796 SDNode *N = nullptr;
1797 if ((N = lookupNode(Key: ID, DL, InsertToken)))
1798 if (!VT.isVector())
1799 return SDValue(N, 0);
1800
1801 if (!N) {
1802 N = newSDNode<ConstantSDNode>(Args&: isT, Args&: isO, Args&: Elt, Args&: VTs);
1803 if (!isT)
1804 N->setDebugLoc(DL.getDebugLoc());
1805 CSEMap.insert(N, Token: InsertToken);
1806 InsertNode(N);
1807 NewSDValueDbgMsg(V: SDValue(N, 0), Msg: "Creating constant: ", G: this);
1808 }
1809
1810 SDValue Result(N, 0);
1811 if (VT.isVector())
1812 Result = getSplat(VT, DL, Op: Result);
1813 return Result;
1814}
1815
1816SDValue SelectionDAG::getSignedConstant(int64_t Val, const SDLoc &DL, EVT VT,
1817 bool isT, bool isO) {
1818 unsigned Size = VT.getScalarSizeInBits();
1819 return getConstant(Val: APInt(Size, Val, /*isSigned=*/true), DL, VT, isT, isO);
1820}
1821
1822SDValue SelectionDAG::getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget,
1823 bool IsOpaque) {
1824 return getConstant(Val: APInt::getAllOnes(numBits: VT.getScalarSizeInBits()), DL, VT,
1825 isT: IsTarget, isO: IsOpaque);
1826}
1827
1828SDValue SelectionDAG::getIntPtrConstant(uint64_t Val, const SDLoc &DL,
1829 bool isTarget) {
1830 return getConstant(Val, DL, VT: TLI->getPointerTy(DL: getDataLayout()), isT: isTarget);
1831}
1832
1833SDValue SelectionDAG::getShiftAmountConstant(uint64_t Val, EVT VT,
1834 const SDLoc &DL) {
1835 assert(VT.isInteger() && "Shift amount is not an integer type!");
1836 EVT ShiftVT = TLI->getShiftAmountTy(LHSTy: VT, DL: getDataLayout());
1837 return getConstant(Val, DL, VT: ShiftVT);
1838}
1839
1840SDValue SelectionDAG::getShiftAmountConstant(const APInt &Val, EVT VT,
1841 const SDLoc &DL) {
1842 assert(Val.ult(VT.getScalarSizeInBits()) && "Out of range shift");
1843 return getShiftAmountConstant(Val: Val.getZExtValue(), VT, DL);
1844}
1845
1846SDValue SelectionDAG::getVectorIdxConstant(uint64_t Val, const SDLoc &DL,
1847 bool isTarget) {
1848 return getConstant(Val, DL, VT: TLI->getVectorIdxTy(DL: getDataLayout()), isT: isTarget);
1849}
1850
1851SDValue SelectionDAG::getConstantFP(const APFloat &V, const SDLoc &DL, EVT VT,
1852 bool isTarget) {
1853 return getConstantFP(V: *ConstantFP::get(Context&: *getContext(), V), DL, VT, isTarget);
1854}
1855
1856SDValue SelectionDAG::getConstantFP(const ConstantFP &V, const SDLoc &DL,
1857 EVT VT, bool isTarget) {
1858 assert(VT.isFloatingPoint() && "Cannot create integer FP constant!");
1859
1860 EVT EltVT = VT.getScalarType();
1861 const ConstantFP *Elt = &V;
1862
1863 // Vector splats are explicit within the DAG, with ConstantFPSDNode holding
1864 // the to-be-splatted scalar ConstantFP.
1865 if (isa<VectorType>(Val: Elt->getType()))
1866 Elt = ConstantFP::get(Context&: *getContext(), V: Elt->getValue());
1867
1868 // Do the map lookup using the actual bit pattern for the floating point
1869 // value, so that we don't have problems with 0.0 comparing equal to -0.0, and
1870 // we don't have issues with SNANs.
1871 unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP;
1872 SDVTList VTs = getVTList(VT: EltVT);
1873 SDNodeKey ID(Opc, VTs, {});
1874 ID.AddPointer(P: Elt);
1875 FoldingSetInsertToken InsertToken;
1876 SDNode *N = nullptr;
1877 if ((N = lookupNode(Key: ID, DL, InsertToken)))
1878 if (!VT.isVector())
1879 return SDValue(N, 0);
1880
1881 if (!N) {
1882 N = newSDNode<ConstantFPSDNode>(Args&: isTarget, Args&: Elt, Args&: VTs);
1883 CSEMap.insert(N, Token: InsertToken);
1884 InsertNode(N);
1885 }
1886
1887 SDValue Result(N, 0);
1888 if (VT.isVector())
1889 Result = getSplat(VT, DL, Op: Result);
1890 NewSDValueDbgMsg(V: Result, Msg: "Creating fp constant: ", G: this);
1891 return Result;
1892}
1893
1894SDValue SelectionDAG::getConstantFP(double Val, const SDLoc &DL, EVT VT,
1895 bool isTarget) {
1896 EVT EltVT = VT.getScalarType();
1897 if (EltVT == MVT::f32)
1898 return getConstantFP(V: APFloat((float)Val), DL, VT, isTarget);
1899 if (EltVT == MVT::f64)
1900 return getConstantFP(V: APFloat(Val), DL, VT, isTarget);
1901 if (EltVT == MVT::f80 || EltVT == MVT::f128 || EltVT == MVT::ppcf128 ||
1902 EltVT == MVT::f16 || EltVT == MVT::bf16) {
1903 bool Ignored;
1904 APFloat APF = APFloat(Val);
1905 APF.convert(ToSemantics: EltVT.getFltSemantics(), RM: APFloat::rmNearestTiesToEven,
1906 losesInfo: &Ignored);
1907 return getConstantFP(V: APF, DL, VT, isTarget);
1908 }
1909 llvm_unreachable("Unsupported type in getConstantFP");
1910}
1911
1912SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, const SDLoc &DL,
1913 EVT VT, int64_t Offset, bool isTargetGA,
1914 unsigned TargetFlags) {
1915 assert((TargetFlags == 0 || isTargetGA) &&
1916 "Cannot set target flags on target-independent globals");
1917
1918 // Truncate (with sign-extension) the offset value to the pointer size.
1919 unsigned BitWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType());
1920 if (BitWidth < 64)
1921 Offset = SignExtend64(X: Offset, B: BitWidth);
1922
1923 unsigned Opc;
1924 if (GV->isThreadLocal())
1925 Opc = isTargetGA ? ISD::TargetGlobalTLSAddress : ISD::GlobalTLSAddress;
1926 else
1927 Opc = isTargetGA ? ISD::TargetGlobalAddress : ISD::GlobalAddress;
1928
1929 SDVTList VTs = getVTList(VT);
1930 SDNodeKey ID(Opc, VTs, {});
1931 ID.AddPointer(P: GV);
1932 ID.AddInteger(I: Offset);
1933 ID.AddInteger(I: TargetFlags);
1934 FoldingSetInsertToken InsertToken;
1935 if (SDNode *E = lookupNode(Key: ID, DL, InsertToken))
1936 return SDValue(E, 0);
1937
1938 auto *N = newSDNode<GlobalAddressSDNode>(
1939 Args&: Opc, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: GV, Args&: VTs, Args&: Offset, Args&: TargetFlags);
1940 CSEMap.insert(N, Token: InsertToken);
1941 InsertNode(N);
1942 return SDValue(N, 0);
1943}
1944
1945SDValue SelectionDAG::getDeactivationSymbol(const GlobalValue *GV) {
1946 SDVTList VTs = getVTList(VT: MVT::Untyped);
1947 SDNodeKey ID(ISD::DEACTIVATION_SYMBOL, VTs, {});
1948 ID.AddPointer(P: GV);
1949 FoldingSetInsertToken InsertToken;
1950 if (SDNode *E = lookupNode(Key: ID, DL: SDLoc(), InsertToken))
1951 return SDValue(E, 0);
1952
1953 auto *N = newSDNode<DeactivationSymbolSDNode>(Args&: GV, Args&: VTs);
1954 CSEMap.insert(N, Token: InsertToken);
1955 InsertNode(N);
1956 return SDValue(N, 0);
1957}
1958
1959SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) {
1960 unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex;
1961 SDVTList VTs = getVTList(VT);
1962 SDNodeKey ID(Opc, VTs, {});
1963 ID.AddInteger(I: FI);
1964 FoldingSetInsertToken InsertToken;
1965 if (SDNode *E = lookupNode(Key: ID, InsertToken))
1966 return SDValue(E, 0);
1967
1968 auto *N = newSDNode<FrameIndexSDNode>(Args&: FI, Args&: VTs, Args&: isTarget);
1969 CSEMap.insert(N, Token: InsertToken);
1970 InsertNode(N);
1971 return SDValue(N, 0);
1972}
1973
1974SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget,
1975 unsigned TargetFlags) {
1976 assert((TargetFlags == 0 || isTarget) &&
1977 "Cannot set target flags on target-independent jump tables");
1978 unsigned Opc = isTarget ? ISD::TargetJumpTable : ISD::JumpTable;
1979 SDVTList VTs = getVTList(VT);
1980 SDNodeKey ID(Opc, VTs, {});
1981 ID.AddInteger(I: JTI);
1982 ID.AddInteger(I: TargetFlags);
1983 FoldingSetInsertToken InsertToken;
1984 if (SDNode *E = lookupNode(Key: ID, InsertToken))
1985 return SDValue(E, 0);
1986
1987 auto *N = newSDNode<JumpTableSDNode>(Args&: JTI, Args&: VTs, Args&: isTarget, Args&: TargetFlags);
1988 CSEMap.insert(N, Token: InsertToken);
1989 InsertNode(N);
1990 return SDValue(N, 0);
1991}
1992
1993SDValue SelectionDAG::getJumpTableDebugInfo(int JTI, SDValue Chain,
1994 const SDLoc &DL) {
1995 EVT PTy = getTargetLoweringInfo().getPointerTy(DL: getDataLayout());
1996 return getNode(Opcode: ISD::JUMP_TABLE_DEBUG_INFO, DL, VT: MVT::Other, N1: Chain,
1997 N2: getTargetConstant(Val: static_cast<uint64_t>(JTI), DL, VT: PTy, isOpaque: true));
1998}
1999
2000SDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT,
2001 MaybeAlign Alignment, int Offset,
2002 bool isTarget, unsigned TargetFlags) {
2003 assert((TargetFlags == 0 || isTarget) &&
2004 "Cannot set target flags on target-independent globals");
2005 if (!Alignment)
2006 Alignment = shouldOptForSize()
2007 ? getDataLayout().getABITypeAlign(Ty: C->getType())
2008 : getDataLayout().getPrefTypeAlign(Ty: C->getType());
2009 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
2010 SDVTList VTs = getVTList(VT);
2011 SDNodeKey ID(Opc, VTs, {});
2012 ID.AddInteger(I: Alignment->value());
2013 ID.AddInteger(I: Offset);
2014 ID.AddPointer(P: C);
2015 ID.AddInteger(I: TargetFlags);
2016 FoldingSetInsertToken InsertToken;
2017 if (SDNode *E = lookupNode(Key: ID, InsertToken))
2018 return SDValue(E, 0);
2019
2020 auto *N = newSDNode<ConstantPoolSDNode>(Args&: isTarget, Args&: C, Args&: VTs, Args&: Offset, Args&: *Alignment,
2021 Args&: TargetFlags);
2022 CSEMap.insert(N, Token: InsertToken);
2023 InsertNode(N);
2024 SDValue V = SDValue(N, 0);
2025 NewSDValueDbgMsg(V, Msg: "Creating new constant pool: ", G: this);
2026 return V;
2027}
2028
2029SDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT,
2030 MaybeAlign Alignment, int Offset,
2031 bool isTarget, unsigned TargetFlags) {
2032 assert((TargetFlags == 0 || isTarget) &&
2033 "Cannot set target flags on target-independent globals");
2034 if (!Alignment)
2035 Alignment = getDataLayout().getPrefTypeAlign(Ty: C->getType());
2036 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
2037 SDVTList VTs = getVTList(VT);
2038 SDNodeKey ID(Opc, VTs, {});
2039 ID.AddInteger(I: Alignment->value());
2040 ID.AddInteger(I: Offset);
2041 C->addSelectionDAGCSEId(ID&: ID.Tail);
2042 ID.AddInteger(I: TargetFlags);
2043 FoldingSetInsertToken InsertToken;
2044 if (SDNode *E = lookupNode(Key: ID, InsertToken))
2045 return SDValue(E, 0);
2046
2047 auto *N = newSDNode<ConstantPoolSDNode>(Args&: isTarget, Args&: C, Args&: VTs, Args&: Offset, Args&: *Alignment,
2048 Args&: TargetFlags);
2049 CSEMap.insert(N, Token: InsertToken);
2050 InsertNode(N);
2051 return SDValue(N, 0);
2052}
2053
2054SDValue SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) {
2055 SDNodeKey ID(ISD::BasicBlock, getVTList(VT: MVT::Other), {});
2056 ID.AddPointer(P: MBB);
2057 FoldingSetInsertToken InsertToken;
2058 if (SDNode *E = lookupNode(Key: ID, InsertToken))
2059 return SDValue(E, 0);
2060
2061 auto *N = newSDNode<BasicBlockSDNode>(Args&: MBB);
2062 CSEMap.insert(N, Token: InsertToken);
2063 InsertNode(N);
2064 return SDValue(N, 0);
2065}
2066
2067SDValue SelectionDAG::getValueType(EVT VT) {
2068 if (VT.isSimple() && (unsigned)VT.getSimpleVT().SimpleTy >=
2069 ValueTypeNodes.size())
2070 ValueTypeNodes.resize(new_size: VT.getSimpleVT().SimpleTy+1);
2071
2072 SDNode *&N = VT.isExtended() ?
2073 ExtendedValueTypeNodes[VT] : ValueTypeNodes[VT.getSimpleVT().SimpleTy];
2074
2075 if (N) return SDValue(N, 0);
2076 N = newSDNode<VTSDNode>(Args&: VT);
2077 InsertNode(N);
2078 return SDValue(N, 0);
2079}
2080
2081SDValue SelectionDAG::getExternalSymbol(const char *Sym, EVT VT) {
2082 SDNode *&N = ExternalSymbols[Sym];
2083 if (N) return SDValue(N, 0);
2084 N = newSDNode<ExternalSymbolSDNode>(Args: false, Args&: Sym, Args: 0, Args: getVTList(VT));
2085 InsertNode(N);
2086 return SDValue(N, 0);
2087}
2088
2089SDValue SelectionDAG::getExternalSymbol(RTLIB::LibcallImpl Libcall, EVT VT) {
2090 StringRef SymName = RTLIB::RuntimeLibcallsInfo::getLibcallImplName(CallImpl: Libcall);
2091 return getExternalSymbol(Sym: SymName.data(), VT);
2092}
2093
2094SDValue SelectionDAG::getMCSymbol(MCSymbol *Sym, EVT VT) {
2095 SDNode *&N = MCSymbols[Sym];
2096 if (N)
2097 return SDValue(N, 0);
2098 N = newSDNode<MCSymbolSDNode>(Args&: Sym, Args: getVTList(VT));
2099 InsertNode(N);
2100 return SDValue(N, 0);
2101}
2102
2103SDValue SelectionDAG::getTargetExternalSymbol(const char *Sym, EVT VT,
2104 unsigned TargetFlags) {
2105 SDNode *&N =
2106 TargetExternalSymbols[std::pair<std::string, unsigned>(Sym, TargetFlags)];
2107 if (N) return SDValue(N, 0);
2108 N = newSDNode<ExternalSymbolSDNode>(Args: true, Args&: Sym, Args&: TargetFlags, Args: getVTList(VT));
2109 InsertNode(N);
2110 return SDValue(N, 0);
2111}
2112
2113SDValue SelectionDAG::getTargetExternalSymbol(RTLIB::LibcallImpl Libcall,
2114 EVT VT, unsigned TargetFlags) {
2115 StringRef SymName = RTLIB::RuntimeLibcallsInfo::getLibcallImplName(CallImpl: Libcall);
2116 return getTargetExternalSymbol(Sym: SymName.data(), VT, TargetFlags);
2117}
2118
2119SDValue SelectionDAG::getCondCode(ISD::CondCode Cond) {
2120 if ((unsigned)Cond >= CondCodeNodes.size())
2121 CondCodeNodes.resize(new_size: Cond+1);
2122
2123 if (!CondCodeNodes[Cond]) {
2124 auto *N = newSDNode<CondCodeSDNode>(Args&: Cond);
2125 CondCodeNodes[Cond] = N;
2126 InsertNode(N);
2127 }
2128
2129 return SDValue(CondCodeNodes[Cond], 0);
2130}
2131
2132SDValue SelectionDAG::getVScale(const SDLoc &DL, EVT VT, APInt MulImm) {
2133 assert(MulImm.getBitWidth() == VT.getSizeInBits() &&
2134 "APInt size does not match type size!");
2135
2136 if (MulImm == 0)
2137 return getConstant(Val: 0, DL, VT);
2138
2139 const MachineFunction &MF = getMachineFunction();
2140 const Function &F = MF.getFunction();
2141 ConstantRange CR = getVScaleRange(F: &F, BitWidth: 64);
2142 if (const APInt *C = CR.getSingleElement())
2143 return getConstant(Val: MulImm * C->getZExtValue(), DL, VT);
2144
2145 return getNode(Opcode: ISD::VSCALE, DL, VT, Operand: getConstant(Val: MulImm, DL, VT));
2146}
2147
2148/// \returns a value of type \p VT that represents the runtime value of \p
2149/// Quantity, i.e. scaled by vscale if it's scalable, or a fixed constant
2150/// otherwise. Quantity should be a FixedOrScalableQuantity, i.e. ElementCount
2151/// or TypeSize.
2152template <typename Ty>
2153static SDValue getFixedOrScalableQuantity(SelectionDAG &DAG, const SDLoc &DL,
2154 EVT VT, Ty Quantity) {
2155 if (Quantity.isScalable())
2156 return DAG.getVScale(
2157 DL, VT, MulImm: APInt(VT.getSizeInBits(), Quantity.getKnownMinValue()));
2158
2159 return DAG.getConstant(Quantity.getKnownMinValue(), DL, VT);
2160}
2161
2162SDValue SelectionDAG::getElementCount(const SDLoc &DL, EVT VT,
2163 ElementCount EC) {
2164 return getFixedOrScalableQuantity(DAG&: *this, DL, VT, Quantity: EC);
2165}
2166
2167SDValue SelectionDAG::getTypeSize(const SDLoc &DL, EVT VT, TypeSize TS) {
2168 return getFixedOrScalableQuantity(DAG&: *this, DL, VT, Quantity: TS);
2169}
2170
2171SDValue SelectionDAG::getMaskFromElementCount(const SDLoc &DL, EVT DataVT,
2172 ElementCount EC) {
2173 EVT IdxVT = TLI->getVectorIdxTy(DL: getDataLayout());
2174 EVT MaskVT = TLI->getSetCCResultType(DL: getDataLayout(), Context&: *getContext(), VT: DataVT);
2175 return getNode(Opcode: ISD::GET_ACTIVE_LANE_MASK, DL, VT: MaskVT,
2176 N1: getConstant(Val: 0, DL, VT: IdxVT), N2: getElementCount(DL, VT: IdxVT, EC));
2177}
2178
2179SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT) {
2180 APInt One(ResVT.getScalarSizeInBits(), 1);
2181 return getStepVector(DL, ResVT, StepVal: One);
2182}
2183
2184SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT,
2185 const APInt &StepVal) {
2186 assert(ResVT.getScalarSizeInBits() == StepVal.getBitWidth());
2187 if (ResVT.isScalableVector())
2188 return getNode(
2189 Opcode: ISD::STEP_VECTOR, DL, VT: ResVT,
2190 Operand: getTargetConstant(Val: StepVal, DL, VT: ResVT.getVectorElementType()));
2191
2192 SmallVector<SDValue, 16> OpsStepConstants;
2193 for (uint64_t i = 0; i < ResVT.getVectorNumElements(); i++)
2194 OpsStepConstants.push_back(
2195 Elt: getConstant(Val: StepVal * i, DL, VT: ResVT.getVectorElementType()));
2196 return getBuildVector(VT: ResVT, DL, Ops: OpsStepConstants);
2197}
2198
2199/// Swaps the values of N1 and N2. Swaps all indices in the shuffle mask M that
2200/// point at N1 to point at N2 and indices that point at N2 to point at N1.
2201static void commuteShuffle(SDValue &N1, SDValue &N2, MutableArrayRef<int> M) {
2202 std::swap(a&: N1, b&: N2);
2203 ShuffleVectorSDNode::commuteMask(Mask: M);
2204}
2205
2206SDValue SelectionDAG::getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1,
2207 SDValue N2, ArrayRef<int> Mask) {
2208 assert(VT.getVectorNumElements() == Mask.size() &&
2209 "Must have the same number of vector elements as mask elements!");
2210 assert(VT == N1.getValueType() && VT == N2.getValueType() &&
2211 "Invalid VECTOR_SHUFFLE");
2212
2213 // Canonicalize shuffle undef, undef -> undef
2214 if (N1.isUndef() && N2.isUndef()) {
2215 if (N1.getOpcode() == ISD::POISON && N2.getOpcode() == ISD::POISON)
2216 return getPOISON(VT);
2217 return getUNDEF(VT);
2218 }
2219
2220 // Validate that all indices in Mask are within the range of the elements
2221 // input to the shuffle.
2222 int NElts = Mask.size();
2223 assert(llvm::all_of(Mask,
2224 [&](int M) { return M < (NElts * 2) && M >= -1; }) &&
2225 "Index out of range");
2226
2227 // Copy the mask so we can do any needed cleanup.
2228 SmallVector<int, 8> MaskVec(Mask);
2229
2230 // Canonicalize shuffle v, v -> v, poison
2231 if (N1 == N2) {
2232 N2 = getPOISON(VT);
2233 for (int i = 0; i != NElts; ++i)
2234 if (MaskVec[i] >= NElts) MaskVec[i] -= NElts;
2235 }
2236
2237 // Canonicalize shuffle undef, v -> v, undef. Commute the shuffle mask.
2238 if (N1.isUndef())
2239 commuteShuffle(N1, N2, M: MaskVec);
2240
2241 if (TLI->hasVectorBlend()) {
2242 // If shuffling a splat, try to blend the splat instead. We do this here so
2243 // that even when this arises during lowering we don't have to re-handle it.
2244 auto BlendSplat = [&](BuildVectorSDNode *BV, int Offset) {
2245 BitVector UndefElements;
2246 SDValue Splat = BV->getSplatValue(UndefElements: &UndefElements);
2247 if (!Splat)
2248 return;
2249
2250 for (int i = 0; i < NElts; ++i) {
2251 if (MaskVec[i] < Offset || MaskVec[i] >= (Offset + NElts))
2252 continue;
2253
2254 // If this input comes from undef, mark it as such.
2255 if (UndefElements[MaskVec[i] - Offset]) {
2256 MaskVec[i] = -1;
2257 continue;
2258 }
2259
2260 // If we can blend a non-undef lane, use that instead.
2261 if (!UndefElements[i])
2262 MaskVec[i] = i + Offset;
2263 }
2264 };
2265 if (auto *N1BV = dyn_cast<BuildVectorSDNode>(Val&: N1))
2266 BlendSplat(N1BV, 0);
2267 if (auto *N2BV = dyn_cast<BuildVectorSDNode>(Val&: N2))
2268 BlendSplat(N2BV, NElts);
2269 }
2270
2271 // Canonicalize all index into lhs, -> shuffle lhs, poison
2272 // Canonicalize all index into rhs, -> shuffle rhs, poison
2273 bool AllLHS = true, AllRHS = true;
2274 bool N2Undef = N2.isUndef();
2275 for (int i = 0; i != NElts; ++i) {
2276 if (MaskVec[i] >= NElts) {
2277 if (N2Undef)
2278 MaskVec[i] = -1;
2279 else
2280 AllLHS = false;
2281 } else if (MaskVec[i] >= 0) {
2282 AllRHS = false;
2283 }
2284 }
2285 if (AllLHS && AllRHS)
2286 return getPOISON(VT);
2287 if (AllLHS && !N2Undef)
2288 N2 = getPOISON(VT);
2289 if (AllRHS) {
2290 N1 = getPOISON(VT);
2291 commuteShuffle(N1, N2, M: MaskVec);
2292 }
2293 // Reset our undef status after accounting for the mask.
2294 N2Undef = N2.isUndef();
2295 // Re-check whether both sides ended up undef.
2296 if (N1.isUndef() && N2Undef) {
2297 if (N1.getOpcode() == ISD::POISON && N2.getOpcode() == ISD::POISON)
2298 return getPOISON(VT);
2299 return getUNDEF(VT);
2300 }
2301
2302 // If Identity shuffle return that node.
2303 bool Identity = true, AllSame = true;
2304 for (int i = 0; i != NElts; ++i) {
2305 if (MaskVec[i] >= 0 && MaskVec[i] != i) Identity = false;
2306 if (MaskVec[i] != MaskVec[0]) AllSame = false;
2307 }
2308 if (Identity && NElts)
2309 return N1;
2310
2311 // Shuffling a constant splat doesn't change the result.
2312 if (N2Undef) {
2313 SDValue V = N1;
2314
2315 // Look through any bitcasts. We check that these don't change the number
2316 // (and size) of elements and just changes their types.
2317 while (V.getOpcode() == ISD::BITCAST)
2318 V = V->getOperand(Num: 0);
2319
2320 // A splat should always show up as a build vector node.
2321 if (auto *BV = dyn_cast<BuildVectorSDNode>(Val&: V)) {
2322 BitVector UndefElements;
2323 SDValue Splat = BV->getSplatValue(UndefElements: &UndefElements);
2324 // If this is a splat of an undef, shuffling it is also undef.
2325 if (Splat && Splat.isUndef())
2326 return Splat.getOpcode() == ISD::POISON ? getPOISON(VT) : getUNDEF(VT);
2327
2328 bool SameNumElts =
2329 V.getValueType().getVectorNumElements() == VT.getVectorNumElements();
2330
2331 // We only have a splat which can skip shuffles if there is a splatted
2332 // value and no undef lanes rearranged by the shuffle.
2333 if (Splat && UndefElements.none()) {
2334 // Splat of <x, x, ..., x>, return <x, x, ..., x>, provided that the
2335 // number of elements match or the value splatted is a zero constant.
2336 if (SameNumElts || isNullConstant(V: Splat))
2337 return N1;
2338 }
2339
2340 // If the shuffle itself creates a splat, build the vector directly.
2341 if (AllSame && SameNumElts) {
2342 EVT BuildVT = BV->getValueType(ResNo: 0);
2343 const SDValue &Splatted = BV->getOperand(Num: MaskVec[0]);
2344 SDValue NewBV = getSplatBuildVector(VT: BuildVT, DL: dl, Op: Splatted);
2345
2346 // We may have jumped through bitcasts, so the type of the
2347 // BUILD_VECTOR may not match the type of the shuffle.
2348 if (BuildVT != VT)
2349 NewBV = getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: NewBV);
2350 return NewBV;
2351 }
2352 }
2353 }
2354
2355 SDVTList VTs = getVTList(VT);
2356 SDValue Ops[2] = { N1, N2 };
2357 SDNodeKey ID(ISD::VECTOR_SHUFFLE, VTs, Ops);
2358 for (int i = 0; i != NElts; ++i)
2359 ID.AddInteger(I: MaskVec[i]);
2360
2361 FoldingSetInsertToken InsertToken;
2362 if (SDNode *E = lookupNode(Key: ID, DL: dl, InsertToken))
2363 return SDValue(E, 0);
2364
2365 // Allocate the mask array for the node out of the BumpPtrAllocator, since
2366 // SDNode doesn't have access to it. This memory will be "leaked" when
2367 // the node is deallocated, but recovered when the NodeAllocator is released.
2368 int *MaskAlloc = OperandAllocator.Allocate<int>(Num: NElts);
2369 llvm::copy(Range&: MaskVec, Out: MaskAlloc);
2370
2371 auto *N = newSDNode<ShuffleVectorSDNode>(Args&: VTs, Args: dl.getIROrder(),
2372 Args: dl.getDebugLoc(), Args&: MaskAlloc);
2373 createOperands(Node: N, Vals: Ops);
2374
2375 CSEMap.insert(N, Token: InsertToken);
2376 InsertNode(N);
2377 SDValue V = SDValue(N, 0);
2378 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
2379 return V;
2380}
2381
2382SDValue SelectionDAG::getCommutedVectorShuffle(const ShuffleVectorSDNode &SV) {
2383 EVT VT = SV.getValueType(ResNo: 0);
2384 SmallVector<int, 8> MaskVec(SV.getMask());
2385 ShuffleVectorSDNode::commuteMask(Mask: MaskVec);
2386
2387 SDValue Op0 = SV.getOperand(Num: 0);
2388 SDValue Op1 = SV.getOperand(Num: 1);
2389 return getVectorShuffle(VT, dl: SDLoc(&SV), N1: Op1, N2: Op0, Mask: MaskVec);
2390}
2391
2392SDValue SelectionDAG::getRegister(Register Reg, EVT VT) {
2393 SDVTList VTs = getVTList(VT);
2394 SDNodeKey ID(ISD::Register, VTs, {});
2395 ID.AddInteger(I: Reg.id());
2396 FoldingSetInsertToken InsertToken;
2397 if (SDNode *E = lookupNode(Key: ID, InsertToken))
2398 return SDValue(E, 0);
2399
2400 auto *N = newSDNode<RegisterSDNode>(Args&: Reg, Args&: VTs);
2401 N->SDNodeBits.IsDivergent = TLI->isSDNodeSourceOfDivergence(N, FLI, UA);
2402 CSEMap.insert(N, Token: InsertToken);
2403 InsertNode(N);
2404 return SDValue(N, 0);
2405}
2406
2407SDValue SelectionDAG::getRegisterMask(const uint32_t *RegMask) {
2408 SDNodeKey ID(ISD::RegisterMask, getVTList(VT: MVT::Untyped), {});
2409 ID.AddPointer(P: RegMask);
2410 FoldingSetInsertToken InsertToken;
2411 if (SDNode *E = lookupNode(Key: ID, InsertToken))
2412 return SDValue(E, 0);
2413
2414 auto *N = newSDNode<RegisterMaskSDNode>(Args&: RegMask);
2415 CSEMap.insert(N, Token: InsertToken);
2416 InsertNode(N);
2417 return SDValue(N, 0);
2418}
2419
2420SDValue SelectionDAG::getEHLabel(const SDLoc &dl, SDValue Root,
2421 MCSymbol *Label) {
2422 return getLabelNode(Opcode: ISD::EH_LABEL, dl, Root, Label);
2423}
2424
2425SDValue SelectionDAG::getLabelNode(unsigned Opcode, const SDLoc &dl,
2426 SDValue Root, MCSymbol *Label) {
2427 SDValue Ops[] = { Root };
2428 SDNodeKey ID(Opcode, getVTList(VT: MVT::Other), Ops);
2429 ID.AddPointer(P: Label);
2430 FoldingSetInsertToken InsertToken;
2431 if (SDNode *E = lookupNode(Key: ID, InsertToken))
2432 return SDValue(E, 0);
2433
2434 auto *N =
2435 newSDNode<LabelSDNode>(Args&: Opcode, Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: Label);
2436 createOperands(Node: N, Vals: Ops);
2437
2438 CSEMap.insert(N, Token: InsertToken);
2439 InsertNode(N);
2440 return SDValue(N, 0);
2441}
2442
2443SDValue SelectionDAG::getBlockAddress(const BlockAddress *BA, EVT VT,
2444 int64_t Offset, bool isTarget,
2445 unsigned TargetFlags) {
2446 unsigned Opc = isTarget ? ISD::TargetBlockAddress : ISD::BlockAddress;
2447 SDVTList VTs = getVTList(VT);
2448
2449 SDNodeKey ID(Opc, VTs, {});
2450 ID.AddPointer(P: BA);
2451 ID.AddInteger(I: Offset);
2452 ID.AddInteger(I: TargetFlags);
2453 FoldingSetInsertToken InsertToken;
2454 if (SDNode *E = lookupNode(Key: ID, InsertToken))
2455 return SDValue(E, 0);
2456
2457 auto *N = newSDNode<BlockAddressSDNode>(Args&: Opc, Args&: VTs, Args&: BA, Args&: Offset, Args&: TargetFlags);
2458 CSEMap.insert(N, Token: InsertToken);
2459 InsertNode(N);
2460 return SDValue(N, 0);
2461}
2462
2463SDValue SelectionDAG::getSrcValue(const Value *V) {
2464 SDNodeKey ID(ISD::SRCVALUE, getVTList(VT: MVT::Other), {});
2465 ID.AddPointer(P: V);
2466
2467 FoldingSetInsertToken InsertToken;
2468 if (SDNode *E = lookupNode(Key: ID, InsertToken))
2469 return SDValue(E, 0);
2470
2471 auto *N = newSDNode<SrcValueSDNode>(Args&: V);
2472 CSEMap.insert(N, Token: InsertToken);
2473 InsertNode(N);
2474 return SDValue(N, 0);
2475}
2476
2477SDValue SelectionDAG::getMDNode(const MDNode *MD) {
2478 SDNodeKey ID(ISD::MDNODE_SDNODE, getVTList(VT: MVT::Other), {});
2479 ID.AddPointer(P: MD);
2480
2481 FoldingSetInsertToken InsertToken;
2482 if (SDNode *E = lookupNode(Key: ID, InsertToken))
2483 return SDValue(E, 0);
2484
2485 auto *N = newSDNode<MDNodeSDNode>(Args&: MD);
2486 CSEMap.insert(N, Token: InsertToken);
2487 InsertNode(N);
2488 return SDValue(N, 0);
2489}
2490
2491SDValue SelectionDAG::getBitcast(EVT VT, SDValue V) {
2492 if (VT == V.getValueType())
2493 return V;
2494
2495 return getNode(Opcode: ISD::BITCAST, DL: SDLoc(V), VT, Operand: V);
2496}
2497
2498SDValue SelectionDAG::getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr,
2499 unsigned SrcAS, unsigned DestAS,
2500 const SDNodeFlags Flags) {
2501 SDVTList VTs = getVTList(VT);
2502 SDValue Ops[] = {Ptr};
2503 SDNodeKey ID(ISD::ADDRSPACECAST, VTs, Ops);
2504 ID.AddInteger(I: SrcAS);
2505 ID.AddInteger(I: DestAS);
2506
2507 FoldingSetInsertToken InsertToken;
2508 if (SDNode *E = lookupNode(Key: ID, DL: dl, InsertToken)) {
2509 E->intersectFlagsWith(Flags);
2510 return SDValue(E, 0);
2511 }
2512
2513 auto *N = newSDNode<AddrSpaceCastSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(),
2514 Args&: VTs, Args&: SrcAS, Args&: DestAS);
2515 N->setFlags(Flags);
2516 createOperands(Node: N, Vals: Ops);
2517
2518 CSEMap.insert(N, Token: InsertToken);
2519 InsertNode(N);
2520 return SDValue(N, 0);
2521}
2522
2523SDValue SelectionDAG::getFreeze(SDValue V) {
2524 return getNode(Opcode: ISD::FREEZE, DL: SDLoc(V), VT: V.getValueType(), Operand: V);
2525}
2526
2527SDValue SelectionDAG::getFreeze(SDValue V, const APInt &DemandedElts,
2528 UndefPoisonKind Kind) {
2529 if (isGuaranteedNotToBeUndefOrPoison(Op: V, DemandedElts, Kind))
2530 return V;
2531 return getFreeze(V);
2532}
2533
2534/// getShiftAmountOperand - Return the specified value casted to
2535/// the target's desired shift amount type.
2536SDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) {
2537 EVT OpTy = Op.getValueType();
2538 EVT ShTy = TLI->getShiftAmountTy(LHSTy, DL: getDataLayout());
2539 if (OpTy == ShTy || OpTy.isVector()) return Op;
2540
2541 return getZExtOrTrunc(Op, DL: SDLoc(Op), VT: ShTy);
2542}
2543
2544SDValue SelectionDAG::expandVAArg(SDNode *Node) {
2545 SDLoc dl(Node);
2546 const TargetLowering &TLI = getTargetLoweringInfo();
2547 const Value *V = cast<SrcValueSDNode>(Val: Node->getOperand(Num: 2))->getValue();
2548 EVT VT = Node->getValueType(ResNo: 0);
2549 SDValue Tmp1 = Node->getOperand(Num: 0);
2550 SDValue Tmp2 = Node->getOperand(Num: 1);
2551 const MaybeAlign MA(Node->getConstantOperandVal(Num: 3));
2552
2553 SDValue VAListLoad = getLoad(VT: TLI.getPointerTy(DL: getDataLayout()), dl, Chain: Tmp1,
2554 Ptr: Tmp2, PtrInfo: MachinePointerInfo(V));
2555 SDValue VAList = VAListLoad;
2556
2557 if (MA && *MA > TLI.getMinStackArgumentAlignment()) {
2558 VAList = getNode(Opcode: ISD::ADD, DL: dl, VT: VAList.getValueType(), N1: VAList,
2559 N2: getConstant(Val: MA->value() - 1, DL: dl, VT: VAList.getValueType()));
2560
2561 VAList = getNode(
2562 Opcode: ISD::AND, DL: dl, VT: VAList.getValueType(), N1: VAList,
2563 N2: getSignedConstant(Val: -(int64_t)MA->value(), DL: dl, VT: VAList.getValueType()));
2564 }
2565
2566 // Increment the pointer, VAList, to the next vaarg
2567 Tmp1 = getNode(Opcode: ISD::ADD, DL: dl, VT: VAList.getValueType(), N1: VAList,
2568 N2: getConstant(Val: getDataLayout().getTypeAllocSize(
2569 Ty: VT.getTypeForEVT(Context&: *getContext())),
2570 DL: dl, VT: VAList.getValueType()));
2571 // Store the incremented VAList to the legalized pointer
2572 Tmp1 =
2573 getStore(Chain: VAListLoad.getValue(R: 1), dl, Val: Tmp1, Ptr: Tmp2, PtrInfo: MachinePointerInfo(V));
2574 // Load the actual argument out of the pointer VAList
2575 return getLoad(VT, dl, Chain: Tmp1, Ptr: VAList, PtrInfo: MachinePointerInfo());
2576}
2577
2578SDValue SelectionDAG::expandVACopy(SDNode *Node) {
2579 SDLoc dl(Node);
2580 const TargetLowering &TLI = getTargetLoweringInfo();
2581 // This defaults to loading a pointer from the input and storing it to the
2582 // output, returning the chain.
2583 const Value *VD = cast<SrcValueSDNode>(Val: Node->getOperand(Num: 3))->getValue();
2584 const Value *VS = cast<SrcValueSDNode>(Val: Node->getOperand(Num: 4))->getValue();
2585 SDValue Tmp1 =
2586 getLoad(VT: TLI.getPointerTy(DL: getDataLayout()), dl, Chain: Node->getOperand(Num: 0),
2587 Ptr: Node->getOperand(Num: 2), PtrInfo: MachinePointerInfo(VS));
2588 return getStore(Chain: Tmp1.getValue(R: 1), dl, Val: Tmp1, Ptr: Node->getOperand(Num: 1),
2589 PtrInfo: MachinePointerInfo(VD));
2590}
2591
2592Align SelectionDAG::getReducedAlign(EVT VT, bool UseABI) {
2593 const DataLayout &DL = getDataLayout();
2594 Type *Ty = VT.getTypeForEVT(Context&: *getContext());
2595 Align RedAlign = UseABI ? DL.getABITypeAlign(Ty) : DL.getPrefTypeAlign(Ty);
2596
2597 if (TLI->isTypeLegal(VT) || !VT.isVector())
2598 return RedAlign;
2599
2600 const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering();
2601 const Align StackAlign = TFI->getStackAlign();
2602
2603 // See if we can choose a smaller ABI alignment in cases where it's an
2604 // illegal vector type that will get broken down.
2605 if (RedAlign > StackAlign) {
2606 EVT IntermediateVT;
2607 MVT RegisterVT;
2608 unsigned NumIntermediates;
2609 TLI->getVectorTypeBreakdown(Context&: *getContext(), VT, IntermediateVT,
2610 NumIntermediates, RegisterVT);
2611 Ty = IntermediateVT.getTypeForEVT(Context&: *getContext());
2612 Align RedAlign2 = UseABI ? DL.getABITypeAlign(Ty) : DL.getPrefTypeAlign(Ty);
2613 if (RedAlign2 < RedAlign)
2614 RedAlign = RedAlign2;
2615
2616 if (!getMachineFunction().getFrameInfo().isStackRealignable())
2617 // If the stack is not realignable, the alignment should be limited to the
2618 // StackAlignment
2619 RedAlign = std::min(a: RedAlign, b: StackAlign);
2620 }
2621
2622 return RedAlign;
2623}
2624
2625SDValue SelectionDAG::CreateStackTemporary(TypeSize Bytes, Align Alignment) {
2626 MachineFrameInfo &MFI = MF->getFrameInfo();
2627 const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering();
2628 int StackID = 0;
2629 if (Bytes.isScalable())
2630 StackID = TFI->getStackIDForScalableVectors();
2631 // The stack id gives an indication of whether the object is scalable or
2632 // not, so it's safe to pass in the minimum size here.
2633 int FrameIdx = MFI.CreateStackObject(Size: Bytes.getKnownMinValue(), Alignment,
2634 isSpillSlot: false, Alloca: nullptr, ID: StackID);
2635 return getFrameIndex(FI: FrameIdx, VT: TLI->getFrameIndexTy(DL: getDataLayout()));
2636}
2637
2638SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) {
2639 Type *Ty = VT.getTypeForEVT(Context&: *getContext());
2640 Align StackAlign =
2641 std::max(a: getDataLayout().getPrefTypeAlign(Ty), b: Align(minAlign));
2642 return CreateStackTemporary(Bytes: VT.getStoreSize(), Alignment: StackAlign);
2643}
2644
2645SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) {
2646 TypeSize VT1Size = VT1.getStoreSize();
2647 TypeSize VT2Size = VT2.getStoreSize();
2648 assert(VT1Size.isScalable() == VT2Size.isScalable() &&
2649 "Don't know how to choose the maximum size when creating a stack "
2650 "temporary");
2651 TypeSize Bytes = VT1Size.getKnownMinValue() > VT2Size.getKnownMinValue()
2652 ? VT1Size
2653 : VT2Size;
2654
2655 Type *Ty1 = VT1.getTypeForEVT(Context&: *getContext());
2656 Type *Ty2 = VT2.getTypeForEVT(Context&: *getContext());
2657 const DataLayout &DL = getDataLayout();
2658 Align Align = std::max(a: DL.getPrefTypeAlign(Ty: Ty1), b: DL.getPrefTypeAlign(Ty: Ty2));
2659 return CreateStackTemporary(Bytes, Alignment: Align);
2660}
2661
2662SDValue SelectionDAG::emitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
2663 const SDLoc &DL, SDValue Chain) {
2664 EVT SrcVT = SrcOp.getValueType();
2665 Type *DestType = DestVT.getTypeForEVT(Context&: *getContext());
2666 Align DestAlign = getDataLayout().getPrefTypeAlign(Ty: DestType);
2667
2668 // Create the stack frame object.
2669 Align SrcAlign =
2670 getDataLayout().getPrefTypeAlign(Ty: SrcVT.getTypeForEVT(Context&: *getContext()));
2671 SDValue FIPtr = CreateStackTemporary(Bytes: SlotVT.getStoreSize(), Alignment: SrcAlign);
2672
2673 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(Val&: FIPtr);
2674 int SPFI = StackPtrFI->getIndex();
2675 MachinePointerInfo PtrInfo =
2676 MachinePointerInfo::getFixedStack(MF&: getMachineFunction(), FI: SPFI);
2677
2678 // Emit a store to the stack slot. Use a truncstore if the input value is
2679 // later than DestVT.
2680 SDValue Store;
2681
2682 if (SrcVT.bitsGT(VT: SlotVT))
2683 Store = getTruncStore(Chain, dl: DL, Val: SrcOp, Ptr: FIPtr, PtrInfo, SVT: SlotVT, Alignment: SrcAlign);
2684 else {
2685 assert(SrcVT.bitsEq(SlotVT) && "Invalid store");
2686 Store = getStore(Chain, dl: DL, Val: SrcOp, Ptr: FIPtr, PtrInfo, Alignment: SrcAlign);
2687 }
2688
2689 // Result is a load from the stack slot.
2690 if (SlotVT.bitsEq(VT: DestVT))
2691 return getLoad(VT: DestVT, dl: DL, Chain: Store, Ptr: FIPtr, PtrInfo, Alignment: DestAlign);
2692
2693 assert(SlotVT.bitsLT(DestVT) && "Unknown extension!");
2694 return getExtLoad(ExtType: ISD::EXTLOAD, dl: DL, VT: DestVT, Chain: Store, Ptr: FIPtr, PtrInfo, MemVT: SlotVT,
2695 Alignment: DestAlign);
2696}
2697
2698SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1, SDValue N2,
2699 ISD::CondCode Cond, const SDLoc &dl,
2700 SDNodeFlags Flags) {
2701 EVT OpVT = N1.getValueType();
2702
2703 auto GetUndefBooleanConstant = [&]() {
2704 if (VT.getScalarType() == MVT::i1 ||
2705 TLI->getBooleanContents(Type: OpVT) ==
2706 TargetLowering::UndefinedBooleanContent)
2707 return getUNDEF(VT);
2708 // ZeroOrOne / ZeroOrNegative require specific values for the high bits,
2709 // so we cannot use getUNDEF(). Return zero instead.
2710 return getConstant(Val: 0, DL: dl, VT);
2711 };
2712
2713 // These setcc operations always fold.
2714 switch (Cond) {
2715 default: break;
2716 case ISD::SETFALSE:
2717 case ISD::SETFALSE2: return getBoolConstant(V: false, DL: dl, VT, OpVT);
2718 case ISD::SETTRUE:
2719 case ISD::SETTRUE2: return getBoolConstant(V: true, DL: dl, VT, OpVT);
2720
2721 case ISD::SETOEQ:
2722 case ISD::SETOGT:
2723 case ISD::SETOGE:
2724 case ISD::SETOLT:
2725 case ISD::SETOLE:
2726 case ISD::SETONE:
2727 case ISD::SETO:
2728 case ISD::SETUO:
2729 case ISD::SETUEQ:
2730 case ISD::SETUNE:
2731 assert(!OpVT.isInteger() && "Illegal setcc for integer!");
2732 break;
2733 }
2734
2735 if (OpVT.isInteger()) {
2736 // For EQ and NE, we can always pick a value for the undef to make the
2737 // predicate pass or fail, so we can return undef.
2738 // Matches behavior in llvm::ConstantFoldCompareInstruction.
2739 // icmp eq/ne X, undef -> undef.
2740 if ((N1.isUndef() || N2.isUndef()) &&
2741 (Cond == ISD::SETEQ || Cond == ISD::SETNE))
2742 return GetUndefBooleanConstant();
2743
2744 // If both operands are undef, we can return undef for int comparison.
2745 // icmp undef, undef -> undef.
2746 if (N1.isUndef() && N2.isUndef())
2747 return GetUndefBooleanConstant();
2748
2749 // icmp X, X -> true/false
2750 // icmp X, undef -> true/false because undef could be X.
2751 if (N1.isUndef() || N2.isUndef() || N1 == N2)
2752 return getBoolConstant(V: ISD::isTrueWhenEqual(Cond), DL: dl, VT, OpVT);
2753 }
2754
2755 if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(Val&: N2)) {
2756 const APInt &C2 = N2C->getAPIntValue();
2757 if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(Val&: N1)) {
2758 const APInt &C1 = N1C->getAPIntValue();
2759
2760 return getBoolConstant(V: ICmpInst::compare(LHS: C1, RHS: C2, Pred: getICmpCondCode(Pred: Cond)),
2761 DL: dl, VT, OpVT);
2762 }
2763 }
2764
2765 auto *N1CFP = dyn_cast<ConstantFPSDNode>(Val&: N1);
2766 auto *N2CFP = dyn_cast<ConstantFPSDNode>(Val&: N2);
2767
2768 if (N1CFP && N2CFP) {
2769 APFloat::cmpResult R = N1CFP->getValueAPF().compare(RHS: N2CFP->getValueAPF());
2770 switch (Cond) {
2771 default: break;
2772 case ISD::SETEQ: if (R==APFloat::cmpUnordered)
2773 return GetUndefBooleanConstant();
2774 [[fallthrough]];
2775 case ISD::SETOEQ: return getBoolConstant(V: R==APFloat::cmpEqual, DL: dl, VT,
2776 OpVT);
2777 case ISD::SETNE: if (R==APFloat::cmpUnordered)
2778 return GetUndefBooleanConstant();
2779 [[fallthrough]];
2780 case ISD::SETONE: return getBoolConstant(V: R==APFloat::cmpGreaterThan ||
2781 R==APFloat::cmpLessThan, DL: dl, VT,
2782 OpVT);
2783 case ISD::SETLT: if (R==APFloat::cmpUnordered)
2784 return GetUndefBooleanConstant();
2785 [[fallthrough]];
2786 case ISD::SETOLT: return getBoolConstant(V: R==APFloat::cmpLessThan, DL: dl, VT,
2787 OpVT);
2788 case ISD::SETGT: if (R==APFloat::cmpUnordered)
2789 return GetUndefBooleanConstant();
2790 [[fallthrough]];
2791 case ISD::SETOGT: return getBoolConstant(V: R==APFloat::cmpGreaterThan, DL: dl,
2792 VT, OpVT);
2793 case ISD::SETLE: if (R==APFloat::cmpUnordered)
2794 return GetUndefBooleanConstant();
2795 [[fallthrough]];
2796 case ISD::SETOLE: return getBoolConstant(V: R==APFloat::cmpLessThan ||
2797 R==APFloat::cmpEqual, DL: dl, VT,
2798 OpVT);
2799 case ISD::SETGE: if (R==APFloat::cmpUnordered)
2800 return GetUndefBooleanConstant();
2801 [[fallthrough]];
2802 case ISD::SETOGE: return getBoolConstant(V: R==APFloat::cmpGreaterThan ||
2803 R==APFloat::cmpEqual, DL: dl, VT, OpVT);
2804 case ISD::SETO: return getBoolConstant(V: R!=APFloat::cmpUnordered, DL: dl, VT,
2805 OpVT);
2806 case ISD::SETUO: return getBoolConstant(V: R==APFloat::cmpUnordered, DL: dl, VT,
2807 OpVT);
2808 case ISD::SETUEQ: return getBoolConstant(V: R==APFloat::cmpUnordered ||
2809 R==APFloat::cmpEqual, DL: dl, VT,
2810 OpVT);
2811 case ISD::SETUNE: return getBoolConstant(V: R!=APFloat::cmpEqual, DL: dl, VT,
2812 OpVT);
2813 case ISD::SETULT: return getBoolConstant(V: R==APFloat::cmpUnordered ||
2814 R==APFloat::cmpLessThan, DL: dl, VT,
2815 OpVT);
2816 case ISD::SETUGT: return getBoolConstant(V: R==APFloat::cmpGreaterThan ||
2817 R==APFloat::cmpUnordered, DL: dl, VT,
2818 OpVT);
2819 case ISD::SETULE: return getBoolConstant(V: R!=APFloat::cmpGreaterThan, DL: dl,
2820 VT, OpVT);
2821 case ISD::SETUGE: return getBoolConstant(V: R!=APFloat::cmpLessThan, DL: dl, VT,
2822 OpVT);
2823 }
2824 } else if (N1CFP && OpVT.isSimple() && !N2.isUndef()) {
2825 // Ensure that the constant occurs on the RHS.
2826 ISD::CondCode SwappedCond = ISD::getSetCCSwappedOperands(Operation: Cond);
2827 if (!TLI->isCondCodeLegal(CC: SwappedCond, VT: OpVT.getSimpleVT()))
2828 return SDValue();
2829 return getSetCC(DL: dl, VT, LHS: N2, RHS: N1, Cond: SwappedCond, /*Chain=*/{},
2830 /*IsSignaling=*/false, Flags);
2831 } else if ((N2CFP && N2CFP->getValueAPF().isNaN()) ||
2832 (OpVT.isFloatingPoint() && (N1.isUndef() || N2.isUndef()))) {
2833 // If an operand is known to be a nan (or undef that could be a nan), we can
2834 // fold it.
2835 // Choosing NaN for the undef will always make unordered comparison succeed
2836 // and ordered comparison fails.
2837 // Matches behavior in llvm::ConstantFoldCompareInstruction.
2838 switch (ISD::getUnorderedFlavor(Cond)) {
2839 default:
2840 llvm_unreachable("Unknown flavor!");
2841 case 0: // Known false.
2842 return getBoolConstant(V: false, DL: dl, VT, OpVT);
2843 case 1: // Known true.
2844 return getBoolConstant(V: true, DL: dl, VT, OpVT);
2845 case 2: // Undefined.
2846 return GetUndefBooleanConstant();
2847 }
2848 }
2849
2850 // Could not fold it.
2851 return SDValue();
2852}
2853
2854/// SignBitIsZero - Return true if the sign bit of Op is known to be zero. We
2855/// use this predicate to simplify operations downstream.
2856bool SelectionDAG::SignBitIsZero(SDValue Op, unsigned Depth) const {
2857 unsigned BitWidth = Op.getScalarValueSizeInBits();
2858 return MaskedValueIsZero(Op, Mask: APInt::getSignMask(BitWidth), Depth);
2859}
2860
2861// TODO: Should have argument to specify if sign bit of nan is ignorable.
2862bool SelectionDAG::SignBitIsZeroFP(SDValue Op, unsigned Depth) const {
2863 if (Depth >= MaxRecursionDepth)
2864 return false; // Limit search depth.
2865
2866 unsigned Opc = Op.getOpcode();
2867 switch (Opc) {
2868 case ISD::FABS:
2869 return true;
2870 case ISD::AssertNoFPClass: {
2871 FPClassTest NoFPClass =
2872 static_cast<FPClassTest>(Op.getConstantOperandVal(i: 1));
2873
2874 const FPClassTest TestMask = fcNan | fcNegative;
2875 return (NoFPClass & TestMask) == TestMask;
2876 }
2877 case ISD::ARITH_FENCE:
2878 return SignBitIsZeroFP(Op: Op.getOperand(i: 0), Depth: Depth + 1);
2879 case ISD::FEXP:
2880 case ISD::FEXP2:
2881 case ISD::FEXP10:
2882 return Op->getFlags().hasNoNaNs();
2883 case ISD::FMINNUM:
2884 case ISD::FMINNUM_IEEE:
2885 case ISD::FMINIMUM:
2886 case ISD::FMINIMUMNUM:
2887 return SignBitIsZeroFP(Op: Op.getOperand(i: 1), Depth: Depth + 1) &&
2888 SignBitIsZeroFP(Op: Op.getOperand(i: 0), Depth: Depth + 1);
2889 case ISD::FMAXNUM:
2890 case ISD::FMAXNUM_IEEE:
2891 case ISD::FMAXIMUM:
2892 case ISD::FMAXIMUMNUM:
2893 // TODO: If we can ignore the sign bit of nans, only one side being known 0
2894 // is sufficient.
2895 return SignBitIsZeroFP(Op: Op.getOperand(i: 1), Depth: Depth + 1) &&
2896 SignBitIsZeroFP(Op: Op.getOperand(i: 0), Depth: Depth + 1);
2897 default:
2898 return false;
2899 }
2900
2901 llvm_unreachable("covered opcode switch");
2902}
2903
2904/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero. We use
2905/// this predicate to simplify operations downstream. Mask is known to be zero
2906/// for bits that V cannot have.
2907bool SelectionDAG::MaskedValueIsZero(SDValue V, const APInt &Mask,
2908 unsigned Depth) const {
2909 return Mask.isSubsetOf(RHS: computeKnownBits(Op: V, Depth).Zero);
2910}
2911
2912/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero in
2913/// DemandedElts. We use this predicate to simplify operations downstream.
2914/// Mask is known to be zero for bits that V cannot have.
2915bool SelectionDAG::MaskedValueIsZero(SDValue V, const APInt &Mask,
2916 const APInt &DemandedElts,
2917 unsigned Depth) const {
2918 return Mask.isSubsetOf(RHS: computeKnownBits(Op: V, DemandedElts, Depth).Zero);
2919}
2920
2921/// MaskedVectorIsZero - Return true if 'Op' is known to be zero in
2922/// DemandedElts. We use this predicate to simplify operations downstream.
2923bool SelectionDAG::MaskedVectorIsZero(SDValue V, const APInt &DemandedElts,
2924 unsigned Depth /* = 0 */) const {
2925 return computeKnownBits(Op: V, DemandedElts, Depth).isZero();
2926}
2927
2928/// MaskedValueIsAllOnes - Return true if '(Op & Mask) == Mask'.
2929bool SelectionDAG::MaskedValueIsAllOnes(SDValue V, const APInt &Mask,
2930 unsigned Depth) const {
2931 return Mask.isSubsetOf(RHS: computeKnownBits(Op: V, Depth).One);
2932}
2933
2934APInt SelectionDAG::computeVectorKnownZeroElements(SDValue Op,
2935 const APInt &DemandedElts,
2936 unsigned Depth) const {
2937 EVT VT = Op.getValueType();
2938 assert(VT.isVector() && !VT.isScalableVector() && "Only for fixed vectors!");
2939
2940 unsigned NumElts = VT.getVectorNumElements();
2941 assert(DemandedElts.getBitWidth() == NumElts && "Unexpected demanded mask.");
2942
2943 APInt KnownZeroElements = APInt::getZero(numBits: NumElts);
2944 for (unsigned EltIdx = 0; EltIdx != NumElts; ++EltIdx) {
2945 if (!DemandedElts[EltIdx])
2946 continue; // Don't query elements that are not demanded.
2947 APInt Mask = APInt::getOneBitSet(numBits: NumElts, BitNo: EltIdx);
2948 if (MaskedVectorIsZero(V: Op, DemandedElts: Mask, Depth))
2949 KnownZeroElements.setBit(EltIdx);
2950 }
2951 return KnownZeroElements;
2952}
2953
2954/// isSplatValue - Return true if the vector V has the same value
2955/// across all DemandedElts. For scalable vectors, we don't know the
2956/// number of lanes at compile time. Instead, we use a 1 bit APInt
2957/// to represent a conservative value for all lanes; that is, that
2958/// one bit value is implicitly splatted across all lanes.
2959bool SelectionDAG::isSplatValue(SDValue V, const APInt &DemandedElts,
2960 APInt &UndefElts, unsigned Depth) const {
2961 unsigned Opcode = V.getOpcode();
2962 EVT VT = V.getValueType();
2963 assert(VT.isVector() && "Vector type expected");
2964 assert((!VT.isScalableVector() || DemandedElts.getBitWidth() == 1) &&
2965 "scalable demanded bits are ignored");
2966
2967 if (!DemandedElts)
2968 return false; // No demanded elts, better to assume we don't know anything.
2969
2970 if (Depth >= MaxRecursionDepth)
2971 return false; // Limit search depth.
2972
2973 // Deal with some common cases here that work for both fixed and scalable
2974 // vector types.
2975 switch (Opcode) {
2976 case ISD::SPLAT_VECTOR:
2977 UndefElts = V.getOperand(i: 0).isUndef()
2978 ? APInt::getAllOnes(numBits: DemandedElts.getBitWidth())
2979 : APInt(DemandedElts.getBitWidth(), 0);
2980 return true;
2981 case ISD::ADD:
2982 case ISD::SUB:
2983 case ISD::AND:
2984 case ISD::XOR:
2985 case ISD::OR: {
2986 APInt UndefLHS, UndefRHS;
2987 SDValue LHS = V.getOperand(i: 0);
2988 SDValue RHS = V.getOperand(i: 1);
2989 // Only recognize splats with the same demanded undef elements for both
2990 // operands, otherwise we might fail to handle binop-specific undef
2991 // handling.
2992 // e.g. (and undef, 0) -> 0 etc.
2993 if (isSplatValue(V: LHS, DemandedElts, UndefElts&: UndefLHS, Depth: Depth + 1) &&
2994 isSplatValue(V: RHS, DemandedElts, UndefElts&: UndefRHS, Depth: Depth + 1) &&
2995 (DemandedElts & UndefLHS) == (DemandedElts & UndefRHS)) {
2996 UndefElts = UndefLHS | UndefRHS;
2997 return true;
2998 }
2999 return false;
3000 }
3001 case ISD::ABS:
3002 case ISD::ABS_MIN_POISON:
3003 case ISD::TRUNCATE:
3004 case ISD::SIGN_EXTEND:
3005 case ISD::ZERO_EXTEND:
3006 return isSplatValue(V: V.getOperand(i: 0), DemandedElts, UndefElts, Depth: Depth + 1);
3007 default:
3008 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
3009 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)
3010 return TLI->isSplatValueForTargetNode(Op: V, DemandedElts, UndefElts, DAG: *this,
3011 Depth);
3012 break;
3013 }
3014
3015 // We don't support other cases than those above for scalable vectors at
3016 // the moment.
3017 if (VT.isScalableVector())
3018 return false;
3019
3020 unsigned NumElts = VT.getVectorNumElements();
3021 assert(NumElts == DemandedElts.getBitWidth() && "Vector size mismatch");
3022 UndefElts = APInt::getZero(numBits: NumElts);
3023
3024 switch (Opcode) {
3025 case ISD::BUILD_VECTOR: {
3026 SDValue Scl;
3027 for (unsigned i = 0; i != NumElts; ++i) {
3028 SDValue Op = V.getOperand(i);
3029 if (Op.isUndef()) {
3030 UndefElts.setBit(i);
3031 continue;
3032 }
3033 if (!DemandedElts[i])
3034 continue;
3035 if (Scl && Scl != Op)
3036 return false;
3037 Scl = Op;
3038 }
3039 return true;
3040 }
3041 case ISD::VECTOR_SHUFFLE: {
3042 // Check if this is a shuffle node doing a splat or a shuffle of a splat.
3043 APInt DemandedLHS = APInt::getZero(numBits: NumElts);
3044 APInt DemandedRHS = APInt::getZero(numBits: NumElts);
3045 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Val&: V)->getMask();
3046 for (int i = 0; i != (int)NumElts; ++i) {
3047 int M = Mask[i];
3048 if (M < 0) {
3049 UndefElts.setBit(i);
3050 continue;
3051 }
3052 if (!DemandedElts[i])
3053 continue;
3054 if (M < (int)NumElts)
3055 DemandedLHS.setBit(M);
3056 else
3057 DemandedRHS.setBit(M - NumElts);
3058 }
3059
3060 // If we aren't demanding either op, assume there's no splat.
3061 // If we are demanding both ops, assume there's no splat.
3062 if ((DemandedLHS.isZero() && DemandedRHS.isZero()) ||
3063 (!DemandedLHS.isZero() && !DemandedRHS.isZero()))
3064 return false;
3065
3066 // See if the demanded elts of the source op is a splat or we only demand
3067 // one element, which should always be a splat.
3068 // TODO: Handle source ops splats with undefs.
3069 auto CheckSplatSrc = [&](SDValue Src, const APInt &SrcElts) {
3070 APInt SrcUndefs;
3071 return (SrcElts.popcount() == 1) ||
3072 (isSplatValue(V: Src, DemandedElts: SrcElts, UndefElts&: SrcUndefs, Depth: Depth + 1) &&
3073 (SrcElts & SrcUndefs).isZero());
3074 };
3075 if (!DemandedLHS.isZero())
3076 return CheckSplatSrc(V.getOperand(i: 0), DemandedLHS);
3077 return CheckSplatSrc(V.getOperand(i: 1), DemandedRHS);
3078 }
3079 case ISD::EXTRACT_SUBVECTOR: {
3080 // Offset the demanded elts by the subvector index.
3081 SDValue Src = V.getOperand(i: 0);
3082 // We don't support scalable vectors at the moment.
3083 if (Src.getValueType().isScalableVector())
3084 return false;
3085 uint64_t Idx = V.getConstantOperandVal(i: 1);
3086 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3087 APInt UndefSrcElts;
3088 APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
3089 if (isSplatValue(V: Src, DemandedElts: DemandedSrcElts, UndefElts&: UndefSrcElts, Depth: Depth + 1)) {
3090 UndefElts = UndefSrcElts.extractBits(numBits: NumElts, bitPosition: Idx);
3091 return true;
3092 }
3093 break;
3094 }
3095 case ISD::ANY_EXTEND_VECTOR_INREG:
3096 case ISD::SIGN_EXTEND_VECTOR_INREG:
3097 case ISD::ZERO_EXTEND_VECTOR_INREG: {
3098 // Widen the demanded elts by the src element count.
3099 SDValue Src = V.getOperand(i: 0);
3100 // We don't support scalable vectors at the moment.
3101 if (Src.getValueType().isScalableVector())
3102 return false;
3103 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3104 APInt UndefSrcElts;
3105 APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts);
3106 if (isSplatValue(V: Src, DemandedElts: DemandedSrcElts, UndefElts&: UndefSrcElts, Depth: Depth + 1)) {
3107 UndefElts = UndefSrcElts.trunc(width: NumElts);
3108 return true;
3109 }
3110 break;
3111 }
3112 case ISD::BITCAST: {
3113 SDValue Src = V.getOperand(i: 0);
3114 EVT SrcVT = Src.getValueType();
3115 unsigned SrcBitWidth = SrcVT.getScalarSizeInBits();
3116 unsigned BitWidth = VT.getScalarSizeInBits();
3117
3118 // Ignore bitcasts from unsupported types.
3119 // TODO: Add fp support?
3120 if (!SrcVT.isVector() || !SrcVT.isInteger() || !VT.isInteger())
3121 break;
3122
3123 // Bitcast 'small element' vector to 'large element' vector.
3124 if ((BitWidth % SrcBitWidth) == 0) {
3125 // See if each sub element is a splat.
3126 unsigned Scale = BitWidth / SrcBitWidth;
3127 unsigned NumSrcElts = SrcVT.getVectorNumElements();
3128 APInt ScaledDemandedElts =
3129 APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumSrcElts);
3130 for (unsigned I = 0; I != Scale; ++I) {
3131 APInt SubUndefElts;
3132 APInt SubDemandedElt = APInt::getOneBitSet(numBits: Scale, BitNo: I);
3133 APInt SubDemandedElts = APInt::getSplat(NewLen: NumSrcElts, V: SubDemandedElt);
3134 SubDemandedElts &= ScaledDemandedElts;
3135 if (!isSplatValue(V: Src, DemandedElts: SubDemandedElts, UndefElts&: SubUndefElts, Depth: Depth + 1))
3136 return false;
3137 // TODO: Add support for merging sub undef elements.
3138 if (!SubUndefElts.isZero())
3139 return false;
3140 }
3141 return true;
3142 }
3143 break;
3144 }
3145 }
3146
3147 return false;
3148}
3149
3150/// Helper wrapper to main isSplatValue function.
3151bool SelectionDAG::isSplatValue(SDValue V, bool AllowUndefs) const {
3152 EVT VT = V.getValueType();
3153 assert(VT.isVector() && "Vector type expected");
3154
3155 APInt UndefElts;
3156 // Since the number of lanes in a scalable vector is unknown at compile time,
3157 // we track one bit which is implicitly broadcast to all lanes. This means
3158 // that all lanes in a scalable vector are considered demanded.
3159 APInt DemandedElts
3160 = APInt::getAllOnes(numBits: VT.isScalableVector() ? 1 : VT.getVectorNumElements());
3161 return isSplatValue(V, DemandedElts, UndefElts) &&
3162 (AllowUndefs || !UndefElts);
3163}
3164
3165SDValue SelectionDAG::getSplatSourceVector(SDValue V, int &SplatIdx) {
3166 V = peekThroughExtractSubvectors(V);
3167
3168 EVT VT = V.getValueType();
3169 unsigned Opcode = V.getOpcode();
3170 switch (Opcode) {
3171 default: {
3172 APInt UndefElts;
3173 // Since the number of lanes in a scalable vector is unknown at compile time,
3174 // we track one bit which is implicitly broadcast to all lanes. This means
3175 // that all lanes in a scalable vector are considered demanded.
3176 APInt DemandedElts
3177 = APInt::getAllOnes(numBits: VT.isScalableVector() ? 1 : VT.getVectorNumElements());
3178
3179 if (isSplatValue(V, DemandedElts, UndefElts)) {
3180 if (VT.isScalableVector()) {
3181 // DemandedElts and UndefElts are ignored for scalable vectors, since
3182 // the only supported cases are SPLAT_VECTOR nodes.
3183 SplatIdx = 0;
3184 } else {
3185 // Handle case where all demanded elements are UNDEF.
3186 if (DemandedElts.isSubsetOf(RHS: UndefElts)) {
3187 SplatIdx = 0;
3188 return getUNDEF(VT);
3189 }
3190 SplatIdx = (UndefElts & DemandedElts).countr_one();
3191 }
3192 return V;
3193 }
3194 break;
3195 }
3196 case ISD::SPLAT_VECTOR:
3197 SplatIdx = 0;
3198 return V;
3199 case ISD::VECTOR_SHUFFLE: {
3200 assert(!VT.isScalableVector());
3201 // Check if this is a shuffle node doing a splat.
3202 // TODO - remove this and rely purely on SelectionDAG::isSplatValue,
3203 // getTargetVShiftNode currently struggles without the splat source.
3204 auto *SVN = cast<ShuffleVectorSDNode>(Val&: V);
3205 if (!SVN->isSplat())
3206 break;
3207 int Idx = SVN->getSplatIndex();
3208 int NumElts = V.getValueType().getVectorNumElements();
3209 SplatIdx = Idx % NumElts;
3210 return V.getOperand(i: Idx / NumElts);
3211 }
3212 }
3213
3214 return SDValue();
3215}
3216
3217SDValue SelectionDAG::getSplatValue(SDValue V, bool LegalTypes) {
3218 int SplatIdx;
3219 if (SDValue SrcVector = getSplatSourceVector(V, SplatIdx)) {
3220 EVT SVT = SrcVector.getValueType().getScalarType();
3221 EVT LegalSVT = SVT;
3222 if (LegalTypes && !TLI->isTypeLegal(VT: SVT)) {
3223 if (!SVT.isInteger())
3224 return SDValue();
3225 LegalSVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT: LegalSVT);
3226 if (LegalSVT.bitsLT(VT: SVT))
3227 return SDValue();
3228 }
3229 return getExtractVectorElt(DL: SDLoc(V), VT: LegalSVT, Vec: SrcVector, Idx: SplatIdx);
3230 }
3231 return SDValue();
3232}
3233
3234std::optional<ConstantRange>
3235SelectionDAG::getValidShiftAmountRange(SDValue V, const APInt &DemandedElts,
3236 unsigned Depth) const {
3237 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
3238 V.getOpcode() == ISD::SRA) &&
3239 "Unknown shift node");
3240 // Shifting more than the bitwidth is not valid.
3241 unsigned BitWidth = V.getScalarValueSizeInBits();
3242
3243 if (auto *Cst = dyn_cast<ConstantSDNode>(Val: V.getOperand(i: 1))) {
3244 const APInt &ShAmt = Cst->getAPIntValue();
3245 if (ShAmt.uge(RHS: BitWidth))
3246 return std::nullopt;
3247 return ConstantRange(ShAmt);
3248 }
3249
3250 if (auto *BV = dyn_cast<BuildVectorSDNode>(Val: V.getOperand(i: 1))) {
3251 const APInt *MinAmt = nullptr, *MaxAmt = nullptr;
3252 for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) {
3253 if (!DemandedElts[i])
3254 continue;
3255 auto *SA = dyn_cast<ConstantSDNode>(Val: BV->getOperand(Num: i));
3256 if (!SA) {
3257 MinAmt = MaxAmt = nullptr;
3258 break;
3259 }
3260 const APInt &ShAmt = SA->getAPIntValue();
3261 if (ShAmt.uge(RHS: BitWidth))
3262 return std::nullopt;
3263 if (!MinAmt || MinAmt->ugt(RHS: ShAmt))
3264 MinAmt = &ShAmt;
3265 if (!MaxAmt || MaxAmt->ult(RHS: ShAmt))
3266 MaxAmt = &ShAmt;
3267 }
3268 assert(((!MinAmt && !MaxAmt) || (MinAmt && MaxAmt)) &&
3269 "Failed to find matching min/max shift amounts");
3270 if (MinAmt && MaxAmt)
3271 return ConstantRange(*MinAmt, *MaxAmt + 1);
3272 }
3273
3274 // Use computeKnownBits to find a hidden constant/knownbits (usually type
3275 // legalized). e.g. Hidden behind multiple bitcasts/build_vector/casts etc.
3276 KnownBits KnownAmt = computeKnownBits(Op: V.getOperand(i: 1), DemandedElts, Depth);
3277 if (KnownAmt.getMaxValue().ult(RHS: BitWidth))
3278 return ConstantRange::fromKnownBits(Known: KnownAmt, /*IsSigned=*/false);
3279
3280 return std::nullopt;
3281}
3282
3283std::optional<unsigned>
3284SelectionDAG::getValidShiftAmount(SDValue V, const APInt &DemandedElts,
3285 unsigned Depth) const {
3286 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
3287 V.getOpcode() == ISD::SRA) &&
3288 "Unknown shift node");
3289 if (std::optional<ConstantRange> AmtRange =
3290 getValidShiftAmountRange(V, DemandedElts, Depth))
3291 if (const APInt *ShAmt = AmtRange->getSingleElement())
3292 return ShAmt->getZExtValue();
3293 return std::nullopt;
3294}
3295
3296std::optional<unsigned>
3297SelectionDAG::getValidShiftAmount(SDValue V, unsigned Depth) const {
3298 APInt DemandedElts = getDemandAllEltsMask(V);
3299 return getValidShiftAmount(V, DemandedElts, Depth);
3300}
3301
3302std::optional<unsigned>
3303SelectionDAG::getValidMinimumShiftAmount(SDValue V, const APInt &DemandedElts,
3304 unsigned Depth) const {
3305 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
3306 V.getOpcode() == ISD::SRA) &&
3307 "Unknown shift node");
3308 if (std::optional<ConstantRange> AmtRange =
3309 getValidShiftAmountRange(V, DemandedElts, Depth))
3310 return AmtRange->getUnsignedMin().getZExtValue();
3311 return std::nullopt;
3312}
3313
3314std::optional<unsigned>
3315SelectionDAG::getValidMinimumShiftAmount(SDValue V, unsigned Depth) const {
3316 APInt DemandedElts = getDemandAllEltsMask(V);
3317 return getValidMinimumShiftAmount(V, DemandedElts, Depth);
3318}
3319
3320std::optional<unsigned>
3321SelectionDAG::getValidMaximumShiftAmount(SDValue V, const APInt &DemandedElts,
3322 unsigned Depth) const {
3323 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
3324 V.getOpcode() == ISD::SRA) &&
3325 "Unknown shift node");
3326 if (std::optional<ConstantRange> AmtRange =
3327 getValidShiftAmountRange(V, DemandedElts, Depth))
3328 return AmtRange->getUnsignedMax().getZExtValue();
3329 return std::nullopt;
3330}
3331
3332std::optional<unsigned>
3333SelectionDAG::getValidMaximumShiftAmount(SDValue V, unsigned Depth) const {
3334 APInt DemandedElts = getDemandAllEltsMask(V);
3335 return getValidMaximumShiftAmount(V, DemandedElts, Depth);
3336}
3337
3338/// Determine which bits of Op are known to be either zero or one and return
3339/// them in Known. For vectors, the known bits are those that are shared by
3340/// every vector element.
3341KnownBits SelectionDAG::computeKnownBits(SDValue Op, unsigned Depth) const {
3342 APInt DemandedElts = getDemandAllEltsMask(V: Op);
3343 return computeKnownBits(Op, DemandedElts, Depth);
3344}
3345
3346/// Determine which bits of Op are known to be either zero or one and return
3347/// them in Known. The DemandedElts argument allows us to only collect the known
3348/// bits that are shared by the requested vector elements.
3349KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts,
3350 unsigned Depth) const {
3351 unsigned BitWidth = Op.getScalarValueSizeInBits();
3352
3353 KnownBits Known(BitWidth); // Don't know anything.
3354
3355 if (auto OptAPInt = Op->bitcastToAPInt()) {
3356 // We know all of the bits for a constant!
3357 return KnownBits::makeConstant(C: *std::move(OptAPInt));
3358 }
3359
3360 if (Depth >= MaxRecursionDepth)
3361 return Known; // Limit search depth.
3362
3363 KnownBits Known2;
3364 unsigned NumElts = DemandedElts.getBitWidth();
3365 assert((!Op.getValueType().isScalableVector() || NumElts == 1) &&
3366 "DemandedElts for scalable vectors must be 1 to represent all lanes");
3367 assert((!Op.getValueType().isFixedLengthVector() ||
3368 NumElts == Op.getValueType().getVectorNumElements()) &&
3369 "Unexpected vector size");
3370
3371 if (!DemandedElts)
3372 return Known; // No demanded elts, better to assume we don't know anything.
3373
3374 unsigned Opcode = Op.getOpcode();
3375 switch (Opcode) {
3376 case ISD::FREEZE: {
3377 if (isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 0), DemandedElts,
3378 Kind: UndefPoisonKind::UndefOrPoison))
3379 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3380 break;
3381 }
3382 case ISD::MERGE_VALUES:
3383 return computeKnownBits(Op: Op.getOperand(i: Op.getResNo()), DemandedElts,
3384 Depth: Depth + 1);
3385 case ISD::SPLAT_VECTOR: {
3386 SDValue SrcOp = Op.getOperand(i: 0);
3387 assert(SrcOp.getValueSizeInBits() >= BitWidth &&
3388 "Expected SPLAT_VECTOR implicit truncation");
3389 // Implicitly truncate the bits to match the official semantics of
3390 // SPLAT_VECTOR.
3391 Known = computeKnownBits(Op: SrcOp, Depth: Depth + 1).trunc(BitWidth);
3392 break;
3393 }
3394 case ISD::SPLAT_VECTOR_PARTS: {
3395 unsigned ScalarSize = Op.getOperand(i: 0).getScalarValueSizeInBits();
3396 assert(ScalarSize * Op.getNumOperands() == BitWidth &&
3397 "Expected SPLAT_VECTOR_PARTS scalars to cover element width");
3398 for (auto [I, SrcOp] : enumerate(First: Op->ops())) {
3399 Known.insertBits(SubBits: computeKnownBits(Op: SrcOp, Depth: Depth + 1), BitPosition: ScalarSize * I);
3400 }
3401 break;
3402 }
3403 case ISD::STEP_VECTOR: {
3404 const APInt &Step = Op.getConstantOperandAPInt(i: 0);
3405
3406 if (Step.isPowerOf2())
3407 Known.Zero.setLowBits(Step.logBase2());
3408
3409 const Function &F = getMachineFunction().getFunction();
3410
3411 if (!isUIntN(N: BitWidth, x: Op.getValueType().getVectorMinNumElements()))
3412 break;
3413 const APInt MinNumElts =
3414 APInt(BitWidth, Op.getValueType().getVectorMinNumElements());
3415
3416 bool Overflow;
3417 const APInt MaxNumElts = getVScaleRange(F: &F, BitWidth)
3418 .getUnsignedMax()
3419 .umul_ov(RHS: MinNumElts, Overflow);
3420 if (Overflow)
3421 break;
3422
3423 const APInt MaxValue = (MaxNumElts - 1).umul_ov(RHS: Step, Overflow);
3424 if (Overflow)
3425 break;
3426
3427 Known.Zero.setHighBits(MaxValue.countl_zero());
3428 break;
3429 }
3430 case ISD::BUILD_VECTOR:
3431 assert(!Op.getValueType().isScalableVector());
3432 // Collect the known bits that are shared by every demanded vector element.
3433 Known.setAllConflict();
3434 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
3435 if (!DemandedElts[i])
3436 continue;
3437
3438 SDValue SrcOp = Op.getOperand(i);
3439 if (SrcOp.getOpcode() == ISD::POISON)
3440 continue;
3441
3442 Known2 = computeKnownBits(Op: SrcOp, Depth: Depth + 1);
3443
3444 // BUILD_VECTOR can implicitly truncate sources, we must handle this.
3445 if (SrcOp.getValueSizeInBits() != BitWidth) {
3446 assert(SrcOp.getValueSizeInBits() > BitWidth &&
3447 "Expected BUILD_VECTOR implicit truncation");
3448 Known2 = Known2.trunc(BitWidth);
3449 }
3450
3451 // Known bits are the values that are shared by every demanded element.
3452 Known = Known.intersectWith(RHS: Known2);
3453
3454 // If we don't know any bits, early out.
3455 if (Known.isUnknown())
3456 break;
3457 }
3458
3459 // If every demanded element was poison, we know nothing.
3460 if (Known.hasConflict())
3461 Known.resetAll();
3462 break;
3463 case ISD::VECTOR_COMPRESS: {
3464 SDValue Vec = Op.getOperand(i: 0);
3465 SDValue PassThru = Op.getOperand(i: 2);
3466 Known = computeKnownBits(Op: PassThru, DemandedElts, Depth: Depth + 1);
3467 // If we don't know any bits, early out.
3468 if (Known.isUnknown())
3469 break;
3470 Known2 = computeKnownBits(Op: Vec, Depth: Depth + 1);
3471 Known = Known.intersectWith(RHS: Known2);
3472 break;
3473 }
3474 case ISD::VECTOR_SHUFFLE: {
3475 assert(!Op.getValueType().isScalableVector());
3476 // Collect the known bits that are shared by every vector element referenced
3477 // by the shuffle.
3478 APInt DemandedLHS, DemandedRHS;
3479 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
3480 assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
3481 if (!getShuffleDemandedElts(SrcWidth: NumElts, Mask: SVN->getMask(), DemandedElts,
3482 DemandedLHS, DemandedRHS))
3483 break;
3484
3485 // Known bits are the values that are shared by every demanded element.
3486 Known.setAllConflict();
3487 if (!!DemandedLHS) {
3488 SDValue LHS = Op.getOperand(i: 0);
3489 Known2 = computeKnownBits(Op: LHS, DemandedElts: DemandedLHS, Depth: Depth + 1);
3490 Known = Known.intersectWith(RHS: Known2);
3491 }
3492 // If we don't know any bits, early out.
3493 if (Known.isUnknown())
3494 break;
3495 if (!!DemandedRHS) {
3496 SDValue RHS = Op.getOperand(i: 1);
3497 Known2 = computeKnownBits(Op: RHS, DemandedElts: DemandedRHS, Depth: Depth + 1);
3498 Known = Known.intersectWith(RHS: Known2);
3499 }
3500 break;
3501 }
3502 case ISD::VSCALE: {
3503 const Function &F = getMachineFunction().getFunction();
3504 const APInt &Multiplier = Op.getConstantOperandAPInt(i: 0);
3505 Known = getVScaleRange(F: &F, BitWidth).multiply(Other: Multiplier).toKnownBits();
3506 break;
3507 }
3508 case ISD::CONCAT_VECTORS: {
3509 if (Op.getValueType().isScalableVector())
3510 break;
3511 // Split DemandedElts and test each of the demanded subvectors.
3512 Known.setAllConflict();
3513 EVT SubVectorVT = Op.getOperand(i: 0).getValueType();
3514 unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements();
3515 unsigned NumSubVectors = Op.getNumOperands();
3516 for (unsigned i = 0; i != NumSubVectors; ++i) {
3517 APInt DemandedSub =
3518 DemandedElts.extractBits(numBits: NumSubVectorElts, bitPosition: i * NumSubVectorElts);
3519 if (!!DemandedSub) {
3520 SDValue Sub = Op.getOperand(i);
3521 Known2 = computeKnownBits(Op: Sub, DemandedElts: DemandedSub, Depth: Depth + 1);
3522 Known = Known.intersectWith(RHS: Known2);
3523 }
3524 // If we don't know any bits, early out.
3525 if (Known.isUnknown())
3526 break;
3527 }
3528 break;
3529 }
3530 case ISD::INSERT_SUBVECTOR: {
3531 if (Op.getValueType().isScalableVector())
3532 break;
3533 // Demand any elements from the subvector and the remainder from the src its
3534 // inserted into.
3535 SDValue Src = Op.getOperand(i: 0);
3536 SDValue Sub = Op.getOperand(i: 1);
3537 uint64_t Idx = Op.getConstantOperandVal(i: 2);
3538 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
3539 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
3540 APInt DemandedSrcElts = DemandedElts;
3541 DemandedSrcElts.clearBits(LoBit: Idx, HiBit: Idx + NumSubElts);
3542
3543 Known.setAllConflict();
3544 if (!!DemandedSubElts) {
3545 Known = computeKnownBits(Op: Sub, DemandedElts: DemandedSubElts, Depth: Depth + 1);
3546 if (Known.isUnknown())
3547 break; // early-out.
3548 }
3549 if (!!DemandedSrcElts) {
3550 Known2 = computeKnownBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
3551 Known = Known.intersectWith(RHS: Known2);
3552 }
3553 break;
3554 }
3555 case ISD::EXTRACT_SUBVECTOR: {
3556 // Offset the demanded elts by the subvector index.
3557 SDValue Src = Op.getOperand(i: 0);
3558
3559 APInt DemandedSrcElts;
3560 if (Src.getValueType().isScalableVector())
3561 DemandedSrcElts = APInt(1, 1); // <=> 'demand all elements'
3562 else {
3563 uint64_t Idx = Op.getConstantOperandVal(i: 1);
3564 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3565 DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
3566 }
3567 Known = computeKnownBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
3568 break;
3569 }
3570 case ISD::SCALAR_TO_VECTOR: {
3571 if (Op.getValueType().isScalableVector())
3572 break;
3573 // We know about scalar_to_vector as much as we know about it source,
3574 // which becomes the first element of otherwise unknown vector.
3575 if (DemandedElts != 1)
3576 break;
3577
3578 SDValue N0 = Op.getOperand(i: 0);
3579 Known = computeKnownBits(Op: N0, Depth: Depth + 1);
3580 if (N0.getValueSizeInBits() != BitWidth)
3581 Known = Known.trunc(BitWidth);
3582
3583 break;
3584 }
3585 case ISD::BITCAST: {
3586 if (Op.getValueType().isScalableVector())
3587 break;
3588
3589 SDValue N0 = Op.getOperand(i: 0);
3590 EVT SubVT = N0.getValueType();
3591 unsigned SubBitWidth = SubVT.getScalarSizeInBits();
3592
3593 // Ignore bitcasts from unsupported types.
3594 if (!(SubVT.isInteger() || SubVT.isFloatingPoint()))
3595 break;
3596
3597 // Fast handling of 'identity' bitcasts.
3598 if (BitWidth == SubBitWidth) {
3599 Known = computeKnownBits(Op: N0, DemandedElts, Depth: Depth + 1);
3600 break;
3601 }
3602
3603 bool IsLE = getDataLayout().isLittleEndian();
3604
3605 // Bitcast 'small element' vector to 'large element' scalar/vector.
3606 if ((BitWidth % SubBitWidth) == 0) {
3607 assert(N0.getValueType().isVector() && "Expected bitcast from vector");
3608
3609 // Collect known bits for the (larger) output by collecting the known
3610 // bits from each set of sub elements and shift these into place.
3611 // We need to separately call computeKnownBits for each set of
3612 // sub elements as the knownbits for each is likely to be different.
3613 unsigned SubScale = BitWidth / SubBitWidth;
3614 APInt SubDemandedElts(NumElts * SubScale, 0);
3615 for (unsigned i = 0; i != NumElts; ++i)
3616 if (DemandedElts[i])
3617 SubDemandedElts.setBit(i * SubScale);
3618
3619 for (unsigned i = 0; i != SubScale; ++i) {
3620 Known2 = computeKnownBits(Op: N0, DemandedElts: SubDemandedElts.shl(shiftAmt: i),
3621 Depth: Depth + 1);
3622 unsigned Shifts = IsLE ? i : SubScale - 1 - i;
3623 Known.insertBits(SubBits: Known2, BitPosition: SubBitWidth * Shifts);
3624 }
3625 }
3626
3627 // Bitcast 'large element' scalar/vector to 'small element' vector.
3628 if ((SubBitWidth % BitWidth) == 0) {
3629 assert(Op.getValueType().isVector() && "Expected bitcast to vector");
3630
3631 // Collect known bits for the (smaller) output by collecting the known
3632 // bits from the overlapping larger input elements and extracting the
3633 // sub sections we actually care about.
3634 unsigned SubScale = SubBitWidth / BitWidth;
3635 APInt SubDemandedElts =
3636 APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumElts / SubScale);
3637 Known2 = computeKnownBits(Op: N0, DemandedElts: SubDemandedElts, Depth: Depth + 1);
3638
3639 Known.setAllConflict();
3640 for (unsigned i = 0; i != NumElts; ++i)
3641 if (DemandedElts[i]) {
3642 unsigned Shifts = IsLE ? i : NumElts - 1 - i;
3643 unsigned Offset = (Shifts % SubScale) * BitWidth;
3644 Known = Known.intersectWith(RHS: Known2.extractBits(NumBits: BitWidth, BitPosition: Offset));
3645 // If we don't know any bits, early out.
3646 if (Known.isUnknown())
3647 break;
3648 }
3649 }
3650 break;
3651 }
3652 case ISD::AND:
3653 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3654 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3655
3656 Known &= Known2;
3657 break;
3658 case ISD::OR:
3659 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3660 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3661
3662 Known |= Known2;
3663 break;
3664 case ISD::XOR:
3665 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3666 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3667
3668 Known ^= Known2;
3669 break;
3670 case ISD::MUL: {
3671 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3672 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3673 bool SelfMultiply = Op.getOperand(i: 0) == Op.getOperand(i: 1);
3674 // TODO: SelfMultiply can be poison, but not undef.
3675 if (SelfMultiply)
3676 SelfMultiply &= isGuaranteedNotToBeUndefOrPoison(
3677 Op: Op.getOperand(i: 0), DemandedElts, Kind: UndefPoisonKind::UndefOrPoison,
3678 Depth: Depth + 1);
3679 Known = KnownBits::mul(LHS: Known, RHS: Known2, NoUndefSelfMultiply: SelfMultiply);
3680
3681 // If the multiplication is known not to overflow, the product of a number
3682 // with itself is non-negative. Only do this if we didn't already computed
3683 // the opposite value for the sign bit.
3684 if (Op->getFlags().hasNoSignedWrap() &&
3685 Op.getOperand(i: 0) == Op.getOperand(i: 1) &&
3686 !Known.isNegative())
3687 Known.makeNonNegative();
3688 break;
3689 }
3690 case ISD::MULHU: {
3691 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3692 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3693 Known = KnownBits::mulhu(LHS: Known, RHS: Known2);
3694 break;
3695 }
3696 case ISD::MULHS: {
3697 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3698 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3699 Known = KnownBits::mulhs(LHS: Known, RHS: Known2);
3700 break;
3701 }
3702 case ISD::ABDU: {
3703 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3704 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3705 Known = KnownBits::abdu(LHS: Known, RHS: Known2);
3706 break;
3707 }
3708 case ISD::ABDS: {
3709 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3710 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3711 Known = KnownBits::abds(LHS: Known, RHS: Known2);
3712 unsigned SignBits1 =
3713 ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3714 if (SignBits1 == 1)
3715 break;
3716 unsigned SignBits0 =
3717 ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3718 Known.Zero.setHighBits(std::min(a: SignBits0, b: SignBits1) - 1);
3719 break;
3720 }
3721 case ISD::UMUL_LOHI: {
3722 assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result");
3723 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3724 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3725 bool SelfMultiply = Op.getOperand(i: 0) == Op.getOperand(i: 1);
3726 if (Op.getResNo() == 0)
3727 Known = KnownBits::mul(LHS: Known, RHS: Known2, NoUndefSelfMultiply: SelfMultiply);
3728 else
3729 Known = KnownBits::mulhu(LHS: Known, RHS: Known2);
3730 break;
3731 }
3732 case ISD::SMUL_LOHI: {
3733 assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result");
3734 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3735 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3736 bool SelfMultiply = Op.getOperand(i: 0) == Op.getOperand(i: 1);
3737 if (Op.getResNo() == 0)
3738 Known = KnownBits::mul(LHS: Known, RHS: Known2, NoUndefSelfMultiply: SelfMultiply);
3739 else
3740 Known = KnownBits::mulhs(LHS: Known, RHS: Known2);
3741 break;
3742 }
3743 case ISD::AVGFLOORU: {
3744 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3745 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3746 Known = KnownBits::avgFloorU(LHS: Known, RHS: Known2);
3747 break;
3748 }
3749 case ISD::AVGCEILU: {
3750 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3751 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3752 Known = KnownBits::avgCeilU(LHS: Known, RHS: Known2);
3753 break;
3754 }
3755 case ISD::AVGFLOORS: {
3756 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3757 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3758 Known = KnownBits::avgFloorS(LHS: Known, RHS: Known2);
3759 break;
3760 }
3761 case ISD::AVGCEILS: {
3762 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3763 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3764 Known = KnownBits::avgCeilS(LHS: Known, RHS: Known2);
3765 break;
3766 }
3767 case ISD::SELECT:
3768 case ISD::VSELECT:
3769 Known = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth+1);
3770 // If we don't know any bits, early out.
3771 if (Known.isUnknown())
3772 break;
3773 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth+1);
3774
3775 // Only known if known in both the LHS and RHS.
3776 Known = Known.intersectWith(RHS: Known2);
3777 break;
3778 case ISD::SELECT_CC:
3779 Known = computeKnownBits(Op: Op.getOperand(i: 3), DemandedElts, Depth: Depth+1);
3780 // If we don't know any bits, early out.
3781 if (Known.isUnknown())
3782 break;
3783 Known2 = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth+1);
3784
3785 // Only known if known in both the LHS and RHS.
3786 Known = Known.intersectWith(RHS: Known2);
3787 break;
3788 case ISD::SMULO:
3789 case ISD::UMULO:
3790 if (Op.getResNo() != 1)
3791 break;
3792 // The boolean result conforms to getBooleanContents.
3793 // If we know the result of a setcc has the top bits zero, use this info.
3794 // We know that we have an integer-based boolean since these operations
3795 // are only available for integer.
3796 if (TLI->getBooleanContents(isVec: Op.getValueType().isVector(), isFloat: false) ==
3797 TargetLowering::ZeroOrOneBooleanContent &&
3798 BitWidth > 1)
3799 Known.Zero.setBitsFrom(1);
3800 break;
3801 case ISD::SETCC:
3802 case ISD::SETCCCARRY:
3803 case ISD::STRICT_FSETCC:
3804 case ISD::STRICT_FSETCCS: {
3805 unsigned OpNo = Op->isStrictFPOpcode() ? 1 : 0;
3806 // If we know the result of a setcc has the top bits zero, use this info.
3807 if (TLI->getBooleanContents(Type: Op.getOperand(i: OpNo).getValueType()) ==
3808 TargetLowering::ZeroOrOneBooleanContent &&
3809 BitWidth > 1)
3810 Known.Zero.setBitsFrom(1);
3811 break;
3812 }
3813 case ISD::SHL: {
3814 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3815 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3816
3817 bool NUW = Op->getFlags().hasNoUnsignedWrap();
3818 bool NSW = Op->getFlags().hasNoSignedWrap();
3819
3820 bool ShAmtNonZero = Known2.isNonZero();
3821
3822 Known = KnownBits::shl(LHS: Known, RHS: Known2, NUW, NSW, ShAmtNonZero);
3823
3824 // Minimum shift low bits are known zero.
3825 if (std::optional<unsigned> ShMinAmt =
3826 getValidMinimumShiftAmount(V: Op, DemandedElts, Depth: Depth + 1))
3827 Known.Zero.setLowBits(*ShMinAmt);
3828 break;
3829 }
3830 case ISD::SRL:
3831 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3832 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3833 Known = KnownBits::lshr(LHS: Known, RHS: Known2, /*ShAmtNonZero=*/false,
3834 Exact: Op->getFlags().hasExact());
3835
3836 // Minimum shift high bits are known zero.
3837 if (std::optional<unsigned> ShMinAmt =
3838 getValidMinimumShiftAmount(V: Op, DemandedElts, Depth: Depth + 1))
3839 Known.Zero.setHighBits(*ShMinAmt);
3840 break;
3841 case ISD::SRA:
3842 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3843 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3844 Known = KnownBits::ashr(LHS: Known, RHS: Known2, /*ShAmtNonZero=*/false,
3845 Exact: Op->getFlags().hasExact());
3846 break;
3847 case ISD::ROTL:
3848 case ISD::ROTR:
3849 if (ConstantSDNode *C =
3850 isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts)) {
3851 unsigned Amt = C->getAPIntValue().urem(RHS: BitWidth);
3852
3853 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3854
3855 // Canonicalize to ROTR.
3856 if (Opcode == ISD::ROTL && Amt != 0)
3857 Amt = BitWidth - Amt;
3858
3859 Known.Zero = Known.Zero.rotr(rotateAmt: Amt);
3860 Known.One = Known.One.rotr(rotateAmt: Amt);
3861 }
3862 break;
3863 case ISD::FSHL:
3864 case ISD::FSHR:
3865 if (ConstantSDNode *C = isConstOrConstSplat(N: Op.getOperand(i: 2), DemandedElts)) {
3866 unsigned Amt = C->getAPIntValue().urem(RHS: BitWidth);
3867
3868 // For fshl, 0-shift returns the 1st arg.
3869 // For fshr, 0-shift returns the 2nd arg.
3870 if (Amt == 0) {
3871 Known = computeKnownBits(Op: Op.getOperand(i: Opcode == ISD::FSHL ? 0 : 1),
3872 DemandedElts, Depth: Depth + 1);
3873 break;
3874 }
3875
3876 // fshl: (X << (Z % BW)) | (Y >> (BW - (Z % BW)))
3877 // fshr: (X << (BW - (Z % BW))) | (Y >> (Z % BW))
3878 const APInt ShAmt(BitWidth, Amt);
3879 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3880 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3881 Known = Opcode == ISD::FSHL ? KnownBits::fshl(LHS: Known, RHS: Known2, Amt: ShAmt)
3882 : KnownBits::fshr(LHS: Known, RHS: Known2, Amt: ShAmt);
3883 }
3884 break;
3885 case ISD::SHL_PARTS:
3886 case ISD::SRA_PARTS:
3887 case ISD::SRL_PARTS: {
3888 assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result");
3889
3890 // Collect lo/hi source values and concatenate.
3891 unsigned LoBits = Op.getOperand(i: 0).getScalarValueSizeInBits();
3892 unsigned HiBits = Op.getOperand(i: 1).getScalarValueSizeInBits();
3893 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3894 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3895 Known = Known2.concat(Lo: Known);
3896
3897 // Collect shift amount.
3898 Known2 = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth + 1);
3899
3900 if (Opcode == ISD::SHL_PARTS)
3901 Known = KnownBits::shl(LHS: Known, RHS: Known2);
3902 else if (Opcode == ISD::SRA_PARTS)
3903 Known = KnownBits::ashr(LHS: Known, RHS: Known2);
3904 else // if (Opcode == ISD::SRL_PARTS)
3905 Known = KnownBits::lshr(LHS: Known, RHS: Known2);
3906
3907 // TODO: Minimum shift low/high bits are known zero.
3908
3909 if (Op.getResNo() == 0)
3910 Known = Known.extractBits(NumBits: LoBits, BitPosition: 0);
3911 else
3912 Known = Known.extractBits(NumBits: HiBits, BitPosition: LoBits);
3913 break;
3914 }
3915 case ISD::SIGN_EXTEND_INREG: {
3916 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3917 EVT EVT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
3918 Known = Known.sextInReg(SrcBitWidth: EVT.getScalarSizeInBits());
3919 break;
3920 }
3921 case ISD::CTTZ:
3922 case ISD::CTTZ_ZERO_POISON: {
3923 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3924 // If we have a known 1, its position is our upper bound.
3925 unsigned PossibleTZ = Known2.countMaxTrailingZeros();
3926 unsigned LowBits = llvm::bit_width(Value: PossibleTZ);
3927 Known.Zero.setBitsFrom(LowBits);
3928 break;
3929 }
3930 case ISD::CTLZ:
3931 case ISD::CTLZ_ZERO_POISON: {
3932 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3933 // If we have a known 1, its position is our upper bound.
3934 unsigned PossibleLZ = Known2.countMaxLeadingZeros();
3935 unsigned LowBits = llvm::bit_width(Value: PossibleLZ);
3936 Known.Zero.setBitsFrom(LowBits);
3937 break;
3938 }
3939 case ISD::CTLS: {
3940 unsigned MinRedundantSignBits =
3941 ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1) - 1;
3942 ConstantRange Range(APInt(BitWidth, MinRedundantSignBits),
3943 APInt(BitWidth, BitWidth));
3944 Known = Range.toKnownBits();
3945 break;
3946 }
3947 case ISD::CTPOP: {
3948 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3949 // If we know some of the bits are zero, they can't be one.
3950 unsigned PossibleOnes = Known2.countMaxPopulation();
3951 Known.Zero.setBitsFrom(llvm::bit_width(Value: PossibleOnes));
3952 break;
3953 }
3954 case ISD::PARITY: {
3955 // Parity returns 0 everywhere but the LSB.
3956 Known.Zero.setBitsFrom(1);
3957 break;
3958 }
3959 case ISD::PDEP: {
3960 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3961 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3962 Known = KnownBits::pdep(Val: Known2, Mask: Known);
3963 break;
3964 }
3965 case ISD::PEXT: {
3966 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3967 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3968 Known = KnownBits::pext(Val: Known2, Mask: Known);
3969 break;
3970 }
3971 case ISD::CLMUL: {
3972 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3973 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3974 Known = KnownBits::clmul(LHS: Known, RHS: Known2);
3975 break;
3976 }
3977 case ISD::MGATHER:
3978 case ISD::MLOAD: {
3979 ISD::LoadExtType ETy =
3980 (Opcode == ISD::MGATHER)
3981 ? cast<MaskedGatherSDNode>(Val&: Op)->getExtensionType()
3982 : cast<MaskedLoadSDNode>(Val&: Op)->getExtensionType();
3983 if (ETy == ISD::ZEXTLOAD) {
3984 EVT MemVT = cast<MemSDNode>(Val&: Op)->getMemoryVT();
3985 KnownBits Known0(MemVT.getScalarSizeInBits());
3986 return Known0.zext(BitWidth);
3987 }
3988 break;
3989 }
3990 case ISD::LOAD: {
3991 LoadSDNode *LD = cast<LoadSDNode>(Val&: Op);
3992 const Constant *Cst = TLI->getTargetConstantFromLoad(LD);
3993 if (ISD::isNON_EXTLoad(N: LD) && Cst) {
3994 // Determine any common known bits from the loaded constant pool value.
3995 Type *CstTy = Cst->getType();
3996 if ((NumElts * BitWidth) == CstTy->getPrimitiveSizeInBits() &&
3997 !Op.getValueType().isScalableVector()) {
3998 // If its a vector splat, then we can (quickly) reuse the scalar path.
3999 // NOTE: We assume all elements match and none are UNDEF.
4000 if (CstTy->isVectorTy()) {
4001 if (const Constant *Splat = Cst->getSplatValue()) {
4002 Cst = Splat;
4003 CstTy = Cst->getType();
4004 }
4005 }
4006 // TODO - do we need to handle different bitwidths?
4007 if (CstTy->isVectorTy() && BitWidth == CstTy->getScalarSizeInBits()) {
4008 // Iterate across all vector elements finding common known bits.
4009 Known.setAllConflict();
4010 for (unsigned i = 0; i != NumElts; ++i) {
4011 if (!DemandedElts[i])
4012 continue;
4013 if (Constant *Elt = Cst->getAggregateElement(Elt: i)) {
4014 if (auto *CInt = dyn_cast<ConstantInt>(Val: Elt)) {
4015 const APInt &Value = CInt->getValue();
4016 Known.One &= Value;
4017 Known.Zero &= ~Value;
4018 continue;
4019 }
4020 if (auto *CFP = dyn_cast<ConstantFP>(Val: Elt)) {
4021 APInt Value = CFP->getValueAPF().bitcastToAPInt();
4022 Known.One &= Value;
4023 Known.Zero &= ~Value;
4024 continue;
4025 }
4026 }
4027 Known.One.clearAllBits();
4028 Known.Zero.clearAllBits();
4029 break;
4030 }
4031 } else if (BitWidth == CstTy->getPrimitiveSizeInBits()) {
4032 if (auto *CInt = dyn_cast<ConstantInt>(Val: Cst)) {
4033 Known = KnownBits::makeConstant(C: CInt->getValue());
4034 } else if (auto *CFP = dyn_cast<ConstantFP>(Val: Cst)) {
4035 Known =
4036 KnownBits::makeConstant(C: CFP->getValueAPF().bitcastToAPInt());
4037 }
4038 }
4039 }
4040 } else if (Op.getResNo() == 0) {
4041 unsigned ScalarMemorySize = LD->getMemoryVT().getScalarSizeInBits();
4042 KnownBits KnownScalarMemory(ScalarMemorySize);
4043 if (const MDNode *MD = LD->getRanges())
4044 computeKnownBitsFromRangeMetadata(Ranges: *MD, Known&: KnownScalarMemory);
4045
4046 // Extend the Known bits from memory to the size of the scalar result.
4047 if (ISD::isZEXTLoad(N: Op.getNode()))
4048 Known = KnownScalarMemory.zext(BitWidth);
4049 else if (ISD::isSEXTLoad(N: Op.getNode()))
4050 Known = KnownScalarMemory.sext(BitWidth);
4051 else if (ISD::isEXTLoad(N: Op.getNode()))
4052 Known = KnownScalarMemory.anyext(BitWidth);
4053 else
4054 Known = KnownScalarMemory;
4055 assert(Known.getBitWidth() == BitWidth);
4056 return Known;
4057 }
4058 break;
4059 }
4060 case ISD::ZERO_EXTEND_VECTOR_INREG: {
4061 if (Op.getValueType().isScalableVector())
4062 break;
4063 EVT InVT = Op.getOperand(i: 0).getValueType();
4064 APInt InDemandedElts = DemandedElts.zext(width: InVT.getVectorNumElements());
4065 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts: InDemandedElts, Depth: Depth + 1);
4066 Known = Known.zext(BitWidth);
4067 break;
4068 }
4069 case ISD::ZERO_EXTEND: {
4070 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4071 Known = Known.zext(BitWidth);
4072 break;
4073 }
4074 case ISD::SIGN_EXTEND_VECTOR_INREG: {
4075 if (Op.getValueType().isScalableVector())
4076 break;
4077 EVT InVT = Op.getOperand(i: 0).getValueType();
4078 APInt InDemandedElts = DemandedElts.zext(width: InVT.getVectorNumElements());
4079 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts: InDemandedElts, Depth: Depth + 1);
4080 // If the sign bit is known to be zero or one, then sext will extend
4081 // it to the top bits, else it will just zext.
4082 Known = Known.sext(BitWidth);
4083 break;
4084 }
4085 case ISD::SIGN_EXTEND: {
4086 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4087 // If the sign bit is known to be zero or one, then sext will extend
4088 // it to the top bits, else it will just zext.
4089 Known = Known.sext(BitWidth);
4090 break;
4091 }
4092 case ISD::ANY_EXTEND_VECTOR_INREG: {
4093 if (Op.getValueType().isScalableVector())
4094 break;
4095 EVT InVT = Op.getOperand(i: 0).getValueType();
4096 APInt InDemandedElts = DemandedElts.zext(width: InVT.getVectorNumElements());
4097 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts: InDemandedElts, Depth: Depth + 1);
4098 Known = Known.anyext(BitWidth);
4099 break;
4100 }
4101 case ISD::ANY_EXTEND: {
4102 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4103 Known = Known.anyext(BitWidth);
4104 break;
4105 }
4106 case ISD::TRUNCATE: {
4107 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4108 Known = Known.trunc(BitWidth);
4109 break;
4110 }
4111 case ISD::TRUNCATE_SSAT_S: {
4112 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4113 Known = Known.truncSSat(BitWidth);
4114 break;
4115 }
4116 case ISD::TRUNCATE_SSAT_U: {
4117 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4118 Known = Known.truncSSatU(BitWidth);
4119 break;
4120 }
4121 case ISD::TRUNCATE_USAT_U: {
4122 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4123 Known = Known.truncUSat(BitWidth);
4124 break;
4125 }
4126 case ISD::AssertZext: {
4127 EVT VT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
4128 APInt InMask = APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: VT.getSizeInBits());
4129 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4130 Known.Zero |= (~InMask);
4131 Known.One &= (~Known.Zero);
4132 break;
4133 }
4134 case ISD::AssertAlign: {
4135 unsigned LogOfAlign = Log2(A: cast<AssertAlignSDNode>(Val&: Op)->getAlign());
4136 assert(LogOfAlign != 0);
4137
4138 // TODO: Should use maximum with source
4139 // If a node is guaranteed to be aligned, set low zero bits accordingly as
4140 // well as clearing one bits.
4141 Known.Zero.setLowBits(LogOfAlign);
4142 Known.One.clearLowBits(loBits: LogOfAlign);
4143 break;
4144 }
4145 case ISD::AssertNoFPClass: {
4146 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4147
4148 FPClassTest NoFPClass =
4149 static_cast<FPClassTest>(Op.getConstantOperandVal(i: 1));
4150 const FPClassTest NegativeTestMask = fcNan | fcNegative;
4151 if ((NoFPClass & NegativeTestMask) == NegativeTestMask) {
4152 // Cannot be negative.
4153 Known.makeNonNegative();
4154 }
4155
4156 const FPClassTest PositiveTestMask = fcNan | fcPositive;
4157 if ((NoFPClass & PositiveTestMask) == PositiveTestMask) {
4158 // Cannot be positive.
4159 Known.makeNegative();
4160 }
4161
4162 break;
4163 }
4164 case ISD::FABS:
4165 // fabs clears the sign bit
4166 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4167 Known.makeNonNegative();
4168 break;
4169 case ISD::FGETSIGN:
4170 // All bits are zero except the low bit.
4171 Known.Zero.setBitsFrom(1);
4172 break;
4173 case ISD::ADD: {
4174 SDNodeFlags Flags = Op.getNode()->getFlags();
4175 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4176 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4177 bool SelfAdd = Op.getOperand(i: 0) == Op.getOperand(i: 1) &&
4178 isGuaranteedNotToBeUndefOrPoison(
4179 Op: Op.getOperand(i: 0), DemandedElts,
4180 Kind: UndefPoisonKind::UndefOrPoison, Depth: Depth + 1);
4181 Known = KnownBits::add(LHS: Known, RHS: Known2, NSW: Flags.hasNoSignedWrap(),
4182 NUW: Flags.hasNoUnsignedWrap(), SelfAdd);
4183 break;
4184 }
4185 case ISD::SUB: {
4186 SDNodeFlags Flags = Op.getNode()->getFlags();
4187 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4188 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4189 Known = KnownBits::sub(LHS: Known, RHS: Known2, NSW: Flags.hasNoSignedWrap(),
4190 NUW: Flags.hasNoUnsignedWrap());
4191 break;
4192 }
4193 case ISD::USUBO:
4194 case ISD::SSUBO:
4195 case ISD::USUBO_CARRY:
4196 case ISD::SSUBO_CARRY:
4197 if (Op.getResNo() == 1) {
4198 // If we know the result of a setcc has the top bits zero, use this info.
4199 if (TLI->getBooleanContents(Type: Op.getOperand(i: 0).getValueType()) ==
4200 TargetLowering::ZeroOrOneBooleanContent &&
4201 BitWidth > 1)
4202 Known.Zero.setBitsFrom(1);
4203 break;
4204 }
4205 [[fallthrough]];
4206 case ISD::SUBC: {
4207 assert(Op.getResNo() == 0 &&
4208 "We only compute knownbits for the difference here.");
4209
4210 // With USUBO_CARRY and SSUBO_CARRY a borrow bit may be added in.
4211 KnownBits Borrow(1);
4212 if (Opcode == ISD::USUBO_CARRY || Opcode == ISD::SSUBO_CARRY) {
4213 Borrow = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth + 1);
4214 // Borrow has bit width 1
4215 Borrow = Borrow.trunc(BitWidth: 1);
4216 } else {
4217 Borrow.setAllZero();
4218 }
4219
4220 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4221 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4222 Known = KnownBits::computeForSubBorrow(LHS: Known, RHS: Known2, Borrow);
4223 break;
4224 }
4225 case ISD::UADDO:
4226 case ISD::SADDO:
4227 case ISD::UADDO_CARRY:
4228 case ISD::SADDO_CARRY:
4229 if (Op.getResNo() == 1) {
4230 // If we know the result of a setcc has the top bits zero, use this info.
4231 if (TLI->getBooleanContents(Type: Op.getOperand(i: 0).getValueType()) ==
4232 TargetLowering::ZeroOrOneBooleanContent &&
4233 BitWidth > 1)
4234 Known.Zero.setBitsFrom(1);
4235 break;
4236 }
4237 [[fallthrough]];
4238 case ISD::ADDC:
4239 case ISD::ADDE: {
4240 assert(Op.getResNo() == 0 && "We only compute knownbits for the sum here.");
4241
4242 // With ADDE and UADDO_CARRY, a carry bit may be added in.
4243 KnownBits Carry(1);
4244 if (Opcode == ISD::ADDE)
4245 // Can't track carry from glue, set carry to unknown.
4246 Carry.resetAll();
4247 else if (Opcode == ISD::UADDO_CARRY || Opcode == ISD::SADDO_CARRY) {
4248 Carry = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth + 1);
4249 // Carry has bit width 1
4250 Carry = Carry.trunc(BitWidth: 1);
4251 } else {
4252 Carry.setAllZero();
4253 }
4254
4255 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4256 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4257 Known = KnownBits::computeForAddCarry(LHS: Known, RHS: Known2, Carry);
4258 break;
4259 }
4260 case ISD::UDIV: {
4261 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4262 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4263 Known = KnownBits::udiv(LHS: Known, RHS: Known2, Exact: Op->getFlags().hasExact());
4264 break;
4265 }
4266 case ISD::SDIV: {
4267 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4268 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4269 Known = KnownBits::sdiv(LHS: Known, RHS: Known2, Exact: Op->getFlags().hasExact());
4270 break;
4271 }
4272 case ISD::SREM: {
4273 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4274 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4275 Known = KnownBits::srem(LHS: Known, RHS: Known2);
4276 break;
4277 }
4278 case ISD::UREM: {
4279 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4280 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4281 Known = KnownBits::urem(LHS: Known, RHS: Known2);
4282 break;
4283 }
4284 case ISD::EXTRACT_ELEMENT: {
4285 Known = computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth+1);
4286 const unsigned Index = Op.getConstantOperandVal(i: 1);
4287 const unsigned EltBitWidth = Op.getValueSizeInBits();
4288
4289 // Remove low part of known bits mask
4290 Known.Zero = Known.Zero.getHiBits(numBits: Known.getBitWidth() - Index * EltBitWidth);
4291 Known.One = Known.One.getHiBits(numBits: Known.getBitWidth() - Index * EltBitWidth);
4292
4293 // Remove high part of known bit mask
4294 Known = Known.trunc(BitWidth: EltBitWidth);
4295 break;
4296 }
4297 case ISD::EXTRACT_VECTOR_ELT: {
4298 SDValue InVec = Op.getOperand(i: 0);
4299 SDValue EltNo = Op.getOperand(i: 1);
4300 EVT VecVT = InVec.getValueType();
4301 // computeKnownBits not yet implemented for scalable vectors.
4302 if (VecVT.isScalableVector())
4303 break;
4304 const unsigned EltBitWidth = VecVT.getScalarSizeInBits();
4305 const unsigned NumSrcElts = VecVT.getVectorNumElements();
4306
4307 // If BitWidth > EltBitWidth the value is anyext:ed. So we do not know
4308 // anything about the extended bits.
4309 if (BitWidth > EltBitWidth)
4310 Known = Known.trunc(BitWidth: EltBitWidth);
4311
4312 // If we know the element index, just demand that vector element, else for
4313 // an unknown element index, ignore DemandedElts and demand them all.
4314 APInt DemandedSrcElts = APInt::getAllOnes(numBits: NumSrcElts);
4315 auto *ConstEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
4316 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(RHS: NumSrcElts))
4317 DemandedSrcElts =
4318 APInt::getOneBitSet(numBits: NumSrcElts, BitNo: ConstEltNo->getZExtValue());
4319
4320 Known = computeKnownBits(Op: InVec, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
4321 if (BitWidth > EltBitWidth)
4322 Known = Known.anyext(BitWidth);
4323 break;
4324 }
4325 case ISD::INSERT_VECTOR_ELT: {
4326 if (Op.getValueType().isScalableVector())
4327 break;
4328
4329 // If we know the element index, split the demand between the
4330 // source vector and the inserted element, otherwise assume we need
4331 // the original demanded vector elements and the value.
4332 SDValue InVec = Op.getOperand(i: 0);
4333 SDValue InVal = Op.getOperand(i: 1);
4334 SDValue EltNo = Op.getOperand(i: 2);
4335 bool DemandedVal = true;
4336 APInt DemandedVecElts = DemandedElts;
4337 auto *CEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
4338 if (CEltNo && CEltNo->getAPIntValue().ult(RHS: NumElts)) {
4339 unsigned EltIdx = CEltNo->getZExtValue();
4340 DemandedVal = !!DemandedElts[EltIdx];
4341 DemandedVecElts.clearBit(BitPosition: EltIdx);
4342 }
4343 Known.setAllConflict();
4344 if (DemandedVal) {
4345 Known2 = computeKnownBits(Op: InVal, Depth: Depth + 1);
4346 Known = Known.intersectWith(RHS: Known2.zextOrTrunc(BitWidth));
4347 }
4348 if (!!DemandedVecElts) {
4349 Known2 = computeKnownBits(Op: InVec, DemandedElts: DemandedVecElts, Depth: Depth + 1);
4350 Known = Known.intersectWith(RHS: Known2);
4351 }
4352 break;
4353 }
4354 case ISD::BITREVERSE: {
4355 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4356 Known = Known2.reverseBits();
4357 break;
4358 }
4359 case ISD::BSWAP: {
4360 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4361 Known = Known2.byteSwap();
4362 break;
4363 }
4364 case ISD::ABS:
4365 case ISD::ABS_MIN_POISON: {
4366 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4367 Known = Known2.abs();
4368 Known.Zero.setHighBits(
4369 ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1) - 1);
4370 break;
4371 }
4372 case ISD::USUBSAT: {
4373 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4374 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4375 Known = KnownBits::usub_sat(LHS: Known, RHS: Known2);
4376 break;
4377 }
4378 case ISD::UMIN: {
4379 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4380 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4381 Known = KnownBits::umin(LHS: Known, RHS: Known2);
4382 break;
4383 }
4384 case ISD::UMAX: {
4385 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4386 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4387 Known = KnownBits::umax(LHS: Known, RHS: Known2);
4388 break;
4389 }
4390 case ISD::SMIN:
4391 case ISD::SMAX: {
4392 // If we have a clamp pattern, we know that the number of sign bits will be
4393 // the minimum of the clamp min/max range.
4394 bool IsMax = (Opcode == ISD::SMAX);
4395 ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr;
4396 if ((CstLow = isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts)))
4397 if (Op.getOperand(i: 0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX))
4398 CstHigh =
4399 isConstOrConstSplat(N: Op.getOperand(i: 0).getOperand(i: 1), DemandedElts);
4400 if (CstLow && CstHigh) {
4401 if (!IsMax)
4402 std::swap(a&: CstLow, b&: CstHigh);
4403
4404 const APInt &ValueLow = CstLow->getAPIntValue();
4405 const APInt &ValueHigh = CstHigh->getAPIntValue();
4406 if (ValueLow.sle(RHS: ValueHigh)) {
4407 unsigned LowSignBits = ValueLow.getNumSignBits();
4408 unsigned HighSignBits = ValueHigh.getNumSignBits();
4409 unsigned MinSignBits = std::min(a: LowSignBits, b: HighSignBits);
4410 if (ValueLow.isNegative() && ValueHigh.isNegative()) {
4411 Known.One.setHighBits(MinSignBits);
4412 break;
4413 }
4414 if (ValueLow.isNonNegative() && ValueHigh.isNonNegative()) {
4415 Known.Zero.setHighBits(MinSignBits);
4416 break;
4417 }
4418 }
4419 }
4420
4421 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4422 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4423 if (IsMax)
4424 Known = KnownBits::smax(LHS: Known, RHS: Known2);
4425 else
4426 Known = KnownBits::smin(LHS: Known, RHS: Known2);
4427
4428 // For SMAX, if CstLow is non-negative we know the result will be
4429 // non-negative and thus all sign bits are 0.
4430 // TODO: There's an equivalent of this for smin with negative constant for
4431 // known ones.
4432 if (IsMax && CstLow) {
4433 const APInt &ValueLow = CstLow->getAPIntValue();
4434 if (ValueLow.isNonNegative()) {
4435 unsigned SignBits = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
4436 Known.Zero.setHighBits(std::min(a: SignBits, b: ValueLow.getNumSignBits()));
4437 }
4438 }
4439
4440 break;
4441 }
4442 case ISD::UINT_TO_FP: {
4443 Known.makeNonNegative();
4444 break;
4445 }
4446 case ISD::SINT_TO_FP: {
4447 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4448 if (Known2.isNonNegative())
4449 Known.makeNonNegative();
4450 else if (Known2.isNegative())
4451 Known.makeNegative();
4452 break;
4453 }
4454 case ISD::FP_TO_UINT_SAT: {
4455 // FP_TO_UINT_SAT produces an unsigned value that fits in the saturating VT.
4456 EVT VT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
4457 Known.Zero |= APInt::getBitsSetFrom(numBits: BitWidth, loBit: VT.getScalarSizeInBits());
4458 break;
4459 }
4460 case ISD::ATOMIC_LOAD: {
4461 // If we are looking at the loaded value.
4462 if (Op.getResNo() == 0) {
4463 auto *AT = cast<AtomicSDNode>(Val&: Op);
4464 unsigned ScalarMemorySize = AT->getMemoryVT().getScalarSizeInBits();
4465 KnownBits KnownScalarMemory(ScalarMemorySize);
4466 if (const MDNode *MD = AT->getRanges())
4467 computeKnownBitsFromRangeMetadata(Ranges: *MD, Known&: KnownScalarMemory);
4468
4469 switch (AT->getExtensionType()) {
4470 case ISD::ZEXTLOAD:
4471 Known = KnownScalarMemory.zext(BitWidth);
4472 break;
4473 case ISD::SEXTLOAD:
4474 Known = KnownScalarMemory.sext(BitWidth);
4475 break;
4476 case ISD::EXTLOAD:
4477 switch (TLI->getExtendForAtomicOps()) {
4478 case ISD::ZERO_EXTEND:
4479 Known = KnownScalarMemory.zext(BitWidth);
4480 break;
4481 case ISD::SIGN_EXTEND:
4482 Known = KnownScalarMemory.sext(BitWidth);
4483 break;
4484 default:
4485 Known = KnownScalarMemory.anyext(BitWidth);
4486 break;
4487 }
4488 break;
4489 case ISD::NON_EXTLOAD:
4490 Known = KnownScalarMemory;
4491 break;
4492 }
4493 assert(Known.getBitWidth() == BitWidth);
4494 }
4495 break;
4496 }
4497 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
4498 if (Op.getResNo() == 1) {
4499 // The boolean result conforms to getBooleanContents.
4500 // If we know the result of a setcc has the top bits zero, use this info.
4501 // We know that we have an integer-based boolean since these operations
4502 // are only available for integer.
4503 if (TLI->getBooleanContents(isVec: Op.getValueType().isVector(), isFloat: false) ==
4504 TargetLowering::ZeroOrOneBooleanContent &&
4505 BitWidth > 1)
4506 Known.Zero.setBitsFrom(1);
4507 break;
4508 }
4509 [[fallthrough]];
4510 case ISD::ATOMIC_CMP_SWAP:
4511 case ISD::ATOMIC_SWAP:
4512 case ISD::ATOMIC_LOAD_ADD:
4513 case ISD::ATOMIC_LOAD_SUB:
4514 case ISD::ATOMIC_LOAD_AND:
4515 case ISD::ATOMIC_LOAD_CLR:
4516 case ISD::ATOMIC_LOAD_OR:
4517 case ISD::ATOMIC_LOAD_XOR:
4518 case ISD::ATOMIC_LOAD_NAND:
4519 case ISD::ATOMIC_LOAD_MIN:
4520 case ISD::ATOMIC_LOAD_MAX:
4521 case ISD::ATOMIC_LOAD_UMIN:
4522 case ISD::ATOMIC_LOAD_UMAX: {
4523 // If we are looking at the loaded value.
4524 if (Op.getResNo() == 0) {
4525 auto *AT = cast<AtomicSDNode>(Val&: Op);
4526 unsigned MemBits = AT->getMemoryVT().getScalarSizeInBits();
4527
4528 if (TLI->getExtendForAtomicOps() == ISD::ZERO_EXTEND)
4529 Known.Zero.setBitsFrom(MemBits);
4530 }
4531 break;
4532 }
4533 case ISD::FrameIndex:
4534 case ISD::TargetFrameIndex: {
4535 const MachineFunction &MF = getMachineFunction();
4536 int FrameIdx = cast<FrameIndexSDNode>(Val&: Op)->getIndex();
4537 TLI->computeKnownBitsForStackObjectPointer(
4538 Known, MF, Alignment: MF.getFrameInfo().getObjectAlign(ObjectIdx: FrameIdx));
4539 break;
4540 }
4541
4542 default:
4543 if (Opcode < ISD::BUILTIN_OP_END)
4544 break;
4545 [[fallthrough]];
4546 case ISD::INTRINSIC_WO_CHAIN:
4547 case ISD::INTRINSIC_W_CHAIN:
4548 case ISD::INTRINSIC_VOID:
4549 // Allow the target to implement this method for its nodes.
4550 TLI->computeKnownBitsForTargetNode(Op, Known, DemandedElts, DAG: *this, Depth);
4551 break;
4552 }
4553
4554 return Known;
4555}
4556
4557/// Convert ConstantRange OverflowResult into SelectionDAG::OverflowKind.
4558static SelectionDAG::OverflowKind mapOverflowResult(ConstantRange::OverflowResult OR) {
4559 switch (OR) {
4560 case ConstantRange::OverflowResult::MayOverflow:
4561 return SelectionDAG::OFK_Sometime;
4562 case ConstantRange::OverflowResult::AlwaysOverflowsLow:
4563 case ConstantRange::OverflowResult::AlwaysOverflowsHigh:
4564 return SelectionDAG::OFK_Always;
4565 case ConstantRange::OverflowResult::NeverOverflows:
4566 return SelectionDAG::OFK_Never;
4567 }
4568 llvm_unreachable("Unknown OverflowResult");
4569}
4570
4571SelectionDAG::OverflowKind
4572SelectionDAG::computeOverflowForSignedAdd(SDValue N0, SDValue N1) const {
4573 // X + 0 never overflow
4574 if (isNullConstant(V: N1))
4575 return OFK_Never;
4576
4577 // If both operands each have at least two sign bits, the addition
4578 // cannot overflow.
4579 if (ComputeNumSignBits(Op: N0) > 1 && ComputeNumSignBits(Op: N1) > 1)
4580 return OFK_Never;
4581
4582 // TODO: Add ConstantRange::signedAddMayOverflow handling.
4583 return OFK_Sometime;
4584}
4585
4586SelectionDAG::OverflowKind
4587SelectionDAG::computeOverflowForUnsignedAdd(SDValue N0, SDValue N1) const {
4588 // X + 0 never overflow
4589 if (isNullConstant(V: N1))
4590 return OFK_Never;
4591
4592 // mulhi + 1 never overflow
4593 KnownBits N1Known = computeKnownBits(Op: N1);
4594 if (N0.getOpcode() == ISD::UMUL_LOHI && N0.getResNo() == 1 &&
4595 N1Known.getMaxValue().ult(RHS: 2))
4596 return OFK_Never;
4597
4598 KnownBits N0Known = computeKnownBits(Op: N0);
4599 if (N1.getOpcode() == ISD::UMUL_LOHI && N1.getResNo() == 1 &&
4600 N0Known.getMaxValue().ult(RHS: 2))
4601 return OFK_Never;
4602
4603 // Fallback to ConstantRange::unsignedAddMayOverflow handling.
4604 ConstantRange N0Range = ConstantRange::fromKnownBits(Known: N0Known, IsSigned: false);
4605 ConstantRange N1Range = ConstantRange::fromKnownBits(Known: N1Known, IsSigned: false);
4606 return mapOverflowResult(OR: N0Range.unsignedAddMayOverflow(Other: N1Range));
4607}
4608
4609SelectionDAG::OverflowKind
4610SelectionDAG::computeOverflowForSignedSub(SDValue N0, SDValue N1) const {
4611 // X - 0 never overflow
4612 if (isNullConstant(V: N1))
4613 return OFK_Never;
4614
4615 // If both operands each have at least two sign bits, the subtraction
4616 // cannot overflow.
4617 if (ComputeNumSignBits(Op: N0) > 1 && ComputeNumSignBits(Op: N1) > 1)
4618 return OFK_Never;
4619
4620 KnownBits N0Known = computeKnownBits(Op: N0);
4621 KnownBits N1Known = computeKnownBits(Op: N1);
4622 ConstantRange N0Range = ConstantRange::fromKnownBits(Known: N0Known, IsSigned: true);
4623 ConstantRange N1Range = ConstantRange::fromKnownBits(Known: N1Known, IsSigned: true);
4624 return mapOverflowResult(OR: N0Range.signedSubMayOverflow(Other: N1Range));
4625}
4626
4627SelectionDAG::OverflowKind
4628SelectionDAG::computeOverflowForUnsignedSub(SDValue N0, SDValue N1) const {
4629 // X - 0 never overflow
4630 if (isNullConstant(V: N1))
4631 return OFK_Never;
4632
4633 ConstantRange N0Range =
4634 computeConstantRangeIncludingKnownBits(Op: N0, /*ForSigned=*/false);
4635 ConstantRange N1Range =
4636 computeConstantRangeIncludingKnownBits(Op: N1, /*ForSigned=*/false);
4637 return mapOverflowResult(OR: N0Range.unsignedSubMayOverflow(Other: N1Range));
4638}
4639
4640SelectionDAG::OverflowKind
4641SelectionDAG::computeOverflowForUnsignedMul(SDValue N0, SDValue N1) const {
4642 // X * 0 and X * 1 never overflow.
4643 if (isNullConstant(V: N1) || isOneConstant(V: N1))
4644 return OFK_Never;
4645
4646 ConstantRange N0Range = computeConstantRangeIncludingKnownBits(Op: N0, ForSigned: false);
4647 ConstantRange N1Range = computeConstantRangeIncludingKnownBits(Op: N1, ForSigned: false);
4648 return mapOverflowResult(OR: N0Range.unsignedMulMayOverflow(Other: N1Range));
4649}
4650
4651SelectionDAG::OverflowKind
4652SelectionDAG::computeOverflowForSignedMul(SDValue N0, SDValue N1) const {
4653 // X * 0 and X * 1 never overflow.
4654 if (isNullConstant(V: N1) || isOneConstant(V: N1))
4655 return OFK_Never;
4656
4657 // Get the size of the result.
4658 unsigned BitWidth = N0.getScalarValueSizeInBits();
4659
4660 // Sum of the sign bits.
4661 unsigned SignBits = ComputeNumSignBits(Op: N0) + ComputeNumSignBits(Op: N1);
4662
4663 // If we have enough sign bits, then there's no overflow.
4664 if (SignBits > BitWidth + 1)
4665 return OFK_Never;
4666
4667 if (SignBits == BitWidth + 1) {
4668 // The overflow occurs when the true multiplication of the
4669 // the operands is the minimum negative number.
4670 KnownBits N0Known = computeKnownBits(Op: N0);
4671 KnownBits N1Known = computeKnownBits(Op: N1);
4672 // If one of the operands is non-negative, then there's no
4673 // overflow.
4674 if (N0Known.isNonNegative() || N1Known.isNonNegative())
4675 return OFK_Never;
4676 }
4677
4678 return OFK_Sometime;
4679}
4680
4681ConstantRange SelectionDAG::computeConstantRange(SDValue Op, bool ForSigned,
4682 unsigned Depth) const {
4683 APInt DemandedElts = getDemandAllEltsMask(V: Op);
4684 return computeConstantRange(Op, DemandedElts, ForSigned, Depth);
4685}
4686
4687ConstantRange SelectionDAG::computeConstantRange(SDValue Op,
4688 const APInt &DemandedElts,
4689 bool ForSigned,
4690 unsigned Depth) const {
4691 EVT VT = Op.getValueType();
4692 unsigned BitWidth = VT.getScalarSizeInBits();
4693
4694 if (Depth >= MaxRecursionDepth)
4695 return ConstantRange::getFull(BitWidth);
4696
4697 if (ConstantSDNode *C = isConstOrConstSplat(N: Op, DemandedElts))
4698 return ConstantRange(C->getAPIntValue());
4699
4700 unsigned Opcode = Op.getOpcode();
4701 switch (Opcode) {
4702 case ISD::VSCALE: {
4703 const Function &F = getMachineFunction().getFunction();
4704 const APInt &Multiplier = Op.getConstantOperandAPInt(i: 0);
4705 return getVScaleRange(F: &F, BitWidth).multiply(Other: Multiplier);
4706 }
4707 default:
4708 break;
4709 }
4710
4711 return ConstantRange::getFull(BitWidth);
4712}
4713
4714ConstantRange
4715SelectionDAG::computeConstantRangeIncludingKnownBits(SDValue Op, bool ForSigned,
4716 unsigned Depth) const {
4717 APInt DemandedElts = getDemandAllEltsMask(V: Op);
4718 return computeConstantRangeIncludingKnownBits(Op, DemandedElts, ForSigned,
4719 Depth);
4720}
4721
4722ConstantRange SelectionDAG::computeConstantRangeIncludingKnownBits(
4723 SDValue Op, const APInt &DemandedElts, bool ForSigned,
4724 unsigned Depth) const {
4725 KnownBits Known = computeKnownBits(Op, DemandedElts, Depth);
4726 ConstantRange CR1 = ConstantRange::fromKnownBits(Known, IsSigned: ForSigned);
4727 ConstantRange CR2 = computeConstantRange(Op, DemandedElts, ForSigned, Depth);
4728 ConstantRange::PreferredRangeType RangeType =
4729 ForSigned ? ConstantRange::Signed : ConstantRange::Unsigned;
4730 return CR1.intersectWith(CR: CR2, Type: RangeType);
4731}
4732
4733bool SelectionDAG::isKnownToBeAPowerOfTwo(SDValue Val, bool OrZero,
4734 unsigned Depth) const {
4735 APInt DemandedElts = getDemandAllEltsMask(V: Val);
4736 return isKnownToBeAPowerOfTwo(Val, DemandedElts, OrZero, Depth);
4737}
4738
4739bool SelectionDAG::isKnownToBeAPowerOfTwo(SDValue Val,
4740 const APInt &DemandedElts,
4741 bool OrZero, unsigned Depth) const {
4742 if (Depth >= MaxRecursionDepth)
4743 return false; // Limit search depth.
4744
4745 EVT OpVT = Val.getValueType();
4746 unsigned BitWidth = OpVT.getScalarSizeInBits();
4747 [[maybe_unused]] unsigned NumElts = DemandedElts.getBitWidth();
4748 assert((!OpVT.isScalableVector() || NumElts == 1) &&
4749 "DemandedElts for scalable vectors must be 1 to represent all lanes");
4750 assert(
4751 (!OpVT.isFixedLengthVector() || NumElts == OpVT.getVectorNumElements()) &&
4752 "Unexpected vector size");
4753
4754 auto IsPowerOfTwoOrZero = [BitWidth, OrZero](const ConstantSDNode *C) {
4755 APInt V = C->getAPIntValue().zextOrTrunc(width: BitWidth);
4756 return (OrZero && V.isZero()) || V.isPowerOf2();
4757 };
4758
4759 // Is the constant a known power of 2 or zero?
4760 if (ISD::matchUnaryPredicate(Op: Val, DemandedElts, Match: IsPowerOfTwoOrZero,
4761 /*AllowUndefs=*/false, /*AllowTruncation=*/true))
4762 return true;
4763
4764 switch (Val.getOpcode()) {
4765 case ISD::EXTRACT_VECTOR_ELT: {
4766 SDValue InVec = Val.getOperand(i: 0);
4767 SDValue EltNo = Val.getOperand(i: 1);
4768 EVT VecVT = InVec.getValueType();
4769
4770 // Skip scalable vectors or implicit extensions.
4771 if (VecVT.isScalableVector() ||
4772 OpVT.getScalarSizeInBits() != VecVT.getScalarSizeInBits())
4773 break;
4774
4775 // If we know the element index, just demand that vector element, else for
4776 // an unknown element index, ignore DemandedElts and demand them all.
4777 const unsigned NumSrcElts = VecVT.getVectorNumElements();
4778 auto *ConstEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
4779 APInt DemandedSrcElts =
4780 ConstEltNo && ConstEltNo->getAPIntValue().ult(RHS: NumSrcElts)
4781 ? APInt::getOneBitSet(numBits: NumSrcElts, BitNo: ConstEltNo->getZExtValue())
4782 : APInt::getAllOnes(numBits: NumSrcElts);
4783 return isKnownToBeAPowerOfTwo(Val: InVec, DemandedElts: DemandedSrcElts, OrZero, Depth: Depth + 1);
4784 }
4785
4786 case ISD::AND: {
4787 // Looking for `x & -x` pattern:
4788 // If x == 0:
4789 // x & -x -> 0
4790 // If x != 0:
4791 // x & -x -> non-zero pow2
4792 // so if we find the pattern return whether we know `x` is non-zero.
4793 SDValue X, Z;
4794 if (sd_match(N: Val, P: m_And(L: m_Value(N&: X), R: m_Neg(V: m_Deferred(V&: X)))) ||
4795 (sd_match(N: Val, P: m_And(L: m_Value(N&: X), R: m_Sub(L: m_Value(N&: Z), R: m_Deferred(V&: X)))) &&
4796 MaskedVectorIsZero(V: Z, DemandedElts, Depth: Depth + 1)))
4797 return OrZero || isKnownNeverZero(Op: X, DemandedElts, Depth);
4798 break;
4799 }
4800
4801 case ISD::SHL: {
4802 // A left-shift of a constant one will have exactly one bit set because
4803 // shifting the bit off the end is undefined.
4804 auto *C = isConstOrConstSplat(N: Val.getOperand(i: 0), DemandedElts);
4805 if (C && C->getAPIntValue() == 1)
4806 return true;
4807 return (OrZero || isKnownNeverZero(Op: Val, DemandedElts, Depth)) &&
4808 isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), DemandedElts, OrZero,
4809 Depth: Depth + 1);
4810 }
4811
4812 case ISD::SRL: {
4813 // A logical right-shift of a constant sign-bit will have exactly
4814 // one bit set.
4815 auto *C = isConstOrConstSplat(N: Val.getOperand(i: 0), DemandedElts);
4816 if (C && C->getAPIntValue().isSignMask())
4817 return true;
4818 return (OrZero || isKnownNeverZero(Op: Val, DemandedElts, Depth)) &&
4819 isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), DemandedElts, OrZero,
4820 Depth: Depth + 1);
4821 }
4822
4823 case ISD::TRUNCATE:
4824 return (OrZero || isKnownNeverZero(Op: Val, DemandedElts, Depth)) &&
4825 isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), DemandedElts, OrZero,
4826 Depth: Depth + 1);
4827
4828 case ISD::ROTL:
4829 case ISD::ROTR:
4830 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), DemandedElts, OrZero,
4831 Depth: Depth + 1);
4832 case ISD::BSWAP:
4833 case ISD::BITREVERSE:
4834 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), DemandedElts, OrZero,
4835 Depth: Depth + 1);
4836
4837 case ISD::SMIN:
4838 case ISD::SMAX:
4839 case ISD::UMIN:
4840 case ISD::UMAX:
4841 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 1), DemandedElts, OrZero,
4842 Depth: Depth + 1) &&
4843 isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), DemandedElts, OrZero,
4844 Depth: Depth + 1);
4845
4846 case ISD::SELECT:
4847 case ISD::VSELECT:
4848 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 2), DemandedElts, OrZero,
4849 Depth: Depth + 1) &&
4850 isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 1), DemandedElts, OrZero,
4851 Depth: Depth + 1);
4852
4853 case ISD::ZERO_EXTEND:
4854 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), DemandedElts, OrZero,
4855 Depth: Depth + 1);
4856
4857 case ISD::VSCALE:
4858 // vscale(power-of-two) is a power-of-two
4859 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), /*OrZero=*/false,
4860 Depth: Depth + 1);
4861
4862 case ISD::VECTOR_SHUFFLE: {
4863 assert(!Val.getValueType().isScalableVector());
4864 // Demanded elements with undef shuffle mask elements are unknown
4865 // - we cannot guarantee they are a power of two, so return false.
4866 APInt DemandedLHS, DemandedRHS;
4867 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val);
4868 assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
4869 if (!getShuffleDemandedElts(SrcWidth: NumElts, Mask: SVN->getMask(), DemandedElts,
4870 DemandedLHS, DemandedRHS))
4871 return false;
4872
4873 // All demanded elements from LHS must be known power of two.
4874 if (!!DemandedLHS && !isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), DemandedElts: DemandedLHS,
4875 OrZero, Depth: Depth + 1))
4876 return false;
4877
4878 // All demanded elements from RHS must be known power of two.
4879 if (!!DemandedRHS && !isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 1), DemandedElts: DemandedRHS,
4880 OrZero, Depth: Depth + 1))
4881 return false;
4882
4883 return true;
4884 }
4885 }
4886
4887 // More could be done here, though the above checks are enough
4888 // to handle some common cases.
4889 return false;
4890}
4891
4892bool SelectionDAG::isKnownToBeAPowerOfTwoFP(SDValue Val, unsigned Depth) const {
4893 if (ConstantFPSDNode *C1 = isConstOrConstSplatFP(N: Val, AllowUndefs: true))
4894 return C1->getValueAPF().getExactLog2Abs() >= 0;
4895
4896 if (Val.getOpcode() == ISD::UINT_TO_FP || Val.getOpcode() == ISD::SINT_TO_FP)
4897 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), OrZero: Depth + 1);
4898
4899 return false;
4900}
4901
4902unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const {
4903 APInt DemandedElts = getDemandAllEltsMask(V: Op);
4904 return ComputeNumSignBits(Op, DemandedElts, Depth);
4905}
4906
4907unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, const APInt &DemandedElts,
4908 unsigned Depth) const {
4909 EVT VT = Op.getValueType();
4910 assert((VT.isInteger() || VT.isFloatingPoint()) && "Invalid VT!");
4911 unsigned VTBits = VT.getScalarSizeInBits();
4912 unsigned NumElts = DemandedElts.getBitWidth();
4913 unsigned Tmp, Tmp2;
4914 unsigned FirstAnswer = 1;
4915
4916 assert((!VT.isScalableVector() || NumElts == 1) &&
4917 "DemandedElts for scalable vectors must be 1 to represent all lanes");
4918
4919 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op)) {
4920 const APInt &Val = C->getAPIntValue();
4921 return Val.getNumSignBits();
4922 }
4923
4924 if (Depth >= MaxRecursionDepth)
4925 return 1; // Limit search depth.
4926
4927 if (!DemandedElts)
4928 return 1; // No demanded elts, better to assume we don't know anything.
4929
4930 unsigned Opcode = Op.getOpcode();
4931 switch (Opcode) {
4932 default: break;
4933 case ISD::AssertSext:
4934 Tmp = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT().getSizeInBits();
4935 return VTBits-Tmp+1;
4936 case ISD::AssertZext:
4937 Tmp = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT().getSizeInBits();
4938 return VTBits-Tmp;
4939 case ISD::FREEZE:
4940 if (isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 0), DemandedElts,
4941 Kind: UndefPoisonKind::UndefOrPoison))
4942 return ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4943 break;
4944 case ISD::MERGE_VALUES:
4945 return ComputeNumSignBits(Op: Op.getOperand(i: Op.getResNo()), DemandedElts,
4946 Depth: Depth + 1);
4947 case ISD::SPLAT_VECTOR: {
4948 // Check if the sign bits of source go down as far as the truncated value.
4949 unsigned NumSrcBits = Op.getOperand(i: 0).getValueSizeInBits();
4950 unsigned NumSrcSignBits = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
4951 if (NumSrcSignBits > (NumSrcBits - VTBits))
4952 return NumSrcSignBits - (NumSrcBits - VTBits);
4953 break;
4954 }
4955 case ISD::BUILD_VECTOR:
4956 assert(!VT.isScalableVector());
4957 Tmp = VTBits;
4958 for (unsigned i = 0, e = Op.getNumOperands(); (i < e) && (Tmp > 1); ++i) {
4959 if (!DemandedElts[i])
4960 continue;
4961
4962 SDValue SrcOp = Op.getOperand(i);
4963 // BUILD_VECTOR can implicitly truncate sources, we handle this specially
4964 // for constant nodes to ensure we only look at the sign bits.
4965 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: SrcOp)) {
4966 APInt T = C->getAPIntValue().trunc(width: VTBits);
4967 Tmp2 = T.getNumSignBits();
4968 } else if (SrcOp.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
4969 SrcOp.getOperand(i: 0).getScalarValueSizeInBits() >= VTBits) {
4970 // EXTRACT_VECTOR_ELT can extend the value with high bits undefined. If
4971 // this BUILD_VECTOR truncates those undefined bits we can just look
4972 // through the SrcOp and query the vector directly.
4973 SDValue InVec = SrcOp.getOperand(i: 0);
4974 EVT InVecVT = InVec.getValueType();
4975
4976 APInt DemandedSrcElts;
4977 if (InVecVT.isScalableVector())
4978 // Demand all elements.
4979 DemandedSrcElts = APInt(1, 1);
4980 else {
4981 unsigned NumSrcElts = InVecVT.getVectorNumElements();
4982 auto *ConstEltNo = dyn_cast<ConstantSDNode>(Val: SrcOp.getOperand(i: 1));
4983 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(RHS: NumSrcElts))
4984 DemandedSrcElts =
4985 APInt::getOneBitSet(numBits: NumSrcElts, BitNo: ConstEltNo->getZExtValue());
4986 else
4987 DemandedSrcElts = APInt::getAllOnes(numBits: NumSrcElts);
4988 }
4989
4990 Tmp2 = ComputeNumSignBits(Op: InVec, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
4991 unsigned ExtraBits = InVec.getScalarValueSizeInBits() - VTBits;
4992 if (ExtraBits)
4993 Tmp2 = (Tmp2 > ExtraBits ? Tmp2 - ExtraBits : 1);
4994 } else {
4995 Tmp2 = ComputeNumSignBits(Op: SrcOp, Depth: Depth + 1);
4996
4997 if (SrcOp.getValueSizeInBits() != VTBits) {
4998 assert(SrcOp.getValueSizeInBits() > VTBits &&
4999 "Expected BUILD_VECTOR implicit truncation");
5000 unsigned ExtraBits = SrcOp.getValueSizeInBits() - VTBits;
5001 Tmp2 = (Tmp2 > ExtraBits ? Tmp2 - ExtraBits : 1);
5002 }
5003 }
5004 Tmp = std::min(a: Tmp, b: Tmp2);
5005 }
5006 return Tmp;
5007
5008 case ISD::VECTOR_COMPRESS: {
5009 SDValue Vec = Op.getOperand(i: 0);
5010 SDValue PassThru = Op.getOperand(i: 2);
5011 Tmp = ComputeNumSignBits(Op: PassThru, DemandedElts, Depth: Depth + 1);
5012 if (Tmp == 1)
5013 return 1;
5014 Tmp2 = ComputeNumSignBits(Op: Vec, Depth: Depth + 1);
5015 Tmp = std::min(a: Tmp, b: Tmp2);
5016 return Tmp;
5017 }
5018
5019 case ISD::VECTOR_SHUFFLE: {
5020 // Collect the minimum number of sign bits that are shared by every vector
5021 // element referenced by the shuffle.
5022 APInt DemandedLHS, DemandedRHS;
5023 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
5024 assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
5025 if (!getShuffleDemandedElts(SrcWidth: NumElts, Mask: SVN->getMask(), DemandedElts,
5026 DemandedLHS, DemandedRHS))
5027 return 1;
5028
5029 Tmp = std::numeric_limits<unsigned>::max();
5030 if (!!DemandedLHS)
5031 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts: DemandedLHS, Depth: Depth + 1);
5032 if (!!DemandedRHS) {
5033 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts: DemandedRHS, Depth: Depth + 1);
5034 Tmp = std::min(a: Tmp, b: Tmp2);
5035 }
5036 // If we don't know anything, early out and try computeKnownBits fall-back.
5037 if (Tmp == 1)
5038 break;
5039 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
5040 return Tmp;
5041 }
5042
5043 case ISD::BITCAST: {
5044 if (VT.isScalableVector())
5045 break;
5046 SDValue N0 = Op.getOperand(i: 0);
5047 EVT SrcVT = N0.getValueType();
5048 unsigned SrcBits = SrcVT.getScalarSizeInBits();
5049
5050 // Ignore bitcasts from unsupported types..
5051 if (!(SrcVT.isInteger() || SrcVT.isFloatingPoint()))
5052 break;
5053
5054 // Fast handling of 'identity' bitcasts.
5055 if (VTBits == SrcBits)
5056 return ComputeNumSignBits(Op: N0, DemandedElts, Depth: Depth + 1);
5057
5058 bool IsLE = getDataLayout().isLittleEndian();
5059
5060 // Bitcast 'large element' scalar/vector to 'small element' vector.
5061 if ((SrcBits % VTBits) == 0) {
5062 assert(VT.isVector() && "Expected bitcast to vector");
5063
5064 unsigned Scale = SrcBits / VTBits;
5065 APInt SrcDemandedElts =
5066 APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumElts / Scale);
5067
5068 // Fast case - sign splat can be simply split across the small elements.
5069 Tmp = ComputeNumSignBits(Op: N0, DemandedElts: SrcDemandedElts, Depth: Depth + 1);
5070 if (Tmp == SrcBits)
5071 return VTBits;
5072
5073 // Slow case - determine how far the sign extends into each sub-element.
5074 Tmp2 = VTBits;
5075 for (unsigned i = 0; i != NumElts; ++i)
5076 if (DemandedElts[i]) {
5077 unsigned SubOffset = i % Scale;
5078 SubOffset = (IsLE ? ((Scale - 1) - SubOffset) : SubOffset);
5079 SubOffset = SubOffset * VTBits;
5080 if (Tmp <= SubOffset)
5081 return 1;
5082 Tmp2 = std::min(a: Tmp2, b: Tmp - SubOffset);
5083 }
5084 return Tmp2;
5085 }
5086 break;
5087 }
5088
5089 case ISD::FP_TO_SINT_SAT:
5090 // FP_TO_SINT_SAT produces a signed value that fits in the saturating VT.
5091 Tmp = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT().getScalarSizeInBits();
5092 return VTBits - Tmp + 1;
5093 case ISD::SIGN_EXTEND:
5094 Tmp = VTBits - Op.getOperand(i: 0).getScalarValueSizeInBits();
5095 return ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth+1) + Tmp;
5096 case ISD::SIGN_EXTEND_INREG:
5097 // Max of the input and what this extends.
5098 Tmp = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT().getScalarSizeInBits();
5099 Tmp = VTBits-Tmp+1;
5100 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth+1);
5101 return std::max(a: Tmp, b: Tmp2);
5102 case ISD::SIGN_EXTEND_VECTOR_INREG: {
5103 if (VT.isScalableVector())
5104 break;
5105 SDValue Src = Op.getOperand(i: 0);
5106 EVT SrcVT = Src.getValueType();
5107 APInt DemandedSrcElts = DemandedElts.zext(width: SrcVT.getVectorNumElements());
5108 Tmp = VTBits - SrcVT.getScalarSizeInBits();
5109 return ComputeNumSignBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth+1) + Tmp;
5110 }
5111 case ISD::SRA:
5112 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5113 // SRA X, C -> adds C sign bits.
5114 if (std::optional<unsigned> ShAmt =
5115 getValidMinimumShiftAmount(V: Op, DemandedElts, Depth: Depth + 1))
5116 Tmp = std::min(a: Tmp + *ShAmt, b: VTBits);
5117 return Tmp;
5118 case ISD::SHL:
5119 if (std::optional<ConstantRange> ShAmtRange =
5120 getValidShiftAmountRange(V: Op, DemandedElts, Depth: Depth + 1)) {
5121 unsigned MaxShAmt = ShAmtRange->getUnsignedMax().getZExtValue();
5122 unsigned MinShAmt = ShAmtRange->getUnsignedMin().getZExtValue();
5123 // Try to look through ZERO/SIGN/ANY_EXTEND. If all extended bits are
5124 // shifted out, then we can compute the number of sign bits for the
5125 // operand being extended. A future improvement could be to pass along the
5126 // "shifted left by" information in the recursive calls to
5127 // ComputeKnownSignBits. Allowing us to handle this more generically.
5128 if (ISD::isExtOpcode(Opcode: Op.getOperand(i: 0).getOpcode())) {
5129 SDValue Ext = Op.getOperand(i: 0);
5130 EVT ExtVT = Ext.getValueType();
5131 SDValue Extendee = Ext.getOperand(i: 0);
5132 EVT ExtendeeVT = Extendee.getValueType();
5133 unsigned SizeDifference =
5134 ExtVT.getScalarSizeInBits() - ExtendeeVT.getScalarSizeInBits();
5135 if (SizeDifference <= MinShAmt) {
5136 Tmp = SizeDifference +
5137 ComputeNumSignBits(Op: Extendee, DemandedElts, Depth: Depth + 1);
5138 if (MaxShAmt < Tmp)
5139 return Tmp - MaxShAmt;
5140 }
5141 }
5142 // shl destroys sign bits, ensure it doesn't shift out all sign bits.
5143 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5144 if (MaxShAmt < Tmp)
5145 return Tmp - MaxShAmt;
5146 }
5147 break;
5148 case ISD::AND:
5149 case ISD::OR:
5150 case ISD::XOR: // NOT is handled here.
5151 // Logical binary ops preserve the number of sign bits at the worst.
5152 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth+1);
5153 if (Tmp != 1) {
5154 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth+1);
5155 FirstAnswer = std::min(a: Tmp, b: Tmp2);
5156 // We computed what we know about the sign bits as our first
5157 // answer. Now proceed to the generic code that uses
5158 // computeKnownBits, and pick whichever answer is better.
5159 }
5160 break;
5161
5162 case ISD::SELECT:
5163 case ISD::VSELECT:
5164 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth+1);
5165 if (Tmp == 1) return 1; // Early out.
5166 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth+1);
5167 return std::min(a: Tmp, b: Tmp2);
5168 case ISD::SELECT_CC:
5169 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth+1);
5170 if (Tmp == 1) return 1; // Early out.
5171 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 3), DemandedElts, Depth: Depth+1);
5172 return std::min(a: Tmp, b: Tmp2);
5173
5174 case ISD::SMIN:
5175 case ISD::SMAX: {
5176 // If we have a clamp pattern, we know that the number of sign bits will be
5177 // the minimum of the clamp min/max range.
5178 bool IsMax = (Opcode == ISD::SMAX);
5179 ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr;
5180 if ((CstLow = isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts)))
5181 if (Op.getOperand(i: 0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX))
5182 CstHigh =
5183 isConstOrConstSplat(N: Op.getOperand(i: 0).getOperand(i: 1), DemandedElts);
5184 if (CstLow && CstHigh) {
5185 if (!IsMax)
5186 std::swap(a&: CstLow, b&: CstHigh);
5187 if (CstLow->getAPIntValue().sle(RHS: CstHigh->getAPIntValue())) {
5188 Tmp = CstLow->getAPIntValue().getNumSignBits();
5189 Tmp2 = CstHigh->getAPIntValue().getNumSignBits();
5190 return std::min(a: Tmp, b: Tmp2);
5191 }
5192 }
5193
5194 // Fallback - just get the minimum number of sign bits of the operands.
5195 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5196 if (Tmp == 1)
5197 return 1; // Early out.
5198 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5199 return std::min(a: Tmp, b: Tmp2);
5200 }
5201 case ISD::UMIN:
5202 case ISD::UMAX:
5203 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5204 if (Tmp == 1)
5205 return 1; // Early out.
5206 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5207 return std::min(a: Tmp, b: Tmp2);
5208 case ISD::SSUBO_CARRY:
5209 case ISD::USUBO_CARRY:
5210 // sub_carry(x,x,c) -> 0/-1 (sext carry)
5211 if (Op.getResNo() == 0 && Op.getOperand(i: 0) == Op.getOperand(i: 1))
5212 return VTBits;
5213 [[fallthrough]];
5214 case ISD::SADDO:
5215 case ISD::UADDO:
5216 case ISD::SADDO_CARRY:
5217 case ISD::UADDO_CARRY:
5218 case ISD::SSUBO:
5219 case ISD::USUBO:
5220 case ISD::SMULO:
5221 case ISD::UMULO:
5222 if (Op.getResNo() != 1)
5223 break;
5224 // The boolean result conforms to getBooleanContents. Fall through.
5225 // If setcc returns 0/-1, all bits are sign bits.
5226 // We know that we have an integer-based boolean since these operations
5227 // are only available for integer.
5228 if (TLI->getBooleanContents(isVec: VT.isVector(), isFloat: false) ==
5229 TargetLowering::ZeroOrNegativeOneBooleanContent)
5230 return VTBits;
5231 break;
5232 case ISD::SETCC:
5233 case ISD::SETCCCARRY:
5234 case ISD::STRICT_FSETCC:
5235 case ISD::STRICT_FSETCCS: {
5236 unsigned OpNo = Op->isStrictFPOpcode() ? 1 : 0;
5237 // If setcc returns 0/-1, all bits are sign bits.
5238 if (TLI->getBooleanContents(Type: Op.getOperand(i: OpNo).getValueType()) ==
5239 TargetLowering::ZeroOrNegativeOneBooleanContent)
5240 return VTBits;
5241 break;
5242 }
5243 case ISD::GET_ACTIVE_LANE_MASK:
5244 // Semantically similar to icmp ult.
5245 if (TLI->getBooleanContents(isVec: VT.isVector(), /*isFloat=*/false) ==
5246 TargetLowering::ZeroOrNegativeOneBooleanContent)
5247 return VTBits;
5248 break;
5249 case ISD::ROTL:
5250 case ISD::ROTR: {
5251 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5252 ConstantSDNode *C = isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts);
5253 FirstAnswer = SignBitsOps::rot(
5254 SrcSignBits: Tmp, BitWidth: VTBits, RotAmt: C ? std::optional(C->getAPIntValue()) : std::nullopt,
5255 IsRotateRight: Opcode == ISD::ROTR);
5256 break;
5257 }
5258 case ISD::ADD:
5259 case ISD::ADDC:
5260 // TODO: Move Operand 1 check before Operand 0 check
5261 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5262 if (Tmp == 1) return 1; // Early out.
5263
5264 // Special case decrementing a value (ADD X, -1):
5265 if (ConstantSDNode *CRHS =
5266 isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts))
5267 if (CRHS->isAllOnes()) {
5268 KnownBits Known =
5269 computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5270
5271 // If the input is known to be 0 or 1, the output is 0/-1, which is all
5272 // sign bits set.
5273 if ((Known.Zero | 1).isAllOnes())
5274 return VTBits;
5275
5276 // If we are subtracting one from a positive number, there is no carry
5277 // out of the result.
5278 if (Known.isNonNegative())
5279 return Tmp;
5280 }
5281
5282 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5283 if (Tmp2 == 1) return 1; // Early out.
5284
5285 // Add can have at most one carry bit. Thus we know that the output
5286 // is, at worst, one more bit than the inputs.
5287 return std::min(a: Tmp, b: Tmp2) - 1;
5288 case ISD::SUB:
5289 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5290 if (Tmp2 == 1) return 1; // Early out.
5291
5292 // Handle NEG.
5293 if (ConstantSDNode *CLHS =
5294 isConstOrConstSplat(N: Op.getOperand(i: 0), DemandedElts))
5295 if (CLHS->isZero()) {
5296 KnownBits Known =
5297 computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5298 // If the input is known to be 0 or 1, the output is 0/-1, which is all
5299 // sign bits set.
5300 if ((Known.Zero | 1).isAllOnes())
5301 return VTBits;
5302
5303 // If the input is known to be positive (the sign bit is known clear),
5304 // the output of the NEG has the same number of sign bits as the input.
5305 if (Known.isNonNegative())
5306 return Tmp2;
5307
5308 // Otherwise, we treat this like a SUB.
5309 }
5310
5311 // Sub can have at most one carry bit. Thus we know that the output
5312 // is, at worst, one more bit than the inputs.
5313 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5314 if (Tmp == 1) return 1; // Early out.
5315 return std::min(a: Tmp, b: Tmp2) - 1;
5316 case ISD::MUL: {
5317 // The output of the Mul can be at most twice the valid bits in the inputs.
5318 unsigned SignBitsOp0 = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
5319 if (SignBitsOp0 == 1)
5320 break;
5321 unsigned SignBitsOp1 = ComputeNumSignBits(Op: Op.getOperand(i: 1), Depth: Depth + 1);
5322 if (SignBitsOp1 == 1)
5323 break;
5324 unsigned OutValidBits =
5325 (VTBits - SignBitsOp0 + 1) + (VTBits - SignBitsOp1 + 1);
5326 return OutValidBits > VTBits ? 1 : VTBits - OutValidBits + 1;
5327 }
5328 case ISD::AVGCEILS:
5329 case ISD::AVGFLOORS:
5330 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5331 if (Tmp == 1)
5332 return 1; // Early out.
5333 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5334 return std::min(a: Tmp, b: Tmp2);
5335 case ISD::SREM:
5336 // The sign bit is the LHS's sign bit, except when the result of the
5337 // remainder is zero. The magnitude of the result should be less than or
5338 // equal to the magnitude of the LHS. Therefore, the result should have
5339 // at least as many sign bits as the left hand side.
5340 return ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5341 case ISD::TRUNCATE: {
5342 // Check if the sign bits of source go down as far as the truncated value.
5343 unsigned NumSrcBits = Op.getOperand(i: 0).getScalarValueSizeInBits();
5344 unsigned NumSrcSignBits = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
5345 if (NumSrcSignBits > (NumSrcBits - VTBits))
5346 return NumSrcSignBits - (NumSrcBits - VTBits);
5347 break;
5348 }
5349 case ISD::EXTRACT_ELEMENT: {
5350 if (VT.isScalableVector())
5351 break;
5352 const int KnownSign = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth+1);
5353 const int BitWidth = Op.getValueSizeInBits();
5354 const int Items = Op.getOperand(i: 0).getValueSizeInBits() / BitWidth;
5355
5356 // Get reverse index (starting from 1), Op1 value indexes elements from
5357 // little end. Sign starts at big end.
5358 const int rIndex = Items - 1 - Op.getConstantOperandVal(i: 1);
5359
5360 // If the sign portion ends in our element the subtraction gives correct
5361 // result. Otherwise it gives either negative or > bitwidth result
5362 return std::clamp(val: KnownSign - rIndex * BitWidth, lo: 1, hi: BitWidth);
5363 }
5364 case ISD::INSERT_VECTOR_ELT: {
5365 if (VT.isScalableVector())
5366 break;
5367 // If we know the element index, split the demand between the
5368 // source vector and the inserted element, otherwise assume we need
5369 // the original demanded vector elements and the value.
5370 SDValue InVec = Op.getOperand(i: 0);
5371 SDValue InVal = Op.getOperand(i: 1);
5372 SDValue EltNo = Op.getOperand(i: 2);
5373 bool DemandedVal = true;
5374 APInt DemandedVecElts = DemandedElts;
5375 auto *CEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
5376 if (CEltNo && CEltNo->getAPIntValue().ult(RHS: NumElts)) {
5377 unsigned EltIdx = CEltNo->getZExtValue();
5378 DemandedVal = !!DemandedElts[EltIdx];
5379 DemandedVecElts.clearBit(BitPosition: EltIdx);
5380 }
5381 Tmp = std::numeric_limits<unsigned>::max();
5382 if (DemandedVal) {
5383 // TODO - handle implicit truncation of inserted elements.
5384 if (InVal.getScalarValueSizeInBits() != VTBits)
5385 break;
5386 Tmp2 = ComputeNumSignBits(Op: InVal, Depth: Depth + 1);
5387 Tmp = std::min(a: Tmp, b: Tmp2);
5388 }
5389 if (!!DemandedVecElts) {
5390 Tmp2 = ComputeNumSignBits(Op: InVec, DemandedElts: DemandedVecElts, Depth: Depth + 1);
5391 Tmp = std::min(a: Tmp, b: Tmp2);
5392 }
5393 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
5394 return Tmp;
5395 }
5396 case ISD::EXTRACT_VECTOR_ELT: {
5397 SDValue InVec = Op.getOperand(i: 0);
5398 SDValue EltNo = Op.getOperand(i: 1);
5399 EVT VecVT = InVec.getValueType();
5400 // ComputeNumSignBits not yet implemented for scalable vectors.
5401 if (VecVT.isScalableVector())
5402 break;
5403 const unsigned BitWidth = Op.getValueSizeInBits();
5404 const unsigned EltBitWidth = Op.getOperand(i: 0).getScalarValueSizeInBits();
5405 const unsigned NumSrcElts = VecVT.getVectorNumElements();
5406
5407 // If BitWidth > EltBitWidth the value is anyext:ed, and we do not know
5408 // anything about sign bits. But if the sizes match we can derive knowledge
5409 // about sign bits from the vector operand.
5410 if (BitWidth != EltBitWidth)
5411 break;
5412
5413 // If we know the element index, just demand that vector element, else for
5414 // an unknown element index, ignore DemandedElts and demand them all.
5415 APInt DemandedSrcElts = APInt::getAllOnes(numBits: NumSrcElts);
5416 auto *ConstEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
5417 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(RHS: NumSrcElts))
5418 DemandedSrcElts =
5419 APInt::getOneBitSet(numBits: NumSrcElts, BitNo: ConstEltNo->getZExtValue());
5420
5421 return ComputeNumSignBits(Op: InVec, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
5422 }
5423 case ISD::EXTRACT_SUBVECTOR: {
5424 // Offset the demanded elts by the subvector index.
5425 SDValue Src = Op.getOperand(i: 0);
5426
5427 APInt DemandedSrcElts;
5428 if (Src.getValueType().isScalableVector())
5429 DemandedSrcElts = APInt(1, 1);
5430 else {
5431 uint64_t Idx = Op.getConstantOperandVal(i: 1);
5432 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
5433 DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
5434 }
5435 return ComputeNumSignBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
5436 }
5437 case ISD::CONCAT_VECTORS: {
5438 if (VT.isScalableVector())
5439 break;
5440 // Determine the minimum number of sign bits across all demanded
5441 // elts of the input vectors. Early out if the result is already 1.
5442 Tmp = std::numeric_limits<unsigned>::max();
5443 EVT SubVectorVT = Op.getOperand(i: 0).getValueType();
5444 unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements();
5445 unsigned NumSubVectors = Op.getNumOperands();
5446 for (unsigned i = 0; (i < NumSubVectors) && (Tmp > 1); ++i) {
5447 APInt DemandedSub =
5448 DemandedElts.extractBits(numBits: NumSubVectorElts, bitPosition: i * NumSubVectorElts);
5449 if (!DemandedSub)
5450 continue;
5451 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i), DemandedElts: DemandedSub, Depth: Depth + 1);
5452 Tmp = std::min(a: Tmp, b: Tmp2);
5453 }
5454 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
5455 return Tmp;
5456 }
5457 case ISD::INSERT_SUBVECTOR: {
5458 SDValue Src = Op.getOperand(i: 0);
5459 SDValue Sub = Op.getOperand(i: 1);
5460 if (VT.isScalableVector()) {
5461 Tmp = ComputeNumSignBits(Op: Sub, Depth: Depth + 1);
5462 Tmp = std::min(a: Tmp, b: ComputeNumSignBits(Op: Src, Depth: Depth + 1));
5463 return Tmp;
5464 }
5465 // Demand any elements from the subvector and the remainder from the src its
5466 // inserted into.
5467 uint64_t Idx = Op.getConstantOperandVal(i: 2);
5468 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
5469 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
5470 APInt DemandedSrcElts = DemandedElts;
5471 DemandedSrcElts.clearBits(LoBit: Idx, HiBit: Idx + NumSubElts);
5472
5473 Tmp = std::numeric_limits<unsigned>::max();
5474 if (!!DemandedSubElts) {
5475 Tmp = ComputeNumSignBits(Op: Sub, DemandedElts: DemandedSubElts, Depth: Depth + 1);
5476 if (Tmp == 1)
5477 return 1; // early-out
5478 }
5479 if (!!DemandedSrcElts) {
5480 Tmp2 = ComputeNumSignBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
5481 Tmp = std::min(a: Tmp, b: Tmp2);
5482 }
5483 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
5484 return Tmp;
5485 }
5486 case ISD::LOAD: {
5487 // If we are looking at the loaded value of the SDNode.
5488 if (Op.getResNo() != 0)
5489 break;
5490
5491 LoadSDNode *LD = cast<LoadSDNode>(Val&: Op);
5492 if (const MDNode *Ranges = LD->getRanges()) {
5493 if (DemandedElts != 1)
5494 break;
5495
5496 ConstantRange CR = getConstantRangeFromMetadata(RangeMD: *Ranges);
5497 if (VTBits > CR.getBitWidth()) {
5498 switch (LD->getExtensionType()) {
5499 case ISD::SEXTLOAD:
5500 CR = CR.signExtend(BitWidth: VTBits);
5501 break;
5502 case ISD::ZEXTLOAD:
5503 CR = CR.zeroExtend(BitWidth: VTBits);
5504 break;
5505 default:
5506 break;
5507 }
5508 }
5509
5510 if (VTBits != CR.getBitWidth())
5511 break;
5512 return std::min(a: CR.getSignedMin().getNumSignBits(),
5513 b: CR.getSignedMax().getNumSignBits());
5514 }
5515
5516 unsigned ExtType = LD->getExtensionType();
5517 switch (ExtType) {
5518 default:
5519 break;
5520 case ISD::SEXTLOAD: // e.g. i16->i32 = '17' bits known.
5521 Tmp = LD->getMemoryVT().getScalarSizeInBits();
5522 return VTBits - Tmp + 1;
5523 case ISD::ZEXTLOAD: // e.g. i16->i32 = '16' bits known.
5524 Tmp = LD->getMemoryVT().getScalarSizeInBits();
5525 return VTBits - Tmp;
5526 case ISD::NON_EXTLOAD:
5527 if (const Constant *Cst = TLI->getTargetConstantFromLoad(LD)) {
5528 // We only need to handle vectors - computeKnownBits should handle
5529 // scalar cases.
5530 Type *CstTy = Cst->getType();
5531 if (CstTy->isVectorTy() && !VT.isScalableVector() &&
5532 (NumElts * VTBits) == CstTy->getPrimitiveSizeInBits() &&
5533 VTBits == CstTy->getScalarSizeInBits()) {
5534 Tmp = VTBits;
5535 for (unsigned i = 0; i != NumElts; ++i) {
5536 if (!DemandedElts[i])
5537 continue;
5538 if (Constant *Elt = Cst->getAggregateElement(Elt: i)) {
5539 if (auto *CInt = dyn_cast<ConstantInt>(Val: Elt)) {
5540 const APInt &Value = CInt->getValue();
5541 Tmp = std::min(a: Tmp, b: Value.getNumSignBits());
5542 continue;
5543 }
5544 if (auto *CFP = dyn_cast<ConstantFP>(Val: Elt)) {
5545 APInt Value = CFP->getValueAPF().bitcastToAPInt();
5546 Tmp = std::min(a: Tmp, b: Value.getNumSignBits());
5547 continue;
5548 }
5549 }
5550 // Unknown type. Conservatively assume no bits match sign bit.
5551 return 1;
5552 }
5553 return Tmp;
5554 }
5555 }
5556 break;
5557 }
5558
5559 break;
5560 }
5561 case ISD::ATOMIC_CMP_SWAP:
5562 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
5563 case ISD::ATOMIC_SWAP:
5564 case ISD::ATOMIC_LOAD_ADD:
5565 case ISD::ATOMIC_LOAD_SUB:
5566 case ISD::ATOMIC_LOAD_AND:
5567 case ISD::ATOMIC_LOAD_CLR:
5568 case ISD::ATOMIC_LOAD_OR:
5569 case ISD::ATOMIC_LOAD_XOR:
5570 case ISD::ATOMIC_LOAD_NAND:
5571 case ISD::ATOMIC_LOAD_MIN:
5572 case ISD::ATOMIC_LOAD_MAX:
5573 case ISD::ATOMIC_LOAD_UMIN:
5574 case ISD::ATOMIC_LOAD_UMAX:
5575 case ISD::ATOMIC_LOAD: {
5576 auto *AT = cast<AtomicSDNode>(Val&: Op);
5577 // If we are looking at the loaded value.
5578 if (Op.getResNo() == 0) {
5579 Tmp = AT->getMemoryVT().getScalarSizeInBits();
5580 if (Tmp == VTBits)
5581 return 1; // early-out
5582
5583 // For atomic_load, prefer to use the extension type.
5584 if (Op->getOpcode() == ISD::ATOMIC_LOAD) {
5585 switch (AT->getExtensionType()) {
5586 default:
5587 break;
5588 case ISD::SEXTLOAD:
5589 return VTBits - Tmp + 1;
5590 case ISD::ZEXTLOAD:
5591 return VTBits - Tmp;
5592 }
5593 }
5594
5595 if (TLI->getExtendForAtomicOps() == ISD::SIGN_EXTEND)
5596 return VTBits - Tmp + 1;
5597 if (TLI->getExtendForAtomicOps() == ISD::ZERO_EXTEND)
5598 return VTBits - Tmp;
5599 }
5600 break;
5601 }
5602 }
5603
5604 // Allow the target to implement this method for its nodes.
5605 if (Opcode >= ISD::BUILTIN_OP_END ||
5606 Opcode == ISD::INTRINSIC_WO_CHAIN ||
5607 Opcode == ISD::INTRINSIC_W_CHAIN ||
5608 Opcode == ISD::INTRINSIC_VOID) {
5609 // TODO: This can probably be removed once target code is audited. This
5610 // is here purely to reduce patch size and review complexity.
5611 if (!VT.isScalableVector()) {
5612 unsigned NumBits =
5613 TLI->ComputeNumSignBitsForTargetNode(Op, DemandedElts, DAG: *this, Depth);
5614 if (NumBits > 1)
5615 FirstAnswer = std::max(a: FirstAnswer, b: NumBits);
5616 }
5617 }
5618
5619 // Finally, if we can prove that the top bits of the result are 0's or 1's,
5620 // use this information.
5621 KnownBits Known = computeKnownBits(Op, DemandedElts, Depth);
5622 return std::max(a: FirstAnswer, b: Known.countMinSignBits());
5623}
5624
5625unsigned SelectionDAG::ComputeMaxSignificantBits(SDValue Op,
5626 unsigned Depth) const {
5627 unsigned SignBits = ComputeNumSignBits(Op, Depth);
5628 return Op.getScalarValueSizeInBits() - SignBits + 1;
5629}
5630
5631unsigned SelectionDAG::ComputeMaxSignificantBits(SDValue Op,
5632 const APInt &DemandedElts,
5633 unsigned Depth) const {
5634 unsigned SignBits = ComputeNumSignBits(Op, DemandedElts, Depth);
5635 return Op.getScalarValueSizeInBits() - SignBits + 1;
5636}
5637
5638bool SelectionDAG::isGuaranteedNotToBeUndefOrPoison(SDValue Op,
5639 UndefPoisonKind Kind,
5640 unsigned Depth) const {
5641 // Early out for FREEZE.
5642 if (Op.getOpcode() == ISD::FREEZE)
5643 return true;
5644
5645 APInt DemandedElts = getDemandAllEltsMask(V: Op);
5646 return isGuaranteedNotToBeUndefOrPoison(Op, DemandedElts, Kind, Depth);
5647}
5648
5649bool SelectionDAG::isGuaranteedNotToBeUndefOrPoison(SDValue Op,
5650 const APInt &DemandedElts,
5651 UndefPoisonKind Kind,
5652 unsigned Depth) const {
5653 unsigned Opcode = Op.getOpcode();
5654
5655 // Early out for FREEZE.
5656 if (Opcode == ISD::FREEZE)
5657 return true;
5658
5659 if (Depth >= MaxRecursionDepth)
5660 return false; // Limit search depth.
5661
5662 if (isIntOrFPConstant(V: Op))
5663 return true;
5664
5665 switch (Opcode) {
5666 case ISD::CONDCODE:
5667 case ISD::VALUETYPE:
5668 case ISD::FrameIndex:
5669 case ISD::TargetFrameIndex:
5670 case ISD::CopyFromReg:
5671 return true;
5672
5673 case ISD::POISON:
5674 return !includesPoison(Kind);
5675
5676 case ISD::UNDEF:
5677 return !includesUndef(Kind);
5678
5679 case ISD::BITCAST: {
5680 SDValue Src = Op.getOperand(i: 0);
5681 EVT SrcVT = Src.getValueType();
5682 EVT DstVT = Op.getValueType();
5683
5684 if (!SrcVT.isVector() || !DstVT.isVector())
5685 return isGuaranteedNotToBeUndefOrPoison(Op: Src, Kind, Depth: Depth + 1);
5686
5687 unsigned SrcEltBits = SrcVT.getScalarSizeInBits();
5688 unsigned DstEltBits = DstVT.getScalarSizeInBits();
5689 ElementCount NumSrcElts = SrcVT.getVectorElementCount();
5690 [[maybe_unused]] ElementCount NumDstElts = DstVT.getVectorElementCount();
5691
5692 if (SrcEltBits == DstEltBits)
5693 return isGuaranteedNotToBeUndefOrPoison(Op: Src, DemandedElts, Kind,
5694 Depth: Depth + 1);
5695
5696 if (SrcEltBits < DstEltBits) {
5697 if (DstEltBits % SrcEltBits != 0)
5698 return isGuaranteedNotToBeUndefOrPoison(Op: Src, Kind, Depth: Depth + 1);
5699
5700 assert(NumSrcElts == NumDstElts * (DstEltBits / SrcEltBits) &&
5701 "Unexpected vector bitcast");
5702 APInt DemandedSrcElts =
5703 APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumSrcElts.getKnownMinValue());
5704 return isGuaranteedNotToBeUndefOrPoison(Op: Src, DemandedElts: DemandedSrcElts, Kind,
5705 Depth: Depth + 1);
5706 }
5707
5708 if (SrcEltBits % DstEltBits != 0)
5709 return isGuaranteedNotToBeUndefOrPoison(Op: Src, Kind, Depth: Depth + 1);
5710
5711 assert(NumDstElts == NumSrcElts * (SrcEltBits / DstEltBits) &&
5712 "Unexpected vector bitcast");
5713 APInt DemandedSrcElts =
5714 APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumSrcElts.getKnownMinValue());
5715 return isGuaranteedNotToBeUndefOrPoison(Op: Src, DemandedElts: DemandedSrcElts, Kind,
5716 Depth: Depth + 1);
5717 }
5718
5719 case ISD::BUILD_VECTOR:
5720 // NOTE: BUILD_VECTOR has implicit truncation of wider scalar elements -
5721 // this shouldn't affect the result.
5722 for (unsigned i = 0, e = Op.getNumOperands(); i < e; ++i) {
5723 if (!DemandedElts[i])
5724 continue;
5725 if (!isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i), Kind, Depth: Depth + 1))
5726 return false;
5727 }
5728 return true;
5729
5730 case ISD::CONCAT_VECTORS: {
5731 EVT VT = Op.getValueType();
5732 if (!VT.isFixedLengthVector())
5733 break;
5734
5735 EVT SubVT = Op.getOperand(i: 0).getValueType();
5736 unsigned NumSubElts = SubVT.getVectorNumElements();
5737 for (unsigned I = 0, E = Op.getNumOperands(); I != E; ++I) {
5738 APInt DemandedSubElts =
5739 DemandedElts.extractBits(numBits: NumSubElts, bitPosition: I * NumSubElts);
5740 if (!!DemandedSubElts &&
5741 !isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: I), DemandedElts: DemandedSubElts,
5742 Kind, Depth: Depth + 1))
5743 return false;
5744 }
5745 return true;
5746 }
5747
5748 case ISD::EXTRACT_SUBVECTOR: {
5749 SDValue Src = Op.getOperand(i: 0);
5750 if (Src.getValueType().isScalableVector())
5751 break;
5752 uint64_t Idx = Op.getConstantOperandVal(i: 1);
5753 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
5754 APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
5755 return isGuaranteedNotToBeUndefOrPoison(Op: Src, DemandedElts: DemandedSrcElts, Kind,
5756 Depth: Depth + 1);
5757 }
5758
5759 case ISD::INSERT_SUBVECTOR: {
5760 if (Op.getValueType().isScalableVector())
5761 break;
5762 SDValue Src = Op.getOperand(i: 0);
5763 SDValue Sub = Op.getOperand(i: 1);
5764 uint64_t Idx = Op.getConstantOperandVal(i: 2);
5765 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
5766 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
5767 APInt DemandedSrcElts = DemandedElts;
5768 DemandedSrcElts.clearBits(LoBit: Idx, HiBit: Idx + NumSubElts);
5769
5770 if (!!DemandedSubElts && !isGuaranteedNotToBeUndefOrPoison(
5771 Op: Sub, DemandedElts: DemandedSubElts, Kind, Depth: Depth + 1))
5772 return false;
5773 if (!!DemandedSrcElts && !isGuaranteedNotToBeUndefOrPoison(
5774 Op: Src, DemandedElts: DemandedSrcElts, Kind, Depth: Depth + 1))
5775 return false;
5776 return true;
5777 }
5778
5779 case ISD::EXTRACT_VECTOR_ELT: {
5780 SDValue Src = Op.getOperand(i: 0);
5781 auto *IndexC = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1));
5782 EVT SrcVT = Src.getValueType();
5783 if (SrcVT.isFixedLengthVector() && IndexC &&
5784 IndexC->getAPIntValue().ult(RHS: SrcVT.getVectorNumElements())) {
5785 APInt DemandedSrcElts = APInt::getOneBitSet(numBits: SrcVT.getVectorNumElements(),
5786 BitNo: IndexC->getZExtValue());
5787 return isGuaranteedNotToBeUndefOrPoison(Op: Src, DemandedElts: DemandedSrcElts, Kind,
5788 Depth: Depth + 1);
5789 }
5790 break;
5791 }
5792
5793 case ISD::INSERT_VECTOR_ELT: {
5794 SDValue InVec = Op.getOperand(i: 0);
5795 SDValue InVal = Op.getOperand(i: 1);
5796 SDValue EltNo = Op.getOperand(i: 2);
5797 EVT VT = InVec.getValueType();
5798 auto *IndexC = dyn_cast<ConstantSDNode>(Val&: EltNo);
5799 if (IndexC && VT.isFixedLengthVector() &&
5800 IndexC->getAPIntValue().ult(RHS: VT.getVectorNumElements())) {
5801 if (DemandedElts[IndexC->getZExtValue()] &&
5802 !isGuaranteedNotToBeUndefOrPoison(Op: InVal, Kind, Depth: Depth + 1))
5803 return false;
5804 APInt InVecDemandedElts = DemandedElts;
5805 InVecDemandedElts.clearBit(BitPosition: IndexC->getZExtValue());
5806 if (!!InVecDemandedElts &&
5807 !isGuaranteedNotToBeUndefOrPoison(
5808 Op: peekThroughInsertVectorElt(V: InVec, DemandedElts: InVecDemandedElts),
5809 DemandedElts: InVecDemandedElts, Kind, Depth: Depth + 1))
5810 return false;
5811 return true;
5812 }
5813 break;
5814 }
5815
5816 case ISD::SCALAR_TO_VECTOR:
5817 // Check upper (known undef) elements.
5818 if (DemandedElts.ugt(RHS: 1) && includesUndef(Kind))
5819 return false;
5820 // Check element zero.
5821 if (DemandedElts[0] &&
5822 !isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 0), Kind, Depth: Depth + 1))
5823 return false;
5824 return true;
5825
5826 case ISD::SPLAT_VECTOR:
5827 return isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 0), Kind, Depth: Depth + 1);
5828
5829 case ISD::SELECT: {
5830 return !canCreateUndefOrPoison(Op, DemandedElts, Kind,
5831 /*ConsiderFlags*/ true, Depth) &&
5832 isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 0), Kind,
5833 Depth: Depth + 1) &&
5834 isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 1), DemandedElts,
5835 Kind, Depth: Depth + 1) &&
5836 isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 2), DemandedElts,
5837 Kind, Depth: Depth + 1);
5838 }
5839
5840 case ISD::VECTOR_SHUFFLE: {
5841 APInt DemandedLHS, DemandedRHS;
5842 auto *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
5843 if (!getShuffleDemandedElts(SrcWidth: DemandedElts.getBitWidth(), Mask: SVN->getMask(),
5844 DemandedElts, DemandedLHS, DemandedRHS,
5845 /*AllowUndefElts=*/false))
5846 return false;
5847 if (!DemandedLHS.isZero() &&
5848 !isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 0), DemandedElts: DemandedLHS, Kind,
5849 Depth: Depth + 1))
5850 return false;
5851 if (!DemandedRHS.isZero() &&
5852 !isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 1), DemandedElts: DemandedRHS, Kind,
5853 Depth: Depth + 1))
5854 return false;
5855 return true;
5856 }
5857
5858 case ISD::SHL:
5859 case ISD::SRL:
5860 case ISD::SRA:
5861 // Shift amount operand is checked by canCreateUndefOrPoison. So it is
5862 // enough to check operand 0 if Op can't create undef/poison.
5863 return !canCreateUndefOrPoison(Op, DemandedElts, Kind,
5864 /*ConsiderFlags*/ true, Depth) &&
5865 isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 0), DemandedElts,
5866 Kind, Depth: Depth + 1);
5867
5868 case ISD::BSWAP:
5869 case ISD::CTPOP:
5870 case ISD::BITREVERSE:
5871 case ISD::AND:
5872 case ISD::OR:
5873 case ISD::XOR:
5874 case ISD::ADD:
5875 case ISD::SUB:
5876 case ISD::MUL:
5877 case ISD::SADDSAT:
5878 case ISD::UADDSAT:
5879 case ISD::SSUBSAT:
5880 case ISD::USUBSAT:
5881 case ISD::SSHLSAT:
5882 case ISD::USHLSAT:
5883 case ISD::SMIN:
5884 case ISD::SMAX:
5885 case ISD::UMIN:
5886 case ISD::UMAX:
5887 case ISD::ZERO_EXTEND:
5888 case ISD::SIGN_EXTEND:
5889 case ISD::ANY_EXTEND:
5890 case ISD::TRUNCATE:
5891 case ISD::VSELECT: {
5892 // If Op can't create undef/poison and none of its operands are undef/poison
5893 // then Op is never undef/poison. A difference from the more common check
5894 // below, outside the switch, is that we handle elementwise operations for
5895 // which the DemandedElts mask is valid for all operands here.
5896 return !canCreateUndefOrPoison(Op, DemandedElts, Kind,
5897 /*ConsiderFlags*/ true, Depth) &&
5898 all_of(Range: Op->ops(), P: [&](SDValue V) {
5899 return isGuaranteedNotToBeUndefOrPoison(Op: V, DemandedElts, Kind,
5900 Depth: Depth + 1);
5901 });
5902 }
5903
5904 // TODO: Search for noundef attributes from library functions.
5905
5906 // TODO: Pointers dereferenced by ISD::LOAD/STORE ops are noundef.
5907
5908 default:
5909 // Allow the target to implement this method for its nodes.
5910 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
5911 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)
5912 return TLI->isGuaranteedNotToBeUndefOrPoisonForTargetNode(
5913 Op, DemandedElts, DAG: *this, Kind, Depth);
5914 break;
5915 }
5916
5917 // If Op can't create undef/poison and none of its operands are undef/poison
5918 // then Op is never undef/poison.
5919 // NOTE: TargetNodes can handle this in themselves in
5920 // isGuaranteedNotToBeUndefOrPoisonForTargetNode or let
5921 // TargetLowering::isGuaranteedNotToBeUndefOrPoisonForTargetNode handle it.
5922 return !canCreateUndefOrPoison(Op, Kind, /*ConsiderFlags*/ true, Depth) &&
5923 all_of(Range: Op->ops(), P: [&](SDValue V) {
5924 return isGuaranteedNotToBeUndefOrPoison(Op: V, Kind, Depth: Depth + 1);
5925 });
5926}
5927
5928bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, UndefPoisonKind Kind,
5929 bool ConsiderFlags,
5930 unsigned Depth) const {
5931 APInt DemandedElts = getDemandAllEltsMask(V: Op);
5932 return canCreateUndefOrPoison(Op, DemandedElts, Kind, ConsiderFlags, Depth);
5933}
5934
5935bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, const APInt &DemandedElts,
5936 UndefPoisonKind Kind,
5937 bool ConsiderFlags,
5938 unsigned Depth) const {
5939 if (ConsiderFlags && includesPoison(Kind) && Op->hasPoisonGeneratingFlags())
5940 return true;
5941
5942 unsigned Opcode = Op.getOpcode();
5943 switch (Opcode) {
5944 case ISD::AssertSext:
5945 case ISD::AssertZext:
5946 case ISD::AssertAlign:
5947 case ISD::AssertNoFPClass:
5948 // Assertion nodes can create poison if the assertion fails.
5949 return includesPoison(Kind);
5950
5951 case ISD::FREEZE:
5952 case ISD::CONCAT_VECTORS:
5953 case ISD::INSERT_SUBVECTOR:
5954 case ISD::EXTRACT_SUBVECTOR:
5955 case ISD::SADDSAT:
5956 case ISD::UADDSAT:
5957 case ISD::SSUBSAT:
5958 case ISD::USUBSAT:
5959 case ISD::MULHU:
5960 case ISD::MULHS:
5961 case ISD::AVGFLOORS:
5962 case ISD::AVGFLOORU:
5963 case ISD::AVGCEILS:
5964 case ISD::AVGCEILU:
5965 case ISD::ABDU:
5966 case ISD::ABDS:
5967 case ISD::SMIN:
5968 case ISD::SMAX:
5969 case ISD::SCMP:
5970 case ISD::UMIN:
5971 case ISD::UMAX:
5972 case ISD::UCMP:
5973 case ISD::AND:
5974 case ISD::XOR:
5975 case ISD::ROTL:
5976 case ISD::ROTR:
5977 case ISD::FSHL:
5978 case ISD::FSHR:
5979 case ISD::BSWAP:
5980 case ISD::CTTZ:
5981 case ISD::CTLZ:
5982 case ISD::CTLS:
5983 case ISD::CTPOP:
5984 case ISD::BITREVERSE:
5985 case ISD::PARITY:
5986 case ISD::SIGN_EXTEND:
5987 case ISD::TRUNCATE:
5988 case ISD::SIGN_EXTEND_INREG:
5989 case ISD::SIGN_EXTEND_VECTOR_INREG:
5990 case ISD::ZERO_EXTEND_VECTOR_INREG:
5991 case ISD::BITCAST:
5992 case ISD::BUILD_VECTOR:
5993 case ISD::BUILD_PAIR:
5994 case ISD::SPLAT_VECTOR:
5995 case ISD::FABS:
5996 case ISD::FCEIL:
5997 case ISD::FFLOOR:
5998 case ISD::FTRUNC:
5999 case ISD::FRINT:
6000 case ISD::FNEARBYINT:
6001 case ISD::FROUND:
6002 case ISD::FROUNDEVEN:
6003 return false;
6004
6005 case ISD::ABS:
6006 // ISD::ABS defines abs(INT_MIN) -> INT_MIN and never generates poison.
6007 // Different to Intrinsic::abs.
6008 return false;
6009 case ISD::ABS_MIN_POISON:
6010 // ABS_MIN_POISON may produce poison if the input is INT_MIN.
6011 return ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1) <= 1;
6012
6013 case ISD::ADDC:
6014 case ISD::SUBC:
6015 case ISD::ADDE:
6016 case ISD::SUBE:
6017 case ISD::SADDO:
6018 case ISD::SSUBO:
6019 case ISD::SMULO:
6020 case ISD::SADDO_CARRY:
6021 case ISD::SSUBO_CARRY:
6022 case ISD::UADDO:
6023 case ISD::USUBO:
6024 case ISD::UMULO:
6025 case ISD::UADDO_CARRY:
6026 case ISD::USUBO_CARRY:
6027 // No poison on result or overflow flags.
6028 return false;
6029
6030 case ISD::SELECT_CC:
6031 case ISD::SETCC: {
6032 // Integer setcc cannot create undef or poison.
6033 if (Op.getOperand(i: 0).getValueType().isInteger())
6034 return false;
6035
6036 // FP compares are more complicated. They can create poison for nan/infinity
6037 // based on options and flags. The options and flags also cause special
6038 // nonan condition codes to be used. Those condition codes may be preserved
6039 // even if the nonan flag is dropped somewhere.
6040 unsigned CCOp = Opcode == ISD::SETCC ? 2 : 4;
6041 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: Op.getOperand(i: CCOp))->get();
6042 return (unsigned)CCCode & 0x10U;
6043 }
6044
6045 case ISD::OR:
6046 case ISD::ZERO_EXTEND:
6047 case ISD::SELECT:
6048 case ISD::VSELECT:
6049 case ISD::ADD:
6050 case ISD::SUB:
6051 case ISD::MUL:
6052 case ISD::FNEG:
6053 case ISD::FADD:
6054 case ISD::FSUB:
6055 case ISD::FMUL:
6056 case ISD::FDIV:
6057 case ISD::FREM:
6058 case ISD::FCOPYSIGN:
6059 case ISD::FMA:
6060 case ISD::FMAD:
6061 case ISD::FMULADD:
6062 case ISD::FP_EXTEND:
6063 case ISD::FMINNUM:
6064 case ISD::FMAXNUM:
6065 case ISD::FMINNUM_IEEE:
6066 case ISD::FMAXNUM_IEEE:
6067 case ISD::FMINIMUM:
6068 case ISD::FMAXIMUM:
6069 case ISD::FMINIMUMNUM:
6070 case ISD::FMAXIMUMNUM:
6071 case ISD::FP_TO_SINT_SAT:
6072 case ISD::FP_TO_UINT_SAT:
6073 case ISD::TRUNCATE_SSAT_S:
6074 case ISD::TRUNCATE_SSAT_U:
6075 case ISD::TRUNCATE_USAT_U:
6076 // No poison except from flags (which is handled above)
6077 return false;
6078
6079 case ISD::SHL:
6080 case ISD::SRL:
6081 case ISD::SRA:
6082 // If the max shift amount isn't in range, then the shift can
6083 // create poison.
6084 return includesPoison(Kind) &&
6085 !getValidMaximumShiftAmount(V: Op, DemandedElts, Depth: Depth + 1);
6086
6087 case ISD::CTTZ_ZERO_POISON:
6088 case ISD::CTLZ_ZERO_POISON:
6089 // If the amount is zero then the result will be poison.
6090 // TODO: Add isKnownNeverZero DemandedElts handling.
6091 return includesPoison(Kind) &&
6092 !isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6093
6094 case ISD::SCALAR_TO_VECTOR:
6095 // Check if we demand any upper (undef) elements.
6096 return includesUndef(Kind) && DemandedElts.ugt(RHS: 1);
6097
6098 case ISD::INSERT_VECTOR_ELT:
6099 case ISD::EXTRACT_VECTOR_ELT: {
6100 // Ensure that the element index is in bounds.
6101 if (includesPoison(Kind)) {
6102 EVT VecVT = Op.getOperand(i: 0).getValueType();
6103 SDValue Idx = Op.getOperand(i: Opcode == ISD::INSERT_VECTOR_ELT ? 2 : 1);
6104 KnownBits KnownIdx = computeKnownBits(Op: Idx, Depth: Depth + 1);
6105 return KnownIdx.getMaxValue().uge(RHS: VecVT.getVectorMinNumElements());
6106 }
6107 return false;
6108 }
6109
6110 case ISD::VECTOR_SHUFFLE: {
6111 // Check for any demanded shuffle element that is undef.
6112 auto *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
6113 for (auto [Idx, Elt] : enumerate(First: SVN->getMask()))
6114 if (Elt < 0 && DemandedElts[Idx])
6115 return true;
6116 return false;
6117 }
6118
6119 case ISD::VECTOR_COMPRESS:
6120 return false;
6121
6122 default:
6123 // Allow the target to implement this method for its nodes.
6124 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
6125 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)
6126 return TLI->canCreateUndefOrPoisonForTargetNode(
6127 Op, DemandedElts, DAG: *this, Kind, ConsiderFlags, Depth);
6128 break;
6129 }
6130
6131 // Be conservative and return true.
6132 return true;
6133}
6134
6135bool SelectionDAG::isADDLike(SDValue Op, bool NoWrap) const {
6136 unsigned Opcode = Op.getOpcode();
6137 if (Opcode == ISD::OR)
6138 return Op->getFlags().hasDisjoint() ||
6139 haveNoCommonBitsSet(A: Op.getOperand(i: 0), B: Op.getOperand(i: 1));
6140 if (Opcode == ISD::XOR)
6141 return !NoWrap && isMinSignedConstant(V: Op.getOperand(i: 1));
6142 return false;
6143}
6144
6145bool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const {
6146 return Op.getNumOperands() == 2 && isa<ConstantSDNode>(Val: Op.getOperand(i: 1)) &&
6147 (Op.isAnyAdd() || isADDLike(Op));
6148}
6149
6150KnownFPClass SelectionDAG::computeKnownFPClass(SDValue Op,
6151 FPClassTest InterestedClasses,
6152 unsigned Depth) const {
6153 APInt DemandedElts = getDemandAllEltsMask(V: Op);
6154 return computeKnownFPClass(Op, DemandedElts, InterestedClasses, Depth);
6155}
6156
6157KnownFPClass SelectionDAG::computeKnownFPClass(SDValue Op,
6158 const APInt &DemandedElts,
6159 FPClassTest InterestedClasses,
6160 unsigned Depth) const {
6161 KnownFPClass Known;
6162
6163 if (const auto *CFP = dyn_cast<ConstantFPSDNode>(Val&: Op))
6164 return KnownFPClass(CFP->getValueAPF());
6165
6166 if (Depth >= MaxRecursionDepth)
6167 return Known;
6168
6169 if (Op.getOpcode() == ISD::UNDEF)
6170 return Known;
6171
6172 EVT VT = Op.getValueType();
6173 assert(VT.isFloatingPoint() && "Computing KnownFPClass on non-FP op!");
6174 assert((!VT.isFixedLengthVector() ||
6175 DemandedElts.getBitWidth() == VT.getVectorNumElements()) &&
6176 "Unexpected vector size");
6177
6178 if (!DemandedElts)
6179 return Known;
6180
6181 unsigned Opcode = Op.getOpcode();
6182 switch (Opcode) {
6183 case ISD::POISON: {
6184 Known.setKnownFPClasses(fcNone);
6185 Known.setSignBit(false);
6186 break;
6187 }
6188 case ISD::FNEG: {
6189 Known = computeKnownFPClass(Op: Op.getOperand(i: 0), DemandedElts,
6190 InterestedClasses, Depth: Depth + 1);
6191 Known.fneg();
6192 break;
6193 }
6194 case ISD::BUILD_VECTOR: {
6195 assert(!VT.isScalableVector());
6196 bool First = true;
6197 for (unsigned I = 0, E = Op.getNumOperands(); I != E; ++I) {
6198 if (!DemandedElts[I])
6199 continue;
6200
6201 if (First) {
6202 Known =
6203 computeKnownFPClass(Op: Op.getOperand(i: I), InterestedClasses, Depth: Depth + 1);
6204 First = false;
6205 } else {
6206 Known |=
6207 computeKnownFPClass(Op: Op.getOperand(i: I), InterestedClasses, Depth: Depth + 1);
6208 }
6209
6210 if (Known.isUnknown())
6211 break;
6212 }
6213 break;
6214 }
6215 case ISD::EXTRACT_VECTOR_ELT: {
6216 SDValue Src = Op.getOperand(i: 0);
6217 auto *CIdx = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1));
6218 EVT SrcVT = Src.getValueType();
6219 if (SrcVT.isFixedLengthVector() && CIdx) {
6220 if (CIdx->getAPIntValue().ult(RHS: SrcVT.getVectorNumElements())) {
6221 APInt DemandedSrcElts = APInt::getOneBitSet(
6222 numBits: SrcVT.getVectorNumElements(), BitNo: CIdx->getZExtValue());
6223 Known = computeKnownFPClass(Op: Src, DemandedElts: DemandedSrcElts, InterestedClasses,
6224 Depth: Depth + 1);
6225 } else {
6226 // Out of bounds index is poison.
6227 Known.setKnownFPClasses(fcNone);
6228 }
6229 } else {
6230 Known = computeKnownFPClass(Op: Src, InterestedClasses, Depth: Depth + 1);
6231 }
6232 break;
6233 }
6234 case ISD::SPLAT_VECTOR: {
6235 Known = computeKnownFPClass(Op: Op.getOperand(i: 0), InterestedClasses, Depth: Depth + 1);
6236 break;
6237 }
6238 case ISD::BITCAST: {
6239 // FIXME: It should not be necessary to check for an elementwise bitcast.
6240 // If a bitcast is not elementwise between vector / scalar types,
6241 // computeKnownBits already splices the known bits of the source elements
6242 // appropriately so as to line up with the bits of the result's demanded
6243 // elements.
6244 EVT SrcVT = Op.getOperand(i: 0).getValueType();
6245 if (VT.isScalableVector() || SrcVT.isScalableVector())
6246 break;
6247 unsigned VTNumElts = VT.isVector() ? VT.getVectorNumElements() : 1;
6248 unsigned SrcVTNumElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
6249 if (VTNumElts != SrcVTNumElts)
6250 break;
6251
6252 KnownBits Bits = computeKnownBits(Op, DemandedElts, Depth: Depth + 1);
6253 Known = KnownFPClass::bitcast(FltSemantics: VT.getFltSemantics(), Bits);
6254 break;
6255 }
6256 case ISD::FABS: {
6257 Known = computeKnownFPClass(Op: Op.getOperand(i: 0), DemandedElts,
6258 InterestedClasses, Depth: Depth + 1);
6259 Known.fabs();
6260 break;
6261 }
6262 case ISD::FCOPYSIGN: {
6263 Known = computeKnownFPClass(Op: Op.getOperand(i: 0), DemandedElts,
6264 InterestedClasses, Depth: Depth + 1);
6265 KnownFPClass KnownSign = computeKnownFPClass(Op: Op.getOperand(i: 1), DemandedElts,
6266 InterestedClasses, Depth: Depth + 1);
6267 Known.copysign(Sign: KnownSign);
6268 break;
6269 }
6270 case ISD::AssertNoFPClass: {
6271 Known = computeKnownFPClass(Op: Op.getOperand(i: 0), DemandedElts,
6272 InterestedClasses, Depth: Depth + 1);
6273 FPClassTest AssertedClasses =
6274 static_cast<FPClassTest>(Op->getConstantOperandVal(Num: 1));
6275 Known.setKnownFPClasses(Known.getKnownFPClasses() & ~AssertedClasses);
6276 break;
6277 }
6278 case ISD::EXTRACT_SUBVECTOR: {
6279 SDValue Src = Op.getOperand(i: 0);
6280 EVT SrcVT = Src.getValueType();
6281 if (SrcVT.isFixedLengthVector()) {
6282 unsigned Idx = Op.getConstantOperandVal(i: 1);
6283 unsigned NumSrcElts = SrcVT.getVectorNumElements();
6284
6285 APInt DemandedSrcElts = DemandedElts.zextOrTrunc(width: NumSrcElts).shl(shiftAmt: Idx);
6286 Known = computeKnownFPClass(Op: Src, DemandedElts: DemandedSrcElts, InterestedClasses,
6287 Depth: Depth + 1);
6288 } else {
6289 Known = computeKnownFPClass(Op: Src, InterestedClasses, Depth: Depth + 1);
6290 }
6291 break;
6292 }
6293 case ISD::INSERT_SUBVECTOR: {
6294 SDValue BaseVector = Op.getOperand(i: 0);
6295 SDValue SubVector = Op.getOperand(i: 1);
6296 EVT BaseVT = BaseVector.getValueType();
6297 if (BaseVT.isFixedLengthVector()) {
6298 unsigned Idx = Op.getConstantOperandVal(i: 2);
6299 unsigned NumBaseElts = BaseVT.getVectorNumElements();
6300 unsigned NumSubElts = SubVector.getValueType().getVectorNumElements();
6301
6302 APInt DemandedMask =
6303 APInt::getBitsSet(numBits: NumBaseElts, loBit: Idx, hiBit: Idx + NumSubElts);
6304 APInt DemandedSrcElts = DemandedElts & ~DemandedMask;
6305 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
6306
6307 if (!DemandedSrcElts.isZero())
6308 Known = computeKnownFPClass(Op: BaseVector, DemandedElts: DemandedSrcElts,
6309 InterestedClasses, Depth: Depth + 1);
6310 if (!DemandedSubElts.isZero()) {
6311 KnownFPClass SubKnown = computeKnownFPClass(
6312 Op: SubVector, DemandedElts: DemandedSubElts, InterestedClasses, Depth: Depth + 1);
6313 Known = DemandedSrcElts.isZero() ? SubKnown : (Known | SubKnown);
6314 }
6315 } else {
6316 Known = computeKnownFPClass(Op: SubVector, InterestedClasses, Depth: Depth + 1);
6317 if (!Known.isUnknown())
6318 Known |= computeKnownFPClass(Op: BaseVector, InterestedClasses, Depth: Depth + 1);
6319 }
6320 break;
6321 }
6322 case ISD::SELECT:
6323 case ISD::VSELECT: {
6324 // TODO: Add adjustKnownFPClassForSelectArm clamp recognition as in
6325 // IR-level ValueTracking.
6326 KnownFPClass KnownFalseClass = computeKnownFPClass(
6327 Op: Op.getOperand(i: 2), DemandedElts, InterestedClasses, Depth: Depth + 1);
6328 if (KnownFalseClass.isUnknown())
6329 break;
6330 KnownFPClass KnownTrueClass = computeKnownFPClass(
6331 Op: Op.getOperand(i: 1), DemandedElts, InterestedClasses, Depth: Depth + 1);
6332 Known = KnownTrueClass.intersectWith(RHS: KnownFalseClass);
6333 break;
6334 }
6335 default:
6336 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
6337 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID) {
6338 TLI->computeKnownFPClassForTargetNode(Op, Known, DemandedElts, DAG: *this,
6339 Depth);
6340 }
6341 break;
6342 }
6343
6344 return Known;
6345}
6346
6347bool SelectionDAG::isKnownNeverNaN(SDValue Op, bool SNaN,
6348 unsigned Depth) const {
6349 APInt DemandedElts = getDemandAllEltsMask(V: Op);
6350 return isKnownNeverNaN(Op, DemandedElts, SNaN, Depth);
6351}
6352
6353bool SelectionDAG::isKnownNeverNaN(SDValue Op, const APInt &DemandedElts,
6354 bool SNaN, unsigned Depth) const {
6355 assert(!DemandedElts.isZero() && "No demanded elements");
6356
6357 // If we're told that NaNs won't happen, assume they won't.
6358 if (Op->getFlags().hasNoNaNs())
6359 return true;
6360
6361 if (Depth >= MaxRecursionDepth)
6362 return false; // Limit search depth.
6363
6364 unsigned Opcode = Op.getOpcode();
6365 switch (Opcode) {
6366 case ISD::FADD:
6367 case ISD::FSUB:
6368 case ISD::FMUL:
6369 case ISD::FDIV:
6370 case ISD::FREM:
6371 case ISD::FSIN:
6372 case ISD::FCOS:
6373 case ISD::FTAN:
6374 case ISD::FASIN:
6375 case ISD::FACOS:
6376 case ISD::FATAN:
6377 case ISD::FATAN2:
6378 case ISD::FSINH:
6379 case ISD::FCOSH:
6380 case ISD::FTANH:
6381 case ISD::FMA:
6382 case ISD::FMULADD:
6383 case ISD::FMAD: {
6384 if (SNaN)
6385 return true;
6386 // TODO: Need isKnownNeverInfinity
6387 return false;
6388 }
6389 case ISD::FCANONICALIZE:
6390 case ISD::FEXP:
6391 case ISD::FEXP2:
6392 case ISD::FEXP10:
6393 case ISD::FTRUNC:
6394 case ISD::FFLOOR:
6395 case ISD::FCEIL:
6396 case ISD::FROUND:
6397 case ISD::FROUNDEVEN:
6398 case ISD::LROUND:
6399 case ISD::LLROUND:
6400 case ISD::FRINT:
6401 case ISD::LRINT:
6402 case ISD::LLRINT:
6403 case ISD::FNEARBYINT:
6404 case ISD::FLDEXP: {
6405 if (SNaN)
6406 return true;
6407 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1);
6408 }
6409 case ISD::FABS:
6410 case ISD::FNEG:
6411 case ISD::FCOPYSIGN: {
6412 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1);
6413 }
6414 case ISD::SELECT:
6415 return isKnownNeverNaN(Op: Op.getOperand(i: 1), DemandedElts, SNaN, Depth: Depth + 1) &&
6416 isKnownNeverNaN(Op: Op.getOperand(i: 2), DemandedElts, SNaN, Depth: Depth + 1);
6417 case ISD::FP_EXTEND:
6418 case ISD::FP_ROUND: {
6419 if (SNaN)
6420 return true;
6421 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1);
6422 }
6423 case ISD::SINT_TO_FP:
6424 case ISD::UINT_TO_FP:
6425 return true;
6426 case ISD::FSQRT: // Need is known positive
6427 case ISD::FLOG:
6428 case ISD::FLOG2:
6429 case ISD::FLOG10:
6430 case ISD::FPOWI:
6431 case ISD::FPOW: {
6432 if (SNaN)
6433 return true;
6434 // TODO: Refine on operand
6435 return false;
6436 }
6437 case ISD::FMINNUM:
6438 case ISD::FMAXNUM:
6439 case ISD::FMINIMUMNUM:
6440 case ISD::FMAXIMUMNUM: {
6441 // Only one needs to be known not-nan, since it will be returned if the
6442 // other ends up being one.
6443 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1) ||
6444 isKnownNeverNaN(Op: Op.getOperand(i: 1), DemandedElts, SNaN, Depth: Depth + 1);
6445 }
6446 case ISD::FMINNUM_IEEE:
6447 case ISD::FMAXNUM_IEEE: {
6448 if (SNaN)
6449 return true;
6450 // This can return a NaN if either operand is an sNaN, or if both operands
6451 // are NaN.
6452 return (isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN: false, Depth: Depth + 1) &&
6453 isKnownNeverSNaN(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1)) ||
6454 (isKnownNeverNaN(Op: Op.getOperand(i: 1), DemandedElts, SNaN: false, Depth: Depth + 1) &&
6455 isKnownNeverSNaN(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1));
6456 }
6457 case ISD::FMINIMUM:
6458 case ISD::FMAXIMUM: {
6459 // TODO: Does this quiet or return the origina NaN as-is?
6460 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1) &&
6461 isKnownNeverNaN(Op: Op.getOperand(i: 1), DemandedElts, SNaN, Depth: Depth + 1);
6462 }
6463 case ISD::EXTRACT_VECTOR_ELT: {
6464 SDValue Src = Op.getOperand(i: 0);
6465 auto *Idx = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1));
6466 EVT SrcVT = Src.getValueType();
6467 if (SrcVT.isFixedLengthVector() && Idx &&
6468 Idx->getAPIntValue().ult(RHS: SrcVT.getVectorNumElements())) {
6469 APInt DemandedSrcElts = APInt::getOneBitSet(numBits: SrcVT.getVectorNumElements(),
6470 BitNo: Idx->getZExtValue());
6471 return isKnownNeverNaN(Op: Src, DemandedElts: DemandedSrcElts, SNaN, Depth: Depth + 1);
6472 }
6473 return isKnownNeverNaN(Op: Src, SNaN, Depth: Depth + 1);
6474 }
6475 case ISD::EXTRACT_SUBVECTOR: {
6476 SDValue Src = Op.getOperand(i: 0);
6477 if (Src.getValueType().isFixedLengthVector()) {
6478 unsigned Idx = Op.getConstantOperandVal(i: 1);
6479 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
6480 APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
6481 return isKnownNeverNaN(Op: Src, DemandedElts: DemandedSrcElts, SNaN, Depth: Depth + 1);
6482 }
6483 return isKnownNeverNaN(Op: Src, SNaN, Depth: Depth + 1);
6484 }
6485 case ISD::INSERT_SUBVECTOR: {
6486 SDValue BaseVector = Op.getOperand(i: 0);
6487 SDValue SubVector = Op.getOperand(i: 1);
6488 EVT BaseVectorVT = BaseVector.getValueType();
6489 if (BaseVectorVT.isFixedLengthVector()) {
6490 unsigned Idx = Op.getConstantOperandVal(i: 2);
6491 unsigned NumBaseElts = BaseVectorVT.getVectorNumElements();
6492 unsigned NumSubElts = SubVector.getValueType().getVectorNumElements();
6493
6494 // Clear/Extract the bits at the position where the subvector will be
6495 // inserted.
6496 APInt DemandedMask =
6497 APInt::getBitsSet(numBits: NumBaseElts, loBit: Idx, hiBit: Idx + NumSubElts);
6498 APInt DemandedSrcElts = DemandedElts & ~DemandedMask;
6499 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
6500
6501 bool NeverNaN = true;
6502 if (!DemandedSrcElts.isZero())
6503 NeverNaN &=
6504 isKnownNeverNaN(Op: BaseVector, DemandedElts: DemandedSrcElts, SNaN, Depth: Depth + 1);
6505 if (NeverNaN && !DemandedSubElts.isZero())
6506 NeverNaN &=
6507 isKnownNeverNaN(Op: SubVector, DemandedElts: DemandedSubElts, SNaN, Depth: Depth + 1);
6508 return NeverNaN;
6509 }
6510 return isKnownNeverNaN(Op: BaseVector, SNaN, Depth: Depth + 1) &&
6511 isKnownNeverNaN(Op: SubVector, SNaN, Depth: Depth + 1);
6512 }
6513 case ISD::BUILD_VECTOR: {
6514 unsigned NumElts = Op.getNumOperands();
6515 for (unsigned I = 0; I != NumElts; ++I)
6516 if (DemandedElts[I] &&
6517 !isKnownNeverNaN(Op: Op.getOperand(i: I), SNaN, Depth: Depth + 1))
6518 return false;
6519 return true;
6520 }
6521 case ISD::SPLAT_VECTOR:
6522 return isKnownNeverNaN(Op: Op.getOperand(i: 0), SNaN, Depth: Depth + 1);
6523 case ISD::AssertNoFPClass: {
6524 FPClassTest NoFPClass =
6525 static_cast<FPClassTest>(Op.getConstantOperandVal(i: 1));
6526 if ((NoFPClass & fcNan) == fcNan)
6527 return true;
6528 if (SNaN && (NoFPClass & fcSNan) == fcSNan)
6529 return true;
6530 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1);
6531 }
6532 default:
6533 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
6534 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID) {
6535 return TLI->isKnownNeverNaNForTargetNode(Op, DemandedElts, DAG: *this, SNaN,
6536 Depth);
6537 }
6538 break;
6539 }
6540
6541 FPClassTest NanMask = SNaN ? fcSNan : fcNan;
6542 KnownFPClass Known = computeKnownFPClass(Op, DemandedElts, InterestedClasses: NanMask, Depth);
6543 return Known.isKnownNever(Mask: NanMask);
6544}
6545
6546bool SelectionDAG::isKnownNeverLogicalZero(SDValue Op, unsigned Depth) const {
6547 APInt DemandedElts = getDemandAllEltsMask(V: Op);
6548 return isKnownNeverLogicalZero(Op, DemandedElts, Depth);
6549}
6550
6551bool SelectionDAG::isKnownNeverLogicalZero(SDValue Op,
6552 const APInt &DemandedElts,
6553 unsigned Depth) const {
6554 assert(!DemandedElts.isZero() && "No demanded elements");
6555 EVT VT = Op.getValueType();
6556 KnownFPClass Known =
6557 computeKnownFPClass(Op, DemandedElts, InterestedClasses: fcZero | fcSubnormal, Depth);
6558 return Known.isKnownNeverLogicalZero(Mode: getDenormalMode(VT));
6559}
6560
6561bool SelectionDAG::isKnownNeverZero(SDValue Op, unsigned Depth) const {
6562 APInt DemandedElts = getDemandAllEltsMask(V: Op);
6563 return isKnownNeverZero(Op, DemandedElts, Depth);
6564}
6565
6566bool SelectionDAG::isKnownNeverZero(SDValue Op, const APInt &DemandedElts,
6567 unsigned Depth) const {
6568 if (Depth >= MaxRecursionDepth)
6569 return false; // Limit search depth.
6570
6571 EVT OpVT = Op.getValueType();
6572 unsigned BitWidth = OpVT.getScalarSizeInBits();
6573
6574 assert(!Op.getValueType().isFloatingPoint() &&
6575 "Floating point types unsupported - use isKnownNeverLogicalZero");
6576
6577 // If the value is a constant, we can obviously see if it is a zero or not.
6578 auto IsNeverZero = [BitWidth](const ConstantSDNode *C) {
6579 APInt V = C->getAPIntValue().zextOrTrunc(width: BitWidth);
6580 return !V.isZero();
6581 };
6582
6583 if (ISD::matchUnaryPredicate(Op, DemandedElts, Match: IsNeverZero,
6584 /*AllowUndefs=*/false, /*AllowTruncation=*/true))
6585 return true;
6586
6587 // TODO: Recognize more cases here. Most of the cases are also incomplete to
6588 // some degree.
6589 switch (Op.getOpcode()) {
6590 default:
6591 break;
6592
6593 case ISD::EXTRACT_VECTOR_ELT: {
6594 SDValue InVec = Op.getOperand(i: 0);
6595 SDValue EltNo = Op.getOperand(i: 1);
6596 EVT VecVT = InVec.getValueType();
6597
6598 // Skip scalable vectors or implicit extensions.
6599 if (VecVT.isScalableVector() ||
6600 OpVT.getScalarSizeInBits() != VecVT.getScalarSizeInBits())
6601 break;
6602
6603 // If we know the element index, just demand that vector element, else for
6604 // an unknown element index, ignore DemandedElts and demand them all.
6605 const unsigned NumSrcElts = VecVT.getVectorNumElements();
6606 APInt DemandedSrcElts = APInt::getAllOnes(numBits: NumSrcElts);
6607 auto *ConstEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
6608 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(RHS: NumSrcElts))
6609 DemandedSrcElts =
6610 APInt::getOneBitSet(numBits: NumSrcElts, BitNo: ConstEltNo->getZExtValue());
6611
6612 return isKnownNeverZero(Op: InVec, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
6613 }
6614
6615 case ISD::OR:
6616 return isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1) ||
6617 isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6618
6619 case ISD::VSELECT:
6620 case ISD::SELECT:
6621 return isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1) &&
6622 isKnownNeverZero(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth + 1);
6623
6624 case ISD::SHL: {
6625 if (Op->getFlags().hasNoSignedWrap() || Op->getFlags().hasNoUnsignedWrap())
6626 return isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6627 KnownBits ValKnown =
6628 computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6629 // 1 << X is never zero.
6630 if (ValKnown.One[0])
6631 return true;
6632 // If max shift cnt of known ones is non-zero, result is non-zero.
6633 APInt MaxCnt = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1)
6634 .getMaxValue();
6635 if (MaxCnt.ult(RHS: ValKnown.getBitWidth()) &&
6636 !ValKnown.One.shl(ShiftAmt: MaxCnt).isZero())
6637 return true;
6638 break;
6639 }
6640
6641 case ISD::VECTOR_SHUFFLE: {
6642 if (Op.getValueType().isScalableVector())
6643 return false;
6644
6645 unsigned NumElts = DemandedElts.getBitWidth();
6646
6647 // All demanded elements from LHS and RHS must be known non-zero.
6648 // Demanded elements with undef shuffle mask elements are unknown.
6649
6650 APInt DemandedLHS, DemandedRHS;
6651 auto *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
6652 assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
6653 if (!getShuffleDemandedElts(SrcWidth: NumElts, Mask: SVN->getMask(), DemandedElts,
6654 DemandedLHS, DemandedRHS))
6655 return false;
6656
6657 return (!DemandedLHS ||
6658 isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts: DemandedLHS, Depth: Depth + 1)) &&
6659 (!DemandedRHS ||
6660 isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts: DemandedRHS, Depth: Depth + 1));
6661 }
6662
6663 case ISD::UADDSAT:
6664 case ISD::UMAX:
6665 return isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1) ||
6666 isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6667
6668 case ISD::UMIN:
6669 return isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1) &&
6670 isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6671
6672 // For smin/smax: If either operand is known negative/positive
6673 // respectively we don't need the other to be known at all.
6674 case ISD::SMAX: {
6675 KnownBits Op1 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
6676 if (Op1.isStrictlyPositive())
6677 return true;
6678
6679 KnownBits Op0 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6680 if (Op0.isStrictlyPositive())
6681 return true;
6682
6683 if (Op1.isNonZero() && Op0.isNonZero())
6684 return true;
6685
6686 return isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1) &&
6687 isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6688 }
6689 case ISD::SMIN: {
6690 KnownBits Op1 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
6691 if (Op1.isNegative())
6692 return true;
6693
6694 KnownBits Op0 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6695 if (Op0.isNegative())
6696 return true;
6697
6698 if (Op1.isNonZero() && Op0.isNonZero())
6699 return true;
6700
6701 return isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1) &&
6702 isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6703 }
6704
6705 case ISD::ROTL:
6706 case ISD::ROTR:
6707 case ISD::BITREVERSE:
6708 case ISD::BSWAP:
6709 case ISD::CTPOP:
6710 case ISD::ABS:
6711 case ISD::ABS_MIN_POISON:
6712 return isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6713
6714 case ISD::SRA:
6715 case ISD::SRL: {
6716 if (Op->getFlags().hasExact())
6717 return isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6718 KnownBits ValKnown =
6719 computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6720 if (ValKnown.isNegative())
6721 return true;
6722 // If max shift cnt of known ones is non-zero, result is non-zero.
6723 APInt MaxCnt = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1)
6724 .getMaxValue();
6725 if (MaxCnt.ult(RHS: ValKnown.getBitWidth()) &&
6726 !ValKnown.One.lshr(ShiftAmt: MaxCnt).isZero())
6727 return true;
6728 break;
6729 }
6730 case ISD::UDIV:
6731 case ISD::SDIV:
6732 // div exact can only produce a zero if the dividend is zero.
6733 // TODO: For udiv this is also true if Op1 u<= Op0
6734 if (Op->getFlags().hasExact())
6735 return isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6736 break;
6737
6738 case ISD::ADD:
6739 if (Op->getFlags().hasNoUnsignedWrap())
6740 if (isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1) ||
6741 isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1))
6742 return true;
6743 // TODO: There are a lot more cases we can prove for add.
6744 break;
6745
6746 case ISD::SUB: {
6747 if (isNullConstant(V: Op.getOperand(i: 0)))
6748 return isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
6749
6750 std::optional<bool> ne = KnownBits::ne(
6751 LHS: computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1),
6752 RHS: computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1));
6753 return ne && *ne;
6754 }
6755
6756 case ISD::MUL:
6757 if (Op->getFlags().hasNoSignedWrap() || Op->getFlags().hasNoUnsignedWrap())
6758 if (isKnownNeverZero(Op: Op.getOperand(i: 1), Depth: Depth + 1) &&
6759 isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1))
6760 return true;
6761 break;
6762
6763 case ISD::ZERO_EXTEND:
6764 case ISD::SIGN_EXTEND:
6765 return isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6766 case ISD::VSCALE: {
6767 const Function &F = getMachineFunction().getFunction();
6768 const APInt &Multiplier = Op.getConstantOperandAPInt(i: 0);
6769 ConstantRange CR =
6770 getVScaleRange(F: &F, BitWidth: Op.getScalarValueSizeInBits()).multiply(Other: Multiplier);
6771 if (!CR.contains(Val: APInt(CR.getBitWidth(), 0)))
6772 return true;
6773 break;
6774 }
6775 }
6776
6777 return computeKnownBits(Op, DemandedElts, Depth).isNonZero();
6778}
6779
6780bool SelectionDAG::cannotBeOrderedNegativeFP(SDValue Op) const {
6781 if (ConstantFPSDNode *C1 = isConstOrConstSplatFP(N: Op, AllowUndefs: true))
6782 return !C1->isNegative();
6783
6784 switch (Op.getOpcode()) {
6785 case ISD::FABS:
6786 case ISD::FEXP:
6787 case ISD::FEXP2:
6788 case ISD::FEXP10:
6789 return true;
6790 default:
6791 return false;
6792 }
6793
6794 llvm_unreachable("covered opcode switch");
6795}
6796
6797bool SelectionDAG::canIgnoreSignBitOfZero(const SDUse &Use) const {
6798 assert(Use.getValueType().isFloatingPoint());
6799 const SDNode *User = Use.getUser();
6800 if (User->getFlags().hasNoSignedZeros())
6801 return true;
6802
6803 unsigned OperandNo = Use.getOperandNo();
6804 // Check if this use is insensitive to the sign of zero
6805 switch (User->getOpcode()) {
6806 case ISD::SETCC:
6807 // Comparisons: IEEE-754 specifies +0.0 == -0.0.
6808 case ISD::FABS:
6809 // fabs always produces +0.0.
6810 return true;
6811 case ISD::FCOPYSIGN:
6812 // copysign overwrites the sign bit of the first operand.
6813 return OperandNo == 0;
6814 case ISD::FADD:
6815 case ISD::FSUB: {
6816 // Arithmetic with non-zero constants fixes the uncertainty around the
6817 // sign bit.
6818 SDValue Other = User->getOperand(Num: 1 - OperandNo);
6819 return isKnownNeverLogicalZero(Op: Other);
6820 }
6821 case ISD::FP_TO_SINT:
6822 case ISD::FP_TO_UINT:
6823 // fp-to-int conversions normalize signed zeros.
6824 return true;
6825 default:
6826 return false;
6827 }
6828}
6829
6830bool SelectionDAG::canIgnoreSignBitOfZero(SDValue Op) const {
6831 if (Op->getFlags().hasNoSignedZeros())
6832 return true;
6833 // FIXME: Limit the amount of checked uses to not introduce a compile-time
6834 // regression. Ideally, this should be implemented as a demanded-bits
6835 // optimization that stems from the users.
6836 if (Op->use_size() > 2)
6837 return false;
6838 return all_of(Range: Op->uses(),
6839 P: [&](const SDUse &Use) { return canIgnoreSignBitOfZero(Use); });
6840}
6841
6842bool SelectionDAG::isEqualTo(SDValue A, SDValue B) const {
6843 // Check the obvious case.
6844 if (A == B) return true;
6845
6846 // For negative and positive zero.
6847 if (const ConstantFPSDNode *CA = dyn_cast<ConstantFPSDNode>(Val&: A))
6848 if (const ConstantFPSDNode *CB = dyn_cast<ConstantFPSDNode>(Val&: B))
6849 if (CA->isZero() && CB->isZero()) return true;
6850
6851 // Otherwise they may not be equal.
6852 return false;
6853}
6854
6855// Only bits set in Mask must be negated, other bits may be arbitrary.
6856SDValue llvm::getBitwiseNotOperand(SDValue V, SDValue Mask, bool AllowUndefs) {
6857 if (isBitwiseNot(V, AllowUndefs))
6858 return V.getOperand(i: 0);
6859
6860 // Handle any_extend (not (truncate X)) pattern, where Mask only sets
6861 // bits in the non-extended part.
6862 ConstantSDNode *MaskC = isConstOrConstSplat(N: Mask);
6863 if (!MaskC || V.getOpcode() != ISD::ANY_EXTEND)
6864 return SDValue();
6865 SDValue ExtArg = V.getOperand(i: 0);
6866 if (ExtArg.getScalarValueSizeInBits() >=
6867 MaskC->getAPIntValue().getActiveBits() &&
6868 isBitwiseNot(V: ExtArg, AllowUndefs) &&
6869 ExtArg.getOperand(i: 0).getOpcode() == ISD::TRUNCATE &&
6870 ExtArg.getOperand(i: 0).getOperand(i: 0).getValueType() == V.getValueType())
6871 return ExtArg.getOperand(i: 0).getOperand(i: 0);
6872 return SDValue();
6873}
6874
6875static bool haveNoCommonBitsSetCommutative(SDValue A, SDValue B) {
6876 // Match masked merge pattern (X & ~M) op (Y & M)
6877 // Including degenerate case (X & ~M) op M
6878 auto MatchNoCommonBitsPattern = [&](SDValue Not, SDValue Mask,
6879 SDValue Other) {
6880 if (SDValue NotOperand =
6881 getBitwiseNotOperand(V: Not, Mask, /* AllowUndefs */ true)) {
6882 if (NotOperand->getOpcode() == ISD::ZERO_EXTEND ||
6883 NotOperand->getOpcode() == ISD::TRUNCATE)
6884 NotOperand = NotOperand->getOperand(Num: 0);
6885
6886 if (Other == NotOperand)
6887 return true;
6888 if (Other->getOpcode() == ISD::AND)
6889 return NotOperand == Other->getOperand(Num: 0) ||
6890 NotOperand == Other->getOperand(Num: 1);
6891 }
6892 return false;
6893 };
6894
6895 if (A->getOpcode() == ISD::ZERO_EXTEND || A->getOpcode() == ISD::TRUNCATE)
6896 A = A->getOperand(Num: 0);
6897
6898 if (B->getOpcode() == ISD::ZERO_EXTEND || B->getOpcode() == ISD::TRUNCATE)
6899 B = B->getOperand(Num: 0);
6900
6901 if (A->getOpcode() == ISD::AND)
6902 return MatchNoCommonBitsPattern(A->getOperand(Num: 0), A->getOperand(Num: 1), B) ||
6903 MatchNoCommonBitsPattern(A->getOperand(Num: 1), A->getOperand(Num: 0), B);
6904 return false;
6905}
6906
6907// FIXME: unify with llvm::haveNoCommonBitsSet.
6908bool SelectionDAG::haveNoCommonBitsSet(SDValue A, SDValue B) const {
6909 assert(A.getValueType() == B.getValueType() &&
6910 "Values must have the same type");
6911 if (haveNoCommonBitsSetCommutative(A, B) ||
6912 haveNoCommonBitsSetCommutative(A: B, B: A))
6913 return true;
6914 return KnownBits::haveNoCommonBitsSet(LHS: computeKnownBits(Op: A),
6915 RHS: computeKnownBits(Op: B));
6916}
6917
6918static SDValue FoldSTEP_VECTOR(const SDLoc &DL, EVT VT, SDValue Step,
6919 SelectionDAG &DAG) {
6920 if (cast<ConstantSDNode>(Val&: Step)->isZero())
6921 return DAG.getConstant(Val: 0, DL, VT);
6922
6923 return SDValue();
6924}
6925
6926static SDValue FoldBUILD_VECTOR(const SDLoc &DL, EVT VT,
6927 ArrayRef<SDValue> Ops,
6928 SelectionDAG &DAG) {
6929 int NumOps = Ops.size();
6930 assert(NumOps != 0 && "Can't build an empty vector!");
6931 assert(!VT.isScalableVector() &&
6932 "BUILD_VECTOR cannot be used with scalable types");
6933 assert(VT.getVectorNumElements() == (unsigned)NumOps &&
6934 "Incorrect element count in BUILD_VECTOR!");
6935
6936 // BUILD_VECTOR of UNDEFs is UNDEF.
6937 bool AllPoison = true;
6938 if (llvm::all_of(Range&: Ops, P: [&AllPoison](SDValue Op) {
6939 AllPoison &= Op.getOpcode() == ISD::POISON;
6940 return Op.isUndef();
6941 }))
6942 return AllPoison ? DAG.getPOISON(VT) : DAG.getUNDEF(VT);
6943
6944 // BUILD_VECTOR of seq extract/insert from the same vector + type is Identity.
6945 SDValue IdentitySrc;
6946 bool IsIdentity = true;
6947 for (int i = 0; i != NumOps; ++i) {
6948 if (Ops[i].getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
6949 Ops[i].getOperand(i: 0).getValueType() != VT ||
6950 (IdentitySrc && Ops[i].getOperand(i: 0) != IdentitySrc) ||
6951 !isa<ConstantSDNode>(Val: Ops[i].getOperand(i: 1)) ||
6952 Ops[i].getConstantOperandAPInt(i: 1) != i) {
6953 IsIdentity = false;
6954 break;
6955 }
6956 IdentitySrc = Ops[i].getOperand(i: 0);
6957 }
6958 if (IsIdentity)
6959 return IdentitySrc;
6960
6961 return SDValue();
6962}
6963
6964/// Try to simplify vector concatenation to an input value, undef, or build
6965/// vector.
6966static SDValue foldCONCAT_VECTORS(const SDLoc &DL, EVT VT,
6967 ArrayRef<SDValue> Ops,
6968 SelectionDAG &DAG) {
6969 assert(!Ops.empty() && "Can't concatenate an empty list of vectors!");
6970 assert(llvm::all_of(Ops,
6971 [Ops](SDValue Op) {
6972 return Ops[0].getValueType() == Op.getValueType();
6973 }) &&
6974 "Concatenation of vectors with inconsistent value types!");
6975 assert((Ops[0].getValueType().getVectorElementCount() * Ops.size()) ==
6976 VT.getVectorElementCount() &&
6977 "Incorrect element count in vector concatenation!");
6978
6979 if (Ops.size() == 1)
6980 return Ops[0];
6981
6982 // Concat of UNDEFs is UNDEF.
6983 bool AllPoison = true;
6984 if (llvm::all_of(Range&: Ops, P: [&AllPoison](SDValue Op) {
6985 AllPoison &= Op.getOpcode() == ISD::POISON;
6986 return Op.isUndef();
6987 }))
6988 return AllPoison ? DAG.getPOISON(VT) : DAG.getUNDEF(VT);
6989
6990 // Scan the operands and look for extract operations from a single source
6991 // that correspond to insertion at the same location via this concatenation:
6992 // concat (extract X, 0*subvec_elts), (extract X, 1*subvec_elts), ...
6993 SDValue IdentitySrc;
6994 bool IsIdentity = true;
6995 for (unsigned i = 0, e = Ops.size(); i != e; ++i) {
6996 SDValue Op = Ops[i];
6997 unsigned IdentityIndex = i * Op.getValueType().getVectorMinNumElements();
6998 if (Op.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
6999 Op.getOperand(i: 0).getValueType() != VT ||
7000 (IdentitySrc && Op.getOperand(i: 0) != IdentitySrc) ||
7001 Op.getConstantOperandVal(i: 1) != IdentityIndex) {
7002 IsIdentity = false;
7003 break;
7004 }
7005 assert((!IdentitySrc || IdentitySrc == Op.getOperand(0)) &&
7006 "Unexpected identity source vector for concat of extracts");
7007 IdentitySrc = Op.getOperand(i: 0);
7008 }
7009 if (IsIdentity) {
7010 assert(IdentitySrc && "Failed to set source vector of extracts");
7011 return IdentitySrc;
7012 }
7013
7014 // The code below this point is only designed to work for fixed width
7015 // vectors, so we bail out for now.
7016 if (VT.isScalableVector())
7017 return SDValue();
7018
7019 // A CONCAT_VECTOR of scalar sources, such as UNDEF, BUILD_VECTOR and
7020 // single-element INSERT_VECTOR_ELT operands can be simplified to one big
7021 // BUILD_VECTOR.
7022 // FIXME: Add support for SCALAR_TO_VECTOR as well.
7023 EVT SVT = VT.getScalarType();
7024 SmallVector<SDValue, 16> Elts;
7025 for (SDValue Op : Ops) {
7026 EVT OpVT = Op.getValueType();
7027 if (Op.getOpcode() == ISD::POISON)
7028 Elts.append(NumInputs: OpVT.getVectorNumElements(), Elt: DAG.getPOISON(VT: SVT));
7029 else if (Op.getOpcode() == ISD::UNDEF)
7030 Elts.append(NumInputs: OpVT.getVectorNumElements(), Elt: DAG.getUNDEF(VT: SVT));
7031 else if (Op.getOpcode() == ISD::BUILD_VECTOR)
7032 Elts.append(in_start: Op->op_begin(), in_end: Op->op_end());
7033 else if (Op.getOpcode() == ISD::INSERT_VECTOR_ELT &&
7034 OpVT.getVectorNumElements() == 1 &&
7035 isNullConstant(V: Op.getOperand(i: 2)))
7036 Elts.push_back(Elt: Op.getOperand(i: 1));
7037 else
7038 return SDValue();
7039 }
7040
7041 // BUILD_VECTOR requires all inputs to be of the same type, find the
7042 // maximum type and extend them all.
7043 for (SDValue Op : Elts)
7044 SVT = (SVT.bitsLT(VT: Op.getValueType()) ? Op.getValueType() : SVT);
7045
7046 if (SVT.bitsGT(VT: VT.getScalarType())) {
7047 for (SDValue &Op : Elts) {
7048 if (Op.getOpcode() == ISD::POISON)
7049 Op = DAG.getPOISON(VT: SVT);
7050 else if (Op.getOpcode() == ISD::UNDEF)
7051 Op = DAG.getUNDEF(VT: SVT);
7052 else
7053 Op = DAG.getTargetLoweringInfo().isZExtFree(FromTy: Op.getValueType(), ToTy: SVT)
7054 ? DAG.getZExtOrTrunc(Op, DL, VT: SVT)
7055 : DAG.getSExtOrTrunc(Op, DL, VT: SVT);
7056 }
7057 }
7058
7059 SDValue V = DAG.getBuildVector(VT, DL, Ops: Elts);
7060 NewSDValueDbgMsg(V, Msg: "New node fold concat vectors: ", G: &DAG);
7061 return V;
7062}
7063
7064/// Gets or creates the specified node.
7065SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT) {
7066 SDVTList VTs = getVTList(VT);
7067 SDNodeKey ID(Opcode, VTs, {});
7068 FoldingSetInsertToken InsertToken;
7069 if (SDNode *E = lookupNode(Key: ID, DL, InsertToken))
7070 return SDValue(E, 0);
7071
7072 auto *N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
7073 CSEMap.insert(N, Token: InsertToken);
7074
7075 InsertNode(N);
7076 SDValue V = SDValue(N, 0);
7077 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
7078 return V;
7079}
7080
7081SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7082 SDValue N1) {
7083 SDNodeFlags Flags;
7084 if (Inserter)
7085 Flags = Inserter->getFlags();
7086 return getNode(Opcode, DL, VT, Operand: N1, Flags);
7087}
7088
7089SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7090 SDValue N1, const SDNodeFlags Flags) {
7091 assert(N1.getOpcode() != ISD::DELETED_NODE && "Operand is DELETED_NODE!");
7092
7093 // Constant fold unary operations with a vector integer or float operand.
7094 switch (Opcode) {
7095 default:
7096 // FIXME: Entirely reasonable to perform folding of other unary
7097 // operations here as the need arises.
7098 break;
7099 case ISD::FNEG:
7100 case ISD::FABS:
7101 case ISD::FCEIL:
7102 case ISD::FTRUNC:
7103 case ISD::FFLOOR:
7104 case ISD::FP_EXTEND:
7105 case ISD::FP_TO_SINT:
7106 case ISD::FP_TO_UINT:
7107 case ISD::FP_TO_FP16:
7108 case ISD::FP_TO_BF16:
7109 case ISD::TRUNCATE:
7110 case ISD::ANY_EXTEND:
7111 case ISD::ZERO_EXTEND:
7112 case ISD::SIGN_EXTEND:
7113 case ISD::UINT_TO_FP:
7114 case ISD::SINT_TO_FP:
7115 case ISD::FP16_TO_FP:
7116 case ISD::BF16_TO_FP:
7117 case ISD::BITCAST:
7118 case ISD::ABS:
7119 case ISD::ABS_MIN_POISON:
7120 case ISD::BITREVERSE:
7121 case ISD::BSWAP:
7122 case ISD::CTLZ:
7123 case ISD::CTLZ_ZERO_POISON:
7124 case ISD::CTTZ:
7125 case ISD::CTTZ_ZERO_POISON:
7126 case ISD::CTPOP:
7127 case ISD::PARITY:
7128 case ISD::CTLS:
7129 case ISD::VECREDUCE_ADD:
7130 case ISD::VECREDUCE_SMAX:
7131 case ISD::VECREDUCE_SMIN:
7132 case ISD::VECREDUCE_UMAX:
7133 case ISD::VECREDUCE_UMIN:
7134 case ISD::VECREDUCE_MUL:
7135 case ISD::VECREDUCE_AND:
7136 case ISD::VECREDUCE_OR:
7137 case ISD::VECREDUCE_XOR:
7138 case ISD::STEP_VECTOR: {
7139 SDValue Ops = {N1};
7140 if (SDValue Fold = FoldConstantArithmetic(Opcode, DL, VT, Ops))
7141 return Fold;
7142 }
7143 }
7144
7145 unsigned OpOpcode = N1.getNode()->getOpcode();
7146 switch (Opcode) {
7147 case ISD::STEP_VECTOR:
7148 assert(VT.isScalableVector() &&
7149 "STEP_VECTOR can only be used with scalable types");
7150 assert(OpOpcode == ISD::TargetConstant &&
7151 VT.getVectorElementType() == N1.getValueType() &&
7152 "Unexpected step operand");
7153 break;
7154 case ISD::FREEZE:
7155 assert(VT == N1.getValueType() && "Unexpected VT!");
7156 if (isGuaranteedNotToBeUndefOrPoison(Op: N1, Kind: UndefPoisonKind::UndefOrPoison))
7157 return N1;
7158 break;
7159 case ISD::TokenFactor:
7160 case ISD::MERGE_VALUES:
7161 case ISD::CONCAT_VECTORS:
7162 return N1; // Factor, merge or concat of one node? No need.
7163 case ISD::BUILD_VECTOR: {
7164 // Attempt to simplify BUILD_VECTOR.
7165 SDValue Ops[] = {N1};
7166 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, DAG&: *this))
7167 return V;
7168 break;
7169 }
7170 case ISD::FP_ROUND: llvm_unreachable("Invalid method to make FP_ROUND node");
7171 case ISD::FP_EXTEND:
7172 assert(VT.isFloatingPoint() && N1.getValueType().isFloatingPoint() &&
7173 "Invalid FP cast!");
7174 if (N1.getValueType() == VT) return N1; // noop conversion.
7175 assert((!VT.isVector() || VT.getVectorElementCount() ==
7176 N1.getValueType().getVectorElementCount()) &&
7177 "Vector element count mismatch!");
7178 assert(N1.getValueType().bitsLT(VT) && "Invalid fpext node, dst < src!");
7179 if (N1.isUndef())
7180 return getUNDEF(VT);
7181 break;
7182 case ISD::FP_TO_SINT:
7183 case ISD::FP_TO_UINT:
7184 if (N1.isUndef())
7185 return getUNDEF(VT);
7186 break;
7187 case ISD::SINT_TO_FP:
7188 case ISD::UINT_TO_FP:
7189 // [us]itofp(undef) = 0, because the result value is bounded.
7190 if (N1.isUndef())
7191 return getConstantFP(Val: 0.0, DL, VT);
7192 break;
7193 case ISD::SIGN_EXTEND:
7194 assert(VT.isInteger() && N1.getValueType().isInteger() &&
7195 "Invalid SIGN_EXTEND!");
7196 assert(VT.isVector() == N1.getValueType().isVector() &&
7197 "SIGN_EXTEND result type type should be vector iff the operand "
7198 "type is vector!");
7199 if (N1.getValueType() == VT) return N1; // noop extension
7200 assert((!VT.isVector() || VT.getVectorElementCount() ==
7201 N1.getValueType().getVectorElementCount()) &&
7202 "Vector element count mismatch!");
7203 assert(N1.getValueType().bitsLT(VT) && "Invalid sext node, dst < src!");
7204 if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND) {
7205 SDNodeFlags Flags;
7206 if (OpOpcode == ISD::ZERO_EXTEND)
7207 Flags.setNonNeg(N1->getFlags().hasNonNeg());
7208 SDValue NewVal = getNode(Opcode: OpOpcode, DL, VT, N1: N1.getOperand(i: 0), Flags);
7209 transferDbgValues(From: N1, To: NewVal);
7210 return NewVal;
7211 }
7212
7213 if (OpOpcode == ISD::POISON)
7214 return getPOISON(VT);
7215
7216 if (N1.isUndef())
7217 // sext(undef) = 0, because the top bits will all be the same.
7218 return getConstant(Val: 0, DL, VT);
7219
7220 // Skip unnecessary sext_inreg pattern:
7221 // (sext (trunc x)) -> x iff the upper bits are all signbits.
7222 if (OpOpcode == ISD::TRUNCATE) {
7223 SDValue OpOp = N1.getOperand(i: 0);
7224 if (OpOp.getValueType() == VT) {
7225 unsigned NumSignExtBits =
7226 VT.getScalarSizeInBits() - N1.getScalarValueSizeInBits();
7227 if (ComputeNumSignBits(Op: OpOp) > NumSignExtBits) {
7228 transferDbgValues(From: N1, To: OpOp);
7229 return OpOp;
7230 }
7231 }
7232 }
7233 break;
7234 case ISD::ZERO_EXTEND:
7235 assert(VT.isInteger() && N1.getValueType().isInteger() &&
7236 "Invalid ZERO_EXTEND!");
7237 assert(VT.isVector() == N1.getValueType().isVector() &&
7238 "ZERO_EXTEND result type type should be vector iff the operand "
7239 "type is vector!");
7240 if (N1.getValueType() == VT) return N1; // noop extension
7241 assert((!VT.isVector() || VT.getVectorElementCount() ==
7242 N1.getValueType().getVectorElementCount()) &&
7243 "Vector element count mismatch!");
7244 assert(N1.getValueType().bitsLT(VT) && "Invalid zext node, dst < src!");
7245 if (OpOpcode == ISD::ZERO_EXTEND) { // (zext (zext x)) -> (zext x)
7246 SDNodeFlags Flags;
7247 Flags.setNonNeg(N1->getFlags().hasNonNeg());
7248 SDValue NewVal =
7249 getNode(Opcode: ISD::ZERO_EXTEND, DL, VT, N1: N1.getOperand(i: 0), Flags);
7250 transferDbgValues(From: N1, To: NewVal);
7251 return NewVal;
7252 }
7253
7254 if (OpOpcode == ISD::POISON)
7255 return getPOISON(VT);
7256
7257 if (N1.isUndef())
7258 // zext(undef) = 0, because the top bits will be zero.
7259 return getConstant(Val: 0, DL, VT);
7260
7261 // Skip unnecessary zext_inreg pattern:
7262 // (zext (trunc x)) -> x iff the upper bits are known zero.
7263 // TODO: Remove (zext (trunc (and x, c))) exception which some targets
7264 // use to recognise zext_inreg patterns.
7265 if (OpOpcode == ISD::TRUNCATE) {
7266 SDValue OpOp = N1.getOperand(i: 0);
7267 if (OpOp.getValueType() == VT) {
7268 if (OpOp.getOpcode() != ISD::AND) {
7269 APInt HiBits = APInt::getBitsSetFrom(numBits: VT.getScalarSizeInBits(),
7270 loBit: N1.getScalarValueSizeInBits());
7271 if (MaskedValueIsZero(V: OpOp, Mask: HiBits)) {
7272 transferDbgValues(From: N1, To: OpOp);
7273 return OpOp;
7274 }
7275 }
7276 }
7277 }
7278 break;
7279 case ISD::ANY_EXTEND:
7280 assert(VT.isInteger() && N1.getValueType().isInteger() &&
7281 "Invalid ANY_EXTEND!");
7282 assert(VT.isVector() == N1.getValueType().isVector() &&
7283 "ANY_EXTEND result type type should be vector iff the operand "
7284 "type is vector!");
7285 if (N1.getValueType() == VT) return N1; // noop extension
7286 assert((!VT.isVector() || VT.getVectorElementCount() ==
7287 N1.getValueType().getVectorElementCount()) &&
7288 "Vector element count mismatch!");
7289 assert(N1.getValueType().bitsLT(VT) && "Invalid anyext node, dst < src!");
7290
7291 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
7292 OpOpcode == ISD::ANY_EXTEND) {
7293 SDNodeFlags Flags;
7294 if (OpOpcode == ISD::ZERO_EXTEND)
7295 Flags.setNonNeg(N1->getFlags().hasNonNeg());
7296 // (ext (zext x)) -> (zext x) and (ext (sext x)) -> (sext x)
7297 return getNode(Opcode: OpOpcode, DL, VT, N1: N1.getOperand(i: 0), Flags);
7298 }
7299 if (N1.isUndef())
7300 return getUNDEF(VT);
7301
7302 // (ext (trunc x)) -> x
7303 if (OpOpcode == ISD::TRUNCATE) {
7304 SDValue OpOp = N1.getOperand(i: 0);
7305 if (OpOp.getValueType() == VT) {
7306 transferDbgValues(From: N1, To: OpOp);
7307 return OpOp;
7308 }
7309 }
7310 break;
7311 case ISD::TRUNCATE:
7312 assert(VT.isInteger() && N1.getValueType().isInteger() &&
7313 "Invalid TRUNCATE!");
7314 assert(VT.isVector() == N1.getValueType().isVector() &&
7315 "TRUNCATE result type type should be vector iff the operand "
7316 "type is vector!");
7317 if (N1.getValueType() == VT) return N1; // noop truncate
7318 assert((!VT.isVector() || VT.getVectorElementCount() ==
7319 N1.getValueType().getVectorElementCount()) &&
7320 "Vector element count mismatch!");
7321 assert(N1.getValueType().bitsGT(VT) && "Invalid truncate node, src < dst!");
7322 if (OpOpcode == ISD::TRUNCATE)
7323 return getNode(Opcode: ISD::TRUNCATE, DL, VT, N1: N1.getOperand(i: 0));
7324 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
7325 OpOpcode == ISD::ANY_EXTEND) {
7326 // If the source is smaller than the dest, we still need an extend.
7327 if (N1.getOperand(i: 0).getValueType().getScalarType().bitsLT(
7328 VT: VT.getScalarType())) {
7329 SDNodeFlags Flags;
7330 if (OpOpcode == ISD::ZERO_EXTEND)
7331 Flags.setNonNeg(N1->getFlags().hasNonNeg());
7332 return getNode(Opcode: OpOpcode, DL, VT, N1: N1.getOperand(i: 0), Flags);
7333 }
7334 if (N1.getOperand(i: 0).getValueType().bitsGT(VT))
7335 return getNode(Opcode: ISD::TRUNCATE, DL, VT, N1: N1.getOperand(i: 0));
7336 return N1.getOperand(i: 0);
7337 }
7338 if (N1.isUndef())
7339 return getUNDEF(VT);
7340 if (OpOpcode == ISD::VSCALE && !NewNodesMustHaveLegalTypes)
7341 return getVScale(DL, VT,
7342 MulImm: N1.getConstantOperandAPInt(i: 0).trunc(width: VT.getSizeInBits()));
7343 break;
7344 case ISD::ANY_EXTEND_VECTOR_INREG:
7345 case ISD::ZERO_EXTEND_VECTOR_INREG:
7346 case ISD::SIGN_EXTEND_VECTOR_INREG:
7347 assert(VT.isVector() && "This DAG node is restricted to vector types.");
7348 assert(N1.getValueType().bitsLE(VT) &&
7349 "The input must be the same size or smaller than the result.");
7350 assert(VT.getVectorMinNumElements() <
7351 N1.getValueType().getVectorMinNumElements() &&
7352 "The destination vector type must have fewer lanes than the input.");
7353 break;
7354 case ISD::ABS:
7355 assert(VT.isInteger() && VT == N1.getValueType() && "Invalid ABS!");
7356 if (N1.isUndef())
7357 return getConstant(Val: 0, DL, VT);
7358 break;
7359 case ISD::ABS_MIN_POISON:
7360 assert(VT.isInteger() && VT == N1.getValueType() &&
7361 "Invalid ABS_MIN_POISON!");
7362 if (N1.isUndef())
7363 return getConstant(Val: 0, DL, VT);
7364 break;
7365 case ISD::BSWAP:
7366 assert(VT.isInteger() && VT == N1.getValueType() && "Invalid BSWAP!");
7367 assert((VT.getScalarSizeInBits() % 16 == 0) &&
7368 "BSWAP types must be a multiple of 16 bits!");
7369 if (N1.isUndef())
7370 return getUNDEF(VT);
7371 // bswap(bswap(X)) -> X.
7372 if (OpOpcode == ISD::BSWAP)
7373 return N1.getOperand(i: 0);
7374 break;
7375 case ISD::BITREVERSE:
7376 assert(VT.isInteger() && VT == N1.getValueType() && "Invalid BITREVERSE!");
7377 if (N1.isUndef())
7378 return getUNDEF(VT);
7379 break;
7380 case ISD::BITCAST:
7381 assert(VT.getSizeInBits() == N1.getValueSizeInBits() &&
7382 "Cannot BITCAST between types of different sizes!");
7383 if (VT == N1.getValueType()) return N1; // noop conversion.
7384 if (OpOpcode == ISD::BITCAST) // bitconv(bitconv(x)) -> bitconv(x)
7385 return getNode(Opcode: ISD::BITCAST, DL, VT, N1: N1.getOperand(i: 0));
7386 if (N1.isUndef())
7387 return getUNDEF(VT);
7388 break;
7389 case ISD::SCALAR_TO_VECTOR:
7390 assert(VT.isVector() && !N1.getValueType().isVector() &&
7391 (VT.getVectorElementType() == N1.getValueType() ||
7392 (VT.getVectorElementType().isInteger() &&
7393 N1.getValueType().isInteger() &&
7394 VT.getVectorElementType().bitsLE(N1.getValueType()))) &&
7395 "Illegal SCALAR_TO_VECTOR node!");
7396 if (N1.isUndef())
7397 return getUNDEF(VT);
7398 // scalar_to_vector(extract_vector_elt V, 0) -> V, top bits are undefined.
7399 if (OpOpcode == ISD::EXTRACT_VECTOR_ELT &&
7400 isa<ConstantSDNode>(Val: N1.getOperand(i: 1)) &&
7401 N1.getConstantOperandVal(i: 1) == 0 &&
7402 N1.getOperand(i: 0).getValueType() == VT)
7403 return N1.getOperand(i: 0);
7404 break;
7405 case ISD::FNEG:
7406 // Negation of an unknown bag of bits is still completely undefined.
7407 if (N1.isUndef())
7408 return getUNDEF(VT);
7409
7410 if (OpOpcode == ISD::FNEG) // --X -> X
7411 return N1.getOperand(i: 0);
7412 break;
7413 case ISD::FABS:
7414 if (OpOpcode == ISD::FNEG) // abs(-X) -> abs(X)
7415 return getNode(Opcode: ISD::FABS, DL, VT, N1: N1.getOperand(i: 0));
7416 break;
7417 case ISD::VSCALE:
7418 assert(VT == N1.getValueType() && "Unexpected VT!");
7419 break;
7420 case ISD::CTPOP:
7421 if (N1.getValueType().getScalarType() == MVT::i1)
7422 return N1;
7423 break;
7424 case ISD::CTLZ:
7425 case ISD::CTTZ:
7426 if (N1.getValueType().getScalarType() == MVT::i1)
7427 return getNOT(DL, Val: N1, VT: N1.getValueType());
7428 break;
7429 case ISD::CTLS:
7430 if (N1.getValueType().getScalarType() == MVT::i1)
7431 return getConstant(Val: 0, DL, VT);
7432 break;
7433 case ISD::VECREDUCE_ADD:
7434 if (N1.getValueType().getScalarType() == MVT::i1)
7435 return getNode(Opcode: ISD::VECREDUCE_XOR, DL, VT, N1);
7436 break;
7437 case ISD::VECREDUCE_SMIN:
7438 case ISD::VECREDUCE_UMAX:
7439 if (N1.getValueType().getScalarType() == MVT::i1)
7440 return getNode(Opcode: ISD::VECREDUCE_OR, DL, VT, N1);
7441 break;
7442 case ISD::VECREDUCE_SMAX:
7443 case ISD::VECREDUCE_UMIN:
7444 if (N1.getValueType().getScalarType() == MVT::i1)
7445 return getNode(Opcode: ISD::VECREDUCE_AND, DL, VT, N1);
7446 break;
7447 case ISD::SPLAT_VECTOR:
7448 assert(VT.isVector() && "Wrong return type!");
7449 // FIXME: Hexagon uses i32 scalar for a floating point zero vector so allow
7450 // that for now.
7451 assert((VT.getVectorElementType() == N1.getValueType() ||
7452 (VT.isFloatingPoint() && N1.getValueType() == MVT::i32) ||
7453 (VT.getVectorElementType().isInteger() &&
7454 N1.getValueType().isInteger() &&
7455 VT.getVectorElementType().bitsLE(N1.getValueType()))) &&
7456 "Wrong operand type!");
7457 break;
7458 }
7459
7460 SDNode *N;
7461 SDVTList VTs = getVTList(VT);
7462 SDValue Ops[] = {N1};
7463 if (VT != MVT::Glue) { // Don't CSE glue producing nodes
7464 SDNodeKey ID(Opcode, VTs, Ops);
7465 FoldingSetInsertToken InsertToken;
7466 if (SDNode *E = lookupNode(Key: ID, DL, InsertToken)) {
7467 E->intersectFlagsWith(Flags);
7468 return SDValue(E, 0);
7469 }
7470
7471 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
7472 N->setFlags(Flags);
7473 createOperands(Node: N, Vals: Ops);
7474 CSEMap.insert(N, Token: InsertToken);
7475 } else {
7476 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
7477 createOperands(Node: N, Vals: Ops);
7478 }
7479
7480 InsertNode(N);
7481 SDValue V = SDValue(N, 0);
7482 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
7483 return V;
7484}
7485
7486static APInt getIntegerIdentity(unsigned Opcode, unsigned BitWidth) {
7487 switch (Opcode) {
7488 default:
7489 llvm_unreachable("Unexpected integer identity opcode");
7490 case ISD::ADD:
7491 case ISD::OR:
7492 case ISD::XOR:
7493 case ISD::UMAX:
7494 return APInt::getZero(numBits: BitWidth);
7495 case ISD::MUL:
7496 return APInt(BitWidth, 1);
7497 case ISD::AND:
7498 case ISD::UMIN:
7499 return APInt::getAllOnes(numBits: BitWidth);
7500 case ISD::SMAX:
7501 return APInt::getSignedMinValue(numBits: BitWidth);
7502 case ISD::SMIN:
7503 return APInt::getSignedMaxValue(numBits: BitWidth);
7504 }
7505}
7506
7507static std::optional<APInt> FoldValue(unsigned Opcode, const APInt &C1,
7508 const APInt &C2) {
7509 switch (Opcode) {
7510 case ISD::ADD: return C1 + C2;
7511 case ISD::SUB: return C1 - C2;
7512 case ISD::MUL: return C1 * C2;
7513 case ISD::AND: return C1 & C2;
7514 case ISD::OR: return C1 | C2;
7515 case ISD::XOR: return C1 ^ C2;
7516 case ISD::SHL: return C1 << C2;
7517 case ISD::SRL: return C1.lshr(ShiftAmt: C2);
7518 case ISD::SRA: return C1.ashr(ShiftAmt: C2);
7519 case ISD::ROTL: return C1.rotl(rotateAmt: C2);
7520 case ISD::ROTR: return C1.rotr(rotateAmt: C2);
7521 case ISD::SMIN: return C1.sle(RHS: C2) ? C1 : C2;
7522 case ISD::SMAX: return C1.sge(RHS: C2) ? C1 : C2;
7523 case ISD::UMIN: return C1.ule(RHS: C2) ? C1 : C2;
7524 case ISD::UMAX: return C1.uge(RHS: C2) ? C1 : C2;
7525 case ISD::SADDSAT: return C1.sadd_sat(RHS: C2);
7526 case ISD::UADDSAT: return C1.uadd_sat(RHS: C2);
7527 case ISD::SSUBSAT: return C1.ssub_sat(RHS: C2);
7528 case ISD::USUBSAT: return C1.usub_sat(RHS: C2);
7529 case ISD::SSHLSAT: return C1.sshl_sat(RHS: C2);
7530 case ISD::USHLSAT: return C1.ushl_sat(RHS: C2);
7531 case ISD::UDIV:
7532 if (!C2.getBoolValue())
7533 break;
7534 return C1.udiv(RHS: C2);
7535 case ISD::UREM:
7536 if (!C2.getBoolValue())
7537 break;
7538 return C1.urem(RHS: C2);
7539 case ISD::SDIV:
7540 if (!C2.getBoolValue())
7541 break;
7542 return C1.sdiv(RHS: C2);
7543 case ISD::SREM:
7544 if (!C2.getBoolValue())
7545 break;
7546 return C1.srem(RHS: C2);
7547 case ISD::AVGFLOORS:
7548 return APIntOps::avgFloorS(C1, C2);
7549 case ISD::AVGFLOORU:
7550 return APIntOps::avgFloorU(C1, C2);
7551 case ISD::AVGCEILS:
7552 return APIntOps::avgCeilS(C1, C2);
7553 case ISD::AVGCEILU:
7554 return APIntOps::avgCeilU(C1, C2);
7555 case ISD::ABDS:
7556 return APIntOps::abds(A: C1, B: C2);
7557 case ISD::ABDU:
7558 return APIntOps::abdu(A: C1, B: C2);
7559 case ISD::MULHS:
7560 return APIntOps::mulhs(C1, C2);
7561 case ISD::MULHU:
7562 return APIntOps::mulhu(C1, C2);
7563 case ISD::CLMUL:
7564 return APIntOps::clmul(LHS: C1, RHS: C2);
7565 case ISD::CLMULR:
7566 return APIntOps::clmulr(LHS: C1, RHS: C2);
7567 case ISD::CLMULH:
7568 return APIntOps::clmulh(LHS: C1, RHS: C2);
7569 case ISD::PEXT:
7570 return APIntOps::pext(Val: C1, Mask: C2);
7571 case ISD::PDEP:
7572 return APIntOps::pdep(Val: C1, Mask: C2);
7573 }
7574 return std::nullopt;
7575}
7576// Handle constant folding with UNDEF.
7577// TODO: Handle more cases.
7578static std::optional<APInt> FoldValueWithUndef(unsigned Opcode, const APInt &C1,
7579 bool IsUndef1, const APInt &C2,
7580 bool IsUndef2) {
7581 if (!(IsUndef1 || IsUndef2))
7582 return FoldValue(Opcode, C1, C2);
7583
7584 // Fold and(x, undef) -> 0
7585 // Fold mul(x, undef) -> 0
7586 if (Opcode == ISD::AND || Opcode == ISD::MUL)
7587 return APInt::getZero(numBits: C1.getBitWidth());
7588
7589 return std::nullopt;
7590}
7591
7592SDValue SelectionDAG::FoldSymbolOffset(unsigned Opcode, EVT VT,
7593 const GlobalAddressSDNode *GA,
7594 const SDNode *N2) {
7595 if (GA->getOpcode() != ISD::GlobalAddress)
7596 return SDValue();
7597 if (!TLI->isOffsetFoldingLegal(GA))
7598 return SDValue();
7599 auto *C2 = dyn_cast<ConstantSDNode>(Val: N2);
7600 if (!C2)
7601 return SDValue();
7602 int64_t Offset = C2->getSExtValue();
7603 switch (Opcode) {
7604 case ISD::ADD:
7605 case ISD::PTRADD:
7606 break;
7607 case ISD::SUB: Offset = -uint64_t(Offset); break;
7608 default: return SDValue();
7609 }
7610 return getGlobalAddress(GV: GA->getGlobal(), DL: SDLoc(C2), VT,
7611 Offset: GA->getOffset() + uint64_t(Offset));
7612}
7613
7614bool SelectionDAG::isUndef(unsigned Opcode, ArrayRef<SDValue> Ops) {
7615 switch (Opcode) {
7616 case ISD::SDIV:
7617 case ISD::UDIV:
7618 case ISD::SREM:
7619 case ISD::UREM: {
7620 // If a divisor is zero/undef or any element of a divisor vector is
7621 // zero/undef, the whole op is undef.
7622 assert(Ops.size() == 2 && "Div/rem should have 2 operands");
7623 SDValue Divisor = Ops[1];
7624 if (Divisor.isUndef() || isNullConstant(V: Divisor))
7625 return true;
7626
7627 return ISD::isBuildVectorOfConstantSDNodes(N: Divisor.getNode()) &&
7628 llvm::any_of(Range: Divisor->op_values(),
7629 P: [](SDValue V) { return V.isUndef() ||
7630 isNullConstant(V); });
7631 // TODO: Handle signed overflow.
7632 }
7633 // TODO: Handle oversized shifts.
7634 default:
7635 return false;
7636 }
7637}
7638
7639SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL,
7640 EVT VT, ArrayRef<SDValue> Ops,
7641 SDNodeFlags Flags) {
7642 // If the opcode is a target-specific ISD node, there's nothing we can
7643 // do here and the operand rules may not line up with the below, so
7644 // bail early.
7645 // We can't create a scalar CONCAT_VECTORS so skip it. It will break
7646 // for concats involving SPLAT_VECTOR. Concats of BUILD_VECTORS are handled by
7647 // foldCONCAT_VECTORS in getNode before this is called.
7648 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::CONCAT_VECTORS)
7649 return SDValue();
7650
7651 unsigned NumOps = Ops.size();
7652 if (NumOps == 0)
7653 return SDValue();
7654
7655 if (isUndef(Opcode, Ops))
7656 return getUNDEF(VT);
7657
7658 // Handle unary special cases.
7659 if (NumOps == 1) {
7660 SDValue N1 = Ops[0];
7661
7662 // Constant fold unary operations with an integer constant operand. Even
7663 // opaque constant will be folded, because the folding of unary operations
7664 // doesn't create new constants with different values. Nevertheless, the
7665 // opaque flag is preserved during folding to prevent future folding with
7666 // other constants.
7667 if (auto *C = dyn_cast<ConstantSDNode>(Val&: N1)) {
7668 const APInt &Val = C->getAPIntValue();
7669 switch (Opcode) {
7670 case ISD::SIGN_EXTEND:
7671 return getConstant(Val: Val.sextOrTrunc(width: VT.getSizeInBits()), DL, VT,
7672 isT: C->isTargetOpcode(), isO: C->isOpaque());
7673 case ISD::TRUNCATE:
7674 if (C->isOpaque())
7675 break;
7676 [[fallthrough]];
7677 case ISD::ZERO_EXTEND:
7678 return getConstant(Val: Val.zextOrTrunc(width: VT.getSizeInBits()), DL, VT,
7679 isT: C->isTargetOpcode(), isO: C->isOpaque());
7680 case ISD::ANY_EXTEND:
7681 // Some targets like RISCV prefer to sign extend some types.
7682 if (TLI->isSExtCheaperThanZExt(FromTy: N1.getValueType(), ToTy: VT))
7683 return getConstant(Val: Val.sextOrTrunc(width: VT.getSizeInBits()), DL, VT,
7684 isT: C->isTargetOpcode(), isO: C->isOpaque());
7685 return getConstant(Val: Val.zextOrTrunc(width: VT.getSizeInBits()), DL, VT,
7686 isT: C->isTargetOpcode(), isO: C->isOpaque());
7687 case ISD::ABS:
7688 return getConstant(Val: Val.abs(), DL, VT, isT: C->isTargetOpcode(),
7689 isO: C->isOpaque());
7690 case ISD::ABS_MIN_POISON:
7691 if (Val.isMinSignedValue())
7692 return getPOISON(VT);
7693 return getConstant(Val: Val.abs(), DL, VT, isT: C->isTargetOpcode(),
7694 isO: C->isOpaque());
7695 case ISD::BITREVERSE:
7696 return getConstant(Val: Val.reverseBits(), DL, VT, isT: C->isTargetOpcode(),
7697 isO: C->isOpaque());
7698 case ISD::BSWAP:
7699 return getConstant(Val: Val.byteSwap(), DL, VT, isT: C->isTargetOpcode(),
7700 isO: C->isOpaque());
7701 case ISD::CTPOP:
7702 return getConstant(Val: Val.popcount(), DL, VT, isT: C->isTargetOpcode(),
7703 isO: C->isOpaque());
7704 case ISD::PARITY:
7705 return getConstant(Val: Val.popcount() & 1, DL, VT, isT: C->isTargetOpcode(),
7706 isO: C->isOpaque());
7707 case ISD::CTLZ:
7708 case ISD::CTLZ_ZERO_POISON:
7709 return getConstant(Val: Val.countl_zero(), DL, VT, isT: C->isTargetOpcode(),
7710 isO: C->isOpaque());
7711 case ISD::CTTZ:
7712 case ISD::CTTZ_ZERO_POISON:
7713 return getConstant(Val: Val.countr_zero(), DL, VT, isT: C->isTargetOpcode(),
7714 isO: C->isOpaque());
7715 case ISD::CTLS:
7716 // CTLS returns the number of extra sign bits so subtract one.
7717 return getConstant(Val: Val.getNumSignBits() - 1, DL, VT,
7718 isT: C->isTargetOpcode(), isO: C->isOpaque());
7719 case ISD::UINT_TO_FP:
7720 case ISD::SINT_TO_FP: {
7721 APFloat FPV(VT.getFltSemantics(), APInt::getZero(numBits: VT.getSizeInBits()));
7722 (void)FPV.convertFromAPInt(Input: Val, IsSigned: Opcode == ISD::SINT_TO_FP,
7723 RM: APFloat::rmNearestTiesToEven);
7724 return getConstantFP(V: FPV, DL, VT);
7725 }
7726 case ISD::FP16_TO_FP:
7727 case ISD::BF16_TO_FP: {
7728 bool Ignored;
7729 APFloat FPV(Opcode == ISD::FP16_TO_FP ? APFloat::IEEEhalf()
7730 : APFloat::BFloat(),
7731 (Val.getBitWidth() == 16) ? Val : Val.trunc(width: 16));
7732
7733 // This can return overflow, underflow, or inexact; we don't care.
7734 // FIXME need to be more flexible about rounding mode.
7735 (void)FPV.convert(ToSemantics: VT.getFltSemantics(), RM: APFloat::rmNearestTiesToEven,
7736 losesInfo: &Ignored);
7737 return getConstantFP(V: FPV, DL, VT);
7738 }
7739 case ISD::STEP_VECTOR:
7740 if (SDValue V = FoldSTEP_VECTOR(DL, VT, Step: N1, DAG&: *this))
7741 return V;
7742 break;
7743 case ISD::BITCAST:
7744 if (VT == MVT::f16 && C->getValueType(ResNo: 0) == MVT::i16)
7745 return getConstantFP(V: APFloat(APFloat::IEEEhalf(), Val), DL, VT);
7746 if (VT == MVT::f32 && C->getValueType(ResNo: 0) == MVT::i32)
7747 return getConstantFP(V: APFloat(APFloat::IEEEsingle(), Val), DL, VT);
7748 if (VT == MVT::f64 && C->getValueType(ResNo: 0) == MVT::i64)
7749 return getConstantFP(V: APFloat(APFloat::IEEEdouble(), Val), DL, VT);
7750 if (VT == MVT::f128 && C->getValueType(ResNo: 0) == MVT::i128)
7751 return getConstantFP(V: APFloat(APFloat::IEEEquad(), Val), DL, VT);
7752 break;
7753 }
7754 }
7755
7756 // Constant fold unary operations with a floating point constant operand.
7757 if (auto *C = dyn_cast<ConstantFPSDNode>(Val&: N1)) {
7758 APFloat V = C->getValueAPF(); // make copy
7759 switch (Opcode) {
7760 case ISD::FNEG:
7761 V.changeSign();
7762 return getConstantFP(V, DL, VT);
7763 case ISD::FABS:
7764 V.clearSign();
7765 return getConstantFP(V, DL, VT);
7766 case ISD::FCEIL: {
7767 APFloat::opStatus fs = V.roundToIntegral(RM: APFloat::rmTowardPositive);
7768 if (fs == APFloat::opOK || fs == APFloat::opInexact)
7769 return getConstantFP(V, DL, VT);
7770 return SDValue();
7771 }
7772 case ISD::FTRUNC: {
7773 APFloat::opStatus fs = V.roundToIntegral(RM: APFloat::rmTowardZero);
7774 if (fs == APFloat::opOK || fs == APFloat::opInexact)
7775 return getConstantFP(V, DL, VT);
7776 return SDValue();
7777 }
7778 case ISD::FFLOOR: {
7779 APFloat::opStatus fs = V.roundToIntegral(RM: APFloat::rmTowardNegative);
7780 if (fs == APFloat::opOK || fs == APFloat::opInexact)
7781 return getConstantFP(V, DL, VT);
7782 return SDValue();
7783 }
7784 case ISD::FP_EXTEND: {
7785 bool ignored;
7786 // This can return overflow, underflow, or inexact; we don't care.
7787 // FIXME need to be more flexible about rounding mode.
7788 (void)V.convert(ToSemantics: VT.getFltSemantics(), RM: APFloat::rmNearestTiesToEven,
7789 losesInfo: &ignored);
7790 return getConstantFP(V, DL, VT);
7791 }
7792 case ISD::FP_TO_SINT:
7793 case ISD::FP_TO_UINT: {
7794 bool ignored;
7795 APSInt IntVal(VT.getSizeInBits(), Opcode == ISD::FP_TO_UINT);
7796 // FIXME need to be more flexible about rounding mode.
7797 APFloat::opStatus s =
7798 V.convertToInteger(Result&: IntVal, RM: APFloat::rmTowardZero, IsExact: &ignored);
7799 if (s == APFloat::opInvalidOp) // inexact is OK, in fact usual
7800 break;
7801 return getConstant(Val: IntVal, DL, VT);
7802 }
7803 case ISD::FP_TO_FP16:
7804 case ISD::FP_TO_BF16: {
7805 bool Ignored;
7806 // This can return overflow, underflow, or inexact; we don't care.
7807 // FIXME need to be more flexible about rounding mode.
7808 (void)V.convert(ToSemantics: Opcode == ISD::FP_TO_FP16 ? APFloat::IEEEhalf()
7809 : APFloat::BFloat(),
7810 RM: APFloat::rmNearestTiesToEven, losesInfo: &Ignored);
7811 return getConstant(Val: V.bitcastToAPInt().getZExtValue(), DL, VT);
7812 }
7813 case ISD::BITCAST:
7814 if (VT == MVT::i16 && C->getValueType(ResNo: 0) == MVT::f16)
7815 return getConstant(Val: (uint16_t)V.bitcastToAPInt().getZExtValue(), DL,
7816 VT);
7817 if (VT == MVT::i16 && C->getValueType(ResNo: 0) == MVT::bf16)
7818 return getConstant(Val: (uint16_t)V.bitcastToAPInt().getZExtValue(), DL,
7819 VT);
7820 if (VT == MVT::i32 && C->getValueType(ResNo: 0) == MVT::f32)
7821 return getConstant(Val: (uint32_t)V.bitcastToAPInt().getZExtValue(), DL,
7822 VT);
7823 if (VT == MVT::i64 && C->getValueType(ResNo: 0) == MVT::f64)
7824 return getConstant(Val: V.bitcastToAPInt().getZExtValue(), DL, VT);
7825 break;
7826 }
7827 }
7828
7829 // Early-out if we failed to constant fold a bitcast.
7830 if (Opcode == ISD::BITCAST)
7831 return SDValue();
7832
7833 // Constant fold integer vector reductions with constant BUILD_VECTORs.
7834 if ((Opcode == ISD::VECREDUCE_ADD || Opcode == ISD::VECREDUCE_SMAX ||
7835 Opcode == ISD::VECREDUCE_SMIN || Opcode == ISD::VECREDUCE_UMAX ||
7836 Opcode == ISD::VECREDUCE_UMIN || Opcode == ISD::VECREDUCE_MUL ||
7837 Opcode == ISD::VECREDUCE_OR || Opcode == ISD::VECREDUCE_XOR ||
7838 Opcode == ISD::VECREDUCE_AND) &&
7839 ISD::isBuildVectorOfConstantSDNodes(N: N1.getNode())) {
7840 unsigned EltBits = N1.getValueType().getScalarSizeInBits();
7841 unsigned BaseOpcode = ISD::getVecReduceBaseOpcode(VecReduceOpcode: Opcode);
7842 APInt Acc = getIntegerIdentity(Opcode: BaseOpcode, BitWidth: EltBits);
7843 for (SDValue Elt : N1->op_values()) {
7844 if (Elt.getOpcode() == ISD::POISON)
7845 return getPOISON(VT);
7846 if (Elt.isUndef() || cast<ConstantSDNode>(Val&: Elt)->isOpaque())
7847 return SDValue();
7848 APInt Value = cast<ConstantSDNode>(Val&: Elt)->getAPIntValue().trunc(width: EltBits);
7849 std::optional<APInt> Folded = FoldValue(Opcode: BaseOpcode, C1: Acc, C2: Value);
7850 assert(Folded &&
7851 "Expected vector reduction base opcode to be foldable");
7852 Acc = *Folded;
7853 }
7854 EVT EltVT = N1.getValueType().getScalarType();
7855 return getAnyExtOrTrunc(Op: getConstant(Val: Acc, DL, VT: EltVT), DL, VT);
7856 }
7857 }
7858
7859 // Handle binops special cases.
7860 if (NumOps == 2) {
7861 if (SDValue CFP = foldConstantFPMath(Opcode, DL, VT, Ops))
7862 return CFP;
7863
7864 if (auto *C1 = dyn_cast<ConstantSDNode>(Val: Ops[0])) {
7865 if (auto *C2 = dyn_cast<ConstantSDNode>(Val: Ops[1])) {
7866 if (C1->isOpaque() || C2->isOpaque())
7867 return SDValue();
7868
7869 std::optional<APInt> FoldAttempt =
7870 FoldValue(Opcode, C1: C1->getAPIntValue(), C2: C2->getAPIntValue());
7871 if (!FoldAttempt)
7872 return SDValue();
7873
7874 SDValue Folded = getConstant(Val: *FoldAttempt, DL, VT);
7875 assert((!Folded || !VT.isVector()) &&
7876 "Can't fold vectors ops with scalar operands");
7877 return Folded;
7878 }
7879 }
7880
7881 // fold (add Sym, c) -> Sym+c
7882 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Val: Ops[0]))
7883 return FoldSymbolOffset(Opcode, VT, GA, N2: Ops[1].getNode());
7884 if (TLI->isCommutativeBinOp(Opcode))
7885 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Val: Ops[1]))
7886 return FoldSymbolOffset(Opcode, VT, GA, N2: Ops[0].getNode());
7887
7888 // fold (sext_in_reg c1) -> c2
7889 if (Opcode == ISD::SIGN_EXTEND_INREG) {
7890 EVT EVT = cast<VTSDNode>(Val: Ops[1])->getVT();
7891
7892 auto SignExtendInReg = [&](APInt Val, llvm::EVT ConstantVT) {
7893 unsigned FromBits = EVT.getScalarSizeInBits();
7894 Val <<= Val.getBitWidth() - FromBits;
7895 Val.ashrInPlace(ShiftAmt: Val.getBitWidth() - FromBits);
7896 return getConstant(Val, DL, VT: ConstantVT);
7897 };
7898
7899 if (auto *C1 = dyn_cast<ConstantSDNode>(Val: Ops[0])) {
7900 const APInt &Val = C1->getAPIntValue();
7901 return SignExtendInReg(Val, VT);
7902 }
7903
7904 if (ISD::isBuildVectorOfConstantSDNodes(N: Ops[0].getNode())) {
7905 SmallVector<SDValue, 8> ScalarOps;
7906 llvm::EVT OpVT = Ops[0].getOperand(i: 0).getValueType();
7907 for (int I = 0, E = VT.getVectorNumElements(); I != E; ++I) {
7908 SDValue Op = Ops[0].getOperand(i: I);
7909 if (Op.isUndef()) {
7910 ScalarOps.push_back(Elt: getUNDEF(VT: OpVT));
7911 continue;
7912 }
7913 const APInt &Val = cast<ConstantSDNode>(Val&: Op)->getAPIntValue();
7914 ScalarOps.push_back(Elt: SignExtendInReg(Val, OpVT));
7915 }
7916 return getBuildVector(VT, DL, Ops: ScalarOps);
7917 }
7918
7919 if (Ops[0].getOpcode() == ISD::SPLAT_VECTOR &&
7920 isa<ConstantSDNode>(Val: Ops[0].getOperand(i: 0)))
7921 return getNode(Opcode: ISD::SPLAT_VECTOR, DL, VT,
7922 N1: SignExtendInReg(Ops[0].getConstantOperandAPInt(i: 0),
7923 Ops[0].getOperand(i: 0).getValueType()));
7924 }
7925 }
7926
7927 // Handle fshl/fshr special cases.
7928 if (Opcode == ISD::FSHL || Opcode == ISD::FSHR) {
7929 auto *C1 = dyn_cast<ConstantSDNode>(Val: Ops[0]);
7930 auto *C2 = dyn_cast<ConstantSDNode>(Val: Ops[1]);
7931 auto *C3 = dyn_cast<ConstantSDNode>(Val: Ops[2]);
7932
7933 if (C1 && C2 && C3) {
7934 if (C1->isOpaque() || C2->isOpaque() || C3->isOpaque())
7935 return SDValue();
7936 const APInt &V1 = C1->getAPIntValue(), &V2 = C2->getAPIntValue(),
7937 &V3 = C3->getAPIntValue();
7938
7939 APInt FoldedVal = Opcode == ISD::FSHL ? APIntOps::fshl(Hi: V1, Lo: V2, Shift: V3)
7940 : APIntOps::fshr(Hi: V1, Lo: V2, Shift: V3);
7941 return getConstant(Val: FoldedVal, DL, VT);
7942 }
7943 }
7944
7945 // Handle fma/fmad special cases.
7946 if (Opcode == ISD::FMA || Opcode == ISD::FMAD || Opcode == ISD::FMULADD) {
7947 assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
7948 assert(Ops[0].getValueType() == VT && Ops[1].getValueType() == VT &&
7949 Ops[2].getValueType() == VT && "FMA types must match!");
7950 ConstantFPSDNode *C1 = dyn_cast<ConstantFPSDNode>(Val: Ops[0]);
7951 ConstantFPSDNode *C2 = dyn_cast<ConstantFPSDNode>(Val: Ops[1]);
7952 ConstantFPSDNode *C3 = dyn_cast<ConstantFPSDNode>(Val: Ops[2]);
7953 if (C1 && C2 && C3) {
7954 APFloat V1 = C1->getValueAPF();
7955 const APFloat &V2 = C2->getValueAPF();
7956 const APFloat &V3 = C3->getValueAPF();
7957 if (Opcode == ISD::FMAD || Opcode == ISD::FMULADD) {
7958 V1.multiply(RHS: V2, RM: APFloat::rmNearestTiesToEven);
7959 V1.add(RHS: V3, RM: APFloat::rmNearestTiesToEven);
7960 } else
7961 V1.fusedMultiplyAdd(Multiplicand: V2, Addend: V3, RM: APFloat::rmNearestTiesToEven);
7962 return getConstantFP(V: V1, DL, VT);
7963 }
7964 }
7965
7966 // This is for vector folding only from here on.
7967 if (!VT.isVector())
7968 return SDValue();
7969
7970 // Constant fold integer partial reductions with constant BUILD_VECTOR
7971 // operands. The reduction order is deliberately unspecified. Use the same
7972 // subvector layout as TargetLowering::expandPartialReduceMLA(), where input
7973 // lane I contributes to accumulator lane I % NumAccElts.
7974 if (Opcode == ISD::PARTIAL_REDUCE_SMLA ||
7975 Opcode == ISD::PARTIAL_REDUCE_UMLA ||
7976 Opcode == ISD::PARTIAL_REDUCE_SUMLA) {
7977 // These nodes have no scalar form, so unsupported cases must not fall
7978 // through to generic per-lane vector folding.
7979 if (!llvm::all_of(Range&: Ops, P: [](SDValue Op) {
7980 return ISD::isBuildVectorOfConstantSDNodes(N: Op.getNode());
7981 }))
7982 return SDValue();
7983
7984 unsigned AccEltBits = VT.getScalarSizeInBits();
7985 unsigned InputEltBits = Ops[1].getScalarValueSizeInBits();
7986 unsigned NumAccElts = VT.getVectorNumElements();
7987 unsigned NumInputElts = Ops[1].getValueType().getVectorNumElements();
7988 SmallVector<APInt, 8> Results(NumAccElts, APInt::getZero(numBits: AccEltBits));
7989 BitVector PoisonElts(NumAccElts);
7990
7991 for (unsigned I = 0; I != NumAccElts; ++I) {
7992 SDValue Elt = Ops[0].getOperand(i: I);
7993 if (Elt.getOpcode() == ISD::POISON) {
7994 PoisonElts.set(I);
7995 continue;
7996 }
7997 auto *C = dyn_cast<ConstantSDNode>(Val&: Elt);
7998 if (!C || C->isOpaque())
7999 return SDValue();
8000 Results[I] = C->getAPIntValue().trunc(width: AccEltBits);
8001 }
8002
8003 bool IsLHSSigned = Opcode != ISD::PARTIAL_REDUCE_UMLA;
8004 bool IsRHSSigned = Opcode == ISD::PARTIAL_REDUCE_SMLA;
8005 for (unsigned I = 0; I != NumInputElts; ++I) {
8006 const unsigned AccIdx = I % NumAccElts;
8007 SDValue LHSElt = Ops[1].getOperand(i: I);
8008 SDValue RHSElt = Ops[2].getOperand(i: I);
8009 if (LHSElt.getOpcode() == ISD::POISON ||
8010 RHSElt.getOpcode() == ISD::POISON) {
8011 PoisonElts.set(AccIdx);
8012 continue;
8013 }
8014
8015 auto *LHS = dyn_cast<ConstantSDNode>(Val&: LHSElt);
8016 auto *RHS = dyn_cast<ConstantSDNode>(Val&: RHSElt);
8017 if (!LHS || !RHS || LHS->isOpaque() || RHS->isOpaque())
8018 return SDValue();
8019
8020 APInt LHSVal = LHS->getAPIntValue().trunc(width: InputEltBits);
8021 APInt RHSVal = RHS->getAPIntValue().trunc(width: InputEltBits);
8022 LHSVal = IsLHSSigned ? LHSVal.sext(width: AccEltBits) : LHSVal.zext(width: AccEltBits);
8023 RHSVal = IsRHSSigned ? RHSVal.sext(width: AccEltBits) : RHSVal.zext(width: AccEltBits);
8024 Results[AccIdx] += LHSVal * RHSVal;
8025 }
8026
8027 // After type legalization the vector element type may not be a legal
8028 // scalar type (e.g. i16 on AArch64). Create the folded constants in the
8029 // promoted legal scalar type instead, matching the generic per-lane path
8030 // below. Bail out if legalization would narrow the type, since the lane
8031 // value would not fit.
8032 EVT AccEltVT = VT.getVectorElementType();
8033 EVT LegalSVT = AccEltVT;
8034 if (NewNodesMustHaveLegalTypes && LegalSVT.isInteger()) {
8035 LegalSVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT: LegalSVT);
8036 if (LegalSVT.bitsLT(VT: AccEltVT))
8037 return SDValue();
8038 }
8039
8040 SmallVector<SDValue, 8> ResultOps;
8041 for (unsigned I = 0; I != NumAccElts; ++I)
8042 ResultOps.push_back(
8043 Elt: PoisonElts[I] ? getPOISON(VT: LegalSVT)
8044 : getConstant(Val: Results[I].sext(width: LegalSVT.getSizeInBits()),
8045 DL, VT: LegalSVT));
8046 return getBuildVector(VT, DL, Ops: ResultOps);
8047 }
8048
8049 ElementCount NumElts = VT.getVectorElementCount();
8050
8051 // See if we can fold through any bitcasted integer ops.
8052 if (NumOps == 2 && VT.isFixedLengthVector() && VT.isInteger() &&
8053 Ops[0].getValueType() == VT && Ops[1].getValueType() == VT &&
8054 (Ops[0].getOpcode() == ISD::BITCAST ||
8055 Ops[1].getOpcode() == ISD::BITCAST)) {
8056 SDValue N1 = peekThroughBitcasts(V: Ops[0]);
8057 SDValue N2 = peekThroughBitcasts(V: Ops[1]);
8058 auto *BV1 = dyn_cast<BuildVectorSDNode>(Val&: N1);
8059 auto *BV2 = dyn_cast<BuildVectorSDNode>(Val&: N2);
8060 if (BV1 && BV2 && N1.getValueType().isInteger() &&
8061 N2.getValueType().isInteger()) {
8062 bool IsLE = getDataLayout().isLittleEndian();
8063 unsigned EltBits = VT.getScalarSizeInBits();
8064 SmallVector<APInt> RawBits1, RawBits2;
8065 BitVector UndefElts1, UndefElts2;
8066 if (BV1->getConstantRawBits(IsLittleEndian: IsLE, DstEltSizeInBits: EltBits, RawBitElements&: RawBits1, UndefElements&: UndefElts1) &&
8067 BV2->getConstantRawBits(IsLittleEndian: IsLE, DstEltSizeInBits: EltBits, RawBitElements&: RawBits2, UndefElements&: UndefElts2)) {
8068 SmallVector<APInt> RawBits;
8069 for (unsigned I = 0, E = NumElts.getFixedValue(); I != E; ++I) {
8070 std::optional<APInt> Fold = FoldValueWithUndef(
8071 Opcode, C1: RawBits1[I], IsUndef1: UndefElts1[I], C2: RawBits2[I], IsUndef2: UndefElts2[I]);
8072 if (!Fold)
8073 break;
8074 RawBits.push_back(Elt: *Fold);
8075 }
8076 if (RawBits.size() == NumElts.getFixedValue()) {
8077 // We have constant folded, but we might need to cast this again back
8078 // to the original (possibly legalized) type.
8079 EVT BVVT, BVEltVT;
8080 if (N1.getValueType() == VT) {
8081 BVVT = N1.getValueType();
8082 BVEltVT = BV1->getOperand(Num: 0).getValueType();
8083 } else {
8084 BVVT = N2.getValueType();
8085 BVEltVT = BV2->getOperand(Num: 0).getValueType();
8086 }
8087 unsigned BVEltBits = BVEltVT.getSizeInBits();
8088 SmallVector<APInt> DstBits;
8089 BitVector DstUndefs;
8090 BuildVectorSDNode::recastRawBits(IsLittleEndian: IsLE, DstEltSizeInBits: BVVT.getScalarSizeInBits(),
8091 DstBitElements&: DstBits, SrcBitElements: RawBits, DstUndefElements&: DstUndefs,
8092 SrcUndefElements: BitVector(RawBits.size(), false));
8093 SmallVector<SDValue> Ops(DstBits.size(), getUNDEF(VT: BVEltVT));
8094 for (unsigned I = 0, E = DstBits.size(); I != E; ++I) {
8095 if (DstUndefs[I])
8096 continue;
8097 Ops[I] = getConstant(Val: DstBits[I].sext(width: BVEltBits), DL, VT: BVEltVT);
8098 }
8099 return getBitcast(VT, V: getBuildVector(VT: BVVT, DL, Ops));
8100 }
8101 }
8102 }
8103 // Logic ops can be folded from raw integer bits - mainly for AVX512 masks.
8104 if (ISD::isBitwiseLogicOp(Opcode) && isa<ConstantSDNode>(Val: N1) &&
8105 isa<ConstantSDNode>(Val: N2)) {
8106 if (SDValue Res = FoldConstantArithmetic(Opcode, DL, VT: N1.getValueType(),
8107 Ops: {N1, N2}, Flags))
8108 return getBitcast(VT, V: Res);
8109 }
8110 }
8111
8112 // Fold (mul step_vector(C0), C1) to (step_vector(C0 * C1)).
8113 // (shl step_vector(C0), C1) -> (step_vector(C0 << C1))
8114 if ((Opcode == ISD::MUL || Opcode == ISD::SHL) &&
8115 Ops[0].getOpcode() == ISD::STEP_VECTOR) {
8116 APInt RHSVal;
8117 if (ISD::isConstantSplatVector(N: Ops[1].getNode(), SplatVal&: RHSVal)) {
8118 APInt NewStep = Opcode == ISD::MUL
8119 ? Ops[0].getConstantOperandAPInt(i: 0) * RHSVal
8120 : Ops[0].getConstantOperandAPInt(i: 0) << RHSVal;
8121 return getStepVector(DL, ResVT: VT, StepVal: NewStep);
8122 }
8123 }
8124
8125 auto IsScalarOrSameVectorSize = [NumElts](const SDValue &Op) {
8126 return !Op.getValueType().isVector() ||
8127 Op.getValueType().getVectorElementCount() == NumElts;
8128 };
8129
8130 auto IsBuildVectorSplatVectorOrUndef = [](const SDValue &Op) {
8131 return Op.isUndef() || Op.getOpcode() == ISD::CONDCODE ||
8132 Op.getOpcode() == ISD::BUILD_VECTOR ||
8133 Op.getOpcode() == ISD::SPLAT_VECTOR;
8134 };
8135
8136 // All operands must be vector types with the same number of elements as
8137 // the result type and must be either UNDEF or a build/splat vector
8138 // or UNDEF scalars.
8139 if (!llvm::all_of(Range&: Ops, P: IsBuildVectorSplatVectorOrUndef) ||
8140 !llvm::all_of(Range&: Ops, P: IsScalarOrSameVectorSize))
8141 return SDValue();
8142
8143 // If we are comparing vectors, then the result needs to be a i1 boolean that
8144 // is then extended back to the legal result type depending on how booleans
8145 // are represented.
8146 EVT SVT = (Opcode == ISD::SETCC ? MVT::i1 : VT.getScalarType());
8147 ISD::NodeType ExtendCode =
8148 (Opcode == ISD::SETCC && SVT != VT.getScalarType())
8149 ? TargetLowering::getExtendForContent(Content: TLI->getBooleanContents(Type: VT))
8150 : ISD::SIGN_EXTEND;
8151
8152 // Find legal integer scalar type for constant promotion and
8153 // ensure that its scalar size is at least as large as source.
8154 EVT LegalSVT = VT.getScalarType();
8155 if (NewNodesMustHaveLegalTypes && LegalSVT.isInteger()) {
8156 LegalSVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT: LegalSVT);
8157 if (LegalSVT.bitsLT(VT: VT.getScalarType()))
8158 return SDValue();
8159 }
8160
8161 // For scalable vector types we know we're dealing with SPLAT_VECTORs. We
8162 // only have one operand to check. For fixed-length vector types we may have
8163 // a combination of BUILD_VECTOR and SPLAT_VECTOR.
8164 unsigned NumVectorElts = NumElts.isScalable() ? 1 : NumElts.getFixedValue();
8165
8166 // Constant fold each scalar lane separately.
8167 SmallVector<SDValue, 4> ScalarResults;
8168 for (unsigned I = 0; I != NumVectorElts; I++) {
8169 SmallVector<SDValue, 4> ScalarOps;
8170 for (SDValue Op : Ops) {
8171 EVT InSVT = Op.getValueType().getScalarType();
8172 if (Op.getOpcode() != ISD::BUILD_VECTOR &&
8173 Op.getOpcode() != ISD::SPLAT_VECTOR) {
8174 if (Op.isUndef())
8175 ScalarOps.push_back(Elt: getUNDEF(VT: InSVT));
8176 else
8177 ScalarOps.push_back(Elt: Op);
8178 continue;
8179 }
8180
8181 SDValue ScalarOp =
8182 Op.getOperand(i: Op.getOpcode() == ISD::SPLAT_VECTOR ? 0 : I);
8183 EVT ScalarVT = ScalarOp.getValueType();
8184
8185 // Build vector (integer) scalar operands may need implicit
8186 // truncation - do this before constant folding.
8187 if (ScalarVT.isInteger() && ScalarVT.bitsGT(VT: InSVT)) {
8188 // Don't create illegally-typed nodes unless they're constants or undef
8189 // - if we fail to constant fold we can't guarantee the (dead) nodes
8190 // we're creating will be cleaned up before being visited for
8191 // legalization.
8192 if (NewNodesMustHaveLegalTypes && !ScalarOp.isUndef() &&
8193 !isa<ConstantSDNode>(Val: ScalarOp) &&
8194 TLI->getTypeAction(Context&: *getContext(), VT: InSVT) !=
8195 TargetLowering::TypeLegal)
8196 return SDValue();
8197 ScalarOp = getNode(Opcode: ISD::TRUNCATE, DL, VT: InSVT, N1: ScalarOp);
8198 }
8199
8200 ScalarOps.push_back(Elt: ScalarOp);
8201 }
8202
8203 // Constant fold the scalar operands.
8204 SDValue ScalarResult = getNode(Opcode, DL, VT: SVT, Ops: ScalarOps, Flags);
8205
8206 // Scalar folding only succeeded if the result is a constant or UNDEF.
8207 if (!ScalarResult.isUndef() && ScalarResult.getOpcode() != ISD::Constant &&
8208 ScalarResult.getOpcode() != ISD::ConstantFP)
8209 return SDValue();
8210
8211 // Legalize the (integer) scalar constant if necessary. We only do
8212 // this once we know the folding succeeded, since otherwise we would
8213 // get a node with illegal type which has a user.
8214 if (LegalSVT != SVT)
8215 ScalarResult = getNode(Opcode: ExtendCode, DL, VT: LegalSVT, N1: ScalarResult);
8216
8217 ScalarResults.push_back(Elt: ScalarResult);
8218 }
8219
8220 SDValue V = NumElts.isScalable() ? getSplatVector(VT, DL, Op: ScalarResults[0])
8221 : getBuildVector(VT, DL, Ops: ScalarResults);
8222 NewSDValueDbgMsg(V, Msg: "New node fold constant vector: ", G: this);
8223 return V;
8224}
8225
8226SDValue SelectionDAG::foldConstantFPMath(unsigned Opcode, const SDLoc &DL,
8227 EVT VT, ArrayRef<SDValue> Ops) {
8228 // TODO: Add support for unary/ternary fp opcodes.
8229 if (Ops.size() != 2)
8230 return SDValue();
8231
8232 // TODO: We don't do any constant folding for strict FP opcodes here, but we
8233 // should. That will require dealing with a potentially non-default
8234 // rounding mode, checking the "opStatus" return value from the APFloat
8235 // math calculations, and possibly other variations.
8236 SDValue N1 = Ops[0];
8237 SDValue N2 = Ops[1];
8238 ConstantFPSDNode *N1CFP = isConstOrConstSplatFP(N: N1, /*AllowUndefs*/ false);
8239 ConstantFPSDNode *N2CFP = isConstOrConstSplatFP(N: N2, /*AllowUndefs*/ false);
8240 if (N1CFP && N2CFP) {
8241 APFloat C1 = N1CFP->getValueAPF(); // make copy
8242 const APFloat &C2 = N2CFP->getValueAPF();
8243 switch (Opcode) {
8244 case ISD::FADD:
8245 C1.add(RHS: C2, RM: APFloat::rmNearestTiesToEven);
8246 return getConstantFP(V: C1, DL, VT);
8247 case ISD::FSUB:
8248 C1.subtract(RHS: C2, RM: APFloat::rmNearestTiesToEven);
8249 return getConstantFP(V: C1, DL, VT);
8250 case ISD::FMUL:
8251 C1.multiply(RHS: C2, RM: APFloat::rmNearestTiesToEven);
8252 return getConstantFP(V: C1, DL, VT);
8253 case ISD::FDIV:
8254 C1.divide(RHS: C2, RM: APFloat::rmNearestTiesToEven);
8255 return getConstantFP(V: C1, DL, VT);
8256 case ISD::FREM:
8257 C1.mod(RHS: C2);
8258 return getConstantFP(V: C1, DL, VT);
8259 case ISD::FCOPYSIGN:
8260 C1.copySign(RHS: C2);
8261 return getConstantFP(V: C1, DL, VT);
8262 case ISD::FMINNUM:
8263 return getConstantFP(V: minnum(A: C1, B: C2), DL, VT);
8264 case ISD::FMAXNUM:
8265 return getConstantFP(V: maxnum(A: C1, B: C2), DL, VT);
8266 case ISD::FMINIMUM:
8267 return getConstantFP(V: minimum(A: C1, B: C2), DL, VT);
8268 case ISD::FMAXIMUM:
8269 return getConstantFP(V: maximum(A: C1, B: C2), DL, VT);
8270 case ISD::FMINIMUMNUM:
8271 return getConstantFP(V: minimumnum(A: C1, B: C2), DL, VT);
8272 case ISD::FMAXIMUMNUM:
8273 return getConstantFP(V: maximumnum(A: C1, B: C2), DL, VT);
8274 default: break;
8275 }
8276 }
8277 if (N1CFP && Opcode == ISD::FP_ROUND) {
8278 APFloat C1 = N1CFP->getValueAPF(); // make copy
8279 bool Unused;
8280 // This can return overflow, underflow, or inexact; we don't care.
8281 // FIXME need to be more flexible about rounding mode.
8282 (void)C1.convert(ToSemantics: VT.getFltSemantics(), RM: APFloat::rmNearestTiesToEven,
8283 losesInfo: &Unused);
8284 return getConstantFP(V: C1, DL, VT);
8285 }
8286
8287 switch (Opcode) {
8288 case ISD::FSUB:
8289 // -0.0 - undef --> undef (consistent with "fneg undef")
8290 if (ConstantFPSDNode *N1C = isConstOrConstSplatFP(N: N1, /*AllowUndefs*/ true))
8291 if (N1C && N1C->getValueAPF().isNegZero() && N2.isUndef())
8292 return getUNDEF(VT);
8293 [[fallthrough]];
8294
8295 case ISD::FADD:
8296 case ISD::FMUL:
8297 case ISD::FDIV:
8298 case ISD::FREM:
8299 // If both operands are undef, the result is undef. If 1 operand is undef,
8300 // the result is NaN. This should match the behavior of the IR optimizer.
8301 if (N1.isUndef() && N2.isUndef())
8302 return getUNDEF(VT);
8303 if (N1.isUndef() || N2.isUndef())
8304 return getConstantFP(V: APFloat::getNaN(Sem: VT.getFltSemantics()), DL, VT);
8305 }
8306 return SDValue();
8307}
8308
8309SDValue SelectionDAG::FoldConstantBuildVector(BuildVectorSDNode *BV,
8310 const SDLoc &DL, EVT DstEltVT) {
8311 EVT SrcEltVT = BV->getValueType(ResNo: 0).getVectorElementType();
8312
8313 // If this is already the right type, we're done.
8314 if (SrcEltVT == DstEltVT)
8315 return SDValue(BV, 0);
8316
8317 unsigned SrcBitSize = SrcEltVT.getSizeInBits();
8318 unsigned DstBitSize = DstEltVT.getSizeInBits();
8319
8320 // If this is a conversion of N elements of one type to N elements of another
8321 // type, convert each element. This handles FP<->INT cases.
8322 if (SrcBitSize == DstBitSize) {
8323 SmallVector<SDValue, 8> Ops;
8324 for (SDValue Op : BV->op_values()) {
8325 // If the vector element type is not legal, the BUILD_VECTOR operands
8326 // are promoted and implicitly truncated. Make that explicit here.
8327 if (Op.getValueType() != SrcEltVT)
8328 Op = getNode(Opcode: ISD::TRUNCATE, DL, VT: SrcEltVT, N1: Op);
8329 Ops.push_back(Elt: getBitcast(VT: DstEltVT, V: Op));
8330 }
8331 EVT VT = EVT::getVectorVT(Context&: *getContext(), VT: DstEltVT,
8332 NumElements: BV->getValueType(ResNo: 0).getVectorNumElements());
8333 return getBuildVector(VT, DL, Ops);
8334 }
8335
8336 // Otherwise, we're growing or shrinking the elements. To avoid having to
8337 // handle annoying details of growing/shrinking FP values, we convert them to
8338 // int first.
8339 if (SrcEltVT.isFloatingPoint()) {
8340 // Convert the input float vector to a int vector where the elements are the
8341 // same sizes.
8342 EVT IntEltVT = EVT::getIntegerVT(Context&: *getContext(), BitWidth: SrcEltVT.getSizeInBits());
8343 if (SDValue Tmp = FoldConstantBuildVector(BV, DL, DstEltVT: IntEltVT))
8344 return FoldConstantBuildVector(BV: cast<BuildVectorSDNode>(Val&: Tmp), DL,
8345 DstEltVT);
8346 return SDValue();
8347 }
8348
8349 // Now we know the input is an integer vector. If the output is a FP type,
8350 // convert to integer first, then to FP of the right size.
8351 if (DstEltVT.isFloatingPoint()) {
8352 EVT IntEltVT = EVT::getIntegerVT(Context&: *getContext(), BitWidth: DstEltVT.getSizeInBits());
8353 if (SDValue Tmp = FoldConstantBuildVector(BV, DL, DstEltVT: IntEltVT))
8354 return FoldConstantBuildVector(BV: cast<BuildVectorSDNode>(Val&: Tmp), DL,
8355 DstEltVT);
8356 return SDValue();
8357 }
8358
8359 // Okay, we know the src/dst types are both integers of differing types.
8360 assert(SrcEltVT.isInteger() && DstEltVT.isInteger());
8361
8362 // Extract the constant raw bit data.
8363 BitVector UndefElements;
8364 SmallVector<APInt> RawBits;
8365 bool IsLE = getDataLayout().isLittleEndian();
8366 if (!BV->getConstantRawBits(IsLittleEndian: IsLE, DstEltSizeInBits: DstBitSize, RawBitElements&: RawBits, UndefElements))
8367 return SDValue();
8368
8369 SmallVector<SDValue, 8> Ops;
8370 for (unsigned I = 0, E = RawBits.size(); I != E; ++I) {
8371 if (UndefElements[I])
8372 Ops.push_back(Elt: getUNDEF(VT: DstEltVT));
8373 else
8374 Ops.push_back(Elt: getConstant(Val: RawBits[I], DL, VT: DstEltVT));
8375 }
8376
8377 EVT VT = EVT::getVectorVT(Context&: *getContext(), VT: DstEltVT, NumElements: Ops.size());
8378 return getBuildVector(VT, DL, Ops);
8379}
8380
8381SDValue SelectionDAG::getAssertAlign(const SDLoc &DL, SDValue Val, Align A) {
8382 assert(Val.getValueType().isInteger() && "Invalid AssertAlign!");
8383
8384 // There's no need to assert on a byte-aligned pointer. All pointers are at
8385 // least byte aligned.
8386 if (A == Align(1))
8387 return Val;
8388
8389 SDVTList VTs = getVTList(VT: Val.getValueType());
8390 SDValue Ops[] = {Val};
8391 SDNodeKey ID(ISD::AssertAlign, VTs, Ops);
8392 ID.AddInteger(I: A.value());
8393
8394 FoldingSetInsertToken InsertToken;
8395 if (SDNode *E = lookupNode(Key: ID, DL, InsertToken))
8396 return SDValue(E, 0);
8397
8398 auto *N =
8399 newSDNode<AssertAlignSDNode>(Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs, Args&: A);
8400 createOperands(Node: N, Vals: {Val});
8401
8402 CSEMap.insert(N, Token: InsertToken);
8403 InsertNode(N);
8404
8405 SDValue V(N, 0);
8406 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
8407 return V;
8408}
8409
8410SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
8411 SDValue N1, SDValue N2) {
8412 SDNodeFlags Flags;
8413 if (Inserter)
8414 Flags = Inserter->getFlags();
8415 return getNode(Opcode, DL, VT, N1, N2, Flags);
8416}
8417
8418void SelectionDAG::canonicalizeCommutativeBinop(unsigned Opcode, SDValue &N1,
8419 SDValue &N2) const {
8420 if (!TLI->isCommutativeBinOp(Opcode))
8421 return;
8422
8423 // Canonicalize:
8424 // binop(const, nonconst) -> binop(nonconst, const)
8425 bool N1C = isConstantIntBuildVectorOrConstantInt(N: N1);
8426 bool N2C = isConstantIntBuildVectorOrConstantInt(N: N2);
8427 bool N1CFP = isConstantFPBuildVectorOrConstantFP(N: N1);
8428 bool N2CFP = isConstantFPBuildVectorOrConstantFP(N: N2);
8429 if ((N1C && !N2C) || (N1CFP && !N2CFP))
8430 std::swap(a&: N1, b&: N2);
8431
8432 // Canonicalize:
8433 // binop(splat(x), step_vector) -> binop(step_vector, splat(x))
8434 else if (N1.getOpcode() == ISD::SPLAT_VECTOR &&
8435 N2.getOpcode() == ISD::STEP_VECTOR)
8436 std::swap(a&: N1, b&: N2);
8437}
8438
8439SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
8440 SDValue N1, SDValue N2, const SDNodeFlags Flags) {
8441 assert(N1.getOpcode() != ISD::DELETED_NODE &&
8442 N2.getOpcode() != ISD::DELETED_NODE &&
8443 "Operand is DELETED_NODE!");
8444
8445 canonicalizeCommutativeBinop(Opcode, N1, N2);
8446
8447 auto *N1C = dyn_cast<ConstantSDNode>(Val&: N1);
8448 auto *N2C = dyn_cast<ConstantSDNode>(Val&: N2);
8449
8450 // Don't allow undefs in vector splats - we might be returning N2 when folding
8451 // to zero etc.
8452 ConstantSDNode *N2CV =
8453 isConstOrConstSplat(N: N2, /*AllowUndefs*/ false, /*AllowTruncation*/ true);
8454
8455 switch (Opcode) {
8456 default: break;
8457 case ISD::TokenFactor:
8458 assert(VT == MVT::Other && N1.getValueType() == MVT::Other &&
8459 N2.getValueType() == MVT::Other && "Invalid token factor!");
8460 // Fold trivial token factors.
8461 if (N1.getOpcode() == ISD::EntryToken) return N2;
8462 if (N2.getOpcode() == ISD::EntryToken) return N1;
8463 if (N1 == N2) return N1;
8464 break;
8465 case ISD::BUILD_VECTOR: {
8466 // Attempt to simplify BUILD_VECTOR.
8467 SDValue Ops[] = {N1, N2};
8468 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, DAG&: *this))
8469 return V;
8470 break;
8471 }
8472 case ISD::CONCAT_VECTORS: {
8473 SDValue Ops[] = {N1, N2};
8474 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, DAG&: *this))
8475 return V;
8476 break;
8477 }
8478 case ISD::AND:
8479 assert(VT.isInteger() && "This operator does not apply to FP types!");
8480 assert(N1.getValueType() == N2.getValueType() &&
8481 N1.getValueType() == VT && "Binary operator types must match!");
8482 // (X & 0) -> 0. This commonly occurs when legalizing i64 values, so it's
8483 // worth handling here.
8484 if (N2CV && N2CV->isZero())
8485 return N2;
8486 if (N2CV && N2CV->isAllOnes()) // X & -1 -> X
8487 return N1;
8488 break;
8489 case ISD::OR:
8490 case ISD::XOR:
8491 case ISD::ADD:
8492 case ISD::PTRADD:
8493 case ISD::SUB:
8494 assert(VT.isInteger() && "This operator does not apply to FP types!");
8495 assert(N1.getValueType() == N2.getValueType() &&
8496 N1.getValueType() == VT && "Binary operator types must match!");
8497 // The equal operand types requirement is unnecessarily strong for PTRADD.
8498 // However, the SelectionDAGBuilder does not generate PTRADDs with different
8499 // operand types, and we'd need to re-implement GEP's non-standard wrapping
8500 // logic everywhere where PTRADDs may be folded or combined to properly
8501 // support them. If/when we introduce pointer types to the SDAG, we will
8502 // need to relax this constraint.
8503
8504 // (X ^|+- 0) -> X. This commonly occurs when legalizing i64 values, so
8505 // it's worth handling here.
8506 if (N2CV && N2CV->isZero())
8507 return N1;
8508 if ((Opcode == ISD::ADD || Opcode == ISD::SUB) &&
8509 VT.getScalarType() == MVT::i1)
8510 return getNode(Opcode: ISD::XOR, DL, VT, N1, N2);
8511 // Fold (add (vscale * C0), (vscale * C1)) to (vscale * (C0 + C1)).
8512 if (Opcode == ISD::ADD && N1.getOpcode() == ISD::VSCALE &&
8513 N2.getOpcode() == ISD::VSCALE) {
8514 const APInt &C1 = N1->getConstantOperandAPInt(Num: 0);
8515 const APInt &C2 = N2->getConstantOperandAPInt(Num: 0);
8516 return getVScale(DL, VT, MulImm: C1 + C2);
8517 }
8518 break;
8519 case ISD::MUL:
8520 assert(VT.isInteger() && "This operator does not apply to FP types!");
8521 assert(N1.getValueType() == N2.getValueType() &&
8522 N1.getValueType() == VT && "Binary operator types must match!");
8523 if (VT.getScalarType() == MVT::i1)
8524 return getNode(Opcode: ISD::AND, DL, VT, N1, N2);
8525 if (N2CV && N2CV->isZero())
8526 return N2;
8527 if (N2C && (N1.getOpcode() == ISD::VSCALE) && Flags.hasNoSignedWrap()) {
8528 const APInt &MulImm = N1->getConstantOperandAPInt(Num: 0);
8529 const APInt &N2CImm = N2C->getAPIntValue();
8530 return getVScale(DL, VT, MulImm: MulImm * N2CImm);
8531 }
8532 break;
8533 case ISD::UDIV:
8534 case ISD::UREM:
8535 case ISD::MULHU:
8536 case ISD::MULHS:
8537 case ISD::SDIV:
8538 case ISD::SREM:
8539 case ISD::SADDSAT:
8540 case ISD::SSUBSAT:
8541 case ISD::UADDSAT:
8542 case ISD::USUBSAT:
8543 assert(VT.isInteger() && "This operator does not apply to FP types!");
8544 assert(N1.getValueType() == N2.getValueType() &&
8545 N1.getValueType() == VT && "Binary operator types must match!");
8546 if (VT.getScalarType() == MVT::i1) {
8547 // fold (add_sat x, y) -> (or x, y) for bool types.
8548 if (Opcode == ISD::SADDSAT || Opcode == ISD::UADDSAT)
8549 return getNode(Opcode: ISD::OR, DL, VT, N1, N2);
8550 // fold (sub_sat x, y) -> (and x, ~y) for bool types.
8551 if (Opcode == ISD::SSUBSAT || Opcode == ISD::USUBSAT)
8552 return getNode(Opcode: ISD::AND, DL, VT, N1, N2: getNOT(DL, Val: N2, VT));
8553 }
8554 break;
8555 case ISD::SCMP:
8556 case ISD::UCMP:
8557 assert(N1.getValueType() == N2.getValueType() &&
8558 "Types of operands of UCMP/SCMP must match");
8559 assert(N1.getValueType().isVector() == VT.isVector() &&
8560 "Operands and return type of must both be scalars or vectors");
8561 if (VT.isVector())
8562 assert(VT.getVectorElementCount() ==
8563 N1.getValueType().getVectorElementCount() &&
8564 "Result and operands must have the same number of elements");
8565 break;
8566 case ISD::AVGFLOORS:
8567 case ISD::AVGFLOORU:
8568 case ISD::AVGCEILS:
8569 case ISD::AVGCEILU:
8570 assert(VT.isInteger() && "This operator does not apply to FP types!");
8571 assert(N1.getValueType() == N2.getValueType() &&
8572 N1.getValueType() == VT && "Binary operator types must match!");
8573 break;
8574 case ISD::ABDS:
8575 case ISD::ABDU:
8576 assert(VT.isInteger() && "This operator does not apply to FP types!");
8577 assert(N1.getValueType() == N2.getValueType() &&
8578 N1.getValueType() == VT && "Binary operator types must match!");
8579 if (VT.getScalarType() == MVT::i1)
8580 return getNode(Opcode: ISD::XOR, DL, VT, N1, N2);
8581 break;
8582 case ISD::SMIN:
8583 case ISD::UMAX:
8584 assert(VT.isInteger() && "This operator does not apply to FP types!");
8585 assert(N1.getValueType() == N2.getValueType() &&
8586 N1.getValueType() == VT && "Binary operator types must match!");
8587 if (VT.getScalarType() == MVT::i1)
8588 return getNode(Opcode: ISD::OR, DL, VT, N1, N2);
8589 break;
8590 case ISD::SMAX:
8591 case ISD::UMIN:
8592 assert(VT.isInteger() && "This operator does not apply to FP types!");
8593 assert(N1.getValueType() == N2.getValueType() &&
8594 N1.getValueType() == VT && "Binary operator types must match!");
8595 if (VT.getScalarType() == MVT::i1)
8596 return getNode(Opcode: ISD::AND, DL, VT, N1, N2);
8597 break;
8598 case ISD::FADD:
8599 case ISD::FSUB:
8600 case ISD::FMUL:
8601 case ISD::FDIV:
8602 case ISD::FREM:
8603 assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
8604 assert(N1.getValueType() == N2.getValueType() &&
8605 N1.getValueType() == VT && "Binary operator types must match!");
8606 if (SDValue V = simplifyFPBinop(Opcode, X: N1, Y: N2, Flags))
8607 return V;
8608 break;
8609 case ISD::FCOPYSIGN: // N1 and result must match. N1/N2 need not match.
8610 assert(N1.getValueType() == VT &&
8611 N1.getValueType().isFloatingPoint() &&
8612 N2.getValueType().isFloatingPoint() &&
8613 "Invalid FCOPYSIGN!");
8614 break;
8615 case ISD::SHL:
8616 if (N2C && (N1.getOpcode() == ISD::VSCALE) && Flags.hasNoSignedWrap()) {
8617 const APInt &MulImm = N1->getConstantOperandAPInt(Num: 0);
8618 const APInt &ShiftImm = N2C->getAPIntValue();
8619 return getVScale(DL, VT, MulImm: MulImm << ShiftImm);
8620 }
8621 [[fallthrough]];
8622 case ISD::SRA:
8623 case ISD::SRL:
8624 if (SDValue V = simplifyShift(X: N1, Y: N2))
8625 return V;
8626 [[fallthrough]];
8627 case ISD::ROTL:
8628 case ISD::ROTR:
8629 case ISD::SSHLSAT:
8630 case ISD::USHLSAT:
8631 assert(VT == N1.getValueType() &&
8632 "Shift operators return type must be the same as their first arg");
8633 assert(VT.isInteger() && N2.getValueType().isInteger() &&
8634 "Shifts only work on integers");
8635 assert((!VT.isVector() || VT == N2.getValueType()) &&
8636 "Vector shift amounts must be in the same as their first arg");
8637 // Verify that the shift amount VT is big enough to hold valid shift
8638 // amounts. This catches things like trying to shift an i1024 value by an
8639 // i8, which is easy to fall into in generic code that uses
8640 // TLI.getShiftAmount().
8641 assert(N2.getValueType().getScalarSizeInBits() >=
8642 Log2_32_Ceil(VT.getScalarSizeInBits()) &&
8643 "Invalid use of small shift amount with oversized value!");
8644
8645 // Always fold shifts of i1 values so the code generator doesn't need to
8646 // handle them. Since we know the size of the shift has to be less than the
8647 // size of the value, the shift/rotate count is guaranteed to be zero.
8648 if (VT == MVT::i1)
8649 return N1;
8650 if (N2CV && N2CV->isZero())
8651 return N1;
8652 break;
8653 case ISD::FP_ROUND:
8654 assert(VT.isFloatingPoint() && N1.getValueType().isFloatingPoint() &&
8655 VT.bitsLE(N1.getValueType()) && N2C &&
8656 (N2C->getZExtValue() == 0 || N2C->getZExtValue() == 1) &&
8657 N2.getOpcode() == ISD::TargetConstant && "Invalid FP_ROUND!");
8658 if (N1.getValueType() == VT) return N1; // noop conversion.
8659 break;
8660 case ISD::IS_FPCLASS: {
8661 assert(N1.getValueType().isFloatingPoint() &&
8662 "IS_FPCLASS is used for a non-floating type");
8663 assert(isa<ConstantSDNode>(N2) && "FPClassTest is not Constant");
8664 // is.fpclass(poison, mask) -> poison
8665 if (N1.getOpcode() == ISD::POISON)
8666 return getPOISON(VT);
8667 FPClassTest Mask = static_cast<FPClassTest>(N2->getAsZExtVal());
8668 // If all tests are made, it doesn't matter what the value is.
8669 if ((Mask & fcAllFlags) == fcAllFlags)
8670 return getBoolConstant(V: true, DL, VT, OpVT: N1.getValueType());
8671 if ((Mask & fcAllFlags) == 0)
8672 return getBoolConstant(V: false, DL, VT, OpVT: N1.getValueType());
8673 break;
8674 }
8675 case ISD::AssertNoFPClass: {
8676 assert(N1.getValueType().isFloatingPoint() &&
8677 "AssertNoFPClass is used for a non-floating type");
8678 assert(isa<ConstantSDNode>(N2) && "NoFPClass is not Constant");
8679 FPClassTest NoFPClass = static_cast<FPClassTest>(N2->getAsZExtVal());
8680 assert(llvm::to_underlying(NoFPClass) <=
8681 BitmaskEnumDetail::Mask<FPClassTest>() &&
8682 "FPClassTest value too large");
8683 (void)NoFPClass;
8684 break;
8685 }
8686 case ISD::AssertSext:
8687 case ISD::AssertZext: {
8688 EVT EVT = cast<VTSDNode>(Val&: N2)->getVT();
8689 assert(VT == N1.getValueType() && "Not an inreg extend!");
8690 assert(VT.isInteger() && EVT.isInteger() &&
8691 "Cannot *_EXTEND_INREG FP types");
8692 assert(!EVT.isVector() &&
8693 "AssertSExt/AssertZExt type should be the vector element type "
8694 "rather than the vector type!");
8695 assert(EVT.bitsLE(VT.getScalarType()) && "Not extending!");
8696 if (VT.getScalarType() == EVT) return N1; // noop assertion.
8697 break;
8698 }
8699 case ISD::SIGN_EXTEND_INREG: {
8700 EVT EVT = cast<VTSDNode>(Val&: N2)->getVT();
8701 assert(VT == N1.getValueType() && "Not an inreg extend!");
8702 assert(VT.isInteger() && EVT.isInteger() &&
8703 "Cannot *_EXTEND_INREG FP types");
8704 assert(EVT.isVector() == VT.isVector() &&
8705 "SIGN_EXTEND_INREG type should be vector iff the operand "
8706 "type is vector!");
8707 assert((!EVT.isVector() ||
8708 EVT.getVectorElementCount() == VT.getVectorElementCount()) &&
8709 "Vector element counts must match in SIGN_EXTEND_INREG");
8710 assert(EVT.getScalarType().bitsLE(VT.getScalarType()) && "Not extending!");
8711 if (EVT == VT) return N1; // Not actually extending
8712 break;
8713 }
8714 case ISD::FP_TO_SINT_SAT:
8715 case ISD::FP_TO_UINT_SAT: {
8716 assert(VT.isInteger() && cast<VTSDNode>(N2)->getVT().isInteger() &&
8717 N1.getValueType().isFloatingPoint() && "Invalid FP_TO_*INT_SAT");
8718 assert(N1.getValueType().isVector() == VT.isVector() &&
8719 "FP_TO_*INT_SAT type should be vector iff the operand type is "
8720 "vector!");
8721 assert((!VT.isVector() || VT.getVectorElementCount() ==
8722 N1.getValueType().getVectorElementCount()) &&
8723 "Vector element counts must match in FP_TO_*INT_SAT");
8724 assert(!cast<VTSDNode>(N2)->getVT().isVector() &&
8725 "Type to saturate to must be a scalar.");
8726 assert(cast<VTSDNode>(N2)->getVT().bitsLE(VT.getScalarType()) &&
8727 "Not extending!");
8728 break;
8729 }
8730 case ISD::EXTRACT_VECTOR_ELT:
8731 assert(VT.getSizeInBits() >= N1.getValueType().getScalarSizeInBits() &&
8732 "The result of EXTRACT_VECTOR_ELT must be at least as wide as the \
8733 element type of the vector.");
8734
8735 // Extract from an undefined value or using an undefined index is undefined.
8736 if (N1.isUndef() || N2.isUndef())
8737 return getUNDEF(VT);
8738
8739 // EXTRACT_VECTOR_ELT of out-of-bounds element is POISON for fixed length
8740 // vectors. For scalable vectors we will provide appropriate support for
8741 // dealing with arbitrary indices.
8742 if (N2C && N1.getValueType().isFixedLengthVector() &&
8743 N2C->getAPIntValue().uge(RHS: N1.getValueType().getVectorNumElements()))
8744 return getPOISON(VT);
8745
8746 // EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is
8747 // expanding copies of large vectors from registers. This only works for
8748 // fixed length vectors, since we need to know the exact number of
8749 // elements.
8750 if (N2C && N1.getOpcode() == ISD::CONCAT_VECTORS &&
8751 N1.getOperand(i: 0).getValueType().isFixedLengthVector()) {
8752 unsigned Factor = N1.getOperand(i: 0).getValueType().getVectorNumElements();
8753 return getExtractVectorElt(DL, VT,
8754 Vec: N1.getOperand(i: N2C->getZExtValue() / Factor),
8755 Idx: N2C->getZExtValue() % Factor);
8756 }
8757
8758 // EXTRACT_VECTOR_ELT of BUILD_VECTOR or SPLAT_VECTOR is often formed while
8759 // lowering is expanding large vector constants.
8760 if (N2C && (N1.getOpcode() == ISD::BUILD_VECTOR ||
8761 N1.getOpcode() == ISD::SPLAT_VECTOR)) {
8762 assert((N1.getOpcode() != ISD::BUILD_VECTOR ||
8763 N1.getValueType().isFixedLengthVector()) &&
8764 "BUILD_VECTOR used for scalable vectors");
8765 unsigned Index =
8766 N1.getOpcode() == ISD::BUILD_VECTOR ? N2C->getZExtValue() : 0;
8767 SDValue Elt = N1.getOperand(i: Index);
8768
8769 if (VT != Elt.getValueType())
8770 // If the vector element type is not legal, the BUILD_VECTOR operands
8771 // are promoted and implicitly truncated, and the result implicitly
8772 // extended. Make that explicit here.
8773 Elt = getAnyExtOrTrunc(Op: Elt, DL, VT);
8774
8775 return Elt;
8776 }
8777
8778 // EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector
8779 // operations are lowered to scalars.
8780 if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT) {
8781 // If the indices are the same, return the inserted element else
8782 // if the indices are known different, extract the element from
8783 // the original vector.
8784 SDValue N1Op2 = N1.getOperand(i: 2);
8785 ConstantSDNode *N1Op2C = dyn_cast<ConstantSDNode>(Val&: N1Op2);
8786
8787 if (N1Op2C && N2C) {
8788 if (N1Op2C->getZExtValue() == N2C->getZExtValue()) {
8789 if (VT == N1.getOperand(i: 1).getValueType())
8790 return N1.getOperand(i: 1);
8791 if (VT.isFloatingPoint()) {
8792 assert(VT.getSizeInBits() > N1.getOperand(1).getValueType().getSizeInBits());
8793 return getFPExtendOrRound(Op: N1.getOperand(i: 1), DL, VT);
8794 }
8795 return getSExtOrTrunc(Op: N1.getOperand(i: 1), DL, VT);
8796 }
8797 return getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT, N1: N1.getOperand(i: 0), N2);
8798 }
8799 }
8800
8801 // EXTRACT_VECTOR_ELT of v1iX EXTRACT_SUBVECTOR could be formed
8802 // when vector types are scalarized and v1iX is legal.
8803 // vextract (v1iX extract_subvector(vNiX, Idx)) -> vextract(vNiX,Idx).
8804 // Here we are completely ignoring the extract element index (N2),
8805 // which is fine for fixed width vectors, since any index other than 0
8806 // is undefined anyway. However, this cannot be ignored for scalable
8807 // vectors - in theory we could support this, but we don't want to do this
8808 // without a profitability check.
8809 if (N1.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
8810 N1.getValueType().isFixedLengthVector() &&
8811 N1.getValueType().getVectorNumElements() == 1) {
8812 return getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT, N1: N1.getOperand(i: 0),
8813 N2: N1.getOperand(i: 1));
8814 }
8815 break;
8816 case ISD::EXTRACT_ELEMENT:
8817 assert(N2C && (unsigned)N2C->getZExtValue() < 2 && "Bad EXTRACT_ELEMENT!");
8818 assert(!N1.getValueType().isVector() && !VT.isVector() &&
8819 (N1.getValueType().isInteger() == VT.isInteger()) &&
8820 N1.getValueType() != VT &&
8821 "Wrong types for EXTRACT_ELEMENT!");
8822
8823 // EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding
8824 // 64-bit integers into 32-bit parts. Instead of building the extract of
8825 // the BUILD_PAIR, only to have legalize rip it apart, just do it now.
8826 if (N1.getOpcode() == ISD::BUILD_PAIR)
8827 return N1.getOperand(i: N2C->getZExtValue());
8828
8829 // EXTRACT_ELEMENT of a constant int is also very common.
8830 if (N1C) {
8831 unsigned ElementSize = VT.getSizeInBits();
8832 unsigned Shift = ElementSize * N2C->getZExtValue();
8833 const APInt &Val = N1C->getAPIntValue();
8834 return getConstant(Val: Val.extractBits(numBits: ElementSize, bitPosition: Shift), DL, VT);
8835 }
8836 break;
8837 case ISD::EXTRACT_SUBVECTOR: {
8838 EVT N1VT = N1.getValueType();
8839 assert(VT.isVector() && N1VT.isVector() &&
8840 "Extract subvector VTs must be vectors!");
8841 assert(VT.getVectorElementType() == N1VT.getVectorElementType() &&
8842 "Extract subvector VTs must have the same element type!");
8843 assert((VT.isFixedLengthVector() || N1VT.isScalableVector()) &&
8844 "Cannot extract a scalable vector from a fixed length vector!");
8845 assert((VT.isScalableVector() != N1VT.isScalableVector() ||
8846 VT.getVectorMinNumElements() <= N1VT.getVectorMinNumElements()) &&
8847 "Extract subvector must be from larger vector to smaller vector!");
8848 assert(N2C && "Extract subvector index must be a constant");
8849 assert((VT.isScalableVector() != N1VT.isScalableVector() ||
8850 (VT.getVectorMinNumElements() + N2C->getZExtValue()) <=
8851 N1VT.getVectorMinNumElements()) &&
8852 "Extract subvector overflow!");
8853 assert(N2C->getAPIntValue().getBitWidth() ==
8854 TLI->getVectorIdxWidth(getDataLayout()) &&
8855 "Constant index for EXTRACT_SUBVECTOR has an invalid size");
8856 assert(N2C->getZExtValue() % VT.getVectorMinNumElements() == 0 &&
8857 "Extract index is not a multiple of the output vector length");
8858
8859 // Trivial extraction.
8860 if (VT == N1VT)
8861 return N1;
8862
8863 // EXTRACT_SUBVECTOR of an UNDEF is an UNDEF.
8864 if (N1.isUndef())
8865 return getUNDEF(VT);
8866
8867 // EXTRACT_SUBVECTOR of CONCAT_VECTOR can be simplified if the pieces of
8868 // the concat have the same type as the extract.
8869 if (N1.getOpcode() == ISD::CONCAT_VECTORS &&
8870 VT == N1.getOperand(i: 0).getValueType()) {
8871 unsigned Factor = VT.getVectorMinNumElements();
8872 return N1.getOperand(i: N2C->getZExtValue() / Factor);
8873 }
8874
8875 // EXTRACT_SUBVECTOR of INSERT_SUBVECTOR is often created
8876 // during shuffle legalization.
8877 if (N1.getOpcode() == ISD::INSERT_SUBVECTOR && N2 == N1.getOperand(i: 2) &&
8878 VT == N1.getOperand(i: 1).getValueType())
8879 return N1.getOperand(i: 1);
8880 break;
8881 }
8882 }
8883
8884 if (N1.getOpcode() == ISD::POISON || N2.getOpcode() == ISD::POISON) {
8885 switch (Opcode) {
8886 case ISD::XOR:
8887 case ISD::ADD:
8888 case ISD::PTRADD:
8889 case ISD::SUB:
8890 case ISD::SIGN_EXTEND_INREG:
8891 case ISD::UDIV:
8892 case ISD::SDIV:
8893 case ISD::UREM:
8894 case ISD::SREM:
8895 case ISD::MUL:
8896 case ISD::AND:
8897 case ISD::SSUBSAT:
8898 case ISD::USUBSAT:
8899 case ISD::UMIN:
8900 case ISD::OR:
8901 case ISD::SADDSAT:
8902 case ISD::UADDSAT:
8903 case ISD::UMAX:
8904 case ISD::SMAX:
8905 case ISD::SMIN:
8906 // fold op(arg1, poison) -> poison, fold op(poison, arg2) -> poison.
8907 return N2.getOpcode() == ISD::POISON ? N2 : N1;
8908 }
8909 }
8910
8911 // Canonicalize an UNDEF to the RHS, even over a constant.
8912 if (N1.getOpcode() == ISD::UNDEF && N2.getOpcode() != ISD::UNDEF) {
8913 if (TLI->isCommutativeBinOp(Opcode)) {
8914 std::swap(a&: N1, b&: N2);
8915 } else {
8916 switch (Opcode) {
8917 case ISD::PTRADD:
8918 case ISD::SUB:
8919 // fold op(undef, non_undef_arg2) -> undef.
8920 return N1;
8921 case ISD::SIGN_EXTEND_INREG:
8922 case ISD::UDIV:
8923 case ISD::SDIV:
8924 case ISD::UREM:
8925 case ISD::SREM:
8926 case ISD::SSUBSAT:
8927 case ISD::USUBSAT:
8928 // fold op(undef, non_undef_arg2) -> 0.
8929 return getConstant(Val: 0, DL, VT);
8930 }
8931 }
8932 }
8933
8934 // Fold a bunch of operators when the RHS is undef.
8935 if (N2.getOpcode() == ISD::UNDEF) {
8936 switch (Opcode) {
8937 case ISD::XOR:
8938 if (N1.getOpcode() == ISD::UNDEF)
8939 // Handle undef ^ undef -> 0 special case. This is a common
8940 // idiom (misuse).
8941 return getConstant(Val: 0, DL, VT);
8942 [[fallthrough]];
8943 case ISD::ADD:
8944 case ISD::PTRADD:
8945 case ISD::SUB:
8946 // fold op(arg1, undef) -> undef.
8947 return N2;
8948 case ISD::UDIV:
8949 case ISD::SDIV:
8950 case ISD::UREM:
8951 case ISD::SREM:
8952 // fold op(arg1, undef) -> poison.
8953 return getPOISON(VT);
8954 case ISD::MUL:
8955 case ISD::AND:
8956 case ISD::SSUBSAT:
8957 case ISD::USUBSAT:
8958 case ISD::UMIN:
8959 // fold op(undef, undef) -> undef, fold op(arg1, undef) -> 0.
8960 return N1.getOpcode() == ISD::UNDEF ? N2 : getConstant(Val: 0, DL, VT);
8961 case ISD::OR:
8962 case ISD::SADDSAT:
8963 case ISD::UADDSAT:
8964 case ISD::UMAX:
8965 // fold op(undef, undef) -> undef, fold op(arg1, undef) -> -1.
8966 return N1.getOpcode() == ISD::UNDEF ? N2 : getAllOnesConstant(DL, VT);
8967 case ISD::SMAX:
8968 // fold op(undef, undef) -> undef, fold op(arg1, undef) -> MAX_INT.
8969 return N1.getOpcode() == ISD::UNDEF
8970 ? N2
8971 : getConstant(
8972 Val: APInt::getSignedMaxValue(numBits: VT.getScalarSizeInBits()), DL,
8973 VT);
8974 case ISD::SMIN:
8975 // fold op(undef, undef) -> undef, fold op(arg1, undef) -> MIN_INT.
8976 return N1.getOpcode() == ISD::UNDEF
8977 ? N2
8978 : getConstant(
8979 Val: APInt::getSignedMinValue(numBits: VT.getScalarSizeInBits()), DL,
8980 VT);
8981 }
8982 }
8983
8984 // Perform trivial constant folding.
8985 if (SDValue SV = FoldConstantArithmetic(Opcode, DL, VT, Ops: {N1, N2}, Flags))
8986 return SV;
8987
8988 // Memoize this node if possible.
8989 SDNode *N;
8990 SDVTList VTs = getVTList(VT);
8991 SDValue Ops[] = {N1, N2};
8992 if (VT != MVT::Glue) {
8993 SDNodeKey ID(Opcode, VTs, Ops);
8994 FoldingSetInsertToken InsertToken;
8995 if (SDNode *E = lookupNode(Key: ID, DL, InsertToken)) {
8996 E->intersectFlagsWith(Flags);
8997 return SDValue(E, 0);
8998 }
8999
9000 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
9001 N->setFlags(Flags);
9002 createOperands(Node: N, Vals: Ops);
9003 CSEMap.insert(N, Token: InsertToken);
9004 } else {
9005 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
9006 createOperands(Node: N, Vals: Ops);
9007 }
9008
9009 InsertNode(N);
9010 SDValue V = SDValue(N, 0);
9011 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
9012 return V;
9013}
9014
9015SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
9016 SDValue N1, SDValue N2, SDValue N3) {
9017 SDNodeFlags Flags;
9018 if (Inserter)
9019 Flags = Inserter->getFlags();
9020 return getNode(Opcode, DL, VT, N1, N2, N3, Flags);
9021}
9022
9023SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
9024 SDValue N1, SDValue N2, SDValue N3,
9025 const SDNodeFlags Flags) {
9026 assert(N1.getOpcode() != ISD::DELETED_NODE &&
9027 N2.getOpcode() != ISD::DELETED_NODE &&
9028 N3.getOpcode() != ISD::DELETED_NODE &&
9029 "Operand is DELETED_NODE!");
9030 // Perform various simplifications.
9031 switch (Opcode) {
9032 case ISD::BUILD_VECTOR: {
9033 // Attempt to simplify BUILD_VECTOR.
9034 SDValue Ops[] = {N1, N2, N3};
9035 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, DAG&: *this))
9036 return V;
9037 break;
9038 }
9039 case ISD::CONCAT_VECTORS: {
9040 SDValue Ops[] = {N1, N2, N3};
9041 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, DAG&: *this))
9042 return V;
9043 break;
9044 }
9045 case ISD::SETCC: {
9046 assert(VT.isInteger() && "SETCC result type must be an integer!");
9047 assert(N1.getValueType() == N2.getValueType() &&
9048 "SETCC operands must have the same type!");
9049 assert(VT.isVector() == N1.getValueType().isVector() &&
9050 "SETCC type should be vector iff the operand type is vector!");
9051 assert((!VT.isVector() || VT.getVectorElementCount() ==
9052 N1.getValueType().getVectorElementCount()) &&
9053 "SETCC vector element counts must match!");
9054 // Use FoldSetCC to simplify SETCC's.
9055 if (SDValue V =
9056 FoldSetCC(VT, N1, N2, Cond: cast<CondCodeSDNode>(Val&: N3)->get(), dl: DL, Flags))
9057 return V;
9058 break;
9059 }
9060 case ISD::SELECT:
9061 case ISD::VSELECT:
9062 if (SDValue V = simplifySelect(Cond: N1, TVal: N2, FVal: N3))
9063 return V;
9064 break;
9065 case ISD::VECTOR_SHUFFLE:
9066 llvm_unreachable("should use getVectorShuffle constructor!");
9067 case ISD::VECTOR_SPLICE_LEFT:
9068 if (isNullConstant(V: N3))
9069 return N1;
9070 break;
9071 case ISD::VECTOR_SPLICE_RIGHT:
9072 if (isNullConstant(V: N3))
9073 return N2;
9074 break;
9075 case ISD::INSERT_VECTOR_ELT: {
9076 assert(VT.isVector() && VT == N1.getValueType() &&
9077 "INSERT_VECTOR_ELT vector type mismatch");
9078 assert(VT.isFloatingPoint() == N2.getValueType().isFloatingPoint() &&
9079 "INSERT_VECTOR_ELT scalar fp/int mismatch");
9080 assert((!VT.isFloatingPoint() ||
9081 VT.getVectorElementType() == N2.getValueType()) &&
9082 "INSERT_VECTOR_ELT fp scalar type mismatch");
9083 assert((!VT.isInteger() ||
9084 VT.getScalarSizeInBits() <= N2.getScalarValueSizeInBits()) &&
9085 "INSERT_VECTOR_ELT int scalar size mismatch");
9086
9087 auto *N3C = dyn_cast<ConstantSDNode>(Val&: N3);
9088 // INSERT_VECTOR_ELT into out-of-bounds element is an UNDEF, except
9089 // for scalable vectors where we will generate appropriate code to
9090 // deal with out-of-bounds cases correctly.
9091 if (N3C && VT.isFixedLengthVector() &&
9092 N3C->getZExtValue() >= VT.getVectorNumElements())
9093 return getUNDEF(VT);
9094
9095 // Undefined index can be assumed out-of-bounds, so that's UNDEF too.
9096 if (N3.isUndef())
9097 return getUNDEF(VT);
9098
9099 // If inserting poison, just use the input vector.
9100 if (N2.getOpcode() == ISD::POISON)
9101 return N1;
9102
9103 // Inserting undef into undef/poison is still undef.
9104 if (N2.getOpcode() == ISD::UNDEF && N1.isUndef())
9105 return getUNDEF(VT);
9106
9107 // If the inserted element is an UNDEF, just use the input vector.
9108 // But not if skipping the insert could make the result more poisonous.
9109 if (N2.isUndef()) {
9110 if (N3C && VT.isFixedLengthVector()) {
9111 APInt EltMask =
9112 APInt::getOneBitSet(numBits: VT.getVectorNumElements(), BitNo: N3C->getZExtValue());
9113 if (isGuaranteedNotToBePoison(Op: N1, DemandedElts: EltMask))
9114 return N1;
9115 } else if (isGuaranteedNotToBePoison(Op: N1))
9116 return N1;
9117 }
9118 break;
9119 }
9120 case ISD::INSERT_SUBVECTOR: {
9121 // If inserting poison, just use the input vector,
9122 if (N2.getOpcode() == ISD::POISON)
9123 return N1;
9124
9125 // Inserting undef into undef/poison is still undef.
9126 if (N2.getOpcode() == ISD::UNDEF && N1.isUndef())
9127 return getUNDEF(VT);
9128
9129 EVT N2VT = N2.getValueType();
9130 assert(VT == N1.getValueType() &&
9131 "Dest and insert subvector source types must match!");
9132 assert(VT.isVector() && N2VT.isVector() &&
9133 "Insert subvector VTs must be vectors!");
9134 assert(VT.getVectorElementType() == N2VT.getVectorElementType() &&
9135 "Insert subvector VTs must have the same element type!");
9136 assert((VT.isScalableVector() || N2VT.isFixedLengthVector()) &&
9137 "Cannot insert a scalable vector into a fixed length vector!");
9138 assert((VT.isScalableVector() != N2VT.isScalableVector() ||
9139 VT.getVectorMinNumElements() >= N2VT.getVectorMinNumElements()) &&
9140 "Insert subvector must be from smaller vector to larger vector!");
9141 assert(isa<ConstantSDNode>(N3) &&
9142 "Insert subvector index must be constant");
9143 assert((VT.isScalableVector() != N2VT.isScalableVector() ||
9144 (N2VT.getVectorMinNumElements() + N3->getAsZExtVal()) <=
9145 VT.getVectorMinNumElements()) &&
9146 "Insert subvector overflow!");
9147 assert(N3->getAsAPIntVal().getBitWidth() ==
9148 TLI->getVectorIdxWidth(getDataLayout()) &&
9149 "Constant index for INSERT_SUBVECTOR has an invalid size");
9150
9151 // Trivial insertion.
9152 if (VT == N2VT)
9153 return N2;
9154
9155 // If this is an insert of an extracted vector into an undef/poison vector,
9156 // we can just use the input to the extract. But not if skipping the
9157 // extract+insert could make the result more poisonous.
9158 if (N1.isUndef() && N2.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
9159 N2.getOperand(i: 1) == N3 && N2.getOperand(i: 0).getValueType() == VT) {
9160 if (N1.getOpcode() == ISD::POISON)
9161 return N2.getOperand(i: 0);
9162 if (VT.isFixedLengthVector() && N2VT.isFixedLengthVector()) {
9163 unsigned LoBit = N3->getAsZExtVal();
9164 unsigned HiBit = LoBit + N2VT.getVectorNumElements();
9165 APInt EltMask =
9166 APInt::getBitsSet(numBits: VT.getVectorNumElements(), loBit: LoBit, hiBit: HiBit);
9167 if (isGuaranteedNotToBePoison(Op: N2.getOperand(i: 0), DemandedElts: ~EltMask))
9168 return N2.getOperand(i: 0);
9169 } else if (isGuaranteedNotToBePoison(Op: N2.getOperand(i: 0)))
9170 return N2.getOperand(i: 0);
9171 }
9172
9173 // If the inserted subvector is UNDEF, just use the input vector.
9174 // But not if skipping the insert could make the result more poisonous.
9175 if (N2.isUndef()) {
9176 if (VT.isFixedLengthVector()) {
9177 unsigned LoBit = N3->getAsZExtVal();
9178 unsigned HiBit = LoBit + N2VT.getVectorNumElements();
9179 APInt EltMask =
9180 APInt::getBitsSet(numBits: VT.getVectorNumElements(), loBit: LoBit, hiBit: HiBit);
9181 if (isGuaranteedNotToBePoison(Op: N1, DemandedElts: EltMask))
9182 return N1;
9183 } else if (isGuaranteedNotToBePoison(Op: N1))
9184 return N1;
9185 }
9186 break;
9187 }
9188 case ISD::BITCAST:
9189 // Fold bit_convert nodes from a type to themselves.
9190 if (N1.getValueType() == VT)
9191 return N1;
9192 break;
9193 case ISD::VECTOR_COMPRESS: {
9194 [[maybe_unused]] EVT VecVT = N1.getValueType();
9195 [[maybe_unused]] EVT MaskVT = N2.getValueType();
9196 [[maybe_unused]] EVT PassthruVT = N3.getValueType();
9197 assert(VT == VecVT && "Vector and result type don't match.");
9198 assert(VecVT.isVector() && MaskVT.isVector() && PassthruVT.isVector() &&
9199 "All inputs must be vectors.");
9200 assert(VecVT == PassthruVT && "Vector and passthru types don't match.");
9201 assert(VecVT.getVectorElementCount() == MaskVT.getVectorElementCount() &&
9202 "Vector and mask must have same number of elements.");
9203
9204 if (N1.isUndef() || N2.isUndef())
9205 return N3;
9206
9207 break;
9208 }
9209 case ISD::PARTIAL_REDUCE_UMLA:
9210 case ISD::PARTIAL_REDUCE_SMLA:
9211 case ISD::PARTIAL_REDUCE_SUMLA:
9212 case ISD::PARTIAL_REDUCE_FMLA: {
9213 [[maybe_unused]] EVT AccVT = N1.getValueType();
9214 [[maybe_unused]] EVT Input1VT = N2.getValueType();
9215 [[maybe_unused]] EVT Input2VT = N3.getValueType();
9216 assert(Input1VT.isVector() && Input1VT == Input2VT &&
9217 "Expected the second and third operands of the PARTIAL_REDUCE_MLA "
9218 "node to have the same type!");
9219 assert(VT.isVector() && VT == AccVT &&
9220 "Expected the first operand of the PARTIAL_REDUCE_MLA node to have "
9221 "the same type as its result!");
9222 assert(Input1VT.getVectorElementCount().hasKnownScalarFactor(
9223 AccVT.getVectorElementCount()) &&
9224 "Expected the element count of the second and third operands of the "
9225 "PARTIAL_REDUCE_MLA node to be a positive integer multiple of the "
9226 "element count of the first operand and the result!");
9227 assert(N2.getScalarValueSizeInBits() <= N1.getScalarValueSizeInBits() &&
9228 "Expected the second and third operands of the PARTIAL_REDUCE_MLA "
9229 "node to have an element type which is the same as or smaller than "
9230 "the element type of the first operand and result!");
9231 break;
9232 }
9233 }
9234
9235 // Perform trivial constant folding for arithmetic operators.
9236 switch (Opcode) {
9237 case ISD::PARTIAL_REDUCE_SMLA:
9238 case ISD::PARTIAL_REDUCE_UMLA:
9239 case ISD::PARTIAL_REDUCE_SUMLA:
9240 case ISD::FMA:
9241 case ISD::FMAD:
9242 case ISD::SETCC:
9243 case ISD::FSHL:
9244 case ISD::FSHR:
9245 if (SDValue SV =
9246 FoldConstantArithmetic(Opcode, DL, VT, Ops: {N1, N2, N3}, Flags))
9247 return SV;
9248 break;
9249 }
9250
9251 // Memoize node if it doesn't produce a glue result.
9252 SDNode *N;
9253 SDVTList VTs = getVTList(VT);
9254 SDValue Ops[] = {N1, N2, N3};
9255 if (VT != MVT::Glue) {
9256 SDNodeKey ID(Opcode, VTs, Ops);
9257 FoldingSetInsertToken InsertToken;
9258 if (SDNode *E = lookupNode(Key: ID, DL, InsertToken)) {
9259 E->intersectFlagsWith(Flags);
9260 return SDValue(E, 0);
9261 }
9262
9263 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
9264 N->setFlags(Flags);
9265 createOperands(Node: N, Vals: Ops);
9266 CSEMap.insert(N, Token: InsertToken);
9267 } else {
9268 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
9269 createOperands(Node: N, Vals: Ops);
9270 }
9271
9272 InsertNode(N);
9273 SDValue V = SDValue(N, 0);
9274 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
9275 return V;
9276}
9277
9278SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
9279 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
9280 const SDNodeFlags Flags) {
9281 SDValue Ops[] = { N1, N2, N3, N4 };
9282 return getNode(Opcode, DL, VT, Ops, Flags);
9283}
9284
9285SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
9286 SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
9287 SDNodeFlags Flags;
9288 if (Inserter)
9289 Flags = Inserter->getFlags();
9290 return getNode(Opcode, DL, VT, N1, N2, N3, N4, Flags);
9291}
9292
9293SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
9294 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
9295 SDValue N5, const SDNodeFlags Flags) {
9296 SDValue Ops[] = { N1, N2, N3, N4, N5 };
9297 return getNode(Opcode, DL, VT, Ops, Flags);
9298}
9299
9300SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
9301 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
9302 SDValue N5) {
9303 SDNodeFlags Flags;
9304 if (Inserter)
9305 Flags = Inserter->getFlags();
9306 return getNode(Opcode, DL, VT, N1, N2, N3, N4, N5, Flags);
9307}
9308
9309/// getStackArgumentTokenFactor - Compute a TokenFactor to force all
9310/// the incoming stack arguments to be loaded from the stack.
9311SDValue SelectionDAG::getStackArgumentTokenFactor(SDValue Chain) {
9312 SmallVector<SDValue, 8> ArgChains;
9313
9314 // Include the original chain at the beginning of the list. When this is
9315 // used by target LowerCall hooks, this helps legalize find the
9316 // CALLSEQ_BEGIN node.
9317 ArgChains.push_back(Elt: Chain);
9318
9319 // Add a chain value for each stack argument.
9320 for (SDNode *U : getEntryNode().getNode()->users())
9321 if (LoadSDNode *L = dyn_cast<LoadSDNode>(Val: U))
9322 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val: L->getBasePtr()))
9323 if (FI->getIndex() < 0)
9324 ArgChains.push_back(Elt: SDValue(L, 1));
9325
9326 // Build a tokenfactor for all the chains.
9327 return getNode(Opcode: ISD::TokenFactor, DL: SDLoc(Chain), VT: MVT::Other, Ops: ArgChains);
9328}
9329
9330/// getMemsetValue - Vectorized representation of the memset value
9331/// operand.
9332static SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG,
9333 const SDLoc &dl) {
9334 assert(!Value.isUndef());
9335
9336 unsigned NumBits = VT.getScalarSizeInBits();
9337 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: Value)) {
9338 assert(C->getAPIntValue().getBitWidth() == 8);
9339 APInt Val = APInt::getSplat(NewLen: NumBits, V: C->getAPIntValue());
9340 if (VT.isInteger()) {
9341 bool IsOpaque = VT.getSizeInBits() > 64 ||
9342 !DAG.getTargetLoweringInfo().isLegalStoreImmediate(Value: C->getSExtValue());
9343 return DAG.getConstant(Val, DL: dl, VT, isT: false, isO: IsOpaque);
9344 }
9345 return DAG.getConstantFP(V: APFloat(VT.getFltSemantics(), Val), DL: dl, VT);
9346 }
9347
9348 assert(Value.getValueType() == MVT::i8 && "memset with non-byte fill value?");
9349 EVT IntVT = VT.getScalarType();
9350 if (!IntVT.isInteger())
9351 IntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: IntVT.getSizeInBits());
9352
9353 Value = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: IntVT, N1: Value);
9354 if (NumBits > 8) {
9355 // Use a multiplication with 0x010101... to extend the input to the
9356 // required length.
9357 APInt Magic = APInt::getSplat(NewLen: NumBits, V: APInt(8, 0x01));
9358 Value = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: IntVT, N1: Value,
9359 N2: DAG.getConstant(Val: Magic, DL: dl, VT: IntVT));
9360 }
9361
9362 if (VT != Value.getValueType() && !VT.isInteger())
9363 Value = DAG.getBitcast(VT: VT.getScalarType(), V: Value);
9364 if (VT != Value.getValueType())
9365 Value = DAG.getSplatBuildVector(VT, DL: dl, Op: Value);
9366
9367 return Value;
9368}
9369
9370/// getMemsetStringVal - Similar to getMemsetValue. Except this is only
9371/// used when a memcpy is turned into a memset when the source is a constant
9372/// string ptr.
9373static SDValue getMemsetStringVal(EVT VT, const SDLoc &dl, SelectionDAG &DAG,
9374 const TargetLowering &TLI,
9375 const ConstantDataArraySlice &Slice) {
9376 // Handle vector with all elements zero.
9377 if (Slice.Array == nullptr) {
9378 if (VT.isInteger())
9379 return DAG.getConstant(Val: 0, DL: dl, VT);
9380 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT,
9381 N1: DAG.getConstant(Val: 0, DL: dl, VT: VT.changeTypeToInteger()));
9382 }
9383
9384 assert(!VT.isVector() && "Can't handle vector type here!");
9385 unsigned NumVTBits = VT.getSizeInBits();
9386 unsigned NumVTBytes = NumVTBits / 8;
9387 unsigned NumBytes = std::min(a: NumVTBytes, b: unsigned(Slice.Length));
9388
9389 APInt Val(NumVTBits, 0);
9390 if (DAG.getDataLayout().isLittleEndian()) {
9391 for (unsigned i = 0; i != NumBytes; ++i)
9392 Val |= (uint64_t)(unsigned char)Slice[i] << i*8;
9393 } else {
9394 for (unsigned i = 0; i != NumBytes; ++i)
9395 Val |= (uint64_t)(unsigned char)Slice[i] << (NumVTBytes-i-1)*8;
9396 }
9397
9398 // If the "cost" of materializing the integer immediate is less than the cost
9399 // of a load, then it is cost effective to turn the load into the immediate.
9400 Type *Ty = VT.getTypeForEVT(Context&: *DAG.getContext());
9401 if (TLI.shouldConvertConstantLoadToIntImm(Imm: Val, Ty))
9402 return DAG.getConstant(Val, DL: dl, VT);
9403 return SDValue();
9404}
9405
9406SDValue SelectionDAG::getMemBasePlusOffset(SDValue Base, TypeSize Offset,
9407 const SDLoc &DL,
9408 const SDNodeFlags Flags) {
9409 SDValue Index = getTypeSize(DL, VT: Base.getValueType(), TS: Offset);
9410 return getMemBasePlusOffset(Base, Offset: Index, DL, Flags);
9411}
9412
9413SDValue SelectionDAG::getMemBasePlusOffset(SDValue Ptr, SDValue Offset,
9414 const SDLoc &DL,
9415 const SDNodeFlags Flags) {
9416 assert(Offset.getValueType().isInteger());
9417 EVT BasePtrVT = Ptr.getValueType();
9418 if (TLI->shouldPreservePtrArith(F: this->getMachineFunction().getFunction(),
9419 PtrVT: BasePtrVT))
9420 return getNode(Opcode: ISD::PTRADD, DL, VT: BasePtrVT, N1: Ptr, N2: Offset, Flags);
9421 // InBounds only applies to PTRADD, don't set it if we generate ADD.
9422 SDNodeFlags AddFlags = Flags;
9423 AddFlags.setInBounds(false);
9424 return getNode(Opcode: ISD::ADD, DL, VT: BasePtrVT, N1: Ptr, N2: Offset, Flags: AddFlags);
9425}
9426
9427/// Returns true if memcpy source is constant data.
9428static bool isMemSrcFromConstant(SDValue Src, ConstantDataArraySlice &Slice) {
9429 uint64_t SrcDelta = 0;
9430 GlobalAddressSDNode *G = nullptr;
9431 if (Src.getOpcode() == ISD::GlobalAddress)
9432 G = cast<GlobalAddressSDNode>(Val&: Src);
9433 else if (Src->isAnyAdd() &&
9434 Src.getOperand(i: 0).getOpcode() == ISD::GlobalAddress &&
9435 Src.getOperand(i: 1).getOpcode() == ISD::Constant) {
9436 G = cast<GlobalAddressSDNode>(Val: Src.getOperand(i: 0));
9437 SrcDelta = Src.getConstantOperandVal(i: 1);
9438 }
9439 if (!G)
9440 return false;
9441
9442 return getConstantDataArrayInfo(V: G->getGlobal(), Slice, ElementSize: 8,
9443 Offset: SrcDelta + G->getOffset());
9444}
9445
9446static bool shouldLowerMemFuncForSize(const MachineFunction &MF,
9447 SelectionDAG &DAG) {
9448 // On Darwin, -Os means optimize for size without hurting performance, so
9449 // only really optimize for size when -Oz (MinSize) is used.
9450 if (MF.getTarget().getTargetTriple().isOSDarwin())
9451 return MF.getFunction().hasMinSize();
9452 return DAG.shouldOptForSize();
9453}
9454
9455static void chainLoadsAndStoresForMemcpy(SelectionDAG &DAG, const SDLoc &dl,
9456 SmallVector<SDValue, 32> &OutChains, unsigned From,
9457 unsigned To, SmallVector<SDValue, 16> &OutLoadChains,
9458 SmallVector<SDValue, 16> &OutStoreChains) {
9459 assert(OutLoadChains.size() && "Missing loads in memcpy inlining");
9460 assert(OutStoreChains.size() && "Missing stores in memcpy inlining");
9461 SmallVector<SDValue, 16> GluedLoadChains;
9462 for (unsigned i = From; i < To; ++i) {
9463 OutChains.push_back(Elt: OutLoadChains[i]);
9464 GluedLoadChains.push_back(Elt: OutLoadChains[i]);
9465 }
9466
9467 // Chain for all loads.
9468 SDValue LoadToken = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other,
9469 Ops: GluedLoadChains);
9470
9471 for (unsigned i = From; i < To; ++i) {
9472 StoreSDNode *ST = dyn_cast<StoreSDNode>(Val&: OutStoreChains[i]);
9473 SDValue NewStore = DAG.getTruncStore(Chain: LoadToken, dl, Val: ST->getValue(),
9474 Ptr: ST->getBasePtr(), SVT: ST->getMemoryVT(),
9475 MMO: ST->getMemOperand());
9476 OutChains.push_back(Elt: NewStore);
9477 }
9478}
9479
9480static SDValue
9481getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain,
9482 SDValue Dst, SDValue Src, uint64_t Size, Align DstAlign,
9483 Align SrcAlign, bool isVol, bool AlwaysInline,
9484 MachinePointerInfo DstPtrInfo,
9485 MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo,
9486 BatchAAResults *BatchAA, const MDNode *DstMemCacheHint,
9487 const MDNode *SrcMemCacheHint) {
9488 // Turn a memcpy of undef to nop.
9489 // FIXME: We need to honor volatile even is Src is undef.
9490 if (Src.isUndef())
9491 return Chain;
9492
9493 // Expand memcpy to a series of load and store ops if the size operand falls
9494 // below a certain threshold.
9495 // TODO: In the AlwaysInline case, if the size is big then generate a loop
9496 // rather than maybe a humongous number of loads and stores.
9497 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9498 const DataLayout &DL = DAG.getDataLayout();
9499 LLVMContext &C = *DAG.getContext();
9500 std::vector<EVT> MemOps;
9501 bool DstAlignCanChange = false;
9502 MachineFunction &MF = DAG.getMachineFunction();
9503 MachineFrameInfo &MFI = MF.getFrameInfo();
9504 bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
9505 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Dst);
9506 if (FI && !MFI.isFixedObjectIndex(ObjectIdx: FI->getIndex()))
9507 DstAlignCanChange = true;
9508 SrcAlign = std::max(a: SrcAlign, b: DAG.InferPtrAlign(Ptr: Src).valueOrOne());
9509 ConstantDataArraySlice Slice;
9510 // If marked as volatile, perform a copy even when marked as constant.
9511 bool CopyFromConstant = !isVol && isMemSrcFromConstant(Src, Slice);
9512 bool isZeroConstant = CopyFromConstant && Slice.Array == nullptr;
9513 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(OptSize);
9514 const MemOp Op = isZeroConstant
9515 ? MemOp::Set(Size, DstAlignCanChange, DstAlign,
9516 /*IsZeroMemset*/ true, IsVolatile: isVol)
9517 : MemOp::Copy(Size, DstAlignCanChange, DstAlign,
9518 SrcAlign, IsVolatile: isVol, MemcpyStrSrc: CopyFromConstant);
9519 if (!TLI.findOptimalMemOpLowering(
9520 Context&: C, MemOps, Limit, Op, DstAS: DstPtrInfo.getAddrSpace(),
9521 SrcAS: SrcPtrInfo.getAddrSpace(), FuncAttributes: MF.getFunction().getAttributes(), LargestVT: nullptr))
9522 return SDValue();
9523
9524 if (DstAlignCanChange) {
9525 Type *Ty = MemOps[0].getTypeForEVT(Context&: C);
9526 Align NewDstAlign = DL.getABITypeAlign(Ty);
9527
9528 // Don't promote to an alignment that would require dynamic stack
9529 // realignment which may conflict with optimizations such as tail call
9530 // optimization.
9531 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
9532 if (!TRI->hasStackRealignment(MF))
9533 if (MaybeAlign StackAlign = DL.getStackAlignment())
9534 NewDstAlign = std::min(a: NewDstAlign, b: *StackAlign);
9535
9536 if (NewDstAlign > DstAlign) {
9537 // Give the stack frame object a larger alignment if needed.
9538 if (MFI.getObjectAlign(ObjectIdx: FI->getIndex()) < NewDstAlign)
9539 MFI.setObjectAlignment(ObjectIdx: FI->getIndex(), Alignment: NewDstAlign);
9540 DstAlign = NewDstAlign;
9541 }
9542 }
9543
9544 // Prepare AAInfo for loads/stores after lowering this memcpy.
9545 AAMDNodes NewAAInfo = AAInfo;
9546 NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
9547
9548 const Value *SrcVal = dyn_cast_if_present<const Value *>(Val&: SrcPtrInfo.V);
9549 bool isConstant =
9550 BatchAA && SrcVal &&
9551 BatchAA->pointsToConstantMemory(Loc: MemoryLocation(SrcVal, Size, AAInfo));
9552
9553 MachineMemOperand::Flags MMOFlags =
9554 isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone;
9555 SmallVector<SDValue, 16> OutLoadChains;
9556 SmallVector<SDValue, 16> OutStoreChains;
9557 SmallVector<SDValue, 32> OutChains;
9558 unsigned NumMemOps = MemOps.size();
9559 uint64_t SrcOff = 0, DstOff = 0;
9560 for (unsigned i = 0; i != NumMemOps; ++i) {
9561 EVT VT = MemOps[i];
9562 unsigned VTSize = VT.getSizeInBits() / 8;
9563 SDValue Value, Store;
9564
9565 if (VTSize > Size) {
9566 // Issuing an unaligned load / store pair that overlaps with the previous
9567 // pair. Adjust the offset accordingly.
9568 assert(i == NumMemOps-1 && i != 0);
9569 SrcOff -= VTSize - Size;
9570 DstOff -= VTSize - Size;
9571 }
9572
9573 if (CopyFromConstant &&
9574 (isZeroConstant || (VT.isInteger() && !VT.isVector()))) {
9575 // It's unlikely a store of a vector immediate can be done in a single
9576 // instruction. It would require a load from a constantpool first.
9577 // We only handle zero vectors here.
9578 // FIXME: Handle other cases where store of vector immediate is done in
9579 // a single instruction.
9580 ConstantDataArraySlice SubSlice;
9581 if (SrcOff < Slice.Length) {
9582 SubSlice = Slice;
9583 SubSlice.move(Delta: SrcOff);
9584 } else {
9585 // This is an out-of-bounds access and hence UB. Pretend we read zero.
9586 SubSlice.Array = nullptr;
9587 SubSlice.Offset = 0;
9588 SubSlice.Length = VTSize;
9589 }
9590 Value = getMemsetStringVal(VT, dl, DAG, TLI, Slice: SubSlice);
9591 if (Value.getNode()) {
9592 Store = DAG.getStore(
9593 Chain, dl, Val: Value,
9594 Ptr: DAG.getObjectPtrOffset(SL: dl, Ptr: Dst, Offset: TypeSize::getFixed(ExactSize: DstOff)),
9595 PtrInfo: DstPtrInfo.getWithOffset(O: DstOff), Alignment: DstAlign, MMOFlags,
9596 Metadata: MMOMetadata(NewAAInfo, /*Ranges=*/nullptr, DstMemCacheHint));
9597 OutChains.push_back(Elt: Store);
9598 }
9599 }
9600
9601 if (!Store.getNode()) {
9602 // The type might not be legal for the target. This should only happen
9603 // if the type is smaller than a legal type, as on PPC, so the right
9604 // thing to do is generate a LoadExt/StoreTrunc pair. These simplify
9605 // to Load/Store if NVT==VT.
9606 // FIXME does the case above also need this?
9607 EVT NVT = TLI.getTypeToTransformTo(Context&: C, VT);
9608 assert(NVT.bitsGE(VT));
9609
9610 bool isDereferenceable =
9611 SrcPtrInfo.getWithOffset(O: SrcOff).isDereferenceable(Size: VTSize, C, DL);
9612 MachineMemOperand::Flags SrcMMOFlags = MMOFlags;
9613 if (isDereferenceable)
9614 SrcMMOFlags |= MachineMemOperand::MODereferenceable;
9615 if (isConstant)
9616 SrcMMOFlags |= MachineMemOperand::MOInvariant;
9617
9618 Value = DAG.getExtLoad(
9619 ExtType: ISD::EXTLOAD, dl, VT: NVT, Chain,
9620 Ptr: DAG.getObjectPtrOffset(SL: dl, Ptr: Src, Offset: TypeSize::getFixed(ExactSize: SrcOff)),
9621 PtrInfo: SrcPtrInfo.getWithOffset(O: SrcOff), MemVT: VT,
9622 Alignment: commonAlignment(A: SrcAlign, Offset: SrcOff), MMOFlags: SrcMMOFlags,
9623 Metadata: MMOMetadata(NewAAInfo, /*Ranges=*/nullptr, SrcMemCacheHint));
9624 OutLoadChains.push_back(Elt: Value.getValue(R: 1));
9625
9626 Store = DAG.getTruncStore(
9627 Chain, dl, Val: Value,
9628 Ptr: DAG.getObjectPtrOffset(SL: dl, Ptr: Dst, Offset: TypeSize::getFixed(ExactSize: DstOff)),
9629 PtrInfo: DstPtrInfo.getWithOffset(O: DstOff), SVT: VT, Alignment: DstAlign, MMOFlags,
9630 Metadata: MMOMetadata(NewAAInfo, /*Ranges=*/nullptr, DstMemCacheHint));
9631 OutStoreChains.push_back(Elt: Store);
9632 }
9633 SrcOff += VTSize;
9634 DstOff += VTSize;
9635 Size -= VTSize;
9636 }
9637
9638 unsigned GluedLdStLimit = MaxLdStGlue == 0 ?
9639 TLI.getMaxGluedStoresPerMemcpy() : MaxLdStGlue;
9640 unsigned NumLdStInMemcpy = OutStoreChains.size();
9641
9642 if (NumLdStInMemcpy) {
9643 // It may be that memcpy might be converted to memset if it's memcpy
9644 // of constants. In such a case, we won't have loads and stores, but
9645 // just stores. In the absence of loads, there is nothing to gang up.
9646 if ((GluedLdStLimit <= 1) || !EnableMemCpyDAGOpt) {
9647 // If target does not care, just leave as it.
9648 for (unsigned i = 0; i < NumLdStInMemcpy; ++i) {
9649 OutChains.push_back(Elt: OutLoadChains[i]);
9650 OutChains.push_back(Elt: OutStoreChains[i]);
9651 }
9652 } else {
9653 // Ld/St less than/equal limit set by target.
9654 if (NumLdStInMemcpy <= GluedLdStLimit) {
9655 chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, From: 0,
9656 To: NumLdStInMemcpy, OutLoadChains,
9657 OutStoreChains);
9658 } else {
9659 unsigned NumberLdChain = NumLdStInMemcpy / GluedLdStLimit;
9660 unsigned RemainingLdStInMemcpy = NumLdStInMemcpy % GluedLdStLimit;
9661 unsigned GlueIter = 0;
9662
9663 // Residual ld/st.
9664 if (RemainingLdStInMemcpy) {
9665 chainLoadsAndStoresForMemcpy(
9666 DAG, dl, OutChains, From: NumLdStInMemcpy - RemainingLdStInMemcpy,
9667 To: NumLdStInMemcpy, OutLoadChains, OutStoreChains);
9668 }
9669
9670 for (unsigned cnt = 0; cnt < NumberLdChain; ++cnt) {
9671 unsigned IndexFrom = NumLdStInMemcpy - RemainingLdStInMemcpy -
9672 GlueIter - GluedLdStLimit;
9673 unsigned IndexTo = NumLdStInMemcpy - RemainingLdStInMemcpy - GlueIter;
9674 chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, From: IndexFrom, To: IndexTo,
9675 OutLoadChains, OutStoreChains);
9676 GlueIter += GluedLdStLimit;
9677 }
9678 }
9679 }
9680 }
9681 return DAG.getTokenFactor(DL: dl, Vals&: OutChains);
9682}
9683
9684static SDValue getMemmoveLoadsAndStores(
9685 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
9686 uint64_t Size, Align DstAlign, Align SrcAlign, bool isVol,
9687 bool AlwaysInline, MachinePointerInfo DstPtrInfo,
9688 MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo) {
9689 // Turn a memmove of undef to nop.
9690 // FIXME: We need to honor volatile even is Src is undef.
9691 if (Src.isUndef())
9692 return Chain;
9693
9694 // Expand memmove to a series of load and store ops if the size operand falls
9695 // below a certain threshold.
9696 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9697 const DataLayout &DL = DAG.getDataLayout();
9698 LLVMContext &C = *DAG.getContext();
9699 std::vector<EVT> MemOps;
9700 bool DstAlignCanChange = false;
9701 MachineFunction &MF = DAG.getMachineFunction();
9702 MachineFrameInfo &MFI = MF.getFrameInfo();
9703 bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
9704 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Dst);
9705 if (FI && !MFI.isFixedObjectIndex(ObjectIdx: FI->getIndex()))
9706 DstAlignCanChange = true;
9707 SrcAlign = std::max(a: SrcAlign, b: DAG.InferPtrAlign(Ptr: Src).valueOrOne());
9708 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize);
9709 if (!TLI.findOptimalMemOpLowering(
9710 Context&: C, MemOps, Limit,
9711 Op: MemOp::Move(Size, DstAlignCanChange, DstAlign, SrcAlign, IsVolatile: isVol),
9712 DstAS: DstPtrInfo.getAddrSpace(), SrcAS: SrcPtrInfo.getAddrSpace(),
9713 FuncAttributes: MF.getFunction().getAttributes(), LargestVT: nullptr))
9714 return SDValue();
9715
9716 if (DstAlignCanChange) {
9717 Type *Ty = MemOps[0].getTypeForEVT(Context&: C);
9718 Align NewDstAlign = DL.getABITypeAlign(Ty);
9719
9720 // Don't promote to an alignment that would require dynamic stack
9721 // realignment which may conflict with optimizations such as tail call
9722 // optimization.
9723 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
9724 if (!TRI->hasStackRealignment(MF))
9725 if (MaybeAlign StackAlign = DL.getStackAlignment())
9726 NewDstAlign = std::min(a: NewDstAlign, b: *StackAlign);
9727
9728 if (NewDstAlign > DstAlign) {
9729 // Give the stack frame object a larger alignment if needed.
9730 if (MFI.getObjectAlign(ObjectIdx: FI->getIndex()) < NewDstAlign)
9731 MFI.setObjectAlignment(ObjectIdx: FI->getIndex(), Alignment: NewDstAlign);
9732 DstAlign = NewDstAlign;
9733 }
9734 }
9735
9736 // Prepare AAInfo for loads/stores after lowering this memmove.
9737 AAMDNodes NewAAInfo = AAInfo;
9738 NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
9739
9740 MachineMemOperand::Flags MMOFlags =
9741 isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone;
9742 uint64_t SrcOff = 0;
9743 SmallVector<SDValue, 8> LoadValues;
9744 SmallVector<SDValue, 8> LoadChains;
9745 SmallVector<SDValue, 8> OutChains;
9746 unsigned NumMemOps = MemOps.size();
9747 for (unsigned i = 0; i < NumMemOps; i++) {
9748 EVT VT = MemOps[i];
9749 unsigned VTSize = VT.getSizeInBits() / 8;
9750 SDValue Value;
9751 bool IsOverlapping = false;
9752
9753 if (i == NumMemOps - 1 && i != 0 && VTSize > Size - SrcOff) {
9754 // Issuing an unaligned load / store pair that overlaps with the previous
9755 // pair. Adjust the offset accordingly.
9756 SrcOff = Size - VTSize;
9757 IsOverlapping = true;
9758 }
9759
9760 // Calculate the actual alignment at the current offset. The alignment at
9761 // SrcOff may be lower than the base alignment, especially when using
9762 // overlapping loads.
9763 Align SrcAlignAtOffset = commonAlignment(A: SrcAlign, Offset: SrcOff);
9764 if (IsOverlapping) {
9765 // Verify that the target allows misaligned memory accesses at the
9766 // adjusted offset when using overlapping loads.
9767 unsigned Fast;
9768 if (!TLI.allowsMisalignedMemoryAccesses(VT, AddrSpace: SrcPtrInfo.getAddrSpace(),
9769 Alignment: SrcAlignAtOffset, Flags: MMOFlags,
9770 &Fast) ||
9771 !Fast) {
9772 // This should have been caught by findOptimalMemOpLowering, but verify
9773 // here for safety.
9774 return SDValue();
9775 }
9776 }
9777
9778 bool isDereferenceable =
9779 SrcPtrInfo.getWithOffset(O: SrcOff).isDereferenceable(Size: VTSize, C, DL);
9780 MachineMemOperand::Flags SrcMMOFlags = MMOFlags;
9781 if (isDereferenceable)
9782 SrcMMOFlags |= MachineMemOperand::MODereferenceable;
9783 Value =
9784 DAG.getLoad(VT, dl, Chain,
9785 Ptr: DAG.getObjectPtrOffset(SL: dl, Ptr: Src, Offset: TypeSize::getFixed(ExactSize: SrcOff)),
9786 PtrInfo: SrcPtrInfo.getWithOffset(O: SrcOff), Alignment: SrcAlignAtOffset,
9787 MMOFlags: SrcMMOFlags, Metadata: NewAAInfo);
9788 LoadValues.push_back(Elt: Value);
9789 LoadChains.push_back(Elt: Value.getValue(R: 1));
9790 SrcOff += VTSize;
9791 }
9792 Chain = DAG.getTokenFactor(DL: dl, Vals&: LoadChains);
9793 OutChains.clear();
9794 uint64_t DstOff = 0;
9795 for (unsigned i = 0; i < NumMemOps; i++) {
9796 EVT VT = MemOps[i];
9797 unsigned VTSize = VT.getSizeInBits() / 8;
9798 SDValue Store;
9799 bool IsOverlapping = false;
9800
9801 if (i == NumMemOps - 1 && i != 0 && VTSize > Size - DstOff) {
9802 // Issuing an unaligned load / store pair that overlaps with the previous
9803 // pair. Adjust the offset accordingly.
9804 DstOff = Size - VTSize;
9805 IsOverlapping = true;
9806 }
9807
9808 // Calculate the actual alignment at the current offset. The alignment at
9809 // DstOff may be lower than the base alignment, especially when using
9810 // overlapping stores.
9811 Align DstAlignAtOffset = commonAlignment(A: DstAlign, Offset: DstOff);
9812 if (IsOverlapping) {
9813 // Verify that the target allows misaligned memory accesses at the
9814 // adjusted offset when using overlapping stores.
9815 unsigned Fast;
9816 if (!TLI.allowsMisalignedMemoryAccesses(VT, AddrSpace: DstPtrInfo.getAddrSpace(),
9817 Alignment: DstAlignAtOffset, Flags: MMOFlags,
9818 &Fast) ||
9819 !Fast) {
9820 // This should have been caught by findOptimalMemOpLowering, but verify
9821 // here for safety.
9822 return SDValue();
9823 }
9824 }
9825 Store = DAG.getStore(
9826 Chain, dl, Val: LoadValues[i],
9827 Ptr: DAG.getObjectPtrOffset(SL: dl, Ptr: Dst, Offset: TypeSize::getFixed(ExactSize: DstOff)),
9828 PtrInfo: DstPtrInfo.getWithOffset(O: DstOff), Alignment: DstAlignAtOffset, MMOFlags,
9829 Metadata: NewAAInfo);
9830 OutChains.push_back(Elt: Store);
9831 DstOff += VTSize;
9832 }
9833
9834 return DAG.getTokenFactor(DL: dl, Vals&: OutChains);
9835}
9836
9837/// Lower the call to 'memset' intrinsic function into a series of store
9838/// operations.
9839///
9840/// \param DAG Selection DAG where lowered code is placed.
9841/// \param dl Link to corresponding IR location.
9842/// \param Chain Control flow dependency.
9843/// \param Dst Pointer to destination memory location.
9844/// \param Src Value of byte to write into the memory.
9845/// \param Size Number of bytes to write.
9846/// \param Alignment Alignment of the destination in bytes.
9847/// \param isVol True if destination is volatile.
9848/// \param AlwaysInline Makes sure no function call is generated.
9849/// \param DstPtrInfo IR information on the memory pointer.
9850/// \returns New head in the control flow, if lowering was successful, empty
9851/// SDValue otherwise.
9852///
9853/// The function tries to replace 'llvm.memset' intrinsic with several store
9854/// operations and value calculation code. This is usually profitable for small
9855/// memory size or when the semantic requires inlining.
9856static SDValue getMemsetStores(SelectionDAG &DAG, const SDLoc &dl,
9857 SDValue Chain, SDValue Dst, SDValue Src,
9858 uint64_t Size, Align Alignment, bool isVol,
9859 bool AlwaysInline, MachinePointerInfo DstPtrInfo,
9860 const AAMDNodes &AAInfo) {
9861 // Turn a memset of undef to nop.
9862 // FIXME: We need to honor volatile even is Src is undef.
9863 if (Src.isUndef())
9864 return Chain;
9865
9866 // Expand memset to a series of load/store ops if the size operand
9867 // falls below a certain threshold.
9868 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9869 std::vector<EVT> MemOps;
9870 bool DstAlignCanChange = false;
9871 LLVMContext &C = *DAG.getContext();
9872 MachineFunction &MF = DAG.getMachineFunction();
9873 MachineFrameInfo &MFI = MF.getFrameInfo();
9874 bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
9875 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Dst);
9876 if (FI && !MFI.isFixedObjectIndex(ObjectIdx: FI->getIndex()))
9877 DstAlignCanChange = true;
9878 bool IsZeroVal = isNullConstant(V: Src);
9879 unsigned Limit = AlwaysInline ? ~0 : TLI.getMaxStoresPerMemset(OptSize);
9880
9881 EVT LargestVT;
9882 if (!TLI.findOptimalMemOpLowering(
9883 Context&: C, MemOps, Limit,
9884 Op: MemOp::Set(Size, DstAlignCanChange, DstAlign: Alignment, IsZeroMemset: IsZeroVal, IsVolatile: isVol),
9885 DstAS: DstPtrInfo.getAddrSpace(), SrcAS: ~0u, FuncAttributes: MF.getFunction().getAttributes(),
9886 LargestVT: &LargestVT))
9887 return SDValue();
9888
9889 if (DstAlignCanChange) {
9890 Type *Ty = MemOps[0].getTypeForEVT(Context&: *DAG.getContext());
9891 const DataLayout &DL = DAG.getDataLayout();
9892 Align NewAlign = DL.getABITypeAlign(Ty);
9893
9894 // Don't promote to an alignment that would require dynamic stack
9895 // realignment which may conflict with optimizations such as tail call
9896 // optimization.
9897 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
9898 if (!TRI->hasStackRealignment(MF))
9899 if (MaybeAlign StackAlign = DL.getStackAlignment())
9900 NewAlign = std::min(a: NewAlign, b: *StackAlign);
9901
9902 if (NewAlign > Alignment) {
9903 // Give the stack frame object a larger alignment if needed.
9904 if (MFI.getObjectAlign(ObjectIdx: FI->getIndex()) < NewAlign)
9905 MFI.setObjectAlignment(ObjectIdx: FI->getIndex(), Alignment: NewAlign);
9906 Alignment = NewAlign;
9907 }
9908 }
9909
9910 SmallVector<SDValue, 8> OutChains;
9911 uint64_t DstOff = 0;
9912 unsigned NumMemOps = MemOps.size();
9913
9914 // Find the largest store and generate the bit pattern for it.
9915 // If target didn't set LargestVT, compute it from MemOps.
9916 if (!LargestVT.isSimple()) {
9917 LargestVT = MemOps[0];
9918 for (unsigned i = 1; i < NumMemOps; i++)
9919 if (MemOps[i].bitsGT(VT: LargestVT))
9920 LargestVT = MemOps[i];
9921 }
9922 SDValue MemSetValue = getMemsetValue(Value: Src, VT: LargestVT, DAG, dl);
9923
9924 // Prepare AAInfo for loads/stores after lowering this memset.
9925 AAMDNodes NewAAInfo = AAInfo;
9926 NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
9927
9928 for (unsigned i = 0; i < NumMemOps; i++) {
9929 EVT VT = MemOps[i];
9930 unsigned VTSize = VT.getSizeInBits() / 8;
9931 // The target should specify store types that exactly cover the memset size
9932 // (with the last store potentially being oversized for overlapping stores).
9933 assert(Size > 0 && "Target specified more stores than needed in "
9934 "findOptimalMemOpLowering");
9935 if (VTSize > Size) {
9936 // Issuing an unaligned load / store pair that overlaps with the previous
9937 // pair. Adjust the offset accordingly.
9938 assert(i == NumMemOps-1 && i != 0);
9939 DstOff -= VTSize - Size;
9940 }
9941
9942 // If this store is smaller than the largest store see whether we can get
9943 // the smaller value for free with a truncate or extract vector element and
9944 // then store.
9945 SDValue Value = MemSetValue;
9946 if (VT.bitsLT(VT: LargestVT)) {
9947 unsigned Index;
9948 unsigned NElts = LargestVT.getSizeInBits() / VT.getSizeInBits();
9949 EVT SVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: VT.getScalarType(), NumElements: NElts);
9950 if (!LargestVT.isVector() && !VT.isVector() &&
9951 TLI.isTruncateFree(FromVT: LargestVT, ToVT: VT))
9952 Value = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, N1: MemSetValue);
9953 else if (LargestVT.isVector() && !VT.isVector() &&
9954 TLI.shallExtractConstSplatVectorElementToStore(
9955 VectorTy: LargestVT.getTypeForEVT(Context&: *DAG.getContext()),
9956 ElemSizeInBits: VT.getSizeInBits(), Index) &&
9957 TLI.isTypeLegal(VT: SVT) &&
9958 LargestVT.getSizeInBits() == SVT.getSizeInBits()) {
9959 // Target which can combine store(extractelement VectorTy, Idx) can get
9960 // the smaller value for free.
9961 SDValue TailValue = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: SVT, N1: MemSetValue);
9962 Value = DAG.getExtractVectorElt(DL: dl, VT, Vec: TailValue, Idx: Index);
9963 } else
9964 Value = getMemsetValue(Value: Src, VT, DAG, dl);
9965 }
9966 assert(Value.getValueType() == VT && "Value with wrong type.");
9967 SDValue Store = DAG.getStore(
9968 Chain, dl, Val: Value,
9969 Ptr: DAG.getObjectPtrOffset(SL: dl, Ptr: Dst, Offset: TypeSize::getFixed(ExactSize: DstOff)),
9970 PtrInfo: DstPtrInfo.getWithOffset(O: DstOff), Alignment,
9971 MMOFlags: isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone,
9972 Metadata: NewAAInfo);
9973 OutChains.push_back(Elt: Store);
9974 DstOff += VT.getSizeInBits() / 8;
9975 // For oversized overlapping stores, only subtract the remaining bytes.
9976 // For normal stores, subtract the full store size.
9977 if (VTSize > Size) {
9978 Size = 0;
9979 } else {
9980 Size -= VTSize;
9981 }
9982 }
9983
9984 // After processing all stores, Size should be exactly 0. Any remaining bytes
9985 // indicate a bug in the target's findOptimalMemOpLowering implementation.
9986 assert(Size == 0 && "Target's findOptimalMemOpLowering did not specify "
9987 "stores that exactly cover the memset size");
9988
9989 return DAG.getTokenFactor(DL: dl, Vals&: OutChains);
9990}
9991
9992static void checkAddrSpaceIsValidForLibcall(const TargetLowering *TLI,
9993 unsigned AS) {
9994 // Lowering memcpy / memset / memmove intrinsics to calls is only valid if all
9995 // pointer operands can be losslessly bitcasted to pointers of address space 0
9996 if (AS != 0 && !TLI->getTargetMachine().isNoopAddrSpaceCast(SrcAS: AS, DestAS: 0)) {
9997 report_fatal_error(reason: "cannot lower memory intrinsic in address space " +
9998 Twine(AS));
9999 }
10000}
10001
10002static bool isInTailCallPositionWrapper(const CallInst *CI,
10003 const SelectionDAG *SelDAG,
10004 bool AllowReturnsFirstArg) {
10005 if (!CI || !CI->isTailCall())
10006 return false;
10007 // TODO: Fix "returns-first-arg" determination so it doesn't depend on which
10008 // helper symbol we lower to.
10009 return isInTailCallPosition(Call: *CI, TM: SelDAG->getTarget(),
10010 ReturnsFirstArg: AllowReturnsFirstArg &&
10011 funcReturnsFirstArgOfCall(CI: *CI));
10012}
10013
10014static std::pair<SDValue, SDValue>
10015getRuntimeCallSDValueHelper(SDValue Chain, const SDLoc &dl,
10016 TargetLowering::ArgListTy &&Args,
10017 const CallInst *CI, RTLIB::Libcall Call,
10018 SelectionDAG *DAG, const TargetLowering *TLI) {
10019 RTLIB::LibcallImpl LCImpl = DAG->getLibcalls().getLibcallImpl(Call);
10020
10021 if (LCImpl == RTLIB::Unsupported)
10022 return {};
10023
10024 TargetLowering::CallLoweringInfo CLI(*DAG);
10025 bool IsTailCall =
10026 isInTailCallPositionWrapper(CI, SelDAG: DAG, /*AllowReturnsFirstArg=*/true) &&
10027 // Lowering doesn't support tail calling inside a function with
10028 // a swifterror argument yet.
10029 !DAG->hasSwiftErrorArg();
10030 SDValue Callee =
10031 DAG->getExternalSymbol(Libcall: LCImpl, VT: TLI->getPointerTy(DL: DAG->getDataLayout()));
10032
10033 CLI.setDebugLoc(dl)
10034 .setChain(Chain)
10035 .setLibCallee(CC: DAG->getLibcalls().getLibcallImplCallingConv(Call: LCImpl),
10036 ResultType: CI->getType(), Target: Callee, ArgsList: std::move(Args))
10037 .setTailCall(IsTailCall);
10038
10039 return TLI->LowerCallTo(CLI);
10040}
10041
10042std::pair<SDValue, SDValue> SelectionDAG::getStrcmp(SDValue Chain,
10043 const SDLoc &dl, SDValue S1,
10044 SDValue S2,
10045 const CallInst *CI) {
10046 PointerType *PT = PointerType::getUnqual(C&: *getContext());
10047 TargetLowering::ArgListTy Args = {{S1, PT}, {S2, PT}};
10048 return getRuntimeCallSDValueHelper(Chain, dl, Args: std::move(Args), CI,
10049 Call: RTLIB::STRCMP, DAG: this, TLI);
10050}
10051
10052std::pair<SDValue, SDValue> SelectionDAG::getStrstr(SDValue Chain,
10053 const SDLoc &dl, SDValue S1,
10054 SDValue S2,
10055 const CallInst *CI) {
10056 PointerType *PT = PointerType::getUnqual(C&: *getContext());
10057 TargetLowering::ArgListTy Args = {{S1, PT}, {S2, PT}};
10058 return getRuntimeCallSDValueHelper(Chain, dl, Args: std::move(Args), CI,
10059 Call: RTLIB::STRSTR, DAG: this, TLI);
10060}
10061
10062std::pair<SDValue, SDValue> SelectionDAG::getMemccpy(SDValue Chain,
10063 const SDLoc &dl,
10064 SDValue Dst, SDValue Src,
10065 SDValue C, SDValue Size,
10066 const CallInst *CI) {
10067 PointerType *PT = PointerType::getUnqual(C&: *getContext());
10068
10069 TargetLowering::ArgListTy Args = {
10070 {Dst, PT},
10071 {Src, PT},
10072 {C, Type::getInt32Ty(C&: *getContext())},
10073 {Size, getDataLayout().getIntPtrType(C&: *getContext())}};
10074 return getRuntimeCallSDValueHelper(Chain, dl, Args: std::move(Args), CI,
10075 Call: RTLIB::MEMCCPY, DAG: this, TLI);
10076}
10077
10078std::pair<SDValue, SDValue>
10079SelectionDAG::getMemcmp(SDValue Chain, const SDLoc &dl, SDValue Mem0,
10080 SDValue Mem1, SDValue Size, const CallInst *CI) {
10081 PointerType *PT = PointerType::getUnqual(C&: *getContext());
10082 TargetLowering::ArgListTy Args = {
10083 {Mem0, PT},
10084 {Mem1, PT},
10085 {Size, getDataLayout().getIntPtrType(C&: *getContext())}};
10086 return getRuntimeCallSDValueHelper(Chain, dl, Args: std::move(Args), CI,
10087 Call: RTLIB::MEMCMP, DAG: this, TLI);
10088}
10089
10090std::pair<SDValue, SDValue> SelectionDAG::getStrcpy(SDValue Chain,
10091 const SDLoc &dl,
10092 SDValue Dst, SDValue Src,
10093 const CallInst *CI) {
10094 PointerType *PT = PointerType::getUnqual(C&: *getContext());
10095 TargetLowering::ArgListTy Args = {{Dst, PT}, {Src, PT}};
10096 return getRuntimeCallSDValueHelper(Chain, dl, Args: std::move(Args), CI,
10097 Call: RTLIB::STRCPY, DAG: this, TLI);
10098}
10099
10100std::pair<SDValue, SDValue> SelectionDAG::getStrlen(SDValue Chain,
10101 const SDLoc &dl,
10102 SDValue Src,
10103 const CallInst *CI) {
10104 // Emit a library call.
10105 TargetLowering::ArgListTy Args = {
10106 {Src, PointerType::getUnqual(C&: *getContext())}};
10107 return getRuntimeCallSDValueHelper(Chain, dl, Args: std::move(Args), CI,
10108 Call: RTLIB::STRLEN, DAG: this, TLI);
10109}
10110
10111bool SelectionDAG::hasSwiftErrorArg() const {
10112 return TLI->supportSwiftError() &&
10113 MF->getFunction().getAttributes().hasAttrSomewhere(
10114 Kind: Attribute::SwiftError);
10115}
10116
10117SDValue SelectionDAG::getMemcpy(
10118 SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size,
10119 Align DstAlign, Align SrcAlign, bool isVol, bool AlwaysInline,
10120 const CallInst *CI, std::optional<bool> OverrideTailCall,
10121 MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo,
10122 const AAMDNodes &AAInfo, BatchAAResults *BatchAA) {
10123 // Check to see if we should lower the memcpy to loads and stores first.
10124 // For cases within the target-specified limits, this is the best choice.
10125 const MDNode *DstMemCacheHint =
10126 CI ? getMemCacheHintMetadata(I: *CI, /*OperandNo=*/0) : nullptr;
10127 const MDNode *SrcMemCacheHint =
10128 CI ? getMemCacheHintMetadata(I: *CI, /*OperandNo=*/1) : nullptr;
10129
10130 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Val&: Size);
10131 if (ConstantSize) {
10132 // Memcpy with size zero? Just return the original chain.
10133 if (ConstantSize->isZero())
10134 return Chain;
10135
10136 SDValue Result = getMemcpyLoadsAndStores(
10137 DAG&: *this, dl, Chain, Dst, Src, Size: ConstantSize->getZExtValue(), DstAlign,
10138 SrcAlign, isVol, AlwaysInline: false, DstPtrInfo, SrcPtrInfo, AAInfo, BatchAA,
10139 DstMemCacheHint, SrcMemCacheHint);
10140 if (Result.getNode())
10141 return Result;
10142 }
10143
10144 // Then check to see if we should lower the memcpy with target-specific
10145 // code. If the target chooses to do this, this is the next best.
10146 if (TSI) {
10147 SDValue Result = TSI->EmitTargetCodeForMemcpy(
10148 DAG&: *this, dl, Chain, Op1: Dst, Op2: Src, Op3: Size, DstAlign, SrcAlign, isVolatile: isVol,
10149 AlwaysInline, DstPtrInfo, SrcPtrInfo);
10150 if (Result.getNode())
10151 return Result;
10152 }
10153
10154 // If we really need inline code and the target declined to provide it,
10155 // use a (potentially long) sequence of loads and stores.
10156 if (AlwaysInline) {
10157 assert(ConstantSize && "AlwaysInline requires a constant size!");
10158 return getMemcpyLoadsAndStores(
10159 DAG&: *this, dl, Chain, Dst, Src, Size: ConstantSize->getZExtValue(), DstAlign,
10160 SrcAlign, isVol, AlwaysInline: true, DstPtrInfo, SrcPtrInfo, AAInfo, BatchAA,
10161 DstMemCacheHint, SrcMemCacheHint);
10162 }
10163
10164 checkAddrSpaceIsValidForLibcall(TLI, AS: DstPtrInfo.getAddrSpace());
10165 checkAddrSpaceIsValidForLibcall(TLI, AS: SrcPtrInfo.getAddrSpace());
10166
10167 // FIXME: If the memcpy is volatile (isVol), lowering it to a plain libc
10168 // memcpy is not guaranteed to be safe. libc memcpys aren't required to
10169 // respect volatile, so they may do things like read or write memory
10170 // beyond the given memory regions. But fixing this isn't easy, and most
10171 // people don't care.
10172
10173 // Emit a library call.
10174 TargetLowering::ArgListTy Args;
10175 Type *PtrTy = PointerType::getUnqual(C&: *getContext());
10176 Args.emplace_back(args&: Dst, args&: PtrTy);
10177 Args.emplace_back(args&: Src, args&: PtrTy);
10178 Args.emplace_back(args&: Size, args: getDataLayout().getIntPtrType(C&: *getContext()));
10179 // FIXME: pass in SDLoc
10180 TargetLowering::CallLoweringInfo CLI(*this);
10181 bool IsTailCall = false;
10182 RTLIB::LibcallImpl MemCpyImpl = TLI->getMemcpyImpl();
10183
10184 if (OverrideTailCall.has_value()) {
10185 IsTailCall = *OverrideTailCall;
10186 } else {
10187 bool LowersToMemcpy = MemCpyImpl == RTLIB::impl_memcpy;
10188 IsTailCall = isInTailCallPositionWrapper(CI, SelDAG: this, AllowReturnsFirstArg: LowersToMemcpy);
10189 }
10190 // Lowering doesn't support tail calling inside a function with a
10191 // swifterror argument yet.
10192 IsTailCall &= !hasSwiftErrorArg();
10193
10194 CLI.setDebugLoc(dl)
10195 .setChain(Chain)
10196 .setLibCallee(
10197 CC: Libcalls->getLibcallImplCallingConv(Call: MemCpyImpl),
10198 ResultType: Dst.getValueType().getTypeForEVT(Context&: *getContext()),
10199 Target: getExternalSymbol(Libcall: MemCpyImpl, VT: TLI->getPointerTy(DL: getDataLayout())),
10200 ArgsList: std::move(Args))
10201 .setDiscardResult()
10202 .setTailCall(IsTailCall);
10203
10204 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
10205 return CallResult.second;
10206}
10207
10208SDValue SelectionDAG::getAtomicMemcpy(SDValue Chain, const SDLoc &dl,
10209 SDValue Dst, SDValue Src, SDValue Size,
10210 Type *SizeTy, unsigned ElemSz,
10211 bool isTailCall,
10212 MachinePointerInfo DstPtrInfo,
10213 MachinePointerInfo SrcPtrInfo) {
10214 // Lowering doesn't support tail calling inside a function with a
10215 // swifterror argument yet.
10216 isTailCall &= !hasSwiftErrorArg();
10217
10218 // Emit a library call.
10219 TargetLowering::ArgListTy Args;
10220 Type *ArgTy = getDataLayout().getIntPtrType(C&: *getContext());
10221 Args.emplace_back(args&: Dst, args&: ArgTy);
10222 Args.emplace_back(args&: Src, args&: ArgTy);
10223 Args.emplace_back(args&: Size, args&: SizeTy);
10224
10225 RTLIB::Libcall LibraryCall =
10226 RTLIB::getMEMCPY_ELEMENT_UNORDERED_ATOMIC(ElementSize: ElemSz);
10227 RTLIB::LibcallImpl LibcallImpl = Libcalls->getLibcallImpl(Call: LibraryCall);
10228 if (LibcallImpl == RTLIB::Unsupported)
10229 report_fatal_error(reason: "Unsupported element size");
10230
10231 TargetLowering::CallLoweringInfo CLI(*this);
10232 CLI.setDebugLoc(dl)
10233 .setChain(Chain)
10234 .setLibCallee(
10235 CC: Libcalls->getLibcallImplCallingConv(Call: LibcallImpl),
10236 ResultType: Type::getVoidTy(C&: *getContext()),
10237 Target: getExternalSymbol(Libcall: LibcallImpl, VT: TLI->getPointerTy(DL: getDataLayout())),
10238 ArgsList: std::move(Args))
10239 .setDiscardResult()
10240 .setTailCall(isTailCall);
10241
10242 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
10243 return CallResult.second;
10244}
10245
10246SDValue SelectionDAG::getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst,
10247 SDValue Src, SDValue Size, Align DstAlign,
10248 Align SrcAlign, bool isVol, const CallInst *CI,
10249 std::optional<bool> OverrideTailCall,
10250 MachinePointerInfo DstPtrInfo,
10251 MachinePointerInfo SrcPtrInfo,
10252 const AAMDNodes &AAInfo,
10253 BatchAAResults *BatchAA) {
10254 // Check to see if we should lower the memmove to loads and stores first.
10255 // For cases within the target-specified limits, this is the best choice.
10256 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Val&: Size);
10257 if (ConstantSize) {
10258 // Memmove with size zero? Just return the original chain.
10259 if (ConstantSize->isZero())
10260 return Chain;
10261
10262 SDValue Result = getMemmoveLoadsAndStores(
10263 DAG&: *this, dl, Chain, Dst, Src, Size: ConstantSize->getZExtValue(), DstAlign,
10264 SrcAlign, isVol, AlwaysInline: false, DstPtrInfo, SrcPtrInfo, AAInfo);
10265 if (Result.getNode())
10266 return Result;
10267 }
10268
10269 // Then check to see if we should lower the memmove with target-specific
10270 // code. If the target chooses to do this, this is the next best.
10271 if (TSI) {
10272 SDValue Result = TSI->EmitTargetCodeForMemmove(
10273 DAG&: *this, dl, Chain, Op1: Dst, Op2: Src, Op3: Size, DstAlign, SrcAlign, isVolatile: isVol, DstPtrInfo,
10274 SrcPtrInfo);
10275 if (Result.getNode())
10276 return Result;
10277 }
10278
10279 checkAddrSpaceIsValidForLibcall(TLI, AS: DstPtrInfo.getAddrSpace());
10280 checkAddrSpaceIsValidForLibcall(TLI, AS: SrcPtrInfo.getAddrSpace());
10281
10282 // FIXME: If the memmove is volatile, lowering it to plain libc memmove may
10283 // not be safe. See memcpy above for more details.
10284
10285 // Emit a library call.
10286 TargetLowering::ArgListTy Args;
10287 Type *PtrTy = PointerType::getUnqual(C&: *getContext());
10288 Args.emplace_back(args&: Dst, args&: PtrTy);
10289 Args.emplace_back(args&: Src, args&: PtrTy);
10290 Args.emplace_back(args&: Size, args: getDataLayout().getIntPtrType(C&: *getContext()));
10291 // FIXME: pass in SDLoc
10292 TargetLowering::CallLoweringInfo CLI(*this);
10293
10294 RTLIB::LibcallImpl MemmoveImpl = Libcalls->getLibcallImpl(Call: RTLIB::MEMMOVE);
10295
10296 bool IsTailCall = false;
10297 if (OverrideTailCall.has_value()) {
10298 IsTailCall = *OverrideTailCall;
10299 } else {
10300 bool LowersToMemmove = MemmoveImpl == RTLIB::impl_memmove;
10301 IsTailCall = isInTailCallPositionWrapper(CI, SelDAG: this, AllowReturnsFirstArg: LowersToMemmove);
10302 }
10303 // Lowering doesn't support tail calling inside a function with a
10304 // swifterror argument yet.
10305 IsTailCall &= !hasSwiftErrorArg();
10306
10307 CLI.setDebugLoc(dl)
10308 .setChain(Chain)
10309 .setLibCallee(
10310 CC: Libcalls->getLibcallImplCallingConv(Call: MemmoveImpl),
10311 ResultType: Dst.getValueType().getTypeForEVT(Context&: *getContext()),
10312 Target: getExternalSymbol(Libcall: MemmoveImpl, VT: TLI->getPointerTy(DL: getDataLayout())),
10313 ArgsList: std::move(Args))
10314 .setDiscardResult()
10315 .setTailCall(IsTailCall);
10316
10317 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
10318 return CallResult.second;
10319}
10320
10321SDValue SelectionDAG::getAtomicMemmove(SDValue Chain, const SDLoc &dl,
10322 SDValue Dst, SDValue Src, SDValue Size,
10323 Type *SizeTy, unsigned ElemSz,
10324 bool isTailCall,
10325 MachinePointerInfo DstPtrInfo,
10326 MachinePointerInfo SrcPtrInfo) {
10327 // Lowering doesn't support tail calling inside a function with a
10328 // swifterror argument yet.
10329 isTailCall &= !hasSwiftErrorArg();
10330
10331 // Emit a library call.
10332 TargetLowering::ArgListTy Args;
10333 Type *IntPtrTy = getDataLayout().getIntPtrType(C&: *getContext());
10334 Args.emplace_back(args&: Dst, args&: IntPtrTy);
10335 Args.emplace_back(args&: Src, args&: IntPtrTy);
10336 Args.emplace_back(args&: Size, args&: SizeTy);
10337
10338 RTLIB::Libcall LibraryCall =
10339 RTLIB::getMEMMOVE_ELEMENT_UNORDERED_ATOMIC(ElementSize: ElemSz);
10340 RTLIB::LibcallImpl LibcallImpl = Libcalls->getLibcallImpl(Call: LibraryCall);
10341 if (LibcallImpl == RTLIB::Unsupported)
10342 report_fatal_error(reason: "Unsupported element size");
10343
10344 TargetLowering::CallLoweringInfo CLI(*this);
10345 CLI.setDebugLoc(dl)
10346 .setChain(Chain)
10347 .setLibCallee(
10348 CC: Libcalls->getLibcallImplCallingConv(Call: LibcallImpl),
10349 ResultType: Type::getVoidTy(C&: *getContext()),
10350 Target: getExternalSymbol(Libcall: LibcallImpl, VT: TLI->getPointerTy(DL: getDataLayout())),
10351 ArgsList: std::move(Args))
10352 .setDiscardResult()
10353 .setTailCall(isTailCall);
10354
10355 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
10356 return CallResult.second;
10357}
10358
10359SDValue SelectionDAG::getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst,
10360 SDValue Src, SDValue Size, Align Alignment,
10361 bool isVol, bool AlwaysInline,
10362 const CallInst *CI,
10363 MachinePointerInfo DstPtrInfo,
10364 const AAMDNodes &AAInfo) {
10365 // Check to see if we should lower the memset to stores first.
10366 // For cases within the target-specified limits, this is the best choice.
10367 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Val&: Size);
10368 if (ConstantSize) {
10369 // Memset with size zero? Just return the original chain.
10370 if (ConstantSize->isZero())
10371 return Chain;
10372
10373 SDValue Result = getMemsetStores(DAG&: *this, dl, Chain, Dst, Src,
10374 Size: ConstantSize->getZExtValue(), Alignment,
10375 isVol, AlwaysInline: false, DstPtrInfo, AAInfo);
10376
10377 if (Result.getNode())
10378 return Result;
10379 }
10380
10381 // Then check to see if we should lower the memset with target-specific
10382 // code. If the target chooses to do this, this is the next best.
10383 if (TSI) {
10384 SDValue Result = TSI->EmitTargetCodeForMemset(
10385 DAG&: *this, dl, Chain, Op1: Dst, Op2: Src, Op3: Size, Alignment, isVolatile: isVol, AlwaysInline, DstPtrInfo);
10386 if (Result.getNode())
10387 return Result;
10388 }
10389
10390 // If we really need inline code and the target declined to provide it,
10391 // use a (potentially long) sequence of loads and stores.
10392 if (AlwaysInline) {
10393 assert(ConstantSize && "AlwaysInline requires a constant size!");
10394 SDValue Result = getMemsetStores(DAG&: *this, dl, Chain, Dst, Src,
10395 Size: ConstantSize->getZExtValue(), Alignment,
10396 isVol, AlwaysInline: true, DstPtrInfo, AAInfo);
10397 assert(Result &&
10398 "getMemsetStores must return a valid sequence when AlwaysInline");
10399 return Result;
10400 }
10401
10402 checkAddrSpaceIsValidForLibcall(TLI, AS: DstPtrInfo.getAddrSpace());
10403
10404 // Emit a library call.
10405 auto &Ctx = *getContext();
10406 const auto& DL = getDataLayout();
10407
10408 TargetLowering::CallLoweringInfo CLI(*this);
10409 // FIXME: pass in SDLoc
10410 CLI.setDebugLoc(dl).setChain(Chain);
10411
10412 RTLIB::LibcallImpl BzeroImpl = Libcalls->getLibcallImpl(Call: RTLIB::BZERO);
10413 bool UseBZero = BzeroImpl != RTLIB::Unsupported && isNullConstant(V: Src);
10414
10415 // If zeroing out and bzero is present, use it.
10416 if (UseBZero) {
10417 TargetLowering::ArgListTy Args;
10418 Args.emplace_back(args&: Dst, args: PointerType::getUnqual(C&: Ctx));
10419 Args.emplace_back(args&: Size, args: DL.getIntPtrType(C&: Ctx));
10420 CLI.setLibCallee(
10421 CC: Libcalls->getLibcallImplCallingConv(Call: BzeroImpl), ResultType: Type::getVoidTy(C&: Ctx),
10422 Target: getExternalSymbol(Libcall: BzeroImpl, VT: TLI->getPointerTy(DL)), ArgsList: std::move(Args));
10423 } else {
10424 RTLIB::LibcallImpl MemsetImpl = Libcalls->getLibcallImpl(Call: RTLIB::MEMSET);
10425
10426 TargetLowering::ArgListTy Args;
10427 Args.emplace_back(args&: Dst, args: PointerType::getUnqual(C&: Ctx));
10428 Args.emplace_back(args&: Src, args: Src.getValueType().getTypeForEVT(Context&: Ctx));
10429 Args.emplace_back(args&: Size, args: DL.getIntPtrType(C&: Ctx));
10430 CLI.setLibCallee(CC: Libcalls->getLibcallImplCallingConv(Call: MemsetImpl),
10431 ResultType: Dst.getValueType().getTypeForEVT(Context&: Ctx),
10432 Target: getExternalSymbol(Libcall: MemsetImpl, VT: TLI->getPointerTy(DL)),
10433 ArgsList: std::move(Args));
10434 }
10435
10436 RTLIB::LibcallImpl MemsetImpl = Libcalls->getLibcallImpl(Call: RTLIB::MEMSET);
10437 bool LowersToMemset = MemsetImpl == RTLIB::impl_memset;
10438
10439 // If we're going to use bzero, make sure not to tail call unless the
10440 // subsequent return doesn't need a value, as bzero doesn't return the first
10441 // arg unlike memset.
10442 bool ReturnsFirstArg = CI && funcReturnsFirstArgOfCall(CI: *CI) && !UseBZero;
10443 bool IsTailCall = CI && CI->isTailCall() &&
10444 isInTailCallPosition(Call: *CI, TM: getTarget(),
10445 ReturnsFirstArg: ReturnsFirstArg && LowersToMemset) &&
10446 // Lowering doesn't support tail calling inside a function
10447 // with a swifterror argument yet.
10448 !hasSwiftErrorArg();
10449 CLI.setDiscardResult().setTailCall(IsTailCall);
10450
10451 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
10452 return CallResult.second;
10453}
10454
10455SDValue SelectionDAG::getAtomicMemset(SDValue Chain, const SDLoc &dl,
10456 SDValue Dst, SDValue Value, SDValue Size,
10457 Type *SizeTy, unsigned ElemSz,
10458 bool isTailCall,
10459 MachinePointerInfo DstPtrInfo) {
10460 // Lowering doesn't support tail calling inside a function with a
10461 // swifterror argument yet.
10462 isTailCall &= !hasSwiftErrorArg();
10463
10464 // Emit a library call.
10465 TargetLowering::ArgListTy Args;
10466 Args.emplace_back(args&: Dst, args: getDataLayout().getIntPtrType(C&: *getContext()));
10467 Args.emplace_back(args&: Value, args: Type::getInt8Ty(C&: *getContext()));
10468 Args.emplace_back(args&: Size, args&: SizeTy);
10469
10470 RTLIB::Libcall LibraryCall =
10471 RTLIB::getMEMSET_ELEMENT_UNORDERED_ATOMIC(ElementSize: ElemSz);
10472 RTLIB::LibcallImpl LibcallImpl = Libcalls->getLibcallImpl(Call: LibraryCall);
10473 if (LibcallImpl == RTLIB::Unsupported)
10474 report_fatal_error(reason: "Unsupported element size");
10475
10476 TargetLowering::CallLoweringInfo CLI(*this);
10477 CLI.setDebugLoc(dl)
10478 .setChain(Chain)
10479 .setLibCallee(
10480 CC: Libcalls->getLibcallImplCallingConv(Call: LibcallImpl),
10481 ResultType: Type::getVoidTy(C&: *getContext()),
10482 Target: getExternalSymbol(Libcall: LibcallImpl, VT: TLI->getPointerTy(DL: getDataLayout())),
10483 ArgsList: std::move(Args))
10484 .setDiscardResult()
10485 .setTailCall(isTailCall);
10486
10487 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
10488 return CallResult.second;
10489}
10490
10491SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
10492 SDVTList VTList, ArrayRef<SDValue> Ops,
10493 MachineMemOperand *MMO,
10494 ISD::LoadExtType ExtType) {
10495 SDNodeKey ID(Opcode, VTList, Ops);
10496 ID.AddInteger(I: MemVT.getRawBits());
10497 ID.AddInteger(I: getSyntheticNodeSubclassData<AtomicSDNode>(
10498 IROrder: dl.getIROrder(), Args&: Opcode, Args&: VTList, Args&: MemVT, Args&: MMO, Args&: ExtType));
10499 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10500 ID.AddInteger(I: MMO->getFlags());
10501 FoldingSetInsertToken InsertToken;
10502 if (auto *E = cast_or_null<AtomicSDNode>(Val: lookupNode(Key: ID, DL: dl, InsertToken))) {
10503 E->refineAlignment(NewMMO: MMO);
10504 E->refineMMOMetadata(NewMMO: MMO);
10505 return SDValue(E, 0);
10506 }
10507
10508 auto *N = newSDNode<AtomicSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: Opcode,
10509 Args&: VTList, Args&: MemVT, Args&: MMO, Args&: ExtType);
10510 createOperands(Node: N, Vals: Ops);
10511
10512 CSEMap.insert(N, Token: InsertToken);
10513 InsertNode(N);
10514 SDValue V(N, 0);
10515 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10516 return V;
10517}
10518
10519SDValue SelectionDAG::getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl,
10520 EVT MemVT, SDVTList VTs, SDValue Chain,
10521 SDValue Ptr, SDValue Cmp, SDValue Swp,
10522 MachineMemOperand *MMO) {
10523 assert(Opcode == ISD::ATOMIC_CMP_SWAP ||
10524 Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
10525 assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
10526
10527 SDValue Ops[] = {Chain, Ptr, Cmp, Swp};
10528 return getAtomic(Opcode, dl, MemVT, VTList: VTs, Ops, MMO);
10529}
10530
10531SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
10532 SDValue Chain, SDValue Ptr, SDValue Val,
10533 MachineMemOperand *MMO) {
10534 assert((Opcode == ISD::ATOMIC_LOAD_ADD || Opcode == ISD::ATOMIC_LOAD_SUB ||
10535 Opcode == ISD::ATOMIC_LOAD_AND || Opcode == ISD::ATOMIC_LOAD_CLR ||
10536 Opcode == ISD::ATOMIC_LOAD_OR || Opcode == ISD::ATOMIC_LOAD_XOR ||
10537 Opcode == ISD::ATOMIC_LOAD_NAND || Opcode == ISD::ATOMIC_LOAD_MIN ||
10538 Opcode == ISD::ATOMIC_LOAD_MAX || Opcode == ISD::ATOMIC_LOAD_UMIN ||
10539 Opcode == ISD::ATOMIC_LOAD_UMAX || Opcode == ISD::ATOMIC_LOAD_FADD ||
10540 Opcode == ISD::ATOMIC_LOAD_FSUB || Opcode == ISD::ATOMIC_LOAD_FMAX ||
10541 Opcode == ISD::ATOMIC_LOAD_FMIN ||
10542 Opcode == ISD::ATOMIC_LOAD_FMINIMUM ||
10543 Opcode == ISD::ATOMIC_LOAD_FMAXIMUM ||
10544 Opcode == ISD::ATOMIC_LOAD_UINC_WRAP ||
10545 Opcode == ISD::ATOMIC_LOAD_UDEC_WRAP ||
10546 Opcode == ISD::ATOMIC_LOAD_USUB_COND ||
10547 Opcode == ISD::ATOMIC_LOAD_USUB_SAT || Opcode == ISD::ATOMIC_SWAP ||
10548 Opcode == ISD::ATOMIC_STORE) &&
10549 "Invalid Atomic Op");
10550
10551 EVT VT = Val.getValueType();
10552
10553 SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(VT: MVT::Other) :
10554 getVTList(VT1: VT, VT2: MVT::Other);
10555 SDValue Ops[] = {Chain, Ptr, Val};
10556 return getAtomic(Opcode, dl, MemVT, VTList: VTs, Ops, MMO);
10557}
10558
10559SDValue SelectionDAG::getAtomicLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
10560 EVT MemVT, EVT VT, SDValue Chain,
10561 SDValue Ptr, MachineMemOperand *MMO) {
10562 SDVTList VTs = getVTList(VT1: VT, VT2: MVT::Other);
10563 SDValue Ops[] = {Chain, Ptr};
10564 return getAtomic(Opcode: ISD::ATOMIC_LOAD, dl, MemVT, VTList: VTs, Ops, MMO, ExtType);
10565}
10566
10567/// getMergeValues - Create a MERGE_VALUES node from the given operands.
10568SDValue SelectionDAG::getMergeValues(ArrayRef<SDValue> Ops, const SDLoc &dl) {
10569 if (Ops.size() == 1)
10570 return Ops[0];
10571
10572 SmallVector<EVT, 4> VTs;
10573 VTs.reserve(N: Ops.size());
10574 for (const SDValue &Op : Ops)
10575 VTs.push_back(Elt: Op.getValueType());
10576 return getNode(Opcode: ISD::MERGE_VALUES, DL: dl, VTList: getVTList(VTs), Ops);
10577}
10578
10579SDValue SelectionDAG::getErrorMergeValues(ArrayRef<EVT> ResultTypes,
10580 SDValue Chain, const SDLoc &dl) {
10581 SmallVector<SDValue, 4> RetValues;
10582 RetValues.reserve(N: ResultTypes.size());
10583 for (EVT VT : ResultTypes)
10584 RetValues.push_back(Elt: VT == MVT::Other ? Chain : getPOISON(VT));
10585 return getMergeValues(Ops: RetValues, dl);
10586}
10587
10588SDValue SelectionDAG::getMemIntrinsicNode(
10589 unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef<SDValue> Ops,
10590 EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment,
10591 MachineMemOperand::Flags Flags, LocationSize Size,
10592 const AAMDNodes &AAInfo) {
10593 if (Size.hasValue() && !Size.getValue())
10594 Size = LocationSize::precise(Value: MemVT.getStoreSize());
10595
10596 MachineFunction &MF = getMachineFunction();
10597 MachineMemOperand *MMO =
10598 MF.getMachineMemOperand(PtrInfo, F: Flags, Size, BaseAlignment: Alignment, Metadata: AAInfo);
10599
10600 return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, MMO);
10601}
10602
10603SDValue SelectionDAG::getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl,
10604 SDVTList VTList,
10605 ArrayRef<SDValue> Ops, EVT MemVT,
10606 MachineMemOperand *MMO) {
10607 return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, MMOs: ArrayRef(MMO));
10608}
10609
10610SDValue SelectionDAG::getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl,
10611 SDVTList VTList,
10612 ArrayRef<SDValue> Ops, EVT MemVT,
10613 ArrayRef<MachineMemOperand *> MMOs) {
10614 assert(!MMOs.empty() && "Must have at least one MMO");
10615 assert(
10616 (Opcode == ISD::INTRINSIC_VOID || Opcode == ISD::INTRINSIC_W_CHAIN ||
10617 Opcode == ISD::PREFETCH ||
10618 (Opcode <= (unsigned)std::numeric_limits<int>::max() &&
10619 Opcode >= ISD::BUILTIN_OP_END && TSI->isTargetMemoryOpcode(Opcode))) &&
10620 "Opcode is not a memory-accessing opcode!");
10621
10622 PointerUnion<MachineMemOperand *, MachineMemOperand **> MemRefs;
10623 if (MMOs.size() == 1) {
10624 MemRefs = MMOs[0];
10625 } else {
10626 // Allocate: [size_t count][MMO*][MMO*]...
10627 size_t AllocSize =
10628 sizeof(size_t) + MMOs.size() * sizeof(MachineMemOperand *);
10629 void *Buffer = Allocator.Allocate(Size: AllocSize, Alignment: alignof(size_t));
10630 size_t *CountPtr = static_cast<size_t *>(Buffer);
10631 *CountPtr = MMOs.size();
10632 MachineMemOperand **Array =
10633 reinterpret_cast<MachineMemOperand **>(CountPtr + 1);
10634 llvm::copy(Range&: MMOs, Out: Array);
10635 MemRefs = Array;
10636 }
10637
10638 // Memoize the node unless it returns a glue result.
10639 MemIntrinsicSDNode *N;
10640 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
10641 SDNodeKey ID(Opcode, VTList, Ops);
10642 ID.AddInteger(I: getSyntheticNodeSubclassData<MemIntrinsicSDNode>(
10643 Opc: Opcode, Order: dl.getIROrder(), VTs: VTList, MemoryVT: MemVT, MemRefs));
10644 ID.AddInteger(I: MemVT.getRawBits());
10645 for (const MachineMemOperand *MMO : MMOs) {
10646 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10647 ID.AddInteger(I: MMO->getFlags());
10648 }
10649 FoldingSetInsertToken InsertToken;
10650 if (SDNode *E = lookupNode(Key: ID, DL: dl, InsertToken)) {
10651 cast<MemIntrinsicSDNode>(Val: E)->refineAlignment(NewMMOs: MMOs);
10652 return SDValue(E, 0);
10653 }
10654
10655 N = newSDNode<MemIntrinsicSDNode>(Args&: Opcode, Args: dl.getIROrder(), Args: dl.getDebugLoc(),
10656 Args&: VTList, Args&: MemVT, Args&: MemRefs);
10657 createOperands(Node: N, Vals: Ops);
10658 CSEMap.insert(N, Token: InsertToken);
10659 } else {
10660 N = newSDNode<MemIntrinsicSDNode>(Args&: Opcode, Args: dl.getIROrder(), Args: dl.getDebugLoc(),
10661 Args&: VTList, Args&: MemVT, Args&: MemRefs);
10662 createOperands(Node: N, Vals: Ops);
10663 }
10664 InsertNode(N);
10665 SDValue V(N, 0);
10666 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10667 return V;
10668}
10669
10670SDValue SelectionDAG::getLifetimeNode(bool IsStart, const SDLoc &dl,
10671 SDValue Chain, int FrameIndex) {
10672 const unsigned Opcode = IsStart ? ISD::LIFETIME_START : ISD::LIFETIME_END;
10673 const auto VTs = getVTList(VT: MVT::Other);
10674 SDValue Ops[2] = {
10675 Chain,
10676 getFrameIndex(FI: FrameIndex,
10677 VT: getTargetLoweringInfo().getFrameIndexTy(DL: getDataLayout()),
10678 isTarget: true)};
10679
10680 SDNodeKey ID(Opcode, VTs, Ops);
10681 FoldingSetInsertToken InsertToken;
10682 if (SDNode *E = lookupNode(Key: ID, DL: dl, InsertToken))
10683 return SDValue(E, 0);
10684
10685 LifetimeSDNode *N =
10686 newSDNode<LifetimeSDNode>(Args: Opcode, Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args: VTs);
10687 createOperands(Node: N, Vals: Ops);
10688 CSEMap.insert(N, Token: InsertToken);
10689 InsertNode(N);
10690 SDValue V(N, 0);
10691 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10692 return V;
10693}
10694
10695SDValue SelectionDAG::getPseudoProbeNode(const SDLoc &Dl, SDValue Chain,
10696 uint64_t Guid, uint64_t Index,
10697 uint32_t Attr) {
10698 const unsigned Opcode = ISD::PSEUDO_PROBE;
10699 const auto VTs = getVTList(VT: MVT::Other);
10700 SDValue Ops[] = {Chain};
10701 SDNodeKey ID(Opcode, VTs, Ops);
10702 ID.AddInteger(I: Guid);
10703 ID.AddInteger(I: Index);
10704 ID.AddInteger(I: Attr);
10705 FoldingSetInsertToken InsertToken;
10706 if (SDNode *E = lookupNode(Key: ID, DL: Dl, InsertToken))
10707 return SDValue(E, 0);
10708
10709 auto *N = newSDNode<PseudoProbeSDNode>(
10710 Args: Opcode, Args: Dl.getIROrder(), Args: Dl.getDebugLoc(), Args: VTs, Args&: Guid, Args&: Index, Args&: Attr);
10711 createOperands(Node: N, Vals: Ops);
10712 CSEMap.insert(N, Token: InsertToken);
10713 InsertNode(N);
10714 SDValue V(N, 0);
10715 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10716 return V;
10717}
10718
10719/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
10720/// MachinePointerInfo record from it. This is particularly useful because the
10721/// code generator has many cases where it doesn't bother passing in a
10722/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
10723static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info,
10724 SelectionDAG &DAG, SDValue Ptr,
10725 int64_t Offset = 0) {
10726 // If this is FI+Offset, we can model it.
10727 if (const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Ptr))
10728 return MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(),
10729 FI: FI->getIndex(), Offset);
10730
10731 // If this is (FI+Offset1)+Offset2, we can model it.
10732 if (Ptr.getOpcode() != ISD::ADD ||
10733 !isa<ConstantSDNode>(Val: Ptr.getOperand(i: 1)) ||
10734 !isa<FrameIndexSDNode>(Val: Ptr.getOperand(i: 0)))
10735 return Info;
10736
10737 int FI = cast<FrameIndexSDNode>(Val: Ptr.getOperand(i: 0))->getIndex();
10738 return MachinePointerInfo::getFixedStack(
10739 MF&: DAG.getMachineFunction(), FI,
10740 Offset: Offset + cast<ConstantSDNode>(Val: Ptr.getOperand(i: 1))->getSExtValue());
10741}
10742
10743/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
10744/// MachinePointerInfo record from it. This is particularly useful because the
10745/// code generator has many cases where it doesn't bother passing in a
10746/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
10747static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info,
10748 SelectionDAG &DAG, SDValue Ptr,
10749 SDValue OffsetOp) {
10750 // If the 'Offset' value isn't a constant, we can't handle this.
10751 if (ConstantSDNode *OffsetNode = dyn_cast<ConstantSDNode>(Val&: OffsetOp))
10752 return InferPointerInfo(Info, DAG, Ptr, Offset: OffsetNode->getSExtValue());
10753 if (OffsetOp.isUndef())
10754 return InferPointerInfo(Info, DAG, Ptr);
10755 return Info;
10756}
10757
10758SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
10759 EVT VT, const SDLoc &dl, SDValue Chain,
10760 SDValue Ptr, SDValue Offset,
10761 MachinePointerInfo PtrInfo, EVT MemVT,
10762 Align Alignment,
10763 MachineMemOperand::Flags MMOFlags,
10764 const MMOMetadata &Metadata) {
10765 assert(Chain.getValueType() == MVT::Other &&
10766 "Invalid chain type");
10767
10768 MMOFlags |= MachineMemOperand::MOLoad;
10769 assert((MMOFlags & MachineMemOperand::MOStore) == 0);
10770 // If we don't have a PtrInfo, infer the trivial frame index case to simplify
10771 // clients.
10772 if (PtrInfo.V.isNull())
10773 PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr, OffsetOp: Offset);
10774
10775 TypeSize Size = MemVT.getStoreSize();
10776 MachineFunction &MF = getMachineFunction();
10777 MachineMemOperand *MMO =
10778 MF.getMachineMemOperand(PtrInfo, F: MMOFlags, Size, BaseAlignment: Alignment, Metadata);
10779 return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO);
10780}
10781
10782SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
10783 EVT VT, const SDLoc &dl, SDValue Chain,
10784 SDValue Ptr, SDValue Offset, EVT MemVT,
10785 MachineMemOperand *MMO) {
10786 if (VT == MemVT) {
10787 ExtType = ISD::NON_EXTLOAD;
10788 } else if (ExtType == ISD::NON_EXTLOAD) {
10789 assert(VT == MemVT && "Non-extending load from different memory type!");
10790 } else {
10791 // Extending load.
10792 assert(MemVT.getScalarType().bitsLT(VT.getScalarType()) &&
10793 "Should only be an extending load, not truncating!");
10794 assert(VT.isInteger() == MemVT.isInteger() &&
10795 "Cannot convert from FP to Int or Int -> FP!");
10796 assert(VT.isVector() == MemVT.isVector() &&
10797 "Cannot use an ext load to convert to or from a vector!");
10798 assert((!VT.isVector() ||
10799 VT.getVectorElementCount() == MemVT.getVectorElementCount()) &&
10800 "Cannot use an ext load to change the number of vector elements!");
10801 }
10802
10803 assert((!MMO->getRanges() ||
10804 (mdconst::extract<ConstantInt>(MMO->getRanges()->getOperand(0))
10805 ->getBitWidth() == MemVT.getScalarSizeInBits() &&
10806 MemVT.isInteger())) &&
10807 "Range metadata and load type must match!");
10808
10809 bool Indexed = AM != ISD::UNINDEXED;
10810 assert((Indexed || Offset.getOpcode() == ISD::POISON) &&
10811 "Unindexed load with an offset!");
10812
10813 SDVTList VTs = Indexed ?
10814 getVTList(VT1: VT, VT2: Ptr.getValueType(), VT3: MVT::Other) : getVTList(VT1: VT, VT2: MVT::Other);
10815 SDValue Ops[] = { Chain, Ptr, Offset };
10816 SDNodeKey ID(ISD::LOAD, VTs, Ops);
10817 ID.AddInteger(I: MemVT.getRawBits());
10818 ID.AddInteger(I: getSyntheticNodeSubclassData<LoadSDNode>(
10819 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: ExtType, Args&: MemVT, Args&: MMO));
10820 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10821 ID.AddInteger(I: MMO->getFlags());
10822 FoldingSetInsertToken InsertToken;
10823 if (auto *E = cast_or_null<LoadSDNode>(Val: lookupNode(Key: ID, DL: dl, InsertToken))) {
10824 E->refineAlignment(NewMMO: MMO);
10825 E->refineMMOMetadata(NewMMO: MMO);
10826 return SDValue(E, 0);
10827 }
10828 auto *N = newSDNode<LoadSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM,
10829 Args&: ExtType, Args&: MemVT, Args&: MMO);
10830 createOperands(Node: N, Vals: Ops);
10831
10832 CSEMap.insert(N, Token: InsertToken);
10833 InsertNode(N);
10834 SDValue V(N, 0);
10835 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10836 return V;
10837}
10838
10839SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain,
10840 SDValue Ptr, MachinePointerInfo PtrInfo,
10841 MaybeAlign Alignment,
10842 MachineMemOperand::Flags MMOFlags,
10843 const MMOMetadata &Metadata) {
10844 SDValue Undef = getPOISON(VT: Ptr.getValueType());
10845 return getLoad(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Offset: Undef,
10846 PtrInfo, MemVT: VT, Alignment, MMOFlags, Metadata);
10847}
10848
10849SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain,
10850 SDValue Ptr, MachineMemOperand *MMO) {
10851 SDValue Undef = getPOISON(VT: Ptr.getValueType());
10852 return getLoad(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Offset: Undef,
10853 MemVT: VT, MMO);
10854}
10855
10856SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
10857 EVT VT, SDValue Chain, SDValue Ptr,
10858 MachinePointerInfo PtrInfo, EVT MemVT,
10859 MaybeAlign Alignment,
10860 MachineMemOperand::Flags MMOFlags,
10861 const MMOMetadata &Metadata) {
10862 SDValue Undef = getPOISON(VT: Ptr.getValueType());
10863 return getLoad(AM: ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Offset: Undef, PtrInfo,
10864 MemVT, Alignment, MMOFlags, Metadata);
10865}
10866
10867SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
10868 EVT VT, SDValue Chain, SDValue Ptr, EVT MemVT,
10869 MachineMemOperand *MMO) {
10870 SDValue Undef = getPOISON(VT: Ptr.getValueType());
10871 return getLoad(AM: ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Offset: Undef,
10872 MemVT, MMO);
10873}
10874
10875SDValue SelectionDAG::getIndexedLoad(SDValue OrigLoad, const SDLoc &dl,
10876 SDValue Base, SDValue Offset,
10877 ISD::MemIndexedMode AM) {
10878 LoadSDNode *LD = cast<LoadSDNode>(Val&: OrigLoad);
10879 assert(LD->getOffset().getOpcode() == ISD::POISON &&
10880 "Load is already a indexed load!");
10881 // Don't propagate the invariant or dereferenceable flags.
10882 auto MMOFlags =
10883 LD->getMemOperand()->getFlags() &
10884 ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
10885 return getLoad(
10886 AM, ExtType: LD->getExtensionType(), VT: OrigLoad.getValueType(), dl, Chain: LD->getChain(),
10887 Ptr: Base, Offset, PtrInfo: LD->getPointerInfo(), MemVT: LD->getMemoryVT(), Alignment: LD->getAlign(),
10888 MMOFlags,
10889 Metadata: MMOMetadata(LD->getAAInfo(), LD->getRanges(), LD->getMemCacheHint()));
10890}
10891
10892SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
10893 SDValue Ptr, MachinePointerInfo PtrInfo,
10894 Align Alignment,
10895 MachineMemOperand::Flags MMOFlags,
10896 const MMOMetadata &Metadata) {
10897 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
10898
10899 MMOFlags |= MachineMemOperand::MOStore;
10900 assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
10901 assert(!Metadata.Ranges && "range metadata is invalid for stores");
10902
10903 if (PtrInfo.V.isNull())
10904 PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr);
10905
10906 MachineFunction &MF = getMachineFunction();
10907 TypeSize Size = Val.getValueType().getStoreSize();
10908 MachineMemOperand *MMO =
10909 MF.getMachineMemOperand(PtrInfo, F: MMOFlags, Size, BaseAlignment: Alignment, Metadata);
10910 return getStore(Chain, dl, Val, Ptr, MMO);
10911}
10912
10913SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
10914 SDValue Ptr, MachineMemOperand *MMO) {
10915 SDValue Undef = getPOISON(VT: Ptr.getValueType());
10916 return getStore(Chain, dl, Val, Ptr, Offset: Undef, SVT: Val.getValueType(), MMO,
10917 AM: ISD::UNINDEXED);
10918}
10919
10920SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
10921 SDValue Ptr, SDValue Offset, EVT SVT,
10922 MachineMemOperand *MMO, ISD::MemIndexedMode AM,
10923 bool IsTruncating) {
10924 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
10925 EVT VT = Val.getValueType();
10926 if (VT == SVT) {
10927 IsTruncating = false;
10928 } else if (!IsTruncating) {
10929 assert(VT == SVT && "No-truncating store from different memory type!");
10930 } else {
10931 assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&
10932 "Should only be a truncating store, not extending!");
10933 assert(VT.isInteger() == SVT.isInteger() && "Can't do FP-INT conversion!");
10934 assert(VT.isVector() == SVT.isVector() &&
10935 "Cannot use trunc store to convert to or from a vector!");
10936 assert((!VT.isVector() ||
10937 VT.getVectorElementCount() == SVT.getVectorElementCount()) &&
10938 "Cannot use trunc store to change the number of vector elements!");
10939 }
10940
10941 bool Indexed = AM != ISD::UNINDEXED;
10942 assert((Indexed || Offset.getOpcode() == ISD::POISON) &&
10943 "Unindexed store with an offset!");
10944 SDVTList VTs = Indexed ? getVTList(VT1: Ptr.getValueType(), VT2: MVT::Other)
10945 : getVTList(VT: MVT::Other);
10946 SDValue Ops[] = {Chain, Val, Ptr, Offset};
10947 SDNodeKey ID(ISD::STORE, VTs, Ops);
10948 ID.AddInteger(I: SVT.getRawBits());
10949 ID.AddInteger(I: getSyntheticNodeSubclassData<StoreSDNode>(
10950 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: IsTruncating, Args&: SVT, Args&: MMO));
10951 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10952 ID.AddInteger(I: MMO->getFlags());
10953 FoldingSetInsertToken InsertToken;
10954 if (SDNode *E = lookupNode(Key: ID, DL: dl, InsertToken)) {
10955 cast<StoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
10956 cast<StoreSDNode>(Val: E)->refineMMOMetadata(NewMMO: MMO);
10957 return SDValue(E, 0);
10958 }
10959 auto *N = newSDNode<StoreSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM,
10960 Args&: IsTruncating, Args&: SVT, Args&: MMO);
10961 createOperands(Node: N, Vals: Ops);
10962
10963 CSEMap.insert(N, Token: InsertToken);
10964 InsertNode(N);
10965 SDValue V(N, 0);
10966 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10967 return V;
10968}
10969
10970SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
10971 SDValue Ptr, SDValue Offset,
10972 MachinePointerInfo PtrInfo, EVT SVT,
10973 Align Alignment,
10974 MachineMemOperand::Flags MMOFlags,
10975 const MMOMetadata &Metadata) {
10976 assert(Chain.getValueType() == MVT::Other &&
10977 "Invalid chain type");
10978
10979 MMOFlags |= MachineMemOperand::MOStore;
10980 assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
10981 assert(!Metadata.Ranges && "range metadata is invalid for stores");
10982
10983 if (PtrInfo.V.isNull())
10984 PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr);
10985
10986 MachineFunction &MF = getMachineFunction();
10987 MachineMemOperand *MMO = MF.getMachineMemOperand(
10988 PtrInfo, F: MMOFlags, Size: SVT.getStoreSize(), BaseAlignment: Alignment, Metadata);
10989 return getTruncStore(Chain, dl, Val, Ptr, Offset, SVT, MMO);
10990}
10991
10992SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
10993 SDValue Ptr, MachinePointerInfo PtrInfo,
10994 EVT SVT, Align Alignment,
10995 MachineMemOperand::Flags MMOFlags,
10996 const MMOMetadata &Metadata) {
10997 return getTruncStore(Chain, dl, Val, Ptr, Offset: getPOISON(VT: Ptr.getValueType()),
10998 PtrInfo, SVT, Alignment, MMOFlags, Metadata);
10999}
11000
11001SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
11002 SDValue Ptr, SDValue Offset, EVT SVT,
11003 MachineMemOperand *MMO) {
11004 return getStore(Chain, dl, Val, Ptr, Offset, SVT, MMO, AM: ISD::UNINDEXED, IsTruncating: true);
11005}
11006
11007SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
11008 SDValue Ptr, EVT SVT,
11009 MachineMemOperand *MMO) {
11010 return getStore(Chain, dl, Val, Ptr, Offset: getPOISON(VT: Ptr.getValueType()), SVT, MMO,
11011 AM: ISD::UNINDEXED, IsTruncating: true);
11012}
11013
11014SDValue SelectionDAG::getIndexedStore(SDValue OrigStore, const SDLoc &dl,
11015 SDValue Base, SDValue Offset,
11016 ISD::MemIndexedMode AM) {
11017 StoreSDNode *ST = cast<StoreSDNode>(Val&: OrigStore);
11018 assert(ST->getOffset().getOpcode() == ISD::POISON &&
11019 "Store is already a indexed store!");
11020 return getStore(Chain: ST->getChain(), dl, Val: ST->getValue(), Ptr: Base, Offset,
11021 SVT: ST->getMemoryVT(), MMO: ST->getMemOperand(), AM,
11022 IsTruncating: ST->isTruncatingStore());
11023}
11024
11025SDValue SelectionDAG::getLoadVP(
11026 ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &dl,
11027 SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL,
11028 MachinePointerInfo PtrInfo, EVT MemVT, Align Alignment,
11029 MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo,
11030 const MDNode *Ranges, bool IsExpanding) {
11031 MMOFlags |= MachineMemOperand::MOLoad;
11032 assert((MMOFlags & MachineMemOperand::MOStore) == 0);
11033 // If we don't have a PtrInfo, infer the trivial frame index case to simplify
11034 // clients.
11035 if (PtrInfo.V.isNull())
11036 PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr, OffsetOp: Offset);
11037
11038 TypeSize Size = MemVT.getStoreSize();
11039 MachineFunction &MF = getMachineFunction();
11040 MachineMemOperand *MMO = MF.getMachineMemOperand(
11041 PtrInfo, F: MMOFlags, Size, BaseAlignment: Alignment, Metadata: MMOMetadata(AAInfo, Ranges));
11042 return getLoadVP(AM, ExtType, VT, dl, Chain, Ptr, Offset, Mask, EVL, MemVT,
11043 MMO, IsExpanding);
11044}
11045
11046SDValue SelectionDAG::getLoadVP(ISD::MemIndexedMode AM,
11047 ISD::LoadExtType ExtType, EVT VT,
11048 const SDLoc &dl, SDValue Chain, SDValue Ptr,
11049 SDValue Offset, SDValue Mask, SDValue EVL,
11050 EVT MemVT, MachineMemOperand *MMO,
11051 bool IsExpanding) {
11052 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
11053 assert(Mask.getValueType().getVectorElementCount() ==
11054 VT.getVectorElementCount() &&
11055 "Vector width mismatch between mask and data");
11056
11057 bool Indexed = AM != ISD::UNINDEXED;
11058 assert((Indexed || Offset.getOpcode() == ISD::POISON) &&
11059 "Unindexed load with an offset!");
11060
11061 SDVTList VTs = Indexed ? getVTList(VT1: VT, VT2: Ptr.getValueType(), VT3: MVT::Other)
11062 : getVTList(VT1: VT, VT2: MVT::Other);
11063 SDValue Ops[] = {Chain, Ptr, Offset, Mask, EVL};
11064 SDNodeKey ID(ISD::VP_LOAD, VTs, Ops);
11065 ID.AddInteger(I: MemVT.getRawBits());
11066 ID.AddInteger(I: getSyntheticNodeSubclassData<VPLoadSDNode>(
11067 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: ExtType, Args&: IsExpanding, Args&: MemVT, Args&: MMO));
11068 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11069 ID.AddInteger(I: MMO->getFlags());
11070 FoldingSetInsertToken InsertToken;
11071 if (auto *E = cast_or_null<VPLoadSDNode>(Val: lookupNode(Key: ID, DL: dl, InsertToken))) {
11072 E->refineAlignment(NewMMO: MMO);
11073 E->refineMMOMetadata(NewMMO: MMO);
11074 return SDValue(E, 0);
11075 }
11076 auto *N = newSDNode<VPLoadSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM,
11077 Args&: ExtType, Args&: IsExpanding, Args&: MemVT, Args&: MMO);
11078 createOperands(Node: N, Vals: Ops);
11079
11080 CSEMap.insert(N, Token: InsertToken);
11081 InsertNode(N);
11082 SDValue V(N, 0);
11083 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11084 return V;
11085}
11086
11087SDValue SelectionDAG::getLoadVP(EVT VT, const SDLoc &dl, SDValue Chain,
11088 SDValue Ptr, SDValue Mask, SDValue EVL,
11089 MachinePointerInfo PtrInfo,
11090 MaybeAlign Alignment,
11091 MachineMemOperand::Flags MMOFlags,
11092 const AAMDNodes &AAInfo, const MDNode *Ranges,
11093 bool IsExpanding) {
11094 SDValue Undef = getPOISON(VT: Ptr.getValueType());
11095 return getLoadVP(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Offset: Undef,
11096 Mask, EVL, PtrInfo, MemVT: VT, Alignment, MMOFlags, AAInfo, Ranges,
11097 IsExpanding);
11098}
11099
11100SDValue SelectionDAG::getLoadVP(EVT VT, const SDLoc &dl, SDValue Chain,
11101 SDValue Ptr, SDValue Mask, SDValue EVL,
11102 MachineMemOperand *MMO, bool IsExpanding) {
11103 SDValue Undef = getPOISON(VT: Ptr.getValueType());
11104 return getLoadVP(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Offset: Undef,
11105 Mask, EVL, MemVT: VT, MMO, IsExpanding);
11106}
11107
11108SDValue SelectionDAG::getExtLoadVP(ISD::LoadExtType ExtType, const SDLoc &dl,
11109 EVT VT, SDValue Chain, SDValue Ptr,
11110 SDValue Mask, SDValue EVL,
11111 MachinePointerInfo PtrInfo, EVT MemVT,
11112 MaybeAlign Alignment,
11113 MachineMemOperand::Flags MMOFlags,
11114 const AAMDNodes &AAInfo, bool IsExpanding) {
11115 SDValue Undef = getPOISON(VT: Ptr.getValueType());
11116 return getLoadVP(AM: ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Offset: Undef, Mask,
11117 EVL, PtrInfo, MemVT, Alignment, MMOFlags, AAInfo, Ranges: nullptr,
11118 IsExpanding);
11119}
11120
11121SDValue SelectionDAG::getExtLoadVP(ISD::LoadExtType ExtType, const SDLoc &dl,
11122 EVT VT, SDValue Chain, SDValue Ptr,
11123 SDValue Mask, SDValue EVL, EVT MemVT,
11124 MachineMemOperand *MMO, bool IsExpanding) {
11125 SDValue Undef = getPOISON(VT: Ptr.getValueType());
11126 return getLoadVP(AM: ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Offset: Undef, Mask,
11127 EVL, MemVT, MMO, IsExpanding);
11128}
11129
11130SDValue SelectionDAG::getStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val,
11131 SDValue Ptr, SDValue Offset, SDValue Mask,
11132 SDValue EVL, EVT MemVT, MachineMemOperand *MMO,
11133 ISD::MemIndexedMode AM, bool IsTruncating,
11134 bool IsCompressing) {
11135 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
11136 assert(Mask.getValueType().getVectorElementCount() ==
11137 Val.getValueType().getVectorElementCount() &&
11138 "Vector width mismatch between mask and data");
11139
11140 bool Indexed = AM != ISD::UNINDEXED;
11141 assert((Indexed || Offset.getOpcode() == ISD::POISON) &&
11142 "Unindexed vp_store with an offset!");
11143 SDVTList VTs = Indexed ? getVTList(VT1: Ptr.getValueType(), VT2: MVT::Other)
11144 : getVTList(VT: MVT::Other);
11145 SDValue Ops[] = {Chain, Val, Ptr, Offset, Mask, EVL};
11146 SDNodeKey ID(ISD::VP_STORE, VTs, Ops);
11147 ID.AddInteger(I: MemVT.getRawBits());
11148 ID.AddInteger(I: getSyntheticNodeSubclassData<VPStoreSDNode>(
11149 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO));
11150 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11151 ID.AddInteger(I: MMO->getFlags());
11152 FoldingSetInsertToken InsertToken;
11153 if (SDNode *E = lookupNode(Key: ID, DL: dl, InsertToken)) {
11154 cast<VPStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11155 return SDValue(E, 0);
11156 }
11157 auto *N = newSDNode<VPStoreSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM,
11158 Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO);
11159 createOperands(Node: N, Vals: Ops);
11160
11161 CSEMap.insert(N, Token: InsertToken);
11162 InsertNode(N);
11163 SDValue V(N, 0);
11164 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11165 return V;
11166}
11167
11168SDValue SelectionDAG::getTruncStoreVP(SDValue Chain, const SDLoc &dl,
11169 SDValue Val, SDValue Ptr, SDValue Mask,
11170 SDValue EVL, MachinePointerInfo PtrInfo,
11171 EVT SVT, Align Alignment,
11172 MachineMemOperand::Flags MMOFlags,
11173 const AAMDNodes &AAInfo,
11174 bool IsCompressing) {
11175 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
11176
11177 MMOFlags |= MachineMemOperand::MOStore;
11178 assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
11179
11180 if (PtrInfo.V.isNull())
11181 PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr);
11182
11183 MachineFunction &MF = getMachineFunction();
11184 MachineMemOperand *MMO = MF.getMachineMemOperand(
11185 PtrInfo, F: MMOFlags, Size: SVT.getStoreSize(), BaseAlignment: Alignment, Metadata: AAInfo);
11186 return getTruncStoreVP(Chain, dl, Val, Ptr, Mask, EVL, SVT, MMO,
11187 IsCompressing);
11188}
11189
11190SDValue SelectionDAG::getTruncStoreVP(SDValue Chain, const SDLoc &dl,
11191 SDValue Val, SDValue Ptr, SDValue Mask,
11192 SDValue EVL, EVT SVT,
11193 MachineMemOperand *MMO,
11194 bool IsCompressing) {
11195 EVT VT = Val.getValueType();
11196
11197 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
11198 if (VT == SVT)
11199 return getStoreVP(Chain, dl, Val, Ptr, Offset: getPOISON(VT: Ptr.getValueType()), Mask,
11200 EVL, MemVT: VT, MMO, AM: ISD::UNINDEXED,
11201 /*IsTruncating*/ false, IsCompressing);
11202
11203 assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&
11204 "Should only be a truncating store, not extending!");
11205 assert(VT.isInteger() == SVT.isInteger() && "Can't do FP-INT conversion!");
11206 assert(VT.isVector() == SVT.isVector() &&
11207 "Cannot use trunc store to convert to or from a vector!");
11208 assert((!VT.isVector() ||
11209 VT.getVectorElementCount() == SVT.getVectorElementCount()) &&
11210 "Cannot use trunc store to change the number of vector elements!");
11211
11212 SDVTList VTs = getVTList(VT: MVT::Other);
11213 SDValue Undef = getPOISON(VT: Ptr.getValueType());
11214 SDValue Ops[] = {Chain, Val, Ptr, Undef, Mask, EVL};
11215 SDNodeKey ID(ISD::VP_STORE, VTs, Ops);
11216 ID.AddInteger(I: SVT.getRawBits());
11217 ID.AddInteger(I: getSyntheticNodeSubclassData<VPStoreSDNode>(
11218 IROrder: dl.getIROrder(), Args&: VTs, Args: ISD::UNINDEXED, Args: true, Args&: IsCompressing, Args&: SVT, Args&: MMO));
11219 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11220 ID.AddInteger(I: MMO->getFlags());
11221 FoldingSetInsertToken InsertToken;
11222 if (SDNode *E = lookupNode(Key: ID, DL: dl, InsertToken)) {
11223 cast<VPStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11224 return SDValue(E, 0);
11225 }
11226 auto *N =
11227 newSDNode<VPStoreSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs,
11228 Args: ISD::UNINDEXED, Args: true, Args&: IsCompressing, Args&: SVT, Args&: MMO);
11229 createOperands(Node: N, Vals: Ops);
11230
11231 CSEMap.insert(N, Token: InsertToken);
11232 InsertNode(N);
11233 SDValue V(N, 0);
11234 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11235 return V;
11236}
11237
11238SDValue SelectionDAG::getStridedLoadVP(
11239 ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &DL,
11240 SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask,
11241 SDValue EVL, EVT MemVT, MachineMemOperand *MMO, bool IsExpanding) {
11242 bool Indexed = AM != ISD::UNINDEXED;
11243 assert((Indexed || Offset.getOpcode() == ISD::POISON) &&
11244 "Unindexed load with an offset!");
11245
11246 SDValue Ops[] = {Chain, Ptr, Offset, Stride, Mask, EVL};
11247 SDVTList VTs = Indexed ? getVTList(VT1: VT, VT2: Ptr.getValueType(), VT3: MVT::Other)
11248 : getVTList(VT1: VT, VT2: MVT::Other);
11249 SDNodeKey ID(ISD::EXPERIMENTAL_VP_STRIDED_LOAD, VTs, Ops);
11250 ID.AddInteger(I: MemVT.getRawBits());
11251 ID.AddInteger(I: getSyntheticNodeSubclassData<VPStridedLoadSDNode>(
11252 IROrder: DL.getIROrder(), Args&: VTs, Args&: AM, Args&: ExtType, Args&: IsExpanding, Args&: MemVT, Args&: MMO));
11253 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11254
11255 FoldingSetInsertToken InsertToken;
11256 if (SDNode *E = lookupNode(Key: ID, DL, InsertToken)) {
11257 cast<VPStridedLoadSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11258 return SDValue(E, 0);
11259 }
11260
11261 auto *N =
11262 newSDNode<VPStridedLoadSDNode>(Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs, Args&: AM,
11263 Args&: ExtType, Args&: IsExpanding, Args&: MemVT, Args&: MMO);
11264 createOperands(Node: N, Vals: Ops);
11265 CSEMap.insert(N, Token: InsertToken);
11266 InsertNode(N);
11267 SDValue V(N, 0);
11268 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11269 return V;
11270}
11271
11272SDValue SelectionDAG::getStridedLoadVP(EVT VT, const SDLoc &DL, SDValue Chain,
11273 SDValue Ptr, SDValue Stride,
11274 SDValue Mask, SDValue EVL,
11275 MachineMemOperand *MMO,
11276 bool IsExpanding) {
11277 SDValue Undef = getPOISON(VT: Ptr.getValueType());
11278 return getStridedLoadVP(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, DL, Chain, Ptr,
11279 Offset: Undef, Stride, Mask, EVL, MemVT: VT, MMO, IsExpanding);
11280}
11281
11282SDValue SelectionDAG::getExtStridedLoadVP(
11283 ISD::LoadExtType ExtType, const SDLoc &DL, EVT VT, SDValue Chain,
11284 SDValue Ptr, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT,
11285 MachineMemOperand *MMO, bool IsExpanding) {
11286 SDValue Undef = getPOISON(VT: Ptr.getValueType());
11287 return getStridedLoadVP(AM: ISD::UNINDEXED, ExtType, VT, DL, Chain, Ptr, Offset: Undef,
11288 Stride, Mask, EVL, MemVT, MMO, IsExpanding);
11289}
11290
11291SDValue SelectionDAG::getStridedStoreVP(SDValue Chain, const SDLoc &DL,
11292 SDValue Val, SDValue Ptr,
11293 SDValue Offset, SDValue Stride,
11294 SDValue Mask, SDValue EVL, EVT MemVT,
11295 MachineMemOperand *MMO,
11296 ISD::MemIndexedMode AM,
11297 bool IsTruncating, bool IsCompressing) {
11298 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
11299 bool Indexed = AM != ISD::UNINDEXED;
11300 assert((Indexed || Offset.getOpcode() == ISD::POISON) &&
11301 "Unindexed vp_store with an offset!");
11302 SDVTList VTs = Indexed ? getVTList(VT1: Ptr.getValueType(), VT2: MVT::Other)
11303 : getVTList(VT: MVT::Other);
11304 SDValue Ops[] = {Chain, Val, Ptr, Offset, Stride, Mask, EVL};
11305 SDNodeKey ID(ISD::EXPERIMENTAL_VP_STRIDED_STORE, VTs, Ops);
11306 ID.AddInteger(I: MemVT.getRawBits());
11307 ID.AddInteger(I: getSyntheticNodeSubclassData<VPStridedStoreSDNode>(
11308 IROrder: DL.getIROrder(), Args&: VTs, Args&: AM, Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO));
11309 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11310 FoldingSetInsertToken InsertToken;
11311 if (SDNode *E = lookupNode(Key: ID, DL, InsertToken)) {
11312 cast<VPStridedStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11313 return SDValue(E, 0);
11314 }
11315 auto *N = newSDNode<VPStridedStoreSDNode>(Args: DL.getIROrder(), Args: DL.getDebugLoc(),
11316 Args&: VTs, Args&: AM, Args&: IsTruncating,
11317 Args&: IsCompressing, Args&: MemVT, Args&: MMO);
11318 createOperands(Node: N, Vals: Ops);
11319
11320 CSEMap.insert(N, Token: InsertToken);
11321 InsertNode(N);
11322 SDValue V(N, 0);
11323 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11324 return V;
11325}
11326
11327SDValue SelectionDAG::getGatherVP(SDVTList VTs, EVT VT, const SDLoc &dl,
11328 ArrayRef<SDValue> Ops, MachineMemOperand *MMO,
11329 ISD::MemIndexType IndexType) {
11330 assert(Ops.size() == 6 && "Incompatible number of operands");
11331
11332 SDNodeKey ID(ISD::VP_GATHER, VTs, Ops);
11333 ID.AddInteger(I: VT.getRawBits());
11334 ID.AddInteger(I: getSyntheticNodeSubclassData<VPGatherSDNode>(
11335 IROrder: dl.getIROrder(), Args&: VTs, Args&: VT, Args&: MMO, Args&: IndexType));
11336 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11337 ID.AddInteger(I: MMO->getFlags());
11338 FoldingSetInsertToken InsertToken;
11339 if (SDNode *E = lookupNode(Key: ID, DL: dl, InsertToken)) {
11340 cast<VPGatherSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11341 return SDValue(E, 0);
11342 }
11343
11344 auto *N = newSDNode<VPGatherSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs,
11345 Args&: VT, Args&: MMO, Args&: IndexType);
11346 createOperands(Node: N, Vals: Ops);
11347
11348 assert(N->getMask().getValueType().getVectorElementCount() ==
11349 N->getValueType(0).getVectorElementCount() &&
11350 "Vector width mismatch between mask and data");
11351 assert(N->getIndex().getValueType().getVectorElementCount().isScalable() ==
11352 N->getValueType(0).getVectorElementCount().isScalable() &&
11353 "Scalable flags of index and data do not match");
11354 assert(ElementCount::isKnownGE(
11355 N->getIndex().getValueType().getVectorElementCount(),
11356 N->getValueType(0).getVectorElementCount()) &&
11357 "Vector width mismatch between index and data");
11358 assert(isa<ConstantSDNode>(N->getScale()) &&
11359 N->getScale()->getAsAPIntVal().isPowerOf2() &&
11360 "Scale should be a constant power of 2");
11361
11362 CSEMap.insert(N, Token: InsertToken);
11363 InsertNode(N);
11364 SDValue V(N, 0);
11365 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11366 return V;
11367}
11368
11369SDValue SelectionDAG::getScatterVP(SDVTList VTs, EVT VT, const SDLoc &dl,
11370 ArrayRef<SDValue> Ops,
11371 MachineMemOperand *MMO,
11372 ISD::MemIndexType IndexType) {
11373 assert(Ops.size() == 7 && "Incompatible number of operands");
11374
11375 SDNodeKey ID(ISD::VP_SCATTER, VTs, Ops);
11376 ID.AddInteger(I: VT.getRawBits());
11377 ID.AddInteger(I: getSyntheticNodeSubclassData<VPScatterSDNode>(
11378 IROrder: dl.getIROrder(), Args&: VTs, Args&: VT, Args&: MMO, Args&: IndexType));
11379 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11380 ID.AddInteger(I: MMO->getFlags());
11381 FoldingSetInsertToken InsertToken;
11382 if (SDNode *E = lookupNode(Key: ID, DL: dl, InsertToken)) {
11383 cast<VPScatterSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11384 return SDValue(E, 0);
11385 }
11386 auto *N = newSDNode<VPScatterSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs,
11387 Args&: VT, Args&: MMO, Args&: IndexType);
11388 createOperands(Node: N, Vals: Ops);
11389
11390 assert(N->getMask().getValueType().getVectorElementCount() ==
11391 N->getValue().getValueType().getVectorElementCount() &&
11392 "Vector width mismatch between mask and data");
11393 assert(
11394 N->getIndex().getValueType().getVectorElementCount().isScalable() ==
11395 N->getValue().getValueType().getVectorElementCount().isScalable() &&
11396 "Scalable flags of index and data do not match");
11397 assert(ElementCount::isKnownGE(
11398 N->getIndex().getValueType().getVectorElementCount(),
11399 N->getValue().getValueType().getVectorElementCount()) &&
11400 "Vector width mismatch between index and data");
11401 assert(isa<ConstantSDNode>(N->getScale()) &&
11402 N->getScale()->getAsAPIntVal().isPowerOf2() &&
11403 "Scale should be a constant power of 2");
11404
11405 CSEMap.insert(N, Token: InsertToken);
11406 InsertNode(N);
11407 SDValue V(N, 0);
11408 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11409 return V;
11410}
11411
11412SDValue SelectionDAG::getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain,
11413 SDValue Base, SDValue Offset, SDValue Mask,
11414 SDValue PassThru, EVT MemVT,
11415 MachineMemOperand *MMO,
11416 ISD::MemIndexedMode AM,
11417 ISD::LoadExtType ExtTy, bool isExpanding) {
11418 bool Indexed = AM != ISD::UNINDEXED;
11419 assert((Indexed || Offset.getOpcode() == ISD::POISON) &&
11420 "Unindexed masked load with an offset!");
11421 SDVTList VTs = Indexed ? getVTList(VT1: VT, VT2: Base.getValueType(), VT3: MVT::Other)
11422 : getVTList(VT1: VT, VT2: MVT::Other);
11423 SDValue Ops[] = {Chain, Base, Offset, Mask, PassThru};
11424 SDNodeKey ID(ISD::MLOAD, VTs, Ops);
11425 ID.AddInteger(I: MemVT.getRawBits());
11426 ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedLoadSDNode>(
11427 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: ExtTy, Args&: isExpanding, Args&: MemVT, Args&: MMO));
11428 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11429 ID.AddInteger(I: MMO->getFlags());
11430 FoldingSetInsertToken InsertToken;
11431 if (SDNode *E = lookupNode(Key: ID, DL: dl, InsertToken)) {
11432 cast<MaskedLoadSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11433 return SDValue(E, 0);
11434 }
11435 auto *N = newSDNode<MaskedLoadSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs,
11436 Args&: AM, Args&: ExtTy, Args&: isExpanding, Args&: MemVT, Args&: MMO);
11437 createOperands(Node: N, Vals: Ops);
11438
11439 CSEMap.insert(N, Token: InsertToken);
11440 InsertNode(N);
11441 SDValue V(N, 0);
11442 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11443 return V;
11444}
11445
11446SDValue SelectionDAG::getIndexedMaskedLoad(SDValue OrigLoad, const SDLoc &dl,
11447 SDValue Base, SDValue Offset,
11448 ISD::MemIndexedMode AM) {
11449 MaskedLoadSDNode *LD = cast<MaskedLoadSDNode>(Val&: OrigLoad);
11450 assert(LD->getOffset().getOpcode() == ISD::POISON &&
11451 "Masked load is already a indexed load!");
11452 return getMaskedLoad(VT: OrigLoad.getValueType(), dl, Chain: LD->getChain(), Base,
11453 Offset, Mask: LD->getMask(), PassThru: LD->getPassThru(),
11454 MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand(), AM,
11455 ExtTy: LD->getExtensionType(), isExpanding: LD->isExpandingLoad());
11456}
11457
11458SDValue SelectionDAG::getMaskedStore(SDValue Chain, const SDLoc &dl,
11459 SDValue Val, SDValue Base, SDValue Offset,
11460 SDValue Mask, EVT MemVT,
11461 MachineMemOperand *MMO,
11462 ISD::MemIndexedMode AM, bool IsTruncating,
11463 bool IsCompressing) {
11464 assert(Chain.getValueType() == MVT::Other &&
11465 "Invalid chain type");
11466 bool Indexed = AM != ISD::UNINDEXED;
11467 assert((Indexed || Offset.getOpcode() == ISD::POISON) &&
11468 "Unindexed masked store with an offset!");
11469 SDVTList VTs = Indexed ? getVTList(VT1: Base.getValueType(), VT2: MVT::Other)
11470 : getVTList(VT: MVT::Other);
11471 SDValue Ops[] = {Chain, Val, Base, Offset, Mask};
11472 SDNodeKey ID(ISD::MSTORE, VTs, Ops);
11473 ID.AddInteger(I: MemVT.getRawBits());
11474 ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedStoreSDNode>(
11475 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO));
11476 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11477 ID.AddInteger(I: MMO->getFlags());
11478 FoldingSetInsertToken InsertToken;
11479 if (SDNode *E = lookupNode(Key: ID, DL: dl, InsertToken)) {
11480 cast<MaskedStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11481 return SDValue(E, 0);
11482 }
11483 auto *N =
11484 newSDNode<MaskedStoreSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM,
11485 Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO);
11486 createOperands(Node: N, Vals: Ops);
11487
11488 CSEMap.insert(N, Token: InsertToken);
11489 InsertNode(N);
11490 SDValue V(N, 0);
11491 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11492 return V;
11493}
11494
11495SDValue SelectionDAG::getIndexedMaskedStore(SDValue OrigStore, const SDLoc &dl,
11496 SDValue Base, SDValue Offset,
11497 ISD::MemIndexedMode AM) {
11498 MaskedStoreSDNode *ST = cast<MaskedStoreSDNode>(Val&: OrigStore);
11499 assert(ST->getOffset().getOpcode() == ISD::POISON &&
11500 "Masked store is already a indexed store!");
11501 return getMaskedStore(Chain: ST->getChain(), dl, Val: ST->getValue(), Base, Offset,
11502 Mask: ST->getMask(), MemVT: ST->getMemoryVT(), MMO: ST->getMemOperand(),
11503 AM, IsTruncating: ST->isTruncatingStore(), IsCompressing: ST->isCompressingStore());
11504}
11505
11506SDValue SelectionDAG::getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl,
11507 ArrayRef<SDValue> Ops,
11508 MachineMemOperand *MMO,
11509 ISD::MemIndexType IndexType,
11510 ISD::LoadExtType ExtTy) {
11511 assert(Ops.size() == 6 && "Incompatible number of operands");
11512
11513 SDNodeKey ID(ISD::MGATHER, VTs, Ops);
11514 ID.AddInteger(I: MemVT.getRawBits());
11515 ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedGatherSDNode>(
11516 IROrder: dl.getIROrder(), Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType, Args&: ExtTy));
11517 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11518 ID.AddInteger(I: MMO->getFlags());
11519 FoldingSetInsertToken InsertToken;
11520 if (SDNode *E = lookupNode(Key: ID, DL: dl, InsertToken)) {
11521 cast<MaskedGatherSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11522 return SDValue(E, 0);
11523 }
11524
11525 auto *N = newSDNode<MaskedGatherSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(),
11526 Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType, Args&: ExtTy);
11527 createOperands(Node: N, Vals: Ops);
11528
11529 assert(N->getPassThru().getValueType() == N->getValueType(0) &&
11530 "Incompatible type of the PassThru value in MaskedGatherSDNode");
11531 assert(N->getMask().getValueType().getVectorElementCount() ==
11532 N->getValueType(0).getVectorElementCount() &&
11533 "Vector width mismatch between mask and data");
11534 assert(N->getIndex().getValueType().getVectorElementCount().isScalable() ==
11535 N->getValueType(0).getVectorElementCount().isScalable() &&
11536 "Scalable flags of index and data do not match");
11537 assert(ElementCount::isKnownGE(
11538 N->getIndex().getValueType().getVectorElementCount(),
11539 N->getValueType(0).getVectorElementCount()) &&
11540 "Vector width mismatch between index and data");
11541 assert(isa<ConstantSDNode>(N->getScale()) &&
11542 N->getScale()->getAsAPIntVal().isPowerOf2() &&
11543 "Scale should be a constant power of 2");
11544
11545 CSEMap.insert(N, Token: InsertToken);
11546 InsertNode(N);
11547 SDValue V(N, 0);
11548 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11549 return V;
11550}
11551
11552SDValue SelectionDAG::getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl,
11553 ArrayRef<SDValue> Ops,
11554 MachineMemOperand *MMO,
11555 ISD::MemIndexType IndexType,
11556 bool IsTrunc) {
11557 assert(Ops.size() == 6 && "Incompatible number of operands");
11558
11559 SDNodeKey ID(ISD::MSCATTER, VTs, Ops);
11560 ID.AddInteger(I: MemVT.getRawBits());
11561 ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedScatterSDNode>(
11562 IROrder: dl.getIROrder(), Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType, Args&: IsTrunc));
11563 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11564 ID.AddInteger(I: MMO->getFlags());
11565 FoldingSetInsertToken InsertToken;
11566 if (SDNode *E = lookupNode(Key: ID, DL: dl, InsertToken)) {
11567 cast<MaskedScatterSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11568 return SDValue(E, 0);
11569 }
11570
11571 auto *N = newSDNode<MaskedScatterSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(),
11572 Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType, Args&: IsTrunc);
11573 createOperands(Node: N, Vals: Ops);
11574
11575 assert(N->getMask().getValueType().getVectorElementCount() ==
11576 N->getValue().getValueType().getVectorElementCount() &&
11577 "Vector width mismatch between mask and data");
11578 assert(
11579 N->getIndex().getValueType().getVectorElementCount().isScalable() ==
11580 N->getValue().getValueType().getVectorElementCount().isScalable() &&
11581 "Scalable flags of index and data do not match");
11582 assert(ElementCount::isKnownGE(
11583 N->getIndex().getValueType().getVectorElementCount(),
11584 N->getValue().getValueType().getVectorElementCount()) &&
11585 "Vector width mismatch between index and data");
11586 assert(isa<ConstantSDNode>(N->getScale()) &&
11587 N->getScale()->getAsAPIntVal().isPowerOf2() &&
11588 "Scale should be a constant power of 2");
11589
11590 CSEMap.insert(N, Token: InsertToken);
11591 InsertNode(N);
11592 SDValue V(N, 0);
11593 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11594 return V;
11595}
11596
11597SDValue SelectionDAG::getMaskedHistogram(SDVTList VTs, EVT MemVT,
11598 const SDLoc &dl, ArrayRef<SDValue> Ops,
11599 MachineMemOperand *MMO,
11600 ISD::MemIndexType IndexType) {
11601 assert(Ops.size() == 7 && "Incompatible number of operands");
11602
11603 SDNodeKey ID(ISD::EXPERIMENTAL_VECTOR_HISTOGRAM, VTs, Ops);
11604 ID.AddInteger(I: MemVT.getRawBits());
11605 ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedHistogramSDNode>(
11606 IROrder: dl.getIROrder(), Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType));
11607 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11608 ID.AddInteger(I: MMO->getFlags());
11609 FoldingSetInsertToken InsertToken;
11610 if (SDNode *E = lookupNode(Key: ID, DL: dl, InsertToken)) {
11611 cast<MaskedGatherSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11612 return SDValue(E, 0);
11613 }
11614
11615 auto *N = newSDNode<MaskedHistogramSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(),
11616 Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType);
11617 createOperands(Node: N, Vals: Ops);
11618
11619 assert(N->getMask().getValueType().getVectorElementCount() ==
11620 N->getIndex().getValueType().getVectorElementCount() &&
11621 "Vector width mismatch between mask and data");
11622 assert(isa<ConstantSDNode>(N->getScale()) &&
11623 N->getScale()->getAsAPIntVal().isPowerOf2() &&
11624 "Scale should be a constant power of 2");
11625 assert(N->getInc().getValueType().isInteger() && "Non integer update value");
11626
11627 CSEMap.insert(N, Token: InsertToken);
11628 InsertNode(N);
11629 SDValue V(N, 0);
11630 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11631 return V;
11632}
11633
11634SDValue SelectionDAG::getLoadFFVP(EVT VT, const SDLoc &DL, SDValue Chain,
11635 SDValue Ptr, SDValue Mask, SDValue EVL,
11636 MachineMemOperand *MMO) {
11637 SDVTList VTs = getVTList(VT1: VT, VT2: EVL.getValueType(), VT3: MVT::Other);
11638 SDValue Ops[] = {Chain, Ptr, Mask, EVL};
11639 SDNodeKey ID(ISD::VP_LOAD_FF, VTs, Ops);
11640 ID.AddInteger(I: VT.getRawBits());
11641 ID.AddInteger(I: getSyntheticNodeSubclassData<VPLoadFFSDNode>(IROrder: DL.getIROrder(),
11642 Args&: VTs, Args&: VT, Args&: MMO));
11643 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11644 ID.AddInteger(I: MMO->getFlags());
11645 FoldingSetInsertToken InsertToken;
11646 if (SDNode *E = lookupNode(Key: ID, DL, InsertToken)) {
11647 cast<VPLoadFFSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11648 return SDValue(E, 0);
11649 }
11650 auto *N = newSDNode<VPLoadFFSDNode>(Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs,
11651 Args&: VT, Args&: MMO);
11652 createOperands(Node: N, Vals: Ops);
11653
11654 CSEMap.insert(N, Token: InsertToken);
11655 InsertNode(N);
11656 SDValue V(N, 0);
11657 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11658 return V;
11659}
11660
11661SDValue SelectionDAG::getGetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr,
11662 EVT MemVT, MachineMemOperand *MMO) {
11663 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
11664 SDVTList VTs = getVTList(VT: MVT::Other);
11665 SDValue Ops[] = {Chain, Ptr};
11666 SDNodeKey ID(ISD::GET_FPENV_MEM, VTs, Ops);
11667 ID.AddInteger(I: MemVT.getRawBits());
11668 ID.AddInteger(I: getSyntheticNodeSubclassData<FPStateAccessSDNode>(
11669 Opc: ISD::GET_FPENV_MEM, Order: dl.getIROrder(), VTs, MemoryVT: MemVT, MMO));
11670 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11671 ID.AddInteger(I: MMO->getFlags());
11672 FoldingSetInsertToken InsertToken;
11673 if (SDNode *E = lookupNode(Key: ID, DL: dl, InsertToken))
11674 return SDValue(E, 0);
11675
11676 auto *N = newSDNode<FPStateAccessSDNode>(Args: ISD::GET_FPENV_MEM, Args: dl.getIROrder(),
11677 Args: dl.getDebugLoc(), Args&: VTs, Args&: MemVT, Args&: MMO);
11678 createOperands(Node: N, Vals: Ops);
11679
11680 CSEMap.insert(N, Token: InsertToken);
11681 InsertNode(N);
11682 SDValue V(N, 0);
11683 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11684 return V;
11685}
11686
11687SDValue SelectionDAG::getSetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr,
11688 EVT MemVT, MachineMemOperand *MMO) {
11689 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
11690 SDVTList VTs = getVTList(VT: MVT::Other);
11691 SDValue Ops[] = {Chain, Ptr};
11692 SDNodeKey ID(ISD::SET_FPENV_MEM, VTs, Ops);
11693 ID.AddInteger(I: MemVT.getRawBits());
11694 ID.AddInteger(I: getSyntheticNodeSubclassData<FPStateAccessSDNode>(
11695 Opc: ISD::SET_FPENV_MEM, Order: dl.getIROrder(), VTs, MemoryVT: MemVT, MMO));
11696 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11697 ID.AddInteger(I: MMO->getFlags());
11698 FoldingSetInsertToken InsertToken;
11699 if (SDNode *E = lookupNode(Key: ID, DL: dl, InsertToken))
11700 return SDValue(E, 0);
11701
11702 auto *N = newSDNode<FPStateAccessSDNode>(Args: ISD::SET_FPENV_MEM, Args: dl.getIROrder(),
11703 Args: dl.getDebugLoc(), Args&: VTs, Args&: MemVT, Args&: MMO);
11704 createOperands(Node: N, Vals: Ops);
11705
11706 CSEMap.insert(N, Token: InsertToken);
11707 InsertNode(N);
11708 SDValue V(N, 0);
11709 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11710 return V;
11711}
11712
11713SDValue SelectionDAG::simplifySelect(SDValue Cond, SDValue T, SDValue F) {
11714 // select undef, T, F --> T (if T is a constant), otherwise F
11715 // select, ?, undef, F --> F
11716 // select, ?, T, undef --> T
11717 if (Cond.isUndef())
11718 return isConstantValueOfAnyType(N: T) ? T : F;
11719 if (T.isUndef())
11720 return isGuaranteedNotToBePoison(Op: F) ? F : getFreeze(V: F);
11721 if (F.isUndef())
11722 return isGuaranteedNotToBePoison(Op: T) ? T : getFreeze(V: T);
11723
11724 // select true, T, F --> T
11725 // select false, T, F --> F
11726 if (auto C = isBoolConstant(N: Cond))
11727 return *C ? T : F;
11728
11729 // select ?, T, T --> T
11730 if (T == F)
11731 return T;
11732
11733 return SDValue();
11734}
11735
11736SDValue SelectionDAG::simplifyShift(SDValue X, SDValue Y) {
11737 // shift undef, Y --> 0 (can always assume that the undef value is 0)
11738 if (X.isUndef())
11739 return getConstant(Val: 0, DL: SDLoc(X.getNode()), VT: X.getValueType());
11740 // shift X, undef --> undef (because it may shift by the bitwidth)
11741 if (Y.isUndef())
11742 return getUNDEF(VT: X.getValueType());
11743
11744 // shift 0, Y --> 0
11745 // shift X, 0 --> X
11746 if (isNullOrNullSplat(V: X) || isNullOrNullSplat(V: Y))
11747 return X;
11748
11749 // shift X, C >= bitwidth(X) --> undef
11750 // All vector elements must be too big (or undef) to avoid partial undefs.
11751 auto isShiftTooBig = [X](ConstantSDNode *Val) {
11752 return !Val || Val->getAPIntValue().uge(RHS: X.getScalarValueSizeInBits());
11753 };
11754 if (ISD::matchUnaryPredicate(Op: Y, Match: isShiftTooBig, AllowUndefs: true))
11755 return getUNDEF(VT: X.getValueType());
11756
11757 // shift i1/vXi1 X, Y --> X (any non-zero shift amount is undefined).
11758 if (X.getValueType().getScalarType() == MVT::i1)
11759 return X;
11760
11761 return SDValue();
11762}
11763
11764SDValue SelectionDAG::simplifyFPBinop(unsigned Opcode, SDValue X, SDValue Y,
11765 SDNodeFlags Flags) {
11766 // If this operation has 'nnan' or 'ninf' and at least 1 disallowed operand
11767 // (an undef operand can be chosen to be Nan/Inf), then the result of this
11768 // operation is poison. That result can be relaxed to undef.
11769 ConstantFPSDNode *XC = isConstOrConstSplatFP(N: X, /* AllowUndefs */ true);
11770 ConstantFPSDNode *YC = isConstOrConstSplatFP(N: Y, /* AllowUndefs */ true);
11771 bool HasNan = (XC && XC->getValueAPF().isNaN()) ||
11772 (YC && YC->getValueAPF().isNaN());
11773 bool HasInf = (XC && XC->getValueAPF().isInfinity()) ||
11774 (YC && YC->getValueAPF().isInfinity());
11775
11776 if (Flags.hasNoNaNs() && (HasNan || X.isUndef() || Y.isUndef()))
11777 return getUNDEF(VT: X.getValueType());
11778
11779 if (Flags.hasNoInfs() && (HasInf || X.isUndef() || Y.isUndef()))
11780 return getUNDEF(VT: X.getValueType());
11781
11782 if (!YC)
11783 return SDValue();
11784
11785 // X + -0.0 --> X
11786 if (Opcode == ISD::FADD)
11787 if (YC->getValueAPF().isNegZero())
11788 return X;
11789
11790 // X - +0.0 --> X
11791 if (Opcode == ISD::FSUB)
11792 if (YC->getValueAPF().isPosZero())
11793 return X;
11794
11795 // X * 1.0 --> X
11796 // X / 1.0 --> X
11797 if (Opcode == ISD::FMUL || Opcode == ISD::FDIV)
11798 if (YC->getValueAPF().isOne())
11799 return X;
11800
11801 // X * 0.0 --> 0.0
11802 if (Opcode == ISD::FMUL && Flags.hasNoNaNs() && Flags.hasNoSignedZeros())
11803 if (YC->getValueAPF().isZero())
11804 return getConstantFP(Val: 0.0, DL: SDLoc(Y), VT: Y.getValueType());
11805
11806 return SDValue();
11807}
11808
11809SDValue SelectionDAG::getVAArg(EVT VT, const SDLoc &dl, SDValue Chain,
11810 SDValue Ptr, SDValue SV, unsigned Align) {
11811 SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Val: Align, DL: dl, VT: MVT::i32) };
11812 return getNode(Opcode: ISD::VAARG, DL: dl, VTList: getVTList(VT1: VT, VT2: MVT::Other), Ops);
11813}
11814
11815SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
11816 ArrayRef<SDUse> Ops) {
11817 switch (Ops.size()) {
11818 case 0: return getNode(Opcode, DL, VT);
11819 case 1: return getNode(Opcode, DL, VT, N1: Ops[0].get());
11820 case 2: return getNode(Opcode, DL, VT, N1: Ops[0], N2: Ops[1]);
11821 case 3: return getNode(Opcode, DL, VT, N1: Ops[0], N2: Ops[1], N3: Ops[2]);
11822 default: break;
11823 }
11824
11825 // Copy from an SDUse array into an SDValue array for use with
11826 // the regular getNode logic.
11827 SmallVector<SDValue, 8> NewOps(Ops);
11828 return getNode(Opcode, DL, VT, Ops: NewOps);
11829}
11830
11831SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
11832 ArrayRef<SDValue> Ops) {
11833 SDNodeFlags Flags;
11834 if (Inserter)
11835 Flags = Inserter->getFlags();
11836 return getNode(Opcode, DL, VT, Ops, Flags);
11837}
11838
11839SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
11840 ArrayRef<SDValue> Ops, const SDNodeFlags Flags) {
11841 unsigned NumOps = Ops.size();
11842 switch (NumOps) {
11843 case 0: return getNode(Opcode, DL, VT);
11844 case 1: return getNode(Opcode, DL, VT, N1: Ops[0], Flags);
11845 case 2: return getNode(Opcode, DL, VT, N1: Ops[0], N2: Ops[1], Flags);
11846 case 3: return getNode(Opcode, DL, VT, N1: Ops[0], N2: Ops[1], N3: Ops[2], Flags);
11847 default: break;
11848 }
11849
11850#ifndef NDEBUG
11851 for (const auto &Op : Ops)
11852 assert(Op.getOpcode() != ISD::DELETED_NODE &&
11853 "Operand is DELETED_NODE!");
11854#endif
11855
11856 switch (Opcode) {
11857 default: break;
11858 case ISD::BUILD_VECTOR:
11859 // Attempt to simplify BUILD_VECTOR.
11860 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, DAG&: *this))
11861 return V;
11862 break;
11863 case ISD::CONCAT_VECTORS:
11864 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, DAG&: *this))
11865 return V;
11866 break;
11867 case ISD::SELECT_CC:
11868 assert(NumOps == 5 && "SELECT_CC takes 5 operands!");
11869 assert(Ops[0].getValueType() == Ops[1].getValueType() &&
11870 "LHS and RHS of condition must have same type!");
11871 assert(Ops[2].getValueType() == Ops[3].getValueType() &&
11872 "True and False arms of SelectCC must have same type!");
11873 assert(Ops[2].getValueType() == VT &&
11874 "select_cc node must be of same type as true and false value!");
11875 assert((!Ops[0].getValueType().isVector() ||
11876 Ops[0].getValueType().getVectorElementCount() ==
11877 VT.getVectorElementCount()) &&
11878 "Expected select_cc with vector result to have the same sized "
11879 "comparison type!");
11880 break;
11881 case ISD::BR_CC:
11882 assert(NumOps == 5 && "BR_CC takes 5 operands!");
11883 assert(Ops[2].getValueType() == Ops[3].getValueType() &&
11884 "LHS/RHS of comparison should match types!");
11885 break;
11886 case ISD::VP_REDUCE_MUL:
11887 // If it is VP_REDUCE_MUL mask operation then turn it to VP_REDUCE_AND
11888 if (VT == MVT::i1)
11889 Opcode = ISD::VP_REDUCE_AND;
11890 break;
11891 case ISD::VP_REDUCE_ADD:
11892 // If it is VP_REDUCE_ADD mask operation then turn it to VP_REDUCE_XOR
11893 if (VT == MVT::i1)
11894 Opcode = ISD::VP_REDUCE_XOR;
11895 break;
11896 case ISD::VP_REDUCE_SMAX:
11897 case ISD::VP_REDUCE_UMIN:
11898 // If it is VP_REDUCE_SMAX/VP_REDUCE_UMIN mask operation then turn it to
11899 // VP_REDUCE_AND.
11900 if (VT == MVT::i1)
11901 Opcode = ISD::VP_REDUCE_AND;
11902 break;
11903 case ISD::VP_REDUCE_SMIN:
11904 case ISD::VP_REDUCE_UMAX:
11905 // If it is VP_REDUCE_SMIN/VP_REDUCE_UMAX mask operation then turn it to
11906 // VP_REDUCE_OR.
11907 if (VT == MVT::i1)
11908 Opcode = ISD::VP_REDUCE_OR;
11909 break;
11910 }
11911
11912 // Memoize nodes.
11913 SDNode *N;
11914 SDVTList VTs = getVTList(VT);
11915
11916 if (VT != MVT::Glue) {
11917 SDNodeKey ID(Opcode, VTs, Ops);
11918 FoldingSetInsertToken InsertToken;
11919
11920 if (SDNode *E = lookupNode(Key: ID, DL, InsertToken)) {
11921 E->intersectFlagsWith(Flags);
11922 return SDValue(E, 0);
11923 }
11924
11925 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
11926 createOperands(Node: N, Vals: Ops);
11927
11928 CSEMap.insert(N, Token: InsertToken);
11929 } else {
11930 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
11931 createOperands(Node: N, Vals: Ops);
11932 }
11933
11934 N->setFlags(Flags);
11935 InsertNode(N);
11936 SDValue V(N, 0);
11937 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11938 return V;
11939}
11940
11941SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
11942 ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops) {
11943 SDNodeFlags Flags;
11944 if (Inserter)
11945 Flags = Inserter->getFlags();
11946 return getNode(Opcode, DL, VTList: getVTList(VTs: ResultTys), Ops, Flags);
11947}
11948
11949SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
11950 ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops,
11951 const SDNodeFlags Flags) {
11952 return getNode(Opcode, DL, VTList: getVTList(VTs: ResultTys), Ops, Flags);
11953}
11954
11955SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
11956 ArrayRef<SDValue> Ops) {
11957 SDNodeFlags Flags;
11958 if (Inserter)
11959 Flags = Inserter->getFlags();
11960 return getNode(Opcode, DL, VTList, Ops, Flags);
11961}
11962
11963SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
11964 ArrayRef<SDValue> Ops, const SDNodeFlags Flags) {
11965 if (VTList.NumVTs == 1)
11966 return getNode(Opcode, DL, VT: VTList.VTs[0], Ops, Flags);
11967
11968#ifndef NDEBUG
11969 for (const auto &Op : Ops)
11970 assert(Op.getOpcode() != ISD::DELETED_NODE &&
11971 "Operand is DELETED_NODE!");
11972#endif
11973
11974 switch (Opcode) {
11975 case ISD::SADDO:
11976 case ISD::UADDO:
11977 case ISD::SSUBO:
11978 case ISD::USUBO: {
11979 assert(VTList.NumVTs == 2 && Ops.size() == 2 &&
11980 "Invalid add/sub overflow op!");
11981 assert(VTList.VTs[0].isInteger() && VTList.VTs[1].isInteger() &&
11982 Ops[0].getValueType() == Ops[1].getValueType() &&
11983 Ops[0].getValueType() == VTList.VTs[0] &&
11984 "Binary operator types must match!");
11985 SDValue N1 = Ops[0], N2 = Ops[1];
11986 canonicalizeCommutativeBinop(Opcode, N1, N2);
11987
11988 // (X +- 0) -> X with zero-overflow.
11989 ConstantSDNode *N2CV = isConstOrConstSplat(N: N2, /*AllowUndefs*/ false,
11990 /*AllowTruncation*/ true);
11991 if (N2CV && N2CV->isZero()) {
11992 SDValue ZeroOverFlow = getConstant(Val: 0, DL, VT: VTList.VTs[1]);
11993 return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList, Ops: {N1, ZeroOverFlow}, Flags);
11994 }
11995
11996 if (VTList.VTs[0].getScalarType() == MVT::i1 &&
11997 VTList.VTs[1].getScalarType() == MVT::i1) {
11998 SDValue F1 = getFreeze(V: N1);
11999 SDValue F2 = getFreeze(V: N2);
12000 // {vXi1,vXi1} (u/s)addo(vXi1 x, vXi1y) -> {xor(x,y),and(x,y)}
12001 if (Opcode == ISD::UADDO || Opcode == ISD::SADDO)
12002 return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList,
12003 Ops: {getNode(Opcode: ISD::XOR, DL, VT: VTList.VTs[0], N1: F1, N2: F2),
12004 getNode(Opcode: ISD::AND, DL, VT: VTList.VTs[1], N1: F1, N2: F2)},
12005 Flags);
12006 // {vXi1,vXi1} (u/s)subo(vXi1 x, vXi1y) -> {xor(x,y),and(~x,y)}
12007 if (Opcode == ISD::USUBO || Opcode == ISD::SSUBO) {
12008 SDValue NotF1 = getNOT(DL, Val: F1, VT: VTList.VTs[0]);
12009 return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList,
12010 Ops: {getNode(Opcode: ISD::XOR, DL, VT: VTList.VTs[0], N1: F1, N2: F2),
12011 getNode(Opcode: ISD::AND, DL, VT: VTList.VTs[1], N1: NotF1, N2: F2)},
12012 Flags);
12013 }
12014 }
12015 break;
12016 }
12017 case ISD::SADDO_CARRY:
12018 case ISD::UADDO_CARRY:
12019 case ISD::SSUBO_CARRY:
12020 case ISD::USUBO_CARRY:
12021 assert(VTList.NumVTs == 2 && Ops.size() == 3 &&
12022 "Invalid add/sub overflow op!");
12023 assert(VTList.VTs[0].isInteger() && VTList.VTs[1].isInteger() &&
12024 Ops[0].getValueType() == Ops[1].getValueType() &&
12025 Ops[0].getValueType() == VTList.VTs[0] &&
12026 Ops[2].getValueType() == VTList.VTs[1] &&
12027 "Binary operator types must match!");
12028 break;
12029 case ISD::SMUL_LOHI:
12030 case ISD::UMUL_LOHI: {
12031 assert(VTList.NumVTs == 2 && Ops.size() == 2 && "Invalid mul lo/hi op!");
12032 assert(VTList.VTs[0].isInteger() && VTList.VTs[0] == VTList.VTs[1] &&
12033 VTList.VTs[0] == Ops[0].getValueType() &&
12034 VTList.VTs[0] == Ops[1].getValueType() &&
12035 "Binary operator types must match!");
12036 // Constant fold.
12037 ConstantSDNode *LHS = dyn_cast<ConstantSDNode>(Val: Ops[0]);
12038 ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Val: Ops[1]);
12039 if (LHS && RHS) {
12040 unsigned Width = VTList.VTs[0].getScalarSizeInBits();
12041 unsigned OutWidth = Width * 2;
12042 APInt Val = LHS->getAPIntValue();
12043 APInt Mul = RHS->getAPIntValue();
12044 if (Opcode == ISD::SMUL_LOHI) {
12045 Val = Val.sext(width: OutWidth);
12046 Mul = Mul.sext(width: OutWidth);
12047 } else {
12048 Val = Val.zext(width: OutWidth);
12049 Mul = Mul.zext(width: OutWidth);
12050 }
12051 Val *= Mul;
12052
12053 SDValue Hi =
12054 getConstant(Val: Val.extractBits(numBits: Width, bitPosition: Width), DL, VT: VTList.VTs[0]);
12055 SDValue Lo = getConstant(Val: Val.trunc(width: Width), DL, VT: VTList.VTs[0]);
12056 return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList, Ops: {Lo, Hi}, Flags);
12057 }
12058 break;
12059 }
12060 case ISD::FFREXP: {
12061 assert(VTList.NumVTs == 2 && Ops.size() == 1 && "Invalid ffrexp op!");
12062 assert(VTList.VTs[0].isFloatingPoint() && VTList.VTs[1].isInteger() &&
12063 VTList.VTs[0] == Ops[0].getValueType() && "frexp type mismatch");
12064
12065 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val: Ops[0])) {
12066 int FrexpExp;
12067 APFloat FrexpMant =
12068 frexp(X: C->getValueAPF(), Exp&: FrexpExp, RM: APFloat::rmNearestTiesToEven);
12069 SDValue Result0 = getConstantFP(V: FrexpMant, DL, VT: VTList.VTs[0]);
12070 SDValue Result1 = getSignedConstant(Val: FrexpMant.isFinite() ? FrexpExp : 0,
12071 DL, VT: VTList.VTs[1]);
12072 return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList, Ops: {Result0, Result1}, Flags);
12073 }
12074
12075 break;
12076 }
12077 case ISD::STRICT_FP_EXTEND:
12078 assert(VTList.NumVTs == 2 && Ops.size() == 2 &&
12079 "Invalid STRICT_FP_EXTEND!");
12080 assert(VTList.VTs[0].isFloatingPoint() &&
12081 Ops[1].getValueType().isFloatingPoint() && "Invalid FP cast!");
12082 assert(VTList.VTs[0].isVector() == Ops[1].getValueType().isVector() &&
12083 "STRICT_FP_EXTEND result type should be vector iff the operand "
12084 "type is vector!");
12085 assert((!VTList.VTs[0].isVector() ||
12086 VTList.VTs[0].getVectorElementCount() ==
12087 Ops[1].getValueType().getVectorElementCount()) &&
12088 "Vector element count mismatch!");
12089 assert(Ops[1].getValueType().bitsLT(VTList.VTs[0]) &&
12090 "Invalid fpext node, dst <= src!");
12091 break;
12092 case ISD::STRICT_FP_ROUND:
12093 assert(VTList.NumVTs == 2 && Ops.size() == 3 && "Invalid STRICT_FP_ROUND!");
12094 assert(VTList.VTs[0].isVector() == Ops[1].getValueType().isVector() &&
12095 "STRICT_FP_ROUND result type should be vector iff the operand "
12096 "type is vector!");
12097 assert((!VTList.VTs[0].isVector() ||
12098 VTList.VTs[0].getVectorElementCount() ==
12099 Ops[1].getValueType().getVectorElementCount()) &&
12100 "Vector element count mismatch!");
12101 assert(VTList.VTs[0].isFloatingPoint() &&
12102 Ops[1].getValueType().isFloatingPoint() &&
12103 VTList.VTs[0].bitsLT(Ops[1].getValueType()) &&
12104 Ops[2].getOpcode() == ISD::TargetConstant &&
12105 (Ops[2]->getAsZExtVal() == 0 || Ops[2]->getAsZExtVal() == 1) &&
12106 "Invalid STRICT_FP_ROUND!");
12107 break;
12108 }
12109
12110 // Memoize the node unless it returns a glue result.
12111 SDNode *N;
12112 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
12113 SDNodeKey ID(Opcode, VTList, Ops);
12114 FoldingSetInsertToken InsertToken;
12115 if (SDNode *E = lookupNode(Key: ID, DL, InsertToken)) {
12116 E->intersectFlagsWith(Flags);
12117 return SDValue(E, 0);
12118 }
12119
12120 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTList);
12121 createOperands(Node: N, Vals: Ops);
12122 CSEMap.insert(N, Token: InsertToken);
12123 } else {
12124 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTList);
12125 createOperands(Node: N, Vals: Ops);
12126 }
12127
12128 N->setFlags(Flags);
12129 InsertNode(N);
12130 SDValue V(N, 0);
12131 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
12132 return V;
12133}
12134
12135SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
12136 SDVTList VTList) {
12137 return getNode(Opcode, DL, VTList, Ops: ArrayRef<SDValue>());
12138}
12139
12140SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
12141 SDValue N1) {
12142 SDValue Ops[] = { N1 };
12143 return getNode(Opcode, DL, VTList, Ops);
12144}
12145
12146SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
12147 SDValue N1, SDValue N2) {
12148 SDValue Ops[] = { N1, N2 };
12149 return getNode(Opcode, DL, VTList, Ops);
12150}
12151
12152SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
12153 SDValue N1, SDValue N2, SDValue N3) {
12154 SDValue Ops[] = { N1, N2, N3 };
12155 return getNode(Opcode, DL, VTList, Ops);
12156}
12157
12158SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
12159 SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
12160 SDValue Ops[] = { N1, N2, N3, N4 };
12161 return getNode(Opcode, DL, VTList, Ops);
12162}
12163
12164SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
12165 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
12166 SDValue N5) {
12167 SDValue Ops[] = { N1, N2, N3, N4, N5 };
12168 return getNode(Opcode, DL, VTList, Ops);
12169}
12170
12171SDVTList SelectionDAG::getVTList(EVT VT) {
12172 if (!VT.isExtended())
12173 return makeVTList(VTs: SDNode::getValueTypeList(VT: VT.getSimpleVT()), NumVTs: 1);
12174
12175 EVT VTs[] = {VT};
12176 return getVTList(VTs);
12177}
12178
12179SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2) {
12180 EVT VTs[] = {VT1, VT2};
12181 return getVTList(VTs);
12182}
12183
12184SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3) {
12185 EVT VTs[] = {VT1, VT2, VT3};
12186 return getVTList(VTs);
12187}
12188
12189SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4) {
12190 EVT VTs[] = {VT1, VT2, VT3, VT4};
12191 return getVTList(VTs);
12192}
12193
12194SDVTList SelectionDAG::getVTList(ArrayRef<EVT> VTs) {
12195 auto It = VTLists.find(V: VTs);
12196 if (It == VTLists.end()) {
12197 EVT *Array = Allocator.Allocate<EVT>(Num: VTs.size());
12198 llvm::copy(Range&: VTs, Out: Array);
12199 It = VTLists.insert(V: ArrayRef(Array, VTs.size())).first;
12200 }
12201 return makeVTList(VTs: It->data(), NumVTs: It->size());
12202}
12203
12204/// UpdateNodeOperands - *Mutate* the specified node in-place to have the
12205/// specified operands. If the resultant node already exists in the DAG,
12206/// this does not modify the specified node, instead it returns the node that
12207/// already exists. If the resultant node does not exist in the DAG, the
12208/// input node is returned. As a degenerate case, if you specify the same
12209/// input operands as the node already has, the input node is returned.
12210SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op) {
12211 assert(N->getNumOperands() == 1 && "Update with wrong number of operands");
12212
12213 // Check to see if there is no change.
12214 if (Op == N->getOperand(Num: 0)) return N;
12215
12216 // See if the modified node already exists.
12217 FoldingSetInsertToken InsertToken;
12218 if (SDNode *Existing = FindModifiedNodeSlot(N, Op, InsertToken))
12219 return Existing;
12220
12221 // Nope it doesn't. Remove the node from its current place in the maps.
12222 if (InsertToken)
12223 if (!RemoveNodeFromCSEMaps(N))
12224 InsertToken = {};
12225
12226 // Now we update the operands.
12227 N->OperandList[0].set(Op);
12228
12229 updateDivergence(N);
12230 // If this gets put into a CSE map, add it.
12231 if (InsertToken)
12232 CSEMap.insert(N, Token: InsertToken);
12233 return N;
12234}
12235
12236SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2) {
12237 assert(N->getNumOperands() == 2 && "Update with wrong number of operands");
12238
12239 // Check to see if there is no change.
12240 if (Op1 == N->getOperand(Num: 0) && Op2 == N->getOperand(Num: 1))
12241 return N; // No operands changed, just return the input node.
12242
12243 // See if the modified node already exists.
12244 FoldingSetInsertToken InsertToken;
12245 if (SDNode *Existing = FindModifiedNodeSlot(N, Op1, Op2, InsertToken))
12246 return Existing;
12247
12248 // Nope it doesn't. Remove the node from its current place in the maps.
12249 if (InsertToken)
12250 if (!RemoveNodeFromCSEMaps(N))
12251 InsertToken = {};
12252
12253 // Now we update the operands.
12254 if (N->OperandList[0] != Op1)
12255 N->OperandList[0].set(Op1);
12256 if (N->OperandList[1] != Op2)
12257 N->OperandList[1].set(Op2);
12258
12259 updateDivergence(N);
12260 // If this gets put into a CSE map, add it.
12261 if (InsertToken)
12262 CSEMap.insert(N, Token: InsertToken);
12263 return N;
12264}
12265
12266SDNode *SelectionDAG::
12267UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, SDValue Op3) {
12268 SDValue Ops[] = { Op1, Op2, Op3 };
12269 return UpdateNodeOperands(N, Ops);
12270}
12271
12272SDNode *SelectionDAG::
12273UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
12274 SDValue Op3, SDValue Op4) {
12275 SDValue Ops[] = { Op1, Op2, Op3, Op4 };
12276 return UpdateNodeOperands(N, Ops);
12277}
12278
12279SDNode *SelectionDAG::
12280UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
12281 SDValue Op3, SDValue Op4, SDValue Op5) {
12282 SDValue Ops[] = { Op1, Op2, Op3, Op4, Op5 };
12283 return UpdateNodeOperands(N, Ops);
12284}
12285
12286SDNode *SelectionDAG::
12287UpdateNodeOperands(SDNode *N, ArrayRef<SDValue> Ops) {
12288 unsigned NumOps = Ops.size();
12289 assert(N->getNumOperands() == NumOps &&
12290 "Update with wrong number of operands");
12291
12292 // If no operands changed just return the input node.
12293 if (std::equal(first1: Ops.begin(), last1: Ops.end(), first2: N->op_begin()))
12294 return N;
12295
12296 // See if the modified node already exists.
12297 FoldingSetInsertToken InsertToken;
12298 if (SDNode *Existing = FindModifiedNodeSlot(N, Ops, InsertToken))
12299 return Existing;
12300
12301 // Nope it doesn't. Remove the node from its current place in the maps.
12302 if (InsertToken)
12303 if (!RemoveNodeFromCSEMaps(N))
12304 InsertToken = {};
12305
12306 // Now we update the operands.
12307 for (unsigned i = 0; i != NumOps; ++i)
12308 if (N->OperandList[i] != Ops[i])
12309 N->OperandList[i].set(Ops[i]);
12310
12311 updateDivergence(N);
12312 // If this gets put into a CSE map, add it.
12313 if (InsertToken)
12314 CSEMap.insert(N, Token: InsertToken);
12315 return N;
12316}
12317
12318/// DropOperands - Release the operands and set this node to have
12319/// zero operands.
12320void SDNode::DropOperands() {
12321 // Unlike the code in MorphNodeTo that does this, we don't need to
12322 // watch for dead nodes here.
12323 for (op_iterator I = op_begin(), E = op_end(); I != E; ) {
12324 SDUse &Use = *I++;
12325 Use.set(SDValue());
12326 }
12327}
12328
12329void SelectionDAG::setNodeMemRefs(MachineSDNode *N,
12330 ArrayRef<MachineMemOperand *> NewMemRefs) {
12331 if (NewMemRefs.empty()) {
12332 N->clearMemRefs();
12333 return;
12334 }
12335
12336 // Check if we can avoid allocating by storing a single reference directly.
12337 if (NewMemRefs.size() == 1) {
12338 N->MemRefs = NewMemRefs[0];
12339 N->NumMemRefs = 1;
12340 return;
12341 }
12342
12343 MachineMemOperand **MemRefsBuffer =
12344 Allocator.template Allocate<MachineMemOperand *>(Num: NewMemRefs.size());
12345 llvm::copy(Range&: NewMemRefs, Out: MemRefsBuffer);
12346 N->MemRefs = MemRefsBuffer;
12347 N->NumMemRefs = static_cast<int>(NewMemRefs.size());
12348}
12349
12350/// SelectNodeTo - These are wrappers around MorphNodeTo that accept a
12351/// machine opcode.
12352///
12353SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12354 EVT VT) {
12355 SDVTList VTs = getVTList(VT);
12356 return SelectNodeTo(N, MachineOpc, VTs, Ops: {});
12357}
12358
12359SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12360 EVT VT, SDValue Op1) {
12361 SDVTList VTs = getVTList(VT);
12362 SDValue Ops[] = { Op1 };
12363 return SelectNodeTo(N, MachineOpc, VTs, Ops);
12364}
12365
12366SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12367 EVT VT, SDValue Op1,
12368 SDValue Op2) {
12369 SDVTList VTs = getVTList(VT);
12370 SDValue Ops[] = { Op1, Op2 };
12371 return SelectNodeTo(N, MachineOpc, VTs, Ops);
12372}
12373
12374SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12375 EVT VT, SDValue Op1,
12376 SDValue Op2, SDValue Op3) {
12377 SDVTList VTs = getVTList(VT);
12378 SDValue Ops[] = { Op1, Op2, Op3 };
12379 return SelectNodeTo(N, MachineOpc, VTs, Ops);
12380}
12381
12382SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12383 EVT VT, ArrayRef<SDValue> Ops) {
12384 SDVTList VTs = getVTList(VT);
12385 return SelectNodeTo(N, MachineOpc, VTs, Ops);
12386}
12387
12388SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12389 EVT VT1, EVT VT2, ArrayRef<SDValue> Ops) {
12390 SDVTList VTs = getVTList(VT1, VT2);
12391 return SelectNodeTo(N, MachineOpc, VTs, Ops);
12392}
12393
12394SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12395 EVT VT1, EVT VT2) {
12396 SDVTList VTs = getVTList(VT1, VT2);
12397 return SelectNodeTo(N, MachineOpc, VTs, Ops: {});
12398}
12399
12400SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12401 EVT VT1, EVT VT2, EVT VT3,
12402 ArrayRef<SDValue> Ops) {
12403 SDVTList VTs = getVTList(VT1, VT2, VT3);
12404 return SelectNodeTo(N, MachineOpc, VTs, Ops);
12405}
12406
12407SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12408 EVT VT1, EVT VT2,
12409 SDValue Op1, SDValue Op2) {
12410 SDVTList VTs = getVTList(VT1, VT2);
12411 SDValue Ops[] = { Op1, Op2 };
12412 return SelectNodeTo(N, MachineOpc, VTs, Ops);
12413}
12414
12415SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12416 SDVTList VTs,ArrayRef<SDValue> Ops) {
12417 SDNode *New = MorphNodeTo(N, Opc: ~MachineOpc, VTs, Ops);
12418 // Reset the NodeID to -1.
12419 New->setNodeId(-1);
12420 if (New != N) {
12421 ReplaceAllUsesWith(From: N, To: New);
12422 RemoveDeadNode(N);
12423 }
12424 return New;
12425}
12426
12427/// UpdateSDLocOnMergeSDNode - If the opt level is -O0 then it throws away
12428/// the line number information on the merged node since it is not possible to
12429/// preserve the information that operation is associated with multiple lines.
12430/// This will make the debugger working better at -O0, were there is a higher
12431/// probability having other instructions associated with that line.
12432///
12433/// For IROrder, we keep the smaller of the two
12434SDNode *SelectionDAG::UpdateSDLocOnMergeSDNode(SDNode *N, const SDLoc &OLoc) {
12435 DebugLoc NLoc = N->getDebugLoc();
12436 if (NLoc && OptLevel == CodeGenOptLevel::None && OLoc.getDebugLoc() != NLoc) {
12437 N->setDebugLoc(DebugLoc());
12438 }
12439 unsigned Order = std::min(a: N->getIROrder(), b: OLoc.getIROrder());
12440 N->setIROrder(Order);
12441 return N;
12442}
12443
12444/// MorphNodeTo - This *mutates* the specified node to have the specified
12445/// return type, opcode, and operands.
12446///
12447/// Note that MorphNodeTo returns the resultant node. If there is already a
12448/// node of the specified opcode and operands, it returns that node instead of
12449/// the current one. Note that the SDLoc need not be the same.
12450///
12451/// Using MorphNodeTo is faster than creating a new node and swapping it in
12452/// with ReplaceAllUsesWith both because it often avoids allocating a new
12453/// node, and because it doesn't require CSE recalculation for any of
12454/// the node's users.
12455///
12456/// However, note that MorphNodeTo recursively deletes dead nodes from the DAG.
12457/// As a consequence it isn't appropriate to use from within the DAG combiner or
12458/// the legalizer which maintain worklists that would need to be updated when
12459/// deleting things.
12460SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
12461 SDVTList VTs, ArrayRef<SDValue> Ops) {
12462 // If an identical node already exists, use it.
12463 FoldingSetInsertToken InsertToken;
12464 if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) {
12465 SDNodeKey ID(Opc, VTs, Ops);
12466 AddNodeIDCustom(ID&: ID.Tail, N, Opc);
12467 if (SDNode *ON = lookupNode(Key: ID, DL: SDLoc(N), InsertToken))
12468 return UpdateSDLocOnMergeSDNode(N: ON, OLoc: SDLoc(N));
12469 }
12470
12471 if (!RemoveNodeFromCSEMaps(N))
12472 InsertToken = {};
12473
12474 // Start the morphing.
12475 N->NodeType = Opc;
12476 N->ValueList = VTs.VTs;
12477 N->NumValues = VTs.NumVTs;
12478
12479 // Clear the operands list, updating used nodes to remove this from their
12480 // use list. Keep track of any operands that become dead as a result.
12481 SmallPtrSet<SDNode*, 16> DeadNodeSet;
12482 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
12483 SDUse &Use = *I++;
12484 SDNode *Used = Use.getNode();
12485 Use.set(SDValue());
12486 if (Used->use_empty())
12487 DeadNodeSet.insert(Ptr: Used);
12488 }
12489
12490 // For MachineNode, initialize the memory references information.
12491 if (MachineSDNode *MN = dyn_cast<MachineSDNode>(Val: N))
12492 MN->clearMemRefs();
12493
12494 // Swap for an appropriately sized array from the recycler.
12495 removeOperands(Node: N);
12496 createOperands(Node: N, Vals: Ops);
12497
12498 // Delete any nodes that are still dead after adding the uses for the
12499 // new operands.
12500 if (!DeadNodeSet.empty()) {
12501 SmallVector<SDNode *, 16> DeadNodes;
12502 for (SDNode *N : DeadNodeSet)
12503 if (N->use_empty())
12504 DeadNodes.push_back(Elt: N);
12505 RemoveDeadNodes(DeadNodes);
12506 }
12507
12508 if (InsertToken)
12509 CSEMap.insert(N, Token: InsertToken); // Memoize the new node.
12510 return N;
12511}
12512
12513SDNode* SelectionDAG::mutateStrictFPToFP(SDNode *Node) {
12514 unsigned OrigOpc = Node->getOpcode();
12515 unsigned NewOpc;
12516 switch (OrigOpc) {
12517 default:
12518 llvm_unreachable("mutateStrictFPToFP called with unexpected opcode!");
12519#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
12520 case ISD::STRICT_##DAGN: NewOpc = ISD::DAGN; break;
12521#define CMP_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
12522 case ISD::STRICT_##DAGN: NewOpc = ISD::SETCC; break;
12523#include "llvm/IR/ConstrainedOps.def"
12524 }
12525
12526 assert(Node->getNumValues() == 2 && "Unexpected number of results!");
12527
12528 // We're taking this node out of the chain, so we need to re-link things.
12529 SDValue InputChain = Node->getOperand(Num: 0);
12530 SDValue OutputChain = SDValue(Node, 1);
12531 ReplaceAllUsesOfValueWith(From: OutputChain, To: InputChain);
12532
12533 SmallVector<SDValue, 3> Ops;
12534 for (unsigned i = 1, e = Node->getNumOperands(); i != e; ++i)
12535 Ops.push_back(Elt: Node->getOperand(Num: i));
12536
12537 SDVTList VTs = getVTList(VT: Node->getValueType(ResNo: 0));
12538 SDNode *Res = MorphNodeTo(N: Node, Opc: NewOpc, VTs, Ops);
12539
12540 // MorphNodeTo can operate in two ways: if an existing node with the
12541 // specified operands exists, it can just return it. Otherwise, it
12542 // updates the node in place to have the requested operands.
12543 if (Res == Node) {
12544 // If we updated the node in place, reset the node ID. To the isel,
12545 // this should be just like a newly allocated machine node.
12546 Res->setNodeId(-1);
12547 } else {
12548 ReplaceAllUsesWith(From: Node, To: Res);
12549 RemoveDeadNode(N: Node);
12550 }
12551
12552 return Res;
12553}
12554
12555/// getMachineNode - These are used for target selectors to create a new node
12556/// with specified return type(s), MachineInstr opcode, and operands.
12557///
12558/// Note that getMachineNode returns the resultant node. If there is already a
12559/// node of the specified opcode and operands, it returns that node instead of
12560/// the current one.
12561MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12562 EVT VT) {
12563 SDVTList VTs = getVTList(VT);
12564 return getMachineNode(Opcode, dl, VTs, Ops: {});
12565}
12566
12567MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12568 EVT VT, SDValue Op1) {
12569 SDVTList VTs = getVTList(VT);
12570 SDValue Ops[] = { Op1 };
12571 return getMachineNode(Opcode, dl, VTs, Ops);
12572}
12573
12574MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12575 EVT VT, SDValue Op1, SDValue Op2) {
12576 SDVTList VTs = getVTList(VT);
12577 SDValue Ops[] = { Op1, Op2 };
12578 return getMachineNode(Opcode, dl, VTs, Ops);
12579}
12580
12581MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12582 EVT VT, SDValue Op1, SDValue Op2,
12583 SDValue Op3) {
12584 SDVTList VTs = getVTList(VT);
12585 SDValue Ops[] = { Op1, Op2, Op3 };
12586 return getMachineNode(Opcode, dl, VTs, Ops);
12587}
12588
12589MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12590 EVT VT, ArrayRef<SDValue> Ops) {
12591 SDVTList VTs = getVTList(VT);
12592 return getMachineNode(Opcode, dl, VTs, Ops);
12593}
12594
12595MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12596 EVT VT1, EVT VT2, SDValue Op1,
12597 SDValue Op2) {
12598 SDVTList VTs = getVTList(VT1, VT2);
12599 SDValue Ops[] = { Op1, Op2 };
12600 return getMachineNode(Opcode, dl, VTs, Ops);
12601}
12602
12603MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12604 EVT VT1, EVT VT2, SDValue Op1,
12605 SDValue Op2, SDValue Op3) {
12606 SDVTList VTs = getVTList(VT1, VT2);
12607 SDValue Ops[] = { Op1, Op2, Op3 };
12608 return getMachineNode(Opcode, dl, VTs, Ops);
12609}
12610
12611MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12612 EVT VT1, EVT VT2,
12613 ArrayRef<SDValue> Ops) {
12614 SDVTList VTs = getVTList(VT1, VT2);
12615 return getMachineNode(Opcode, dl, VTs, Ops);
12616}
12617
12618MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12619 EVT VT1, EVT VT2, EVT VT3,
12620 SDValue Op1, SDValue Op2) {
12621 SDVTList VTs = getVTList(VT1, VT2, VT3);
12622 SDValue Ops[] = { Op1, Op2 };
12623 return getMachineNode(Opcode, dl, VTs, Ops);
12624}
12625
12626MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12627 EVT VT1, EVT VT2, EVT VT3,
12628 SDValue Op1, SDValue Op2,
12629 SDValue Op3) {
12630 SDVTList VTs = getVTList(VT1, VT2, VT3);
12631 SDValue Ops[] = { Op1, Op2, Op3 };
12632 return getMachineNode(Opcode, dl, VTs, Ops);
12633}
12634
12635MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12636 EVT VT1, EVT VT2, EVT VT3,
12637 ArrayRef<SDValue> Ops) {
12638 SDVTList VTs = getVTList(VT1, VT2, VT3);
12639 return getMachineNode(Opcode, dl, VTs, Ops);
12640}
12641
12642MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12643 ArrayRef<EVT> ResultTys,
12644 ArrayRef<SDValue> Ops) {
12645 SDVTList VTs = getVTList(VTs: ResultTys);
12646 return getMachineNode(Opcode, dl, VTs, Ops);
12647}
12648
12649MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &DL,
12650 SDVTList VTs,
12651 ArrayRef<SDValue> Ops) {
12652 bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue;
12653 MachineSDNode *N;
12654 FoldingSetInsertToken InsertToken;
12655
12656 if (DoCSE) {
12657 SDNodeKey ID(~Opcode, VTs, Ops);
12658 if (SDNode *E = lookupNode(Key: ID, DL, InsertToken)) {
12659 return cast<MachineSDNode>(Val: UpdateSDLocOnMergeSDNode(N: E, OLoc: DL));
12660 }
12661 }
12662
12663 // Allocate a new MachineSDNode.
12664 N = newSDNode<MachineSDNode>(Args: ~Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
12665 createOperands(Node: N, Vals: Ops);
12666
12667 if (DoCSE)
12668 CSEMap.insert(N, Token: InsertToken);
12669
12670 InsertNode(N);
12671 NewSDValueDbgMsg(V: SDValue(N, 0), Msg: "Creating new machine node: ", G: this);
12672 return N;
12673}
12674
12675/// getTargetExtractSubreg - A convenience function for creating
12676/// TargetOpcode::EXTRACT_SUBREG nodes.
12677SDValue SelectionDAG::getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT,
12678 SDValue Operand) {
12679 SDValue SRIdxVal = getTargetConstant(Val: SRIdx, DL, VT: MVT::i32);
12680 SDNode *Subreg = getMachineNode(Opcode: TargetOpcode::EXTRACT_SUBREG, dl: DL,
12681 VT, Op1: Operand, Op2: SRIdxVal);
12682 return SDValue(Subreg, 0);
12683}
12684
12685/// getTargetInsertSubreg - A convenience function for creating
12686/// TargetOpcode::INSERT_SUBREG nodes.
12687SDValue SelectionDAG::getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT,
12688 SDValue Operand, SDValue Subreg) {
12689 SDValue SRIdxVal = getTargetConstant(Val: SRIdx, DL, VT: MVT::i32);
12690 SDNode *Result = getMachineNode(Opcode: TargetOpcode::INSERT_SUBREG, dl: DL,
12691 VT, Op1: Operand, Op2: Subreg, Op3: SRIdxVal);
12692 return SDValue(Result, 0);
12693}
12694
12695/// getNodeIfExists - Get the specified node if it's already available, or
12696/// else return NULL.
12697SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,
12698 ArrayRef<SDValue> Ops,
12699 bool AllowCommute) {
12700 SDNodeFlags Flags;
12701 if (Inserter)
12702 Flags = Inserter->getFlags();
12703 return getNodeIfExists(Opcode, VTList, Ops, Flags, AllowCommute);
12704}
12705
12706SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,
12707 ArrayRef<SDValue> Ops,
12708 const SDNodeFlags Flags,
12709 bool AllowCommute) {
12710 if (VTList.VTs[VTList.NumVTs - 1] == MVT::Glue)
12711 return nullptr;
12712
12713 auto Lookup = [&](ArrayRef<SDValue> LookupOps) -> SDNode * {
12714 SDNodeKey ID(Opcode, VTList, LookupOps);
12715 FoldingSetInsertToken InsertToken;
12716 if (SDNode *E = lookupNode(Key: ID, InsertToken)) {
12717 E->intersectFlagsWith(Flags);
12718 return E;
12719 }
12720 return nullptr;
12721 };
12722
12723 if (SDNode *Existing = Lookup(Ops))
12724 return Existing;
12725
12726 if (AllowCommute && TLI->isCommutativeBinOp(Opcode))
12727 return Lookup({Ops[1], Ops[0]});
12728
12729 return nullptr;
12730}
12731
12732/// doesNodeExist - Check if a node exists without modifying its flags.
12733bool SelectionDAG::doesNodeExist(unsigned Opcode, SDVTList VTList,
12734 ArrayRef<SDValue> Ops) {
12735 if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) {
12736 SDNodeKey ID(Opcode, VTList, Ops);
12737 FoldingSetInsertToken InsertToken;
12738 if (lookupNode(Key: ID, DL: SDLoc(), InsertToken))
12739 return true;
12740 }
12741 return false;
12742}
12743
12744/// getDbgValue - Creates a SDDbgValue node.
12745///
12746/// SDNode
12747SDDbgValue *SelectionDAG::getDbgValue(DIVariable *Var, DIExpression *Expr,
12748 SDNode *N, unsigned R, bool IsIndirect,
12749 const DebugLoc &DL, unsigned O) {
12750 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
12751 "Expected inlined-at fields to agree");
12752 return new (DbgInfo->getAlloc())
12753 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromNode(Node: N, ResNo: R),
12754 {}, IsIndirect, DL, O,
12755 /*IsVariadic=*/false);
12756}
12757
12758/// Constant
12759SDDbgValue *SelectionDAG::getConstantDbgValue(DIVariable *Var,
12760 DIExpression *Expr,
12761 const Value *C,
12762 const DebugLoc &DL, unsigned O) {
12763 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
12764 "Expected inlined-at fields to agree");
12765 return new (DbgInfo->getAlloc())
12766 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromConst(Const: C), {},
12767 /*IsIndirect=*/false, DL, O,
12768 /*IsVariadic=*/false);
12769}
12770
12771/// FrameIndex
12772SDDbgValue *SelectionDAG::getFrameIndexDbgValue(DIVariable *Var,
12773 DIExpression *Expr, unsigned FI,
12774 bool IsIndirect,
12775 const DebugLoc &DL,
12776 unsigned O) {
12777 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
12778 "Expected inlined-at fields to agree");
12779 return getFrameIndexDbgValue(Var, Expr, FI, Dependencies: {}, IsIndirect, DL, O);
12780}
12781
12782/// FrameIndex with dependencies
12783SDDbgValue *SelectionDAG::getFrameIndexDbgValue(DIVariable *Var,
12784 DIExpression *Expr, unsigned FI,
12785 ArrayRef<SDNode *> Dependencies,
12786 bool IsIndirect,
12787 const DebugLoc &DL,
12788 unsigned O) {
12789 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
12790 "Expected inlined-at fields to agree");
12791 return new (DbgInfo->getAlloc())
12792 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromFrameIdx(FrameIdx: FI),
12793 Dependencies, IsIndirect, DL, O,
12794 /*IsVariadic=*/false);
12795}
12796
12797/// VReg
12798SDDbgValue *SelectionDAG::getVRegDbgValue(DIVariable *Var, DIExpression *Expr,
12799 Register VReg, bool IsIndirect,
12800 const DebugLoc &DL, unsigned O) {
12801 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
12802 "Expected inlined-at fields to agree");
12803 return new (DbgInfo->getAlloc())
12804 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromVReg(VReg),
12805 {}, IsIndirect, DL, O,
12806 /*IsVariadic=*/false);
12807}
12808
12809SDDbgValue *SelectionDAG::getDbgValueList(DIVariable *Var, DIExpression *Expr,
12810 ArrayRef<SDDbgOperand> Locs,
12811 ArrayRef<SDNode *> Dependencies,
12812 bool IsIndirect, const DebugLoc &DL,
12813 unsigned O, bool IsVariadic) {
12814 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
12815 "Expected inlined-at fields to agree");
12816 return new (DbgInfo->getAlloc())
12817 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, Locs, Dependencies, IsIndirect,
12818 DL, O, IsVariadic);
12819}
12820
12821void SelectionDAG::transferDbgValues(SDValue From, SDValue To,
12822 unsigned OffsetInBits, unsigned SizeInBits,
12823 bool InvalidateDbg) {
12824 SDNode *FromNode = From.getNode();
12825 SDNode *ToNode = To.getNode();
12826 assert(FromNode && ToNode && "Can't modify dbg values");
12827
12828 // PR35338
12829 // TODO: assert(From != To && "Redundant dbg value transfer");
12830 // TODO: assert(FromNode != ToNode && "Intranode dbg value transfer");
12831 if (From == To || FromNode == ToNode)
12832 return;
12833
12834 if (!FromNode->getHasDebugValue())
12835 return;
12836
12837 SDDbgOperand FromLocOp =
12838 SDDbgOperand::fromNode(Node: From.getNode(), ResNo: From.getResNo());
12839 SDDbgOperand ToLocOp = SDDbgOperand::fromNode(Node: To.getNode(), ResNo: To.getResNo());
12840
12841 SmallVector<SDDbgValue *, 2> ClonedDVs;
12842 for (SDDbgValue *Dbg : GetDbgValues(SD: FromNode)) {
12843 if (Dbg->isInvalidated())
12844 continue;
12845
12846 // TODO: assert(!Dbg->isInvalidated() && "Transfer of invalid dbg value");
12847
12848 // Create a new location ops vector that is equal to the old vector, but
12849 // with each instance of FromLocOp replaced with ToLocOp.
12850 bool Changed = false;
12851 auto NewLocOps = Dbg->copyLocationOps();
12852 std::replace_if(
12853 first: NewLocOps.begin(), last: NewLocOps.end(),
12854 pred: [&Changed, FromLocOp](const SDDbgOperand &Op) {
12855 bool Match = Op == FromLocOp;
12856 Changed |= Match;
12857 return Match;
12858 },
12859 new_value: ToLocOp);
12860 // Ignore this SDDbgValue if we didn't find a matching location.
12861 if (!Changed)
12862 continue;
12863
12864 DIVariable *Var = Dbg->getVariable();
12865 auto *Expr = Dbg->getExpression();
12866 // If a fragment is requested, update the expression.
12867 if (SizeInBits) {
12868 // When splitting a larger (e.g., sign-extended) value whose
12869 // lower bits are described with an SDDbgValue, do not attempt
12870 // to transfer the SDDbgValue to the upper bits.
12871 if (auto FI = Expr->getFragmentInfo())
12872 if (OffsetInBits + SizeInBits > FI->SizeInBits)
12873 continue;
12874 auto Fragment = DIExpression::createFragmentExpression(Expr, OffsetInBits,
12875 SizeInBits);
12876 if (!Fragment)
12877 continue;
12878 Expr = *Fragment;
12879 }
12880
12881 auto AdditionalDependencies = Dbg->getAdditionalDependencies();
12882 // Clone the SDDbgValue and move it to To.
12883 SDDbgValue *Clone = getDbgValueList(
12884 Var, Expr, Locs: NewLocOps, Dependencies: AdditionalDependencies, IsIndirect: Dbg->isIndirect(),
12885 DL: Dbg->getDebugLoc(), O: std::max(a: ToNode->getIROrder(), b: Dbg->getOrder()),
12886 IsVariadic: Dbg->isVariadic());
12887 ClonedDVs.push_back(Elt: Clone);
12888
12889 if (InvalidateDbg) {
12890 // Invalidate value and indicate the SDDbgValue should not be emitted.
12891 Dbg->setIsInvalidated();
12892 Dbg->setIsEmitted();
12893 }
12894 }
12895
12896 for (SDDbgValue *Dbg : ClonedDVs) {
12897 assert(is_contained(Dbg->getSDNodes(), ToNode) &&
12898 "Transferred DbgValues should depend on the new SDNode");
12899 AddDbgValue(DB: Dbg, isParameter: false);
12900 }
12901}
12902
12903void SelectionDAG::salvageDebugInfo(SDNode &N) {
12904 if (!N.getHasDebugValue())
12905 return;
12906
12907 auto GetLocationOperand = [](SDNode *Node, unsigned ResNo) {
12908 if (auto *FISDN = dyn_cast<FrameIndexSDNode>(Val: Node))
12909 return SDDbgOperand::fromFrameIdx(FrameIdx: FISDN->getIndex());
12910 return SDDbgOperand::fromNode(Node, ResNo);
12911 };
12912
12913 SmallVector<SDDbgValue *, 2> ClonedDVs;
12914 for (auto *DV : GetDbgValues(SD: &N)) {
12915 if (DV->isInvalidated())
12916 continue;
12917 switch (N.getOpcode()) {
12918 default:
12919 break;
12920 case ISD::ADD: {
12921 SDValue N0 = N.getOperand(Num: 0);
12922 SDValue N1 = N.getOperand(Num: 1);
12923 if (!isa<ConstantSDNode>(Val: N0)) {
12924 bool RHSConstant = isa<ConstantSDNode>(Val: N1);
12925 uint64_t Offset;
12926 if (RHSConstant)
12927 Offset = N.getConstantOperandVal(Num: 1);
12928 // We are not allowed to turn indirect debug values variadic, so
12929 // don't salvage those.
12930 if (!RHSConstant && DV->isIndirect())
12931 continue;
12932
12933 // Rewrite an ADD constant node into a DIExpression. Since we are
12934 // performing arithmetic to compute the variable's *value* in the
12935 // DIExpression, we need to mark the expression with a
12936 // DW_OP_stack_value.
12937 auto *DIExpr = DV->getExpression();
12938 auto NewLocOps = DV->copyLocationOps();
12939 bool Changed = false;
12940 size_t OrigLocOpsSize = NewLocOps.size();
12941 for (size_t i = 0; i < OrigLocOpsSize; ++i) {
12942 // We're not given a ResNo to compare against because the whole
12943 // node is going away. We know that any ISD::ADD only has one
12944 // result, so we can assume any node match is using the result.
12945 if (NewLocOps[i].getKind() != SDDbgOperand::SDNODE ||
12946 NewLocOps[i].getSDNode() != &N)
12947 continue;
12948 NewLocOps[i] = GetLocationOperand(N0.getNode(), N0.getResNo());
12949 if (RHSConstant) {
12950 SmallVector<uint64_t, 3> ExprOps;
12951 DIExpression::appendOffset(Ops&: ExprOps, Offset);
12952 DIExpr = DIExpression::appendOpsToArg(Expr: DIExpr, Ops: ExprOps, ArgNo: i, StackValue: true);
12953 } else {
12954 // Convert to a variadic expression (if not already).
12955 // convertToVariadicExpression() returns a const pointer, so we use
12956 // a temporary const variable here.
12957 const auto *TmpDIExpr =
12958 DIExpression::convertToVariadicExpression(Expr: DIExpr);
12959 SmallVector<uint64_t, 3> ExprOps;
12960 ExprOps.push_back(Elt: dwarf::DW_OP_LLVM_arg);
12961 ExprOps.push_back(Elt: NewLocOps.size());
12962 ExprOps.push_back(Elt: dwarf::DW_OP_plus);
12963 SDDbgOperand RHS =
12964 SDDbgOperand::fromNode(Node: N1.getNode(), ResNo: N1.getResNo());
12965 NewLocOps.push_back(Elt: RHS);
12966 DIExpr = DIExpression::appendOpsToArg(Expr: TmpDIExpr, Ops: ExprOps, ArgNo: i, StackValue: true);
12967 }
12968 Changed = true;
12969 }
12970 (void)Changed;
12971 assert(Changed && "Salvage target doesn't use N");
12972
12973 bool IsVariadic =
12974 DV->isVariadic() || OrigLocOpsSize != NewLocOps.size();
12975
12976 auto AdditionalDependencies = DV->getAdditionalDependencies();
12977 SDDbgValue *Clone = getDbgValueList(
12978 Var: DV->getVariable(), Expr: DIExpr, Locs: NewLocOps, Dependencies: AdditionalDependencies,
12979 IsIndirect: DV->isIndirect(), DL: DV->getDebugLoc(), O: DV->getOrder(), IsVariadic);
12980 ClonedDVs.push_back(Elt: Clone);
12981 DV->setIsInvalidated();
12982 DV->setIsEmitted();
12983 LLVM_DEBUG(dbgs() << "SALVAGE: Rewriting";
12984 N0.getNode()->dumprFull(this);
12985 dbgs() << " into " << *DIExpr << '\n');
12986 }
12987 break;
12988 }
12989 case ISD::TRUNCATE: {
12990 SDValue N0 = N.getOperand(Num: 0);
12991 TypeSize FromSize = N0.getValueSizeInBits();
12992 TypeSize ToSize = N.getValueSizeInBits(ResNo: 0);
12993
12994 DIExpression *DbgExpression = DV->getExpression();
12995 auto ExtOps = DIExpression::getExtOps(FromSize, ToSize, Signed: false);
12996 auto NewLocOps = DV->copyLocationOps();
12997 bool Changed = false;
12998 for (size_t i = 0; i < NewLocOps.size(); ++i) {
12999 if (NewLocOps[i].getKind() != SDDbgOperand::SDNODE ||
13000 NewLocOps[i].getSDNode() != &N)
13001 continue;
13002
13003 NewLocOps[i] = GetLocationOperand(N0.getNode(), N0.getResNo());
13004 DbgExpression = DIExpression::appendOpsToArg(Expr: DbgExpression, Ops: ExtOps, ArgNo: i);
13005 Changed = true;
13006 }
13007 assert(Changed && "Salvage target doesn't use N");
13008 (void)Changed;
13009
13010 SDDbgValue *Clone =
13011 getDbgValueList(Var: DV->getVariable(), Expr: DbgExpression, Locs: NewLocOps,
13012 Dependencies: DV->getAdditionalDependencies(), IsIndirect: DV->isIndirect(),
13013 DL: DV->getDebugLoc(), O: DV->getOrder(), IsVariadic: DV->isVariadic());
13014
13015 ClonedDVs.push_back(Elt: Clone);
13016 DV->setIsInvalidated();
13017 DV->setIsEmitted();
13018 LLVM_DEBUG(dbgs() << "SALVAGE: Rewriting"; N0.getNode()->dumprFull(this);
13019 dbgs() << " into " << *DbgExpression << '\n');
13020 break;
13021 }
13022 }
13023 }
13024
13025 for (SDDbgValue *Dbg : ClonedDVs) {
13026 assert((!Dbg->getSDNodes().empty() ||
13027 llvm::any_of(Dbg->getLocationOps(),
13028 [&](const SDDbgOperand &Op) {
13029 return Op.getKind() == SDDbgOperand::FRAMEIX;
13030 })) &&
13031 "Salvaged DbgValue should depend on a new SDNode");
13032 AddDbgValue(DB: Dbg, isParameter: false);
13033 }
13034}
13035
13036/// Creates a SDDbgLabel node.
13037SDDbgLabel *SelectionDAG::getDbgLabel(DILabel *Label,
13038 const DebugLoc &DL, unsigned O) {
13039 assert(cast<DILabel>(Label)->isValidLocationForIntrinsic(DL) &&
13040 "Expected inlined-at fields to agree");
13041 return new (DbgInfo->getAlloc()) SDDbgLabel(Label, DL, O);
13042}
13043
13044namespace {
13045
13046/// RAUWUpdateListener - Helper for ReplaceAllUsesWith - When the node
13047/// pointed to by a use iterator is deleted, increment the use iterator
13048/// so that it doesn't dangle.
13049///
13050class RAUWUpdateListener : public SelectionDAG::DAGUpdateListener {
13051 SDNode::use_iterator &UI;
13052 SDNode::use_iterator &UE;
13053
13054 void NodeDeleted(SDNode *N, SDNode *E) override {
13055 // Increment the iterator as needed.
13056 while (UI != UE && N == UI->getUser())
13057 ++UI;
13058 }
13059
13060public:
13061 RAUWUpdateListener(SelectionDAG &d,
13062 SDNode::use_iterator &ui,
13063 SDNode::use_iterator &ue)
13064 : SelectionDAG::DAGUpdateListener(d), UI(ui), UE(ue) {}
13065};
13066
13067} // end anonymous namespace
13068
13069/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
13070/// This can cause recursive merging of nodes in the DAG.
13071///
13072/// This version assumes From has a single result value.
13073///
13074void SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To) {
13075 SDNode *From = FromN.getNode();
13076 assert(From->getNumValues() == 1 && FromN.getResNo() == 0 &&
13077 "Cannot replace with this method!");
13078 assert(From != To.getNode() && "Cannot replace uses of with self");
13079
13080 // Preserve Debug Values
13081 transferDbgValues(From: FromN, To);
13082 // Preserve extra info.
13083 copyExtraInfo(From, To: To.getNode());
13084
13085 // Iterate over all the existing uses of From. New uses will be added
13086 // to the beginning of the use list, which we avoid visiting.
13087 // This specifically avoids visiting uses of From that arise while the
13088 // replacement is happening, because any such uses would be the result
13089 // of CSE: If an existing node looks like From after one of its operands
13090 // is replaced by To, we don't want to replace of all its users with To
13091 // too. See PR3018 for more info.
13092 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
13093 RAUWUpdateListener Listener(*this, UI, UE);
13094 while (UI != UE) {
13095 SDNode *User = UI->getUser();
13096
13097 // This node is about to morph, remove its old self from the CSE maps.
13098 RemoveNodeFromCSEMaps(N: User);
13099
13100 // A user can appear in a use list multiple times, and when this
13101 // happens the uses are usually next to each other in the list.
13102 // To help reduce the number of CSE recomputations, process all
13103 // the uses of this user that we can find this way.
13104 do {
13105 SDUse &Use = *UI;
13106 ++UI;
13107 Use.set(To);
13108 if (To->isDivergent() != From->isDivergent())
13109 updateDivergence(N: User);
13110 } while (UI != UE && UI->getUser() == User);
13111 // Now that we have modified User, add it back to the CSE maps. If it
13112 // already exists there, recursively merge the results together.
13113 AddModifiedNodeToCSEMaps(N: User);
13114 }
13115
13116 // If we just RAUW'd the root, take note.
13117 if (FromN == getRoot())
13118 setRoot(To);
13119}
13120
13121/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
13122/// This can cause recursive merging of nodes in the DAG.
13123///
13124/// This version assumes that for each value of From, there is a
13125/// corresponding value in To in the same position with the same type.
13126///
13127void SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To) {
13128#ifndef NDEBUG
13129 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
13130 assert((!From->hasAnyUseOfValue(i) ||
13131 From->getValueType(i) == To->getValueType(i)) &&
13132 "Cannot use this version of ReplaceAllUsesWith!");
13133#endif
13134
13135 // Handle the trivial case.
13136 if (From == To)
13137 return;
13138
13139 // Preserve Debug Info. Only do this if there's a use.
13140 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
13141 if (From->hasAnyUseOfValue(Value: i)) {
13142 assert((i < To->getNumValues()) && "Invalid To location");
13143 transferDbgValues(From: SDValue(From, i), To: SDValue(To, i));
13144 }
13145 // Preserve extra info.
13146 copyExtraInfo(From, To);
13147
13148 // Iterate over just the existing users of From. See the comments in
13149 // the ReplaceAllUsesWith above.
13150 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
13151 RAUWUpdateListener Listener(*this, UI, UE);
13152 while (UI != UE) {
13153 SDNode *User = UI->getUser();
13154
13155 // This node is about to morph, remove its old self from the CSE maps.
13156 RemoveNodeFromCSEMaps(N: User);
13157
13158 // A user can appear in a use list multiple times, and when this
13159 // happens the uses are usually next to each other in the list.
13160 // To help reduce the number of CSE recomputations, process all
13161 // the uses of this user that we can find this way.
13162 do {
13163 SDUse &Use = *UI;
13164 ++UI;
13165 Use.setNode(To);
13166 if (To->isDivergent() != From->isDivergent())
13167 updateDivergence(N: User);
13168 } while (UI != UE && UI->getUser() == User);
13169
13170 // Now that we have modified User, add it back to the CSE maps. If it
13171 // already exists there, recursively merge the results together.
13172 AddModifiedNodeToCSEMaps(N: User);
13173 }
13174
13175 // If we just RAUW'd the root, take note.
13176 if (From == getRoot().getNode())
13177 setRoot(SDValue(To, getRoot().getResNo()));
13178}
13179
13180/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
13181/// This can cause recursive merging of nodes in the DAG.
13182///
13183/// This version can replace From with any result values. To must match the
13184/// number and types of values returned by From.
13185void SelectionDAG::ReplaceAllUsesWith(SDNode *From, const SDValue *To) {
13186 if (From->getNumValues() == 1) // Handle the simple case efficiently.
13187 return ReplaceAllUsesWith(FromN: SDValue(From, 0), To: To[0]);
13188
13189 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i) {
13190 // Preserve Debug Info.
13191 transferDbgValues(From: SDValue(From, i), To: To[i]);
13192 // Preserve extra info.
13193 copyExtraInfo(From, To: To[i].getNode());
13194 }
13195
13196 // Iterate over just the existing users of From. See the comments in
13197 // the ReplaceAllUsesWith above.
13198 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
13199 RAUWUpdateListener Listener(*this, UI, UE);
13200 while (UI != UE) {
13201 SDNode *User = UI->getUser();
13202
13203 // This node is about to morph, remove its old self from the CSE maps.
13204 RemoveNodeFromCSEMaps(N: User);
13205
13206 // A user can appear in a use list multiple times, and when this happens the
13207 // uses are usually next to each other in the list. To help reduce the
13208 // number of CSE and divergence recomputations, process all the uses of this
13209 // user that we can find this way.
13210 bool To_IsDivergent = false;
13211 do {
13212 SDUse &Use = *UI;
13213 const SDValue &ToOp = To[Use.getResNo()];
13214 ++UI;
13215 Use.set(ToOp);
13216 if (ToOp.getValueType() != MVT::Other)
13217 To_IsDivergent |= ToOp->isDivergent();
13218 } while (UI != UE && UI->getUser() == User);
13219
13220 if (To_IsDivergent != From->isDivergent())
13221 updateDivergence(N: User);
13222
13223 // Now that we have modified User, add it back to the CSE maps. If it
13224 // already exists there, recursively merge the results together.
13225 AddModifiedNodeToCSEMaps(N: User);
13226 }
13227
13228 // If we just RAUW'd the root, take note.
13229 if (From == getRoot().getNode())
13230 setRoot(SDValue(To[getRoot().getResNo()]));
13231}
13232
13233/// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving
13234/// uses of other values produced by From.getNode() alone. The Deleted
13235/// vector is handled the same way as for ReplaceAllUsesWith.
13236void SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To){
13237 // Handle the really simple, really trivial case efficiently.
13238 if (From == To) return;
13239
13240 // Handle the simple, trivial, case efficiently.
13241 if (From.getNode()->getNumValues() == 1) {
13242 ReplaceAllUsesWith(FromN: From, To);
13243 return;
13244 }
13245
13246 // Preserve Debug Info.
13247 transferDbgValues(From, To);
13248 copyExtraInfo(From: From.getNode(), To: To.getNode());
13249
13250 // Iterate over just the existing users of From. See the comments in
13251 // the ReplaceAllUsesWith above.
13252 SDNode::use_iterator UI = From.getNode()->use_begin(),
13253 UE = From.getNode()->use_end();
13254 RAUWUpdateListener Listener(*this, UI, UE);
13255 while (UI != UE) {
13256 SDNode *User = UI->getUser();
13257 bool UserRemovedFromCSEMaps = false;
13258
13259 // A user can appear in a use list multiple times, and when this
13260 // happens the uses are usually next to each other in the list.
13261 // To help reduce the number of CSE recomputations, process all
13262 // the uses of this user that we can find this way.
13263 do {
13264 SDUse &Use = *UI;
13265
13266 // Skip uses of different values from the same node.
13267 if (Use.getResNo() != From.getResNo()) {
13268 ++UI;
13269 continue;
13270 }
13271
13272 // If this node hasn't been modified yet, it's still in the CSE maps,
13273 // so remove its old self from the CSE maps.
13274 if (!UserRemovedFromCSEMaps) {
13275 RemoveNodeFromCSEMaps(N: User);
13276 UserRemovedFromCSEMaps = true;
13277 }
13278
13279 ++UI;
13280 Use.set(To);
13281 if (To->isDivergent() != From->isDivergent())
13282 updateDivergence(N: User);
13283 } while (UI != UE && UI->getUser() == User);
13284 // We are iterating over all uses of the From node, so if a use
13285 // doesn't use the specific value, no changes are made.
13286 if (!UserRemovedFromCSEMaps)
13287 continue;
13288
13289 // Now that we have modified User, add it back to the CSE maps. If it
13290 // already exists there, recursively merge the results together.
13291 AddModifiedNodeToCSEMaps(N: User);
13292 }
13293
13294 // If we just RAUW'd the root, take note.
13295 if (From == getRoot())
13296 setRoot(To);
13297}
13298
13299namespace {
13300
13301/// UseMemo - This class is used by SelectionDAG::ReplaceAllUsesOfValuesWith
13302/// to record information about a use.
13303struct UseMemo {
13304 SDNode *User;
13305 unsigned Index;
13306 SDUse *Use;
13307};
13308
13309/// operator< - Sort Memos by User.
13310bool operator<(const UseMemo &L, const UseMemo &R) {
13311 return (intptr_t)L.User < (intptr_t)R.User;
13312}
13313
13314/// RAUOVWUpdateListener - Helper for ReplaceAllUsesOfValuesWith - When the node
13315/// pointed to by a UseMemo is deleted, set the User to nullptr to indicate that
13316/// the node already has been taken care of recursively.
13317class RAUOVWUpdateListener : public SelectionDAG::DAGUpdateListener {
13318 SmallVectorImpl<UseMemo> &Uses;
13319
13320 void NodeDeleted(SDNode *N, SDNode *E) override {
13321 for (UseMemo &Memo : Uses)
13322 if (Memo.User == N)
13323 Memo.User = nullptr;
13324 }
13325
13326public:
13327 RAUOVWUpdateListener(SelectionDAG &d, SmallVectorImpl<UseMemo> &uses)
13328 : SelectionDAG::DAGUpdateListener(d), Uses(uses) {}
13329};
13330
13331} // end anonymous namespace
13332
13333/// Return true if a glue output should propagate divergence information.
13334static bool gluePropagatesDivergence(const SDNode *Node) {
13335 switch (Node->getOpcode()) {
13336 case ISD::CopyFromReg:
13337 case ISD::CopyToReg:
13338 return false;
13339 default:
13340 return true;
13341 }
13342
13343 llvm_unreachable("covered opcode switch");
13344}
13345
13346bool SelectionDAG::calculateDivergence(SDNode *N) {
13347 if (TLI->isSDNodeAlwaysUniform(N)) {
13348 assert(!TLI->isSDNodeSourceOfDivergence(N, FLI, UA) &&
13349 "Conflicting divergence information!");
13350 return false;
13351 }
13352 if (TLI->isSDNodeSourceOfDivergence(N, FLI, UA))
13353 return true;
13354 for (const auto &Op : N->ops()) {
13355 EVT VT = Op.getValueType();
13356
13357 // Skip Chain. It does not carry divergence.
13358 if (VT != MVT::Other && Op.getNode()->isDivergent() &&
13359 (VT != MVT::Glue || gluePropagatesDivergence(Node: Op.getNode())))
13360 return true;
13361 }
13362 return false;
13363}
13364
13365void SelectionDAG::updateDivergence(SDNode *N) {
13366 SmallVector<SDNode *, 16> Worklist(1, N);
13367 do {
13368 N = Worklist.pop_back_val();
13369 bool IsDivergent = calculateDivergence(N);
13370 if (N->SDNodeBits.IsDivergent != IsDivergent) {
13371 N->SDNodeBits.IsDivergent = IsDivergent;
13372 llvm::append_range(C&: Worklist, R: N->users());
13373 }
13374 } while (!Worklist.empty());
13375}
13376
13377void SelectionDAG::CreateTopologicalOrder(std::vector<SDNode *> &Order) {
13378 DenseMap<SDNode *, unsigned> Degree;
13379 Order.reserve(n: AllNodes.size());
13380 for (auto &N : allnodes()) {
13381 unsigned NOps = N.getNumOperands();
13382 Degree[&N] = NOps;
13383 if (0 == NOps)
13384 Order.push_back(x: &N);
13385 }
13386 for (size_t I = 0; I != Order.size(); ++I) {
13387 SDNode *N = Order[I];
13388 for (auto *U : N->users()) {
13389 unsigned &UnsortedOps = Degree[U];
13390 if (0 == --UnsortedOps)
13391 Order.push_back(x: U);
13392 }
13393 }
13394}
13395
13396#if !defined(NDEBUG) && LLVM_ENABLE_ABI_BREAKING_CHECKS
13397void SelectionDAG::VerifyDAGDivergence() {
13398 std::vector<SDNode *> TopoOrder;
13399 CreateTopologicalOrder(TopoOrder);
13400 for (auto *N : TopoOrder) {
13401 assert(calculateDivergence(N) == N->isDivergent() &&
13402 "Divergence bit inconsistency detected");
13403 }
13404}
13405#endif
13406
13407/// ReplaceAllUsesOfValuesWith - Replace any uses of From with To, leaving
13408/// uses of other values produced by From.getNode() alone. The same value
13409/// may appear in both the From and To list. The Deleted vector is
13410/// handled the same way as for ReplaceAllUsesWith.
13411void SelectionDAG::ReplaceAllUsesOfValuesWith(const SDValue *From,
13412 const SDValue *To,
13413 unsigned Num){
13414 // Handle the simple, trivial case efficiently.
13415 if (Num == 1)
13416 return ReplaceAllUsesOfValueWith(From: *From, To: *To);
13417
13418 transferDbgValues(From: *From, To: *To);
13419 copyExtraInfo(From: From->getNode(), To: To->getNode());
13420
13421 // Read up all the uses and make records of them. This helps
13422 // processing new uses that are introduced during the
13423 // replacement process.
13424 SmallVector<UseMemo, 4> Uses;
13425 for (unsigned i = 0; i != Num; ++i) {
13426 unsigned FromResNo = From[i].getResNo();
13427 SDNode *FromNode = From[i].getNode();
13428 for (SDUse &Use : FromNode->uses()) {
13429 if (Use.getResNo() == FromResNo) {
13430 UseMemo Memo = {.User: Use.getUser(), .Index: i, .Use: &Use};
13431 Uses.push_back(Elt: Memo);
13432 }
13433 }
13434 }
13435
13436 // Sort the uses, so that all the uses from a given User are together.
13437 llvm::sort(C&: Uses);
13438 RAUOVWUpdateListener Listener(*this, Uses);
13439
13440 for (unsigned UseIndex = 0, UseIndexEnd = Uses.size();
13441 UseIndex != UseIndexEnd; ) {
13442 // We know that this user uses some value of From. If it is the right
13443 // value, update it.
13444 SDNode *User = Uses[UseIndex].User;
13445 // If the node has been deleted by recursive CSE updates when updating
13446 // another node, then just skip this entry.
13447 if (User == nullptr) {
13448 ++UseIndex;
13449 continue;
13450 }
13451
13452 // This node is about to morph, remove its old self from the CSE maps.
13453 RemoveNodeFromCSEMaps(N: User);
13454
13455 // The Uses array is sorted, so all the uses for a given User
13456 // are next to each other in the list.
13457 // To help reduce the number of CSE recomputations, process all
13458 // the uses of this user that we can find this way.
13459 do {
13460 unsigned i = Uses[UseIndex].Index;
13461 SDUse &Use = *Uses[UseIndex].Use;
13462 ++UseIndex;
13463
13464 Use.set(To[i]);
13465 } while (UseIndex != UseIndexEnd && Uses[UseIndex].User == User);
13466
13467 // Now that we have modified User, add it back to the CSE maps. If it
13468 // already exists there, recursively merge the results together.
13469 AddModifiedNodeToCSEMaps(N: User);
13470 }
13471}
13472
13473/// AssignTopologicalOrder - Assign a unique node id for each node in the DAG
13474/// based on their topological order. It returns the maximum id and a vector
13475/// of the SDNodes* in assigned order by reference.
13476unsigned SelectionDAG::AssignTopologicalOrder() {
13477 unsigned DAGSize = 0;
13478
13479 // SortedPos tracks the progress of the algorithm. Nodes before it are
13480 // sorted, nodes after it are unsorted. When the algorithm completes
13481 // it is at the end of the list.
13482 allnodes_iterator SortedPos = allnodes_begin();
13483
13484 // Visit all the nodes. Move nodes with no operands to the front of
13485 // the list immediately. Annotate nodes that do have operands with their
13486 // operand count. Before we do this, the Node Id fields of the nodes
13487 // may contain arbitrary values. After, the Node Id fields for nodes
13488 // before SortedPos will contain the topological sort index, and the
13489 // Node Id fields for nodes At SortedPos and after will contain the
13490 // count of outstanding operands.
13491 for (SDNode &N : llvm::make_early_inc_range(Range: allnodes())) {
13492 checkForCycles(N: &N, DAG: this);
13493 unsigned Degree = N.getNumOperands();
13494 if (Degree == 0) {
13495 // A node with no uses, add it to the result array immediately.
13496 N.setNodeId(DAGSize++);
13497 allnodes_iterator Q(&N);
13498 if (Q != SortedPos)
13499 SortedPos = AllNodes.insert(where: SortedPos, New: AllNodes.remove(IT&: Q));
13500 assert(SortedPos != AllNodes.end() && "Overran node list");
13501 ++SortedPos;
13502 } else {
13503 // Temporarily use the Node Id as scratch space for the degree count.
13504 N.setNodeId(Degree);
13505 }
13506 }
13507
13508 // Visit all the nodes. As we iterate, move nodes into sorted order,
13509 // such that by the time the end is reached all nodes will be sorted.
13510 for (SDNode &Node : allnodes()) {
13511 SDNode *N = &Node;
13512 checkForCycles(N, DAG: this);
13513 // N is in sorted position, so all its uses have one less operand
13514 // that needs to be sorted.
13515 for (SDNode *P : N->users()) {
13516 unsigned Degree = P->getNodeId();
13517 assert(Degree != 0 && "Invalid node degree");
13518 --Degree;
13519 if (Degree == 0) {
13520 // All of P's operands are sorted, so P may sorted now.
13521 P->setNodeId(DAGSize++);
13522 if (P->getIterator() != SortedPos)
13523 SortedPos = AllNodes.insert(where: SortedPos, New: AllNodes.remove(IT: P));
13524 assert(SortedPos != AllNodes.end() && "Overran node list");
13525 ++SortedPos;
13526 } else {
13527 // Update P's outstanding operand count.
13528 P->setNodeId(Degree);
13529 }
13530 }
13531 if (Node.getIterator() == SortedPos) {
13532#ifndef NDEBUG
13533 allnodes_iterator I(N);
13534 SDNode *S = &*++I;
13535 dbgs() << "Overran sorted position:\n";
13536 S->dumprFull(this); dbgs() << "\n";
13537 dbgs() << "Checking if this is due to cycles\n";
13538 checkForCycles(this, true);
13539#endif
13540 llvm_unreachable(nullptr);
13541 }
13542 }
13543
13544 assert(SortedPos == AllNodes.end() &&
13545 "Topological sort incomplete!");
13546 assert(AllNodes.front().getOpcode() == ISD::EntryToken &&
13547 "First node in topological sort is not the entry token!");
13548 assert(AllNodes.front().getNodeId() == 0 &&
13549 "First node in topological sort has non-zero id!");
13550 assert(AllNodes.front().getNumOperands() == 0 &&
13551 "First node in topological sort has operands!");
13552 assert(AllNodes.back().getNodeId() == (int)DAGSize-1 &&
13553 "Last node in topologic sort has unexpected id!");
13554 assert(AllNodes.back().use_empty() &&
13555 "Last node in topologic sort has users!");
13556 assert(DAGSize == allnodes_size() && "Node count mismatch!");
13557 return DAGSize;
13558}
13559
13560void SelectionDAG::getTopologicallyOrderedNodes(
13561 SmallVectorImpl<const SDNode *> &SortedNodes) const {
13562 SortedNodes.clear();
13563 // Node -> remaining number of outstanding operands.
13564 DenseMap<const SDNode *, unsigned> RemainingOperands;
13565
13566 // Put nodes without any operands into SortedNodes first.
13567 for (const SDNode &N : allnodes()) {
13568 checkForCycles(N: &N, DAG: this);
13569 unsigned NumOperands = N.getNumOperands();
13570 if (NumOperands == 0)
13571 SortedNodes.push_back(Elt: &N);
13572 else
13573 // Record their total number of outstanding operands.
13574 RemainingOperands[&N] = NumOperands;
13575 }
13576
13577 // A node is pushed into SortedNodes when all of its operands (predecessors in
13578 // the graph) are also in SortedNodes.
13579 for (unsigned i = 0U; i < SortedNodes.size(); ++i) {
13580 const SDNode *N = SortedNodes[i];
13581 for (const SDNode *U : N->users()) {
13582 // HandleSDNode is never part of a DAG and therefore has no entry in
13583 // RemainingOperands.
13584 if (U->getOpcode() == ISD::HANDLENODE)
13585 continue;
13586 unsigned &NumRemOperands = RemainingOperands[U];
13587 assert(NumRemOperands && "Invalid number of remaining operands");
13588 --NumRemOperands;
13589 if (!NumRemOperands)
13590 SortedNodes.push_back(Elt: U);
13591 }
13592 }
13593
13594 assert(SortedNodes.size() == AllNodes.size() && "Node count mismatch");
13595 assert(SortedNodes.front()->getOpcode() == ISD::EntryToken &&
13596 "First node in topological sort is not the entry token");
13597 assert(SortedNodes.front()->getNumOperands() == 0 &&
13598 "First node in topological sort has operands");
13599}
13600
13601/// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the
13602/// value is produced by SD.
13603void SelectionDAG::AddDbgValue(SDDbgValue *DB, bool isParameter) {
13604 for (SDNode *SD : DB->getSDNodes()) {
13605 if (!SD)
13606 continue;
13607 assert(DbgInfo->getSDDbgValues(SD).empty() || SD->getHasDebugValue());
13608 SD->setHasDebugValue(true);
13609 }
13610 DbgInfo->add(V: DB, isParameter);
13611}
13612
13613void SelectionDAG::AddDbgLabel(SDDbgLabel *DB) { DbgInfo->add(L: DB); }
13614
13615SDValue SelectionDAG::makeEquivalentMemoryOrdering(SDValue OldChain,
13616 SDValue NewMemOpChain) {
13617 assert(isa<MemSDNode>(NewMemOpChain) && "Expected a memop node");
13618 assert(NewMemOpChain.getValueType() == MVT::Other && "Expected a token VT");
13619 // The new memory operation must have the same position as the old load in
13620 // terms of memory dependency. Create a TokenFactor for the old load and new
13621 // memory operation and update uses of the old load's output chain to use that
13622 // TokenFactor.
13623 if (OldChain == NewMemOpChain || OldChain.use_empty())
13624 return NewMemOpChain;
13625
13626 SDValue TokenFactor = getNode(Opcode: ISD::TokenFactor, DL: SDLoc(OldChain), VT: MVT::Other,
13627 N1: OldChain, N2: NewMemOpChain);
13628 ReplaceAllUsesOfValueWith(From: OldChain, To: TokenFactor);
13629 UpdateNodeOperands(N: TokenFactor.getNode(), Op1: OldChain, Op2: NewMemOpChain);
13630 return TokenFactor;
13631}
13632
13633SDValue SelectionDAG::makeEquivalentMemoryOrdering(LoadSDNode *OldLoad,
13634 SDValue NewMemOp) {
13635 assert(isa<MemSDNode>(NewMemOp.getNode()) && "Expected a memop node");
13636 SDValue OldChain = SDValue(OldLoad, 1);
13637 SDValue NewMemOpChain = NewMemOp.getValue(R: 1);
13638 return makeEquivalentMemoryOrdering(OldChain, NewMemOpChain);
13639}
13640
13641SDValue SelectionDAG::getSymbolFunctionGlobalAddress(SDValue Op,
13642 Function **OutFunction) {
13643 assert(isa<ExternalSymbolSDNode>(Op) && "Node should be an ExternalSymbol");
13644
13645 auto *Symbol = cast<ExternalSymbolSDNode>(Val&: Op)->getSymbol();
13646 auto *Module = MF->getFunction().getParent();
13647 auto *Function = Module->getFunction(Name: Symbol);
13648
13649 if (OutFunction != nullptr)
13650 *OutFunction = Function;
13651
13652 if (Function != nullptr) {
13653 auto PtrTy = TLI->getPointerTy(DL: getDataLayout(), AS: Function->getAddressSpace());
13654 return getGlobalAddress(GV: Function, DL: SDLoc(Op), VT: PtrTy);
13655 }
13656
13657 std::string ErrorStr;
13658 raw_string_ostream ErrorFormatter(ErrorStr);
13659 ErrorFormatter << "Undefined external symbol ";
13660 ErrorFormatter << '"' << Symbol << '"';
13661 report_fatal_error(reason: Twine(ErrorStr));
13662}
13663
13664//===----------------------------------------------------------------------===//
13665// SDNode Class
13666//===----------------------------------------------------------------------===//
13667
13668bool llvm::isNullConstant(SDValue V) {
13669 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val&: V);
13670 return Const != nullptr && Const->isZero();
13671}
13672
13673bool llvm::isNullConstantOrUndef(SDValue V) {
13674 return V.isUndef() || isNullConstant(V);
13675}
13676
13677bool llvm::isNullFPConstant(SDValue V) {
13678 ConstantFPSDNode *Const = dyn_cast<ConstantFPSDNode>(Val&: V);
13679 return Const != nullptr && Const->isZero() && !Const->isNegative();
13680}
13681
13682bool llvm::isAllOnesConstant(SDValue V) {
13683 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val&: V);
13684 return Const != nullptr && Const->isAllOnes();
13685}
13686
13687bool llvm::isOneConstant(SDValue V) {
13688 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val&: V);
13689 return Const != nullptr && Const->isOne();
13690}
13691
13692bool llvm::isMinSignedConstant(SDValue V) {
13693 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val&: V);
13694 return Const != nullptr && Const->isMinSignedValue();
13695}
13696
13697bool SelectionDAG::isIdentityElement(unsigned Opcode, SDNodeFlags Flags,
13698 SDValue V, unsigned OperandNo,
13699 unsigned Depth) const {
13700 APInt DemandedElts = getDemandAllEltsMask(V);
13701 return isIdentityElement(Opc: Opcode, Flags, V, DemandedElts, OperandNo, Depth);
13702}
13703
13704bool SelectionDAG::isIdentityElement(unsigned Opcode, SDNodeFlags Flags,
13705 SDValue V, const APInt &DemandedElts,
13706 unsigned OperandNo, unsigned Depth) const {
13707 // NOTE: The cases should match with IR's ConstantExpr::getBinOpIdentity().
13708 // TODO: Target-specific opcodes could be added.
13709 if (V.getValueType().isInteger()) {
13710 KnownBits Known = computeKnownBits(Op: V, DemandedElts, Depth);
13711 if (Known.isConstant()) {
13712 const APInt &Const = Known.getConstant();
13713 switch (Opcode) {
13714 case ISD::ADD:
13715 case ISD::OR:
13716 case ISD::XOR:
13717 case ISD::UMAX:
13718 return Const.isZero();
13719 case ISD::MUL:
13720 return Const.isOne();
13721 case ISD::AND:
13722 case ISD::UMIN:
13723 return Const.isAllOnes();
13724 case ISD::SMAX:
13725 return Const.isMinSignedValue();
13726 case ISD::SMIN:
13727 return Const.isMaxSignedValue();
13728 case ISD::SUB:
13729 case ISD::SHL:
13730 case ISD::SRA:
13731 case ISD::SRL:
13732 return OperandNo == 1 && Const.isZero();
13733 case ISD::UDIV:
13734 case ISD::SDIV:
13735 return OperandNo == 1 && Const.isOne();
13736 }
13737 }
13738 } else if (auto *ConstFP = isConstOrConstSplatFP(N: V, DemandedElts)) {
13739 switch (Opcode) {
13740 case ISD::FADD:
13741 return ConstFP->isZero() &&
13742 (Flags.hasNoSignedZeros() || ConstFP->isNegative());
13743 case ISD::FSUB:
13744 return OperandNo == 1 && ConstFP->isZero() &&
13745 (Flags.hasNoSignedZeros() || !ConstFP->isNegative());
13746 case ISD::FMUL:
13747 return ConstFP->isOne();
13748 case ISD::FDIV:
13749 return OperandNo == 1 && ConstFP->isOne();
13750 case ISD::FMINNUM:
13751 case ISD::FMAXNUM:
13752 case ISD::FMINIMUMNUM:
13753 case ISD::FMAXIMUMNUM: {
13754 // Neutral element for fminnum/fminimumnum is NaN, Inf or FLT_MAX,
13755 // depending on fast-math flags (FMF).
13756 EVT VT = V.getValueType();
13757 const fltSemantics &Semantics = VT.getFltSemantics();
13758 APFloat NeutralAF = !Flags.hasNoNaNs() ? APFloat::getQNaN(Sem: Semantics)
13759 : !Flags.hasNoInfs() ? APFloat::getInf(Sem: Semantics)
13760 : APFloat::getLargest(Sem: Semantics);
13761 if (Opcode == ISD::FMAXNUM || Opcode == ISD::FMAXIMUMNUM)
13762 NeutralAF.changeSign();
13763
13764 return ConstFP->isExactlyValue(V: NeutralAF);
13765 }
13766 case ISD::FMINIMUM:
13767 case ISD::FMAXIMUM: {
13768 // Neutral element for fminimum is Inf or FLT_MAX, depending on FMF.
13769 const APFloat &VAPF = ConstFP->getValueAPF();
13770 bool NeutralNegative = (Opcode == ISD::FMAXIMUM);
13771 if (Flags.hasNoInfs())
13772 return VAPF.isLargest() && VAPF.isNegative() == NeutralNegative;
13773 return VAPF.isInfinity() && VAPF.isNegative() == NeutralNegative;
13774 }
13775 }
13776 }
13777 return false;
13778}
13779
13780SDValue llvm::peekThroughBitcasts(SDValue V) {
13781 while (V.getOpcode() == ISD::BITCAST)
13782 V = V.getOperand(i: 0);
13783 return V;
13784}
13785
13786SDValue llvm::peekThroughOneUseBitcasts(SDValue V) {
13787 while (V.getOpcode() == ISD::BITCAST && V.getOperand(i: 0).hasOneUse())
13788 V = V.getOperand(i: 0);
13789 return V;
13790}
13791
13792SDValue llvm::peekThroughExtractSubvectors(SDValue V) {
13793 while (V.getOpcode() == ISD::EXTRACT_SUBVECTOR)
13794 V = V.getOperand(i: 0);
13795 return V;
13796}
13797
13798SDValue llvm::peekThroughInsertVectorElt(SDValue V, const APInt &DemandedElts) {
13799 while (V.getOpcode() == ISD::INSERT_VECTOR_ELT) {
13800 SDValue InVec = V.getOperand(i: 0);
13801 SDValue EltNo = V.getOperand(i: 2);
13802 EVT VT = InVec.getValueType();
13803 auto *IndexC = dyn_cast<ConstantSDNode>(Val&: EltNo);
13804 if (IndexC && VT.isFixedLengthVector() &&
13805 IndexC->getAPIntValue().ult(RHS: VT.getVectorNumElements()) &&
13806 !DemandedElts[IndexC->getZExtValue()]) {
13807 V = InVec;
13808 continue;
13809 }
13810 break;
13811 }
13812 return V;
13813}
13814
13815SDValue llvm::peekThroughTruncates(SDValue V) {
13816 while (V.getOpcode() == ISD::TRUNCATE)
13817 V = V.getOperand(i: 0);
13818 return V;
13819}
13820
13821bool llvm::isBitwiseNot(SDValue V, bool AllowUndefs) {
13822 if (V.getOpcode() != ISD::XOR)
13823 return false;
13824 V = peekThroughBitcasts(V: V.getOperand(i: 1));
13825 unsigned NumBits = V.getScalarValueSizeInBits();
13826 ConstantSDNode *C =
13827 isConstOrConstSplat(N: V, AllowUndefs, /*AllowTruncation*/ true);
13828 return C && (C->getAPIntValue().countr_one() >= NumBits);
13829}
13830
13831ConstantSDNode *llvm::isConstOrConstSplat(SDValue N, bool AllowUndefs,
13832 bool AllowTruncation) {
13833 APInt DemandedElts = getDemandAllEltsMask(V: N);
13834 return isConstOrConstSplat(N, DemandedElts, AllowUndefs, AllowTruncation);
13835}
13836
13837ConstantSDNode *llvm::isConstOrConstSplat(SDValue N, const APInt &DemandedElts,
13838 bool AllowUndefs,
13839 bool AllowTruncation) {
13840 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val&: N))
13841 return CN;
13842
13843 // SplatVectors can truncate their operands. Ignore that case here unless
13844 // AllowTruncation is set.
13845 if (N->getOpcode() == ISD::SPLAT_VECTOR) {
13846 EVT VecEltVT = N->getValueType(ResNo: 0).getVectorElementType();
13847 if (auto *CN = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 0))) {
13848 EVT CVT = CN->getValueType(ResNo: 0);
13849 assert(CVT.bitsGE(VecEltVT) && "Illegal splat_vector element extension");
13850 if (AllowTruncation || CVT == VecEltVT)
13851 return CN;
13852 }
13853 }
13854
13855 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Val&: N)) {
13856 BitVector UndefElements;
13857 ConstantSDNode *CN = BV->getConstantSplatNode(DemandedElts, UndefElements: &UndefElements);
13858
13859 // BuildVectors can truncate their operands. Ignore that case here unless
13860 // AllowTruncation is set.
13861 // TODO: Look into whether we should allow UndefElements in non-DemandedElts
13862 if (CN && (UndefElements.none() || AllowUndefs)) {
13863 EVT CVT = CN->getValueType(ResNo: 0);
13864 EVT NSVT = N.getValueType().getScalarType();
13865 assert(CVT.bitsGE(NSVT) && "Illegal build vector element extension");
13866 if (AllowTruncation || (CVT == NSVT))
13867 return CN;
13868 }
13869 }
13870
13871 return nullptr;
13872}
13873
13874ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N, bool AllowUndefs) {
13875 APInt DemandedElts = getDemandAllEltsMask(V: N);
13876 return isConstOrConstSplatFP(N, DemandedElts, AllowUndefs);
13877}
13878
13879ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N,
13880 const APInt &DemandedElts,
13881 bool AllowUndefs) {
13882 if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(Val&: N))
13883 return CN;
13884
13885 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Val&: N)) {
13886 BitVector UndefElements;
13887 ConstantFPSDNode *CN =
13888 BV->getConstantFPSplatNode(DemandedElts, UndefElements: &UndefElements);
13889 // TODO: Look into whether we should allow UndefElements in non-DemandedElts
13890 if (CN && (UndefElements.none() || AllowUndefs))
13891 return CN;
13892 }
13893
13894 if (N.getOpcode() == ISD::SPLAT_VECTOR)
13895 if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(Val: N.getOperand(i: 0)))
13896 return CN;
13897
13898 return nullptr;
13899}
13900
13901bool llvm::isNullOrNullSplat(SDValue N, bool AllowUndefs) {
13902 // TODO: may want to use peekThroughBitcast() here.
13903 ConstantSDNode *C =
13904 isConstOrConstSplat(N, AllowUndefs, /*AllowTruncation=*/true);
13905 return C && C->isZero();
13906}
13907
13908bool llvm::isOneOrOneSplat(SDValue N, bool AllowUndefs) {
13909 ConstantSDNode *C =
13910 isConstOrConstSplat(N, AllowUndefs, /*AllowTruncation*/ true);
13911 return C && C->isOne();
13912}
13913
13914bool llvm::isOneOrOneSplatFP(SDValue N, bool AllowUndefs) {
13915 ConstantFPSDNode *C = isConstOrConstSplatFP(N, AllowUndefs);
13916 return C && C->isOne();
13917}
13918
13919bool llvm::isAllOnesOrAllOnesSplat(SDValue N, bool AllowUndefs) {
13920 N = peekThroughBitcasts(V: N);
13921 unsigned BitWidth = N.getScalarValueSizeInBits();
13922 ConstantSDNode *C =
13923 isConstOrConstSplat(N, AllowUndefs, /*AllowTruncation=*/true);
13924 return C && C->getAPIntValue().countTrailingOnes() >= BitWidth;
13925}
13926
13927bool llvm::isOnesOrOnesSplat(SDValue N, bool AllowUndefs) {
13928 ConstantSDNode *C = isConstOrConstSplat(N, AllowUndefs);
13929 return C && APInt::isSameValue(I1: C->getAPIntValue(),
13930 I2: APInt(C->getAPIntValue().getBitWidth(), 1));
13931}
13932
13933bool llvm::isZeroOrZeroSplat(SDValue N, bool AllowUndefs) {
13934 N = peekThroughBitcasts(V: N);
13935 ConstantSDNode *C = isConstOrConstSplat(N, AllowUndefs, AllowTruncation: true);
13936 return C && C->isZero();
13937}
13938
13939bool llvm::isZeroOrZeroSplatFP(SDValue N, bool AllowUndefs) {
13940 ConstantFPSDNode *C = isConstOrConstSplatFP(N, AllowUndefs);
13941 return C && C->isZero();
13942}
13943
13944HandleSDNode::~HandleSDNode() {
13945 DropOperands();
13946}
13947
13948MemSDNode::MemSDNode(
13949 unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTs, EVT memvt,
13950 PointerUnion<MachineMemOperand *, MachineMemOperand **> memrefs)
13951 : SDNode(Opc, Order, dl, VTs), MemoryVT(memvt), MemRefs(memrefs) {
13952 bool IsVolatile = false;
13953 bool IsNonTemporal = false;
13954 bool IsDereferenceable = true;
13955 bool IsInvariant = true;
13956 for (const MachineMemOperand *MMO : memoperands()) {
13957 IsVolatile |= MMO->isVolatile();
13958 IsNonTemporal |= MMO->isNonTemporal();
13959 IsDereferenceable &= MMO->isDereferenceable();
13960 IsInvariant &= MMO->isInvariant();
13961 }
13962 MemSDNodeBits.IsVolatile = IsVolatile;
13963 MemSDNodeBits.IsNonTemporal = IsNonTemporal;
13964 MemSDNodeBits.IsDereferenceable = IsDereferenceable;
13965 MemSDNodeBits.IsInvariant = IsInvariant;
13966
13967 // For the single-MMO case, we check here that the size of the memory operand
13968 // fits within the size of the MMO. This is because the MMO might indicate
13969 // only a possible address range instead of specifying the affected memory
13970 // addresses precisely.
13971 assert((getNumMemOperands() != 1 || !getMemOperand()->getType().isValid() ||
13972 TypeSize::isKnownLE(memvt.getStoreSize(),
13973 getMemOperand()->getSize().getValue())) &&
13974 "Size mismatch!");
13975}
13976
13977namespace {
13978
13979 struct EVTArray {
13980 std::vector<EVT> VTs;
13981
13982 EVTArray() {
13983 VTs.reserve(n: MVT::VALUETYPE_SIZE);
13984 for (unsigned i = 0; i < MVT::VALUETYPE_SIZE; ++i)
13985 VTs.push_back(x: MVT((MVT::SimpleValueType)i));
13986 }
13987 };
13988
13989} // end anonymous namespace
13990
13991/// getValueTypeList - Return a pointer to the specified value type.
13992///
13993const EVT *SDNode::getValueTypeList(MVT VT) {
13994 static EVTArray SimpleVTArray;
13995
13996 assert(VT < MVT::VALUETYPE_SIZE && "Value type out of range!");
13997 return &SimpleVTArray.VTs[VT.SimpleTy];
13998}
13999
14000/// hasAnyUseOfValue - Return true if there are any use of the indicated
14001/// value. This method ignores uses of other values defined by this operation.
14002bool SDNode::hasAnyUseOfValue(unsigned Value) const {
14003 assert(Value < getNumValues() && "Bad value!");
14004
14005 for (SDUse &U : uses())
14006 if (U.getResNo() == Value)
14007 return true;
14008
14009 return false;
14010}
14011
14012/// isOnlyUserOf - Return true if this node is the only use of N.
14013bool SDNode::isOnlyUserOf(const SDNode *N) const {
14014 bool Seen = false;
14015 for (const SDNode *User : N->users()) {
14016 if (User == this)
14017 Seen = true;
14018 else
14019 return false;
14020 }
14021
14022 return Seen;
14023}
14024
14025/// Return true if the only users of N are contained in Nodes.
14026bool SDNode::areOnlyUsersOf(ArrayRef<const SDNode *> Nodes, const SDNode *N) {
14027 bool Seen = false;
14028 for (const SDNode *User : N->users()) {
14029 if (llvm::is_contained(Range&: Nodes, Element: User))
14030 Seen = true;
14031 else
14032 return false;
14033 }
14034
14035 return Seen;
14036}
14037
14038/// Return true if the referenced return value is an operand of N.
14039bool SDValue::isOperandOf(const SDNode *N) const {
14040 return is_contained(Range: N->op_values(), Element: *this);
14041}
14042
14043bool SDNode::isOperandOf(const SDNode *N) const {
14044 return any_of(Range: N->op_values(),
14045 P: [this](SDValue Op) { return this == Op.getNode(); });
14046}
14047
14048/// reachesChainWithoutSideEffects - Return true if this operand (which must
14049/// be a chain) reaches the specified operand without crossing any
14050/// side-effecting instructions on any chain path. In practice, this looks
14051/// through token factors and non-volatile loads. In order to remain efficient,
14052/// this only looks a couple of nodes in, it does not do an exhaustive search.
14053///
14054/// Note that we only need to examine chains when we're searching for
14055/// side-effects; SelectionDAG requires that all side-effects are represented
14056/// by chains, even if another operand would force a specific ordering. This
14057/// constraint is necessary to allow transformations like splitting loads.
14058bool SDValue::reachesChainWithoutSideEffects(SDValue Dest,
14059 unsigned Depth) const {
14060 if (*this == Dest) return true;
14061
14062 // Don't search too deeply, we just want to be able to see through
14063 // TokenFactor's etc.
14064 if (Depth == 0) return false;
14065
14066 // If this is a token factor, all inputs to the TF happen in parallel.
14067 if (getOpcode() == ISD::TokenFactor) {
14068 // First, try a shallow search.
14069 if (is_contained(Range: (*this)->ops(), Element: Dest)) {
14070 // We found the chain we want as an operand of this TokenFactor.
14071 // Essentially, we reach the chain without side-effects if we could
14072 // serialize the TokenFactor into a simple chain of operations with
14073 // Dest as the last operation. This is automatically true if the
14074 // chain has one use: there are no other ordering constraints.
14075 // If the chain has more than one use, we give up: some other
14076 // use of Dest might force a side-effect between Dest and the current
14077 // node.
14078 if (Dest.hasOneUse())
14079 return true;
14080 }
14081 // Next, try a deep search: check whether every operand of the TokenFactor
14082 // reaches Dest.
14083 return llvm::all_of(Range: (*this)->ops(), P: [=](SDValue Op) {
14084 return Op.reachesChainWithoutSideEffects(Dest, Depth: Depth - 1);
14085 });
14086 }
14087
14088 // Loads don't have side effects, look through them.
14089 if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(Val: *this)) {
14090 if (Ld->isUnordered())
14091 return Ld->getChain().reachesChainWithoutSideEffects(Dest, Depth: Depth-1);
14092 }
14093 return false;
14094}
14095
14096bool SDNode::hasPredecessor(const SDNode *N) const {
14097 SmallPtrSet<const SDNode *, 32> Visited;
14098 SmallVector<const SDNode *, 16> Worklist;
14099 Worklist.push_back(Elt: this);
14100 return hasPredecessorHelper(N, Visited, Worklist);
14101}
14102
14103void SDNode::intersectFlagsWith(const SDNodeFlags Flags) {
14104 this->Flags &= Flags;
14105}
14106
14107SDValue
14108SelectionDAG::matchBinOpReduction(SDNode *Extract, ISD::NodeType &BinOp,
14109 ArrayRef<ISD::NodeType> CandidateBinOps,
14110 bool AllowPartials) {
14111 // The pattern must end in an extract from index 0.
14112 if (Extract->getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
14113 !isNullConstant(V: Extract->getOperand(Num: 1)))
14114 return SDValue();
14115
14116 // Match against one of the candidate binary ops.
14117 SDValue Op = Extract->getOperand(Num: 0);
14118 if (llvm::none_of(Range&: CandidateBinOps, P: [Op](ISD::NodeType BinOp) {
14119 return Op.getOpcode() == unsigned(BinOp);
14120 }))
14121 return SDValue();
14122
14123 // Floating-point reductions may require relaxed constraints on the final step
14124 // of the reduction because they may reorder intermediate operations.
14125 unsigned CandidateBinOp = Op.getOpcode();
14126 if (Op.getValueType().isFloatingPoint()) {
14127 SDNodeFlags Flags = Op->getFlags();
14128 switch (CandidateBinOp) {
14129 case ISD::FADD:
14130 if (!Flags.hasNoSignedZeros() || !Flags.hasAllowReassociation())
14131 return SDValue();
14132 break;
14133 default:
14134 llvm_unreachable("Unhandled FP opcode for binop reduction");
14135 }
14136 }
14137
14138 // Matching failed - attempt to see if we did enough stages that a partial
14139 // reduction from a subvector is possible.
14140 auto PartialReduction = [&](SDValue Op, unsigned NumSubElts) {
14141 if (!AllowPartials || !Op)
14142 return SDValue();
14143 EVT OpVT = Op.getValueType();
14144 EVT OpSVT = OpVT.getScalarType();
14145 EVT SubVT = EVT::getVectorVT(Context&: *getContext(), VT: OpSVT, NumElements: NumSubElts);
14146 if (TLI->getExtractSubvectorCost(ResVT: SubVT, SrcVT: OpVT, Index: 0) >
14147 TargetLowering::ExtractSubvectorCost::Cheap)
14148 return SDValue();
14149 BinOp = (ISD::NodeType)CandidateBinOp;
14150 return getExtractSubvector(DL: SDLoc(Op), VT: SubVT, Vec: Op, Idx: 0);
14151 };
14152
14153 // At each stage, we're looking for something that looks like:
14154 // %s = shufflevector <8 x i32> %op, <8 x i32> undef,
14155 // <8 x i32> <i32 2, i32 3, i32 undef, i32 undef,
14156 // i32 undef, i32 undef, i32 undef, i32 undef>
14157 // %a = binop <8 x i32> %op, %s
14158 // Where the mask changes according to the stage. E.g. for a 3-stage pyramid,
14159 // we expect something like:
14160 // <4,5,6,7,u,u,u,u>
14161 // <2,3,u,u,u,u,u,u>
14162 // <1,u,u,u,u,u,u,u>
14163 // While a partial reduction match would be:
14164 // <2,3,u,u,u,u,u,u>
14165 // <1,u,u,u,u,u,u,u>
14166 unsigned Stages = Log2_32(Value: Op.getValueType().getVectorNumElements());
14167 SDValue PrevOp;
14168 for (unsigned i = 0; i < Stages; ++i) {
14169 unsigned MaskEnd = (1 << i);
14170
14171 if (Op.getOpcode() != CandidateBinOp)
14172 return PartialReduction(PrevOp, MaskEnd);
14173
14174 SDValue Op0 = Op.getOperand(i: 0);
14175 SDValue Op1 = Op.getOperand(i: 1);
14176
14177 ShuffleVectorSDNode *Shuffle = dyn_cast<ShuffleVectorSDNode>(Val&: Op0);
14178 if (Shuffle) {
14179 Op = Op1;
14180 } else {
14181 Shuffle = dyn_cast<ShuffleVectorSDNode>(Val&: Op1);
14182 Op = Op0;
14183 }
14184
14185 // The first operand of the shuffle should be the same as the other operand
14186 // of the binop.
14187 if (!Shuffle || Shuffle->getOperand(Num: 0) != Op)
14188 return PartialReduction(PrevOp, MaskEnd);
14189
14190 // Verify the shuffle has the expected (at this stage of the pyramid) mask.
14191 for (int Index = 0; Index < (int)MaskEnd; ++Index)
14192 if (Shuffle->getMaskElt(Idx: Index) != (int)(MaskEnd + Index))
14193 return PartialReduction(PrevOp, MaskEnd);
14194
14195 PrevOp = Op;
14196 }
14197
14198 // Handle subvector reductions, which tend to appear after the shuffle
14199 // reduction stages.
14200 while (Op.getOpcode() == CandidateBinOp) {
14201 unsigned NumElts = Op.getValueType().getVectorNumElements();
14202 SDValue Op0 = Op.getOperand(i: 0);
14203 SDValue Op1 = Op.getOperand(i: 1);
14204 if (Op0.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
14205 Op1.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
14206 Op0.getOperand(i: 0) != Op1.getOperand(i: 0))
14207 break;
14208 SDValue Src = Op0.getOperand(i: 0);
14209 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
14210 if (NumSrcElts != (2 * NumElts))
14211 break;
14212 if (!(Op0.getConstantOperandAPInt(i: 1) == 0 &&
14213 Op1.getConstantOperandAPInt(i: 1) == NumElts) &&
14214 !(Op1.getConstantOperandAPInt(i: 1) == 0 &&
14215 Op0.getConstantOperandAPInt(i: 1) == NumElts))
14216 break;
14217 Op = Src;
14218 }
14219
14220 BinOp = (ISD::NodeType)CandidateBinOp;
14221 return Op;
14222}
14223
14224SDValue SelectionDAG::UnrollVectorOp(SDNode *N, unsigned ResNE) {
14225 EVT VT = N->getValueType(ResNo: 0);
14226 EVT EltVT = VT.getVectorElementType();
14227 unsigned NE = getMaxRuntimeNumElements(VT);
14228
14229 if (VT.isScalableVector() && (NE == 0 || ResNE != 0))
14230 reportFatalUsageError(reason: "Cannot unroll scalable vector!");
14231
14232 assert(NE && "Nothing to unroll!");
14233 SDLoc dl(N);
14234
14235 // If ResNE is 0, fully unroll the vector op.
14236 if (ResNE == 0)
14237 ResNE = NE;
14238 else if (NE > ResNE)
14239 NE = ResNE;
14240
14241 if (N->getNumValues() == 2) {
14242 SmallVector<SDValue, 8> Scalars0, Scalars1;
14243 SmallVector<SDValue, 4> Operands(N->getNumOperands());
14244 EVT VT1 = N->getValueType(ResNo: 1);
14245 EVT EltVT1 = VT1.getVectorElementType();
14246
14247 unsigned i;
14248 for (i = 0; i != NE; ++i) {
14249 for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) {
14250 SDValue Operand = N->getOperand(Num: j);
14251 EVT OperandVT = Operand.getValueType();
14252
14253 // A vector operand; extract a single element.
14254 EVT OperandEltVT = OperandVT.getVectorElementType();
14255 Operands[j] = getExtractVectorElt(DL: dl, VT: OperandEltVT, Vec: Operand, Idx: i);
14256 }
14257
14258 SDValue EltOp = getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {EltVT, EltVT1}, Ops: Operands);
14259 Scalars0.push_back(Elt: EltOp);
14260 Scalars1.push_back(Elt: EltOp.getValue(R: 1));
14261 }
14262
14263 for (; i < ResNE; ++i) {
14264 Scalars0.push_back(Elt: getUNDEF(VT: EltVT));
14265 Scalars1.push_back(Elt: getUNDEF(VT: EltVT1));
14266 }
14267
14268 ElementCount ResEC = VT.isScalableVector() ? VT.getVectorElementCount()
14269 : ElementCount::getFixed(MinVal: ResNE);
14270 EVT VecVT = EVT::getVectorVT(Context&: *getContext(), VT: EltVT, EC: ResEC);
14271 EVT VecVT1 = EVT::getVectorVT(Context&: *getContext(), VT: EltVT1, EC: ResEC);
14272 SDValue Vec0 = buildVectorFromUnrolledParts(VT: VecVT, DL: dl, Scalars: Scalars0);
14273 SDValue Vec1 = buildVectorFromUnrolledParts(VT: VecVT1, DL: dl, Scalars: Scalars1);
14274 return getMergeValues(Ops: {Vec0, Vec1}, dl);
14275 }
14276
14277 assert(N->getNumValues() == 1 &&
14278 "Can't unroll a vector with multiple results!");
14279
14280 SmallVector<SDValue, 8> Scalars;
14281 SmallVector<SDValue, 4> Operands(N->getNumOperands());
14282
14283 unsigned i;
14284 for (i= 0; i != NE; ++i) {
14285 for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) {
14286 SDValue Operand = N->getOperand(Num: j);
14287 EVT OperandVT = Operand.getValueType();
14288 if (OperandVT.isVector()) {
14289 // A vector operand; extract a single element.
14290 EVT OperandEltVT = OperandVT.getVectorElementType();
14291 Operands[j] = getExtractVectorElt(DL: dl, VT: OperandEltVT, Vec: Operand, Idx: i);
14292 } else {
14293 // A scalar operand; just use it as is.
14294 Operands[j] = Operand;
14295 }
14296 }
14297
14298 switch (N->getOpcode()) {
14299 default: {
14300 Scalars.push_back(Elt: getNode(Opcode: N->getOpcode(), DL: dl, VT: EltVT, Ops: Operands,
14301 Flags: N->getFlags()));
14302 break;
14303 }
14304 case ISD::VSELECT:
14305 Scalars.push_back(
14306 Elt: getNode(Opcode: ISD::SELECT, DL: dl, VT: EltVT, Ops: Operands, Flags: N->getFlags()));
14307 break;
14308 case ISD::SHL:
14309 case ISD::SRA:
14310 case ISD::SRL:
14311 case ISD::ROTL:
14312 case ISD::ROTR:
14313 Scalars.push_back(Elt: getNode(Opcode: N->getOpcode(), DL: dl, VT: EltVT, N1: Operands[0],
14314 N2: getShiftAmountOperand(LHSTy: Operands[0].getValueType(),
14315 Op: Operands[1])));
14316 break;
14317 case ISD::SIGN_EXTEND_INREG: {
14318 EVT ExtVT = cast<VTSDNode>(Val&: Operands[1])->getVT().getVectorElementType();
14319 Scalars.push_back(Elt: getNode(Opcode: N->getOpcode(), DL: dl, VT: EltVT,
14320 N1: Operands[0],
14321 N2: getValueType(VT: ExtVT)));
14322 break;
14323 }
14324 case ISD::ADDRSPACECAST: {
14325 const auto *ASC = cast<AddrSpaceCastSDNode>(Val: N);
14326 Scalars.push_back(
14327 Elt: getAddrSpaceCast(dl, VT: EltVT, Ptr: Operands[0], SrcAS: ASC->getSrcAddressSpace(),
14328 DestAS: ASC->getDestAddressSpace(), Flags: ASC->getFlags()));
14329 break;
14330 }
14331 }
14332 }
14333
14334 for (; i < ResNE; ++i)
14335 Scalars.push_back(Elt: getUNDEF(VT: EltVT));
14336
14337 EVT VecVT = VT.isScalableVector()
14338 ? VT
14339 : EVT::getVectorVT(Context&: *getContext(), VT: EltVT, NumElements: ResNE);
14340 return buildVectorFromUnrolledParts(VT: VecVT, DL: dl, Scalars);
14341}
14342
14343std::pair<SDValue, SDValue> SelectionDAG::UnrollVectorOverflowOp(
14344 SDNode *N, unsigned ResNE) {
14345 unsigned Opcode = N->getOpcode();
14346 assert((Opcode == ISD::UADDO || Opcode == ISD::SADDO ||
14347 Opcode == ISD::USUBO || Opcode == ISD::SSUBO ||
14348 Opcode == ISD::UMULO || Opcode == ISD::SMULO) &&
14349 "Expected an overflow opcode");
14350
14351 EVT ResVT = N->getValueType(ResNo: 0);
14352 EVT OvVT = N->getValueType(ResNo: 1);
14353 EVT ResEltVT = ResVT.getVectorElementType();
14354 EVT OvEltVT = OvVT.getVectorElementType();
14355 SDLoc dl(N);
14356
14357 // If ResNE is 0, fully unroll the vector op.
14358 unsigned NE = ResVT.getVectorNumElements();
14359 if (ResNE == 0)
14360 ResNE = NE;
14361 else if (NE > ResNE)
14362 NE = ResNE;
14363
14364 SmallVector<SDValue, 8> LHSScalars;
14365 SmallVector<SDValue, 8> RHSScalars;
14366 ExtractVectorElements(Op: N->getOperand(Num: 0), Args&: LHSScalars, Start: 0, Count: NE);
14367 ExtractVectorElements(Op: N->getOperand(Num: 1), Args&: RHSScalars, Start: 0, Count: NE);
14368
14369 EVT SVT = TLI->getSetCCResultType(DL: getDataLayout(), Context&: *getContext(), VT: ResEltVT);
14370 SDVTList VTs = getVTList(VT1: ResEltVT, VT2: SVT);
14371 SmallVector<SDValue, 8> ResScalars;
14372 SmallVector<SDValue, 8> OvScalars;
14373 for (unsigned i = 0; i < NE; ++i) {
14374 SDValue Res = getNode(Opcode, DL: dl, VTList: VTs, N1: LHSScalars[i], N2: RHSScalars[i]);
14375 SDValue Ov =
14376 getSelect(DL: dl, VT: OvEltVT, Cond: Res.getValue(R: 1),
14377 LHS: getBoolConstant(V: true, DL: dl, VT: OvEltVT, OpVT: ResVT),
14378 RHS: getConstant(Val: 0, DL: dl, VT: OvEltVT));
14379
14380 ResScalars.push_back(Elt: Res);
14381 OvScalars.push_back(Elt: Ov);
14382 }
14383
14384 ResScalars.append(NumInputs: ResNE - NE, Elt: getUNDEF(VT: ResEltVT));
14385 OvScalars.append(NumInputs: ResNE - NE, Elt: getUNDEF(VT: OvEltVT));
14386
14387 EVT NewResVT = EVT::getVectorVT(Context&: *getContext(), VT: ResEltVT, NumElements: ResNE);
14388 EVT NewOvVT = EVT::getVectorVT(Context&: *getContext(), VT: OvEltVT, NumElements: ResNE);
14389 return std::make_pair(x: getBuildVector(VT: NewResVT, DL: dl, Ops: ResScalars),
14390 y: getBuildVector(VT: NewOvVT, DL: dl, Ops: OvScalars));
14391}
14392
14393static bool areNonVolatileConsecutiveLoadsOrStores(LSBaseSDNode *LS,
14394 LSBaseSDNode *Base,
14395 unsigned Bytes, int Dist,
14396 const SelectionDAG &DAG) {
14397 if (LS->isVolatile() || Base->isVolatile())
14398 return false;
14399 // TODO: probably too restrictive for atomics, revisit
14400 if (!LS->isSimple())
14401 return false;
14402 if (LS->isIndexed() || Base->isIndexed())
14403 return false;
14404 if (LS->getChain() != Base->getChain())
14405 return false;
14406 EVT VT = LS->getMemoryVT();
14407 if (VT.getSizeInBits() / 8 != Bytes)
14408 return false;
14409
14410 auto BaseLocDecomp = BaseIndexOffset::match(N: Base, DAG);
14411 auto LocDecomp = BaseIndexOffset::match(N: LS, DAG);
14412
14413 int64_t Offset = 0;
14414 if (BaseLocDecomp.equalBaseIndex(Other: LocDecomp, DAG, Off&: Offset))
14415 return (Dist * (int64_t)Bytes == Offset);
14416 return false;
14417}
14418
14419bool SelectionDAG::areNonVolatileConsecutiveLoads(LoadSDNode *LD,
14420 LoadSDNode *Base,
14421 unsigned Bytes,
14422 int Dist) const {
14423 return areNonVolatileConsecutiveLoadsOrStores(LS: LD, Base, Bytes, Dist, DAG: *this);
14424}
14425
14426bool SelectionDAG::areNonVolatileConsecutiveStores(StoreSDNode *ST,
14427 StoreSDNode *Base,
14428 unsigned Bytes,
14429 int Dist) const {
14430 return areNonVolatileConsecutiveLoadsOrStores(LS: ST, Base, Bytes, Dist, DAG: *this);
14431}
14432
14433/// InferPtrAlignment - Infer alignment of a load / store address. Return
14434/// std::nullopt if it cannot be inferred.
14435MaybeAlign SelectionDAG::InferPtrAlign(SDValue Ptr) const {
14436 // If this is a GlobalAddress + cst, return the alignment.
14437 const GlobalValue *GV = nullptr;
14438 int64_t GVOffset = 0;
14439 if (TLI->isGAPlusOffset(N: Ptr.getNode(), GA&: GV, Offset&: GVOffset)) {
14440 unsigned PtrWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType());
14441 KnownBits Known(PtrWidth);
14442 llvm::computeKnownBits(V: GV, Known, DL: getDataLayout());
14443 unsigned AlignBits = Known.countMinTrailingZeros();
14444 if (AlignBits)
14445 return commonAlignment(A: Align(1ull << std::min(a: 31U, b: AlignBits)), Offset: GVOffset);
14446 }
14447
14448 // If this is a direct reference to a stack slot, use information about the
14449 // stack slot's alignment.
14450 int FrameIdx = INT_MIN;
14451 int64_t FrameOffset = 0;
14452 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Ptr)) {
14453 FrameIdx = FI->getIndex();
14454 } else if (isBaseWithConstantOffset(Op: Ptr) &&
14455 isa<FrameIndexSDNode>(Val: Ptr.getOperand(i: 0))) {
14456 // Handle FI+Cst
14457 FrameIdx = cast<FrameIndexSDNode>(Val: Ptr.getOperand(i: 0))->getIndex();
14458 FrameOffset = Ptr.getConstantOperandVal(i: 1);
14459 }
14460
14461 if (FrameIdx != INT_MIN) {
14462 const MachineFrameInfo &MFI = getMachineFunction().getFrameInfo();
14463 return commonAlignment(A: MFI.getObjectAlign(ObjectIdx: FrameIdx), Offset: FrameOffset);
14464 }
14465
14466 return std::nullopt;
14467}
14468
14469/// Split the scalar node with EXTRACT_ELEMENT using the provided
14470/// VTs and return the low/high part.
14471std::pair<SDValue, SDValue> SelectionDAG::SplitScalar(const SDValue &N,
14472 const SDLoc &DL,
14473 const EVT &LoVT,
14474 const EVT &HiVT) {
14475 assert(!LoVT.isVector() && !HiVT.isVector() && !N.getValueType().isVector() &&
14476 "Split node must be a scalar type");
14477 SDValue Lo =
14478 getNode(Opcode: ISD::EXTRACT_ELEMENT, DL, VT: LoVT, N1: N, N2: getIntPtrConstant(Val: 0, DL));
14479 SDValue Hi =
14480 getNode(Opcode: ISD::EXTRACT_ELEMENT, DL, VT: HiVT, N1: N, N2: getIntPtrConstant(Val: 1, DL));
14481 return std::make_pair(x&: Lo, y&: Hi);
14482}
14483
14484/// GetSplitDestVTs - Compute the VTs needed for the low/hi parts of a type
14485/// which is split (or expanded) into two not necessarily identical pieces.
14486std::pair<EVT, EVT> SelectionDAG::GetSplitDestVTs(const EVT &VT) const {
14487 // Currently all types are split in half.
14488 EVT LoVT, HiVT;
14489 if (!VT.isVector())
14490 LoVT = HiVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT);
14491 else
14492 LoVT = HiVT = VT.getHalfNumVectorElementsVT(Context&: *getContext());
14493
14494 return std::make_pair(x&: LoVT, y&: HiVT);
14495}
14496
14497/// GetDependentSplitDestVTs - Compute the VTs needed for the low/hi parts of a
14498/// type, dependent on an enveloping VT that has been split into two identical
14499/// pieces. Sets the HiIsEmpty flag when hi type has zero storage size.
14500std::pair<EVT, EVT>
14501SelectionDAG::GetDependentSplitDestVTs(const EVT &VT, const EVT &EnvVT,
14502 bool *HiIsEmpty) const {
14503 EVT EltTp = VT.getVectorElementType();
14504 // Examples:
14505 // custom VL=8 with enveloping VL=8/8 yields 8/0 (hi empty)
14506 // custom VL=9 with enveloping VL=8/8 yields 8/1
14507 // custom VL=10 with enveloping VL=8/8 yields 8/2
14508 // etc.
14509 ElementCount VTNumElts = VT.getVectorElementCount();
14510 ElementCount EnvNumElts = EnvVT.getVectorElementCount();
14511 assert(VTNumElts.isScalable() == EnvNumElts.isScalable() &&
14512 "Mixing fixed width and scalable vectors when enveloping a type");
14513 EVT LoVT, HiVT;
14514 if (VTNumElts.getKnownMinValue() > EnvNumElts.getKnownMinValue()) {
14515 LoVT = EVT::getVectorVT(Context&: *getContext(), VT: EltTp, EC: EnvNumElts);
14516 HiVT = EVT::getVectorVT(Context&: *getContext(), VT: EltTp, EC: VTNumElts - EnvNumElts);
14517 *HiIsEmpty = false;
14518 } else {
14519 // Flag that hi type has zero storage size, but return split envelop type
14520 // (this would be easier if vector types with zero elements were allowed).
14521 LoVT = EVT::getVectorVT(Context&: *getContext(), VT: EltTp, EC: VTNumElts);
14522 HiVT = EVT::getVectorVT(Context&: *getContext(), VT: EltTp, EC: EnvNumElts);
14523 *HiIsEmpty = true;
14524 }
14525 return std::make_pair(x&: LoVT, y&: HiVT);
14526}
14527
14528/// SplitVector - Split the vector with EXTRACT_SUBVECTOR and return the
14529/// low/high part.
14530std::pair<SDValue, SDValue>
14531SelectionDAG::SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT,
14532 const EVT &HiVT) {
14533 assert(LoVT.isScalableVector() == HiVT.isScalableVector() &&
14534 LoVT.isScalableVector() == N.getValueType().isScalableVector() &&
14535 "Splitting vector with an invalid mixture of fixed and scalable "
14536 "vector types");
14537 assert(LoVT.getVectorMinNumElements() + HiVT.getVectorMinNumElements() <=
14538 N.getValueType().getVectorMinNumElements() &&
14539 "More vector elements requested than available!");
14540 SDValue Lo, Hi;
14541 Lo = getExtractSubvector(DL, VT: LoVT, Vec: N, Idx: 0);
14542 // For scalable vectors it is safe to use LoVT.getVectorMinNumElements()
14543 // (rather than having to use ElementCount), because EXTRACT_SUBVECTOR scales
14544 // IDX with the runtime scaling factor of the result vector type. For
14545 // fixed-width result vectors, that runtime scaling factor is 1.
14546 Hi = getExtractSubvector(DL, VT: HiVT, Vec: N, Idx: LoVT.getVectorMinNumElements());
14547 return std::make_pair(x&: Lo, y&: Hi);
14548}
14549
14550std::pair<SDValue, SDValue> SelectionDAG::SplitEVL(SDValue N, EVT VecVT,
14551 const SDLoc &DL) {
14552 // Split the vector length parameter.
14553 // %evl -> umin(%evl, %halfnumelts) and usubsat(%evl - %halfnumelts).
14554 EVT VT = N.getValueType();
14555 assert(VecVT.getVectorElementCount().isKnownEven() &&
14556 "Expecting the mask to be an evenly-sized vector");
14557 SDValue HalfNumElts = getElementCount(
14558 DL, VT, EC: VecVT.getVectorElementCount().divideCoefficientBy(RHS: 2));
14559 SDValue Lo = getNode(Opcode: ISD::UMIN, DL, VT, N1: N, N2: HalfNumElts);
14560 SDValue Hi = getNode(Opcode: ISD::USUBSAT, DL, VT, N1: N, N2: HalfNumElts);
14561 return std::make_pair(x&: Lo, y&: Hi);
14562}
14563
14564/// Widen the vector up to the next power of two using INSERT_SUBVECTOR.
14565SDValue SelectionDAG::WidenVector(const SDValue &N, const SDLoc &DL) {
14566 EVT VT = N.getValueType();
14567 EVT WideVT = EVT::getVectorVT(Context&: *getContext(), VT: VT.getVectorElementType(),
14568 NumElements: NextPowerOf2(A: VT.getVectorNumElements()));
14569 return getInsertSubvector(DL, Vec: getPOISON(VT: WideVT), SubVec: N, Idx: 0);
14570}
14571
14572void SelectionDAG::ExtractVectorElements(SDValue Op,
14573 SmallVectorImpl<SDValue> &Args,
14574 unsigned Start, unsigned Count,
14575 EVT EltVT) {
14576 EVT VT = Op.getValueType();
14577 if (Count == 0)
14578 Count = VT.getVectorNumElements();
14579 if (EltVT == EVT())
14580 EltVT = VT.getVectorElementType();
14581 SDLoc SL(Op);
14582 for (unsigned i = Start, e = Start + Count; i != e; ++i) {
14583 Args.push_back(Elt: getExtractVectorElt(DL: SL, VT: EltVT, Vec: Op, Idx: i));
14584 }
14585}
14586
14587// getAddressSpace - Return the address space this GlobalAddress belongs to.
14588unsigned GlobalAddressSDNode::getAddressSpace() const {
14589 return getGlobal()->getType()->getAddressSpace();
14590}
14591
14592Type *ConstantPoolSDNode::getType() const {
14593 if (isMachineConstantPoolEntry())
14594 return Val.MachineCPVal->getType();
14595 return Val.ConstVal->getType();
14596}
14597
14598bool BuildVectorSDNode::isConstantSplat(APInt &SplatValue, APInt &SplatUndef,
14599 unsigned &SplatBitSize,
14600 bool &HasAnyUndefs,
14601 unsigned MinSplatBits,
14602 bool IsBigEndian) const {
14603 EVT VT = getValueType(ResNo: 0);
14604 assert(VT.isVector() && "Expected a vector type");
14605 unsigned VecWidth = VT.getSizeInBits();
14606 if (MinSplatBits > VecWidth)
14607 return false;
14608
14609 // FIXME: The widths are based on this node's type, but build vectors can
14610 // truncate their operands.
14611 SplatValue = APInt(VecWidth, 0);
14612 SplatUndef = APInt(VecWidth, 0);
14613
14614 // Get the bits. Bits with undefined values (when the corresponding element
14615 // of the vector is an ISD::UNDEF value) are set in SplatUndef and cleared
14616 // in SplatValue. If any of the values are not constant, give up and return
14617 // false.
14618 unsigned int NumOps = getNumOperands();
14619 assert(NumOps > 0 && "isConstantSplat has 0-size build vector");
14620 unsigned EltWidth = VT.getScalarSizeInBits();
14621
14622 for (unsigned j = 0; j < NumOps; ++j) {
14623 unsigned i = IsBigEndian ? NumOps - 1 - j : j;
14624 SDValue OpVal = getOperand(Num: i);
14625 unsigned BitPos = j * EltWidth;
14626
14627 if (OpVal.isUndef())
14628 SplatUndef.setBits(loBit: BitPos, hiBit: BitPos + EltWidth);
14629 else if (auto *CN = dyn_cast<ConstantSDNode>(Val&: OpVal))
14630 SplatValue.insertBits(SubBits: CN->getAPIntValue().zextOrTrunc(width: EltWidth), bitPosition: BitPos);
14631 else if (auto *CN = dyn_cast<ConstantFPSDNode>(Val&: OpVal))
14632 SplatValue.insertBits(SubBits: CN->getValueAPF().bitcastToAPInt(), bitPosition: BitPos);
14633 else
14634 return false;
14635 }
14636
14637 // The build_vector is all constants or undefs. Find the smallest element
14638 // size that splats the vector.
14639 HasAnyUndefs = (SplatUndef != 0);
14640
14641 // FIXME: This does not work for vectors with elements less than 8 bits.
14642 while (VecWidth > 8) {
14643 // If we can't split in half, stop here.
14644 if (VecWidth & 1)
14645 break;
14646
14647 unsigned HalfSize = VecWidth / 2;
14648 APInt HighValue = SplatValue.extractBits(numBits: HalfSize, bitPosition: HalfSize);
14649 APInt LowValue = SplatValue.extractBits(numBits: HalfSize, bitPosition: 0);
14650 APInt HighUndef = SplatUndef.extractBits(numBits: HalfSize, bitPosition: HalfSize);
14651 APInt LowUndef = SplatUndef.extractBits(numBits: HalfSize, bitPosition: 0);
14652
14653 // If the two halves do not match (ignoring undef bits), stop here.
14654 if ((HighValue & ~LowUndef) != (LowValue & ~HighUndef) ||
14655 MinSplatBits > HalfSize)
14656 break;
14657
14658 SplatValue = HighValue | LowValue;
14659 SplatUndef = HighUndef & LowUndef;
14660
14661 VecWidth = HalfSize;
14662 }
14663
14664 // FIXME: The loop above only tries to split in halves. But if the input
14665 // vector for example is <3 x i16> it wouldn't be able to detect a
14666 // SplatBitSize of 16. No idea if that is a design flaw currently limiting
14667 // optimizations. I guess that back in the days when this helper was created
14668 // vectors normally was power-of-2 sized.
14669
14670 SplatBitSize = VecWidth;
14671 return true;
14672}
14673
14674SDValue BuildVectorSDNode::getSplatValue(const APInt &DemandedElts,
14675 BitVector *UndefElements) const {
14676 unsigned NumOps = getNumOperands();
14677 if (UndefElements) {
14678 UndefElements->clear();
14679 UndefElements->resize(N: NumOps);
14680 }
14681 assert(NumOps == DemandedElts.getBitWidth() && "Unexpected vector size");
14682 if (!DemandedElts)
14683 return SDValue();
14684 SDValue Splatted;
14685 for (unsigned i = 0; i != NumOps; ++i) {
14686 if (!DemandedElts[i])
14687 continue;
14688 SDValue Op = getOperand(Num: i);
14689 if (Op.isUndef()) {
14690 if (UndefElements)
14691 (*UndefElements)[i] = true;
14692 } else if (!Splatted) {
14693 Splatted = Op;
14694 } else if (Splatted != Op) {
14695 return SDValue();
14696 }
14697 }
14698
14699 if (!Splatted) {
14700 unsigned FirstDemandedIdx = DemandedElts.countr_zero();
14701 assert(getOperand(FirstDemandedIdx).isUndef() &&
14702 "Can only have a splat without a constant for all undefs.");
14703 return getOperand(Num: FirstDemandedIdx);
14704 }
14705
14706 return Splatted;
14707}
14708
14709SDValue BuildVectorSDNode::getSplatValue(BitVector *UndefElements) const {
14710 APInt DemandedElts = APInt::getAllOnes(numBits: getNumOperands());
14711 return getSplatValue(DemandedElts, UndefElements);
14712}
14713
14714bool BuildVectorSDNode::getRepeatedSequence(const APInt &DemandedElts,
14715 SmallVectorImpl<SDValue> &Sequence,
14716 BitVector *UndefElements) const {
14717 unsigned NumOps = getNumOperands();
14718 Sequence.clear();
14719 if (UndefElements) {
14720 UndefElements->clear();
14721 UndefElements->resize(N: NumOps);
14722 }
14723 assert(NumOps == DemandedElts.getBitWidth() && "Unexpected vector size");
14724 if (!DemandedElts || NumOps < 2 || !isPowerOf2_32(Value: NumOps))
14725 return false;
14726
14727 // Set the undefs even if we don't find a sequence (like getSplatValue).
14728 if (UndefElements)
14729 for (unsigned I = 0; I != NumOps; ++I)
14730 if (DemandedElts[I] && getOperand(Num: I).isUndef())
14731 (*UndefElements)[I] = true;
14732
14733 // Iteratively widen the sequence length looking for repetitions.
14734 for (unsigned SeqLen = 1; SeqLen < NumOps; SeqLen *= 2) {
14735 Sequence.append(NumInputs: SeqLen, Elt: SDValue());
14736 for (unsigned I = 0; I != NumOps; ++I) {
14737 if (!DemandedElts[I])
14738 continue;
14739 SDValue &SeqOp = Sequence[I % SeqLen];
14740 SDValue Op = getOperand(Num: I);
14741 if (Op.isUndef()) {
14742 if (!SeqOp)
14743 SeqOp = Op;
14744 continue;
14745 }
14746 if (SeqOp && !SeqOp.isUndef() && SeqOp != Op) {
14747 Sequence.clear();
14748 break;
14749 }
14750 SeqOp = Op;
14751 }
14752 if (!Sequence.empty())
14753 return true;
14754 }
14755
14756 assert(Sequence.empty() && "Failed to empty non-repeating sequence pattern");
14757 return false;
14758}
14759
14760bool BuildVectorSDNode::getRepeatedSequence(SmallVectorImpl<SDValue> &Sequence,
14761 BitVector *UndefElements) const {
14762 APInt DemandedElts = APInt::getAllOnes(numBits: getNumOperands());
14763 return getRepeatedSequence(DemandedElts, Sequence, UndefElements);
14764}
14765
14766ConstantSDNode *
14767BuildVectorSDNode::getConstantSplatNode(const APInt &DemandedElts,
14768 BitVector *UndefElements) const {
14769 return dyn_cast_or_null<ConstantSDNode>(
14770 Val: getSplatValue(DemandedElts, UndefElements));
14771}
14772
14773ConstantSDNode *
14774BuildVectorSDNode::getConstantSplatNode(BitVector *UndefElements) const {
14775 return dyn_cast_or_null<ConstantSDNode>(Val: getSplatValue(UndefElements));
14776}
14777
14778ConstantFPSDNode *
14779BuildVectorSDNode::getConstantFPSplatNode(const APInt &DemandedElts,
14780 BitVector *UndefElements) const {
14781 return dyn_cast_or_null<ConstantFPSDNode>(
14782 Val: getSplatValue(DemandedElts, UndefElements));
14783}
14784
14785ConstantFPSDNode *
14786BuildVectorSDNode::getConstantFPSplatNode(BitVector *UndefElements) const {
14787 return dyn_cast_or_null<ConstantFPSDNode>(Val: getSplatValue(UndefElements));
14788}
14789
14790int32_t
14791BuildVectorSDNode::getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements,
14792 uint32_t BitWidth) const {
14793 if (ConstantFPSDNode *CN =
14794 dyn_cast_or_null<ConstantFPSDNode>(Val: getSplatValue(UndefElements))) {
14795 bool IsExact;
14796 APSInt IntVal(BitWidth);
14797 const APFloat &APF = CN->getValueAPF();
14798 if (APF.convertToInteger(Result&: IntVal, RM: APFloat::rmTowardZero, IsExact: &IsExact) !=
14799 APFloat::opOK ||
14800 !IsExact)
14801 return -1;
14802
14803 return IntVal.exactLogBase2();
14804 }
14805 return -1;
14806}
14807
14808bool BuildVectorSDNode::getConstantRawBits(
14809 bool IsLittleEndian, unsigned DstEltSizeInBits,
14810 SmallVectorImpl<APInt> &RawBitElements, BitVector &UndefElements) const {
14811 // Early-out if this contains anything but Undef/Constant/ConstantFP.
14812 if (!isConstant())
14813 return false;
14814
14815 unsigned NumSrcOps = getNumOperands();
14816 unsigned SrcEltSizeInBits = getValueType(ResNo: 0).getScalarSizeInBits();
14817 assert(((NumSrcOps * SrcEltSizeInBits) % DstEltSizeInBits) == 0 &&
14818 "Invalid bitcast scale");
14819
14820 // Extract raw src bits.
14821 SmallVector<APInt> SrcBitElements(NumSrcOps,
14822 APInt::getZero(numBits: SrcEltSizeInBits));
14823 BitVector SrcUndeElements(NumSrcOps, false);
14824
14825 for (unsigned I = 0; I != NumSrcOps; ++I) {
14826 SDValue Op = getOperand(Num: I);
14827 if (Op.isUndef()) {
14828 SrcUndeElements.set(I);
14829 continue;
14830 }
14831 auto *CInt = dyn_cast<ConstantSDNode>(Val&: Op);
14832 auto *CFP = dyn_cast<ConstantFPSDNode>(Val&: Op);
14833 assert((CInt || CFP) && "Unknown constant");
14834 SrcBitElements[I] = CInt ? CInt->getAPIntValue().trunc(width: SrcEltSizeInBits)
14835 : CFP->getValueAPF().bitcastToAPInt();
14836 }
14837
14838 // Recast to dst width.
14839 recastRawBits(IsLittleEndian, DstEltSizeInBits, DstBitElements&: RawBitElements,
14840 SrcBitElements, DstUndefElements&: UndefElements, SrcUndefElements: SrcUndeElements);
14841 return true;
14842}
14843
14844void BuildVectorSDNode::recastRawBits(bool IsLittleEndian,
14845 unsigned DstEltSizeInBits,
14846 SmallVectorImpl<APInt> &DstBitElements,
14847 ArrayRef<APInt> SrcBitElements,
14848 BitVector &DstUndefElements,
14849 const BitVector &SrcUndefElements) {
14850 unsigned NumSrcOps = SrcBitElements.size();
14851 unsigned SrcEltSizeInBits = SrcBitElements[0].getBitWidth();
14852 assert(((NumSrcOps * SrcEltSizeInBits) % DstEltSizeInBits) == 0 &&
14853 "Invalid bitcast scale");
14854 assert(NumSrcOps == SrcUndefElements.size() &&
14855 "Vector size mismatch");
14856
14857 unsigned NumDstOps = (NumSrcOps * SrcEltSizeInBits) / DstEltSizeInBits;
14858 DstUndefElements.clear();
14859 DstUndefElements.resize(N: NumDstOps, t: false);
14860 DstBitElements.assign(NumElts: NumDstOps, Elt: APInt::getZero(numBits: DstEltSizeInBits));
14861
14862 // Concatenate src elements constant bits together into dst element.
14863 if (SrcEltSizeInBits <= DstEltSizeInBits) {
14864 unsigned Scale = DstEltSizeInBits / SrcEltSizeInBits;
14865 for (unsigned I = 0; I != NumDstOps; ++I) {
14866 DstUndefElements.set(I);
14867 APInt &DstBits = DstBitElements[I];
14868 for (unsigned J = 0; J != Scale; ++J) {
14869 unsigned Idx = (I * Scale) + (IsLittleEndian ? J : (Scale - J - 1));
14870 if (SrcUndefElements[Idx])
14871 continue;
14872 DstUndefElements.reset(Idx: I);
14873 const APInt &SrcBits = SrcBitElements[Idx];
14874 assert(SrcBits.getBitWidth() == SrcEltSizeInBits &&
14875 "Illegal constant bitwidths");
14876 DstBits.insertBits(SubBits: SrcBits, bitPosition: J * SrcEltSizeInBits);
14877 }
14878 }
14879 return;
14880 }
14881
14882 // Split src element constant bits into dst elements.
14883 unsigned Scale = SrcEltSizeInBits / DstEltSizeInBits;
14884 for (unsigned I = 0; I != NumSrcOps; ++I) {
14885 if (SrcUndefElements[I]) {
14886 DstUndefElements.set(I: I * Scale, E: (I + 1) * Scale);
14887 continue;
14888 }
14889 const APInt &SrcBits = SrcBitElements[I];
14890 for (unsigned J = 0; J != Scale; ++J) {
14891 unsigned Idx = (I * Scale) + (IsLittleEndian ? J : (Scale - J - 1));
14892 APInt &DstBits = DstBitElements[Idx];
14893 DstBits = SrcBits.extractBits(numBits: DstEltSizeInBits, bitPosition: J * DstEltSizeInBits);
14894 }
14895 }
14896}
14897
14898bool BuildVectorSDNode::isConstant() const {
14899 for (const SDValue &Op : op_values()) {
14900 unsigned Opc = Op.getOpcode();
14901 if (!Op.isUndef() && Opc != ISD::Constant && Opc != ISD::ConstantFP)
14902 return false;
14903 }
14904 return true;
14905}
14906
14907std::optional<std::pair<APInt, APInt>>
14908BuildVectorSDNode::isArithmeticSequence() const {
14909 unsigned NumOps = getNumOperands();
14910 if (NumOps < 2)
14911 return std::nullopt;
14912
14913 unsigned EltSize = getValueType(ResNo: 0).getScalarSizeInBits();
14914 APInt Start, Stride;
14915 int FirstIdx = -1, SecondIdx = -1;
14916
14917 // Find the first two non-undef constant elements to determine Start and
14918 // Stride, then verify all remaining elements match the sequence.
14919 for (unsigned I = 0; I < NumOps; ++I) {
14920 SDValue Op = getOperand(Num: I);
14921 if (Op->isUndef())
14922 continue;
14923 if (!isa<ConstantSDNode>(Val: Op))
14924 return std::nullopt;
14925
14926 APInt Val = getConstantOperandAPInt(Num: I).trunc(width: EltSize);
14927 if (FirstIdx < 0) {
14928 FirstIdx = I;
14929 Start = Val;
14930 } else if (SecondIdx < 0) {
14931 SecondIdx = I;
14932 // Compute stride using modular arithmetic. Simple division would handle
14933 // common strides (1, 2, -1, etc.), but modular inverse maximizes matches.
14934 // Example: <0, poison, poison, 0xFF> has stride 0x55 since 3*0x55 = 0xFF
14935 // Note that modular arithmetic is agnostic to signed/unsigned.
14936 unsigned IdxDiff = I - FirstIdx;
14937 APInt ValDiff = Val - Start;
14938
14939 // Step 1: Factor out common powers of 2 from IdxDiff and ValDiff.
14940 unsigned CommonPow2Bits = llvm::countr_zero(Val: IdxDiff);
14941 if (ValDiff.countr_zero() < CommonPow2Bits)
14942 return std::nullopt; // ValDiff not divisible by 2^CommonPow2Bits
14943 IdxDiff >>= CommonPow2Bits;
14944 ValDiff.lshrInPlace(ShiftAmt: CommonPow2Bits);
14945
14946 // Step 2: IdxDiff is now odd, so its inverse mod 2^EltSize exists.
14947 // TODO: There are 2^CommonPow2Bits valid strides; currently we only try
14948 // one, but we could try all candidates to handle more cases.
14949 Stride = ValDiff * APInt(EltSize, IdxDiff).multiplicativeInverse();
14950 if (Stride.isZero())
14951 return std::nullopt;
14952
14953 // Step 3: Adjust Start based on the first defined element's index.
14954 Start -= Stride * FirstIdx;
14955 } else {
14956 // Verify this element matches the sequence.
14957 if (Val != Start + Stride * I)
14958 return std::nullopt;
14959 }
14960 }
14961
14962 // Need at least two defined elements.
14963 if (SecondIdx < 0)
14964 return std::nullopt;
14965
14966 return std::make_pair(x&: Start, y&: Stride);
14967}
14968
14969bool ShuffleVectorSDNode::isSplatMask(ArrayRef<int> Mask) {
14970 // Find the first non-undef value in the shuffle mask.
14971 unsigned i, e;
14972 for (i = 0, e = Mask.size(); i != e && Mask[i] < 0; ++i)
14973 /* search */;
14974
14975 // If all elements are undefined, this shuffle can be considered a splat
14976 // (although it should eventually get simplified away completely).
14977 if (i == e)
14978 return true;
14979
14980 // Make sure all remaining elements are either undef or the same as the first
14981 // non-undef value.
14982 for (int Idx = Mask[i]; i != e; ++i)
14983 if (Mask[i] >= 0 && Mask[i] != Idx)
14984 return false;
14985 return true;
14986}
14987
14988// Returns true if it is a constant integer BuildVector or constant integer,
14989// possibly hidden by a bitcast.
14990bool SelectionDAG::isConstantIntBuildVectorOrConstantInt(
14991 SDValue N, bool AllowOpaques) const {
14992 N = peekThroughBitcasts(V: N);
14993
14994 if (auto *C = dyn_cast<ConstantSDNode>(Val&: N))
14995 return AllowOpaques || !C->isOpaque();
14996
14997 if (ISD::isBuildVectorOfConstantSDNodes(N: N.getNode()))
14998 return true;
14999
15000 // Treat a GlobalAddress supporting constant offset folding as a
15001 // constant integer.
15002 if (auto *GA = dyn_cast<GlobalAddressSDNode>(Val&: N))
15003 if (GA->getOpcode() == ISD::GlobalAddress &&
15004 TLI->isOffsetFoldingLegal(GA))
15005 return true;
15006
15007 if ((N.getOpcode() == ISD::SPLAT_VECTOR) &&
15008 isa<ConstantSDNode>(Val: N.getOperand(i: 0)))
15009 return true;
15010 return false;
15011}
15012
15013// Returns true if it is a constant float BuildVector or constant float.
15014bool SelectionDAG::isConstantFPBuildVectorOrConstantFP(SDValue N) const {
15015 if (isa<ConstantFPSDNode>(Val: N))
15016 return true;
15017
15018 if (ISD::isBuildVectorOfConstantFPSDNodes(N: N.getNode()))
15019 return true;
15020
15021 if ((N.getOpcode() == ISD::SPLAT_VECTOR) &&
15022 isa<ConstantFPSDNode>(Val: N.getOperand(i: 0)))
15023 return true;
15024
15025 return false;
15026}
15027
15028std::optional<bool> SelectionDAG::isBoolConstant(SDValue N) const {
15029 ConstantSDNode *Const =
15030 isConstOrConstSplat(N, AllowUndefs: false, /*AllowTruncation=*/true);
15031 if (!Const)
15032 return std::nullopt;
15033
15034 EVT VT = N->getValueType(ResNo: 0);
15035 const APInt CVal = Const->getAPIntValue().trunc(width: VT.getScalarSizeInBits());
15036 switch (TLI->getBooleanContents(Type: N.getValueType())) {
15037 case TargetLowering::ZeroOrOneBooleanContent:
15038 if (CVal.isOne())
15039 return true;
15040 if (CVal.isZero())
15041 return false;
15042 return std::nullopt;
15043 case TargetLowering::ZeroOrNegativeOneBooleanContent:
15044 if (CVal.isAllOnes())
15045 return true;
15046 if (CVal.isZero())
15047 return false;
15048 return std::nullopt;
15049 case TargetLowering::UndefinedBooleanContent:
15050 return CVal[0];
15051 }
15052 llvm_unreachable("Unknown BooleanContent enum");
15053}
15054
15055void SelectionDAG::createOperands(SDNode *Node, ArrayRef<SDValue> Vals) {
15056 assert(!Node->OperandList && "Node already has operands");
15057 assert(SDNode::getMaxNumOperands() >= Vals.size() &&
15058 "too many operands to fit into SDNode");
15059 SDUse *Ops = OperandRecycler.allocate(
15060 Cap: ArrayRecycler<SDUse>::Capacity::get(N: Vals.size()), Allocator&: OperandAllocator);
15061
15062 bool IsDivergent = false;
15063 for (unsigned I = 0; I != Vals.size(); ++I) {
15064 Ops[I].setUser(Node);
15065 Ops[I].setInitial(Vals[I]);
15066 EVT VT = Ops[I].getValueType();
15067
15068 // Skip Chain. It does not carry divergence.
15069 if (VT != MVT::Other &&
15070 (VT != MVT::Glue || gluePropagatesDivergence(Node: Ops[I].getNode())) &&
15071 Ops[I].getNode()->isDivergent()) {
15072 IsDivergent = true;
15073 }
15074 }
15075 Node->NumOperands = Vals.size();
15076 Node->OperandList = Ops;
15077 if (!TLI->isSDNodeAlwaysUniform(N: Node)) {
15078 IsDivergent |= TLI->isSDNodeSourceOfDivergence(N: Node, FLI, UA);
15079 Node->SDNodeBits.IsDivergent = IsDivergent;
15080 }
15081 checkForCycles(N: Node);
15082}
15083
15084SDValue SelectionDAG::getTokenFactor(const SDLoc &DL,
15085 SmallVectorImpl<SDValue> &Vals) {
15086 size_t Limit = SDNode::getMaxNumOperands();
15087 while (Vals.size() > Limit) {
15088 unsigned SliceIdx = Vals.size() - Limit;
15089 auto ExtractedTFs = ArrayRef<SDValue>(Vals).slice(N: SliceIdx, M: Limit);
15090 SDValue NewTF = getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: ExtractedTFs);
15091 Vals.erase(CS: Vals.begin() + SliceIdx, CE: Vals.end());
15092 Vals.emplace_back(Args&: NewTF);
15093 }
15094 return getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: Vals);
15095}
15096
15097SDValue SelectionDAG::getIdentityElement(unsigned Opcode, const SDLoc &DL,
15098 EVT VT, SDNodeFlags Flags) {
15099 switch (Opcode) {
15100 default:
15101 return SDValue();
15102 case ISD::ADD:
15103 case ISD::OR:
15104 case ISD::XOR:
15105 case ISD::UMAX:
15106 case ISD::MUL:
15107 case ISD::AND:
15108 case ISD::UMIN:
15109 case ISD::SMAX:
15110 case ISD::SMIN:
15111 return getConstant(Val: getIntegerIdentity(Opcode, BitWidth: VT.getScalarSizeInBits()), DL,
15112 VT);
15113 case ISD::FADD:
15114 // If flags allow, prefer positive zero since it's generally cheaper
15115 // to materialize on most targets.
15116 return getConstantFP(Val: Flags.hasNoSignedZeros() ? 0.0 : -0.0, DL, VT);
15117 case ISD::FMUL:
15118 return getConstantFP(Val: 1.0, DL, VT);
15119 case ISD::FMINNUM:
15120 case ISD::FMAXNUM:
15121 case ISD::FMINIMUMNUM:
15122 case ISD::FMAXIMUMNUM: {
15123 // Neutral element for fminnum/fminimumnum is NaN, Inf or FLT_MAX,
15124 // depending on fast-math flags (FMF).
15125 const fltSemantics &Semantics = VT.getFltSemantics();
15126 APFloat NeutralAF = !Flags.hasNoNaNs() ? APFloat::getQNaN(Sem: Semantics) :
15127 !Flags.hasNoInfs() ? APFloat::getInf(Sem: Semantics) :
15128 APFloat::getLargest(Sem: Semantics);
15129 if (Opcode == ISD::FMAXNUM || Opcode == ISD::FMAXIMUMNUM)
15130 NeutralAF.changeSign();
15131
15132 return getConstantFP(V: NeutralAF, DL, VT);
15133 }
15134 case ISD::FMINIMUM:
15135 case ISD::FMAXIMUM: {
15136 // Neutral element for fminimum is Inf or FLT_MAX, depending on FMF.
15137 const fltSemantics &Semantics = VT.getFltSemantics();
15138 APFloat NeutralAF = !Flags.hasNoInfs() ? APFloat::getInf(Sem: Semantics)
15139 : APFloat::getLargest(Sem: Semantics);
15140 if (Opcode == ISD::FMAXIMUM)
15141 NeutralAF.changeSign();
15142
15143 return getConstantFP(V: NeutralAF, DL, VT);
15144 }
15145
15146 }
15147}
15148
15149SDValue SelectionDAG::getPartialReduceMLS(unsigned Opc, const SDLoc &DL,
15150 SDValue Acc, SDValue LHS,
15151 SDValue RHS) {
15152 EVT AccVT = Acc.getValueType();
15153 if (AccVT.isFloatingPoint()) {
15154 assert(Opc == ISD::PARTIAL_REDUCE_FMLA && "Unexpected opcode");
15155 SDValue NegRHS = getNode(Opcode: ISD::FNEG, DL, VT: RHS.getValueType(), N1: RHS);
15156 return getNode(Opcode: Opc, DL, VT: AccVT, N1: Acc, N2: LHS, N3: NegRHS);
15157 }
15158 assert((Opc == ISD::PARTIAL_REDUCE_UMLA || Opc == ISD::PARTIAL_REDUCE_SMLA ||
15159 Opc == ISD::PARTIAL_REDUCE_SUMLA) &&
15160 "Unexpected opcode");
15161 SDValue NegAcc = getNegative(Val: Acc, DL, VT: AccVT);
15162 SDValue MLA = getNode(Opcode: Opc, DL, VT: AccVT, N1: NegAcc, N2: LHS, N3: RHS);
15163 return getNegative(Val: MLA, DL, VT: AccVT);
15164}
15165
15166/// Helper used to make a call to a library function that has one argument of
15167/// pointer type.
15168///
15169/// Such functions include 'fegetmode', 'fesetenv' and some others, which are
15170/// used to get or set floating-point state. They have one argument of pointer
15171/// type, which points to the memory region containing bits of the
15172/// floating-point state. The value returned by such function is ignored in the
15173/// created call.
15174///
15175/// \param LibFunc Reference to library function (value of RTLIB::Libcall).
15176/// \param Ptr Pointer used to save/load state.
15177/// \param InChain Ingoing token chain.
15178/// \returns Outgoing chain token.
15179SDValue SelectionDAG::makeStateFunctionCall(unsigned LibFunc, SDValue Ptr,
15180 SDValue InChain,
15181 const SDLoc &DLoc) {
15182 assert(InChain.getValueType() == MVT::Other && "Expected token chain");
15183 TargetLowering::ArgListTy Args;
15184 Args.emplace_back(args&: Ptr, args: Ptr.getValueType().getTypeForEVT(Context&: *getContext()));
15185 RTLIB::LibcallImpl LibcallImpl =
15186 Libcalls->getLibcallImpl(Call: static_cast<RTLIB::Libcall>(LibFunc));
15187 if (LibcallImpl == RTLIB::Unsupported)
15188 reportFatalUsageError(reason: "emitting call to unsupported libcall");
15189
15190 SDValue Callee =
15191 getExternalSymbol(Libcall: LibcallImpl, VT: TLI->getPointerTy(DL: getDataLayout()));
15192 TargetLowering::CallLoweringInfo CLI(*this);
15193 CLI.setDebugLoc(DLoc).setChain(InChain).setLibCallee(
15194 CC: Libcalls->getLibcallImplCallingConv(Call: LibcallImpl),
15195 ResultType: Type::getVoidTy(C&: *getContext()), Target: Callee, ArgsList: std::move(Args));
15196 return TLI->LowerCallTo(CLI).second;
15197}
15198
15199void SelectionDAG::copyExtraInfo(SDNode *From, SDNode *To) {
15200 assert(From && To && "Invalid SDNode; empty source SDValue?");
15201 auto I = SDEI.find(Val: From);
15202 if (I == SDEI.end())
15203 return;
15204
15205 // Use of operator[] on the DenseMap may cause an insertion, which invalidates
15206 // the iterator, hence the need to make a copy to prevent a use-after-free.
15207 NodeExtraInfo NEI = I->second;
15208 if (LLVM_LIKELY(!NEI.PCSections)) {
15209 // No deep copy required for the types of extra info set.
15210 //
15211 // FIXME: Investigate if other types of extra info also need deep copy. This
15212 // depends on the types of nodes they can be attached to: if some extra info
15213 // is only ever attached to nodes where a replacement To node is always the
15214 // node where later use and propagation of the extra info has the intended
15215 // semantics, no deep copy is required.
15216 SDEI[To] = std::move(NEI);
15217 return;
15218 }
15219
15220 const SDNode *EntrySDN = getEntryNode().getNode();
15221
15222 // We need to copy NodeExtraInfo to all _new_ nodes that are being introduced
15223 // through the replacement of From with To. Otherwise, replacements of a node
15224 // (From) with more complex nodes (To and its operands) may result in lost
15225 // extra info where the root node (To) is insignificant in further propagating
15226 // and using extra info when further lowering to MIR.
15227 //
15228 // In the first step pre-populate the visited set with the nodes reachable
15229 // from the old From node. This avoids copying NodeExtraInfo to parts of the
15230 // DAG that is not new and should be left untouched.
15231 SmallVector<const SDNode *> Leafs{From}; // Leafs reachable with VisitFrom.
15232 DenseSet<const SDNode *> FromReach; // The set of nodes reachable from From.
15233 auto VisitFrom = [&](auto &&Self, const SDNode *N, int MaxDepth) {
15234 if (MaxDepth == 0) {
15235 // Remember this node in case we need to increase MaxDepth and continue
15236 // populating FromReach from this node.
15237 Leafs.emplace_back(Args&: N);
15238 return;
15239 }
15240 if (!FromReach.insert(V: N).second)
15241 return;
15242 for (const SDValue &Op : N->op_values())
15243 Self(Self, Op.getNode(), MaxDepth - 1);
15244 };
15245
15246 // Copy extra info to To and all its transitive operands (that are new).
15247 SmallPtrSet<const SDNode *, 8> Visited;
15248 auto DeepCopyTo = [&](auto &&Self, const SDNode *N) {
15249 if (FromReach.contains(V: N))
15250 return true;
15251 if (!Visited.insert(Ptr: N).second)
15252 return true;
15253 if (EntrySDN == N)
15254 return false;
15255 for (const SDValue &Op : N->op_values()) {
15256 if (N == To && Op.getNode() == EntrySDN) {
15257 // Special case: New node's operand is the entry node; just need to
15258 // copy extra info to new node.
15259 break;
15260 }
15261 if (!Self(Self, Op.getNode()))
15262 return false;
15263 }
15264 // Copy only if entry node was not reached.
15265 SDEI[N] = std::move(NEI);
15266 return true;
15267 };
15268
15269 // We first try with a lower MaxDepth, assuming that the path to common
15270 // operands between From and To is relatively short. This significantly
15271 // improves performance in the common case. The initial MaxDepth is big
15272 // enough to avoid retry in the common case; the last MaxDepth is large
15273 // enough to avoid having to use the fallback below (and protects from
15274 // potential stack exhaustion from recursion).
15275 for (int PrevDepth = 0, MaxDepth = 16; MaxDepth <= 1024;
15276 PrevDepth = MaxDepth, MaxDepth *= 2, Visited.clear()) {
15277 // StartFrom is the previous (or initial) set of leafs reachable at the
15278 // previous maximum depth.
15279 SmallVector<const SDNode *> StartFrom;
15280 std::swap(LHS&: StartFrom, RHS&: Leafs);
15281 for (const SDNode *N : StartFrom)
15282 VisitFrom(VisitFrom, N, MaxDepth - PrevDepth);
15283 if (LLVM_LIKELY(DeepCopyTo(DeepCopyTo, To)))
15284 return;
15285 // This should happen very rarely (reached the entry node).
15286 LLVM_DEBUG(dbgs() << __func__ << ": MaxDepth=" << MaxDepth << " too low\n");
15287 assert(!Leafs.empty());
15288 }
15289
15290 // This should not happen - but if it did, that means the subgraph reachable
15291 // from From has depth greater or equal to maximum MaxDepth, and VisitFrom()
15292 // could not visit all reachable common operands. Consequently, we were able
15293 // to reach the entry node.
15294 errs() << "warning: incomplete propagation of SelectionDAG::NodeExtraInfo\n";
15295 assert(false && "From subgraph too complex - increase max. MaxDepth?");
15296 // Best-effort fallback if assertions disabled.
15297 SDEI[To] = std::move(NEI);
15298}
15299
15300unsigned SelectionDAG::getMaxRuntimeNumElements(EVT VT) const {
15301 assert(VT.isVector() && "Can only unroll vector types!");
15302 if (VT.isFixedLengthVector())
15303 return VT.getVectorNumElements();
15304
15305 const MachineFunction &MF = getMachineFunction();
15306 const Function &F = MF.getFunction();
15307
15308 APInt MaxVScale = getVScaleRange(F: &F, BitWidth: sizeof(unsigned) * 8).getUnsignedMax();
15309 if (MaxVScale.ugt(RHS: VScaleUnrollLimit))
15310 return 0;
15311
15312 bool Overflow;
15313 APInt MinNElts(sizeof(unsigned) * 8, VT.getVectorMinNumElements());
15314 APInt MaxNElts = MinNElts.umul_ov(RHS: MaxVScale, Overflow);
15315 if (Overflow)
15316 return 0;
15317
15318 return MaxNElts.getZExtValue();
15319}
15320
15321SDValue SelectionDAG::buildVectorFromUnrolledParts(EVT VT, const SDLoc &DL,
15322 ArrayRef<SDValue> Scalars) {
15323 assert(Scalars.size() == getMaxRuntimeNumElements(VT) &&
15324 "Element count mismatch!");
15325 if (VT.isFixedLengthVector())
15326 return getBuildVector(VT, DL, Ops: Scalars);
15327
15328 SDValue Vec = getPOISON(VT);
15329 // Iterate in reverse so result remains poison until we encounter a lane that
15330 // exists, after which all lower-numbered lanes must also exist.
15331 for (unsigned IdxVal : reverse(C: seq(Size: Scalars.size())))
15332 Vec = getInsertVectorElt(DL, Vec, Elt: Scalars[IdxVal], Idx: IdxVal);
15333
15334 return Vec;
15335}
15336
15337#ifndef NDEBUG
15338static void checkForCyclesHelper(const SDNode *N,
15339 SmallPtrSetImpl<const SDNode*> &Visited,
15340 SmallPtrSetImpl<const SDNode*> &Checked,
15341 const llvm::SelectionDAG *DAG) {
15342 // If this node has already been checked, don't check it again.
15343 if (Checked.count(N))
15344 return;
15345
15346 // If a node has already been visited on this depth-first walk, reject it as
15347 // a cycle.
15348 if (!Visited.insert(N).second) {
15349 errs() << "Detected cycle in SelectionDAG\n";
15350 dbgs() << "Offending node:\n";
15351 N->dumprFull(DAG); dbgs() << "\n";
15352 abort();
15353 }
15354
15355 for (const SDValue &Op : N->op_values())
15356 checkForCyclesHelper(Op.getNode(), Visited, Checked, DAG);
15357
15358 Checked.insert(N);
15359 Visited.erase(N);
15360}
15361#endif
15362
15363void llvm::checkForCycles(const llvm::SDNode *N,
15364 const llvm::SelectionDAG *DAG,
15365 bool force) {
15366#ifndef NDEBUG
15367 bool check = force;
15368#ifdef EXPENSIVE_CHECKS
15369 check = true;
15370#endif // EXPENSIVE_CHECKS
15371 if (check) {
15372 assert(N && "Checking nonexistent SDNode");
15373 SmallPtrSet<const SDNode*, 32> visited;
15374 SmallPtrSet<const SDNode*, 32> checked;
15375 checkForCyclesHelper(N, visited, checked, DAG);
15376 }
15377#endif // !NDEBUG
15378}
15379
15380void llvm::checkForCycles(const llvm::SelectionDAG *DAG, bool force) {
15381 checkForCycles(N: DAG->getRoot().getNode(), DAG, force);
15382}
15383