1//===- SelectionDAG.cpp - Implement the SelectionDAG data structures ------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements the SelectionDAG class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/CodeGen/SelectionDAG.h"
14#include "SDNodeDbgValue.h"
15#include "llvm/ADT/APFloat.h"
16#include "llvm/ADT/APInt.h"
17#include "llvm/ADT/APSInt.h"
18#include "llvm/ADT/ArrayRef.h"
19#include "llvm/ADT/BitVector.h"
20#include "llvm/ADT/DenseSet.h"
21#include "llvm/ADT/FoldingSet.h"
22#include "llvm/ADT/STLExtras.h"
23#include "llvm/ADT/SmallPtrSet.h"
24#include "llvm/ADT/SmallVector.h"
25#include "llvm/ADT/Twine.h"
26#include "llvm/Analysis/AliasAnalysis.h"
27#include "llvm/Analysis/MemoryLocation.h"
28#include "llvm/Analysis/TargetLibraryInfo.h"
29#include "llvm/Analysis/ValueTracking.h"
30#include "llvm/Analysis/VectorUtils.h"
31#include "llvm/BinaryFormat/Dwarf.h"
32#include "llvm/CodeGen/Analysis.h"
33#include "llvm/CodeGen/CodeGenCommonISel.h"
34#include "llvm/CodeGen/FunctionLoweringInfo.h"
35#include "llvm/CodeGen/ISDOpcodes.h"
36#include "llvm/CodeGen/MachineBasicBlock.h"
37#include "llvm/CodeGen/MachineConstantPool.h"
38#include "llvm/CodeGen/MachineFrameInfo.h"
39#include "llvm/CodeGen/MachineFunction.h"
40#include "llvm/CodeGen/MachineMemOperand.h"
41#include "llvm/CodeGen/RuntimeLibcallUtil.h"
42#include "llvm/CodeGen/SDPatternMatch.h"
43#include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
44#include "llvm/CodeGen/SelectionDAGNodes.h"
45#include "llvm/CodeGen/SelectionDAGTargetInfo.h"
46#include "llvm/CodeGen/TargetFrameLowering.h"
47#include "llvm/CodeGen/TargetLowering.h"
48#include "llvm/CodeGen/TargetRegisterInfo.h"
49#include "llvm/CodeGen/TargetSubtargetInfo.h"
50#include "llvm/CodeGen/ValueTypes.h"
51#include "llvm/CodeGenTypes/MachineValueType.h"
52#include "llvm/IR/Constant.h"
53#include "llvm/IR/Constants.h"
54#include "llvm/IR/DataLayout.h"
55#include "llvm/IR/DebugInfoMetadata.h"
56#include "llvm/IR/DebugLoc.h"
57#include "llvm/IR/DerivedTypes.h"
58#include "llvm/IR/Function.h"
59#include "llvm/IR/GlobalValue.h"
60#include "llvm/IR/Metadata.h"
61#include "llvm/IR/Type.h"
62#include "llvm/Support/Casting.h"
63#include "llvm/Support/CodeGen.h"
64#include "llvm/Support/Compiler.h"
65#include "llvm/Support/Debug.h"
66#include "llvm/Support/ErrorHandling.h"
67#include "llvm/Support/KnownBits.h"
68#include "llvm/Support/KnownFPClass.h"
69#include "llvm/Support/MathExtras.h"
70#include "llvm/Support/raw_ostream.h"
71#include "llvm/Target/TargetMachine.h"
72#include "llvm/Target/TargetOptions.h"
73#include "llvm/TargetParser/Triple.h"
74#include "llvm/Transforms/Utils/SizeOpts.h"
75#include <algorithm>
76#include <cassert>
77#include <cstdint>
78#include <cstdlib>
79#include <limits>
80#include <optional>
81#include <string>
82#include <utility>
83#include <vector>
84
85using namespace llvm;
86using namespace llvm::SDPatternMatch;
87
88/// makeVTList - Return an instance of the SDVTList struct initialized with the
89/// specified members.
90static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) {
91 SDVTList Res = {.VTs: VTs, .NumVTs: NumVTs};
92 return Res;
93}
94
95// Default null implementations of the callbacks.
96void SelectionDAG::DAGUpdateListener::NodeDeleted(SDNode*, SDNode*) {}
97void SelectionDAG::DAGUpdateListener::NodeUpdated(SDNode*) {}
98void SelectionDAG::DAGUpdateListener::NodeInserted(SDNode *) {}
99
100void SelectionDAG::DAGNodeDeletedListener::anchor() {}
101void SelectionDAG::DAGNodeInsertedListener::anchor() {}
102
103#define DEBUG_TYPE "selectiondag"
104
105static cl::opt<bool> EnableMemCpyDAGOpt("enable-memcpy-dag-opt",
106 cl::Hidden, cl::init(Val: true),
107 cl::desc("Gang up loads and stores generated by inlining of memcpy"));
108
109static cl::opt<int> MaxLdStGlue("ldstmemcpy-glue-max",
110 cl::desc("Number limit for gluing ld/st of memcpy."),
111 cl::Hidden, cl::init(Val: 0));
112
113static cl::opt<unsigned>
114 MaxSteps("has-predecessor-max-steps", cl::Hidden, cl::init(Val: 8192),
115 cl::desc("DAG combiner limit number of steps when searching DAG "
116 "for predecessor nodes"));
117
118static cl::opt<int> VScaleUnrollLimit(
119 "vscale-unroll-limit",
120 cl::desc("Maximum vscale for which vector unrolling is allowed."),
121 cl::Hidden, cl::init(Val: 64));
122
123static void NewSDValueDbgMsg(SDValue V, StringRef Msg, SelectionDAG *G) {
124 LLVM_DEBUG(dbgs() << Msg; V.getNode()->dump(G););
125}
126
127unsigned SelectionDAG::getHasPredecessorMaxSteps() { return MaxSteps; }
128
129//===----------------------------------------------------------------------===//
130// ConstantFPSDNode Class
131//===----------------------------------------------------------------------===//
132
133/// isExactlyValue - We don't rely on operator== working on double values, as
134/// it returns true for things that are clearly not equal, like -0.0 and 0.0.
135/// As such, this method can be used to do an exact bit-for-bit comparison of
136/// two floating point values.
137bool ConstantFPSDNode::isExactlyValue(const APFloat& V) const {
138 return getValueAPF().bitwiseIsEqual(RHS: V);
139}
140
141bool ConstantFPSDNode::isValueValidForType(EVT VT,
142 const APFloat& Val) {
143 assert(VT.isFloatingPoint() && "Can only convert between FP types");
144
145 // convert modifies in place, so make a copy.
146 APFloat Val2 = APFloat(Val);
147 bool losesInfo;
148 (void)Val2.convert(ToSemantics: VT.getFltSemantics(), RM: APFloat::rmNearestTiesToEven,
149 losesInfo: &losesInfo);
150 return !losesInfo;
151}
152
153//===----------------------------------------------------------------------===//
154// ISD Namespace
155//===----------------------------------------------------------------------===//
156
157bool ISD::isConstantSplatVector(const SDNode *N, APInt &SplatVal) {
158 if (N->getOpcode() == ISD::SPLAT_VECTOR) {
159 if (auto OptAPInt = N->getOperand(Num: 0)->bitcastToAPInt()) {
160 unsigned EltSize =
161 N->getValueType(ResNo: 0).getVectorElementType().getSizeInBits();
162 SplatVal = OptAPInt->trunc(width: EltSize);
163 return true;
164 }
165 }
166
167 auto *BV = dyn_cast<BuildVectorSDNode>(Val: N);
168 if (!BV)
169 return false;
170
171 APInt SplatUndef;
172 unsigned SplatBitSize;
173 bool HasUndefs;
174 unsigned EltSize = N->getValueType(ResNo: 0).getVectorElementType().getSizeInBits();
175 // Endianness does not matter here. We are checking for a splat given the
176 // element size of the vector, and if we find such a splat for little endian
177 // layout, then that should be valid also for big endian (as the full vector
178 // size is known to be a multiple of the element size).
179 const bool IsBigEndian = false;
180 return BV->isConstantSplat(SplatValue&: SplatVal, SplatUndef, SplatBitSize, HasAnyUndefs&: HasUndefs,
181 MinSplatBits: EltSize, isBigEndian: IsBigEndian) &&
182 EltSize == SplatBitSize;
183}
184
185// FIXME: AllOnes and AllZeros duplicate a lot of code. Could these be
186// specializations of the more general isConstantSplatVector()?
187
188bool ISD::isConstantSplatVectorAllOnes(const SDNode *N, bool BuildVectorOnly) {
189 // Look through a bit convert.
190 while (N->getOpcode() == ISD::BITCAST)
191 N = N->getOperand(Num: 0).getNode();
192
193 if (!BuildVectorOnly && N->getOpcode() == ISD::SPLAT_VECTOR) {
194 APInt SplatVal;
195 return isConstantSplatVector(N, SplatVal) && SplatVal.isAllOnes();
196 }
197
198 if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
199
200 unsigned i = 0, e = N->getNumOperands();
201
202 // Skip over all of the undef values.
203 while (i != e && N->getOperand(Num: i).isUndef())
204 ++i;
205
206 // Do not accept an all-undef vector.
207 if (i == e) return false;
208
209 // Do not accept build_vectors that aren't all constants or which have non-~0
210 // elements. We have to be a bit careful here, as the type of the constant
211 // may not be the same as the type of the vector elements due to type
212 // legalization (the elements are promoted to a legal type for the target and
213 // a vector of a type may be legal when the base element type is not).
214 // We only want to check enough bits to cover the vector elements, because
215 // we care if the resultant vector is all ones, not whether the individual
216 // constants are.
217 SDValue NotZero = N->getOperand(Num: i);
218 if (auto OptAPInt = NotZero->bitcastToAPInt()) {
219 unsigned EltSize = N->getValueType(ResNo: 0).getScalarSizeInBits();
220 if (OptAPInt->countr_one() < EltSize)
221 return false;
222 } else
223 return false;
224
225 // Okay, we have at least one ~0 value, check to see if the rest match or are
226 // undefs. Even with the above element type twiddling, this should be OK, as
227 // the same type legalization should have applied to all the elements.
228 for (++i; i != e; ++i)
229 if (N->getOperand(Num: i) != NotZero && !N->getOperand(Num: i).isUndef())
230 return false;
231 return true;
232}
233
234bool ISD::isConstantSplatVectorAllZeros(const SDNode *N, bool BuildVectorOnly) {
235 // Look through a bit convert.
236 while (N->getOpcode() == ISD::BITCAST)
237 N = N->getOperand(Num: 0).getNode();
238
239 if (!BuildVectorOnly && N->getOpcode() == ISD::SPLAT_VECTOR) {
240 APInt SplatVal;
241 return isConstantSplatVector(N, SplatVal) && SplatVal.isZero();
242 }
243
244 if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
245
246 bool IsAllUndef = true;
247 for (const SDValue &Op : N->op_values()) {
248 if (Op.isUndef())
249 continue;
250 IsAllUndef = false;
251 // Do not accept build_vectors that aren't all constants or which have non-0
252 // elements. We have to be a bit careful here, as the type of the constant
253 // may not be the same as the type of the vector elements due to type
254 // legalization (the elements are promoted to a legal type for the target
255 // and a vector of a type may be legal when the base element type is not).
256 // We only want to check enough bits to cover the vector elements, because
257 // we care if the resultant vector is all zeros, not whether the individual
258 // constants are.
259 if (auto OptAPInt = Op->bitcastToAPInt()) {
260 unsigned EltSize = N->getValueType(ResNo: 0).getScalarSizeInBits();
261 if (OptAPInt->countr_zero() < EltSize)
262 return false;
263 } else
264 return false;
265 }
266
267 // Do not accept an all-undef vector.
268 if (IsAllUndef)
269 return false;
270 return true;
271}
272
273bool ISD::isBuildVectorAllOnes(const SDNode *N) {
274 return isConstantSplatVectorAllOnes(N, /*BuildVectorOnly*/ true);
275}
276
277bool ISD::isBuildVectorAllZeros(const SDNode *N) {
278 return isConstantSplatVectorAllZeros(N, /*BuildVectorOnly*/ true);
279}
280
281bool ISD::isBuildVectorOfConstantSDNodes(const SDNode *N) {
282 if (N->getOpcode() != ISD::BUILD_VECTOR)
283 return false;
284
285 for (const SDValue &Op : N->op_values()) {
286 if (Op.isUndef())
287 continue;
288 if (!isa<ConstantSDNode>(Val: Op))
289 return false;
290 }
291 return true;
292}
293
294bool ISD::isBuildVectorOfConstantFPSDNodes(const SDNode *N) {
295 if (N->getOpcode() != ISD::BUILD_VECTOR)
296 return false;
297
298 for (const SDValue &Op : N->op_values()) {
299 if (Op.isUndef())
300 continue;
301 if (!isa<ConstantFPSDNode>(Val: Op))
302 return false;
303 }
304 return true;
305}
306
307bool ISD::isVectorShrinkable(const SDNode *N, unsigned NewEltSize,
308 bool Signed) {
309 assert(N->getValueType(0).isVector() && "Expected a vector!");
310
311 unsigned EltSize = N->getValueType(ResNo: 0).getScalarSizeInBits();
312 if (EltSize <= NewEltSize)
313 return false;
314
315 if (N->getOpcode() == ISD::ZERO_EXTEND) {
316 return (N->getOperand(Num: 0).getValueType().getScalarSizeInBits() <=
317 NewEltSize) &&
318 !Signed;
319 }
320 if (N->getOpcode() == ISD::SIGN_EXTEND) {
321 return (N->getOperand(Num: 0).getValueType().getScalarSizeInBits() <=
322 NewEltSize) &&
323 Signed;
324 }
325 if (N->getOpcode() != ISD::BUILD_VECTOR)
326 return false;
327
328 for (const SDValue &Op : N->op_values()) {
329 if (Op.isUndef())
330 continue;
331 if (!isa<ConstantSDNode>(Val: Op))
332 return false;
333
334 APInt C = Op->getAsAPIntVal().trunc(width: EltSize);
335 if (Signed && C.trunc(width: NewEltSize).sext(width: EltSize) != C)
336 return false;
337 if (!Signed && C.trunc(width: NewEltSize).zext(width: EltSize) != C)
338 return false;
339 }
340
341 return true;
342}
343
344bool ISD::allOperandsUndef(const SDNode *N) {
345 // Return false if the node has no operands.
346 // This is "logically inconsistent" with the definition of "all" but
347 // is probably the desired behavior.
348 if (N->getNumOperands() == 0)
349 return false;
350 return all_of(Range: N->op_values(), P: [](SDValue Op) { return Op.isUndef(); });
351}
352
353bool ISD::isFreezeUndef(const SDNode *N) {
354 return N->getOpcode() == ISD::FREEZE && N->getOperand(Num: 0).isUndef();
355}
356
357template <typename ConstNodeType>
358bool ISD::matchUnaryPredicateImpl(SDValue Op, const APInt &DemandedElts,
359 std::function<bool(ConstNodeType *)> Match,
360 bool AllowUndefs, bool AllowTruncation) {
361 // FIXME: Add support for scalar UNDEF cases?
362 if (auto *C = dyn_cast<ConstNodeType>(Op))
363 return Match(C);
364
365 // FIXME: Add support for vector UNDEF cases?
366 if (ISD::BUILD_VECTOR != Op.getOpcode() &&
367 ISD::SPLAT_VECTOR != Op.getOpcode())
368 return false;
369
370 if (ISD::SPLAT_VECTOR == Op.getOpcode() && !DemandedElts)
371 return true;
372
373 EVT SVT = Op.getValueType().getScalarType();
374 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
375 if (ISD::SPLAT_VECTOR != Op.getOpcode() && !DemandedElts[i])
376 continue;
377
378 if (AllowUndefs && Op.getOperand(i).isUndef()) {
379 if (!Match(nullptr))
380 return false;
381 continue;
382 }
383
384 auto *Cst = dyn_cast<ConstNodeType>(Op.getOperand(i));
385 if (!Cst || (!AllowTruncation && Cst->getValueType(0) != SVT) ||
386 !Match(Cst))
387 return false;
388 }
389 return true;
390}
391// Build used template types.
392template bool ISD::matchUnaryPredicateImpl<ConstantSDNode>(
393 SDValue, const APInt &, std::function<bool(ConstantSDNode *)>, bool, bool);
394template bool ISD::matchUnaryPredicateImpl<ConstantFPSDNode>(
395 SDValue, const APInt &, std::function<bool(ConstantFPSDNode *)>, bool,
396 bool);
397
398bool ISD::matchBinaryPredicate(
399 SDValue LHS, SDValue RHS, const APInt &DemandedElts,
400 std::function<bool(ConstantSDNode *, ConstantSDNode *)> Match,
401 bool AllowUndefs, bool AllowTypeMismatch) {
402 if (!AllowTypeMismatch && LHS.getValueType() != RHS.getValueType())
403 return false;
404
405 // TODO: Add support for scalar UNDEF cases?
406 if (auto *LHSCst = dyn_cast<ConstantSDNode>(Val&: LHS))
407 if (auto *RHSCst = dyn_cast<ConstantSDNode>(Val&: RHS))
408 return Match(LHSCst, RHSCst);
409
410 // TODO: Add support for vector UNDEF cases?
411 if (LHS.getOpcode() != RHS.getOpcode() ||
412 (LHS.getOpcode() != ISD::BUILD_VECTOR &&
413 LHS.getOpcode() != ISD::SPLAT_VECTOR))
414 return false;
415
416 if (ISD::SPLAT_VECTOR == LHS.getOpcode() && !DemandedElts)
417 return true;
418
419 EVT SVT = LHS.getValueType().getScalarType();
420 for (unsigned i = 0, e = LHS.getNumOperands(); i != e; ++i) {
421 if (ISD::SPLAT_VECTOR != LHS.getOpcode() && !DemandedElts[i])
422 continue;
423 SDValue LHSOp = LHS.getOperand(i);
424 SDValue RHSOp = RHS.getOperand(i);
425 bool LHSUndef = AllowUndefs && LHSOp.isUndef();
426 bool RHSUndef = AllowUndefs && RHSOp.isUndef();
427 auto *LHSCst = dyn_cast<ConstantSDNode>(Val&: LHSOp);
428 auto *RHSCst = dyn_cast<ConstantSDNode>(Val&: RHSOp);
429 if ((!LHSCst && !LHSUndef) || (!RHSCst && !RHSUndef))
430 return false;
431 if (!AllowTypeMismatch && (LHSOp.getValueType() != SVT ||
432 LHSOp.getValueType() != RHSOp.getValueType()))
433 return false;
434 if (!Match(LHSCst, RHSCst))
435 return false;
436 }
437 return true;
438}
439
440ISD::NodeType ISD::getInverseMinMaxOpcode(unsigned MinMaxOpc) {
441 switch (MinMaxOpc) {
442 default:
443 llvm_unreachable("unrecognized opcode");
444 case ISD::UMIN:
445 return ISD::UMAX;
446 case ISD::UMAX:
447 return ISD::UMIN;
448 case ISD::SMIN:
449 return ISD::SMAX;
450 case ISD::SMAX:
451 return ISD::SMIN;
452 }
453}
454
455ISD::NodeType ISD::getOppositeSignednessMinMaxOpcode(unsigned MinMaxOpc) {
456 switch (MinMaxOpc) {
457 default:
458 llvm_unreachable("unrecognized min/max opcode");
459 case ISD::SMIN:
460 return ISD::UMIN;
461 case ISD::SMAX:
462 return ISD::UMAX;
463 case ISD::UMIN:
464 return ISD::SMIN;
465 case ISD::UMAX:
466 return ISD::SMAX;
467 }
468}
469
470ISD::NodeType ISD::getVecReduceBaseOpcode(unsigned VecReduceOpcode) {
471 switch (VecReduceOpcode) {
472 default:
473 llvm_unreachable("Expected VECREDUCE opcode");
474 case ISD::VECREDUCE_FADD:
475 case ISD::VECREDUCE_SEQ_FADD:
476 case ISD::VP_REDUCE_FADD:
477 case ISD::VP_REDUCE_SEQ_FADD:
478 return ISD::FADD;
479 case ISD::VECREDUCE_FMUL:
480 case ISD::VECREDUCE_SEQ_FMUL:
481 case ISD::VP_REDUCE_FMUL:
482 case ISD::VP_REDUCE_SEQ_FMUL:
483 return ISD::FMUL;
484 case ISD::VECREDUCE_ADD:
485 case ISD::VP_REDUCE_ADD:
486 return ISD::ADD;
487 case ISD::VECREDUCE_MUL:
488 case ISD::VP_REDUCE_MUL:
489 return ISD::MUL;
490 case ISD::VECREDUCE_AND:
491 case ISD::VP_REDUCE_AND:
492 return ISD::AND;
493 case ISD::VECREDUCE_OR:
494 case ISD::VP_REDUCE_OR:
495 return ISD::OR;
496 case ISD::VECREDUCE_XOR:
497 case ISD::VP_REDUCE_XOR:
498 return ISD::XOR;
499 case ISD::VECREDUCE_SMAX:
500 case ISD::VP_REDUCE_SMAX:
501 return ISD::SMAX;
502 case ISD::VECREDUCE_SMIN:
503 case ISD::VP_REDUCE_SMIN:
504 return ISD::SMIN;
505 case ISD::VECREDUCE_UMAX:
506 case ISD::VP_REDUCE_UMAX:
507 return ISD::UMAX;
508 case ISD::VECREDUCE_UMIN:
509 case ISD::VP_REDUCE_UMIN:
510 return ISD::UMIN;
511 case ISD::VECREDUCE_FMAX:
512 case ISD::VP_REDUCE_FMAX:
513 return ISD::FMAXNUM;
514 case ISD::VECREDUCE_FMIN:
515 case ISD::VP_REDUCE_FMIN:
516 return ISD::FMINNUM;
517 case ISD::VECREDUCE_FMAXIMUM:
518 case ISD::VP_REDUCE_FMAXIMUM:
519 return ISD::FMAXIMUM;
520 case ISD::VECREDUCE_FMINIMUM:
521 case ISD::VP_REDUCE_FMINIMUM:
522 return ISD::FMINIMUM;
523 case ISD::VECREDUCE_FMAXIMUMNUM:
524 return ISD::FMAXIMUMNUM;
525 case ISD::VECREDUCE_FMINIMUMNUM:
526 return ISD::FMINIMUMNUM;
527 }
528}
529
530ISD::NodeType ISD::getUnmaskedBinOpOpcode(unsigned MaskedOpc) {
531 switch (MaskedOpc) {
532 case ISD::MASKED_UDIV:
533 return ISD::UDIV;
534 case ISD::MASKED_SDIV:
535 return ISD::SDIV;
536 case ISD::MASKED_UREM:
537 return ISD::UREM;
538 case ISD::MASKED_SREM:
539 return ISD::SREM;
540 default:
541 llvm_unreachable("Expected masked binop opcode");
542 }
543}
544
545bool ISD::isVPOpcode(unsigned Opcode) {
546 switch (Opcode) {
547 default:
548 return false;
549#define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) \
550 case ISD::VPSD: \
551 return true;
552#include "llvm/IR/VPIntrinsics.def"
553 }
554}
555
556bool ISD::isVPBinaryOp(unsigned Opcode) {
557 switch (Opcode) {
558 default:
559 break;
560#define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) case ISD::VPSD:
561#define VP_PROPERTY_BINARYOP return true;
562#define END_REGISTER_VP_SDNODE(VPSD) break;
563#include "llvm/IR/VPIntrinsics.def"
564 }
565 return false;
566}
567
568bool ISD::isVPReduction(unsigned Opcode) {
569 switch (Opcode) {
570 default:
571 return false;
572 case ISD::VP_REDUCE_ADD:
573 case ISD::VP_REDUCE_MUL:
574 case ISD::VP_REDUCE_AND:
575 case ISD::VP_REDUCE_OR:
576 case ISD::VP_REDUCE_XOR:
577 case ISD::VP_REDUCE_SMAX:
578 case ISD::VP_REDUCE_SMIN:
579 case ISD::VP_REDUCE_UMAX:
580 case ISD::VP_REDUCE_UMIN:
581 case ISD::VP_REDUCE_FMAX:
582 case ISD::VP_REDUCE_FMIN:
583 case ISD::VP_REDUCE_FMAXIMUM:
584 case ISD::VP_REDUCE_FMINIMUM:
585 case ISD::VP_REDUCE_FADD:
586 case ISD::VP_REDUCE_FMUL:
587 case ISD::VP_REDUCE_SEQ_FADD:
588 case ISD::VP_REDUCE_SEQ_FMUL:
589 return true;
590 }
591}
592
593/// The operand position of the vector mask.
594std::optional<unsigned> ISD::getVPMaskIdx(unsigned Opcode) {
595 switch (Opcode) {
596 default:
597 return std::nullopt;
598#define BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, TDNAME, MASKPOS, ...) \
599 case ISD::VPSD: \
600 return MASKPOS;
601#include "llvm/IR/VPIntrinsics.def"
602 }
603}
604
605/// The operand position of the explicit vector length parameter.
606std::optional<unsigned> ISD::getVPExplicitVectorLengthIdx(unsigned Opcode) {
607 switch (Opcode) {
608 default:
609 return std::nullopt;
610#define BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, TDNAME, MASKPOS, EVLPOS) \
611 case ISD::VPSD: \
612 return EVLPOS;
613#include "llvm/IR/VPIntrinsics.def"
614 }
615}
616
617std::optional<unsigned> ISD::getBaseOpcodeForVP(unsigned VPOpcode,
618 bool hasFPExcept) {
619 // FIXME: Return strict opcodes in case of fp exceptions.
620 switch (VPOpcode) {
621 default:
622 return std::nullopt;
623#define BEGIN_REGISTER_VP_SDNODE(VPOPC, ...) case ISD::VPOPC:
624#define VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) return ISD::SDOPC;
625#define END_REGISTER_VP_SDNODE(VPOPC) break;
626#include "llvm/IR/VPIntrinsics.def"
627 }
628 return std::nullopt;
629}
630
631std::optional<unsigned> ISD::getVPForBaseOpcode(unsigned Opcode) {
632 switch (Opcode) {
633 default:
634 return std::nullopt;
635#define BEGIN_REGISTER_VP_SDNODE(VPOPC, ...) break;
636#define VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) case ISD::SDOPC:
637#define END_REGISTER_VP_SDNODE(VPOPC) return ISD::VPOPC;
638#include "llvm/IR/VPIntrinsics.def"
639 }
640}
641
642ISD::NodeType ISD::getExtForLoadExtType(bool IsFP, ISD::LoadExtType ExtType) {
643 switch (ExtType) {
644 case ISD::EXTLOAD:
645 return IsFP ? ISD::FP_EXTEND : ISD::ANY_EXTEND;
646 case ISD::SEXTLOAD:
647 return ISD::SIGN_EXTEND;
648 case ISD::ZEXTLOAD:
649 return ISD::ZERO_EXTEND;
650 default:
651 break;
652 }
653
654 llvm_unreachable("Invalid LoadExtType");
655}
656
657ISD::CondCode ISD::getSetCCSwappedOperands(ISD::CondCode Operation) {
658 // To perform this operation, we just need to swap the L and G bits of the
659 // operation.
660 unsigned OldL = (Operation >> 2) & 1;
661 unsigned OldG = (Operation >> 1) & 1;
662 return ISD::CondCode((Operation & ~6) | // Keep the N, U, E bits
663 (OldL << 1) | // New G bit
664 (OldG << 2)); // New L bit.
665}
666
667static ISD::CondCode getSetCCInverseImpl(ISD::CondCode Op, bool isIntegerLike) {
668 unsigned Operation = Op;
669 if (isIntegerLike)
670 Operation ^= 7; // Flip L, G, E bits, but not U.
671 else
672 Operation ^= 15; // Flip all of the condition bits.
673
674 if (Operation > ISD::SETTRUE2)
675 Operation &= ~8; // Don't let N and U bits get set.
676
677 return ISD::CondCode(Operation);
678}
679
680ISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, EVT Type) {
681 return getSetCCInverseImpl(Op, isIntegerLike: Type.isInteger());
682}
683
684ISD::CondCode ISD::GlobalISel::getSetCCInverse(ISD::CondCode Op,
685 bool isIntegerLike) {
686 return getSetCCInverseImpl(Op, isIntegerLike);
687}
688
689/// For an integer comparison, return 1 if the comparison is a signed operation
690/// and 2 if the result is an unsigned comparison. Return zero if the operation
691/// does not depend on the sign of the input (setne and seteq).
692static int isSignedOp(ISD::CondCode Opcode) {
693 switch (Opcode) {
694 default: llvm_unreachable("Illegal integer setcc operation!");
695 case ISD::SETEQ:
696 case ISD::SETNE: return 0;
697 case ISD::SETLT:
698 case ISD::SETLE:
699 case ISD::SETGT:
700 case ISD::SETGE: return 1;
701 case ISD::SETULT:
702 case ISD::SETULE:
703 case ISD::SETUGT:
704 case ISD::SETUGE: return 2;
705 }
706}
707
708ISD::CondCode ISD::getSetCCOrOperation(ISD::CondCode Op1, ISD::CondCode Op2,
709 EVT Type) {
710 bool IsInteger = Type.isInteger();
711 if (IsInteger && (isSignedOp(Opcode: Op1) | isSignedOp(Opcode: Op2)) == 3)
712 // Cannot fold a signed integer setcc with an unsigned integer setcc.
713 return ISD::SETCC_INVALID;
714
715 unsigned Op = Op1 | Op2; // Combine all of the condition bits.
716
717 // If the N and U bits get set, then the resultant comparison DOES suddenly
718 // care about orderedness, and it is true when ordered.
719 if (Op > ISD::SETTRUE2)
720 Op &= ~16; // Clear the U bit if the N bit is set.
721
722 // Canonicalize illegal integer setcc's.
723 if (IsInteger && Op == ISD::SETUNE) // e.g. SETUGT | SETULT
724 Op = ISD::SETNE;
725
726 return ISD::CondCode(Op);
727}
728
729ISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2,
730 EVT Type) {
731 bool IsInteger = Type.isInteger();
732 if (IsInteger && (isSignedOp(Opcode: Op1) | isSignedOp(Opcode: Op2)) == 3)
733 // Cannot fold a signed setcc with an unsigned setcc.
734 return ISD::SETCC_INVALID;
735
736 // Combine all of the condition bits.
737 ISD::CondCode Result = ISD::CondCode(Op1 & Op2);
738
739 // Canonicalize illegal integer setcc's.
740 if (IsInteger) {
741 switch (Result) {
742 default: break;
743 case ISD::SETUO : Result = ISD::SETFALSE; break; // SETUGT & SETULT
744 case ISD::SETOEQ: // SETEQ & SETU[LG]E
745 case ISD::SETUEQ: Result = ISD::SETEQ ; break; // SETUGE & SETULE
746 case ISD::SETOLT: Result = ISD::SETULT ; break; // SETULT & SETNE
747 case ISD::SETOGT: Result = ISD::SETUGT ; break; // SETUGT & SETNE
748 }
749 }
750
751 return Result;
752}
753
754//===----------------------------------------------------------------------===//
755// SDNode Key Support
756//===----------------------------------------------------------------------===//
757
758/// If this is an SDNode with special info, add this info to the NodeID data.
759/// MorphNodeTo passes the opcode \p N is becoming, so \p Opc may differ from
760/// N->getOpcode() and a case may only cast to a class \p N already is.
761static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N,
762 unsigned Opc) {
763 // Machine nodes are uniqued by opcode, value types and operands alone, as in
764 // getMachineNode; MorphNodeTo's clearMemRefs overwrites the fields below.
765 if (SDNode::isMachineOpcode(Opc))
766 return;
767
768 switch (Opc) {
769 case ISD::TargetExternalSymbol:
770 case ISD::ExternalSymbol:
771 case ISD::MCSymbol:
772 llvm_unreachable("Should only be used on nodes with operands");
773 default: break; // Normal nodes don't need extra info.
774 case ISD::TargetConstant:
775 case ISD::Constant: {
776 const ConstantSDNode *C = cast<ConstantSDNode>(Val: N);
777 ID.AddPointer(Ptr: C->getConstantIntValue());
778 ID.AddBoolean(B: C->isOpaque());
779 break;
780 }
781 case ISD::TargetConstantFP:
782 case ISD::ConstantFP:
783 ID.AddPointer(Ptr: cast<ConstantFPSDNode>(Val: N)->getConstantFPValue());
784 break;
785 case ISD::TargetGlobalAddress:
786 case ISD::GlobalAddress:
787 case ISD::TargetGlobalTLSAddress:
788 case ISD::GlobalTLSAddress: {
789 const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Val: N);
790 ID.AddPointer(Ptr: GA->getGlobal());
791 ID.AddInteger(I: GA->getOffset());
792 ID.AddInteger(I: GA->getTargetFlags());
793 break;
794 }
795 case ISD::BasicBlock:
796 ID.AddPointer(Ptr: cast<BasicBlockSDNode>(Val: N)->getBasicBlock());
797 break;
798 case ISD::EH_LABEL:
799 case ISD::ANNOTATION_LABEL:
800 ID.AddPointer(Ptr: cast<LabelSDNode>(Val: N)->getLabel());
801 break;
802 case ISD::DEACTIVATION_SYMBOL:
803 ID.AddPointer(Ptr: cast<DeactivationSymbolSDNode>(Val: N)->getGlobal());
804 break;
805 case ISD::Register:
806 ID.AddInteger(I: cast<RegisterSDNode>(Val: N)->getReg().id());
807 break;
808 case ISD::RegisterMask:
809 ID.AddPointer(Ptr: cast<RegisterMaskSDNode>(Val: N)->getRegMask());
810 break;
811 case ISD::SRCVALUE:
812 ID.AddPointer(Ptr: cast<SrcValueSDNode>(Val: N)->getValue());
813 break;
814 case ISD::FrameIndex:
815 case ISD::TargetFrameIndex:
816 ID.AddInteger(I: cast<FrameIndexSDNode>(Val: N)->getIndex());
817 break;
818 case ISD::PSEUDO_PROBE:
819 ID.AddInteger(I: cast<PseudoProbeSDNode>(Val: N)->getGuid());
820 ID.AddInteger(I: cast<PseudoProbeSDNode>(Val: N)->getIndex());
821 ID.AddInteger(I: cast<PseudoProbeSDNode>(Val: N)->getAttributes());
822 break;
823 case ISD::JumpTable:
824 case ISD::TargetJumpTable:
825 ID.AddInteger(I: cast<JumpTableSDNode>(Val: N)->getIndex());
826 ID.AddInteger(I: cast<JumpTableSDNode>(Val: N)->getTargetFlags());
827 break;
828 case ISD::ConstantPool:
829 case ISD::TargetConstantPool: {
830 const ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Val: N);
831 ID.AddInteger(I: CP->getAlign().value());
832 ID.AddInteger(I: CP->getOffset());
833 if (CP->isMachineConstantPoolEntry())
834 CP->getMachineCPVal()->addSelectionDAGCSEId(ID);
835 else
836 ID.AddPointer(Ptr: CP->getConstVal());
837 ID.AddInteger(I: CP->getTargetFlags());
838 break;
839 }
840 case ISD::TargetIndex: {
841 const TargetIndexSDNode *TI = cast<TargetIndexSDNode>(Val: N);
842 ID.AddInteger(I: TI->getIndex());
843 ID.AddInteger(I: TI->getOffset());
844 ID.AddInteger(I: TI->getTargetFlags());
845 break;
846 }
847 case ISD::LOAD: {
848 const LoadSDNode *LD = cast<LoadSDNode>(Val: N);
849 ID.AddInteger(I: LD->getMemoryVT().getRawBits());
850 ID.AddInteger(I: LD->getRawSubclassData());
851 ID.AddInteger(I: LD->getPointerInfo().getAddrSpace());
852 ID.AddInteger(I: LD->getMemOperand()->getFlags());
853 break;
854 }
855 case ISD::STORE: {
856 const StoreSDNode *ST = cast<StoreSDNode>(Val: N);
857 ID.AddInteger(I: ST->getMemoryVT().getRawBits());
858 ID.AddInteger(I: ST->getRawSubclassData());
859 ID.AddInteger(I: ST->getPointerInfo().getAddrSpace());
860 ID.AddInteger(I: ST->getMemOperand()->getFlags());
861 break;
862 }
863 case ISD::VP_LOAD: {
864 const VPLoadSDNode *ELD = cast<VPLoadSDNode>(Val: N);
865 ID.AddInteger(I: ELD->getMemoryVT().getRawBits());
866 ID.AddInteger(I: ELD->getRawSubclassData());
867 ID.AddInteger(I: ELD->getPointerInfo().getAddrSpace());
868 ID.AddInteger(I: ELD->getMemOperand()->getFlags());
869 break;
870 }
871 case ISD::VP_LOAD_FF: {
872 const auto *LD = cast<VPLoadFFSDNode>(Val: N);
873 ID.AddInteger(I: LD->getMemoryVT().getRawBits());
874 ID.AddInteger(I: LD->getRawSubclassData());
875 ID.AddInteger(I: LD->getPointerInfo().getAddrSpace());
876 ID.AddInteger(I: LD->getMemOperand()->getFlags());
877 break;
878 }
879 case ISD::VP_STORE: {
880 const VPStoreSDNode *EST = cast<VPStoreSDNode>(Val: N);
881 ID.AddInteger(I: EST->getMemoryVT().getRawBits());
882 ID.AddInteger(I: EST->getRawSubclassData());
883 ID.AddInteger(I: EST->getPointerInfo().getAddrSpace());
884 ID.AddInteger(I: EST->getMemOperand()->getFlags());
885 break;
886 }
887 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD: {
888 const VPStridedLoadSDNode *SLD = cast<VPStridedLoadSDNode>(Val: N);
889 ID.AddInteger(I: SLD->getMemoryVT().getRawBits());
890 ID.AddInteger(I: SLD->getRawSubclassData());
891 ID.AddInteger(I: SLD->getPointerInfo().getAddrSpace());
892 break;
893 }
894 case ISD::EXPERIMENTAL_VP_STRIDED_STORE: {
895 const VPStridedStoreSDNode *SST = cast<VPStridedStoreSDNode>(Val: N);
896 ID.AddInteger(I: SST->getMemoryVT().getRawBits());
897 ID.AddInteger(I: SST->getRawSubclassData());
898 ID.AddInteger(I: SST->getPointerInfo().getAddrSpace());
899 break;
900 }
901 case ISD::VP_GATHER: {
902 const VPGatherSDNode *EG = cast<VPGatherSDNode>(Val: N);
903 ID.AddInteger(I: EG->getMemoryVT().getRawBits());
904 ID.AddInteger(I: EG->getRawSubclassData());
905 ID.AddInteger(I: EG->getPointerInfo().getAddrSpace());
906 ID.AddInteger(I: EG->getMemOperand()->getFlags());
907 break;
908 }
909 case ISD::VP_SCATTER: {
910 const VPScatterSDNode *ES = cast<VPScatterSDNode>(Val: N);
911 ID.AddInteger(I: ES->getMemoryVT().getRawBits());
912 ID.AddInteger(I: ES->getRawSubclassData());
913 ID.AddInteger(I: ES->getPointerInfo().getAddrSpace());
914 ID.AddInteger(I: ES->getMemOperand()->getFlags());
915 break;
916 }
917 case ISD::MLOAD: {
918 const MaskedLoadSDNode *MLD = cast<MaskedLoadSDNode>(Val: N);
919 ID.AddInteger(I: MLD->getMemoryVT().getRawBits());
920 ID.AddInteger(I: MLD->getRawSubclassData());
921 ID.AddInteger(I: MLD->getPointerInfo().getAddrSpace());
922 ID.AddInteger(I: MLD->getMemOperand()->getFlags());
923 break;
924 }
925 case ISD::MSTORE: {
926 const MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(Val: N);
927 ID.AddInteger(I: MST->getMemoryVT().getRawBits());
928 ID.AddInteger(I: MST->getRawSubclassData());
929 ID.AddInteger(I: MST->getPointerInfo().getAddrSpace());
930 ID.AddInteger(I: MST->getMemOperand()->getFlags());
931 break;
932 }
933 case ISD::MGATHER: {
934 const MaskedGatherSDNode *MG = cast<MaskedGatherSDNode>(Val: N);
935 ID.AddInteger(I: MG->getMemoryVT().getRawBits());
936 ID.AddInteger(I: MG->getRawSubclassData());
937 ID.AddInteger(I: MG->getPointerInfo().getAddrSpace());
938 ID.AddInteger(I: MG->getMemOperand()->getFlags());
939 break;
940 }
941 case ISD::MSCATTER:
942 case ISD::GET_FPENV_MEM:
943 case ISD::SET_FPENV_MEM:
944 case ISD::EXPERIMENTAL_VECTOR_HISTOGRAM: {
945 const MemSDNode *M = cast<MemSDNode>(Val: N);
946 ID.AddInteger(I: M->getMemoryVT().getRawBits());
947 ID.AddInteger(I: M->getRawSubclassData());
948 ID.AddInteger(I: M->getPointerInfo().getAddrSpace());
949 ID.AddInteger(I: M->getMemOperand()->getFlags());
950 break;
951 }
952 case ISD::VECTOR_SHUFFLE: {
953 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Val: N)->getMask();
954 for (int M : Mask)
955 ID.AddInteger(I: M);
956 break;
957 }
958 case ISD::ADDRSPACECAST: {
959 const AddrSpaceCastSDNode *ASC = cast<AddrSpaceCastSDNode>(Val: N);
960 ID.AddInteger(I: ASC->getSrcAddressSpace());
961 ID.AddInteger(I: ASC->getDestAddressSpace());
962 break;
963 }
964 case ISD::TargetBlockAddress:
965 case ISD::BlockAddress: {
966 const BlockAddressSDNode *BA = cast<BlockAddressSDNode>(Val: N);
967 ID.AddPointer(Ptr: BA->getBlockAddress());
968 ID.AddInteger(I: BA->getOffset());
969 ID.AddInteger(I: BA->getTargetFlags());
970 break;
971 }
972 case ISD::AssertAlign:
973 ID.AddInteger(I: cast<AssertAlignSDNode>(Val: N)->getAlign().value());
974 break;
975 case ISD::PREFETCH:
976 case ISD::INTRINSIC_VOID:
977 case ISD::INTRINSIC_W_CHAIN:
978 // Handled by MemIntrinsicSDNode check after the switch.
979 break;
980 case ISD::MDNODE_SDNODE:
981 ID.AddPointer(Ptr: cast<MDNodeSDNode>(Val: N)->getMD());
982 break;
983 } // end switch (Opc)
984
985 if (auto *AT = dyn_cast<AtomicSDNode>(Val: N)) {
986 ID.AddInteger(I: AT->getMemoryVT().getRawBits());
987 ID.AddInteger(I: AT->getRawSubclassData());
988 ID.AddInteger(I: AT->getPointerInfo().getAddrSpace());
989 ID.AddInteger(I: AT->getMemOperand()->getFlags());
990 }
991
992 // MemIntrinsic nodes could also have subclass data, address spaces, and flags
993 // to check.
994 if (auto *MN = dyn_cast<MemIntrinsicSDNode>(Val: N)) {
995 ID.AddInteger(I: MN->getRawSubclassData());
996 ID.AddInteger(I: MN->getMemoryVT().getRawBits());
997 for (const MachineMemOperand *MMO : MN->memoperands()) {
998 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
999 ID.AddInteger(I: MMO->getFlags());
1000 }
1001 }
1002}
1003
1004SDNodeKey::SDNodeKey(const SDNode &N)
1005 : Opcode(N.getOpcode()), VTs(N.getVTList().VTs),
1006 OpStorage(N.op_begin(), N.op_end()) {
1007 Ops = OpStorage;
1008 AddNodeIDCustom(ID&: Tail, N: &N, Opc: N.getOpcode());
1009}
1010
1011#ifndef NDEBUG
1012/// Serialize a node the way SDNodes were uniqued before SDNodeKey, to
1013/// cross-check the typed comparison against. Deliberately reads the node
1014/// rather than going through SDNodeKey, so the key constructor is checked too
1015/// and not used as its own oracle.
1016static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) {
1017 ID.AddInteger(N->getOpcode());
1018 ID.AddPointer(N->getVTList().VTs);
1019 for (const SDUse &Op : N->ops()) {
1020 ID.AddPointer(Op.getNode());
1021 ID.AddInteger(Op.getResNo());
1022 }
1023 AddNodeIDCustom(ID, N, N->getOpcode());
1024}
1025
1026static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNodeKey &Key) {
1027 ID.AddInteger(Key.Opcode);
1028 ID.AddPointer(Key.VTs);
1029 for (const SDValue &Op : Key.Ops) {
1030 ID.AddPointer(Op.getNode());
1031 ID.AddInteger(Op.getResNo());
1032 }
1033 ID.AddNodeID(Key.Tail);
1034}
1035#endif
1036
1037static bool keyMatches(const SDNodeKey &Key, const SDNode &N) {
1038 if (N.getOpcode() != Key.Opcode || N.getVTList().VTs != Key.VTs)
1039 return false;
1040 if (!llvm::equal(LRange: N.ops(), RRange: Key.Ops))
1041 return false;
1042 // Build only what AddNodeIDCustom would have added, which is nothing for
1043 // most opcodes, rather than the whole profile.
1044 FoldingSetNodeID Tail;
1045 AddNodeIDCustom(ID&: Tail, N: &N, Opc: N.getOpcode());
1046 return Tail == Key.Tail;
1047}
1048
1049bool SDNodeKeyInfo::isEqual(const SDNodeKey &Key, const SDNode &N) {
1050 bool Result = keyMatches(Key, N);
1051
1052#ifndef NDEBUG
1053 FoldingSetNodeID NodeProfile, KeyProfile;
1054 AddNodeIDNode(NodeProfile, &N);
1055 AddNodeIDNode(KeyProfile, Key);
1056 assert((NodeProfile == KeyProfile) == Result &&
1057 "SDNodeKey equality disagrees with profile");
1058#endif
1059 return Result;
1060}
1061
1062//===----------------------------------------------------------------------===//
1063// SelectionDAG Class
1064//===----------------------------------------------------------------------===//
1065
1066/// doNotCSE - Return true if CSE should not be performed for this node.
1067static bool doNotCSE(SDNode *N) {
1068 if (N->getValueType(ResNo: 0) == MVT::Glue)
1069 return true; // Never CSE anything that produces a glue result.
1070
1071 switch (N->getOpcode()) {
1072 default: break;
1073 case ISD::HANDLENODE:
1074 case ISD::EH_LABEL:
1075 return true; // Never CSE these nodes.
1076 }
1077
1078 // Check that remaining values produced are not flags.
1079 for (unsigned i = 1, e = N->getNumValues(); i != e; ++i)
1080 if (N->getValueType(ResNo: i) == MVT::Glue)
1081 return true; // Never CSE anything that produces a glue result.
1082
1083 return false;
1084}
1085
1086/// Construct a DemandedElts mask which demands all elements of \p V.
1087/// If \p V is not a fixed-length vector, then this will return a single bit.
1088static APInt getDemandAllEltsMask(SDValue V) {
1089 EVT VT = V.getValueType();
1090 // Since the number of lanes in a scalable vector is unknown at compile time,
1091 // we track one bit which is implicitly broadcast to all lanes. This means
1092 // that all lanes in a scalable vector are considered demanded.
1093 return VT.isFixedLengthVector() ? APInt::getAllOnes(numBits: VT.getVectorNumElements())
1094 : APInt(1, 1);
1095}
1096
1097/// RemoveDeadNodes - This method deletes all unreachable nodes in the
1098/// SelectionDAG.
1099void SelectionDAG::RemoveDeadNodes() {
1100 // Create a dummy node (which is not added to allnodes), that adds a reference
1101 // to the root node, preventing it from being deleted.
1102 HandleSDNode Dummy(getRoot());
1103
1104 SmallVector<SDNode*, 128> DeadNodes;
1105
1106 // Add all obviously-dead nodes to the DeadNodes worklist.
1107 for (SDNode &Node : allnodes())
1108 if (Node.use_empty())
1109 DeadNodes.push_back(Elt: &Node);
1110
1111 RemoveDeadNodes(DeadNodes);
1112
1113 // If the root changed (e.g. it was a dead load, update the root).
1114 setRoot(Dummy.getValue());
1115}
1116
1117/// RemoveDeadNodes - This method deletes the unreachable nodes in the
1118/// given list, and any nodes that become unreachable as a result.
1119void SelectionDAG::RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes) {
1120
1121 // Process the worklist, deleting the nodes and adding their uses to the
1122 // worklist.
1123 while (!DeadNodes.empty()) {
1124 SDNode *N = DeadNodes.pop_back_val();
1125 // Skip to next node if we've already managed to delete the node. This could
1126 // happen if replacing a node causes a node previously added to the node to
1127 // be deleted.
1128 if (N->getOpcode() == ISD::DELETED_NODE)
1129 continue;
1130
1131 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1132 DUL->NodeDeleted(N, nullptr);
1133
1134 // Take the node out of the appropriate CSE map.
1135 RemoveNodeFromCSEMaps(N);
1136
1137 // Next, brutally remove the operand list. This is safe to do, as there are
1138 // no cycles in the graph.
1139 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
1140 SDUse &Use = *I++;
1141 SDNode *Operand = Use.getNode();
1142 Use.set(SDValue());
1143
1144 // Now that we removed this operand, see if there are no uses of it left.
1145 if (Operand->use_empty())
1146 DeadNodes.push_back(Elt: Operand);
1147 }
1148
1149 DeallocateNode(N);
1150 }
1151}
1152
1153void SelectionDAG::RemoveDeadNode(SDNode *N){
1154 SmallVector<SDNode*, 16> DeadNodes(1, N);
1155
1156 // Create a dummy node that adds a reference to the root node, preventing
1157 // it from being deleted. (This matters if the root is an operand of the
1158 // dead node.)
1159 HandleSDNode Dummy(getRoot());
1160
1161 RemoveDeadNodes(DeadNodes);
1162}
1163
1164void SelectionDAG::DeleteNode(SDNode *N) {
1165 // First take this out of the appropriate CSE map.
1166 RemoveNodeFromCSEMaps(N);
1167
1168 // Finally, remove uses due to operands of this node, remove from the
1169 // AllNodes list, and delete the node.
1170 DeleteNodeNotInCSEMaps(N);
1171}
1172
1173void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) {
1174 assert(N->getIterator() != AllNodes.begin() &&
1175 "Cannot delete the entry node!");
1176 assert(N->use_empty() && "Cannot delete a node that is not dead!");
1177
1178 // Drop all of the operands and decrement used node's use counts.
1179 N->DropOperands();
1180
1181 DeallocateNode(N);
1182}
1183
1184void SDDbgInfo::add(SDDbgValue *V, bool isParameter) {
1185 assert(!(V->isVariadic() && isParameter));
1186 if (isParameter)
1187 ByvalParmDbgValues.push_back(Elt: V);
1188 else
1189 DbgValues.push_back(Elt: V);
1190 for (const SDNode *Node : V->getSDNodes())
1191 if (Node)
1192 DbgValMap[Node].push_back(Elt: V);
1193}
1194
1195void SDDbgInfo::erase(const SDNode *Node) {
1196 DbgValMapType::iterator I = DbgValMap.find(Val: Node);
1197 if (I == DbgValMap.end())
1198 return;
1199 for (auto &Val: I->second)
1200 Val->setIsInvalidated();
1201 DbgValMap.erase(I);
1202}
1203
1204void SelectionDAG::DeallocateNode(SDNode *N) {
1205 // If we have operands, deallocate them.
1206 removeOperands(Node: N);
1207
1208 NodeAllocator.Deallocate(E: AllNodes.remove(IT: N));
1209
1210 // Set the opcode to DELETED_NODE to help catch bugs when node
1211 // memory is reallocated.
1212 // FIXME: There are places in SDag that have grown a dependency on the opcode
1213 // value in the released node.
1214 __asan_unpoison_memory_region(&N->NodeType, sizeof(N->NodeType));
1215 N->NodeType = ISD::DELETED_NODE;
1216
1217 // If any of the SDDbgValue nodes refer to this SDNode, invalidate
1218 // them and forget about that node.
1219 DbgInfo->erase(Node: N);
1220
1221 // Invalidate extra info.
1222 SDEI.erase(Val: N);
1223}
1224
1225#ifndef NDEBUG
1226/// VerifySDNode - Check the given SDNode. Aborts if it is invalid.
1227void SelectionDAG::verifyNode(SDNode *N) const {
1228 switch (N->getOpcode()) {
1229 default:
1230 if (N->isTargetOpcode())
1231 getSelectionDAGInfo().verifyTargetNode(*this, N);
1232 break;
1233 case ISD::BUILD_PAIR: {
1234 EVT VT = N->getValueType(0);
1235 assert(N->getNumValues() == 1 && "Too many results!");
1236 assert(!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) &&
1237 "Wrong return type!");
1238 assert(N->getNumOperands() == 2 && "Wrong number of operands!");
1239 assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() &&
1240 "Mismatched operand types!");
1241 assert(N->getOperand(0).getValueType().isInteger() == VT.isInteger() &&
1242 "Wrong operand type!");
1243 assert(VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() &&
1244 "Wrong return type size");
1245 break;
1246 }
1247 case ISD::BUILD_VECTOR: {
1248 assert(N->getNumValues() == 1 && "Too many results!");
1249 assert(N->getValueType(0).isVector() && "Wrong return type!");
1250 assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() &&
1251 "Wrong number of operands!");
1252 EVT EltVT = N->getValueType(0).getVectorElementType();
1253 for (const SDUse &Op : N->ops()) {
1254 assert((Op.getValueType() == EltVT ||
1255 (EltVT.isInteger() && Op.getValueType().isInteger() &&
1256 EltVT.bitsLE(Op.getValueType()))) &&
1257 "Wrong operand type!");
1258 assert(Op.getValueType() == N->getOperand(0).getValueType() &&
1259 "Operands must all have the same type");
1260 }
1261 break;
1262 }
1263 case ISD::SADDO:
1264 case ISD::UADDO:
1265 case ISD::SSUBO:
1266 case ISD::USUBO:
1267 assert(N->getNumValues() == 2 && "Wrong number of results!");
1268 assert(N->getVTList().NumVTs == 2 && N->getNumOperands() == 2 &&
1269 "Invalid add/sub overflow op!");
1270 assert(N->getVTList().VTs[0].isInteger() &&
1271 N->getVTList().VTs[1].isInteger() &&
1272 N->getOperand(0).getValueType() == N->getOperand(1).getValueType() &&
1273 N->getOperand(0).getValueType() == N->getVTList().VTs[0] &&
1274 "Binary operator types must match!");
1275 break;
1276 }
1277}
1278#endif // NDEBUG
1279
1280/// Insert a newly allocated node into the DAG.
1281///
1282/// Handles insertion into the all nodes list and CSE map, as well as
1283/// verification and other common operations when a new node is allocated.
1284void SelectionDAG::InsertNode(SDNode *N) {
1285 AllNodes.push_back(val: N);
1286#ifndef NDEBUG
1287 N->PersistentId = NextPersistentId++;
1288 verifyNode(N);
1289#endif
1290 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1291 DUL->NodeInserted(N);
1292}
1293
1294/// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that
1295/// correspond to it. This is useful when we're about to delete or repurpose
1296/// the node. We don't want future request for structurally identical nodes
1297/// to return N anymore.
1298bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) {
1299 bool Erased = false;
1300 switch (N->getOpcode()) {
1301 case ISD::HANDLENODE: return false; // noop.
1302 case ISD::CONDCODE:
1303 assert(CondCodeNodes[cast<CondCodeSDNode>(N)->get()] &&
1304 "Cond code doesn't exist!");
1305 Erased = CondCodeNodes[cast<CondCodeSDNode>(Val: N)->get()] != nullptr;
1306 CondCodeNodes[cast<CondCodeSDNode>(Val: N)->get()] = nullptr;
1307 break;
1308 case ISD::ExternalSymbol:
1309 Erased = ExternalSymbols.erase(Key: cast<ExternalSymbolSDNode>(Val: N)->getSymbol());
1310 break;
1311 case ISD::TargetExternalSymbol: {
1312 ExternalSymbolSDNode *ESN = cast<ExternalSymbolSDNode>(Val: N);
1313 Erased = TargetExternalSymbols.erase(x: std::pair<std::string, unsigned>(
1314 ESN->getSymbol(), ESN->getTargetFlags()));
1315 break;
1316 }
1317 case ISD::MCSymbol: {
1318 auto *MCSN = cast<MCSymbolSDNode>(Val: N);
1319 Erased = MCSymbols.erase(Val: MCSN->getMCSymbol());
1320 break;
1321 }
1322 case ISD::VALUETYPE: {
1323 EVT VT = cast<VTSDNode>(Val: N)->getVT();
1324 if (VT.isExtended()) {
1325 Erased = ExtendedValueTypeNodes.erase(x: VT);
1326 } else {
1327 Erased = ValueTypeNodes[VT.getSimpleVT().SimpleTy] != nullptr;
1328 ValueTypeNodes[VT.getSimpleVT().SimpleTy] = nullptr;
1329 }
1330 break;
1331 }
1332 default:
1333 // Remove it from the CSE Map.
1334 assert(N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!");
1335 assert(N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!");
1336 Erased = CSEMap.erase(N);
1337 break;
1338 }
1339#ifndef NDEBUG
1340 // Verify that the node was actually in one of the CSE maps, unless it has a
1341 // glue result (which cannot be CSE'd) or is one of the special cases that are
1342 // not subject to CSE.
1343 if (!Erased && N->getValueType(N->getNumValues()-1) != MVT::Glue &&
1344 !N->isMachineOpcode() && !doNotCSE(N)) {
1345 N->dump(this);
1346 dbgs() << "\n";
1347 llvm_unreachable("Node is not in map!");
1348 }
1349#endif
1350 return Erased;
1351}
1352
1353/// AddModifiedNodeToCSEMaps - The specified node has been removed from the CSE
1354/// maps and modified in place. Add it back to the CSE maps, unless an identical
1355/// node already exists, in which case transfer all its users to the existing
1356/// node. This transfer can potentially trigger recursive merging.
1357void
1358SelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N) {
1359 // For node types that aren't CSE'd, just act as if no identical node
1360 // already exists.
1361 if (!doNotCSE(N)) {
1362 SDNode *Existing = CSEMap.getOrInsert(N);
1363 if (Existing != N) {
1364 // If there was already an existing matching node, use ReplaceAllUsesWith
1365 // to replace the dead one with the existing one. This can cause
1366 // recursive merging of other unrelated nodes down the line.
1367 Existing->intersectFlagsWith(Flags: N->getFlags());
1368 if (auto *MemNode = dyn_cast<MemSDNode>(Val: Existing)) {
1369 ArrayRef<MachineMemOperand *> NewMMOs =
1370 cast<MemSDNode>(Val: N)->memoperands();
1371 // Range and cache hint metadata are not part of the DAG CSE key because
1372 // we prefer to CSE even when metadata does not match. Merge potentially
1373 // differing metadata conservatively.
1374 MemNode->refineMMOMetadata(NewMMOs);
1375 }
1376 ReplaceAllUsesWith(From: N, To: Existing);
1377
1378 // N is now dead. Inform the listeners and delete it.
1379 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1380 DUL->NodeDeleted(N, Existing);
1381 DeleteNodeNotInCSEMaps(N);
1382 return;
1383 }
1384 }
1385
1386 // If the node doesn't already exist, we updated it. Inform listeners.
1387 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1388 DUL->NodeUpdated(N);
1389}
1390
1391/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1392/// were replaced with those specified. If this node is never memoized,
1393/// return null, otherwise return a pointer to the slot it would take. If a
1394/// node already exists with these operands, the slot will be non-null.
1395SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op,
1396 FoldingSetInsertToken &InsertToken) {
1397 if (doNotCSE(N))
1398 return nullptr;
1399
1400 SDValue Ops[] = { Op };
1401 SDNodeKey ID(N->getOpcode(), N->getVTList(), Ops);
1402 AddNodeIDCustom(ID&: ID.Tail, N, Opc: N->getOpcode());
1403 SDNode *Node = lookupNode(Key: ID, DL: SDLoc(N), InsertToken);
1404 if (Node)
1405 Node->intersectFlagsWith(Flags: N->getFlags());
1406 return Node;
1407}
1408
1409/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1410/// were replaced with those specified. If this node is never memoized,
1411/// return null, otherwise return a pointer to the slot it would take. If a
1412/// node already exists with these operands, the slot will be non-null.
1413SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op1, SDValue Op2,
1414 FoldingSetInsertToken &InsertToken) {
1415 if (doNotCSE(N))
1416 return nullptr;
1417
1418 SDValue Ops[] = { Op1, Op2 };
1419 SDNodeKey ID(N->getOpcode(), N->getVTList(), Ops);
1420 AddNodeIDCustom(ID&: ID.Tail, N, Opc: N->getOpcode());
1421 SDNode *Node = lookupNode(Key: ID, DL: SDLoc(N), InsertToken);
1422 if (Node)
1423 Node->intersectFlagsWith(Flags: N->getFlags());
1424 return Node;
1425}
1426
1427/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1428/// were replaced with those specified. If this node is never memoized,
1429/// return null, otherwise return a pointer to the slot it would take. If a
1430/// node already exists with these operands, the slot will be non-null.
1431SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, ArrayRef<SDValue> Ops,
1432 FoldingSetInsertToken &InsertToken) {
1433 if (doNotCSE(N))
1434 return nullptr;
1435
1436 SDNodeKey ID(N->getOpcode(), N->getVTList(), Ops);
1437 AddNodeIDCustom(ID&: ID.Tail, N, Opc: N->getOpcode());
1438 SDNode *Node = lookupNode(Key: ID, DL: SDLoc(N), InsertToken);
1439 if (Node)
1440 Node->intersectFlagsWith(Flags: N->getFlags());
1441 return Node;
1442}
1443
1444Align SelectionDAG::getEVTAlign(EVT VT) const {
1445 Type *Ty = VT == MVT::iPTR ? PointerType::get(C&: *getContext(), AddressSpace: 0)
1446 : VT.getTypeForEVT(Context&: *getContext());
1447
1448 return getDataLayout().getABITypeAlign(Ty);
1449}
1450
1451// EntryNode could meaningfully have debug info if we can find it...
1452SelectionDAG::SelectionDAG(const TargetMachine &tm, CodeGenOptLevel OL)
1453 : TM(tm), OptLevel(OL), EntryNode(ISD::EntryToken, 0, DebugLoc(),
1454 getVTList(VT1: MVT::Other, VT2: MVT::Glue)),
1455 Root(getEntryNode()) {
1456 InsertNode(N: &EntryNode);
1457 DbgInfo = new SDDbgInfo();
1458}
1459
1460void SelectionDAG::init(MachineFunction &NewMF,
1461 const TargetLibraryInfo *LibraryInfo,
1462 const LibcallLoweringInfo *LibcallsInfo,
1463 UniformityInfo *NewUA, ProfileSummaryInfo *PSIin,
1464 BlockFrequencyInfo *BFIin,
1465 FunctionVarLocs const *VarLocs) {
1466 MF = &NewMF;
1467 TLI = getSubtarget().getTargetLowering();
1468 TSI = getSubtarget().getSelectionDAGInfo();
1469 LibInfo = LibraryInfo;
1470 Libcalls = LibcallsInfo;
1471 Context = &MF->getFunction().getContext();
1472 UA = NewUA;
1473 PSI = PSIin;
1474 BFI = BFIin;
1475 FnVarLocs = VarLocs;
1476}
1477
1478SelectionDAG::~SelectionDAG() {
1479 assert(!UpdateListeners && "Dangling registered DAGUpdateListeners");
1480 allnodes_clear();
1481 OperandRecycler.clear(OperandAllocator);
1482 delete DbgInfo;
1483}
1484
1485bool SelectionDAG::shouldOptForSize() const {
1486 return llvm::shouldOptimizeForSize(BB: FLI->MBB->getBasicBlock(), PSI, BFI);
1487}
1488
1489void SelectionDAG::allnodes_clear() {
1490 assert(&*AllNodes.begin() == &EntryNode);
1491 AllNodes.remove(IT: AllNodes.begin());
1492 while (!AllNodes.empty())
1493 DeallocateNode(N: &AllNodes.front());
1494#ifndef NDEBUG
1495 NextPersistentId = 0;
1496#endif
1497}
1498
1499SDNode *SelectionDAG::lookupNode(const SDNodeKey &Key,
1500 FoldingSetInsertToken &InsertToken) {
1501 SDNode *N = CSEMap.lookup(Key, Token&: InsertToken);
1502 if (N) {
1503 switch (N->getOpcode()) {
1504 default: break;
1505 case ISD::Constant:
1506 case ISD::ConstantFP:
1507 llvm_unreachable("Querying for Constant and ConstantFP nodes requires "
1508 "debug location. Use another overload.");
1509 }
1510 }
1511 return N;
1512}
1513
1514SDNode *SelectionDAG::lookupNode(const SDNodeKey &Key, const SDLoc &DL,
1515 FoldingSetInsertToken &InsertToken) {
1516 SDNode *N = CSEMap.lookup(Key, Token&: InsertToken);
1517 if (N) {
1518 switch (N->getOpcode()) {
1519 case ISD::Constant:
1520 case ISD::ConstantFP:
1521 // Erase debug location from the node if the node is used at several
1522 // different places. Do not propagate one location to all uses as it
1523 // will cause a worse single stepping debugging experience.
1524 if (N->getDebugLoc() != DL.getDebugLoc())
1525 N->setDebugLoc(DebugLoc());
1526 break;
1527 default:
1528 // When the node's point of use is located earlier in the instruction
1529 // sequence than its prior point of use, update its debug info to the
1530 // earlier location.
1531 if (DL.getIROrder() && DL.getIROrder() < N->getIROrder())
1532 N->setDebugLoc(DL.getDebugLoc());
1533 break;
1534 }
1535 }
1536 return N;
1537}
1538
1539void SelectionDAG::clear() {
1540 allnodes_clear();
1541 OperandRecycler.clear(OperandAllocator);
1542 OperandAllocator.Reset();
1543 CSEMap.clear();
1544
1545 ExtendedValueTypeNodes.clear();
1546 ExternalSymbols.clear();
1547 TargetExternalSymbols.clear();
1548 MCSymbols.clear();
1549 SDEI.clear();
1550 llvm::fill(Range&: CondCodeNodes, Value: nullptr);
1551 llvm::fill(Range&: ValueTypeNodes, Value: nullptr);
1552
1553 EntryNode.UseList = nullptr;
1554 InsertNode(N: &EntryNode);
1555 Root = getEntryNode();
1556 DbgInfo->clear();
1557}
1558
1559SDValue SelectionDAG::getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT) {
1560 return VT.bitsGT(VT: Op.getValueType())
1561 ? getNode(Opcode: ISD::FP_EXTEND, DL, VT, Operand: Op)
1562 : getNode(Opcode: ISD::FP_ROUND, DL, VT, N1: Op,
1563 N2: getIntPtrConstant(Val: 0, DL, /*isTarget=*/true));
1564}
1565
1566std::pair<SDValue, SDValue>
1567SelectionDAG::getStrictFPExtendOrRound(SDValue Op, SDValue Chain,
1568 const SDLoc &DL, EVT VT) {
1569 assert(!VT.bitsEq(Op.getValueType()) &&
1570 "Strict no-op FP extend/round not allowed.");
1571 SDValue Res =
1572 VT.bitsGT(VT: Op.getValueType())
1573 ? getNode(Opcode: ISD::STRICT_FP_EXTEND, DL, ResultTys: {VT, MVT::Other}, Ops: {Chain, Op})
1574 : getNode(Opcode: ISD::STRICT_FP_ROUND, DL, ResultTys: {VT, MVT::Other},
1575 Ops: {Chain, Op, getIntPtrConstant(Val: 0, DL, /*isTarget=*/true)});
1576
1577 return std::pair<SDValue, SDValue>(Res, SDValue(Res.getNode(), 1));
1578}
1579
1580SDValue SelectionDAG::getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1581 return VT.bitsGT(VT: Op.getValueType()) ?
1582 getNode(Opcode: ISD::ANY_EXTEND, DL, VT, Operand: Op) :
1583 getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Op);
1584}
1585
1586SDValue SelectionDAG::getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1587 return VT.bitsGT(VT: Op.getValueType()) ?
1588 getNode(Opcode: ISD::SIGN_EXTEND, DL, VT, Operand: Op) :
1589 getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Op);
1590}
1591
1592SDValue SelectionDAG::getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1593 return VT.bitsGT(VT: Op.getValueType()) ?
1594 getNode(Opcode: ISD::ZERO_EXTEND, DL, VT, Operand: Op) :
1595 getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Op);
1596}
1597
1598SDValue SelectionDAG::getBitcastedAnyExtOrTrunc(SDValue Op, const SDLoc &DL,
1599 EVT VT) {
1600 assert(!VT.isVector());
1601 auto Type = Op.getValueType();
1602 SDValue DestOp;
1603 if (Type == VT)
1604 return Op;
1605 auto Size = Op.getValueSizeInBits();
1606 DestOp = getBitcast(VT: EVT::getIntegerVT(Context&: *Context, BitWidth: Size), V: Op);
1607 if (DestOp.getValueType() == VT)
1608 return DestOp;
1609
1610 return getAnyExtOrTrunc(Op: DestOp, DL, VT);
1611}
1612
1613SDValue SelectionDAG::getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT,
1614 EVT OpVT) {
1615 if (VT.bitsLE(VT: Op.getValueType()))
1616 return getNode(Opcode: ISD::TRUNCATE, DL: SL, VT, Operand: Op);
1617
1618 TargetLowering::BooleanContent BType = TLI->getBooleanContents(Type: OpVT);
1619 return getNode(Opcode: TLI->getExtendForContent(Content: BType), DL: SL, VT, Operand: Op);
1620}
1621
1622SDValue SelectionDAG::getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) {
1623 EVT OpVT = Op.getValueType();
1624 assert(VT.isInteger() && OpVT.isInteger() &&
1625 "Cannot getZeroExtendInReg FP types");
1626 assert(VT.isVector() == OpVT.isVector() &&
1627 "getZeroExtendInReg type should be vector iff the operand "
1628 "type is vector!");
1629 assert((!VT.isVector() ||
1630 VT.getVectorElementCount() == OpVT.getVectorElementCount()) &&
1631 "Vector element counts must match in getZeroExtendInReg");
1632 assert(VT.getScalarType().bitsLE(OpVT.getScalarType()) && "Not extending!");
1633 if (OpVT == VT)
1634 return Op;
1635 // TODO: Use computeKnownBits instead of AssertZext.
1636 if (Op.getOpcode() == ISD::AssertZext && cast<VTSDNode>(Val: Op.getOperand(i: 1))
1637 ->getVT()
1638 .getScalarType()
1639 .bitsLE(VT: VT.getScalarType()))
1640 return Op;
1641 APInt Imm = APInt::getLowBitsSet(numBits: OpVT.getScalarSizeInBits(),
1642 loBitsSet: VT.getScalarSizeInBits());
1643 return getNode(Opcode: ISD::AND, DL, VT: OpVT, N1: Op, N2: getConstant(Val: Imm, DL, VT: OpVT));
1644}
1645
1646SDValue SelectionDAG::getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1647 // Only unsigned pointer semantics are supported right now. In the future this
1648 // might delegate to TLI to check pointer signedness.
1649 return getZExtOrTrunc(Op, DL, VT);
1650}
1651
1652SDValue SelectionDAG::getPtrExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) {
1653 // Only unsigned pointer semantics are supported right now. In the future this
1654 // might delegate to TLI to check pointer signedness.
1655 return getZeroExtendInReg(Op, DL, VT);
1656}
1657
1658SDValue SelectionDAG::getNegative(SDValue Val, const SDLoc &DL, EVT VT) {
1659 return getNode(Opcode: ISD::SUB, DL, VT, N1: getConstant(Val: 0, DL, VT), N2: Val);
1660}
1661
1662/// getNOT - Create a bitwise NOT operation as (XOR Val, -1).
1663SDValue SelectionDAG::getNOT(const SDLoc &DL, SDValue Val, EVT VT) {
1664 return getNode(Opcode: ISD::XOR, DL, VT, N1: Val, N2: getAllOnesConstant(DL, VT));
1665}
1666
1667SDValue SelectionDAG::getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT) {
1668 SDValue TrueValue = getBoolConstant(V: true, DL, VT, OpVT: VT);
1669 return getNode(Opcode: ISD::XOR, DL, VT, N1: Val, N2: TrueValue);
1670}
1671
1672SDValue SelectionDAG::getBoolConstant(bool V, const SDLoc &DL, EVT VT,
1673 EVT OpVT) {
1674 if (!V)
1675 return getConstant(Val: 0, DL, VT);
1676
1677 switch (TLI->getBooleanContents(Type: OpVT)) {
1678 case TargetLowering::ZeroOrOneBooleanContent:
1679 case TargetLowering::UndefinedBooleanContent:
1680 return getConstant(Val: 1, DL, VT);
1681 case TargetLowering::ZeroOrNegativeOneBooleanContent:
1682 return getAllOnesConstant(DL, VT);
1683 }
1684 llvm_unreachable("Unexpected boolean content enum!");
1685}
1686
1687SDValue SelectionDAG::getConstant(uint64_t Val, const SDLoc &DL, EVT VT,
1688 bool isT, bool isO) {
1689 return getConstant(Val: APInt(VT.getScalarSizeInBits(), Val, /*isSigned=*/false),
1690 DL, VT, isTarget: isT, isOpaque: isO);
1691}
1692
1693SDValue SelectionDAG::getConstant(const APInt &Val, const SDLoc &DL, EVT VT,
1694 bool isT, bool isO) {
1695 return getConstant(Val: *ConstantInt::get(Context&: *Context, V: Val), DL, VT, isTarget: isT, isOpaque: isO);
1696}
1697
1698SDValue SelectionDAG::getConstant(const ConstantInt &Val, const SDLoc &DL,
1699 EVT VT, bool isT, bool isO) {
1700 assert(VT.isInteger() && "Cannot create FP integer constant!");
1701
1702 EVT EltVT = VT.getScalarType();
1703 const ConstantInt *Elt = &Val;
1704
1705 // Vector splats are explicit within the DAG, with ConstantSDNode holding the
1706 // to-be-splatted scalar ConstantInt.
1707 if (isa<VectorType>(Val: Elt->getType()))
1708 Elt = ConstantInt::get(Context&: *getContext(), V: Elt->getValue());
1709
1710 // In some cases the vector type is legal but the element type is illegal and
1711 // needs to be promoted, for example v8i8 on ARM. In this case, promote the
1712 // inserted value (the type does not need to match the vector element type).
1713 // Any extra bits introduced will be truncated away.
1714 if (VT.isVector() && TLI->getTypeAction(Context&: *getContext(), VT: EltVT) ==
1715 TargetLowering::TypePromoteInteger) {
1716 EltVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT: EltVT);
1717 APInt NewVal;
1718 if (TLI->isSExtCheaperThanZExt(FromTy: VT.getScalarType(), ToTy: EltVT))
1719 NewVal = Elt->getValue().sextOrTrunc(width: EltVT.getSizeInBits());
1720 else
1721 NewVal = Elt->getValue().zextOrTrunc(width: EltVT.getSizeInBits());
1722 Elt = ConstantInt::get(Context&: *getContext(), V: NewVal);
1723 }
1724 // In other cases the element type is illegal and needs to be expanded, for
1725 // example v2i64 on MIPS32. In this case, find the nearest legal type, split
1726 // the value into n parts and use a vector type with n-times the elements.
1727 // Then bitcast to the type requested.
1728 // Legalizing constants too early makes the DAGCombiner's job harder so we
1729 // only legalize if the DAG tells us we must produce legal types.
1730 else if (NewNodesMustHaveLegalTypes && VT.isVector() &&
1731 TLI->getTypeAction(Context&: *getContext(), VT: EltVT) ==
1732 TargetLowering::TypeExpandInteger) {
1733 const APInt &NewVal = Elt->getValue();
1734 EVT ViaEltVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT: EltVT);
1735 unsigned ViaEltSizeInBits = ViaEltVT.getSizeInBits();
1736
1737 // For scalable vectors, try to use a SPLAT_VECTOR_PARTS node.
1738 if (VT.isScalableVector() ||
1739 TLI->isOperationLegal(Op: ISD::SPLAT_VECTOR, VT)) {
1740 assert(EltVT.getSizeInBits() % ViaEltSizeInBits == 0 &&
1741 "Can only handle an even split!");
1742 unsigned Parts = EltVT.getSizeInBits() / ViaEltSizeInBits;
1743
1744 SmallVector<SDValue, 2> ScalarParts;
1745 for (unsigned i = 0; i != Parts; ++i)
1746 ScalarParts.push_back(Elt: getConstant(
1747 Val: NewVal.extractBits(numBits: ViaEltSizeInBits, bitPosition: i * ViaEltSizeInBits), DL,
1748 VT: ViaEltVT, isT, isO));
1749
1750 return getNode(Opcode: ISD::SPLAT_VECTOR_PARTS, DL, VT, Ops: ScalarParts);
1751 }
1752
1753 unsigned ViaVecNumElts = VT.getSizeInBits() / ViaEltSizeInBits;
1754 EVT ViaVecVT = EVT::getVectorVT(Context&: *getContext(), VT: ViaEltVT, NumElements: ViaVecNumElts);
1755
1756 // Check the temporary vector is the correct size. If this fails then
1757 // getTypeToTransformTo() probably returned a type whose size (in bits)
1758 // isn't a power-of-2 factor of the requested type size.
1759 assert(ViaVecVT.getSizeInBits() == VT.getSizeInBits());
1760
1761 SmallVector<SDValue, 2> EltParts;
1762 for (unsigned i = 0; i < ViaVecNumElts / VT.getVectorNumElements(); ++i)
1763 EltParts.push_back(Elt: getConstant(
1764 Val: NewVal.extractBits(numBits: ViaEltSizeInBits, bitPosition: i * ViaEltSizeInBits), DL,
1765 VT: ViaEltVT, isT, isO));
1766
1767 // EltParts is currently in little endian order. If we actually want
1768 // big-endian order then reverse it now.
1769 if (getDataLayout().isBigEndian())
1770 std::reverse(first: EltParts.begin(), last: EltParts.end());
1771
1772 // The elements must be reversed when the element order is different
1773 // to the endianness of the elements (because the BITCAST is itself a
1774 // vector shuffle in this situation). However, we do not need any code to
1775 // perform this reversal because getConstant() is producing a vector
1776 // splat.
1777 // This situation occurs in MIPS MSA.
1778
1779 SmallVector<SDValue, 8> Ops;
1780 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i)
1781 llvm::append_range(C&: Ops, R&: EltParts);
1782
1783 SDValue V =
1784 getNode(Opcode: ISD::BITCAST, DL, VT, Operand: getBuildVector(VT: ViaVecVT, DL, Ops));
1785 return V;
1786 }
1787
1788 assert(Elt->getBitWidth() == EltVT.getSizeInBits() &&
1789 "APInt size does not match type size!");
1790 unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant;
1791 SDVTList VTs = getVTList(VT: EltVT);
1792 SDNodeKey ID(Opc, VTs, {});
1793 ID.AddPointer(P: Elt);
1794 ID.AddBoolean(B: isO);
1795 FoldingSetInsertToken InsertToken;
1796 SDNode *N = nullptr;
1797 if ((N = lookupNode(Key: ID, DL, InsertToken)))
1798 if (!VT.isVector())
1799 return SDValue(N, 0);
1800
1801 if (!N) {
1802 N = newSDNode<ConstantSDNode>(Args&: isT, Args&: isO, Args&: Elt, Args&: VTs);
1803 if (!isT)
1804 N->setDebugLoc(DL.getDebugLoc());
1805 CSEMap.insert(N, Token: InsertToken);
1806 InsertNode(N);
1807 NewSDValueDbgMsg(V: SDValue(N, 0), Msg: "Creating constant: ", G: this);
1808 }
1809
1810 SDValue Result(N, 0);
1811 if (VT.isVector())
1812 Result = getSplat(VT, DL, Op: Result);
1813 return Result;
1814}
1815
1816SDValue SelectionDAG::getSignedConstant(int64_t Val, const SDLoc &DL, EVT VT,
1817 bool isT, bool isO) {
1818 unsigned Size = VT.getScalarSizeInBits();
1819 return getConstant(Val: APInt(Size, Val, /*isSigned=*/true), DL, VT, isT, isO);
1820}
1821
1822SDValue SelectionDAG::getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget,
1823 bool IsOpaque) {
1824 return getConstant(Val: APInt::getAllOnes(numBits: VT.getScalarSizeInBits()), DL, VT,
1825 isT: IsTarget, isO: IsOpaque);
1826}
1827
1828SDValue SelectionDAG::getIntPtrConstant(uint64_t Val, const SDLoc &DL,
1829 bool isTarget) {
1830 return getConstant(Val, DL, VT: TLI->getPointerTy(DL: getDataLayout()), isT: isTarget);
1831}
1832
1833SDValue SelectionDAG::getShiftAmountConstant(uint64_t Val, EVT VT,
1834 const SDLoc &DL) {
1835 assert(VT.isInteger() && "Shift amount is not an integer type!");
1836 EVT ShiftVT = TLI->getShiftAmountTy(LHSTy: VT, DL: getDataLayout());
1837 return getConstant(Val, DL, VT: ShiftVT);
1838}
1839
1840SDValue SelectionDAG::getShiftAmountConstant(const APInt &Val, EVT VT,
1841 const SDLoc &DL) {
1842 assert(Val.ult(VT.getScalarSizeInBits()) && "Out of range shift");
1843 return getShiftAmountConstant(Val: Val.getZExtValue(), VT, DL);
1844}
1845
1846SDValue SelectionDAG::getVectorIdxConstant(uint64_t Val, const SDLoc &DL,
1847 bool isTarget) {
1848 return getConstant(Val, DL, VT: TLI->getVectorIdxTy(DL: getDataLayout()), isT: isTarget);
1849}
1850
1851SDValue SelectionDAG::getConstantFP(const APFloat &V, const SDLoc &DL, EVT VT,
1852 bool isTarget) {
1853 return getConstantFP(V: *ConstantFP::get(Context&: *getContext(), V), DL, VT, isTarget);
1854}
1855
1856SDValue SelectionDAG::getConstantFP(const ConstantFP &V, const SDLoc &DL,
1857 EVT VT, bool isTarget) {
1858 assert(VT.isFloatingPoint() && "Cannot create integer FP constant!");
1859
1860 EVT EltVT = VT.getScalarType();
1861 const ConstantFP *Elt = &V;
1862
1863 // Vector splats are explicit within the DAG, with ConstantFPSDNode holding
1864 // the to-be-splatted scalar ConstantFP.
1865 if (isa<VectorType>(Val: Elt->getType()))
1866 Elt = ConstantFP::get(Context&: *getContext(), V: Elt->getValue());
1867
1868 // Do the map lookup using the actual bit pattern for the floating point
1869 // value, so that we don't have problems with 0.0 comparing equal to -0.0, and
1870 // we don't have issues with SNANs.
1871 unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP;
1872 SDVTList VTs = getVTList(VT: EltVT);
1873 SDNodeKey ID(Opc, VTs, {});
1874 ID.AddPointer(P: Elt);
1875 FoldingSetInsertToken InsertToken;
1876 SDNode *N = nullptr;
1877 if ((N = lookupNode(Key: ID, DL, InsertToken)))
1878 if (!VT.isVector())
1879 return SDValue(N, 0);
1880
1881 if (!N) {
1882 N = newSDNode<ConstantFPSDNode>(Args&: isTarget, Args&: Elt, Args&: VTs);
1883 CSEMap.insert(N, Token: InsertToken);
1884 InsertNode(N);
1885 }
1886
1887 SDValue Result(N, 0);
1888 if (VT.isVector())
1889 Result = getSplat(VT, DL, Op: Result);
1890 NewSDValueDbgMsg(V: Result, Msg: "Creating fp constant: ", G: this);
1891 return Result;
1892}
1893
1894SDValue SelectionDAG::getConstantFP(double Val, const SDLoc &DL, EVT VT,
1895 bool isTarget) {
1896 EVT EltVT = VT.getScalarType();
1897 if (EltVT == MVT::f32)
1898 return getConstantFP(V: APFloat((float)Val), DL, VT, isTarget);
1899 if (EltVT == MVT::f64)
1900 return getConstantFP(V: APFloat(Val), DL, VT, isTarget);
1901 if (EltVT == MVT::f80 || EltVT == MVT::f128 || EltVT == MVT::ppcf128 ||
1902 EltVT == MVT::f16 || EltVT == MVT::bf16) {
1903 bool Ignored;
1904 APFloat APF = APFloat(Val);
1905 APF.convert(ToSemantics: EltVT.getFltSemantics(), RM: APFloat::rmNearestTiesToEven,
1906 losesInfo: &Ignored);
1907 return getConstantFP(V: APF, DL, VT, isTarget);
1908 }
1909 llvm_unreachable("Unsupported type in getConstantFP");
1910}
1911
1912SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, const SDLoc &DL,
1913 EVT VT, int64_t Offset, bool isTargetGA,
1914 unsigned TargetFlags) {
1915 assert((TargetFlags == 0 || isTargetGA) &&
1916 "Cannot set target flags on target-independent globals");
1917
1918 // Truncate (with sign-extension) the offset value to the pointer size.
1919 unsigned BitWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType());
1920 if (BitWidth < 64)
1921 Offset = SignExtend64(X: Offset, B: BitWidth);
1922
1923 unsigned Opc;
1924 if (GV->isThreadLocal())
1925 Opc = isTargetGA ? ISD::TargetGlobalTLSAddress : ISD::GlobalTLSAddress;
1926 else
1927 Opc = isTargetGA ? ISD::TargetGlobalAddress : ISD::GlobalAddress;
1928
1929 SDVTList VTs = getVTList(VT);
1930 SDNodeKey ID(Opc, VTs, {});
1931 ID.AddPointer(P: GV);
1932 ID.AddInteger(I: Offset);
1933 ID.AddInteger(I: TargetFlags);
1934 FoldingSetInsertToken InsertToken;
1935 if (SDNode *E = lookupNode(Key: ID, DL, InsertToken))
1936 return SDValue(E, 0);
1937
1938 auto *N = newSDNode<GlobalAddressSDNode>(
1939 Args&: Opc, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: GV, Args&: VTs, Args&: Offset, Args&: TargetFlags);
1940 CSEMap.insert(N, Token: InsertToken);
1941 InsertNode(N);
1942 return SDValue(N, 0);
1943}
1944
1945SDValue SelectionDAG::getDeactivationSymbol(const GlobalValue *GV) {
1946 SDVTList VTs = getVTList(VT: MVT::Untyped);
1947 SDNodeKey ID(ISD::DEACTIVATION_SYMBOL, VTs, {});
1948 ID.AddPointer(P: GV);
1949 FoldingSetInsertToken InsertToken;
1950 if (SDNode *E = lookupNode(Key: ID, DL: SDLoc(), InsertToken))
1951 return SDValue(E, 0);
1952
1953 auto *N = newSDNode<DeactivationSymbolSDNode>(Args&: GV, Args&: VTs);
1954 CSEMap.insert(N, Token: InsertToken);
1955 InsertNode(N);
1956 return SDValue(N, 0);
1957}
1958
1959SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) {
1960 unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex;
1961 SDVTList VTs = getVTList(VT);
1962 SDNodeKey ID(Opc, VTs, {});
1963 ID.AddInteger(I: FI);
1964 FoldingSetInsertToken InsertToken;
1965 if (SDNode *E = lookupNode(Key: ID, InsertToken))
1966 return SDValue(E, 0);
1967
1968 auto *N = newSDNode<FrameIndexSDNode>(Args&: FI, Args&: VTs, Args&: isTarget);
1969 CSEMap.insert(N, Token: InsertToken);
1970 InsertNode(N);
1971 return SDValue(N, 0);
1972}
1973
1974SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget,
1975 unsigned TargetFlags) {
1976 assert((TargetFlags == 0 || isTarget) &&
1977 "Cannot set target flags on target-independent jump tables");
1978 unsigned Opc = isTarget ? ISD::TargetJumpTable : ISD::JumpTable;
1979 SDVTList VTs = getVTList(VT);
1980 SDNodeKey ID(Opc, VTs, {});
1981 ID.AddInteger(I: JTI);
1982 ID.AddInteger(I: TargetFlags);
1983 FoldingSetInsertToken InsertToken;
1984 if (SDNode *E = lookupNode(Key: ID, InsertToken))
1985 return SDValue(E, 0);
1986
1987 auto *N = newSDNode<JumpTableSDNode>(Args&: JTI, Args&: VTs, Args&: isTarget, Args&: TargetFlags);
1988 CSEMap.insert(N, Token: InsertToken);
1989 InsertNode(N);
1990 return SDValue(N, 0);
1991}
1992
1993SDValue SelectionDAG::getJumpTableDebugInfo(int JTI, SDValue Chain,
1994 const SDLoc &DL) {
1995 EVT PTy = getTargetLoweringInfo().getPointerTy(DL: getDataLayout());
1996 return getNode(Opcode: ISD::JUMP_TABLE_DEBUG_INFO, DL, VT: MVT::Other, N1: Chain,
1997 N2: getTargetConstant(Val: static_cast<uint64_t>(JTI), DL, VT: PTy, isOpaque: true));
1998}
1999
2000SDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT,
2001 MaybeAlign Alignment, int Offset,
2002 bool isTarget, unsigned TargetFlags) {
2003 assert((TargetFlags == 0 || isTarget) &&
2004 "Cannot set target flags on target-independent globals");
2005 if (!Alignment)
2006 Alignment = shouldOptForSize()
2007 ? getDataLayout().getABITypeAlign(Ty: C->getType())
2008 : getDataLayout().getPrefTypeAlign(Ty: C->getType());
2009 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
2010 SDVTList VTs = getVTList(VT);
2011 SDNodeKey ID(Opc, VTs, {});
2012 ID.AddInteger(I: Alignment->value());
2013 ID.AddInteger(I: Offset);
2014 ID.AddPointer(P: C);
2015 ID.AddInteger(I: TargetFlags);
2016 FoldingSetInsertToken InsertToken;
2017 if (SDNode *E = lookupNode(Key: ID, InsertToken))
2018 return SDValue(E, 0);
2019
2020 auto *N = newSDNode<ConstantPoolSDNode>(Args&: isTarget, Args&: C, Args&: VTs, Args&: Offset, Args&: *Alignment,
2021 Args&: TargetFlags);
2022 CSEMap.insert(N, Token: InsertToken);
2023 InsertNode(N);
2024 SDValue V = SDValue(N, 0);
2025 NewSDValueDbgMsg(V, Msg: "Creating new constant pool: ", G: this);
2026 return V;
2027}
2028
2029SDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT,
2030 MaybeAlign Alignment, int Offset,
2031 bool isTarget, unsigned TargetFlags) {
2032 assert((TargetFlags == 0 || isTarget) &&
2033 "Cannot set target flags on target-independent globals");
2034 if (!Alignment)
2035 Alignment = getDataLayout().getPrefTypeAlign(Ty: C->getType());
2036 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
2037 SDVTList VTs = getVTList(VT);
2038 SDNodeKey ID(Opc, VTs, {});
2039 ID.AddInteger(I: Alignment->value());
2040 ID.AddInteger(I: Offset);
2041 C->addSelectionDAGCSEId(ID&: ID.Tail);
2042 ID.AddInteger(I: TargetFlags);
2043 FoldingSetInsertToken InsertToken;
2044 if (SDNode *E = lookupNode(Key: ID, InsertToken))
2045 return SDValue(E, 0);
2046
2047 auto *N = newSDNode<ConstantPoolSDNode>(Args&: isTarget, Args&: C, Args&: VTs, Args&: Offset, Args&: *Alignment,
2048 Args&: TargetFlags);
2049 CSEMap.insert(N, Token: InsertToken);
2050 InsertNode(N);
2051 return SDValue(N, 0);
2052}
2053
2054SDValue SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) {
2055 SDNodeKey ID(ISD::BasicBlock, getVTList(VT: MVT::Other), {});
2056 ID.AddPointer(P: MBB);
2057 FoldingSetInsertToken InsertToken;
2058 if (SDNode *E = lookupNode(Key: ID, InsertToken))
2059 return SDValue(E, 0);
2060
2061 auto *N = newSDNode<BasicBlockSDNode>(Args&: MBB);
2062 CSEMap.insert(N, Token: InsertToken);
2063 InsertNode(N);
2064 return SDValue(N, 0);
2065}
2066
2067SDValue SelectionDAG::getValueType(EVT VT) {
2068 if (VT.isSimple() && (unsigned)VT.getSimpleVT().SimpleTy >=
2069 ValueTypeNodes.size())
2070 ValueTypeNodes.resize(new_size: VT.getSimpleVT().SimpleTy+1);
2071
2072 SDNode *&N = VT.isExtended() ?
2073 ExtendedValueTypeNodes[VT] : ValueTypeNodes[VT.getSimpleVT().SimpleTy];
2074
2075 if (N) return SDValue(N, 0);
2076 N = newSDNode<VTSDNode>(Args&: VT);
2077 InsertNode(N);
2078 return SDValue(N, 0);
2079}
2080
2081SDValue SelectionDAG::getExternalSymbol(const char *Sym, EVT VT) {
2082 SDNode *&N = ExternalSymbols[Sym];
2083 if (N) return SDValue(N, 0);
2084 N = newSDNode<ExternalSymbolSDNode>(Args: false, Args&: Sym, Args: 0, Args: getVTList(VT));
2085 InsertNode(N);
2086 return SDValue(N, 0);
2087}
2088
2089SDValue SelectionDAG::getExternalSymbol(RTLIB::LibcallImpl Libcall, EVT VT) {
2090 StringRef SymName = RTLIB::RuntimeLibcallsInfo::getLibcallImplName(CallImpl: Libcall);
2091 return getExternalSymbol(Sym: SymName.data(), VT);
2092}
2093
2094SDValue SelectionDAG::getMCSymbol(MCSymbol *Sym, EVT VT) {
2095 SDNode *&N = MCSymbols[Sym];
2096 if (N)
2097 return SDValue(N, 0);
2098 N = newSDNode<MCSymbolSDNode>(Args&: Sym, Args: getVTList(VT));
2099 InsertNode(N);
2100 return SDValue(N, 0);
2101}
2102
2103SDValue SelectionDAG::getTargetExternalSymbol(const char *Sym, EVT VT,
2104 unsigned TargetFlags) {
2105 SDNode *&N =
2106 TargetExternalSymbols[std::pair<std::string, unsigned>(Sym, TargetFlags)];
2107 if (N) return SDValue(N, 0);
2108 N = newSDNode<ExternalSymbolSDNode>(Args: true, Args&: Sym, Args&: TargetFlags, Args: getVTList(VT));
2109 InsertNode(N);
2110 return SDValue(N, 0);
2111}
2112
2113SDValue SelectionDAG::getTargetExternalSymbol(RTLIB::LibcallImpl Libcall,
2114 EVT VT, unsigned TargetFlags) {
2115 StringRef SymName = RTLIB::RuntimeLibcallsInfo::getLibcallImplName(CallImpl: Libcall);
2116 return getTargetExternalSymbol(Sym: SymName.data(), VT, TargetFlags);
2117}
2118
2119SDValue SelectionDAG::getCondCode(ISD::CondCode Cond) {
2120 if ((unsigned)Cond >= CondCodeNodes.size())
2121 CondCodeNodes.resize(new_size: Cond+1);
2122
2123 if (!CondCodeNodes[Cond]) {
2124 auto *N = newSDNode<CondCodeSDNode>(Args&: Cond);
2125 CondCodeNodes[Cond] = N;
2126 InsertNode(N);
2127 }
2128
2129 return SDValue(CondCodeNodes[Cond], 0);
2130}
2131
2132SDValue SelectionDAG::getVScale(const SDLoc &DL, EVT VT, APInt MulImm) {
2133 assert(MulImm.getBitWidth() == VT.getSizeInBits() &&
2134 "APInt size does not match type size!");
2135
2136 if (MulImm == 0)
2137 return getConstant(Val: 0, DL, VT);
2138
2139 const MachineFunction &MF = getMachineFunction();
2140 const Function &F = MF.getFunction();
2141 ConstantRange CR = getVScaleRange(F: &F, BitWidth: 64);
2142 if (const APInt *C = CR.getSingleElement())
2143 return getConstant(Val: MulImm * C->getZExtValue(), DL, VT);
2144
2145 return getNode(Opcode: ISD::VSCALE, DL, VT, Operand: getConstant(Val: MulImm, DL, VT));
2146}
2147
2148/// \returns a value of type \p VT that represents the runtime value of \p
2149/// Quantity, i.e. scaled by vscale if it's scalable, or a fixed constant
2150/// otherwise. Quantity should be a FixedOrScalableQuantity, i.e. ElementCount
2151/// or TypeSize.
2152template <typename Ty>
2153static SDValue getFixedOrScalableQuantity(SelectionDAG &DAG, const SDLoc &DL,
2154 EVT VT, Ty Quantity) {
2155 if (Quantity.isScalable())
2156 return DAG.getVScale(
2157 DL, VT, MulImm: APInt(VT.getSizeInBits(), Quantity.getKnownMinValue()));
2158
2159 return DAG.getConstant(Quantity.getKnownMinValue(), DL, VT);
2160}
2161
2162SDValue SelectionDAG::getElementCount(const SDLoc &DL, EVT VT,
2163 ElementCount EC) {
2164 return getFixedOrScalableQuantity(DAG&: *this, DL, VT, Quantity: EC);
2165}
2166
2167SDValue SelectionDAG::getTypeSize(const SDLoc &DL, EVT VT, TypeSize TS) {
2168 return getFixedOrScalableQuantity(DAG&: *this, DL, VT, Quantity: TS);
2169}
2170
2171SDValue SelectionDAG::getMaskFromElementCount(const SDLoc &DL, EVT DataVT,
2172 ElementCount EC) {
2173 EVT IdxVT = TLI->getVectorIdxTy(DL: getDataLayout());
2174 EVT MaskVT = TLI->getSetCCResultType(DL: getDataLayout(), Context&: *getContext(), VT: DataVT);
2175 return getNode(Opcode: ISD::GET_ACTIVE_LANE_MASK, DL, VT: MaskVT,
2176 N1: getConstant(Val: 0, DL, VT: IdxVT), N2: getElementCount(DL, VT: IdxVT, EC));
2177}
2178
2179SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT) {
2180 APInt One(ResVT.getScalarSizeInBits(), 1);
2181 return getStepVector(DL, ResVT, StepVal: One);
2182}
2183
2184SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT,
2185 const APInt &StepVal) {
2186 assert(ResVT.getScalarSizeInBits() == StepVal.getBitWidth());
2187 if (ResVT.isScalableVector())
2188 return getNode(
2189 Opcode: ISD::STEP_VECTOR, DL, VT: ResVT,
2190 Operand: getTargetConstant(Val: StepVal, DL, VT: ResVT.getVectorElementType()));
2191
2192 SmallVector<SDValue, 16> OpsStepConstants;
2193 for (uint64_t i = 0; i < ResVT.getVectorNumElements(); i++)
2194 OpsStepConstants.push_back(
2195 Elt: getConstant(Val: StepVal * i, DL, VT: ResVT.getVectorElementType()));
2196 return getBuildVector(VT: ResVT, DL, Ops: OpsStepConstants);
2197}
2198
2199/// Swaps the values of N1 and N2. Swaps all indices in the shuffle mask M that
2200/// point at N1 to point at N2 and indices that point at N2 to point at N1.
2201static void commuteShuffle(SDValue &N1, SDValue &N2, MutableArrayRef<int> M) {
2202 std::swap(a&: N1, b&: N2);
2203 ShuffleVectorSDNode::commuteMask(Mask: M);
2204}
2205
2206SDValue SelectionDAG::getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1,
2207 SDValue N2, ArrayRef<int> Mask) {
2208 assert(VT.getVectorNumElements() == Mask.size() &&
2209 "Must have the same number of vector elements as mask elements!");
2210 assert(VT == N1.getValueType() && VT == N2.getValueType() &&
2211 "Invalid VECTOR_SHUFFLE");
2212
2213 // Canonicalize shuffle undef, undef -> undef
2214 if (N1.isUndef() && N2.isUndef()) {
2215 if (N1.getOpcode() == ISD::POISON && N2.getOpcode() == ISD::POISON)
2216 return getPOISON(VT);
2217 return getUNDEF(VT);
2218 }
2219
2220 // Validate that all indices in Mask are within the range of the elements
2221 // input to the shuffle.
2222 int NElts = Mask.size();
2223 assert(llvm::all_of(Mask,
2224 [&](int M) { return M < (NElts * 2) && M >= -1; }) &&
2225 "Index out of range");
2226
2227 // Copy the mask so we can do any needed cleanup.
2228 SmallVector<int, 8> MaskVec(Mask);
2229
2230 // Canonicalize shuffle v, v -> v, poison
2231 if (N1 == N2) {
2232 N2 = getPOISON(VT);
2233 for (int i = 0; i != NElts; ++i)
2234 if (MaskVec[i] >= NElts) MaskVec[i] -= NElts;
2235 }
2236
2237 // Canonicalize shuffle undef, v -> v, undef. Commute the shuffle mask.
2238 if (N1.isUndef())
2239 commuteShuffle(N1, N2, M: MaskVec);
2240
2241 if (TLI->hasVectorBlend()) {
2242 // If shuffling a splat, try to blend the splat instead. We do this here so
2243 // that even when this arises during lowering we don't have to re-handle it.
2244 auto BlendSplat = [&](BuildVectorSDNode *BV, int Offset) {
2245 BitVector UndefElements;
2246 SDValue Splat = BV->getSplatValue(UndefElements: &UndefElements);
2247 if (!Splat)
2248 return;
2249
2250 for (int i = 0; i < NElts; ++i) {
2251 if (MaskVec[i] < Offset || MaskVec[i] >= (Offset + NElts))
2252 continue;
2253
2254 // If this input comes from undef, mark it as such.
2255 if (UndefElements[MaskVec[i] - Offset]) {
2256 MaskVec[i] = -1;
2257 continue;
2258 }
2259
2260 // If we can blend a non-undef lane, use that instead.
2261 if (!UndefElements[i])
2262 MaskVec[i] = i + Offset;
2263 }
2264 };
2265 if (auto *N1BV = dyn_cast<BuildVectorSDNode>(Val&: N1))
2266 BlendSplat(N1BV, 0);
2267 if (auto *N2BV = dyn_cast<BuildVectorSDNode>(Val&: N2))
2268 BlendSplat(N2BV, NElts);
2269 }
2270
2271 // Canonicalize all index into lhs, -> shuffle lhs, poison
2272 // Canonicalize all index into rhs, -> shuffle rhs, poison
2273 bool AllLHS = true, AllRHS = true;
2274 bool N2Undef = N2.isUndef();
2275 for (int i = 0; i != NElts; ++i) {
2276 if (MaskVec[i] >= NElts) {
2277 if (N2Undef)
2278 MaskVec[i] = -1;
2279 else
2280 AllLHS = false;
2281 } else if (MaskVec[i] >= 0) {
2282 AllRHS = false;
2283 }
2284 }
2285 if (AllLHS && AllRHS)
2286 return getPOISON(VT);
2287 if (AllLHS && !N2Undef)
2288 N2 = getPOISON(VT);
2289 if (AllRHS) {
2290 N1 = getPOISON(VT);
2291 commuteShuffle(N1, N2, M: MaskVec);
2292 }
2293 // Reset our undef status after accounting for the mask.
2294 N2Undef = N2.isUndef();
2295 // Re-check whether both sides ended up undef.
2296 if (N1.isUndef() && N2Undef) {
2297 if (N1.getOpcode() == ISD::POISON && N2.getOpcode() == ISD::POISON)
2298 return getPOISON(VT);
2299 return getUNDEF(VT);
2300 }
2301
2302 // If Identity shuffle return that node.
2303 bool Identity = true, AllSame = true;
2304 for (int i = 0; i != NElts; ++i) {
2305 if (MaskVec[i] >= 0 && MaskVec[i] != i) Identity = false;
2306 if (MaskVec[i] != MaskVec[0]) AllSame = false;
2307 }
2308 if (Identity && NElts)
2309 return N1;
2310
2311 // Shuffling a constant splat doesn't change the result.
2312 if (N2Undef) {
2313 SDValue V = N1;
2314
2315 // Look through any bitcasts. We check that these don't change the number
2316 // (and size) of elements and just changes their types.
2317 while (V.getOpcode() == ISD::BITCAST)
2318 V = V->getOperand(Num: 0);
2319
2320 // A splat should always show up as a build vector node.
2321 if (auto *BV = dyn_cast<BuildVectorSDNode>(Val&: V)) {
2322 BitVector UndefElements;
2323 SDValue Splat = BV->getSplatValue(UndefElements: &UndefElements);
2324 // If this is a splat of an undef, shuffling it is also undef.
2325 if (Splat && Splat.isUndef())
2326 return Splat.getOpcode() == ISD::POISON ? getPOISON(VT) : getUNDEF(VT);
2327
2328 bool SameNumElts =
2329 V.getValueType().getVectorNumElements() == VT.getVectorNumElements();
2330
2331 // We only have a splat which can skip shuffles if there is a splatted
2332 // value and no undef lanes rearranged by the shuffle.
2333 if (Splat && UndefElements.none()) {
2334 // Splat of <x, x, ..., x>, return <x, x, ..., x>, provided that the
2335 // number of elements match or the value splatted is a zero constant.
2336 if (SameNumElts || isNullConstant(V: Splat))
2337 return N1;
2338 }
2339
2340 // If the shuffle itself creates a splat, build the vector directly.
2341 if (AllSame && SameNumElts) {
2342 EVT BuildVT = BV->getValueType(ResNo: 0);
2343 const SDValue &Splatted = BV->getOperand(Num: MaskVec[0]);
2344 SDValue NewBV = getSplatBuildVector(VT: BuildVT, DL: dl, Op: Splatted);
2345
2346 // We may have jumped through bitcasts, so the type of the
2347 // BUILD_VECTOR may not match the type of the shuffle.
2348 if (BuildVT != VT)
2349 NewBV = getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: NewBV);
2350 return NewBV;
2351 }
2352 }
2353 }
2354
2355 SDVTList VTs = getVTList(VT);
2356 SDValue Ops[2] = { N1, N2 };
2357 SDNodeKey ID(ISD::VECTOR_SHUFFLE, VTs, Ops);
2358 for (int i = 0; i != NElts; ++i)
2359 ID.AddInteger(I: MaskVec[i]);
2360
2361 FoldingSetInsertToken InsertToken;
2362 if (SDNode *E = lookupNode(Key: ID, DL: dl, InsertToken))
2363 return SDValue(E, 0);
2364
2365 // Allocate the mask array for the node out of the BumpPtrAllocator, since
2366 // SDNode doesn't have access to it. This memory will be "leaked" when
2367 // the node is deallocated, but recovered when the NodeAllocator is released.
2368 int *MaskAlloc = OperandAllocator.Allocate<int>(Num: NElts);
2369 llvm::copy(Range&: MaskVec, Out: MaskAlloc);
2370
2371 auto *N = newSDNode<ShuffleVectorSDNode>(Args&: VTs, Args: dl.getIROrder(),
2372 Args: dl.getDebugLoc(), Args&: MaskAlloc);
2373 createOperands(Node: N, Vals: Ops);
2374
2375 CSEMap.insert(N, Token: InsertToken);
2376 InsertNode(N);
2377 SDValue V = SDValue(N, 0);
2378 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
2379 return V;
2380}
2381
2382SDValue SelectionDAG::getCommutedVectorShuffle(const ShuffleVectorSDNode &SV) {
2383 EVT VT = SV.getValueType(ResNo: 0);
2384 SmallVector<int, 8> MaskVec(SV.getMask());
2385 ShuffleVectorSDNode::commuteMask(Mask: MaskVec);
2386
2387 SDValue Op0 = SV.getOperand(Num: 0);
2388 SDValue Op1 = SV.getOperand(Num: 1);
2389 return getVectorShuffle(VT, dl: SDLoc(&SV), N1: Op1, N2: Op0, Mask: MaskVec);
2390}
2391
2392SDValue SelectionDAG::getRegister(Register Reg, EVT VT) {
2393 SDVTList VTs = getVTList(VT);
2394 SDNodeKey ID(ISD::Register, VTs, {});
2395 ID.AddInteger(I: Reg.id());
2396 FoldingSetInsertToken InsertToken;
2397 if (SDNode *E = lookupNode(Key: ID, InsertToken))
2398 return SDValue(E, 0);
2399
2400 auto *N = newSDNode<RegisterSDNode>(Args&: Reg, Args&: VTs);
2401 N->SDNodeBits.IsDivergent = TLI->isSDNodeSourceOfDivergence(N, FLI, UA);
2402 CSEMap.insert(N, Token: InsertToken);
2403 InsertNode(N);
2404 return SDValue(N, 0);
2405}
2406
2407SDValue SelectionDAG::getRegisterMask(const uint32_t *RegMask) {
2408 SDNodeKey ID(ISD::RegisterMask, getVTList(VT: MVT::Untyped), {});
2409 ID.AddPointer(P: RegMask);
2410 FoldingSetInsertToken InsertToken;
2411 if (SDNode *E = lookupNode(Key: ID, InsertToken))
2412 return SDValue(E, 0);
2413
2414 auto *N = newSDNode<RegisterMaskSDNode>(Args&: RegMask);
2415 CSEMap.insert(N, Token: InsertToken);
2416 InsertNode(N);
2417 return SDValue(N, 0);
2418}
2419
2420SDValue SelectionDAG::getEHLabel(const SDLoc &dl, SDValue Root,
2421 MCSymbol *Label) {
2422 return getLabelNode(Opcode: ISD::EH_LABEL, dl, Root, Label);
2423}
2424
2425SDValue SelectionDAG::getLabelNode(unsigned Opcode, const SDLoc &dl,
2426 SDValue Root, MCSymbol *Label) {
2427 SDValue Ops[] = { Root };
2428 SDNodeKey ID(Opcode, getVTList(VT: MVT::Other), Ops);
2429 ID.AddPointer(P: Label);
2430 FoldingSetInsertToken InsertToken;
2431 if (SDNode *E = lookupNode(Key: ID, InsertToken))
2432 return SDValue(E, 0);
2433
2434 auto *N =
2435 newSDNode<LabelSDNode>(Args&: Opcode, Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: Label);
2436 createOperands(Node: N, Vals: Ops);
2437
2438 CSEMap.insert(N, Token: InsertToken);
2439 InsertNode(N);
2440 return SDValue(N, 0);
2441}
2442
2443SDValue SelectionDAG::getBlockAddress(const BlockAddress *BA, EVT VT,
2444 int64_t Offset, bool isTarget,
2445 unsigned TargetFlags) {
2446 unsigned Opc = isTarget ? ISD::TargetBlockAddress : ISD::BlockAddress;
2447 SDVTList VTs = getVTList(VT);
2448
2449 SDNodeKey ID(Opc, VTs, {});
2450 ID.AddPointer(P: BA);
2451 ID.AddInteger(I: Offset);
2452 ID.AddInteger(I: TargetFlags);
2453 FoldingSetInsertToken InsertToken;
2454 if (SDNode *E = lookupNode(Key: ID, InsertToken))
2455 return SDValue(E, 0);
2456
2457 auto *N = newSDNode<BlockAddressSDNode>(Args&: Opc, Args&: VTs, Args&: BA, Args&: Offset, Args&: TargetFlags);
2458 CSEMap.insert(N, Token: InsertToken);
2459 InsertNode(N);
2460 return SDValue(N, 0);
2461}
2462
2463SDValue SelectionDAG::getSrcValue(const Value *V) {
2464 SDNodeKey ID(ISD::SRCVALUE, getVTList(VT: MVT::Other), {});
2465 ID.AddPointer(P: V);
2466
2467 FoldingSetInsertToken InsertToken;
2468 if (SDNode *E = lookupNode(Key: ID, InsertToken))
2469 return SDValue(E, 0);
2470
2471 auto *N = newSDNode<SrcValueSDNode>(Args&: V);
2472 CSEMap.insert(N, Token: InsertToken);
2473 InsertNode(N);
2474 return SDValue(N, 0);
2475}
2476
2477SDValue SelectionDAG::getMDNode(const MDNode *MD) {
2478 SDNodeKey ID(ISD::MDNODE_SDNODE, getVTList(VT: MVT::Other), {});
2479 ID.AddPointer(P: MD);
2480
2481 FoldingSetInsertToken InsertToken;
2482 if (SDNode *E = lookupNode(Key: ID, InsertToken))
2483 return SDValue(E, 0);
2484
2485 auto *N = newSDNode<MDNodeSDNode>(Args&: MD);
2486 CSEMap.insert(N, Token: InsertToken);
2487 InsertNode(N);
2488 return SDValue(N, 0);
2489}
2490
2491SDValue SelectionDAG::getBitcast(EVT VT, SDValue V) {
2492 if (VT == V.getValueType())
2493 return V;
2494
2495 return getNode(Opcode: ISD::BITCAST, DL: SDLoc(V), VT, Operand: V);
2496}
2497
2498SDValue SelectionDAG::getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr,
2499 unsigned SrcAS, unsigned DestAS,
2500 const SDNodeFlags Flags) {
2501 SDVTList VTs = getVTList(VT);
2502 SDValue Ops[] = {Ptr};
2503 SDNodeKey ID(ISD::ADDRSPACECAST, VTs, Ops);
2504 ID.AddInteger(I: SrcAS);
2505 ID.AddInteger(I: DestAS);
2506
2507 FoldingSetInsertToken InsertToken;
2508 if (SDNode *E = lookupNode(Key: ID, DL: dl, InsertToken)) {
2509 E->intersectFlagsWith(Flags);
2510 return SDValue(E, 0);
2511 }
2512
2513 auto *N = newSDNode<AddrSpaceCastSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(),
2514 Args&: VTs, Args&: SrcAS, Args&: DestAS);
2515 N->setFlags(Flags);
2516 createOperands(Node: N, Vals: Ops);
2517
2518 CSEMap.insert(N, Token: InsertToken);
2519 InsertNode(N);
2520 return SDValue(N, 0);
2521}
2522
2523SDValue SelectionDAG::getFreeze(SDValue V) {
2524 return getNode(Opcode: ISD::FREEZE, DL: SDLoc(V), VT: V.getValueType(), Operand: V);
2525}
2526
2527SDValue SelectionDAG::getFreeze(SDValue V, const APInt &DemandedElts,
2528 UndefPoisonKind Kind) {
2529 if (isGuaranteedNotToBeUndefOrPoison(Op: V, DemandedElts, Kind))
2530 return V;
2531 return getFreeze(V);
2532}
2533
2534/// getShiftAmountOperand - Return the specified value casted to
2535/// the target's desired shift amount type.
2536SDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) {
2537 EVT OpTy = Op.getValueType();
2538 EVT ShTy = TLI->getShiftAmountTy(LHSTy, DL: getDataLayout());
2539 if (OpTy == ShTy || OpTy.isVector()) return Op;
2540
2541 return getZExtOrTrunc(Op, DL: SDLoc(Op), VT: ShTy);
2542}
2543
2544SDValue SelectionDAG::expandVAArg(SDNode *Node) {
2545 SDLoc dl(Node);
2546 const TargetLowering &TLI = getTargetLoweringInfo();
2547 const Value *V = cast<SrcValueSDNode>(Val: Node->getOperand(Num: 2))->getValue();
2548 EVT VT = Node->getValueType(ResNo: 0);
2549 SDValue Tmp1 = Node->getOperand(Num: 0);
2550 SDValue Tmp2 = Node->getOperand(Num: 1);
2551 const MaybeAlign MA(Node->getConstantOperandVal(Num: 3));
2552
2553 SDValue VAListLoad = getLoad(VT: TLI.getPointerTy(DL: getDataLayout()), dl, Chain: Tmp1,
2554 Ptr: Tmp2, PtrInfo: MachinePointerInfo(V));
2555 SDValue VAList = VAListLoad;
2556
2557 if (MA && *MA > TLI.getMinStackArgumentAlignment()) {
2558 VAList = getNode(Opcode: ISD::ADD, DL: dl, VT: VAList.getValueType(), N1: VAList,
2559 N2: getConstant(Val: MA->value() - 1, DL: dl, VT: VAList.getValueType()));
2560
2561 VAList = getNode(
2562 Opcode: ISD::AND, DL: dl, VT: VAList.getValueType(), N1: VAList,
2563 N2: getSignedConstant(Val: -(int64_t)MA->value(), DL: dl, VT: VAList.getValueType()));
2564 }
2565
2566 // Increment the pointer, VAList, to the next vaarg
2567 Tmp1 = getNode(Opcode: ISD::ADD, DL: dl, VT: VAList.getValueType(), N1: VAList,
2568 N2: getConstant(Val: getDataLayout().getTypeAllocSize(
2569 Ty: VT.getTypeForEVT(Context&: *getContext())),
2570 DL: dl, VT: VAList.getValueType()));
2571 // Store the incremented VAList to the legalized pointer
2572 Tmp1 =
2573 getStore(Chain: VAListLoad.getValue(R: 1), dl, Val: Tmp1, Ptr: Tmp2, PtrInfo: MachinePointerInfo(V));
2574 // Load the actual argument out of the pointer VAList
2575 return getLoad(VT, dl, Chain: Tmp1, Ptr: VAList, PtrInfo: MachinePointerInfo());
2576}
2577
2578SDValue SelectionDAG::expandVACopy(SDNode *Node) {
2579 SDLoc dl(Node);
2580 const TargetLowering &TLI = getTargetLoweringInfo();
2581 // This defaults to loading a pointer from the input and storing it to the
2582 // output, returning the chain.
2583 const Value *VD = cast<SrcValueSDNode>(Val: Node->getOperand(Num: 3))->getValue();
2584 const Value *VS = cast<SrcValueSDNode>(Val: Node->getOperand(Num: 4))->getValue();
2585 SDValue Tmp1 =
2586 getLoad(VT: TLI.getPointerTy(DL: getDataLayout()), dl, Chain: Node->getOperand(Num: 0),
2587 Ptr: Node->getOperand(Num: 2), PtrInfo: MachinePointerInfo(VS));
2588 return getStore(Chain: Tmp1.getValue(R: 1), dl, Val: Tmp1, Ptr: Node->getOperand(Num: 1),
2589 PtrInfo: MachinePointerInfo(VD));
2590}
2591
2592Align SelectionDAG::getReducedAlign(EVT VT, bool UseABI) {
2593 const DataLayout &DL = getDataLayout();
2594 Type *Ty = VT.getTypeForEVT(Context&: *getContext());
2595 Align RedAlign = UseABI ? DL.getABITypeAlign(Ty) : DL.getPrefTypeAlign(Ty);
2596
2597 if (TLI->isTypeLegal(VT) || !VT.isVector())
2598 return RedAlign;
2599
2600 const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering();
2601 const Align StackAlign = TFI->getStackAlign();
2602
2603 // See if we can choose a smaller ABI alignment in cases where it's an
2604 // illegal vector type that will get broken down.
2605 if (RedAlign > StackAlign) {
2606 EVT IntermediateVT;
2607 MVT RegisterVT;
2608 unsigned NumIntermediates;
2609 TLI->getVectorTypeBreakdown(Context&: *getContext(), VT, IntermediateVT,
2610 NumIntermediates, RegisterVT);
2611 Ty = IntermediateVT.getTypeForEVT(Context&: *getContext());
2612 Align RedAlign2 = UseABI ? DL.getABITypeAlign(Ty) : DL.getPrefTypeAlign(Ty);
2613 if (RedAlign2 < RedAlign)
2614 RedAlign = RedAlign2;
2615
2616 if (!getMachineFunction().getFrameInfo().isStackRealignable())
2617 // If the stack is not realignable, the alignment should be limited to the
2618 // StackAlignment
2619 RedAlign = std::min(a: RedAlign, b: StackAlign);
2620 }
2621
2622 return RedAlign;
2623}
2624
2625SDValue SelectionDAG::CreateStackTemporary(TypeSize Bytes, Align Alignment) {
2626 MachineFrameInfo &MFI = MF->getFrameInfo();
2627 const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering();
2628 int StackID = 0;
2629 if (Bytes.isScalable())
2630 StackID = TFI->getStackIDForScalableVectors();
2631 // The stack id gives an indication of whether the object is scalable or
2632 // not, so it's safe to pass in the minimum size here.
2633 int FrameIdx = MFI.CreateStackObject(Size: Bytes.getKnownMinValue(), Alignment,
2634 isSpillSlot: false, Alloca: nullptr, ID: StackID);
2635 return getFrameIndex(FI: FrameIdx, VT: TLI->getFrameIndexTy(DL: getDataLayout()));
2636}
2637
2638SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) {
2639 Type *Ty = VT.getTypeForEVT(Context&: *getContext());
2640 Align StackAlign =
2641 std::max(a: getDataLayout().getPrefTypeAlign(Ty), b: Align(minAlign));
2642 return CreateStackTemporary(Bytes: VT.getStoreSize(), Alignment: StackAlign);
2643}
2644
2645SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) {
2646 TypeSize VT1Size = VT1.getStoreSize();
2647 TypeSize VT2Size = VT2.getStoreSize();
2648 assert(VT1Size.isScalable() == VT2Size.isScalable() &&
2649 "Don't know how to choose the maximum size when creating a stack "
2650 "temporary");
2651 TypeSize Bytes = VT1Size.getKnownMinValue() > VT2Size.getKnownMinValue()
2652 ? VT1Size
2653 : VT2Size;
2654
2655 Type *Ty1 = VT1.getTypeForEVT(Context&: *getContext());
2656 Type *Ty2 = VT2.getTypeForEVT(Context&: *getContext());
2657 const DataLayout &DL = getDataLayout();
2658 Align Align = std::max(a: DL.getPrefTypeAlign(Ty: Ty1), b: DL.getPrefTypeAlign(Ty: Ty2));
2659 return CreateStackTemporary(Bytes, Alignment: Align);
2660}
2661
2662SDValue SelectionDAG::emitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
2663 const SDLoc &DL, SDValue Chain) {
2664 EVT SrcVT = SrcOp.getValueType();
2665 Type *DestType = DestVT.getTypeForEVT(Context&: *getContext());
2666 Align DestAlign = getDataLayout().getPrefTypeAlign(Ty: DestType);
2667
2668 // Create the stack frame object.
2669 Align SrcAlign =
2670 getDataLayout().getPrefTypeAlign(Ty: SrcVT.getTypeForEVT(Context&: *getContext()));
2671 SDValue FIPtr = CreateStackTemporary(Bytes: SlotVT.getStoreSize(), Alignment: SrcAlign);
2672
2673 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(Val&: FIPtr);
2674 int SPFI = StackPtrFI->getIndex();
2675 MachinePointerInfo PtrInfo =
2676 MachinePointerInfo::getFixedStack(MF&: getMachineFunction(), FI: SPFI);
2677
2678 // Emit a store to the stack slot. Use a truncstore if the input value is
2679 // later than DestVT.
2680 SDValue Store;
2681
2682 if (SrcVT.bitsGT(VT: SlotVT))
2683 Store = getTruncStore(Chain, dl: DL, Val: SrcOp, Ptr: FIPtr, PtrInfo, SVT: SlotVT, Alignment: SrcAlign);
2684 else {
2685 assert(SrcVT.bitsEq(SlotVT) && "Invalid store");
2686 Store = getStore(Chain, dl: DL, Val: SrcOp, Ptr: FIPtr, PtrInfo, Alignment: SrcAlign);
2687 }
2688
2689 // Result is a load from the stack slot.
2690 if (SlotVT.bitsEq(VT: DestVT))
2691 return getLoad(VT: DestVT, dl: DL, Chain: Store, Ptr: FIPtr, PtrInfo, Alignment: DestAlign);
2692
2693 assert(SlotVT.bitsLT(DestVT) && "Unknown extension!");
2694 return getExtLoad(ExtType: ISD::EXTLOAD, dl: DL, VT: DestVT, Chain: Store, Ptr: FIPtr, PtrInfo, MemVT: SlotVT,
2695 Alignment: DestAlign);
2696}
2697
2698SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1, SDValue N2,
2699 ISD::CondCode Cond, const SDLoc &dl,
2700 SDNodeFlags Flags) {
2701 EVT OpVT = N1.getValueType();
2702
2703 auto GetUndefBooleanConstant = [&]() {
2704 if (VT.getScalarType() == MVT::i1 ||
2705 TLI->getBooleanContents(Type: OpVT) ==
2706 TargetLowering::UndefinedBooleanContent)
2707 return getUNDEF(VT);
2708 // ZeroOrOne / ZeroOrNegative require specific values for the high bits,
2709 // so we cannot use getUNDEF(). Return zero instead.
2710 return getConstant(Val: 0, DL: dl, VT);
2711 };
2712
2713 // These setcc operations always fold.
2714 switch (Cond) {
2715 default: break;
2716 case ISD::SETFALSE:
2717 case ISD::SETFALSE2: return getBoolConstant(V: false, DL: dl, VT, OpVT);
2718 case ISD::SETTRUE:
2719 case ISD::SETTRUE2: return getBoolConstant(V: true, DL: dl, VT, OpVT);
2720
2721 case ISD::SETOEQ:
2722 case ISD::SETOGT:
2723 case ISD::SETOGE:
2724 case ISD::SETOLT:
2725 case ISD::SETOLE:
2726 case ISD::SETONE:
2727 case ISD::SETO:
2728 case ISD::SETUO:
2729 case ISD::SETUEQ:
2730 case ISD::SETUNE:
2731 assert(!OpVT.isInteger() && "Illegal setcc for integer!");
2732 break;
2733 }
2734
2735 if (OpVT.isInteger()) {
2736 // For EQ and NE, we can always pick a value for the undef to make the
2737 // predicate pass or fail, so we can return undef.
2738 // Matches behavior in llvm::ConstantFoldCompareInstruction.
2739 // icmp eq/ne X, undef -> undef.
2740 if ((N1.isUndef() || N2.isUndef()) &&
2741 (Cond == ISD::SETEQ || Cond == ISD::SETNE))
2742 return GetUndefBooleanConstant();
2743
2744 // If both operands are undef, we can return undef for int comparison.
2745 // icmp undef, undef -> undef.
2746 if (N1.isUndef() && N2.isUndef())
2747 return GetUndefBooleanConstant();
2748
2749 // icmp X, X -> true/false
2750 // icmp X, undef -> true/false because undef could be X.
2751 if (N1.isUndef() || N2.isUndef() || N1 == N2)
2752 return getBoolConstant(V: ISD::isTrueWhenEqual(Cond), DL: dl, VT, OpVT);
2753 }
2754
2755 if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(Val&: N2)) {
2756 const APInt &C2 = N2C->getAPIntValue();
2757 if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(Val&: N1)) {
2758 const APInt &C1 = N1C->getAPIntValue();
2759
2760 return getBoolConstant(V: ICmpInst::compare(LHS: C1, RHS: C2, Pred: getICmpCondCode(Pred: Cond)),
2761 DL: dl, VT, OpVT);
2762 }
2763 }
2764
2765 auto *N1CFP = dyn_cast<ConstantFPSDNode>(Val&: N1);
2766 auto *N2CFP = dyn_cast<ConstantFPSDNode>(Val&: N2);
2767
2768 if (N1CFP && N2CFP) {
2769 APFloat::cmpResult R = N1CFP->getValueAPF().compare(RHS: N2CFP->getValueAPF());
2770 switch (Cond) {
2771 default: break;
2772 case ISD::SETEQ: if (R==APFloat::cmpUnordered)
2773 return GetUndefBooleanConstant();
2774 [[fallthrough]];
2775 case ISD::SETOEQ: return getBoolConstant(V: R==APFloat::cmpEqual, DL: dl, VT,
2776 OpVT);
2777 case ISD::SETNE: if (R==APFloat::cmpUnordered)
2778 return GetUndefBooleanConstant();
2779 [[fallthrough]];
2780 case ISD::SETONE: return getBoolConstant(V: R==APFloat::cmpGreaterThan ||
2781 R==APFloat::cmpLessThan, DL: dl, VT,
2782 OpVT);
2783 case ISD::SETLT: if (R==APFloat::cmpUnordered)
2784 return GetUndefBooleanConstant();
2785 [[fallthrough]];
2786 case ISD::SETOLT: return getBoolConstant(V: R==APFloat::cmpLessThan, DL: dl, VT,
2787 OpVT);
2788 case ISD::SETGT: if (R==APFloat::cmpUnordered)
2789 return GetUndefBooleanConstant();
2790 [[fallthrough]];
2791 case ISD::SETOGT: return getBoolConstant(V: R==APFloat::cmpGreaterThan, DL: dl,
2792 VT, OpVT);
2793 case ISD::SETLE: if (R==APFloat::cmpUnordered)
2794 return GetUndefBooleanConstant();
2795 [[fallthrough]];
2796 case ISD::SETOLE: return getBoolConstant(V: R==APFloat::cmpLessThan ||
2797 R==APFloat::cmpEqual, DL: dl, VT,
2798 OpVT);
2799 case ISD::SETGE: if (R==APFloat::cmpUnordered)
2800 return GetUndefBooleanConstant();
2801 [[fallthrough]];
2802 case ISD::SETOGE: return getBoolConstant(V: R==APFloat::cmpGreaterThan ||
2803 R==APFloat::cmpEqual, DL: dl, VT, OpVT);
2804 case ISD::SETO: return getBoolConstant(V: R!=APFloat::cmpUnordered, DL: dl, VT,
2805 OpVT);
2806 case ISD::SETUO: return getBoolConstant(V: R==APFloat::cmpUnordered, DL: dl, VT,
2807 OpVT);
2808 case ISD::SETUEQ: return getBoolConstant(V: R==APFloat::cmpUnordered ||
2809 R==APFloat::cmpEqual, DL: dl, VT,
2810 OpVT);
2811 case ISD::SETUNE: return getBoolConstant(V: R!=APFloat::cmpEqual, DL: dl, VT,
2812 OpVT);
2813 case ISD::SETULT: return getBoolConstant(V: R==APFloat::cmpUnordered ||
2814 R==APFloat::cmpLessThan, DL: dl, VT,
2815 OpVT);
2816 case ISD::SETUGT: return getBoolConstant(V: R==APFloat::cmpGreaterThan ||
2817 R==APFloat::cmpUnordered, DL: dl, VT,
2818 OpVT);
2819 case ISD::SETULE: return getBoolConstant(V: R!=APFloat::cmpGreaterThan, DL: dl,
2820 VT, OpVT);
2821 case ISD::SETUGE: return getBoolConstant(V: R!=APFloat::cmpLessThan, DL: dl, VT,
2822 OpVT);
2823 }
2824 } else if (N1CFP && OpVT.isSimple() && !N2.isUndef()) {
2825 // Ensure that the constant occurs on the RHS.
2826 ISD::CondCode SwappedCond = ISD::getSetCCSwappedOperands(Operation: Cond);
2827 if (!TLI->isCondCodeLegal(CC: SwappedCond, VT: OpVT.getSimpleVT()))
2828 return SDValue();
2829 return getSetCC(DL: dl, VT, LHS: N2, RHS: N1, Cond: SwappedCond, /*Chain=*/{},
2830 /*IsSignaling=*/false, Flags);
2831 } else if ((N2CFP && N2CFP->getValueAPF().isNaN()) ||
2832 (OpVT.isFloatingPoint() && (N1.isUndef() || N2.isUndef()))) {
2833 // If an operand is known to be a nan (or undef that could be a nan), we can
2834 // fold it.
2835 // Choosing NaN for the undef will always make unordered comparison succeed
2836 // and ordered comparison fails.
2837 // Matches behavior in llvm::ConstantFoldCompareInstruction.
2838 switch (ISD::getUnorderedFlavor(Cond)) {
2839 default:
2840 llvm_unreachable("Unknown flavor!");
2841 case 0: // Known false.
2842 return getBoolConstant(V: false, DL: dl, VT, OpVT);
2843 case 1: // Known true.
2844 return getBoolConstant(V: true, DL: dl, VT, OpVT);
2845 case 2: // Undefined.
2846 return GetUndefBooleanConstant();
2847 }
2848 }
2849
2850 // Could not fold it.
2851 return SDValue();
2852}
2853
2854/// SignBitIsZero - Return true if the sign bit of Op is known to be zero. We
2855/// use this predicate to simplify operations downstream.
2856bool SelectionDAG::SignBitIsZero(SDValue Op, unsigned Depth) const {
2857 unsigned BitWidth = Op.getScalarValueSizeInBits();
2858 return MaskedValueIsZero(Op, Mask: APInt::getSignMask(BitWidth), Depth);
2859}
2860
2861// TODO: Should have argument to specify if sign bit of nan is ignorable.
2862bool SelectionDAG::SignBitIsZeroFP(SDValue Op, unsigned Depth) const {
2863 if (Depth >= MaxRecursionDepth)
2864 return false; // Limit search depth.
2865
2866 unsigned Opc = Op.getOpcode();
2867 switch (Opc) {
2868 case ISD::FABS:
2869 return true;
2870 case ISD::AssertNoFPClass: {
2871 FPClassTest NoFPClass =
2872 static_cast<FPClassTest>(Op.getConstantOperandVal(i: 1));
2873
2874 const FPClassTest TestMask = fcNan | fcNegative;
2875 return (NoFPClass & TestMask) == TestMask;
2876 }
2877 case ISD::ARITH_FENCE:
2878 return SignBitIsZeroFP(Op: Op.getOperand(i: 0), Depth: Depth + 1);
2879 case ISD::FEXP:
2880 case ISD::FEXP2:
2881 case ISD::FEXP10:
2882 return Op->getFlags().hasNoNaNs();
2883 case ISD::FMINNUM:
2884 case ISD::FMINNUM_IEEE:
2885 case ISD::FMINIMUM:
2886 case ISD::FMINIMUMNUM:
2887 return SignBitIsZeroFP(Op: Op.getOperand(i: 1), Depth: Depth + 1) &&
2888 SignBitIsZeroFP(Op: Op.getOperand(i: 0), Depth: Depth + 1);
2889 case ISD::FMAXNUM:
2890 case ISD::FMAXNUM_IEEE:
2891 case ISD::FMAXIMUM:
2892 case ISD::FMAXIMUMNUM:
2893 // TODO: If we can ignore the sign bit of nans, only one side being known 0
2894 // is sufficient.
2895 return SignBitIsZeroFP(Op: Op.getOperand(i: 1), Depth: Depth + 1) &&
2896 SignBitIsZeroFP(Op: Op.getOperand(i: 0), Depth: Depth + 1);
2897 default:
2898 return false;
2899 }
2900
2901 llvm_unreachable("covered opcode switch");
2902}
2903
2904/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero. We use
2905/// this predicate to simplify operations downstream. Mask is known to be zero
2906/// for bits that V cannot have.
2907bool SelectionDAG::MaskedValueIsZero(SDValue V, const APInt &Mask,
2908 unsigned Depth) const {
2909 return Mask.isSubsetOf(RHS: computeKnownBits(Op: V, Depth).Zero);
2910}
2911
2912/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero in
2913/// DemandedElts. We use this predicate to simplify operations downstream.
2914/// Mask is known to be zero for bits that V cannot have.
2915bool SelectionDAG::MaskedValueIsZero(SDValue V, const APInt &Mask,
2916 const APInt &DemandedElts,
2917 unsigned Depth) const {
2918 return Mask.isSubsetOf(RHS: computeKnownBits(Op: V, DemandedElts, Depth).Zero);
2919}
2920
2921/// MaskedVectorIsZero - Return true if 'Op' is known to be zero in
2922/// DemandedElts. We use this predicate to simplify operations downstream.
2923bool SelectionDAG::MaskedVectorIsZero(SDValue V, const APInt &DemandedElts,
2924 unsigned Depth /* = 0 */) const {
2925 return computeKnownBits(Op: V, DemandedElts, Depth).isZero();
2926}
2927
2928/// MaskedValueIsAllOnes - Return true if '(Op & Mask) == Mask'.
2929bool SelectionDAG::MaskedValueIsAllOnes(SDValue V, const APInt &Mask,
2930 unsigned Depth) const {
2931 return Mask.isSubsetOf(RHS: computeKnownBits(Op: V, Depth).One);
2932}
2933
2934APInt SelectionDAG::computeVectorKnownZeroElements(SDValue Op,
2935 const APInt &DemandedElts,
2936 unsigned Depth) const {
2937 EVT VT = Op.getValueType();
2938 assert(VT.isVector() && !VT.isScalableVector() && "Only for fixed vectors!");
2939
2940 unsigned NumElts = VT.getVectorNumElements();
2941 assert(DemandedElts.getBitWidth() == NumElts && "Unexpected demanded mask.");
2942
2943 APInt KnownZeroElements = APInt::getZero(numBits: NumElts);
2944 for (unsigned EltIdx = 0; EltIdx != NumElts; ++EltIdx) {
2945 if (!DemandedElts[EltIdx])
2946 continue; // Don't query elements that are not demanded.
2947 APInt Mask = APInt::getOneBitSet(numBits: NumElts, BitNo: EltIdx);
2948 if (MaskedVectorIsZero(V: Op, DemandedElts: Mask, Depth))
2949 KnownZeroElements.setBit(EltIdx);
2950 }
2951 return KnownZeroElements;
2952}
2953
2954/// isSplatValue - Return true if the vector V has the same value
2955/// across all DemandedElts. For scalable vectors, we don't know the
2956/// number of lanes at compile time. Instead, we use a 1 bit APInt
2957/// to represent a conservative value for all lanes; that is, that
2958/// one bit value is implicitly splatted across all lanes.
2959bool SelectionDAG::isSplatValue(SDValue V, const APInt &DemandedElts,
2960 APInt &UndefElts, unsigned Depth) const {
2961 unsigned Opcode = V.getOpcode();
2962 EVT VT = V.getValueType();
2963 assert(VT.isVector() && "Vector type expected");
2964 assert((!VT.isScalableVector() || DemandedElts.getBitWidth() == 1) &&
2965 "scalable demanded bits are ignored");
2966
2967 if (!DemandedElts)
2968 return false; // No demanded elts, better to assume we don't know anything.
2969
2970 if (Depth >= MaxRecursionDepth)
2971 return false; // Limit search depth.
2972
2973 // Deal with some common cases here that work for both fixed and scalable
2974 // vector types.
2975 switch (Opcode) {
2976 case ISD::SPLAT_VECTOR:
2977 UndefElts = V.getOperand(i: 0).isUndef()
2978 ? APInt::getAllOnes(numBits: DemandedElts.getBitWidth())
2979 : APInt(DemandedElts.getBitWidth(), 0);
2980 return true;
2981 case ISD::ADD:
2982 case ISD::SUB:
2983 case ISD::AND:
2984 case ISD::XOR:
2985 case ISD::OR: {
2986 APInt UndefLHS, UndefRHS;
2987 SDValue LHS = V.getOperand(i: 0);
2988 SDValue RHS = V.getOperand(i: 1);
2989 // Only recognize splats with the same demanded undef elements for both
2990 // operands, otherwise we might fail to handle binop-specific undef
2991 // handling.
2992 // e.g. (and undef, 0) -> 0 etc.
2993 if (isSplatValue(V: LHS, DemandedElts, UndefElts&: UndefLHS, Depth: Depth + 1) &&
2994 isSplatValue(V: RHS, DemandedElts, UndefElts&: UndefRHS, Depth: Depth + 1) &&
2995 (DemandedElts & UndefLHS) == (DemandedElts & UndefRHS)) {
2996 UndefElts = UndefLHS | UndefRHS;
2997 return true;
2998 }
2999 return false;
3000 }
3001 case ISD::ABS:
3002 case ISD::ABS_MIN_POISON:
3003 case ISD::TRUNCATE:
3004 case ISD::SIGN_EXTEND:
3005 case ISD::ZERO_EXTEND:
3006 return isSplatValue(V: V.getOperand(i: 0), DemandedElts, UndefElts, Depth: Depth + 1);
3007 default:
3008 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
3009 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)
3010 return TLI->isSplatValueForTargetNode(Op: V, DemandedElts, UndefElts, DAG: *this,
3011 Depth);
3012 break;
3013 }
3014
3015 // We don't support other cases than those above for scalable vectors at
3016 // the moment.
3017 if (VT.isScalableVector())
3018 return false;
3019
3020 unsigned NumElts = VT.getVectorNumElements();
3021 assert(NumElts == DemandedElts.getBitWidth() && "Vector size mismatch");
3022 UndefElts = APInt::getZero(numBits: NumElts);
3023
3024 switch (Opcode) {
3025 case ISD::BUILD_VECTOR: {
3026 SDValue Scl;
3027 for (unsigned i = 0; i != NumElts; ++i) {
3028 SDValue Op = V.getOperand(i);
3029 if (Op.isUndef()) {
3030 UndefElts.setBit(i);
3031 continue;
3032 }
3033 if (!DemandedElts[i])
3034 continue;
3035 if (Scl && Scl != Op)
3036 return false;
3037 Scl = Op;
3038 }
3039 return true;
3040 }
3041 case ISD::VECTOR_SHUFFLE: {
3042 // Check if this is a shuffle node doing a splat or a shuffle of a splat.
3043 APInt DemandedLHS = APInt::getZero(numBits: NumElts);
3044 APInt DemandedRHS = APInt::getZero(numBits: NumElts);
3045 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Val&: V)->getMask();
3046 for (int i = 0; i != (int)NumElts; ++i) {
3047 int M = Mask[i];
3048 if (M < 0) {
3049 UndefElts.setBit(i);
3050 continue;
3051 }
3052 if (!DemandedElts[i])
3053 continue;
3054 if (M < (int)NumElts)
3055 DemandedLHS.setBit(M);
3056 else
3057 DemandedRHS.setBit(M - NumElts);
3058 }
3059
3060 // If we aren't demanding either op, assume there's no splat.
3061 // If we are demanding both ops, assume there's no splat.
3062 if ((DemandedLHS.isZero() && DemandedRHS.isZero()) ||
3063 (!DemandedLHS.isZero() && !DemandedRHS.isZero()))
3064 return false;
3065
3066 // See if the demanded elts of the source op is a splat or we only demand
3067 // one element, which should always be a splat.
3068 // TODO: Handle source ops splats with undefs.
3069 auto CheckSplatSrc = [&](SDValue Src, const APInt &SrcElts) {
3070 APInt SrcUndefs;
3071 return (SrcElts.popcount() == 1) ||
3072 (isSplatValue(V: Src, DemandedElts: SrcElts, UndefElts&: SrcUndefs, Depth: Depth + 1) &&
3073 (SrcElts & SrcUndefs).isZero());
3074 };
3075 if (!DemandedLHS.isZero())
3076 return CheckSplatSrc(V.getOperand(i: 0), DemandedLHS);
3077 return CheckSplatSrc(V.getOperand(i: 1), DemandedRHS);
3078 }
3079 case ISD::EXTRACT_SUBVECTOR: {
3080 // Offset the demanded elts by the subvector index.
3081 SDValue Src = V.getOperand(i: 0);
3082 // We don't support scalable vectors at the moment.
3083 if (Src.getValueType().isScalableVector())
3084 return false;
3085 uint64_t Idx = V.getConstantOperandVal(i: 1);
3086 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3087 APInt UndefSrcElts;
3088 APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
3089 if (isSplatValue(V: Src, DemandedElts: DemandedSrcElts, UndefElts&: UndefSrcElts, Depth: Depth + 1)) {
3090 UndefElts = UndefSrcElts.extractBits(numBits: NumElts, bitPosition: Idx);
3091 return true;
3092 }
3093 break;
3094 }
3095 case ISD::ANY_EXTEND_VECTOR_INREG:
3096 case ISD::SIGN_EXTEND_VECTOR_INREG:
3097 case ISD::ZERO_EXTEND_VECTOR_INREG: {
3098 // Widen the demanded elts by the src element count.
3099 SDValue Src = V.getOperand(i: 0);
3100 // We don't support scalable vectors at the moment.
3101 if (Src.getValueType().isScalableVector())
3102 return false;
3103 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3104 APInt UndefSrcElts;
3105 APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts);
3106 if (isSplatValue(V: Src, DemandedElts: DemandedSrcElts, UndefElts&: UndefSrcElts, Depth: Depth + 1)) {
3107 UndefElts = UndefSrcElts.trunc(width: NumElts);
3108 return true;
3109 }
3110 break;
3111 }
3112 case ISD::BITCAST: {
3113 SDValue Src = V.getOperand(i: 0);
3114 EVT SrcVT = Src.getValueType();
3115 unsigned SrcBitWidth = SrcVT.getScalarSizeInBits();
3116 unsigned BitWidth = VT.getScalarSizeInBits();
3117
3118 // Ignore bitcasts from unsupported types.
3119 // TODO: Add fp support?
3120 if (!SrcVT.isVector() || !SrcVT.isInteger() || !VT.isInteger())
3121 break;
3122
3123 // Bitcast 'small element' vector to 'large element' vector.
3124 if ((BitWidth % SrcBitWidth) == 0) {
3125 // See if each sub element is a splat.
3126 unsigned Scale = BitWidth / SrcBitWidth;
3127 unsigned NumSrcElts = SrcVT.getVectorNumElements();
3128 APInt ScaledDemandedElts =
3129 APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumSrcElts);
3130 for (unsigned I = 0; I != Scale; ++I) {
3131 APInt SubUndefElts;
3132 APInt SubDemandedElt = APInt::getOneBitSet(numBits: Scale, BitNo: I);
3133 APInt SubDemandedElts = APInt::getSplat(NewLen: NumSrcElts, V: SubDemandedElt);
3134 SubDemandedElts &= ScaledDemandedElts;
3135 if (!isSplatValue(V: Src, DemandedElts: SubDemandedElts, UndefElts&: SubUndefElts, Depth: Depth + 1))
3136 return false;
3137 // TODO: Add support for merging sub undef elements.
3138 if (!SubUndefElts.isZero())
3139 return false;
3140 }
3141 return true;
3142 }
3143 break;
3144 }
3145 }
3146
3147 return false;
3148}
3149
3150/// Helper wrapper to main isSplatValue function.
3151bool SelectionDAG::isSplatValue(SDValue V, bool AllowUndefs) const {
3152 EVT VT = V.getValueType();
3153 assert(VT.isVector() && "Vector type expected");
3154
3155 APInt UndefElts;
3156 // Since the number of lanes in a scalable vector is unknown at compile time,
3157 // we track one bit which is implicitly broadcast to all lanes. This means
3158 // that all lanes in a scalable vector are considered demanded.
3159 APInt DemandedElts
3160 = APInt::getAllOnes(numBits: VT.isScalableVector() ? 1 : VT.getVectorNumElements());
3161 return isSplatValue(V, DemandedElts, UndefElts) &&
3162 (AllowUndefs || !UndefElts);
3163}
3164
3165SDValue SelectionDAG::getSplatSourceVector(SDValue V, int &SplatIdx) {
3166 V = peekThroughExtractSubvectors(V);
3167
3168 EVT VT = V.getValueType();
3169 unsigned Opcode = V.getOpcode();
3170 switch (Opcode) {
3171 default: {
3172 APInt UndefElts;
3173 // Since the number of lanes in a scalable vector is unknown at compile time,
3174 // we track one bit which is implicitly broadcast to all lanes. This means
3175 // that all lanes in a scalable vector are considered demanded.
3176 APInt DemandedElts
3177 = APInt::getAllOnes(numBits: VT.isScalableVector() ? 1 : VT.getVectorNumElements());
3178
3179 if (isSplatValue(V, DemandedElts, UndefElts)) {
3180 if (VT.isScalableVector()) {
3181 // DemandedElts and UndefElts are ignored for scalable vectors, since
3182 // the only supported cases are SPLAT_VECTOR nodes.
3183 SplatIdx = 0;
3184 } else {
3185 // Handle case where all demanded elements are UNDEF.
3186 if (DemandedElts.isSubsetOf(RHS: UndefElts)) {
3187 SplatIdx = 0;
3188 return getUNDEF(VT);
3189 }
3190 SplatIdx = (UndefElts & DemandedElts).countr_one();
3191 }
3192 return V;
3193 }
3194 break;
3195 }
3196 case ISD::SPLAT_VECTOR:
3197 SplatIdx = 0;
3198 return V;
3199 case ISD::VECTOR_SHUFFLE: {
3200 assert(!VT.isScalableVector());
3201 // Check if this is a shuffle node doing a splat.
3202 // TODO - remove this and rely purely on SelectionDAG::isSplatValue,
3203 // getTargetVShiftNode currently struggles without the splat source.
3204 auto *SVN = cast<ShuffleVectorSDNode>(Val&: V);
3205 if (!SVN->isSplat())
3206 break;
3207 int Idx = SVN->getSplatIndex();
3208 int NumElts = V.getValueType().getVectorNumElements();
3209 SplatIdx = Idx % NumElts;
3210 return V.getOperand(i: Idx / NumElts);
3211 }
3212 }
3213
3214 return SDValue();
3215}
3216
3217SDValue SelectionDAG::getSplatValue(SDValue V, bool LegalTypes) {
3218 int SplatIdx;
3219 if (SDValue SrcVector = getSplatSourceVector(V, SplatIdx)) {
3220 EVT SVT = SrcVector.getValueType().getScalarType();
3221 EVT LegalSVT = SVT;
3222 if (LegalTypes && !TLI->isTypeLegal(VT: SVT)) {
3223 if (!SVT.isInteger())
3224 return SDValue();
3225 LegalSVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT: LegalSVT);
3226 if (LegalSVT.bitsLT(VT: SVT))
3227 return SDValue();
3228 }
3229 return getExtractVectorElt(DL: SDLoc(V), VT: LegalSVT, Vec: SrcVector, Idx: SplatIdx);
3230 }
3231 return SDValue();
3232}
3233
3234std::optional<ConstantRange>
3235SelectionDAG::getValidShiftAmountRange(SDValue V, const APInt &DemandedElts,
3236 unsigned Depth) const {
3237 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
3238 V.getOpcode() == ISD::SRA) &&
3239 "Unknown shift node");
3240 // Shifting more than the bitwidth is not valid.
3241 unsigned BitWidth = V.getScalarValueSizeInBits();
3242
3243 if (auto *Cst = dyn_cast<ConstantSDNode>(Val: V.getOperand(i: 1))) {
3244 const APInt &ShAmt = Cst->getAPIntValue();
3245 if (ShAmt.uge(RHS: BitWidth))
3246 return std::nullopt;
3247 return ConstantRange(ShAmt);
3248 }
3249
3250 if (auto *BV = dyn_cast<BuildVectorSDNode>(Val: V.getOperand(i: 1))) {
3251 const APInt *MinAmt = nullptr, *MaxAmt = nullptr;
3252 for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) {
3253 if (!DemandedElts[i])
3254 continue;
3255 auto *SA = dyn_cast<ConstantSDNode>(Val: BV->getOperand(Num: i));
3256 if (!SA) {
3257 MinAmt = MaxAmt = nullptr;
3258 break;
3259 }
3260 const APInt &ShAmt = SA->getAPIntValue();
3261 if (ShAmt.uge(RHS: BitWidth))
3262 return std::nullopt;
3263 if (!MinAmt || MinAmt->ugt(RHS: ShAmt))
3264 MinAmt = &ShAmt;
3265 if (!MaxAmt || MaxAmt->ult(RHS: ShAmt))
3266 MaxAmt = &ShAmt;
3267 }
3268 assert(((!MinAmt && !MaxAmt) || (MinAmt && MaxAmt)) &&
3269 "Failed to find matching min/max shift amounts");
3270 if (MinAmt && MaxAmt)
3271 return ConstantRange(*MinAmt, *MaxAmt + 1);
3272 }
3273
3274 // Use computeKnownBits to find a hidden constant/knownbits (usually type
3275 // legalized). e.g. Hidden behind multiple bitcasts/build_vector/casts etc.
3276 KnownBits KnownAmt = computeKnownBits(Op: V.getOperand(i: 1), DemandedElts, Depth);
3277 if (KnownAmt.getMaxValue().ult(RHS: BitWidth))
3278 return ConstantRange::fromKnownBits(Known: KnownAmt, /*IsSigned=*/false);
3279
3280 return std::nullopt;
3281}
3282
3283std::optional<unsigned>
3284SelectionDAG::getValidShiftAmount(SDValue V, const APInt &DemandedElts,
3285 unsigned Depth) const {
3286 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
3287 V.getOpcode() == ISD::SRA) &&
3288 "Unknown shift node");
3289 if (std::optional<ConstantRange> AmtRange =
3290 getValidShiftAmountRange(V, DemandedElts, Depth))
3291 if (const APInt *ShAmt = AmtRange->getSingleElement())
3292 return ShAmt->getZExtValue();
3293 return std::nullopt;
3294}
3295
3296std::optional<unsigned>
3297SelectionDAG::getValidShiftAmount(SDValue V, unsigned Depth) const {
3298 APInt DemandedElts = getDemandAllEltsMask(V);
3299 return getValidShiftAmount(V, DemandedElts, Depth);
3300}
3301
3302std::optional<unsigned>
3303SelectionDAG::getValidMinimumShiftAmount(SDValue V, const APInt &DemandedElts,
3304 unsigned Depth) const {
3305 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
3306 V.getOpcode() == ISD::SRA) &&
3307 "Unknown shift node");
3308 if (std::optional<ConstantRange> AmtRange =
3309 getValidShiftAmountRange(V, DemandedElts, Depth))
3310 return AmtRange->getUnsignedMin().getZExtValue();
3311 return std::nullopt;
3312}
3313
3314std::optional<unsigned>
3315SelectionDAG::getValidMinimumShiftAmount(SDValue V, unsigned Depth) const {
3316 APInt DemandedElts = getDemandAllEltsMask(V);
3317 return getValidMinimumShiftAmount(V, DemandedElts, Depth);
3318}
3319
3320std::optional<unsigned>
3321SelectionDAG::getValidMaximumShiftAmount(SDValue V, const APInt &DemandedElts,
3322 unsigned Depth) const {
3323 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
3324 V.getOpcode() == ISD::SRA) &&
3325 "Unknown shift node");
3326 if (std::optional<ConstantRange> AmtRange =
3327 getValidShiftAmountRange(V, DemandedElts, Depth))
3328 return AmtRange->getUnsignedMax().getZExtValue();
3329 return std::nullopt;
3330}
3331
3332std::optional<unsigned>
3333SelectionDAG::getValidMaximumShiftAmount(SDValue V, unsigned Depth) const {
3334 APInt DemandedElts = getDemandAllEltsMask(V);
3335 return getValidMaximumShiftAmount(V, DemandedElts, Depth);
3336}
3337
3338/// Determine which bits of Op are known to be either zero or one and return
3339/// them in Known. For vectors, the known bits are those that are shared by
3340/// every vector element.
3341KnownBits SelectionDAG::computeKnownBits(SDValue Op, unsigned Depth) const {
3342 APInt DemandedElts = getDemandAllEltsMask(V: Op);
3343 return computeKnownBits(Op, DemandedElts, Depth);
3344}
3345
3346/// Determine which bits of Op are known to be either zero or one and return
3347/// them in Known. The DemandedElts argument allows us to only collect the known
3348/// bits that are shared by the requested vector elements.
3349KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts,
3350 unsigned Depth) const {
3351 unsigned BitWidth = Op.getScalarValueSizeInBits();
3352
3353 KnownBits Known(BitWidth); // Don't know anything.
3354
3355 if (auto OptAPInt = Op->bitcastToAPInt()) {
3356 // We know all of the bits for a constant!
3357 APInt V = *std::move(OptAPInt);
3358
3359 // Swap the low-order and high-order double of a ppc_fp128 when casting to
3360 // i128, see #44482.
3361 //
3362 // A ppc_fp128 is two doubles, with the high-order double stored at the
3363 // lower address. Reading that as an integer therefore puts the high-order
3364 // double in the high 64 bits on big-endian targets and in the low 64 bits
3365 // on little-endian targets.
3366 //
3367 // But APFloat::bitcastToAPInt is endianness-agnostic and always places the
3368 // high-order double in the low 64 bits. Hence the two doubles must be
3369 // flipped on big-endian targets.
3370 if (getDataLayout().isBigEndian() && Op.getValueType() == MVT::ppcf128)
3371 V = V.rotl(rotateAmt: 64);
3372
3373 return KnownBits::makeConstant(C: V);
3374 }
3375
3376 if (Depth >= MaxRecursionDepth)
3377 return Known; // Limit search depth.
3378
3379 KnownBits Known2;
3380 unsigned NumElts = DemandedElts.getBitWidth();
3381 assert((!Op.getValueType().isScalableVector() || NumElts == 1) &&
3382 "DemandedElts for scalable vectors must be 1 to represent all lanes");
3383 assert((!Op.getValueType().isFixedLengthVector() ||
3384 NumElts == Op.getValueType().getVectorNumElements()) &&
3385 "Unexpected vector size");
3386
3387 if (!DemandedElts)
3388 return Known; // No demanded elts, better to assume we don't know anything.
3389
3390 unsigned Opcode = Op.getOpcode();
3391 switch (Opcode) {
3392 case ISD::FREEZE: {
3393 if (isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 0), DemandedElts,
3394 Kind: UndefPoisonKind::UndefOrPoison))
3395 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3396 break;
3397 }
3398 case ISD::MERGE_VALUES:
3399 return computeKnownBits(Op: Op.getOperand(i: Op.getResNo()), DemandedElts,
3400 Depth: Depth + 1);
3401 case ISD::SPLAT_VECTOR: {
3402 SDValue SrcOp = Op.getOperand(i: 0);
3403 assert(SrcOp.getValueSizeInBits() >= BitWidth &&
3404 "Expected SPLAT_VECTOR implicit truncation");
3405 // Implicitly truncate the bits to match the official semantics of
3406 // SPLAT_VECTOR.
3407 Known = computeKnownBits(Op: SrcOp, Depth: Depth + 1).trunc(BitWidth);
3408 break;
3409 }
3410 case ISD::SPLAT_VECTOR_PARTS: {
3411 unsigned ScalarSize = Op.getOperand(i: 0).getScalarValueSizeInBits();
3412 assert(ScalarSize * Op.getNumOperands() == BitWidth &&
3413 "Expected SPLAT_VECTOR_PARTS scalars to cover element width");
3414 for (auto [I, SrcOp] : enumerate(First: Op->ops())) {
3415 Known.insertBits(SubBits: computeKnownBits(Op: SrcOp, Depth: Depth + 1), BitPosition: ScalarSize * I);
3416 }
3417 break;
3418 }
3419 case ISD::STEP_VECTOR: {
3420 const APInt &Step = Op.getConstantOperandAPInt(i: 0);
3421
3422 if (Step.isPowerOf2())
3423 Known.Zero.setLowBits(Step.logBase2());
3424
3425 const Function &F = getMachineFunction().getFunction();
3426
3427 if (!isUIntN(N: BitWidth, x: Op.getValueType().getVectorMinNumElements()))
3428 break;
3429 const APInt MinNumElts =
3430 APInt(BitWidth, Op.getValueType().getVectorMinNumElements());
3431
3432 bool Overflow;
3433 const APInt MaxNumElts = getVScaleRange(F: &F, BitWidth)
3434 .getUnsignedMax()
3435 .umul_ov(RHS: MinNumElts, Overflow);
3436 if (Overflow)
3437 break;
3438
3439 const APInt MaxValue = (MaxNumElts - 1).umul_ov(RHS: Step, Overflow);
3440 if (Overflow)
3441 break;
3442
3443 Known.Zero.setHighBits(MaxValue.countl_zero());
3444 break;
3445 }
3446 case ISD::BUILD_VECTOR:
3447 assert(!Op.getValueType().isScalableVector());
3448 // Collect the known bits that are shared by every demanded vector element.
3449 Known.setAllConflict();
3450 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
3451 if (!DemandedElts[i])
3452 continue;
3453
3454 SDValue SrcOp = Op.getOperand(i);
3455 if (SrcOp.getOpcode() == ISD::POISON)
3456 continue;
3457
3458 Known2 = computeKnownBits(Op: SrcOp, Depth: Depth + 1);
3459
3460 // BUILD_VECTOR can implicitly truncate sources, we must handle this.
3461 if (SrcOp.getValueSizeInBits() != BitWidth) {
3462 assert(SrcOp.getValueSizeInBits() > BitWidth &&
3463 "Expected BUILD_VECTOR implicit truncation");
3464 Known2 = Known2.trunc(BitWidth);
3465 }
3466
3467 // Known bits are the values that are shared by every demanded element.
3468 Known = Known.intersectWith(RHS: Known2);
3469
3470 // If we don't know any bits, early out.
3471 if (Known.isUnknown())
3472 break;
3473 }
3474
3475 // If every demanded element was poison, we know nothing.
3476 if (Known.hasConflict())
3477 Known.resetAll();
3478 break;
3479 case ISD::VECTOR_COMPRESS: {
3480 SDValue Vec = Op.getOperand(i: 0);
3481 SDValue PassThru = Op.getOperand(i: 2);
3482 Known = computeKnownBits(Op: PassThru, DemandedElts, Depth: Depth + 1);
3483 // If we don't know any bits, early out.
3484 if (Known.isUnknown())
3485 break;
3486 Known2 = computeKnownBits(Op: Vec, Depth: Depth + 1);
3487 Known = Known.intersectWith(RHS: Known2);
3488 break;
3489 }
3490 case ISD::VECTOR_SHUFFLE: {
3491 assert(!Op.getValueType().isScalableVector());
3492 // Collect the known bits that are shared by every vector element referenced
3493 // by the shuffle.
3494 APInt DemandedLHS, DemandedRHS;
3495 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
3496 assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
3497 if (!getShuffleDemandedElts(SrcWidth: NumElts, Mask: SVN->getMask(), DemandedElts,
3498 DemandedLHS, DemandedRHS))
3499 break;
3500
3501 // Known bits are the values that are shared by every demanded element.
3502 Known.setAllConflict();
3503 if (!!DemandedLHS) {
3504 SDValue LHS = Op.getOperand(i: 0);
3505 Known2 = computeKnownBits(Op: LHS, DemandedElts: DemandedLHS, Depth: Depth + 1);
3506 Known = Known.intersectWith(RHS: Known2);
3507 }
3508 // If we don't know any bits, early out.
3509 if (Known.isUnknown())
3510 break;
3511 if (!!DemandedRHS) {
3512 SDValue RHS = Op.getOperand(i: 1);
3513 Known2 = computeKnownBits(Op: RHS, DemandedElts: DemandedRHS, Depth: Depth + 1);
3514 Known = Known.intersectWith(RHS: Known2);
3515 }
3516 break;
3517 }
3518 case ISD::VSCALE: {
3519 const Function &F = getMachineFunction().getFunction();
3520 const APInt &Multiplier = Op.getConstantOperandAPInt(i: 0);
3521 Known = getVScaleRange(F: &F, BitWidth).multiply(Other: Multiplier).toKnownBits();
3522 break;
3523 }
3524 case ISD::CONCAT_VECTORS: {
3525 if (Op.getValueType().isScalableVector())
3526 break;
3527 // Split DemandedElts and test each of the demanded subvectors.
3528 Known.setAllConflict();
3529 EVT SubVectorVT = Op.getOperand(i: 0).getValueType();
3530 unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements();
3531 unsigned NumSubVectors = Op.getNumOperands();
3532 for (unsigned i = 0; i != NumSubVectors; ++i) {
3533 APInt DemandedSub =
3534 DemandedElts.extractBits(numBits: NumSubVectorElts, bitPosition: i * NumSubVectorElts);
3535 if (!!DemandedSub) {
3536 SDValue Sub = Op.getOperand(i);
3537 Known2 = computeKnownBits(Op: Sub, DemandedElts: DemandedSub, Depth: Depth + 1);
3538 Known = Known.intersectWith(RHS: Known2);
3539 }
3540 // If we don't know any bits, early out.
3541 if (Known.isUnknown())
3542 break;
3543 }
3544 break;
3545 }
3546 case ISD::INSERT_SUBVECTOR: {
3547 if (Op.getValueType().isScalableVector())
3548 break;
3549 // Demand any elements from the subvector and the remainder from the src its
3550 // inserted into.
3551 SDValue Src = Op.getOperand(i: 0);
3552 SDValue Sub = Op.getOperand(i: 1);
3553 uint64_t Idx = Op.getConstantOperandVal(i: 2);
3554 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
3555 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
3556 APInt DemandedSrcElts = DemandedElts;
3557 DemandedSrcElts.clearBits(LoBit: Idx, HiBit: Idx + NumSubElts);
3558
3559 Known.setAllConflict();
3560 if (!!DemandedSubElts) {
3561 Known = computeKnownBits(Op: Sub, DemandedElts: DemandedSubElts, Depth: Depth + 1);
3562 if (Known.isUnknown())
3563 break; // early-out.
3564 }
3565 if (!!DemandedSrcElts) {
3566 Known2 = computeKnownBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
3567 Known = Known.intersectWith(RHS: Known2);
3568 }
3569 break;
3570 }
3571 case ISD::EXTRACT_SUBVECTOR: {
3572 // Offset the demanded elts by the subvector index.
3573 SDValue Src = Op.getOperand(i: 0);
3574
3575 APInt DemandedSrcElts;
3576 if (Src.getValueType().isScalableVector())
3577 DemandedSrcElts = APInt(1, 1); // <=> 'demand all elements'
3578 else {
3579 uint64_t Idx = Op.getConstantOperandVal(i: 1);
3580 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3581 DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
3582 }
3583 Known = computeKnownBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
3584 break;
3585 }
3586 case ISD::SCALAR_TO_VECTOR: {
3587 if (Op.getValueType().isScalableVector())
3588 break;
3589 // We know about scalar_to_vector as much as we know about it source,
3590 // which becomes the first element of otherwise unknown vector.
3591 if (DemandedElts != 1)
3592 break;
3593
3594 SDValue N0 = Op.getOperand(i: 0);
3595 Known = computeKnownBits(Op: N0, Depth: Depth + 1);
3596 if (N0.getValueSizeInBits() != BitWidth)
3597 Known = Known.trunc(BitWidth);
3598
3599 break;
3600 }
3601 case ISD::BITCAST: {
3602 if (Op.getValueType().isScalableVector())
3603 break;
3604
3605 SDValue N0 = Op.getOperand(i: 0);
3606 EVT SubVT = N0.getValueType();
3607 unsigned SubBitWidth = SubVT.getScalarSizeInBits();
3608
3609 // Ignore bitcasts from unsupported types.
3610 if (!(SubVT.isInteger() || SubVT.isFloatingPoint()))
3611 break;
3612
3613 // Fast handling of 'identity' bitcasts.
3614 if (BitWidth == SubBitWidth) {
3615 Known = computeKnownBits(Op: N0, DemandedElts, Depth: Depth + 1);
3616 break;
3617 }
3618
3619 bool IsLE = getDataLayout().isLittleEndian();
3620
3621 // Bitcast 'small element' vector to 'large element' scalar/vector.
3622 if ((BitWidth % SubBitWidth) == 0) {
3623 assert(N0.getValueType().isVector() && "Expected bitcast from vector");
3624
3625 // Collect known bits for the (larger) output by collecting the known
3626 // bits from each set of sub elements and shift these into place.
3627 // We need to separately call computeKnownBits for each set of
3628 // sub elements as the knownbits for each is likely to be different.
3629 unsigned SubScale = BitWidth / SubBitWidth;
3630 APInt SubDemandedElts(NumElts * SubScale, 0);
3631 for (unsigned i = 0; i != NumElts; ++i)
3632 if (DemandedElts[i])
3633 SubDemandedElts.setBit(i * SubScale);
3634
3635 for (unsigned i = 0; i != SubScale; ++i) {
3636 Known2 = computeKnownBits(Op: N0, DemandedElts: SubDemandedElts.shl(shiftAmt: i),
3637 Depth: Depth + 1);
3638 unsigned Shifts = IsLE ? i : SubScale - 1 - i;
3639 Known.insertBits(SubBits: Known2, BitPosition: SubBitWidth * Shifts);
3640 }
3641 }
3642
3643 // Bitcast 'large element' scalar/vector to 'small element' vector.
3644 if ((SubBitWidth % BitWidth) == 0) {
3645 assert(Op.getValueType().isVector() && "Expected bitcast to vector");
3646
3647 // Collect known bits for the (smaller) output by collecting the known
3648 // bits from the overlapping larger input elements and extracting the
3649 // sub sections we actually care about.
3650 unsigned SubScale = SubBitWidth / BitWidth;
3651 APInt SubDemandedElts =
3652 APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumElts / SubScale);
3653 Known2 = computeKnownBits(Op: N0, DemandedElts: SubDemandedElts, Depth: Depth + 1);
3654
3655 Known.setAllConflict();
3656 for (unsigned i = 0; i != NumElts; ++i)
3657 if (DemandedElts[i]) {
3658 unsigned Shifts = IsLE ? i : NumElts - 1 - i;
3659 unsigned Offset = (Shifts % SubScale) * BitWidth;
3660 Known = Known.intersectWith(RHS: Known2.extractBits(NumBits: BitWidth, BitPosition: Offset));
3661 // If we don't know any bits, early out.
3662 if (Known.isUnknown())
3663 break;
3664 }
3665 }
3666 break;
3667 }
3668 case ISD::AND:
3669 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3670 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3671
3672 Known &= Known2;
3673 break;
3674 case ISD::OR:
3675 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3676 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3677
3678 Known |= Known2;
3679 break;
3680 case ISD::XOR:
3681 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3682 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3683
3684 Known ^= Known2;
3685 break;
3686 case ISD::MUL: {
3687 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3688 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3689 bool SelfMultiply = Op.getOperand(i: 0) == Op.getOperand(i: 1);
3690 // TODO: SelfMultiply can be poison, but not undef.
3691 if (SelfMultiply)
3692 SelfMultiply &= isGuaranteedNotToBeUndefOrPoison(
3693 Op: Op.getOperand(i: 0), DemandedElts, Kind: UndefPoisonKind::UndefOrPoison,
3694 Depth: Depth + 1);
3695 Known = KnownBits::mul(LHS: Known, RHS: Known2, NoUndefSelfMultiply: SelfMultiply);
3696
3697 // If the multiplication is known not to overflow, the product of a number
3698 // with itself is non-negative. Only do this if we didn't already computed
3699 // the opposite value for the sign bit.
3700 if (Op->getFlags().hasNoSignedWrap() &&
3701 Op.getOperand(i: 0) == Op.getOperand(i: 1) &&
3702 !Known.isNegative())
3703 Known.makeNonNegative();
3704 break;
3705 }
3706 case ISD::MULHU: {
3707 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3708 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3709 Known = KnownBits::mulhu(LHS: Known, RHS: Known2);
3710 break;
3711 }
3712 case ISD::MULHS: {
3713 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3714 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3715 Known = KnownBits::mulhs(LHS: Known, RHS: Known2);
3716 break;
3717 }
3718 case ISD::ABDU: {
3719 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3720 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3721 Known = KnownBits::abdu(LHS: Known, RHS: Known2);
3722 break;
3723 }
3724 case ISD::ABDS: {
3725 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3726 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3727 Known = KnownBits::abds(LHS: Known, RHS: Known2);
3728 unsigned SignBits1 =
3729 ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3730 if (SignBits1 == 1)
3731 break;
3732 unsigned SignBits0 =
3733 ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3734 Known.Zero.setHighBits(std::min(a: SignBits0, b: SignBits1) - 1);
3735 break;
3736 }
3737 case ISD::UMUL_LOHI: {
3738 assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result");
3739 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3740 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3741 bool SelfMultiply = Op.getOperand(i: 0) == Op.getOperand(i: 1);
3742 if (Op.getResNo() == 0)
3743 Known = KnownBits::mul(LHS: Known, RHS: Known2, NoUndefSelfMultiply: SelfMultiply);
3744 else
3745 Known = KnownBits::mulhu(LHS: Known, RHS: Known2);
3746 break;
3747 }
3748 case ISD::SMUL_LOHI: {
3749 assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result");
3750 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3751 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3752 bool SelfMultiply = Op.getOperand(i: 0) == Op.getOperand(i: 1);
3753 if (Op.getResNo() == 0)
3754 Known = KnownBits::mul(LHS: Known, RHS: Known2, NoUndefSelfMultiply: SelfMultiply);
3755 else
3756 Known = KnownBits::mulhs(LHS: Known, RHS: Known2);
3757 break;
3758 }
3759 case ISD::AVGFLOORU: {
3760 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3761 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3762 Known = KnownBits::avgFloorU(LHS: Known, RHS: Known2);
3763 break;
3764 }
3765 case ISD::AVGCEILU: {
3766 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3767 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3768 Known = KnownBits::avgCeilU(LHS: Known, RHS: Known2);
3769 break;
3770 }
3771 case ISD::AVGFLOORS: {
3772 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3773 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3774 Known = KnownBits::avgFloorS(LHS: Known, RHS: Known2);
3775 break;
3776 }
3777 case ISD::AVGCEILS: {
3778 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3779 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3780 Known = KnownBits::avgCeilS(LHS: Known, RHS: Known2);
3781 break;
3782 }
3783 case ISD::SELECT:
3784 case ISD::VSELECT:
3785 Known = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth+1);
3786 // If we don't know any bits, early out.
3787 if (Known.isUnknown())
3788 break;
3789 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth+1);
3790
3791 // Only known if known in both the LHS and RHS.
3792 Known = Known.intersectWith(RHS: Known2);
3793 break;
3794 case ISD::SELECT_CC:
3795 Known = computeKnownBits(Op: Op.getOperand(i: 3), DemandedElts, Depth: Depth+1);
3796 // If we don't know any bits, early out.
3797 if (Known.isUnknown())
3798 break;
3799 Known2 = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth+1);
3800
3801 // Only known if known in both the LHS and RHS.
3802 Known = Known.intersectWith(RHS: Known2);
3803 break;
3804 case ISD::SMULO:
3805 case ISD::UMULO:
3806 if (Op.getResNo() != 1)
3807 break;
3808 // The boolean result conforms to getBooleanContents.
3809 // If we know the result of a setcc has the top bits zero, use this info.
3810 // We know that we have an integer-based boolean since these operations
3811 // are only available for integer.
3812 if (TLI->getBooleanContents(isVec: Op.getValueType().isVector(), isFloat: false) ==
3813 TargetLowering::ZeroOrOneBooleanContent &&
3814 BitWidth > 1)
3815 Known.Zero.setBitsFrom(1);
3816 break;
3817 case ISD::SETCC:
3818 case ISD::SETCCCARRY:
3819 case ISD::STRICT_FSETCC:
3820 case ISD::STRICT_FSETCCS: {
3821 unsigned OpNo = Op->isStrictFPOpcode() ? 1 : 0;
3822 // If we know the result of a setcc has the top bits zero, use this info.
3823 if (TLI->getBooleanContents(Type: Op.getOperand(i: OpNo).getValueType()) ==
3824 TargetLowering::ZeroOrOneBooleanContent &&
3825 BitWidth > 1)
3826 Known.Zero.setBitsFrom(1);
3827 break;
3828 }
3829 case ISD::SHL: {
3830 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3831 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3832
3833 bool NUW = Op->getFlags().hasNoUnsignedWrap();
3834 bool NSW = Op->getFlags().hasNoSignedWrap();
3835
3836 bool ShAmtNonZero = Known2.isNonZero();
3837
3838 Known = KnownBits::shl(LHS: Known, RHS: Known2, NUW, NSW, ShAmtNonZero);
3839
3840 // Minimum shift low bits are known zero.
3841 if (std::optional<unsigned> ShMinAmt =
3842 getValidMinimumShiftAmount(V: Op, DemandedElts, Depth: Depth + 1))
3843 Known.Zero.setLowBits(*ShMinAmt);
3844 break;
3845 }
3846 case ISD::SRL:
3847 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3848 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3849 Known = KnownBits::lshr(LHS: Known, RHS: Known2, /*ShAmtNonZero=*/false,
3850 Exact: Op->getFlags().hasExact());
3851
3852 // Minimum shift high bits are known zero.
3853 if (std::optional<unsigned> ShMinAmt =
3854 getValidMinimumShiftAmount(V: Op, DemandedElts, Depth: Depth + 1))
3855 Known.Zero.setHighBits(*ShMinAmt);
3856 break;
3857 case ISD::SRA:
3858 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3859 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3860 Known = KnownBits::ashr(LHS: Known, RHS: Known2, /*ShAmtNonZero=*/false,
3861 Exact: Op->getFlags().hasExact());
3862 break;
3863 case ISD::ROTL:
3864 case ISD::ROTR:
3865 if (ConstantSDNode *C =
3866 isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts)) {
3867 unsigned Amt = C->getAPIntValue().urem(RHS: BitWidth);
3868
3869 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3870
3871 // Canonicalize to ROTR.
3872 if (Opcode == ISD::ROTL && Amt != 0)
3873 Amt = BitWidth - Amt;
3874
3875 Known.Zero = Known.Zero.rotr(rotateAmt: Amt);
3876 Known.One = Known.One.rotr(rotateAmt: Amt);
3877 }
3878 break;
3879 case ISD::FSHL:
3880 case ISD::FSHR:
3881 if (ConstantSDNode *C = isConstOrConstSplat(N: Op.getOperand(i: 2), DemandedElts)) {
3882 unsigned Amt = C->getAPIntValue().urem(RHS: BitWidth);
3883
3884 // For fshl, 0-shift returns the 1st arg.
3885 // For fshr, 0-shift returns the 2nd arg.
3886 if (Amt == 0) {
3887 Known = computeKnownBits(Op: Op.getOperand(i: Opcode == ISD::FSHL ? 0 : 1),
3888 DemandedElts, Depth: Depth + 1);
3889 break;
3890 }
3891
3892 // fshl: (X << (Z % BW)) | (Y >> (BW - (Z % BW)))
3893 // fshr: (X << (BW - (Z % BW))) | (Y >> (Z % BW))
3894 const APInt ShAmt(BitWidth, Amt);
3895 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3896 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3897 Known = Opcode == ISD::FSHL ? KnownBits::fshl(LHS: Known, RHS: Known2, Amt: ShAmt)
3898 : KnownBits::fshr(LHS: Known, RHS: Known2, Amt: ShAmt);
3899 }
3900 break;
3901 case ISD::SHL_PARTS:
3902 case ISD::SRA_PARTS:
3903 case ISD::SRL_PARTS: {
3904 assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result");
3905
3906 // Collect lo/hi source values and concatenate.
3907 unsigned LoBits = Op.getOperand(i: 0).getScalarValueSizeInBits();
3908 unsigned HiBits = Op.getOperand(i: 1).getScalarValueSizeInBits();
3909 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3910 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3911 Known = Known2.concat(Lo: Known);
3912
3913 // Collect shift amount.
3914 Known2 = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth + 1);
3915
3916 if (Opcode == ISD::SHL_PARTS)
3917 Known = KnownBits::shl(LHS: Known, RHS: Known2);
3918 else if (Opcode == ISD::SRA_PARTS)
3919 Known = KnownBits::ashr(LHS: Known, RHS: Known2);
3920 else // if (Opcode == ISD::SRL_PARTS)
3921 Known = KnownBits::lshr(LHS: Known, RHS: Known2);
3922
3923 // TODO: Minimum shift low/high bits are known zero.
3924
3925 if (Op.getResNo() == 0)
3926 Known = Known.extractBits(NumBits: LoBits, BitPosition: 0);
3927 else
3928 Known = Known.extractBits(NumBits: HiBits, BitPosition: LoBits);
3929 break;
3930 }
3931 case ISD::SIGN_EXTEND_INREG: {
3932 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3933 EVT EVT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
3934 Known = Known.sextInReg(SrcBitWidth: EVT.getScalarSizeInBits());
3935 break;
3936 }
3937 case ISD::CTTZ:
3938 case ISD::CTTZ_ZERO_POISON: {
3939 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3940 // If we have a known 1, its position is our upper bound.
3941 unsigned PossibleTZ = Known2.countMaxTrailingZeros();
3942 unsigned LowBits = llvm::bit_width(Value: PossibleTZ);
3943 Known.Zero.setBitsFrom(LowBits);
3944 break;
3945 }
3946 case ISD::CTLZ:
3947 case ISD::CTLZ_ZERO_POISON: {
3948 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3949 // If we have a known 1, its position is our upper bound.
3950 unsigned PossibleLZ = Known2.countMaxLeadingZeros();
3951 unsigned LowBits = llvm::bit_width(Value: PossibleLZ);
3952 Known.Zero.setBitsFrom(LowBits);
3953 break;
3954 }
3955 case ISD::CTLS: {
3956 unsigned MinRedundantSignBits =
3957 ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1) - 1;
3958 ConstantRange Range(APInt(BitWidth, MinRedundantSignBits),
3959 APInt(BitWidth, BitWidth));
3960 Known = Range.toKnownBits();
3961 break;
3962 }
3963 case ISD::CTPOP: {
3964 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3965 // If we know some of the bits are zero, they can't be one.
3966 unsigned PossibleOnes = Known2.countMaxPopulation();
3967 Known.Zero.setBitsFrom(llvm::bit_width(Value: PossibleOnes));
3968 break;
3969 }
3970 case ISD::PARITY: {
3971 // Parity returns 0 everywhere but the LSB.
3972 Known.Zero.setBitsFrom(1);
3973 break;
3974 }
3975 case ISD::PDEP: {
3976 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3977 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3978 Known = KnownBits::pdep(Val: Known2, Mask: Known);
3979 break;
3980 }
3981 case ISD::PEXT: {
3982 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3983 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3984 Known = KnownBits::pext(Val: Known2, Mask: Known);
3985 break;
3986 }
3987 case ISD::CLMUL: {
3988 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3989 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3990 Known = KnownBits::clmul(LHS: Known, RHS: Known2);
3991 break;
3992 }
3993 case ISD::MGATHER:
3994 case ISD::MLOAD: {
3995 ISD::LoadExtType ETy =
3996 (Opcode == ISD::MGATHER)
3997 ? cast<MaskedGatherSDNode>(Val&: Op)->getExtensionType()
3998 : cast<MaskedLoadSDNode>(Val&: Op)->getExtensionType();
3999 if (ETy == ISD::ZEXTLOAD) {
4000 EVT MemVT = cast<MemSDNode>(Val&: Op)->getMemoryVT();
4001 KnownBits Known0(MemVT.getScalarSizeInBits());
4002 return Known0.zext(BitWidth);
4003 }
4004 break;
4005 }
4006 case ISD::LOAD: {
4007 LoadSDNode *LD = cast<LoadSDNode>(Val&: Op);
4008 const Constant *Cst = TLI->getTargetConstantFromLoad(LD);
4009 if (ISD::isNON_EXTLoad(N: LD) && Cst) {
4010 // Determine any common known bits from the loaded constant pool value.
4011 Type *CstTy = Cst->getType();
4012 if ((NumElts * BitWidth) == CstTy->getPrimitiveSizeInBits() &&
4013 !Op.getValueType().isScalableVector()) {
4014 // If its a vector splat, then we can (quickly) reuse the scalar path.
4015 // NOTE: We assume all elements match and none are UNDEF.
4016 if (CstTy->isVectorTy()) {
4017 if (const Constant *Splat = Cst->getSplatValue()) {
4018 Cst = Splat;
4019 CstTy = Cst->getType();
4020 }
4021 }
4022 // TODO - do we need to handle different bitwidths?
4023 if (CstTy->isVectorTy() && BitWidth == CstTy->getScalarSizeInBits()) {
4024 // Iterate across all vector elements finding common known bits.
4025 Known.setAllConflict();
4026 for (unsigned i = 0; i != NumElts; ++i) {
4027 if (!DemandedElts[i])
4028 continue;
4029 if (Constant *Elt = Cst->getAggregateElement(Elt: i)) {
4030 if (auto *CInt = dyn_cast<ConstantInt>(Val: Elt)) {
4031 const APInt &Value = CInt->getValue();
4032 Known.One &= Value;
4033 Known.Zero &= ~Value;
4034 continue;
4035 }
4036 if (auto *CFP = dyn_cast<ConstantFP>(Val: Elt)) {
4037 APInt Value = CFP->getValueAPF().bitcastToAPInt();
4038 Known.One &= Value;
4039 Known.Zero &= ~Value;
4040 continue;
4041 }
4042 }
4043 Known.One.clearAllBits();
4044 Known.Zero.clearAllBits();
4045 break;
4046 }
4047 } else if (BitWidth == CstTy->getPrimitiveSizeInBits()) {
4048 if (auto *CInt = dyn_cast<ConstantInt>(Val: Cst)) {
4049 Known = KnownBits::makeConstant(C: CInt->getValue());
4050 } else if (auto *CFP = dyn_cast<ConstantFP>(Val: Cst)) {
4051 Known =
4052 KnownBits::makeConstant(C: CFP->getValueAPF().bitcastToAPInt());
4053 }
4054 }
4055 }
4056 } else if (Op.getResNo() == 0) {
4057 unsigned ScalarMemorySize = LD->getMemoryVT().getScalarSizeInBits();
4058 KnownBits KnownScalarMemory(ScalarMemorySize);
4059 if (const MDNode *MD = LD->getRanges())
4060 computeKnownBitsFromRangeMetadata(Ranges: *MD, Known&: KnownScalarMemory);
4061
4062 // Extend the Known bits from memory to the size of the scalar result.
4063 if (ISD::isZEXTLoad(N: Op.getNode()))
4064 Known = KnownScalarMemory.zext(BitWidth);
4065 else if (ISD::isSEXTLoad(N: Op.getNode()))
4066 Known = KnownScalarMemory.sext(BitWidth);
4067 else if (ISD::isEXTLoad(N: Op.getNode()))
4068 Known = KnownScalarMemory.anyext(BitWidth);
4069 else
4070 Known = KnownScalarMemory;
4071 assert(Known.getBitWidth() == BitWidth);
4072 return Known;
4073 }
4074 break;
4075 }
4076 case ISD::ZERO_EXTEND_VECTOR_INREG: {
4077 if (Op.getValueType().isScalableVector())
4078 break;
4079 EVT InVT = Op.getOperand(i: 0).getValueType();
4080 APInt InDemandedElts = DemandedElts.zext(width: InVT.getVectorNumElements());
4081 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts: InDemandedElts, Depth: Depth + 1);
4082 Known = Known.zext(BitWidth);
4083 break;
4084 }
4085 case ISD::ZERO_EXTEND: {
4086 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4087 Known = Known.zext(BitWidth);
4088 break;
4089 }
4090 case ISD::SIGN_EXTEND_VECTOR_INREG: {
4091 if (Op.getValueType().isScalableVector())
4092 break;
4093 EVT InVT = Op.getOperand(i: 0).getValueType();
4094 APInt InDemandedElts = DemandedElts.zext(width: InVT.getVectorNumElements());
4095 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts: InDemandedElts, Depth: Depth + 1);
4096 // If the sign bit is known to be zero or one, then sext will extend
4097 // it to the top bits, else it will just zext.
4098 Known = Known.sext(BitWidth);
4099 break;
4100 }
4101 case ISD::SIGN_EXTEND: {
4102 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4103 // If the sign bit is known to be zero or one, then sext will extend
4104 // it to the top bits, else it will just zext.
4105 Known = Known.sext(BitWidth);
4106 break;
4107 }
4108 case ISD::ANY_EXTEND_VECTOR_INREG: {
4109 if (Op.getValueType().isScalableVector())
4110 break;
4111 EVT InVT = Op.getOperand(i: 0).getValueType();
4112 APInt InDemandedElts = DemandedElts.zext(width: InVT.getVectorNumElements());
4113 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts: InDemandedElts, Depth: Depth + 1);
4114 Known = Known.anyext(BitWidth);
4115 break;
4116 }
4117 case ISD::ANY_EXTEND: {
4118 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4119 Known = Known.anyext(BitWidth);
4120 break;
4121 }
4122 case ISD::TRUNCATE: {
4123 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4124 Known = Known.trunc(BitWidth);
4125 break;
4126 }
4127 case ISD::TRUNCATE_SSAT_S: {
4128 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4129 Known = Known.truncSSat(BitWidth);
4130 break;
4131 }
4132 case ISD::TRUNCATE_SSAT_U: {
4133 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4134 Known = Known.truncSSatU(BitWidth);
4135 break;
4136 }
4137 case ISD::TRUNCATE_USAT_U: {
4138 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4139 Known = Known.truncUSat(BitWidth);
4140 break;
4141 }
4142 case ISD::AssertZext: {
4143 EVT VT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
4144 APInt InMask = APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: VT.getSizeInBits());
4145 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4146 Known.Zero |= (~InMask);
4147 Known.One &= (~Known.Zero);
4148 break;
4149 }
4150 case ISD::AssertAlign: {
4151 unsigned LogOfAlign = Log2(A: cast<AssertAlignSDNode>(Val&: Op)->getAlign());
4152 assert(LogOfAlign != 0);
4153
4154 // TODO: Should use maximum with source
4155 // If a node is guaranteed to be aligned, set low zero bits accordingly as
4156 // well as clearing one bits.
4157 Known.Zero.setLowBits(LogOfAlign);
4158 Known.One.clearLowBits(loBits: LogOfAlign);
4159 break;
4160 }
4161 case ISD::AssertNoFPClass: {
4162 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4163
4164 FPClassTest NoFPClass =
4165 static_cast<FPClassTest>(Op.getConstantOperandVal(i: 1));
4166 const FPClassTest NegativeTestMask = fcNan | fcNegative;
4167 if ((NoFPClass & NegativeTestMask) == NegativeTestMask) {
4168 // Cannot be negative.
4169 Known.makeNonNegative();
4170 }
4171
4172 const FPClassTest PositiveTestMask = fcNan | fcPositive;
4173 if ((NoFPClass & PositiveTestMask) == PositiveTestMask) {
4174 // Cannot be positive.
4175 Known.makeNegative();
4176 }
4177
4178 break;
4179 }
4180 case ISD::FABS:
4181 // fabs clears the sign bit
4182 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4183 Known.Zero.setSignBit();
4184 Known.One.clearSignBit();
4185 break;
4186 case ISD::FGETSIGN:
4187 // All bits are zero except the low bit.
4188 Known.Zero.setBitsFrom(1);
4189 break;
4190 case ISD::ADD: {
4191 SDNodeFlags Flags = Op.getNode()->getFlags();
4192 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4193 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4194 bool SelfAdd = Op.getOperand(i: 0) == Op.getOperand(i: 1) &&
4195 isGuaranteedNotToBeUndefOrPoison(
4196 Op: Op.getOperand(i: 0), DemandedElts,
4197 Kind: UndefPoisonKind::UndefOrPoison, Depth: Depth + 1);
4198 Known = KnownBits::add(LHS: Known, RHS: Known2, NSW: Flags.hasNoSignedWrap(),
4199 NUW: Flags.hasNoUnsignedWrap(), SelfAdd);
4200 break;
4201 }
4202 case ISD::SUB: {
4203 SDNodeFlags Flags = Op.getNode()->getFlags();
4204 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4205 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4206 Known = KnownBits::sub(LHS: Known, RHS: Known2, NSW: Flags.hasNoSignedWrap(),
4207 NUW: Flags.hasNoUnsignedWrap());
4208 break;
4209 }
4210 case ISD::USUBO:
4211 case ISD::SSUBO:
4212 case ISD::USUBO_CARRY:
4213 case ISD::SSUBO_CARRY:
4214 if (Op.getResNo() == 1) {
4215 // If we know the result of a setcc has the top bits zero, use this info.
4216 if (TLI->getBooleanContents(Type: Op.getOperand(i: 0).getValueType()) ==
4217 TargetLowering::ZeroOrOneBooleanContent &&
4218 BitWidth > 1)
4219 Known.Zero.setBitsFrom(1);
4220 break;
4221 }
4222 [[fallthrough]];
4223 case ISD::SUBC: {
4224 assert(Op.getResNo() == 0 &&
4225 "We only compute knownbits for the difference here.");
4226
4227 // With USUBO_CARRY and SSUBO_CARRY a borrow bit may be added in.
4228 KnownBits Borrow(1);
4229 if (Opcode == ISD::USUBO_CARRY || Opcode == ISD::SSUBO_CARRY) {
4230 Borrow = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth + 1);
4231 // Borrow has bit width 1
4232 Borrow = Borrow.trunc(BitWidth: 1);
4233 } else {
4234 Borrow.setAllZero();
4235 }
4236
4237 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4238 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4239 Known = KnownBits::computeForSubBorrow(LHS: Known, RHS: Known2, Borrow);
4240 break;
4241 }
4242 case ISD::UADDO:
4243 case ISD::SADDO:
4244 case ISD::UADDO_CARRY:
4245 case ISD::SADDO_CARRY:
4246 if (Op.getResNo() == 1) {
4247 // If we know the result of a setcc has the top bits zero, use this info.
4248 if (TLI->getBooleanContents(Type: Op.getOperand(i: 0).getValueType()) ==
4249 TargetLowering::ZeroOrOneBooleanContent &&
4250 BitWidth > 1)
4251 Known.Zero.setBitsFrom(1);
4252 break;
4253 }
4254 [[fallthrough]];
4255 case ISD::ADDC:
4256 case ISD::ADDE: {
4257 assert(Op.getResNo() == 0 && "We only compute knownbits for the sum here.");
4258
4259 // With ADDE and UADDO_CARRY, a carry bit may be added in.
4260 KnownBits Carry(1);
4261 if (Opcode == ISD::ADDE)
4262 // Can't track carry from glue, set carry to unknown.
4263 Carry.resetAll();
4264 else if (Opcode == ISD::UADDO_CARRY || Opcode == ISD::SADDO_CARRY) {
4265 Carry = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth + 1);
4266 // Carry has bit width 1
4267 Carry = Carry.trunc(BitWidth: 1);
4268 } else {
4269 Carry.setAllZero();
4270 }
4271
4272 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4273 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4274 Known = KnownBits::computeForAddCarry(LHS: Known, RHS: Known2, Carry);
4275 break;
4276 }
4277 case ISD::UDIV: {
4278 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4279 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4280 Known = KnownBits::udiv(LHS: Known, RHS: Known2, Exact: Op->getFlags().hasExact());
4281 break;
4282 }
4283 case ISD::SDIV: {
4284 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4285 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4286 Known = KnownBits::sdiv(LHS: Known, RHS: Known2, Exact: Op->getFlags().hasExact());
4287 break;
4288 }
4289 case ISD::SREM: {
4290 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4291 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4292 Known = KnownBits::srem(LHS: Known, RHS: Known2);
4293 break;
4294 }
4295 case ISD::UREM: {
4296 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4297 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4298 Known = KnownBits::urem(LHS: Known, RHS: Known2);
4299 break;
4300 }
4301 case ISD::EXTRACT_ELEMENT: {
4302 Known = computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth+1);
4303 const unsigned Index = Op.getConstantOperandVal(i: 1);
4304 const unsigned EltBitWidth = Op.getValueSizeInBits();
4305
4306 Known = Known.extractBits(NumBits: EltBitWidth, BitPosition: Index * EltBitWidth);
4307 break;
4308 }
4309 case ISD::EXTRACT_VECTOR_ELT: {
4310 SDValue InVec = Op.getOperand(i: 0);
4311 SDValue EltNo = Op.getOperand(i: 1);
4312 EVT VecVT = InVec.getValueType();
4313 // computeKnownBits not yet implemented for scalable vectors.
4314 if (VecVT.isScalableVector())
4315 break;
4316 const unsigned EltBitWidth = VecVT.getScalarSizeInBits();
4317 const unsigned NumSrcElts = VecVT.getVectorNumElements();
4318
4319 // If BitWidth > EltBitWidth the value is anyext:ed. So we do not know
4320 // anything about the extended bits.
4321 if (BitWidth > EltBitWidth)
4322 Known = Known.trunc(BitWidth: EltBitWidth);
4323
4324 // If we know the element index, just demand that vector element, else for
4325 // an unknown element index, ignore DemandedElts and demand them all.
4326 APInt DemandedSrcElts = APInt::getAllOnes(numBits: NumSrcElts);
4327 auto *ConstEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
4328 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(RHS: NumSrcElts))
4329 DemandedSrcElts =
4330 APInt::getOneBitSet(numBits: NumSrcElts, BitNo: ConstEltNo->getZExtValue());
4331
4332 Known = computeKnownBits(Op: InVec, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
4333 if (BitWidth > EltBitWidth)
4334 Known = Known.anyext(BitWidth);
4335 break;
4336 }
4337 case ISD::BUILD_PAIR: {
4338 // Operand 0 is the low half and operand 1 the high half,
4339 // KnownBits::concat places its argument in the low bits.
4340 Known = computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
4341 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), Depth: Depth + 1);
4342 Known = Known2.concat(Lo: Known);
4343 break;
4344 }
4345 case ISD::INSERT_VECTOR_ELT: {
4346 if (Op.getValueType().isScalableVector())
4347 break;
4348
4349 // If we know the element index, split the demand between the
4350 // source vector and the inserted element, otherwise assume we need
4351 // the original demanded vector elements and the value.
4352 SDValue InVec = Op.getOperand(i: 0);
4353 SDValue InVal = Op.getOperand(i: 1);
4354 SDValue EltNo = Op.getOperand(i: 2);
4355 bool DemandedVal = true;
4356 APInt DemandedVecElts = DemandedElts;
4357 auto *CEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
4358 if (CEltNo && CEltNo->getAPIntValue().ult(RHS: NumElts)) {
4359 unsigned EltIdx = CEltNo->getZExtValue();
4360 DemandedVal = !!DemandedElts[EltIdx];
4361 DemandedVecElts.clearBit(BitPosition: EltIdx);
4362 }
4363 Known.setAllConflict();
4364 if (DemandedVal) {
4365 Known2 = computeKnownBits(Op: InVal, Depth: Depth + 1);
4366 Known = Known.intersectWith(RHS: Known2.zextOrTrunc(BitWidth));
4367 }
4368 if (!!DemandedVecElts) {
4369 Known2 = computeKnownBits(Op: InVec, DemandedElts: DemandedVecElts, Depth: Depth + 1);
4370 Known = Known.intersectWith(RHS: Known2);
4371 }
4372 break;
4373 }
4374 case ISD::BITREVERSE: {
4375 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4376 Known = Known2.reverseBits();
4377 break;
4378 }
4379 case ISD::BSWAP: {
4380 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4381 Known = Known2.byteSwap();
4382 break;
4383 }
4384 case ISD::ABS:
4385 case ISD::ABS_MIN_POISON: {
4386 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4387 Known = Known2.abs();
4388 Known.Zero.setHighBits(
4389 ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1) - 1);
4390 break;
4391 }
4392 case ISD::USUBSAT: {
4393 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4394 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4395 Known = KnownBits::usub_sat(LHS: Known, RHS: Known2);
4396 break;
4397 }
4398 case ISD::UMIN: {
4399 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4400 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4401 Known = KnownBits::umin(LHS: Known, RHS: Known2);
4402 break;
4403 }
4404 case ISD::UMAX: {
4405 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4406 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4407 Known = KnownBits::umax(LHS: Known, RHS: Known2);
4408 break;
4409 }
4410 case ISD::SMIN:
4411 case ISD::SMAX: {
4412 // If we have a clamp pattern, we know that the number of sign bits will be
4413 // the minimum of the clamp min/max range.
4414 bool IsMax = (Opcode == ISD::SMAX);
4415 ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr;
4416 if ((CstLow = isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts)))
4417 if (Op.getOperand(i: 0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX))
4418 CstHigh =
4419 isConstOrConstSplat(N: Op.getOperand(i: 0).getOperand(i: 1), DemandedElts);
4420 if (CstLow && CstHigh) {
4421 if (!IsMax)
4422 std::swap(a&: CstLow, b&: CstHigh);
4423
4424 const APInt &ValueLow = CstLow->getAPIntValue();
4425 const APInt &ValueHigh = CstHigh->getAPIntValue();
4426 if (ValueLow.sle(RHS: ValueHigh)) {
4427 unsigned LowSignBits = ValueLow.getNumSignBits();
4428 unsigned HighSignBits = ValueHigh.getNumSignBits();
4429 unsigned MinSignBits = std::min(a: LowSignBits, b: HighSignBits);
4430 if (ValueLow.isNegative() && ValueHigh.isNegative()) {
4431 Known.One.setHighBits(MinSignBits);
4432 break;
4433 }
4434 if (ValueLow.isNonNegative() && ValueHigh.isNonNegative()) {
4435 Known.Zero.setHighBits(MinSignBits);
4436 break;
4437 }
4438 }
4439 }
4440
4441 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4442 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4443 if (IsMax)
4444 Known = KnownBits::smax(LHS: Known, RHS: Known2);
4445 else
4446 Known = KnownBits::smin(LHS: Known, RHS: Known2);
4447
4448 // For SMAX, if CstLow is non-negative we know the result will be
4449 // non-negative and thus all sign bits are 0.
4450 // TODO: There's an equivalent of this for smin with negative constant for
4451 // known ones.
4452 if (IsMax && CstLow) {
4453 const APInt &ValueLow = CstLow->getAPIntValue();
4454 if (ValueLow.isNonNegative()) {
4455 unsigned SignBits = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
4456 Known.Zero.setHighBits(std::min(a: SignBits, b: ValueLow.getNumSignBits()));
4457 }
4458 }
4459
4460 break;
4461 }
4462 case ISD::UINT_TO_FP: {
4463 Known.makeNonNegative();
4464 break;
4465 }
4466 case ISD::SINT_TO_FP: {
4467 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4468 if (Known2.isNonNegative())
4469 Known.makeNonNegative();
4470 else if (Known2.isNegative())
4471 Known.makeNegative();
4472 break;
4473 }
4474 case ISD::FP_TO_UINT_SAT: {
4475 // FP_TO_UINT_SAT produces an unsigned value that fits in the saturating VT.
4476 EVT VT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
4477 Known.Zero |= APInt::getBitsSetFrom(numBits: BitWidth, loBit: VT.getScalarSizeInBits());
4478 break;
4479 }
4480 case ISD::ATOMIC_LOAD: {
4481 // If we are looking at the loaded value.
4482 if (Op.getResNo() == 0) {
4483 auto *AT = cast<AtomicSDNode>(Val&: Op);
4484 unsigned ScalarMemorySize = AT->getMemoryVT().getScalarSizeInBits();
4485 KnownBits KnownScalarMemory(ScalarMemorySize);
4486 if (const MDNode *MD = AT->getRanges())
4487 computeKnownBitsFromRangeMetadata(Ranges: *MD, Known&: KnownScalarMemory);
4488
4489 switch (AT->getExtensionType()) {
4490 case ISD::ZEXTLOAD:
4491 Known = KnownScalarMemory.zext(BitWidth);
4492 break;
4493 case ISD::SEXTLOAD:
4494 Known = KnownScalarMemory.sext(BitWidth);
4495 break;
4496 case ISD::EXTLOAD:
4497 switch (TLI->getExtendForAtomicOps()) {
4498 case ISD::ZERO_EXTEND:
4499 Known = KnownScalarMemory.zext(BitWidth);
4500 break;
4501 case ISD::SIGN_EXTEND:
4502 Known = KnownScalarMemory.sext(BitWidth);
4503 break;
4504 default:
4505 Known = KnownScalarMemory.anyext(BitWidth);
4506 break;
4507 }
4508 break;
4509 case ISD::NON_EXTLOAD:
4510 Known = KnownScalarMemory;
4511 break;
4512 }
4513 assert(Known.getBitWidth() == BitWidth);
4514 }
4515 break;
4516 }
4517 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
4518 if (Op.getResNo() == 1) {
4519 // The boolean result conforms to getBooleanContents.
4520 // If we know the result of a setcc has the top bits zero, use this info.
4521 // We know that we have an integer-based boolean since these operations
4522 // are only available for integer.
4523 if (TLI->getBooleanContents(isVec: Op.getValueType().isVector(), isFloat: false) ==
4524 TargetLowering::ZeroOrOneBooleanContent &&
4525 BitWidth > 1)
4526 Known.Zero.setBitsFrom(1);
4527 break;
4528 }
4529 [[fallthrough]];
4530 case ISD::ATOMIC_CMP_SWAP:
4531 case ISD::ATOMIC_SWAP:
4532 case ISD::ATOMIC_LOAD_ADD:
4533 case ISD::ATOMIC_LOAD_SUB:
4534 case ISD::ATOMIC_LOAD_AND:
4535 case ISD::ATOMIC_LOAD_CLR:
4536 case ISD::ATOMIC_LOAD_OR:
4537 case ISD::ATOMIC_LOAD_XOR:
4538 case ISD::ATOMIC_LOAD_NAND:
4539 case ISD::ATOMIC_LOAD_MIN:
4540 case ISD::ATOMIC_LOAD_MAX:
4541 case ISD::ATOMIC_LOAD_UMIN:
4542 case ISD::ATOMIC_LOAD_UMAX: {
4543 // If we are looking at the loaded value.
4544 if (Op.getResNo() == 0) {
4545 auto *AT = cast<AtomicSDNode>(Val&: Op);
4546 unsigned MemBits = AT->getMemoryVT().getScalarSizeInBits();
4547
4548 if (TLI->getExtendForAtomicOps() == ISD::ZERO_EXTEND)
4549 Known.Zero.setBitsFrom(MemBits);
4550 }
4551 break;
4552 }
4553 case ISD::FrameIndex:
4554 case ISD::TargetFrameIndex: {
4555 const MachineFunction &MF = getMachineFunction();
4556 int FrameIdx = cast<FrameIndexSDNode>(Val&: Op)->getIndex();
4557 TLI->computeKnownBitsForStackObjectPointer(
4558 Known, MF, Alignment: MF.getFrameInfo().getObjectAlign(ObjectIdx: FrameIdx));
4559 break;
4560 }
4561 case ISD::VP_LOAD_FF: {
4562 if (Op.getResNo() != 1)
4563 break;
4564 // The second result of vp.load.ff is an unsigned value that is less than or
4565 // equal to the EVL operand.
4566 KnownBits VLKB =
4567 computeKnownBits(Op: Op.getOperand(i: 3), DemandedElts, Depth: Depth + 1);
4568 // The new VL is also bounded by the largest vector length.
4569 EVT ResVT = Op->getValueType(ResNo: 0);
4570 auto ResKB = KnownBits::makeConstant(
4571 C: APInt(BitWidth, ResVT.getVectorMinNumElements()));
4572 if (ResVT.isScalableVector()) {
4573 const Function &F = getMachineFunction().getFunction();
4574 ResKB = KnownBits::mul(LHS: getVScaleRange(F: &F, BitWidth).toKnownBits(), RHS: ResKB);
4575 }
4576 Known.Zero.setHighBits(KnownBits::umin(LHS: VLKB, RHS: ResKB).countMinLeadingZeros());
4577 break;
4578 }
4579
4580 default:
4581 if (Opcode < ISD::BUILTIN_OP_END)
4582 break;
4583 [[fallthrough]];
4584 case ISD::INTRINSIC_WO_CHAIN:
4585 case ISD::INTRINSIC_W_CHAIN:
4586 case ISD::INTRINSIC_VOID:
4587 // Allow the target to implement this method for its nodes.
4588 TLI->computeKnownBitsForTargetNode(Op, Known, DemandedElts, DAG: *this, Depth);
4589 break;
4590 }
4591
4592 return Known;
4593}
4594
4595/// Convert ConstantRange OverflowResult into SelectionDAG::OverflowKind.
4596static SelectionDAG::OverflowKind mapOverflowResult(ConstantRange::OverflowResult OR) {
4597 switch (OR) {
4598 case ConstantRange::OverflowResult::MayOverflow:
4599 return SelectionDAG::OFK_Sometime;
4600 case ConstantRange::OverflowResult::AlwaysOverflowsLow:
4601 case ConstantRange::OverflowResult::AlwaysOverflowsHigh:
4602 return SelectionDAG::OFK_Always;
4603 case ConstantRange::OverflowResult::NeverOverflows:
4604 return SelectionDAG::OFK_Never;
4605 }
4606 llvm_unreachable("Unknown OverflowResult");
4607}
4608
4609SelectionDAG::OverflowKind
4610SelectionDAG::computeOverflowForSignedAdd(SDValue N0, SDValue N1) const {
4611 // X + 0 never overflow
4612 if (isNullConstant(V: N1))
4613 return OFK_Never;
4614
4615 // If both operands each have at least two sign bits, the addition
4616 // cannot overflow.
4617 if (ComputeNumSignBits(Op: N0) > 1 && ComputeNumSignBits(Op: N1) > 1)
4618 return OFK_Never;
4619
4620 // TODO: Add ConstantRange::signedAddMayOverflow handling.
4621 return OFK_Sometime;
4622}
4623
4624SelectionDAG::OverflowKind
4625SelectionDAG::computeOverflowForUnsignedAdd(SDValue N0, SDValue N1) const {
4626 // X + 0 never overflow
4627 if (isNullConstant(V: N1))
4628 return OFK_Never;
4629
4630 // mulhi + 1 never overflow
4631 KnownBits N1Known = computeKnownBits(Op: N1);
4632 if (N0.getOpcode() == ISD::UMUL_LOHI && N0.getResNo() == 1 &&
4633 N1Known.getMaxValue().ult(RHS: 2))
4634 return OFK_Never;
4635
4636 KnownBits N0Known = computeKnownBits(Op: N0);
4637 if (N1.getOpcode() == ISD::UMUL_LOHI && N1.getResNo() == 1 &&
4638 N0Known.getMaxValue().ult(RHS: 2))
4639 return OFK_Never;
4640
4641 // Fallback to ConstantRange::unsignedAddMayOverflow handling.
4642 ConstantRange N0Range = ConstantRange::fromKnownBits(Known: N0Known, IsSigned: false);
4643 ConstantRange N1Range = ConstantRange::fromKnownBits(Known: N1Known, IsSigned: false);
4644 return mapOverflowResult(OR: N0Range.unsignedAddMayOverflow(Other: N1Range));
4645}
4646
4647SelectionDAG::OverflowKind
4648SelectionDAG::computeOverflowForSignedSub(SDValue N0, SDValue N1) const {
4649 // X - 0 never overflow
4650 if (isNullConstant(V: N1))
4651 return OFK_Never;
4652
4653 // If both operands each have at least two sign bits, the subtraction
4654 // cannot overflow.
4655 if (ComputeNumSignBits(Op: N0) > 1 && ComputeNumSignBits(Op: N1) > 1)
4656 return OFK_Never;
4657
4658 KnownBits N0Known = computeKnownBits(Op: N0);
4659 KnownBits N1Known = computeKnownBits(Op: N1);
4660 ConstantRange N0Range = ConstantRange::fromKnownBits(Known: N0Known, IsSigned: true);
4661 ConstantRange N1Range = ConstantRange::fromKnownBits(Known: N1Known, IsSigned: true);
4662 return mapOverflowResult(OR: N0Range.signedSubMayOverflow(Other: N1Range));
4663}
4664
4665SelectionDAG::OverflowKind
4666SelectionDAG::computeOverflowForUnsignedSub(SDValue N0, SDValue N1) const {
4667 // X - 0 never overflow
4668 if (isNullConstant(V: N1))
4669 return OFK_Never;
4670
4671 ConstantRange N0Range =
4672 computeConstantRangeIncludingKnownBits(Op: N0, /*ForSigned=*/false);
4673 ConstantRange N1Range =
4674 computeConstantRangeIncludingKnownBits(Op: N1, /*ForSigned=*/false);
4675 return mapOverflowResult(OR: N0Range.unsignedSubMayOverflow(Other: N1Range));
4676}
4677
4678SelectionDAG::OverflowKind
4679SelectionDAG::computeOverflowForUnsignedMul(SDValue N0, SDValue N1) const {
4680 // X * 0 and X * 1 never overflow.
4681 if (isNullConstant(V: N1) || isOneConstant(V: N1))
4682 return OFK_Never;
4683
4684 ConstantRange N0Range = computeConstantRangeIncludingKnownBits(Op: N0, ForSigned: false);
4685 ConstantRange N1Range = computeConstantRangeIncludingKnownBits(Op: N1, ForSigned: false);
4686 return mapOverflowResult(OR: N0Range.unsignedMulMayOverflow(Other: N1Range));
4687}
4688
4689SelectionDAG::OverflowKind
4690SelectionDAG::computeOverflowForSignedMul(SDValue N0, SDValue N1) const {
4691 // X * 0 and X * 1 never overflow.
4692 if (isNullConstant(V: N1) || isOneConstant(V: N1))
4693 return OFK_Never;
4694
4695 // Get the size of the result.
4696 unsigned BitWidth = N0.getScalarValueSizeInBits();
4697
4698 // Sum of the sign bits.
4699 unsigned SignBits = ComputeNumSignBits(Op: N0) + ComputeNumSignBits(Op: N1);
4700
4701 // If we have enough sign bits, then there's no overflow.
4702 if (SignBits > BitWidth + 1)
4703 return OFK_Never;
4704
4705 if (SignBits == BitWidth + 1) {
4706 // The overflow occurs when the true multiplication of the
4707 // the operands is the minimum negative number.
4708 KnownBits N0Known = computeKnownBits(Op: N0);
4709 KnownBits N1Known = computeKnownBits(Op: N1);
4710 // If one of the operands is non-negative, then there's no
4711 // overflow.
4712 if (N0Known.isNonNegative() || N1Known.isNonNegative())
4713 return OFK_Never;
4714 }
4715
4716 return OFK_Sometime;
4717}
4718
4719ConstantRange SelectionDAG::computeConstantRange(SDValue Op, bool ForSigned,
4720 unsigned Depth) const {
4721 APInt DemandedElts = getDemandAllEltsMask(V: Op);
4722 return computeConstantRange(Op, DemandedElts, ForSigned, Depth);
4723}
4724
4725ConstantRange SelectionDAG::computeConstantRange(SDValue Op,
4726 const APInt &DemandedElts,
4727 bool ForSigned,
4728 unsigned Depth) const {
4729 EVT VT = Op.getValueType();
4730 unsigned BitWidth = VT.getScalarSizeInBits();
4731
4732 if (Depth >= MaxRecursionDepth)
4733 return ConstantRange::getFull(BitWidth);
4734
4735 if (ConstantSDNode *C = isConstOrConstSplat(N: Op, DemandedElts))
4736 return ConstantRange(C->getAPIntValue());
4737
4738 unsigned Opcode = Op.getOpcode();
4739 switch (Opcode) {
4740 case ISD::VSCALE: {
4741 const Function &F = getMachineFunction().getFunction();
4742 const APInt &Multiplier = Op.getConstantOperandAPInt(i: 0);
4743 return getVScaleRange(F: &F, BitWidth).multiply(Other: Multiplier);
4744 }
4745 default:
4746 break;
4747 }
4748
4749 return ConstantRange::getFull(BitWidth);
4750}
4751
4752ConstantRange
4753SelectionDAG::computeConstantRangeIncludingKnownBits(SDValue Op, bool ForSigned,
4754 unsigned Depth) const {
4755 APInt DemandedElts = getDemandAllEltsMask(V: Op);
4756 return computeConstantRangeIncludingKnownBits(Op, DemandedElts, ForSigned,
4757 Depth);
4758}
4759
4760ConstantRange SelectionDAG::computeConstantRangeIncludingKnownBits(
4761 SDValue Op, const APInt &DemandedElts, bool ForSigned,
4762 unsigned Depth) const {
4763 KnownBits Known = computeKnownBits(Op, DemandedElts, Depth);
4764 ConstantRange CR1 = ConstantRange::fromKnownBits(Known, IsSigned: ForSigned);
4765 ConstantRange CR2 = computeConstantRange(Op, DemandedElts, ForSigned, Depth);
4766 ConstantRange::PreferredRangeType RangeType =
4767 ForSigned ? ConstantRange::Signed : ConstantRange::Unsigned;
4768 return CR1.intersectWith(CR: CR2, Type: RangeType);
4769}
4770
4771bool SelectionDAG::isKnownToBeAPowerOfTwo(SDValue Val, bool OrZero,
4772 unsigned Depth) const {
4773 APInt DemandedElts = getDemandAllEltsMask(V: Val);
4774 return isKnownToBeAPowerOfTwo(Val, DemandedElts, OrZero, Depth);
4775}
4776
4777bool SelectionDAG::isKnownToBeAPowerOfTwo(SDValue Val,
4778 const APInt &DemandedElts,
4779 bool OrZero, unsigned Depth) const {
4780 if (Depth >= MaxRecursionDepth)
4781 return false; // Limit search depth.
4782
4783 EVT OpVT = Val.getValueType();
4784 unsigned BitWidth = OpVT.getScalarSizeInBits();
4785 [[maybe_unused]] unsigned NumElts = DemandedElts.getBitWidth();
4786 assert((!OpVT.isScalableVector() || NumElts == 1) &&
4787 "DemandedElts for scalable vectors must be 1 to represent all lanes");
4788 assert(
4789 (!OpVT.isFixedLengthVector() || NumElts == OpVT.getVectorNumElements()) &&
4790 "Unexpected vector size");
4791
4792 auto IsPowerOfTwoOrZero = [BitWidth, OrZero](const ConstantSDNode *C) {
4793 APInt V = C->getAPIntValue().zextOrTrunc(width: BitWidth);
4794 return (OrZero && V.isZero()) || V.isPowerOf2();
4795 };
4796
4797 // Is the constant a known power of 2 or zero?
4798 if (ISD::matchUnaryPredicate(Op: Val, DemandedElts, Match: IsPowerOfTwoOrZero,
4799 /*AllowUndefs=*/false, /*AllowTruncation=*/true))
4800 return true;
4801
4802 switch (Val.getOpcode()) {
4803 case ISD::EXTRACT_VECTOR_ELT: {
4804 SDValue InVec = Val.getOperand(i: 0);
4805 SDValue EltNo = Val.getOperand(i: 1);
4806 EVT VecVT = InVec.getValueType();
4807
4808 // Skip scalable vectors or implicit extensions.
4809 if (VecVT.isScalableVector() ||
4810 OpVT.getScalarSizeInBits() != VecVT.getScalarSizeInBits())
4811 break;
4812
4813 // If we know the element index, just demand that vector element, else for
4814 // an unknown element index, ignore DemandedElts and demand them all.
4815 const unsigned NumSrcElts = VecVT.getVectorNumElements();
4816 auto *ConstEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
4817 APInt DemandedSrcElts =
4818 ConstEltNo && ConstEltNo->getAPIntValue().ult(RHS: NumSrcElts)
4819 ? APInt::getOneBitSet(numBits: NumSrcElts, BitNo: ConstEltNo->getZExtValue())
4820 : APInt::getAllOnes(numBits: NumSrcElts);
4821 return isKnownToBeAPowerOfTwo(Val: InVec, DemandedElts: DemandedSrcElts, OrZero, Depth: Depth + 1);
4822 }
4823
4824 case ISD::AND: {
4825 // Looking for `x & -x` pattern:
4826 // If x == 0:
4827 // x & -x -> 0
4828 // If x != 0:
4829 // x & -x -> non-zero pow2
4830 // so if we find the pattern return whether we know `x` is non-zero.
4831 SDValue X, Z;
4832 if (sd_match(N: Val, P: m_And(L: m_Value(N&: X), R: m_Neg(V: m_Deferred(V&: X)))) ||
4833 (sd_match(N: Val, P: m_And(L: m_Value(N&: X), R: m_Sub(L: m_Value(N&: Z), R: m_Deferred(V&: X)))) &&
4834 MaskedVectorIsZero(V: Z, DemandedElts, Depth: Depth + 1)))
4835 return OrZero || isKnownNeverZero(Op: X, DemandedElts, Depth);
4836 break;
4837 }
4838
4839 case ISD::SHL: {
4840 // A left-shift of a constant one will have exactly one bit set because
4841 // shifting the bit off the end is undefined.
4842 auto *C = isConstOrConstSplat(N: Val.getOperand(i: 0), DemandedElts);
4843 if (C && C->getAPIntValue() == 1)
4844 return true;
4845 return (OrZero || isKnownNeverZero(Op: Val, DemandedElts, Depth)) &&
4846 isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), DemandedElts, OrZero,
4847 Depth: Depth + 1);
4848 }
4849
4850 case ISD::SRL: {
4851 // A logical right-shift of a constant sign-bit will have exactly
4852 // one bit set.
4853 auto *C = isConstOrConstSplat(N: Val.getOperand(i: 0), DemandedElts);
4854 if (C && C->getAPIntValue().isSignMask())
4855 return true;
4856 return (OrZero || isKnownNeverZero(Op: Val, DemandedElts, Depth)) &&
4857 isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), DemandedElts, OrZero,
4858 Depth: Depth + 1);
4859 }
4860
4861 case ISD::TRUNCATE:
4862 return (OrZero || isKnownNeverZero(Op: Val, DemandedElts, Depth)) &&
4863 isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), DemandedElts, OrZero,
4864 Depth: Depth + 1);
4865
4866 case ISD::ROTL:
4867 case ISD::ROTR:
4868 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), DemandedElts, OrZero,
4869 Depth: Depth + 1);
4870 case ISD::BSWAP:
4871 case ISD::BITREVERSE:
4872 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), DemandedElts, OrZero,
4873 Depth: Depth + 1);
4874
4875 case ISD::SMIN:
4876 case ISD::SMAX:
4877 case ISD::UMIN:
4878 case ISD::UMAX:
4879 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 1), DemandedElts, OrZero,
4880 Depth: Depth + 1) &&
4881 isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), DemandedElts, OrZero,
4882 Depth: Depth + 1);
4883
4884 case ISD::SELECT:
4885 case ISD::VSELECT:
4886 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 2), DemandedElts, OrZero,
4887 Depth: Depth + 1) &&
4888 isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 1), DemandedElts, OrZero,
4889 Depth: Depth + 1);
4890
4891 case ISD::ZERO_EXTEND:
4892 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), DemandedElts, OrZero,
4893 Depth: Depth + 1);
4894
4895 case ISD::VSCALE:
4896 // vscale(power-of-two) is a power-of-two
4897 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), /*OrZero=*/false,
4898 Depth: Depth + 1);
4899
4900 case ISD::VECTOR_SHUFFLE: {
4901 assert(!Val.getValueType().isScalableVector());
4902 // Demanded elements with undef shuffle mask elements are unknown
4903 // - we cannot guarantee they are a power of two, so return false.
4904 APInt DemandedLHS, DemandedRHS;
4905 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val);
4906 assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
4907 if (!getShuffleDemandedElts(SrcWidth: NumElts, Mask: SVN->getMask(), DemandedElts,
4908 DemandedLHS, DemandedRHS))
4909 return false;
4910
4911 // All demanded elements from LHS must be known power of two.
4912 if (!!DemandedLHS && !isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), DemandedElts: DemandedLHS,
4913 OrZero, Depth: Depth + 1))
4914 return false;
4915
4916 // All demanded elements from RHS must be known power of two.
4917 if (!!DemandedRHS && !isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 1), DemandedElts: DemandedRHS,
4918 OrZero, Depth: Depth + 1))
4919 return false;
4920
4921 return true;
4922 }
4923 }
4924
4925 // More could be done here, though the above checks are enough
4926 // to handle some common cases.
4927 return false;
4928}
4929
4930bool SelectionDAG::isKnownToBeAPowerOfTwoFP(SDValue Val, unsigned Depth) const {
4931 if (ConstantFPSDNode *C1 = isConstOrConstSplatFP(N: Val, AllowUndefs: true))
4932 return C1->getValueAPF().getExactLog2Abs() >= 0;
4933
4934 if (Val.getOpcode() == ISD::UINT_TO_FP || Val.getOpcode() == ISD::SINT_TO_FP)
4935 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), OrZero: Depth + 1);
4936
4937 return false;
4938}
4939
4940unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const {
4941 APInt DemandedElts = getDemandAllEltsMask(V: Op);
4942 return ComputeNumSignBits(Op, DemandedElts, Depth);
4943}
4944
4945unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, const APInt &DemandedElts,
4946 unsigned Depth) const {
4947 EVT VT = Op.getValueType();
4948 assert((VT.isInteger() || VT.isFloatingPoint()) && "Invalid VT!");
4949 unsigned VTBits = VT.getScalarSizeInBits();
4950 unsigned NumElts = DemandedElts.getBitWidth();
4951 unsigned Tmp, Tmp2;
4952 unsigned FirstAnswer = 1;
4953
4954 assert((!VT.isScalableVector() || NumElts == 1) &&
4955 "DemandedElts for scalable vectors must be 1 to represent all lanes");
4956
4957 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op)) {
4958 const APInt &Val = C->getAPIntValue();
4959 return Val.getNumSignBits();
4960 }
4961
4962 if (Depth >= MaxRecursionDepth)
4963 return 1; // Limit search depth.
4964
4965 if (!DemandedElts)
4966 return 1; // No demanded elts, better to assume we don't know anything.
4967
4968 unsigned Opcode = Op.getOpcode();
4969 switch (Opcode) {
4970 default: break;
4971 case ISD::AssertSext:
4972 Tmp = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT().getSizeInBits();
4973 return VTBits-Tmp+1;
4974 case ISD::AssertZext:
4975 Tmp = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT().getSizeInBits();
4976 return VTBits-Tmp;
4977 case ISD::FREEZE:
4978 if (isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 0), DemandedElts,
4979 Kind: UndefPoisonKind::UndefOrPoison))
4980 return ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4981 break;
4982 case ISD::MERGE_VALUES:
4983 return ComputeNumSignBits(Op: Op.getOperand(i: Op.getResNo()), DemandedElts,
4984 Depth: Depth + 1);
4985 case ISD::SPLAT_VECTOR: {
4986 // Check if the sign bits of source go down as far as the truncated value.
4987 unsigned NumSrcBits = Op.getOperand(i: 0).getValueSizeInBits();
4988 unsigned NumSrcSignBits = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
4989 if (NumSrcSignBits > (NumSrcBits - VTBits))
4990 return NumSrcSignBits - (NumSrcBits - VTBits);
4991 break;
4992 }
4993 case ISD::BUILD_VECTOR:
4994 assert(!VT.isScalableVector());
4995 Tmp = VTBits;
4996 for (unsigned i = 0, e = Op.getNumOperands(); (i < e) && (Tmp > 1); ++i) {
4997 if (!DemandedElts[i])
4998 continue;
4999
5000 SDValue SrcOp = Op.getOperand(i);
5001 // BUILD_VECTOR can implicitly truncate sources, we handle this specially
5002 // for constant nodes to ensure we only look at the sign bits.
5003 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: SrcOp)) {
5004 APInt T = C->getAPIntValue().trunc(width: VTBits);
5005 Tmp2 = T.getNumSignBits();
5006 } else if (SrcOp.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
5007 SrcOp.getOperand(i: 0).getScalarValueSizeInBits() >= VTBits) {
5008 // EXTRACT_VECTOR_ELT can extend the value with high bits undefined. If
5009 // this BUILD_VECTOR truncates those undefined bits we can just look
5010 // through the SrcOp and query the vector directly.
5011 SDValue InVec = SrcOp.getOperand(i: 0);
5012 EVT InVecVT = InVec.getValueType();
5013
5014 APInt DemandedSrcElts;
5015 if (InVecVT.isScalableVector())
5016 // Demand all elements.
5017 DemandedSrcElts = APInt(1, 1);
5018 else {
5019 unsigned NumSrcElts = InVecVT.getVectorNumElements();
5020 auto *ConstEltNo = dyn_cast<ConstantSDNode>(Val: SrcOp.getOperand(i: 1));
5021 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(RHS: NumSrcElts))
5022 DemandedSrcElts =
5023 APInt::getOneBitSet(numBits: NumSrcElts, BitNo: ConstEltNo->getZExtValue());
5024 else
5025 DemandedSrcElts = APInt::getAllOnes(numBits: NumSrcElts);
5026 }
5027
5028 Tmp2 = ComputeNumSignBits(Op: InVec, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
5029 unsigned ExtraBits = InVec.getScalarValueSizeInBits() - VTBits;
5030 if (ExtraBits)
5031 Tmp2 = (Tmp2 > ExtraBits ? Tmp2 - ExtraBits : 1);
5032 } else {
5033 Tmp2 = ComputeNumSignBits(Op: SrcOp, Depth: Depth + 1);
5034
5035 if (SrcOp.getValueSizeInBits() != VTBits) {
5036 assert(SrcOp.getValueSizeInBits() > VTBits &&
5037 "Expected BUILD_VECTOR implicit truncation");
5038 unsigned ExtraBits = SrcOp.getValueSizeInBits() - VTBits;
5039 Tmp2 = (Tmp2 > ExtraBits ? Tmp2 - ExtraBits : 1);
5040 }
5041 }
5042 Tmp = std::min(a: Tmp, b: Tmp2);
5043 }
5044 return Tmp;
5045
5046 case ISD::VECTOR_COMPRESS: {
5047 SDValue Vec = Op.getOperand(i: 0);
5048 SDValue PassThru = Op.getOperand(i: 2);
5049 Tmp = ComputeNumSignBits(Op: PassThru, DemandedElts, Depth: Depth + 1);
5050 if (Tmp == 1)
5051 return 1;
5052 Tmp2 = ComputeNumSignBits(Op: Vec, Depth: Depth + 1);
5053 Tmp = std::min(a: Tmp, b: Tmp2);
5054 return Tmp;
5055 }
5056
5057 case ISD::VECTOR_SHUFFLE: {
5058 // Collect the minimum number of sign bits that are shared by every vector
5059 // element referenced by the shuffle.
5060 APInt DemandedLHS, DemandedRHS;
5061 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
5062 assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
5063 if (!getShuffleDemandedElts(SrcWidth: NumElts, Mask: SVN->getMask(), DemandedElts,
5064 DemandedLHS, DemandedRHS))
5065 return 1;
5066
5067 Tmp = std::numeric_limits<unsigned>::max();
5068 if (!!DemandedLHS)
5069 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts: DemandedLHS, Depth: Depth + 1);
5070 if (!!DemandedRHS) {
5071 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts: DemandedRHS, Depth: Depth + 1);
5072 Tmp = std::min(a: Tmp, b: Tmp2);
5073 }
5074 // If we don't know anything, early out and try computeKnownBits fall-back.
5075 if (Tmp == 1)
5076 break;
5077 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
5078 return Tmp;
5079 }
5080
5081 case ISD::BITCAST: {
5082 if (VT.isScalableVector())
5083 break;
5084 SDValue N0 = Op.getOperand(i: 0);
5085 EVT SrcVT = N0.getValueType();
5086 unsigned SrcBits = SrcVT.getScalarSizeInBits();
5087
5088 // Ignore bitcasts from unsupported types..
5089 if (!(SrcVT.isInteger() || SrcVT.isFloatingPoint()))
5090 break;
5091
5092 // Fast handling of 'identity' bitcasts.
5093 if (VTBits == SrcBits)
5094 return ComputeNumSignBits(Op: N0, DemandedElts, Depth: Depth + 1);
5095
5096 bool IsLE = getDataLayout().isLittleEndian();
5097
5098 // Bitcast 'large element' scalar/vector to 'small element' vector.
5099 if ((SrcBits % VTBits) == 0) {
5100 assert(VT.isVector() && "Expected bitcast to vector");
5101
5102 unsigned Scale = SrcBits / VTBits;
5103 APInt SrcDemandedElts =
5104 APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumElts / Scale);
5105
5106 // Fast case - sign splat can be simply split across the small elements.
5107 Tmp = ComputeNumSignBits(Op: N0, DemandedElts: SrcDemandedElts, Depth: Depth + 1);
5108 if (Tmp == SrcBits)
5109 return VTBits;
5110
5111 // Slow case - determine how far the sign extends into each sub-element.
5112 Tmp2 = VTBits;
5113 for (unsigned i = 0; i != NumElts; ++i)
5114 if (DemandedElts[i]) {
5115 unsigned SubOffset = i % Scale;
5116 SubOffset = (IsLE ? ((Scale - 1) - SubOffset) : SubOffset);
5117 SubOffset = SubOffset * VTBits;
5118 if (Tmp <= SubOffset)
5119 return 1;
5120 Tmp2 = std::min(a: Tmp2, b: Tmp - SubOffset);
5121 }
5122 return Tmp2;
5123 }
5124 break;
5125 }
5126
5127 case ISD::FP_TO_SINT_SAT:
5128 // FP_TO_SINT_SAT produces a signed value that fits in the saturating VT.
5129 Tmp = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT().getScalarSizeInBits();
5130 return VTBits - Tmp + 1;
5131 case ISD::SIGN_EXTEND:
5132 Tmp = VTBits - Op.getOperand(i: 0).getScalarValueSizeInBits();
5133 return ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth+1) + Tmp;
5134 case ISD::SIGN_EXTEND_INREG:
5135 // Max of the input and what this extends.
5136 Tmp = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT().getScalarSizeInBits();
5137 Tmp = VTBits-Tmp+1;
5138 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth+1);
5139 return std::max(a: Tmp, b: Tmp2);
5140 case ISD::SIGN_EXTEND_VECTOR_INREG: {
5141 if (VT.isScalableVector())
5142 break;
5143 SDValue Src = Op.getOperand(i: 0);
5144 EVT SrcVT = Src.getValueType();
5145 APInt DemandedSrcElts = DemandedElts.zext(width: SrcVT.getVectorNumElements());
5146 Tmp = VTBits - SrcVT.getScalarSizeInBits();
5147 return ComputeNumSignBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth+1) + Tmp;
5148 }
5149 case ISD::SRA:
5150 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5151 // SRA X, C -> adds C sign bits.
5152 if (std::optional<unsigned> ShAmt =
5153 getValidMinimumShiftAmount(V: Op, DemandedElts, Depth: Depth + 1))
5154 Tmp = std::min(a: Tmp + *ShAmt, b: VTBits);
5155 return Tmp;
5156 case ISD::SHL:
5157 if (std::optional<ConstantRange> ShAmtRange =
5158 getValidShiftAmountRange(V: Op, DemandedElts, Depth: Depth + 1)) {
5159 unsigned MaxShAmt = ShAmtRange->getUnsignedMax().getZExtValue();
5160 unsigned MinShAmt = ShAmtRange->getUnsignedMin().getZExtValue();
5161 // Try to look through ZERO/SIGN/ANY_EXTEND. If all extended bits are
5162 // shifted out, then we can compute the number of sign bits for the
5163 // operand being extended. A future improvement could be to pass along the
5164 // "shifted left by" information in the recursive calls to
5165 // ComputeKnownSignBits. Allowing us to handle this more generically.
5166 if (ISD::isExtOpcode(Opcode: Op.getOperand(i: 0).getOpcode())) {
5167 SDValue Ext = Op.getOperand(i: 0);
5168 EVT ExtVT = Ext.getValueType();
5169 SDValue Extendee = Ext.getOperand(i: 0);
5170 EVT ExtendeeVT = Extendee.getValueType();
5171 unsigned SizeDifference =
5172 ExtVT.getScalarSizeInBits() - ExtendeeVT.getScalarSizeInBits();
5173 if (SizeDifference <= MinShAmt) {
5174 Tmp = SizeDifference +
5175 ComputeNumSignBits(Op: Extendee, DemandedElts, Depth: Depth + 1);
5176 if (MaxShAmt < Tmp)
5177 return Tmp - MaxShAmt;
5178 }
5179 }
5180 // shl destroys sign bits, ensure it doesn't shift out all sign bits.
5181 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5182 if (MaxShAmt < Tmp)
5183 return Tmp - MaxShAmt;
5184 }
5185 break;
5186 case ISD::AND:
5187 case ISD::OR:
5188 case ISD::XOR: // NOT is handled here.
5189 // Logical binary ops preserve the number of sign bits at the worst.
5190 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth+1);
5191 if (Tmp != 1) {
5192 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth+1);
5193 FirstAnswer = std::min(a: Tmp, b: Tmp2);
5194 // We computed what we know about the sign bits as our first
5195 // answer. Now proceed to the generic code that uses
5196 // computeKnownBits, and pick whichever answer is better.
5197 }
5198 break;
5199
5200 case ISD::SELECT:
5201 case ISD::VSELECT:
5202 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth+1);
5203 if (Tmp == 1) return 1; // Early out.
5204 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth+1);
5205 return std::min(a: Tmp, b: Tmp2);
5206 case ISD::SELECT_CC:
5207 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth+1);
5208 if (Tmp == 1) return 1; // Early out.
5209 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 3), DemandedElts, Depth: Depth+1);
5210 return std::min(a: Tmp, b: Tmp2);
5211
5212 case ISD::SMIN:
5213 case ISD::SMAX: {
5214 // If we have a clamp pattern, we know that the number of sign bits will be
5215 // the minimum of the clamp min/max range.
5216 bool IsMax = (Opcode == ISD::SMAX);
5217 ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr;
5218 if ((CstLow = isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts)))
5219 if (Op.getOperand(i: 0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX))
5220 CstHigh =
5221 isConstOrConstSplat(N: Op.getOperand(i: 0).getOperand(i: 1), DemandedElts);
5222 if (CstLow && CstHigh) {
5223 if (!IsMax)
5224 std::swap(a&: CstLow, b&: CstHigh);
5225 if (CstLow->getAPIntValue().sle(RHS: CstHigh->getAPIntValue())) {
5226 Tmp = CstLow->getAPIntValue().getNumSignBits();
5227 Tmp2 = CstHigh->getAPIntValue().getNumSignBits();
5228 return std::min(a: Tmp, b: Tmp2);
5229 }
5230 }
5231
5232 // Fallback - just get the minimum number of sign bits of the operands.
5233 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5234 if (Tmp == 1)
5235 return 1; // Early out.
5236 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5237 return std::min(a: Tmp, b: Tmp2);
5238 }
5239 case ISD::UMIN:
5240 case ISD::UMAX:
5241 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5242 if (Tmp == 1)
5243 return 1; // Early out.
5244 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5245 return std::min(a: Tmp, b: Tmp2);
5246 case ISD::SSUBO_CARRY:
5247 case ISD::USUBO_CARRY:
5248 // sub_carry(x,x,c) -> 0/-1 (sext carry)
5249 if (Op.getResNo() == 0 && Op.getOperand(i: 0) == Op.getOperand(i: 1))
5250 return VTBits;
5251 [[fallthrough]];
5252 case ISD::SADDO:
5253 case ISD::UADDO:
5254 case ISD::SADDO_CARRY:
5255 case ISD::UADDO_CARRY:
5256 case ISD::SSUBO:
5257 case ISD::USUBO:
5258 case ISD::SMULO:
5259 case ISD::UMULO:
5260 if (Op.getResNo() != 1)
5261 break;
5262 // The boolean result conforms to getBooleanContents. Fall through.
5263 // If setcc returns 0/-1, all bits are sign bits.
5264 // We know that we have an integer-based boolean since these operations
5265 // are only available for integer.
5266 if (TLI->getBooleanContents(isVec: VT.isVector(), isFloat: false) ==
5267 TargetLowering::ZeroOrNegativeOneBooleanContent)
5268 return VTBits;
5269 break;
5270 case ISD::SETCC:
5271 case ISD::SETCCCARRY:
5272 case ISD::STRICT_FSETCC:
5273 case ISD::STRICT_FSETCCS: {
5274 unsigned OpNo = Op->isStrictFPOpcode() ? 1 : 0;
5275 // If setcc returns 0/-1, all bits are sign bits.
5276 if (TLI->getBooleanContents(Type: Op.getOperand(i: OpNo).getValueType()) ==
5277 TargetLowering::ZeroOrNegativeOneBooleanContent)
5278 return VTBits;
5279 break;
5280 }
5281 case ISD::GET_ACTIVE_LANE_MASK:
5282 // Semantically similar to icmp ult.
5283 if (TLI->getBooleanContents(isVec: VT.isVector(), /*isFloat=*/false) ==
5284 TargetLowering::ZeroOrNegativeOneBooleanContent)
5285 return VTBits;
5286 break;
5287 case ISD::ROTL:
5288 case ISD::ROTR: {
5289 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5290 ConstantSDNode *C = isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts);
5291 FirstAnswer = SignBitsOps::rot(
5292 SrcSignBits: Tmp, BitWidth: VTBits, RotAmt: C ? std::optional(C->getAPIntValue()) : std::nullopt,
5293 IsRotateRight: Opcode == ISD::ROTR);
5294 break;
5295 }
5296 case ISD::ADD:
5297 case ISD::ADDC:
5298 // TODO: Move Operand 1 check before Operand 0 check
5299 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5300 if (Tmp == 1) return 1; // Early out.
5301
5302 // Special case decrementing a value (ADD X, -1):
5303 if (ConstantSDNode *CRHS =
5304 isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts))
5305 if (CRHS->isAllOnes()) {
5306 KnownBits Known =
5307 computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5308
5309 // If the input is known to be 0 or 1, the output is 0/-1, which is all
5310 // sign bits set.
5311 if ((Known.Zero | 1).isAllOnes())
5312 return VTBits;
5313
5314 // If we are subtracting one from a positive number, there is no carry
5315 // out of the result.
5316 if (Known.isNonNegative())
5317 return Tmp;
5318 }
5319
5320 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5321 if (Tmp2 == 1) return 1; // Early out.
5322
5323 // Add can have at most one carry bit. Thus we know that the output
5324 // is, at worst, one more bit than the inputs.
5325 return std::min(a: Tmp, b: Tmp2) - 1;
5326 case ISD::SUB:
5327 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5328 if (Tmp2 == 1) return 1; // Early out.
5329
5330 // Handle NEG.
5331 if (ConstantSDNode *CLHS =
5332 isConstOrConstSplat(N: Op.getOperand(i: 0), DemandedElts))
5333 if (CLHS->isZero()) {
5334 KnownBits Known =
5335 computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5336 // If the input is known to be 0 or 1, the output is 0/-1, which is all
5337 // sign bits set.
5338 if ((Known.Zero | 1).isAllOnes())
5339 return VTBits;
5340
5341 // If the input is known to be positive (the sign bit is known clear),
5342 // the output of the NEG has the same number of sign bits as the input.
5343 if (Known.isNonNegative())
5344 return Tmp2;
5345
5346 // Otherwise, we treat this like a SUB.
5347 }
5348
5349 // Sub can have at most one carry bit. Thus we know that the output
5350 // is, at worst, one more bit than the inputs.
5351 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5352 if (Tmp == 1) return 1; // Early out.
5353 return std::min(a: Tmp, b: Tmp2) - 1;
5354 case ISD::MUL: {
5355 // The output of the Mul can be at most twice the valid bits in the inputs.
5356 unsigned SignBitsOp0 = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
5357 if (SignBitsOp0 == 1)
5358 break;
5359 unsigned SignBitsOp1 = ComputeNumSignBits(Op: Op.getOperand(i: 1), Depth: Depth + 1);
5360 if (SignBitsOp1 == 1)
5361 break;
5362 unsigned OutValidBits =
5363 (VTBits - SignBitsOp0 + 1) + (VTBits - SignBitsOp1 + 1);
5364 return OutValidBits > VTBits ? 1 : VTBits - OutValidBits + 1;
5365 }
5366 case ISD::AVGCEILS:
5367 case ISD::AVGFLOORS:
5368 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5369 if (Tmp == 1)
5370 return 1; // Early out.
5371 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5372 return std::min(a: Tmp, b: Tmp2);
5373 case ISD::SREM:
5374 // The sign bit is the LHS's sign bit, except when the result of the
5375 // remainder is zero. The magnitude of the result should be less than or
5376 // equal to the magnitude of the LHS. Therefore, the result should have
5377 // at least as many sign bits as the left hand side.
5378 return ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5379 case ISD::TRUNCATE: {
5380 // Check if the sign bits of source go down as far as the truncated value.
5381 unsigned NumSrcBits = Op.getOperand(i: 0).getScalarValueSizeInBits();
5382 unsigned NumSrcSignBits = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
5383 if (NumSrcSignBits > (NumSrcBits - VTBits))
5384 return NumSrcSignBits - (NumSrcBits - VTBits);
5385 break;
5386 }
5387 case ISD::EXTRACT_ELEMENT: {
5388 if (VT.isScalableVector())
5389 break;
5390 const int KnownSign = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth+1);
5391 const int BitWidth = Op.getValueSizeInBits();
5392 const int Items = Op.getOperand(i: 0).getValueSizeInBits() / BitWidth;
5393
5394 // Get reverse index (starting from 1), Op1 value indexes elements from
5395 // little end. Sign starts at big end.
5396 const int rIndex = Items - 1 - Op.getConstantOperandVal(i: 1);
5397
5398 // If the sign portion ends in our element the subtraction gives correct
5399 // result. Otherwise it gives either negative or > bitwidth result
5400 return std::clamp(val: KnownSign - rIndex * BitWidth, lo: 1, hi: BitWidth);
5401 }
5402 case ISD::INSERT_VECTOR_ELT: {
5403 if (VT.isScalableVector())
5404 break;
5405 // If we know the element index, split the demand between the
5406 // source vector and the inserted element, otherwise assume we need
5407 // the original demanded vector elements and the value.
5408 SDValue InVec = Op.getOperand(i: 0);
5409 SDValue InVal = Op.getOperand(i: 1);
5410 SDValue EltNo = Op.getOperand(i: 2);
5411 bool DemandedVal = true;
5412 APInt DemandedVecElts = DemandedElts;
5413 auto *CEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
5414 if (CEltNo && CEltNo->getAPIntValue().ult(RHS: NumElts)) {
5415 unsigned EltIdx = CEltNo->getZExtValue();
5416 DemandedVal = !!DemandedElts[EltIdx];
5417 DemandedVecElts.clearBit(BitPosition: EltIdx);
5418 }
5419 Tmp = std::numeric_limits<unsigned>::max();
5420 if (DemandedVal) {
5421 // TODO - handle implicit truncation of inserted elements.
5422 if (InVal.getScalarValueSizeInBits() != VTBits)
5423 break;
5424 Tmp2 = ComputeNumSignBits(Op: InVal, Depth: Depth + 1);
5425 Tmp = std::min(a: Tmp, b: Tmp2);
5426 }
5427 if (!!DemandedVecElts) {
5428 Tmp2 = ComputeNumSignBits(Op: InVec, DemandedElts: DemandedVecElts, Depth: Depth + 1);
5429 Tmp = std::min(a: Tmp, b: Tmp2);
5430 }
5431 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
5432 return Tmp;
5433 }
5434 case ISD::EXTRACT_VECTOR_ELT: {
5435 SDValue InVec = Op.getOperand(i: 0);
5436 SDValue EltNo = Op.getOperand(i: 1);
5437 EVT VecVT = InVec.getValueType();
5438 // ComputeNumSignBits not yet implemented for scalable vectors.
5439 if (VecVT.isScalableVector())
5440 break;
5441 const unsigned BitWidth = Op.getValueSizeInBits();
5442 const unsigned EltBitWidth = Op.getOperand(i: 0).getScalarValueSizeInBits();
5443 const unsigned NumSrcElts = VecVT.getVectorNumElements();
5444
5445 // If BitWidth > EltBitWidth the value is anyext:ed, and we do not know
5446 // anything about sign bits. But if the sizes match we can derive knowledge
5447 // about sign bits from the vector operand.
5448 if (BitWidth != EltBitWidth)
5449 break;
5450
5451 // If we know the element index, just demand that vector element, else for
5452 // an unknown element index, ignore DemandedElts and demand them all.
5453 APInt DemandedSrcElts = APInt::getAllOnes(numBits: NumSrcElts);
5454 auto *ConstEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
5455 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(RHS: NumSrcElts))
5456 DemandedSrcElts =
5457 APInt::getOneBitSet(numBits: NumSrcElts, BitNo: ConstEltNo->getZExtValue());
5458
5459 return ComputeNumSignBits(Op: InVec, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
5460 }
5461 case ISD::EXTRACT_SUBVECTOR: {
5462 // Offset the demanded elts by the subvector index.
5463 SDValue Src = Op.getOperand(i: 0);
5464
5465 APInt DemandedSrcElts;
5466 if (Src.getValueType().isScalableVector())
5467 DemandedSrcElts = APInt(1, 1);
5468 else {
5469 uint64_t Idx = Op.getConstantOperandVal(i: 1);
5470 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
5471 DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
5472 }
5473 return ComputeNumSignBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
5474 }
5475 case ISD::CONCAT_VECTORS: {
5476 if (VT.isScalableVector())
5477 break;
5478 // Determine the minimum number of sign bits across all demanded
5479 // elts of the input vectors. Early out if the result is already 1.
5480 Tmp = std::numeric_limits<unsigned>::max();
5481 EVT SubVectorVT = Op.getOperand(i: 0).getValueType();
5482 unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements();
5483 unsigned NumSubVectors = Op.getNumOperands();
5484 for (unsigned i = 0; (i < NumSubVectors) && (Tmp > 1); ++i) {
5485 APInt DemandedSub =
5486 DemandedElts.extractBits(numBits: NumSubVectorElts, bitPosition: i * NumSubVectorElts);
5487 if (!DemandedSub)
5488 continue;
5489 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i), DemandedElts: DemandedSub, Depth: Depth + 1);
5490 Tmp = std::min(a: Tmp, b: Tmp2);
5491 }
5492 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
5493 return Tmp;
5494 }
5495 case ISD::INSERT_SUBVECTOR: {
5496 SDValue Src = Op.getOperand(i: 0);
5497 SDValue Sub = Op.getOperand(i: 1);
5498 if (VT.isScalableVector()) {
5499 Tmp = ComputeNumSignBits(Op: Sub, Depth: Depth + 1);
5500 Tmp = std::min(a: Tmp, b: ComputeNumSignBits(Op: Src, Depth: Depth + 1));
5501 return Tmp;
5502 }
5503 // Demand any elements from the subvector and the remainder from the src its
5504 // inserted into.
5505 uint64_t Idx = Op.getConstantOperandVal(i: 2);
5506 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
5507 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
5508 APInt DemandedSrcElts = DemandedElts;
5509 DemandedSrcElts.clearBits(LoBit: Idx, HiBit: Idx + NumSubElts);
5510
5511 Tmp = std::numeric_limits<unsigned>::max();
5512 if (!!DemandedSubElts) {
5513 Tmp = ComputeNumSignBits(Op: Sub, DemandedElts: DemandedSubElts, Depth: Depth + 1);
5514 if (Tmp == 1)
5515 return 1; // early-out
5516 }
5517 if (!!DemandedSrcElts) {
5518 Tmp2 = ComputeNumSignBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
5519 Tmp = std::min(a: Tmp, b: Tmp2);
5520 }
5521 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
5522 return Tmp;
5523 }
5524 case ISD::LOAD: {
5525 // If we are looking at the loaded value of the SDNode.
5526 if (Op.getResNo() != 0)
5527 break;
5528
5529 LoadSDNode *LD = cast<LoadSDNode>(Val&: Op);
5530 if (const MDNode *Ranges = LD->getRanges()) {
5531 if (DemandedElts != 1)
5532 break;
5533
5534 ConstantRange CR = getConstantRangeFromMetadata(RangeMD: *Ranges);
5535 if (VTBits > CR.getBitWidth()) {
5536 switch (LD->getExtensionType()) {
5537 case ISD::SEXTLOAD:
5538 CR = CR.signExtend(BitWidth: VTBits);
5539 break;
5540 case ISD::ZEXTLOAD:
5541 CR = CR.zeroExtend(BitWidth: VTBits);
5542 break;
5543 default:
5544 break;
5545 }
5546 }
5547
5548 if (VTBits != CR.getBitWidth())
5549 break;
5550 return std::min(a: CR.getSignedMin().getNumSignBits(),
5551 b: CR.getSignedMax().getNumSignBits());
5552 }
5553
5554 unsigned ExtType = LD->getExtensionType();
5555 switch (ExtType) {
5556 default:
5557 break;
5558 case ISD::SEXTLOAD: // e.g. i16->i32 = '17' bits known.
5559 Tmp = LD->getMemoryVT().getScalarSizeInBits();
5560 return VTBits - Tmp + 1;
5561 case ISD::ZEXTLOAD: // e.g. i16->i32 = '16' bits known.
5562 Tmp = LD->getMemoryVT().getScalarSizeInBits();
5563 return VTBits - Tmp;
5564 case ISD::NON_EXTLOAD:
5565 if (const Constant *Cst = TLI->getTargetConstantFromLoad(LD)) {
5566 // We only need to handle vectors - computeKnownBits should handle
5567 // scalar cases.
5568 Type *CstTy = Cst->getType();
5569 if (CstTy->isVectorTy() && !VT.isScalableVector() &&
5570 (NumElts * VTBits) == CstTy->getPrimitiveSizeInBits() &&
5571 VTBits == CstTy->getScalarSizeInBits()) {
5572 Tmp = VTBits;
5573 for (unsigned i = 0; i != NumElts; ++i) {
5574 if (!DemandedElts[i])
5575 continue;
5576 if (Constant *Elt = Cst->getAggregateElement(Elt: i)) {
5577 if (auto *CInt = dyn_cast<ConstantInt>(Val: Elt)) {
5578 const APInt &Value = CInt->getValue();
5579 Tmp = std::min(a: Tmp, b: Value.getNumSignBits());
5580 continue;
5581 }
5582 if (auto *CFP = dyn_cast<ConstantFP>(Val: Elt)) {
5583 APInt Value = CFP->getValueAPF().bitcastToAPInt();
5584 Tmp = std::min(a: Tmp, b: Value.getNumSignBits());
5585 continue;
5586 }
5587 }
5588 // Unknown type. Conservatively assume no bits match sign bit.
5589 return 1;
5590 }
5591 return Tmp;
5592 }
5593 }
5594 break;
5595 }
5596
5597 break;
5598 }
5599 case ISD::ATOMIC_CMP_SWAP:
5600 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
5601 case ISD::ATOMIC_SWAP:
5602 case ISD::ATOMIC_LOAD_ADD:
5603 case ISD::ATOMIC_LOAD_SUB:
5604 case ISD::ATOMIC_LOAD_AND:
5605 case ISD::ATOMIC_LOAD_CLR:
5606 case ISD::ATOMIC_LOAD_OR:
5607 case ISD::ATOMIC_LOAD_XOR:
5608 case ISD::ATOMIC_LOAD_NAND:
5609 case ISD::ATOMIC_LOAD_MIN:
5610 case ISD::ATOMIC_LOAD_MAX:
5611 case ISD::ATOMIC_LOAD_UMIN:
5612 case ISD::ATOMIC_LOAD_UMAX:
5613 case ISD::ATOMIC_LOAD: {
5614 auto *AT = cast<AtomicSDNode>(Val&: Op);
5615 // If we are looking at the loaded value.
5616 if (Op.getResNo() == 0) {
5617 Tmp = AT->getMemoryVT().getScalarSizeInBits();
5618 if (Tmp == VTBits)
5619 return 1; // early-out
5620
5621 // For atomic_load, prefer to use the extension type.
5622 if (Op->getOpcode() == ISD::ATOMIC_LOAD) {
5623 switch (AT->getExtensionType()) {
5624 default:
5625 break;
5626 case ISD::SEXTLOAD:
5627 return VTBits - Tmp + 1;
5628 case ISD::ZEXTLOAD:
5629 return VTBits - Tmp;
5630 }
5631 }
5632
5633 if (TLI->getExtendForAtomicOps() == ISD::SIGN_EXTEND)
5634 return VTBits - Tmp + 1;
5635 if (TLI->getExtendForAtomicOps() == ISD::ZERO_EXTEND)
5636 return VTBits - Tmp;
5637 }
5638 break;
5639 }
5640 }
5641
5642 // Allow the target to implement this method for its nodes.
5643 if (Opcode >= ISD::BUILTIN_OP_END ||
5644 Opcode == ISD::INTRINSIC_WO_CHAIN ||
5645 Opcode == ISD::INTRINSIC_W_CHAIN ||
5646 Opcode == ISD::INTRINSIC_VOID) {
5647 // TODO: This can probably be removed once target code is audited. This
5648 // is here purely to reduce patch size and review complexity.
5649 if (!VT.isScalableVector()) {
5650 unsigned NumBits =
5651 TLI->ComputeNumSignBitsForTargetNode(Op, DemandedElts, DAG: *this, Depth);
5652 if (NumBits > 1)
5653 FirstAnswer = std::max(a: FirstAnswer, b: NumBits);
5654 }
5655 }
5656
5657 // Finally, if we can prove that the top bits of the result are 0's or 1's,
5658 // use this information.
5659 KnownBits Known = computeKnownBits(Op, DemandedElts, Depth);
5660 return std::max(a: FirstAnswer, b: Known.countMinSignBits());
5661}
5662
5663unsigned SelectionDAG::ComputeMaxSignificantBits(SDValue Op,
5664 unsigned Depth) const {
5665 unsigned SignBits = ComputeNumSignBits(Op, Depth);
5666 return Op.getScalarValueSizeInBits() - SignBits + 1;
5667}
5668
5669unsigned SelectionDAG::ComputeMaxSignificantBits(SDValue Op,
5670 const APInt &DemandedElts,
5671 unsigned Depth) const {
5672 unsigned SignBits = ComputeNumSignBits(Op, DemandedElts, Depth);
5673 return Op.getScalarValueSizeInBits() - SignBits + 1;
5674}
5675
5676bool SelectionDAG::isGuaranteedNotToBeUndefOrPoison(SDValue Op,
5677 UndefPoisonKind Kind,
5678 unsigned Depth) const {
5679 // Early out for FREEZE.
5680 if (Op.getOpcode() == ISD::FREEZE)
5681 return true;
5682
5683 APInt DemandedElts = getDemandAllEltsMask(V: Op);
5684 return isGuaranteedNotToBeUndefOrPoison(Op, DemandedElts, Kind, Depth);
5685}
5686
5687bool SelectionDAG::isGuaranteedNotToBeUndefOrPoison(SDValue Op,
5688 const APInt &DemandedElts,
5689 UndefPoisonKind Kind,
5690 unsigned Depth) const {
5691 unsigned Opcode = Op.getOpcode();
5692
5693 // Early out for FREEZE.
5694 if (Opcode == ISD::FREEZE)
5695 return true;
5696
5697 if (Depth >= MaxRecursionDepth)
5698 return false; // Limit search depth.
5699
5700 if (isIntOrFPConstant(V: Op))
5701 return true;
5702
5703 switch (Opcode) {
5704 case ISD::CONDCODE:
5705 case ISD::VALUETYPE:
5706 case ISD::FrameIndex:
5707 case ISD::TargetFrameIndex:
5708 case ISD::CopyFromReg:
5709 return true;
5710
5711 case ISD::POISON:
5712 return !includesPoison(Kind);
5713
5714 case ISD::UNDEF:
5715 return !includesUndef(Kind);
5716
5717 case ISD::BITCAST: {
5718 SDValue Src = Op.getOperand(i: 0);
5719 EVT SrcVT = Src.getValueType();
5720 EVT DstVT = Op.getValueType();
5721
5722 if (!SrcVT.isVector() || !DstVT.isVector())
5723 return isGuaranteedNotToBeUndefOrPoison(Op: Src, Kind, Depth: Depth + 1);
5724
5725 unsigned SrcEltBits = SrcVT.getScalarSizeInBits();
5726 unsigned DstEltBits = DstVT.getScalarSizeInBits();
5727 ElementCount NumSrcElts = SrcVT.getVectorElementCount();
5728 [[maybe_unused]] ElementCount NumDstElts = DstVT.getVectorElementCount();
5729
5730 if (SrcEltBits == DstEltBits)
5731 return isGuaranteedNotToBeUndefOrPoison(Op: Src, DemandedElts, Kind,
5732 Depth: Depth + 1);
5733
5734 if (SrcEltBits < DstEltBits) {
5735 if (DstEltBits % SrcEltBits != 0)
5736 return isGuaranteedNotToBeUndefOrPoison(Op: Src, Kind, Depth: Depth + 1);
5737
5738 assert(NumSrcElts == NumDstElts * (DstEltBits / SrcEltBits) &&
5739 "Unexpected vector bitcast");
5740 APInt DemandedSrcElts =
5741 APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumSrcElts.getKnownMinValue());
5742 return isGuaranteedNotToBeUndefOrPoison(Op: Src, DemandedElts: DemandedSrcElts, Kind,
5743 Depth: Depth + 1);
5744 }
5745
5746 if (SrcEltBits % DstEltBits != 0)
5747 return isGuaranteedNotToBeUndefOrPoison(Op: Src, Kind, Depth: Depth + 1);
5748
5749 assert(NumDstElts == NumSrcElts * (SrcEltBits / DstEltBits) &&
5750 "Unexpected vector bitcast");
5751 APInt DemandedSrcElts =
5752 APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumSrcElts.getKnownMinValue());
5753 return isGuaranteedNotToBeUndefOrPoison(Op: Src, DemandedElts: DemandedSrcElts, Kind,
5754 Depth: Depth + 1);
5755 }
5756
5757 case ISD::BUILD_VECTOR:
5758 // NOTE: BUILD_VECTOR has implicit truncation of wider scalar elements -
5759 // this shouldn't affect the result.
5760 for (unsigned i = 0, e = Op.getNumOperands(); i < e; ++i) {
5761 if (!DemandedElts[i])
5762 continue;
5763 if (!isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i), Kind, Depth: Depth + 1))
5764 return false;
5765 }
5766 return true;
5767
5768 case ISD::CONCAT_VECTORS: {
5769 EVT VT = Op.getValueType();
5770 if (!VT.isFixedLengthVector())
5771 break;
5772
5773 EVT SubVT = Op.getOperand(i: 0).getValueType();
5774 unsigned NumSubElts = SubVT.getVectorNumElements();
5775 for (unsigned I = 0, E = Op.getNumOperands(); I != E; ++I) {
5776 APInt DemandedSubElts =
5777 DemandedElts.extractBits(numBits: NumSubElts, bitPosition: I * NumSubElts);
5778 if (!!DemandedSubElts &&
5779 !isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: I), DemandedElts: DemandedSubElts,
5780 Kind, Depth: Depth + 1))
5781 return false;
5782 }
5783 return true;
5784 }
5785
5786 case ISD::EXTRACT_SUBVECTOR: {
5787 SDValue Src = Op.getOperand(i: 0);
5788 if (Src.getValueType().isScalableVector())
5789 break;
5790 uint64_t Idx = Op.getConstantOperandVal(i: 1);
5791 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
5792 APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
5793 return isGuaranteedNotToBeUndefOrPoison(Op: Src, DemandedElts: DemandedSrcElts, Kind,
5794 Depth: Depth + 1);
5795 }
5796
5797 case ISD::INSERT_SUBVECTOR: {
5798 if (Op.getValueType().isScalableVector())
5799 break;
5800 SDValue Src = Op.getOperand(i: 0);
5801 SDValue Sub = Op.getOperand(i: 1);
5802 uint64_t Idx = Op.getConstantOperandVal(i: 2);
5803 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
5804 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
5805 APInt DemandedSrcElts = DemandedElts;
5806 DemandedSrcElts.clearBits(LoBit: Idx, HiBit: Idx + NumSubElts);
5807
5808 if (!!DemandedSubElts && !isGuaranteedNotToBeUndefOrPoison(
5809 Op: Sub, DemandedElts: DemandedSubElts, Kind, Depth: Depth + 1))
5810 return false;
5811 if (!!DemandedSrcElts && !isGuaranteedNotToBeUndefOrPoison(
5812 Op: Src, DemandedElts: DemandedSrcElts, Kind, Depth: Depth + 1))
5813 return false;
5814 return true;
5815 }
5816
5817 case ISD::EXTRACT_VECTOR_ELT: {
5818 SDValue Src = Op.getOperand(i: 0);
5819 auto *IndexC = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1));
5820 EVT SrcVT = Src.getValueType();
5821 if (SrcVT.isFixedLengthVector() && IndexC &&
5822 IndexC->getAPIntValue().ult(RHS: SrcVT.getVectorNumElements())) {
5823 APInt DemandedSrcElts = APInt::getOneBitSet(numBits: SrcVT.getVectorNumElements(),
5824 BitNo: IndexC->getZExtValue());
5825 return isGuaranteedNotToBeUndefOrPoison(Op: Src, DemandedElts: DemandedSrcElts, Kind,
5826 Depth: Depth + 1);
5827 }
5828 break;
5829 }
5830
5831 case ISD::INSERT_VECTOR_ELT: {
5832 SDValue InVec = Op.getOperand(i: 0);
5833 SDValue InVal = Op.getOperand(i: 1);
5834 SDValue EltNo = Op.getOperand(i: 2);
5835 EVT VT = InVec.getValueType();
5836 auto *IndexC = dyn_cast<ConstantSDNode>(Val&: EltNo);
5837 if (IndexC && VT.isFixedLengthVector() &&
5838 IndexC->getAPIntValue().ult(RHS: VT.getVectorNumElements())) {
5839 if (DemandedElts[IndexC->getZExtValue()] &&
5840 !isGuaranteedNotToBeUndefOrPoison(Op: InVal, Kind, Depth: Depth + 1))
5841 return false;
5842 APInt InVecDemandedElts = DemandedElts;
5843 InVecDemandedElts.clearBit(BitPosition: IndexC->getZExtValue());
5844 if (!!InVecDemandedElts &&
5845 !isGuaranteedNotToBeUndefOrPoison(
5846 Op: peekThroughInsertVectorElt(V: InVec, DemandedElts: InVecDemandedElts),
5847 DemandedElts: InVecDemandedElts, Kind, Depth: Depth + 1))
5848 return false;
5849 return true;
5850 }
5851 break;
5852 }
5853
5854 case ISD::SCALAR_TO_VECTOR:
5855 // Check upper (known poison) elements.
5856 if (DemandedElts.ugt(RHS: 1) && includesPoison(Kind))
5857 return false;
5858 // Check element zero.
5859 if (DemandedElts[0] &&
5860 !isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 0), Kind, Depth: Depth + 1))
5861 return false;
5862 return true;
5863
5864 case ISD::SPLAT_VECTOR:
5865 return isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 0), Kind, Depth: Depth + 1);
5866
5867 case ISD::SELECT: {
5868 return !canCreateUndefOrPoison(Op, DemandedElts, Kind,
5869 /*ConsiderFlags*/ true, Depth) &&
5870 isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 0), Kind,
5871 Depth: Depth + 1) &&
5872 isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 1), DemandedElts,
5873 Kind, Depth: Depth + 1) &&
5874 isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 2), DemandedElts,
5875 Kind, Depth: Depth + 1);
5876 }
5877
5878 case ISD::VECTOR_SHUFFLE: {
5879 APInt DemandedLHS, DemandedRHS;
5880 auto *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
5881 if (!getShuffleDemandedElts(SrcWidth: DemandedElts.getBitWidth(), Mask: SVN->getMask(),
5882 DemandedElts, DemandedLHS, DemandedRHS,
5883 /*AllowUndefElts=*/false))
5884 return false;
5885 if (!DemandedLHS.isZero() &&
5886 !isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 0), DemandedElts: DemandedLHS, Kind,
5887 Depth: Depth + 1))
5888 return false;
5889 if (!DemandedRHS.isZero() &&
5890 !isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 1), DemandedElts: DemandedRHS, Kind,
5891 Depth: Depth + 1))
5892 return false;
5893 return true;
5894 }
5895
5896 case ISD::SHL:
5897 case ISD::SRL:
5898 case ISD::SRA:
5899 // Shift amount operand is checked by canCreateUndefOrPoison. So it is
5900 // enough to check operand 0 if Op can't create undef/poison.
5901 return !canCreateUndefOrPoison(Op, DemandedElts, Kind,
5902 /*ConsiderFlags*/ true, Depth) &&
5903 isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 0), DemandedElts,
5904 Kind, Depth: Depth + 1);
5905
5906 case ISD::BSWAP:
5907 case ISD::CTPOP:
5908 case ISD::BITREVERSE:
5909 case ISD::AND:
5910 case ISD::OR:
5911 case ISD::XOR:
5912 case ISD::ADD:
5913 case ISD::SUB:
5914 case ISD::MUL:
5915 case ISD::SADDSAT:
5916 case ISD::UADDSAT:
5917 case ISD::SSUBSAT:
5918 case ISD::USUBSAT:
5919 case ISD::SSHLSAT:
5920 case ISD::USHLSAT:
5921 case ISD::SMIN:
5922 case ISD::SMAX:
5923 case ISD::UMIN:
5924 case ISD::UMAX:
5925 case ISD::ZERO_EXTEND:
5926 case ISD::SIGN_EXTEND:
5927 case ISD::ANY_EXTEND:
5928 case ISD::TRUNCATE:
5929 case ISD::VSELECT: {
5930 // If Op can't create undef/poison and none of its operands are undef/poison
5931 // then Op is never undef/poison. A difference from the more common check
5932 // below, outside the switch, is that we handle elementwise operations for
5933 // which the DemandedElts mask is valid for all operands here.
5934 return !canCreateUndefOrPoison(Op, DemandedElts, Kind,
5935 /*ConsiderFlags*/ true, Depth) &&
5936 all_of(Range: Op->ops(), P: [&](SDValue V) {
5937 return isGuaranteedNotToBeUndefOrPoison(Op: V, DemandedElts, Kind,
5938 Depth: Depth + 1);
5939 });
5940 }
5941
5942 // TODO: Search for noundef attributes from library functions.
5943
5944 // TODO: Pointers dereferenced by ISD::LOAD/STORE ops are noundef.
5945
5946 default:
5947 // Allow the target to implement this method for its nodes.
5948 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
5949 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)
5950 return TLI->isGuaranteedNotToBeUndefOrPoisonForTargetNode(
5951 Op, DemandedElts, DAG: *this, Kind, Depth);
5952 break;
5953 }
5954
5955 // If Op can't create undef/poison and none of its operands are undef/poison
5956 // then Op is never undef/poison.
5957 // NOTE: TargetNodes can handle this in themselves in
5958 // isGuaranteedNotToBeUndefOrPoisonForTargetNode or let
5959 // TargetLowering::isGuaranteedNotToBeUndefOrPoisonForTargetNode handle it.
5960 return !canCreateUndefOrPoison(Op, Kind, /*ConsiderFlags*/ true, Depth) &&
5961 all_of(Range: Op->ops(), P: [&](SDValue V) {
5962 return isGuaranteedNotToBeUndefOrPoison(Op: V, Kind, Depth: Depth + 1);
5963 });
5964}
5965
5966bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, UndefPoisonKind Kind,
5967 bool ConsiderFlags,
5968 unsigned Depth) const {
5969 APInt DemandedElts = getDemandAllEltsMask(V: Op);
5970 return canCreateUndefOrPoison(Op, DemandedElts, Kind, ConsiderFlags, Depth);
5971}
5972
5973bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, const APInt &DemandedElts,
5974 UndefPoisonKind Kind,
5975 bool ConsiderFlags,
5976 unsigned Depth) const {
5977 if (ConsiderFlags && includesPoison(Kind) && Op->hasPoisonGeneratingFlags())
5978 return true;
5979
5980 unsigned Opcode = Op.getOpcode();
5981 switch (Opcode) {
5982 case ISD::AssertSext:
5983 case ISD::AssertZext:
5984 case ISD::AssertAlign:
5985 case ISD::AssertNoFPClass:
5986 // Assertion nodes can create poison if the assertion fails.
5987 return includesPoison(Kind);
5988
5989 case ISD::FREEZE:
5990 case ISD::CONCAT_VECTORS:
5991 case ISD::INSERT_SUBVECTOR:
5992 case ISD::EXTRACT_SUBVECTOR:
5993 case ISD::SADDSAT:
5994 case ISD::UADDSAT:
5995 case ISD::SSUBSAT:
5996 case ISD::USUBSAT:
5997 case ISD::MULHU:
5998 case ISD::MULHS:
5999 case ISD::AVGFLOORS:
6000 case ISD::AVGFLOORU:
6001 case ISD::AVGCEILS:
6002 case ISD::AVGCEILU:
6003 case ISD::ABDU:
6004 case ISD::ABDS:
6005 case ISD::SMIN:
6006 case ISD::SMAX:
6007 case ISD::SCMP:
6008 case ISD::UMIN:
6009 case ISD::UMAX:
6010 case ISD::UCMP:
6011 case ISD::AND:
6012 case ISD::XOR:
6013 case ISD::ROTL:
6014 case ISD::ROTR:
6015 case ISD::FSHL:
6016 case ISD::FSHR:
6017 case ISD::BSWAP:
6018 case ISD::CTTZ:
6019 case ISD::CTLZ:
6020 case ISD::CTLS:
6021 case ISD::CTPOP:
6022 case ISD::BITREVERSE:
6023 case ISD::PARITY:
6024 case ISD::SIGN_EXTEND:
6025 case ISD::TRUNCATE:
6026 case ISD::SIGN_EXTEND_INREG:
6027 case ISD::SIGN_EXTEND_VECTOR_INREG:
6028 case ISD::ZERO_EXTEND_VECTOR_INREG:
6029 case ISD::BITCAST:
6030 case ISD::BUILD_VECTOR:
6031 case ISD::BUILD_PAIR:
6032 case ISD::SPLAT_VECTOR:
6033 case ISD::FABS:
6034 case ISD::FCEIL:
6035 case ISD::FFLOOR:
6036 case ISD::FTRUNC:
6037 case ISD::FRINT:
6038 case ISD::FNEARBYINT:
6039 case ISD::FROUND:
6040 case ISD::FROUNDEVEN:
6041 return false;
6042
6043 case ISD::ABS:
6044 // ISD::ABS defines abs(INT_MIN) -> INT_MIN and never generates poison.
6045 // Different to Intrinsic::abs.
6046 return false;
6047 case ISD::ABS_MIN_POISON:
6048 // ABS_MIN_POISON may produce poison if the input is INT_MIN.
6049 return ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1) <= 1;
6050
6051 case ISD::ADDC:
6052 case ISD::SUBC:
6053 case ISD::ADDE:
6054 case ISD::SUBE:
6055 case ISD::SADDO:
6056 case ISD::SSUBO:
6057 case ISD::SMULO:
6058 case ISD::SADDO_CARRY:
6059 case ISD::SSUBO_CARRY:
6060 case ISD::UADDO:
6061 case ISD::USUBO:
6062 case ISD::UMULO:
6063 case ISD::UADDO_CARRY:
6064 case ISD::USUBO_CARRY:
6065 // No poison on result or overflow flags.
6066 return false;
6067
6068 case ISD::SELECT_CC:
6069 case ISD::SETCC: {
6070 // Integer setcc cannot create undef or poison.
6071 if (Op.getOperand(i: 0).getValueType().isInteger())
6072 return false;
6073
6074 // FP compares are more complicated. They can create poison for nan/infinity
6075 // based on options and flags. The options and flags also cause special
6076 // nonan condition codes to be used. Those condition codes may be preserved
6077 // even if the nonan flag is dropped somewhere.
6078 unsigned CCOp = Opcode == ISD::SETCC ? 2 : 4;
6079 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: Op.getOperand(i: CCOp))->get();
6080 return (unsigned)CCCode & 0x10U;
6081 }
6082
6083 case ISD::OR:
6084 case ISD::ZERO_EXTEND:
6085 case ISD::SELECT:
6086 case ISD::VSELECT:
6087 case ISD::ADD:
6088 case ISD::SUB:
6089 case ISD::MUL:
6090 case ISD::FNEG:
6091 case ISD::FADD:
6092 case ISD::FSUB:
6093 case ISD::FMUL:
6094 case ISD::FDIV:
6095 case ISD::FREM:
6096 case ISD::FCOPYSIGN:
6097 case ISD::FMA:
6098 case ISD::FMAD:
6099 case ISD::FMULADD:
6100 case ISD::FP_EXTEND:
6101 case ISD::FMINNUM:
6102 case ISD::FMAXNUM:
6103 case ISD::FMINNUM_IEEE:
6104 case ISD::FMAXNUM_IEEE:
6105 case ISD::FMINIMUM:
6106 case ISD::FMAXIMUM:
6107 case ISD::FMINIMUMNUM:
6108 case ISD::FMAXIMUMNUM:
6109 case ISD::FP_TO_SINT_SAT:
6110 case ISD::FP_TO_UINT_SAT:
6111 case ISD::TRUNCATE_SSAT_S:
6112 case ISD::TRUNCATE_SSAT_U:
6113 case ISD::TRUNCATE_USAT_U:
6114 // No poison except from flags (which is handled above)
6115 return false;
6116
6117 case ISD::SHL:
6118 case ISD::SRL:
6119 case ISD::SRA:
6120 // If the max shift amount isn't in range, then the shift can
6121 // create poison.
6122 return includesPoison(Kind) &&
6123 !getValidMaximumShiftAmount(V: Op, DemandedElts, Depth: Depth + 1);
6124
6125 case ISD::CTTZ_ZERO_POISON:
6126 case ISD::CTLZ_ZERO_POISON:
6127 // If the amount is zero then the result will be poison.
6128 // TODO: Add isKnownNeverZero DemandedElts handling.
6129 return includesPoison(Kind) &&
6130 !isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6131
6132 case ISD::SCALAR_TO_VECTOR:
6133 // Check if we demand any upper (poison) elements.
6134 return includesPoison(Kind) && DemandedElts.ugt(RHS: 1);
6135
6136 case ISD::INSERT_VECTOR_ELT:
6137 case ISD::EXTRACT_VECTOR_ELT: {
6138 // Ensure that the element index is in bounds.
6139 if (includesPoison(Kind)) {
6140 EVT VecVT = Op.getOperand(i: 0).getValueType();
6141 SDValue Idx = Op.getOperand(i: Opcode == ISD::INSERT_VECTOR_ELT ? 2 : 1);
6142 KnownBits KnownIdx = computeKnownBits(Op: Idx, Depth: Depth + 1);
6143 return KnownIdx.getMaxValue().uge(RHS: VecVT.getVectorMinNumElements());
6144 }
6145 return false;
6146 }
6147
6148 case ISD::VECTOR_SHUFFLE: {
6149 // Check for any demanded shuffle element that is undef.
6150 auto *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
6151 for (auto [Idx, Elt] : enumerate(First: SVN->getMask()))
6152 if (Elt < 0 && DemandedElts[Idx])
6153 return true;
6154 return false;
6155 }
6156
6157 case ISD::VECTOR_COMPRESS:
6158 return false;
6159
6160 default:
6161 // Allow the target to implement this method for its nodes.
6162 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
6163 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)
6164 return TLI->canCreateUndefOrPoisonForTargetNode(
6165 Op, DemandedElts, DAG: *this, Kind, ConsiderFlags, Depth);
6166 break;
6167 }
6168
6169 // Be conservative and return true.
6170 return true;
6171}
6172
6173bool SelectionDAG::isADDLike(SDValue Op, bool NoWrap) const {
6174 unsigned Opcode = Op.getOpcode();
6175 if (Opcode == ISD::OR)
6176 return Op->getFlags().hasDisjoint() ||
6177 haveNoCommonBitsSet(A: Op.getOperand(i: 0), B: Op.getOperand(i: 1));
6178 if (Opcode == ISD::XOR)
6179 return !NoWrap && isMinSignedConstant(V: Op.getOperand(i: 1));
6180 return false;
6181}
6182
6183bool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const {
6184 return Op.getNumOperands() == 2 && isa<ConstantSDNode>(Val: Op.getOperand(i: 1)) &&
6185 (Op.isAnyAdd() || isADDLike(Op));
6186}
6187
6188KnownFPClass SelectionDAG::computeKnownFPClass(SDValue Op,
6189 FPClassTest InterestedClasses,
6190 unsigned Depth) const {
6191 APInt DemandedElts = getDemandAllEltsMask(V: Op);
6192 return computeKnownFPClass(Op, DemandedElts, InterestedClasses, Depth);
6193}
6194
6195KnownFPClass SelectionDAG::computeKnownFPClass(SDValue Op,
6196 const APInt &DemandedElts,
6197 FPClassTest InterestedClasses,
6198 unsigned Depth) const {
6199 KnownFPClass Known;
6200
6201 if (const auto *CFP = dyn_cast<ConstantFPSDNode>(Val&: Op))
6202 return KnownFPClass(CFP->getValueAPF());
6203
6204 if (Depth >= MaxRecursionDepth)
6205 return Known;
6206
6207 if (Op.getOpcode() == ISD::UNDEF)
6208 return Known;
6209
6210 EVT VT = Op.getValueType();
6211 assert(VT.isFloatingPoint() && "Computing KnownFPClass on non-FP op!");
6212 assert((!VT.isFixedLengthVector() ||
6213 DemandedElts.getBitWidth() == VT.getVectorNumElements()) &&
6214 "Unexpected vector size");
6215
6216 if (!DemandedElts)
6217 return Known;
6218
6219 unsigned Opcode = Op.getOpcode();
6220 switch (Opcode) {
6221 case ISD::POISON: {
6222 Known.setKnownFPClasses(fcNone);
6223 Known.setSignBit(false);
6224 break;
6225 }
6226 case ISD::FNEG: {
6227 Known = computeKnownFPClass(Op: Op.getOperand(i: 0), DemandedElts,
6228 InterestedClasses, Depth: Depth + 1);
6229 Known.fneg();
6230 break;
6231 }
6232 case ISD::BUILD_VECTOR: {
6233 assert(!VT.isScalableVector());
6234 bool First = true;
6235 for (unsigned I = 0, E = Op.getNumOperands(); I != E; ++I) {
6236 if (!DemandedElts[I])
6237 continue;
6238
6239 if (First) {
6240 Known =
6241 computeKnownFPClass(Op: Op.getOperand(i: I), InterestedClasses, Depth: Depth + 1);
6242 First = false;
6243 } else {
6244 Known |=
6245 computeKnownFPClass(Op: Op.getOperand(i: I), InterestedClasses, Depth: Depth + 1);
6246 }
6247
6248 if (Known.isUnknown())
6249 break;
6250 }
6251 break;
6252 }
6253 case ISD::EXTRACT_VECTOR_ELT: {
6254 SDValue Src = Op.getOperand(i: 0);
6255 auto *CIdx = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1));
6256 EVT SrcVT = Src.getValueType();
6257 if (SrcVT.isFixedLengthVector() && CIdx) {
6258 if (CIdx->getAPIntValue().ult(RHS: SrcVT.getVectorNumElements())) {
6259 APInt DemandedSrcElts = APInt::getOneBitSet(
6260 numBits: SrcVT.getVectorNumElements(), BitNo: CIdx->getZExtValue());
6261 Known = computeKnownFPClass(Op: Src, DemandedElts: DemandedSrcElts, InterestedClasses,
6262 Depth: Depth + 1);
6263 } else {
6264 // Out of bounds index is poison.
6265 Known.setKnownFPClasses(fcNone);
6266 }
6267 } else {
6268 Known = computeKnownFPClass(Op: Src, InterestedClasses, Depth: Depth + 1);
6269 }
6270 break;
6271 }
6272 case ISD::SPLAT_VECTOR: {
6273 Known = computeKnownFPClass(Op: Op.getOperand(i: 0), InterestedClasses, Depth: Depth + 1);
6274 break;
6275 }
6276 case ISD::BITCAST: {
6277 // FIXME: It should not be necessary to check for an elementwise bitcast.
6278 // If a bitcast is not elementwise between vector / scalar types,
6279 // computeKnownBits already splices the known bits of the source elements
6280 // appropriately so as to line up with the bits of the result's demanded
6281 // elements.
6282 EVT SrcVT = Op.getOperand(i: 0).getValueType();
6283 if (VT.isScalableVector() || SrcVT.isScalableVector())
6284 break;
6285 unsigned VTNumElts = VT.isVector() ? VT.getVectorNumElements() : 1;
6286 unsigned SrcVTNumElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
6287 if (VTNumElts != SrcVTNumElts)
6288 break;
6289
6290 KnownBits Bits = computeKnownBits(Op, DemandedElts, Depth: Depth + 1);
6291 Known = KnownFPClass::bitcast(FltSemantics: VT.getFltSemantics(), Bits);
6292 break;
6293 }
6294 case ISD::FABS: {
6295 Known = computeKnownFPClass(Op: Op.getOperand(i: 0), DemandedElts,
6296 InterestedClasses, Depth: Depth + 1);
6297 Known.fabs();
6298 break;
6299 }
6300 case ISD::FCOPYSIGN: {
6301 Known = computeKnownFPClass(Op: Op.getOperand(i: 0), DemandedElts,
6302 InterestedClasses, Depth: Depth + 1);
6303 KnownFPClass KnownSign = computeKnownFPClass(Op: Op.getOperand(i: 1), DemandedElts,
6304 InterestedClasses, Depth: Depth + 1);
6305 Known.copysign(Sign: KnownSign);
6306 break;
6307 }
6308 case ISD::AssertNoFPClass: {
6309 Known = computeKnownFPClass(Op: Op.getOperand(i: 0), DemandedElts,
6310 InterestedClasses, Depth: Depth + 1);
6311 FPClassTest AssertedClasses =
6312 static_cast<FPClassTest>(Op->getConstantOperandVal(Num: 1));
6313 Known.setKnownFPClasses(Known.getKnownFPClasses() & ~AssertedClasses);
6314 break;
6315 }
6316 case ISD::EXTRACT_SUBVECTOR: {
6317 SDValue Src = Op.getOperand(i: 0);
6318 EVT SrcVT = Src.getValueType();
6319 if (SrcVT.isFixedLengthVector()) {
6320 unsigned Idx = Op.getConstantOperandVal(i: 1);
6321 unsigned NumSrcElts = SrcVT.getVectorNumElements();
6322
6323 APInt DemandedSrcElts = DemandedElts.zextOrTrunc(width: NumSrcElts).shl(shiftAmt: Idx);
6324 Known = computeKnownFPClass(Op: Src, DemandedElts: DemandedSrcElts, InterestedClasses,
6325 Depth: Depth + 1);
6326 } else {
6327 Known = computeKnownFPClass(Op: Src, InterestedClasses, Depth: Depth + 1);
6328 }
6329 break;
6330 }
6331 case ISD::INSERT_SUBVECTOR: {
6332 SDValue BaseVector = Op.getOperand(i: 0);
6333 SDValue SubVector = Op.getOperand(i: 1);
6334 EVT BaseVT = BaseVector.getValueType();
6335 if (BaseVT.isFixedLengthVector()) {
6336 unsigned Idx = Op.getConstantOperandVal(i: 2);
6337 unsigned NumBaseElts = BaseVT.getVectorNumElements();
6338 unsigned NumSubElts = SubVector.getValueType().getVectorNumElements();
6339
6340 APInt DemandedMask =
6341 APInt::getBitsSet(numBits: NumBaseElts, loBit: Idx, hiBit: Idx + NumSubElts);
6342 APInt DemandedSrcElts = DemandedElts & ~DemandedMask;
6343 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
6344
6345 if (!DemandedSrcElts.isZero())
6346 Known = computeKnownFPClass(Op: BaseVector, DemandedElts: DemandedSrcElts,
6347 InterestedClasses, Depth: Depth + 1);
6348 if (!DemandedSubElts.isZero()) {
6349 KnownFPClass SubKnown = computeKnownFPClass(
6350 Op: SubVector, DemandedElts: DemandedSubElts, InterestedClasses, Depth: Depth + 1);
6351 Known = DemandedSrcElts.isZero() ? SubKnown : (Known | SubKnown);
6352 }
6353 } else {
6354 Known = computeKnownFPClass(Op: SubVector, InterestedClasses, Depth: Depth + 1);
6355 if (!Known.isUnknown())
6356 Known |= computeKnownFPClass(Op: BaseVector, InterestedClasses, Depth: Depth + 1);
6357 }
6358 break;
6359 }
6360 case ISD::SELECT:
6361 case ISD::VSELECT: {
6362 // TODO: Add adjustKnownFPClassForSelectArm clamp recognition as in
6363 // IR-level ValueTracking.
6364 KnownFPClass KnownFalseClass = computeKnownFPClass(
6365 Op: Op.getOperand(i: 2), DemandedElts, InterestedClasses, Depth: Depth + 1);
6366 if (KnownFalseClass.isUnknown())
6367 break;
6368 KnownFPClass KnownTrueClass = computeKnownFPClass(
6369 Op: Op.getOperand(i: 1), DemandedElts, InterestedClasses, Depth: Depth + 1);
6370 Known = KnownTrueClass.intersectWith(RHS: KnownFalseClass);
6371 break;
6372 }
6373 default:
6374 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
6375 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID) {
6376 TLI->computeKnownFPClassForTargetNode(Op, Known, DemandedElts, DAG: *this,
6377 Depth);
6378 }
6379 break;
6380 }
6381
6382 return Known;
6383}
6384
6385bool SelectionDAG::isKnownNeverNaN(SDValue Op, bool SNaN,
6386 unsigned Depth) const {
6387 APInt DemandedElts = getDemandAllEltsMask(V: Op);
6388 return isKnownNeverNaN(Op, DemandedElts, SNaN, Depth);
6389}
6390
6391bool SelectionDAG::isKnownNeverNaN(SDValue Op, const APInt &DemandedElts,
6392 bool SNaN, unsigned Depth) const {
6393 assert(!DemandedElts.isZero() && "No demanded elements");
6394
6395 // If we're told that NaNs won't happen, assume they won't.
6396 if (Op->getFlags().hasNoNaNs())
6397 return true;
6398
6399 if (Depth >= MaxRecursionDepth)
6400 return false; // Limit search depth.
6401
6402 unsigned Opcode = Op.getOpcode();
6403 switch (Opcode) {
6404 case ISD::FADD:
6405 case ISD::FSUB:
6406 case ISD::FMUL:
6407 case ISD::FDIV:
6408 case ISD::FREM:
6409 case ISD::FSIN:
6410 case ISD::FCOS:
6411 case ISD::FTAN:
6412 case ISD::FASIN:
6413 case ISD::FACOS:
6414 case ISD::FATAN:
6415 case ISD::FATAN2:
6416 case ISD::FSINH:
6417 case ISD::FCOSH:
6418 case ISD::FTANH:
6419 case ISD::FMA:
6420 case ISD::FMULADD:
6421 case ISD::FMAD: {
6422 if (SNaN)
6423 return true;
6424 // TODO: Need isKnownNeverInfinity
6425 return false;
6426 }
6427 case ISD::FCANONICALIZE:
6428 case ISD::FEXP:
6429 case ISD::FEXP2:
6430 case ISD::FEXP10:
6431 case ISD::FTRUNC:
6432 case ISD::FFLOOR:
6433 case ISD::FCEIL:
6434 case ISD::FROUND:
6435 case ISD::FROUNDEVEN:
6436 case ISD::LROUND:
6437 case ISD::LLROUND:
6438 case ISD::FRINT:
6439 case ISD::LRINT:
6440 case ISD::LLRINT:
6441 case ISD::FNEARBYINT:
6442 case ISD::FLDEXP: {
6443 if (SNaN)
6444 return true;
6445 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1);
6446 }
6447 case ISD::FABS:
6448 case ISD::FNEG:
6449 case ISD::FCOPYSIGN: {
6450 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1);
6451 }
6452 case ISD::SELECT:
6453 return isKnownNeverNaN(Op: Op.getOperand(i: 1), DemandedElts, SNaN, Depth: Depth + 1) &&
6454 isKnownNeverNaN(Op: Op.getOperand(i: 2), DemandedElts, SNaN, Depth: Depth + 1);
6455 case ISD::FP_EXTEND:
6456 case ISD::FP_ROUND: {
6457 if (SNaN)
6458 return true;
6459 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1);
6460 }
6461 case ISD::SINT_TO_FP:
6462 case ISD::UINT_TO_FP:
6463 return true;
6464 case ISD::FSQRT: // Need is known positive
6465 case ISD::FLOG:
6466 case ISD::FLOG2:
6467 case ISD::FLOG10:
6468 case ISD::FPOWI:
6469 case ISD::FPOW: {
6470 if (SNaN)
6471 return true;
6472 // TODO: Refine on operand
6473 return false;
6474 }
6475 case ISD::FMINNUM:
6476 case ISD::FMAXNUM:
6477 case ISD::FMINIMUMNUM:
6478 case ISD::FMAXIMUMNUM: {
6479 // Only one needs to be known not-nan, since it will be returned if the
6480 // other ends up being one.
6481 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1) ||
6482 isKnownNeverNaN(Op: Op.getOperand(i: 1), DemandedElts, SNaN, Depth: Depth + 1);
6483 }
6484 case ISD::FMINNUM_IEEE:
6485 case ISD::FMAXNUM_IEEE: {
6486 if (SNaN)
6487 return true;
6488 // This can return a NaN if either operand is an sNaN, or if both operands
6489 // are NaN.
6490 return (isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN: false, Depth: Depth + 1) &&
6491 isKnownNeverSNaN(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1)) ||
6492 (isKnownNeverNaN(Op: Op.getOperand(i: 1), DemandedElts, SNaN: false, Depth: Depth + 1) &&
6493 isKnownNeverSNaN(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1));
6494 }
6495 case ISD::FMINIMUM:
6496 case ISD::FMAXIMUM: {
6497 // TODO: Does this quiet or return the origina NaN as-is?
6498 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1) &&
6499 isKnownNeverNaN(Op: Op.getOperand(i: 1), DemandedElts, SNaN, Depth: Depth + 1);
6500 }
6501 case ISD::EXTRACT_VECTOR_ELT: {
6502 SDValue Src = Op.getOperand(i: 0);
6503 auto *Idx = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1));
6504 EVT SrcVT = Src.getValueType();
6505 if (SrcVT.isFixedLengthVector() && Idx &&
6506 Idx->getAPIntValue().ult(RHS: SrcVT.getVectorNumElements())) {
6507 APInt DemandedSrcElts = APInt::getOneBitSet(numBits: SrcVT.getVectorNumElements(),
6508 BitNo: Idx->getZExtValue());
6509 return isKnownNeverNaN(Op: Src, DemandedElts: DemandedSrcElts, SNaN, Depth: Depth + 1);
6510 }
6511 return isKnownNeverNaN(Op: Src, SNaN, Depth: Depth + 1);
6512 }
6513 case ISD::EXTRACT_SUBVECTOR: {
6514 SDValue Src = Op.getOperand(i: 0);
6515 if (Src.getValueType().isFixedLengthVector()) {
6516 unsigned Idx = Op.getConstantOperandVal(i: 1);
6517 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
6518 APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
6519 return isKnownNeverNaN(Op: Src, DemandedElts: DemandedSrcElts, SNaN, Depth: Depth + 1);
6520 }
6521 return isKnownNeverNaN(Op: Src, SNaN, Depth: Depth + 1);
6522 }
6523 case ISD::INSERT_SUBVECTOR: {
6524 SDValue BaseVector = Op.getOperand(i: 0);
6525 SDValue SubVector = Op.getOperand(i: 1);
6526 EVT BaseVectorVT = BaseVector.getValueType();
6527 if (BaseVectorVT.isFixedLengthVector()) {
6528 unsigned Idx = Op.getConstantOperandVal(i: 2);
6529 unsigned NumBaseElts = BaseVectorVT.getVectorNumElements();
6530 unsigned NumSubElts = SubVector.getValueType().getVectorNumElements();
6531
6532 // Clear/Extract the bits at the position where the subvector will be
6533 // inserted.
6534 APInt DemandedMask =
6535 APInt::getBitsSet(numBits: NumBaseElts, loBit: Idx, hiBit: Idx + NumSubElts);
6536 APInt DemandedSrcElts = DemandedElts & ~DemandedMask;
6537 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
6538
6539 bool NeverNaN = true;
6540 if (!DemandedSrcElts.isZero())
6541 NeverNaN &=
6542 isKnownNeverNaN(Op: BaseVector, DemandedElts: DemandedSrcElts, SNaN, Depth: Depth + 1);
6543 if (NeverNaN && !DemandedSubElts.isZero())
6544 NeverNaN &=
6545 isKnownNeverNaN(Op: SubVector, DemandedElts: DemandedSubElts, SNaN, Depth: Depth + 1);
6546 return NeverNaN;
6547 }
6548 return isKnownNeverNaN(Op: BaseVector, SNaN, Depth: Depth + 1) &&
6549 isKnownNeverNaN(Op: SubVector, SNaN, Depth: Depth + 1);
6550 }
6551 case ISD::BUILD_VECTOR: {
6552 unsigned NumElts = Op.getNumOperands();
6553 for (unsigned I = 0; I != NumElts; ++I)
6554 if (DemandedElts[I] &&
6555 !isKnownNeverNaN(Op: Op.getOperand(i: I), SNaN, Depth: Depth + 1))
6556 return false;
6557 return true;
6558 }
6559 case ISD::SPLAT_VECTOR:
6560 return isKnownNeverNaN(Op: Op.getOperand(i: 0), SNaN, Depth: Depth + 1);
6561 case ISD::AssertNoFPClass: {
6562 FPClassTest NoFPClass =
6563 static_cast<FPClassTest>(Op.getConstantOperandVal(i: 1));
6564 if ((NoFPClass & fcNan) == fcNan)
6565 return true;
6566 if (SNaN && (NoFPClass & fcSNan) == fcSNan)
6567 return true;
6568 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1);
6569 }
6570 default:
6571 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
6572 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID) {
6573 return TLI->isKnownNeverNaNForTargetNode(Op, DemandedElts, DAG: *this, SNaN,
6574 Depth);
6575 }
6576 break;
6577 }
6578
6579 FPClassTest NanMask = SNaN ? fcSNan : fcNan;
6580 KnownFPClass Known = computeKnownFPClass(Op, DemandedElts, InterestedClasses: NanMask, Depth);
6581 return Known.isKnownNever(Mask: NanMask);
6582}
6583
6584bool SelectionDAG::isKnownNeverLogicalZero(SDValue Op, unsigned Depth) const {
6585 APInt DemandedElts = getDemandAllEltsMask(V: Op);
6586 return isKnownNeverLogicalZero(Op, DemandedElts, Depth);
6587}
6588
6589bool SelectionDAG::isKnownNeverLogicalZero(SDValue Op,
6590 const APInt &DemandedElts,
6591 unsigned Depth) const {
6592 assert(!DemandedElts.isZero() && "No demanded elements");
6593 EVT VT = Op.getValueType();
6594 KnownFPClass Known =
6595 computeKnownFPClass(Op, DemandedElts, InterestedClasses: fcZero | fcSubnormal, Depth);
6596 return Known.isKnownNeverLogicalZero(Mode: getDenormalMode(VT));
6597}
6598
6599bool SelectionDAG::isKnownNeverZero(SDValue Op, unsigned Depth) const {
6600 APInt DemandedElts = getDemandAllEltsMask(V: Op);
6601 return isKnownNeverZero(Op, DemandedElts, Depth);
6602}
6603
6604bool SelectionDAG::isKnownNeverZero(SDValue Op, const APInt &DemandedElts,
6605 unsigned Depth) const {
6606 if (Depth >= MaxRecursionDepth)
6607 return false; // Limit search depth.
6608
6609 EVT OpVT = Op.getValueType();
6610 unsigned BitWidth = OpVT.getScalarSizeInBits();
6611
6612 assert(!Op.getValueType().isFloatingPoint() &&
6613 "Floating point types unsupported - use isKnownNeverLogicalZero");
6614
6615 // If the value is a constant, we can obviously see if it is a zero or not.
6616 auto IsNeverZero = [BitWidth](const ConstantSDNode *C) {
6617 APInt V = C->getAPIntValue().zextOrTrunc(width: BitWidth);
6618 return !V.isZero();
6619 };
6620
6621 if (ISD::matchUnaryPredicate(Op, DemandedElts, Match: IsNeverZero,
6622 /*AllowUndefs=*/false, /*AllowTruncation=*/true))
6623 return true;
6624
6625 // TODO: Recognize more cases here. Most of the cases are also incomplete to
6626 // some degree.
6627 switch (Op.getOpcode()) {
6628 default:
6629 break;
6630
6631 case ISD::EXTRACT_VECTOR_ELT: {
6632 SDValue InVec = Op.getOperand(i: 0);
6633 SDValue EltNo = Op.getOperand(i: 1);
6634 EVT VecVT = InVec.getValueType();
6635
6636 // Skip scalable vectors or implicit extensions.
6637 if (VecVT.isScalableVector() ||
6638 OpVT.getScalarSizeInBits() != VecVT.getScalarSizeInBits())
6639 break;
6640
6641 // If we know the element index, just demand that vector element, else for
6642 // an unknown element index, ignore DemandedElts and demand them all.
6643 const unsigned NumSrcElts = VecVT.getVectorNumElements();
6644 APInt DemandedSrcElts = APInt::getAllOnes(numBits: NumSrcElts);
6645 auto *ConstEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
6646 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(RHS: NumSrcElts))
6647 DemandedSrcElts =
6648 APInt::getOneBitSet(numBits: NumSrcElts, BitNo: ConstEltNo->getZExtValue());
6649
6650 return isKnownNeverZero(Op: InVec, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
6651 }
6652
6653 case ISD::OR:
6654 return isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1) ||
6655 isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6656
6657 case ISD::VSELECT:
6658 case ISD::SELECT:
6659 return isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1) &&
6660 isKnownNeverZero(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth + 1);
6661
6662 case ISD::SHL: {
6663 if (Op->getFlags().hasNoSignedWrap() || Op->getFlags().hasNoUnsignedWrap())
6664 return isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6665 KnownBits ValKnown =
6666 computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6667 // 1 << X is never zero.
6668 if (ValKnown.One[0])
6669 return true;
6670 // If max shift cnt of known ones is non-zero, result is non-zero.
6671 APInt MaxCnt = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1)
6672 .getMaxValue();
6673 if (MaxCnt.ult(RHS: ValKnown.getBitWidth()) &&
6674 !ValKnown.One.shl(ShiftAmt: MaxCnt).isZero())
6675 return true;
6676 break;
6677 }
6678
6679 case ISD::VECTOR_SHUFFLE: {
6680 if (Op.getValueType().isScalableVector())
6681 return false;
6682
6683 unsigned NumElts = DemandedElts.getBitWidth();
6684
6685 // All demanded elements from LHS and RHS must be known non-zero.
6686 // Demanded elements with undef shuffle mask elements are unknown.
6687
6688 APInt DemandedLHS, DemandedRHS;
6689 auto *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
6690 assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
6691 if (!getShuffleDemandedElts(SrcWidth: NumElts, Mask: SVN->getMask(), DemandedElts,
6692 DemandedLHS, DemandedRHS))
6693 return false;
6694
6695 return (!DemandedLHS ||
6696 isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts: DemandedLHS, Depth: Depth + 1)) &&
6697 (!DemandedRHS ||
6698 isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts: DemandedRHS, Depth: Depth + 1));
6699 }
6700
6701 case ISD::UADDSAT:
6702 case ISD::UMAX:
6703 return isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1) ||
6704 isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6705
6706 case ISD::UMIN:
6707 return isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1) &&
6708 isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6709
6710 // For smin/smax: If either operand is known negative/positive
6711 // respectively we don't need the other to be known at all.
6712 case ISD::SMAX: {
6713 KnownBits Op1 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
6714 if (Op1.isStrictlyPositive())
6715 return true;
6716
6717 KnownBits Op0 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6718 if (Op0.isStrictlyPositive())
6719 return true;
6720
6721 if (Op1.isNonZero() && Op0.isNonZero())
6722 return true;
6723
6724 return isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1) &&
6725 isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6726 }
6727 case ISD::SMIN: {
6728 KnownBits Op1 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
6729 if (Op1.isNegative())
6730 return true;
6731
6732 KnownBits Op0 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6733 if (Op0.isNegative())
6734 return true;
6735
6736 if (Op1.isNonZero() && Op0.isNonZero())
6737 return true;
6738
6739 return isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1) &&
6740 isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6741 }
6742
6743 case ISD::ROTL:
6744 case ISD::ROTR:
6745 case ISD::BITREVERSE:
6746 case ISD::BSWAP:
6747 case ISD::CTPOP:
6748 case ISD::ABS:
6749 case ISD::ABS_MIN_POISON:
6750 return isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6751
6752 case ISD::SRA:
6753 case ISD::SRL: {
6754 if (Op->getFlags().hasExact())
6755 return isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6756 KnownBits ValKnown =
6757 computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6758 if (ValKnown.isNegative())
6759 return true;
6760 // If max shift cnt of known ones is non-zero, result is non-zero.
6761 APInt MaxCnt = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1)
6762 .getMaxValue();
6763 if (MaxCnt.ult(RHS: ValKnown.getBitWidth()) &&
6764 !ValKnown.One.lshr(ShiftAmt: MaxCnt).isZero())
6765 return true;
6766 break;
6767 }
6768 case ISD::UDIV:
6769 case ISD::SDIV:
6770 // div exact can only produce a zero if the dividend is zero.
6771 // TODO: For udiv this is also true if Op1 u<= Op0
6772 if (Op->getFlags().hasExact())
6773 return isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6774 break;
6775
6776 case ISD::ADD:
6777 if (Op->getFlags().hasNoUnsignedWrap())
6778 if (isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1) ||
6779 isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1))
6780 return true;
6781 // TODO: There are a lot more cases we can prove for add.
6782 break;
6783
6784 case ISD::SUB: {
6785 if (isNullConstant(V: Op.getOperand(i: 0)))
6786 return isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
6787
6788 std::optional<bool> ne = KnownBits::ne(
6789 LHS: computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1),
6790 RHS: computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1));
6791 return ne && *ne;
6792 }
6793
6794 case ISD::MUL:
6795 if (Op->getFlags().hasNoSignedWrap() || Op->getFlags().hasNoUnsignedWrap())
6796 if (isKnownNeverZero(Op: Op.getOperand(i: 1), Depth: Depth + 1) &&
6797 isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1))
6798 return true;
6799 break;
6800
6801 case ISD::ZERO_EXTEND:
6802 case ISD::SIGN_EXTEND:
6803 return isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6804 case ISD::VSCALE: {
6805 const Function &F = getMachineFunction().getFunction();
6806 const APInt &Multiplier = Op.getConstantOperandAPInt(i: 0);
6807 ConstantRange CR =
6808 getVScaleRange(F: &F, BitWidth: Op.getScalarValueSizeInBits()).multiply(Other: Multiplier);
6809 if (!CR.contains(Val: APInt(CR.getBitWidth(), 0)))
6810 return true;
6811 break;
6812 }
6813 }
6814
6815 return computeKnownBits(Op, DemandedElts, Depth).isNonZero();
6816}
6817
6818bool SelectionDAG::cannotBeOrderedNegativeFP(SDValue Op) const {
6819 if (ConstantFPSDNode *C1 = isConstOrConstSplatFP(N: Op, AllowUndefs: true))
6820 return !C1->isNegative();
6821
6822 switch (Op.getOpcode()) {
6823 case ISD::FABS:
6824 case ISD::FEXP:
6825 case ISD::FEXP2:
6826 case ISD::FEXP10:
6827 return true;
6828 default:
6829 return false;
6830 }
6831
6832 llvm_unreachable("covered opcode switch");
6833}
6834
6835bool SelectionDAG::canIgnoreSignBitOfZero(const SDUse &Use) const {
6836 assert(Use.getValueType().isFloatingPoint());
6837 const SDNode *User = Use.getUser();
6838 if (User->getFlags().hasNoSignedZeros())
6839 return true;
6840
6841 unsigned OperandNo = Use.getOperandNo();
6842 // Check if this use is insensitive to the sign of zero
6843 switch (User->getOpcode()) {
6844 case ISD::SETCC:
6845 // Comparisons: IEEE-754 specifies +0.0 == -0.0.
6846 case ISD::FABS:
6847 // fabs always produces +0.0.
6848 return true;
6849 case ISD::FCOPYSIGN:
6850 // copysign overwrites the sign bit of the first operand.
6851 return OperandNo == 0;
6852 case ISD::FADD:
6853 case ISD::FSUB: {
6854 // Arithmetic with non-zero constants fixes the uncertainty around the
6855 // sign bit.
6856 SDValue Other = User->getOperand(Num: 1 - OperandNo);
6857 return isKnownNeverLogicalZero(Op: Other);
6858 }
6859 case ISD::FP_TO_SINT:
6860 case ISD::FP_TO_UINT:
6861 // fp-to-int conversions normalize signed zeros.
6862 return true;
6863 default:
6864 return false;
6865 }
6866}
6867
6868bool SelectionDAG::canIgnoreSignBitOfZero(SDValue Op) const {
6869 if (Op->getFlags().hasNoSignedZeros())
6870 return true;
6871 // FIXME: Limit the amount of checked uses to not introduce a compile-time
6872 // regression. Ideally, this should be implemented as a demanded-bits
6873 // optimization that stems from the users.
6874 if (Op->use_size() > 2)
6875 return false;
6876 return all_of(Range: Op->uses(),
6877 P: [&](const SDUse &Use) { return canIgnoreSignBitOfZero(Use); });
6878}
6879
6880bool SelectionDAG::isEqualTo(SDValue A, SDValue B) const {
6881 // Check the obvious case.
6882 if (A == B) return true;
6883
6884 // For negative and positive zero.
6885 if (const ConstantFPSDNode *CA = dyn_cast<ConstantFPSDNode>(Val&: A))
6886 if (const ConstantFPSDNode *CB = dyn_cast<ConstantFPSDNode>(Val&: B))
6887 if (CA->isZero() && CB->isZero()) return true;
6888
6889 // Otherwise they may not be equal.
6890 return false;
6891}
6892
6893// Only bits set in Mask must be negated, other bits may be arbitrary.
6894SDValue llvm::getBitwiseNotOperand(SDValue V, SDValue Mask, bool AllowUndefs) {
6895 if (isBitwiseNot(V, AllowUndefs))
6896 return V.getOperand(i: 0);
6897
6898 // Handle any_extend (not (truncate X)) pattern, where Mask only sets
6899 // bits in the non-extended part.
6900 ConstantSDNode *MaskC = isConstOrConstSplat(N: Mask);
6901 if (!MaskC || V.getOpcode() != ISD::ANY_EXTEND)
6902 return SDValue();
6903 SDValue ExtArg = V.getOperand(i: 0);
6904 if (ExtArg.getScalarValueSizeInBits() >=
6905 MaskC->getAPIntValue().getActiveBits() &&
6906 isBitwiseNot(V: ExtArg, AllowUndefs) &&
6907 ExtArg.getOperand(i: 0).getOpcode() == ISD::TRUNCATE &&
6908 ExtArg.getOperand(i: 0).getOperand(i: 0).getValueType() == V.getValueType())
6909 return ExtArg.getOperand(i: 0).getOperand(i: 0);
6910 return SDValue();
6911}
6912
6913static bool haveNoCommonBitsSetCommutative(SDValue A, SDValue B) {
6914 // Match masked merge pattern (X & ~M) op (Y & M)
6915 // Including degenerate case (X & ~M) op M
6916 auto MatchNoCommonBitsPattern = [&](SDValue Not, SDValue Mask,
6917 SDValue Other) {
6918 if (SDValue NotOperand =
6919 getBitwiseNotOperand(V: Not, Mask, /* AllowUndefs */ true)) {
6920 if (NotOperand->getOpcode() == ISD::ZERO_EXTEND ||
6921 NotOperand->getOpcode() == ISD::TRUNCATE)
6922 NotOperand = NotOperand->getOperand(Num: 0);
6923
6924 if (Other == NotOperand)
6925 return true;
6926 if (Other->getOpcode() == ISD::AND)
6927 return NotOperand == Other->getOperand(Num: 0) ||
6928 NotOperand == Other->getOperand(Num: 1);
6929 }
6930 return false;
6931 };
6932
6933 if (A->getOpcode() == ISD::ZERO_EXTEND || A->getOpcode() == ISD::TRUNCATE)
6934 A = A->getOperand(Num: 0);
6935
6936 if (B->getOpcode() == ISD::ZERO_EXTEND || B->getOpcode() == ISD::TRUNCATE)
6937 B = B->getOperand(Num: 0);
6938
6939 if (A->getOpcode() == ISD::AND)
6940 return MatchNoCommonBitsPattern(A->getOperand(Num: 0), A->getOperand(Num: 1), B) ||
6941 MatchNoCommonBitsPattern(A->getOperand(Num: 1), A->getOperand(Num: 0), B);
6942 return false;
6943}
6944
6945// FIXME: unify with llvm::haveNoCommonBitsSet.
6946bool SelectionDAG::haveNoCommonBitsSet(SDValue A, SDValue B) const {
6947 assert(A.getValueType() == B.getValueType() &&
6948 "Values must have the same type");
6949 if (haveNoCommonBitsSetCommutative(A, B) ||
6950 haveNoCommonBitsSetCommutative(A: B, B: A))
6951 return true;
6952 return KnownBits::haveNoCommonBitsSet(LHS: computeKnownBits(Op: A),
6953 RHS: computeKnownBits(Op: B));
6954}
6955
6956static SDValue FoldSTEP_VECTOR(const SDLoc &DL, EVT VT, SDValue Step,
6957 SelectionDAG &DAG) {
6958 if (cast<ConstantSDNode>(Val&: Step)->isZero())
6959 return DAG.getConstant(Val: 0, DL, VT);
6960
6961 return SDValue();
6962}
6963
6964static SDValue FoldBUILD_VECTOR(const SDLoc &DL, EVT VT,
6965 ArrayRef<SDValue> Ops,
6966 SelectionDAG &DAG) {
6967 int NumOps = Ops.size();
6968 assert(NumOps != 0 && "Can't build an empty vector!");
6969 assert(!VT.isScalableVector() &&
6970 "BUILD_VECTOR cannot be used with scalable types");
6971 assert(VT.getVectorNumElements() == (unsigned)NumOps &&
6972 "Incorrect element count in BUILD_VECTOR!");
6973
6974 // BUILD_VECTOR of UNDEFs is UNDEF.
6975 bool AllPoison = true;
6976 if (llvm::all_of(Range&: Ops, P: [&AllPoison](SDValue Op) {
6977 AllPoison &= Op.getOpcode() == ISD::POISON;
6978 return Op.isUndef();
6979 }))
6980 return AllPoison ? DAG.getPOISON(VT) : DAG.getUNDEF(VT);
6981
6982 // BUILD_VECTOR of seq extract/insert from the same vector + type is Identity.
6983 SDValue IdentitySrc;
6984 bool IsIdentity = true;
6985 for (int i = 0; i != NumOps; ++i) {
6986 if (Ops[i].getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
6987 Ops[i].getOperand(i: 0).getValueType() != VT ||
6988 (IdentitySrc && Ops[i].getOperand(i: 0) != IdentitySrc) ||
6989 !isa<ConstantSDNode>(Val: Ops[i].getOperand(i: 1)) ||
6990 Ops[i].getConstantOperandAPInt(i: 1) != i) {
6991 IsIdentity = false;
6992 break;
6993 }
6994 IdentitySrc = Ops[i].getOperand(i: 0);
6995 }
6996 if (IsIdentity)
6997 return IdentitySrc;
6998
6999 return SDValue();
7000}
7001
7002/// Try to simplify vector concatenation to an input value, undef, or build
7003/// vector.
7004static SDValue foldCONCAT_VECTORS(const SDLoc &DL, EVT VT,
7005 ArrayRef<SDValue> Ops,
7006 SelectionDAG &DAG) {
7007 assert(!Ops.empty() && "Can't concatenate an empty list of vectors!");
7008 assert(llvm::all_of(Ops,
7009 [Ops](SDValue Op) {
7010 return Ops[0].getValueType() == Op.getValueType();
7011 }) &&
7012 "Concatenation of vectors with inconsistent value types!");
7013 assert((Ops[0].getValueType().getVectorElementCount() * Ops.size()) ==
7014 VT.getVectorElementCount() &&
7015 "Incorrect element count in vector concatenation!");
7016
7017 if (Ops.size() == 1)
7018 return Ops[0];
7019
7020 // Concat of UNDEFs is UNDEF.
7021 bool AllPoison = true;
7022 if (llvm::all_of(Range&: Ops, P: [&AllPoison](SDValue Op) {
7023 AllPoison &= Op.getOpcode() == ISD::POISON;
7024 return Op.isUndef();
7025 }))
7026 return AllPoison ? DAG.getPOISON(VT) : DAG.getUNDEF(VT);
7027
7028 // Scan the operands and look for extract operations from a single source
7029 // that correspond to insertion at the same location via this concatenation:
7030 // concat (extract X, 0*subvec_elts), (extract X, 1*subvec_elts), ...
7031 SDValue IdentitySrc;
7032 bool IsIdentity = true;
7033 for (unsigned i = 0, e = Ops.size(); i != e; ++i) {
7034 SDValue Op = Ops[i];
7035 unsigned IdentityIndex = i * Op.getValueType().getVectorMinNumElements();
7036 if (Op.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
7037 Op.getOperand(i: 0).getValueType() != VT ||
7038 (IdentitySrc && Op.getOperand(i: 0) != IdentitySrc) ||
7039 Op.getConstantOperandVal(i: 1) != IdentityIndex) {
7040 IsIdentity = false;
7041 break;
7042 }
7043 assert((!IdentitySrc || IdentitySrc == Op.getOperand(0)) &&
7044 "Unexpected identity source vector for concat of extracts");
7045 IdentitySrc = Op.getOperand(i: 0);
7046 }
7047 if (IsIdentity) {
7048 assert(IdentitySrc && "Failed to set source vector of extracts");
7049 return IdentitySrc;
7050 }
7051
7052 // The code below this point is only designed to work for fixed width
7053 // vectors, so we bail out for now.
7054 if (VT.isScalableVector())
7055 return SDValue();
7056
7057 // A CONCAT_VECTOR of scalar sources, such as UNDEF, BUILD_VECTOR and
7058 // single-element INSERT_VECTOR_ELT operands can be simplified to one big
7059 // BUILD_VECTOR.
7060 // FIXME: Add support for SCALAR_TO_VECTOR as well.
7061 EVT SVT = VT.getScalarType();
7062 SmallVector<SDValue, 16> Elts;
7063 for (SDValue Op : Ops) {
7064 EVT OpVT = Op.getValueType();
7065 if (Op.getOpcode() == ISD::POISON)
7066 Elts.append(NumInputs: OpVT.getVectorNumElements(), Elt: DAG.getPOISON(VT: SVT));
7067 else if (Op.getOpcode() == ISD::UNDEF)
7068 Elts.append(NumInputs: OpVT.getVectorNumElements(), Elt: DAG.getUNDEF(VT: SVT));
7069 else if (Op.getOpcode() == ISD::BUILD_VECTOR)
7070 Elts.append(in_start: Op->op_begin(), in_end: Op->op_end());
7071 else if (Op.getOpcode() == ISD::INSERT_VECTOR_ELT &&
7072 OpVT.getVectorNumElements() == 1 &&
7073 isNullConstant(V: Op.getOperand(i: 2)))
7074 Elts.push_back(Elt: Op.getOperand(i: 1));
7075 else
7076 return SDValue();
7077 }
7078
7079 // BUILD_VECTOR requires all inputs to be of the same type, find the
7080 // maximum type and extend them all.
7081 for (SDValue Op : Elts)
7082 SVT = (SVT.bitsLT(VT: Op.getValueType()) ? Op.getValueType() : SVT);
7083
7084 if (SVT.bitsGT(VT: VT.getScalarType())) {
7085 for (SDValue &Op : Elts) {
7086 if (Op.getOpcode() == ISD::POISON)
7087 Op = DAG.getPOISON(VT: SVT);
7088 else if (Op.getOpcode() == ISD::UNDEF)
7089 Op = DAG.getUNDEF(VT: SVT);
7090 else
7091 Op = DAG.getTargetLoweringInfo().isZExtFree(FromTy: Op.getValueType(), ToTy: SVT)
7092 ? DAG.getZExtOrTrunc(Op, DL, VT: SVT)
7093 : DAG.getSExtOrTrunc(Op, DL, VT: SVT);
7094 }
7095 }
7096
7097 SDValue V = DAG.getBuildVector(VT, DL, Ops: Elts);
7098 NewSDValueDbgMsg(V, Msg: "New node fold concat vectors: ", G: &DAG);
7099 return V;
7100}
7101
7102/// Gets or creates the specified node.
7103SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT) {
7104 SDVTList VTs = getVTList(VT);
7105 SDNodeKey ID(Opcode, VTs, {});
7106 FoldingSetInsertToken InsertToken;
7107 if (SDNode *E = lookupNode(Key: ID, DL, InsertToken))
7108 return SDValue(E, 0);
7109
7110 auto *N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
7111 CSEMap.insert(N, Token: InsertToken);
7112
7113 InsertNode(N);
7114 SDValue V = SDValue(N, 0);
7115 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
7116 return V;
7117}
7118
7119SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7120 SDValue N1) {
7121 SDNodeFlags Flags;
7122 if (Inserter)
7123 Flags = Inserter->getFlags();
7124 return getNode(Opcode, DL, VT, Operand: N1, Flags);
7125}
7126
7127SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7128 SDValue N1, const SDNodeFlags Flags) {
7129 assert(N1.getOpcode() != ISD::DELETED_NODE && "Operand is DELETED_NODE!");
7130
7131 // Constant fold unary operations with a vector integer or float operand.
7132 switch (Opcode) {
7133 default:
7134 // FIXME: Entirely reasonable to perform folding of other unary
7135 // operations here as the need arises.
7136 break;
7137 case ISD::FNEG:
7138 case ISD::FABS:
7139 case ISD::FCEIL:
7140 case ISD::FTRUNC:
7141 case ISD::FFLOOR:
7142 case ISD::FP_EXTEND:
7143 case ISD::FP_TO_SINT:
7144 case ISD::FP_TO_UINT:
7145 case ISD::FP_TO_FP16:
7146 case ISD::FP_TO_BF16:
7147 case ISD::TRUNCATE:
7148 case ISD::ANY_EXTEND:
7149 case ISD::ZERO_EXTEND:
7150 case ISD::SIGN_EXTEND:
7151 case ISD::UINT_TO_FP:
7152 case ISD::SINT_TO_FP:
7153 case ISD::FP16_TO_FP:
7154 case ISD::BF16_TO_FP:
7155 case ISD::BITCAST:
7156 case ISD::ABS:
7157 case ISD::ABS_MIN_POISON:
7158 case ISD::BITREVERSE:
7159 case ISD::BSWAP:
7160 case ISD::CTLZ:
7161 case ISD::CTLZ_ZERO_POISON:
7162 case ISD::CTTZ:
7163 case ISD::CTTZ_ZERO_POISON:
7164 case ISD::CTPOP:
7165 case ISD::PARITY:
7166 case ISD::CTLS:
7167 case ISD::VECREDUCE_ADD:
7168 case ISD::VECREDUCE_SMAX:
7169 case ISD::VECREDUCE_SMIN:
7170 case ISD::VECREDUCE_UMAX:
7171 case ISD::VECREDUCE_UMIN:
7172 case ISD::VECREDUCE_MUL:
7173 case ISD::VECREDUCE_AND:
7174 case ISD::VECREDUCE_OR:
7175 case ISD::VECREDUCE_XOR:
7176 case ISD::STEP_VECTOR: {
7177 SDValue Ops = {N1};
7178 if (SDValue Fold = FoldConstantArithmetic(Opcode, DL, VT, Ops))
7179 return Fold;
7180 }
7181 }
7182
7183 unsigned OpOpcode = N1.getNode()->getOpcode();
7184 switch (Opcode) {
7185 case ISD::STEP_VECTOR:
7186 assert(VT.isScalableVector() &&
7187 "STEP_VECTOR can only be used with scalable types");
7188 assert(OpOpcode == ISD::TargetConstant &&
7189 VT.getVectorElementType() == N1.getValueType() &&
7190 "Unexpected step operand");
7191 break;
7192 case ISD::FREEZE:
7193 assert(VT == N1.getValueType() && "Unexpected VT!");
7194 if (isGuaranteedNotToBeUndefOrPoison(Op: N1, Kind: UndefPoisonKind::UndefOrPoison))
7195 return N1;
7196 break;
7197 case ISD::TokenFactor:
7198 case ISD::MERGE_VALUES:
7199 case ISD::CONCAT_VECTORS:
7200 return N1; // Factor, merge or concat of one node? No need.
7201 case ISD::BUILD_VECTOR: {
7202 // Attempt to simplify BUILD_VECTOR.
7203 SDValue Ops[] = {N1};
7204 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, DAG&: *this))
7205 return V;
7206 break;
7207 }
7208 case ISD::FP_ROUND: llvm_unreachable("Invalid method to make FP_ROUND node");
7209 case ISD::FP_EXTEND:
7210 assert(VT.isFloatingPoint() && N1.getValueType().isFloatingPoint() &&
7211 "Invalid FP cast!");
7212 if (N1.getValueType() == VT) return N1; // noop conversion.
7213 assert((!VT.isVector() || VT.getVectorElementCount() ==
7214 N1.getValueType().getVectorElementCount()) &&
7215 "Vector element count mismatch!");
7216 assert(N1.getValueType().bitsLT(VT) && "Invalid fpext node, dst < src!");
7217 if (N1.isUndef())
7218 return getUNDEF(VT);
7219 break;
7220 case ISD::FP_TO_SINT:
7221 case ISD::FP_TO_UINT:
7222 if (N1.isUndef())
7223 return getUNDEF(VT);
7224 break;
7225 case ISD::SINT_TO_FP:
7226 case ISD::UINT_TO_FP:
7227 // [us]itofp(undef) = 0, because the result value is bounded.
7228 if (N1.isUndef())
7229 return getConstantFP(Val: 0.0, DL, VT);
7230 break;
7231 case ISD::SIGN_EXTEND:
7232 assert(VT.isInteger() && N1.getValueType().isInteger() &&
7233 "Invalid SIGN_EXTEND!");
7234 assert(VT.isVector() == N1.getValueType().isVector() &&
7235 "SIGN_EXTEND result type type should be vector iff the operand "
7236 "type is vector!");
7237 if (N1.getValueType() == VT) return N1; // noop extension
7238 assert((!VT.isVector() || VT.getVectorElementCount() ==
7239 N1.getValueType().getVectorElementCount()) &&
7240 "Vector element count mismatch!");
7241 assert(N1.getValueType().bitsLT(VT) && "Invalid sext node, dst < src!");
7242 if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND) {
7243 SDNodeFlags Flags;
7244 if (OpOpcode == ISD::ZERO_EXTEND)
7245 Flags.setNonNeg(N1->getFlags().hasNonNeg());
7246 SDValue NewVal = getNode(Opcode: OpOpcode, DL, VT, N1: N1.getOperand(i: 0), Flags);
7247 transferDbgValues(From: N1, To: NewVal);
7248 return NewVal;
7249 }
7250
7251 if (OpOpcode == ISD::POISON)
7252 return getPOISON(VT);
7253
7254 if (N1.isUndef())
7255 // sext(undef) = 0, because the top bits will all be the same.
7256 return getConstant(Val: 0, DL, VT);
7257
7258 // Skip unnecessary sext_inreg pattern:
7259 // (sext (trunc x)) -> x iff the upper bits are all signbits.
7260 if (OpOpcode == ISD::TRUNCATE) {
7261 SDValue OpOp = N1.getOperand(i: 0);
7262 if (OpOp.getValueType() == VT) {
7263 unsigned NumSignExtBits =
7264 VT.getScalarSizeInBits() - N1.getScalarValueSizeInBits();
7265 if (ComputeNumSignBits(Op: OpOp) > NumSignExtBits) {
7266 transferDbgValues(From: N1, To: OpOp);
7267 return OpOp;
7268 }
7269 }
7270 }
7271 break;
7272 case ISD::ZERO_EXTEND:
7273 assert(VT.isInteger() && N1.getValueType().isInteger() &&
7274 "Invalid ZERO_EXTEND!");
7275 assert(VT.isVector() == N1.getValueType().isVector() &&
7276 "ZERO_EXTEND result type type should be vector iff the operand "
7277 "type is vector!");
7278 if (N1.getValueType() == VT) return N1; // noop extension
7279 assert((!VT.isVector() || VT.getVectorElementCount() ==
7280 N1.getValueType().getVectorElementCount()) &&
7281 "Vector element count mismatch!");
7282 assert(N1.getValueType().bitsLT(VT) && "Invalid zext node, dst < src!");
7283 if (OpOpcode == ISD::ZERO_EXTEND) { // (zext (zext x)) -> (zext x)
7284 SDNodeFlags Flags;
7285 Flags.setNonNeg(N1->getFlags().hasNonNeg());
7286 SDValue NewVal =
7287 getNode(Opcode: ISD::ZERO_EXTEND, DL, VT, N1: N1.getOperand(i: 0), Flags);
7288 transferDbgValues(From: N1, To: NewVal);
7289 return NewVal;
7290 }
7291
7292 if (OpOpcode == ISD::POISON)
7293 return getPOISON(VT);
7294
7295 if (N1.isUndef())
7296 // zext(undef) = 0, because the top bits will be zero.
7297 return getConstant(Val: 0, DL, VT);
7298
7299 // Skip unnecessary zext_inreg pattern:
7300 // (zext (trunc x)) -> x iff the upper bits are known zero.
7301 // TODO: Remove (zext (trunc (and x, c))) exception which some targets
7302 // use to recognise zext_inreg patterns.
7303 if (OpOpcode == ISD::TRUNCATE) {
7304 SDValue OpOp = N1.getOperand(i: 0);
7305 if (OpOp.getValueType() == VT) {
7306 if (OpOp.getOpcode() != ISD::AND) {
7307 APInt HiBits = APInt::getBitsSetFrom(numBits: VT.getScalarSizeInBits(),
7308 loBit: N1.getScalarValueSizeInBits());
7309 if (MaskedValueIsZero(V: OpOp, Mask: HiBits)) {
7310 transferDbgValues(From: N1, To: OpOp);
7311 return OpOp;
7312 }
7313 }
7314 }
7315 }
7316 break;
7317 case ISD::ANY_EXTEND:
7318 assert(VT.isInteger() && N1.getValueType().isInteger() &&
7319 "Invalid ANY_EXTEND!");
7320 assert(VT.isVector() == N1.getValueType().isVector() &&
7321 "ANY_EXTEND result type type should be vector iff the operand "
7322 "type is vector!");
7323 if (N1.getValueType() == VT) return N1; // noop extension
7324 assert((!VT.isVector() || VT.getVectorElementCount() ==
7325 N1.getValueType().getVectorElementCount()) &&
7326 "Vector element count mismatch!");
7327 assert(N1.getValueType().bitsLT(VT) && "Invalid anyext node, dst < src!");
7328
7329 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
7330 OpOpcode == ISD::ANY_EXTEND) {
7331 SDNodeFlags Flags;
7332 if (OpOpcode == ISD::ZERO_EXTEND)
7333 Flags.setNonNeg(N1->getFlags().hasNonNeg());
7334 // (ext (zext x)) -> (zext x) and (ext (sext x)) -> (sext x)
7335 return getNode(Opcode: OpOpcode, DL, VT, N1: N1.getOperand(i: 0), Flags);
7336 }
7337 if (N1.isUndef())
7338 return getUNDEF(VT);
7339
7340 // (ext (trunc x)) -> x
7341 if (OpOpcode == ISD::TRUNCATE) {
7342 SDValue OpOp = N1.getOperand(i: 0);
7343 if (OpOp.getValueType() == VT) {
7344 transferDbgValues(From: N1, To: OpOp);
7345 return OpOp;
7346 }
7347 }
7348 break;
7349 case ISD::TRUNCATE:
7350 assert(VT.isInteger() && N1.getValueType().isInteger() &&
7351 "Invalid TRUNCATE!");
7352 assert(VT.isVector() == N1.getValueType().isVector() &&
7353 "TRUNCATE result type type should be vector iff the operand "
7354 "type is vector!");
7355 if (N1.getValueType() == VT) return N1; // noop truncate
7356 assert((!VT.isVector() || VT.getVectorElementCount() ==
7357 N1.getValueType().getVectorElementCount()) &&
7358 "Vector element count mismatch!");
7359 assert(N1.getValueType().bitsGT(VT) && "Invalid truncate node, src < dst!");
7360 if (OpOpcode == ISD::TRUNCATE)
7361 return getNode(Opcode: ISD::TRUNCATE, DL, VT, N1: N1.getOperand(i: 0));
7362 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
7363 OpOpcode == ISD::ANY_EXTEND) {
7364 // If the source is smaller than the dest, we still need an extend.
7365 if (N1.getOperand(i: 0).getValueType().getScalarType().bitsLT(
7366 VT: VT.getScalarType())) {
7367 SDNodeFlags Flags;
7368 if (OpOpcode == ISD::ZERO_EXTEND)
7369 Flags.setNonNeg(N1->getFlags().hasNonNeg());
7370 return getNode(Opcode: OpOpcode, DL, VT, N1: N1.getOperand(i: 0), Flags);
7371 }
7372 if (N1.getOperand(i: 0).getValueType().bitsGT(VT))
7373 return getNode(Opcode: ISD::TRUNCATE, DL, VT, N1: N1.getOperand(i: 0));
7374 return N1.getOperand(i: 0);
7375 }
7376 if (N1.isUndef())
7377 return getUNDEF(VT);
7378 if (OpOpcode == ISD::VSCALE && !NewNodesMustHaveLegalTypes)
7379 return getVScale(DL, VT,
7380 MulImm: N1.getConstantOperandAPInt(i: 0).trunc(width: VT.getSizeInBits()));
7381 break;
7382 case ISD::ANY_EXTEND_VECTOR_INREG:
7383 case ISD::ZERO_EXTEND_VECTOR_INREG:
7384 case ISD::SIGN_EXTEND_VECTOR_INREG:
7385 assert(VT.isVector() && "This DAG node is restricted to vector types.");
7386 assert(N1.getValueType().bitsLE(VT) &&
7387 "The input must be the same size or smaller than the result.");
7388 assert(VT.getVectorMinNumElements() <
7389 N1.getValueType().getVectorMinNumElements() &&
7390 "The destination vector type must have fewer lanes than the input.");
7391 break;
7392 case ISD::ABS:
7393 assert(VT.isInteger() && VT == N1.getValueType() && "Invalid ABS!");
7394 if (N1.isUndef())
7395 return getConstant(Val: 0, DL, VT);
7396 break;
7397 case ISD::ABS_MIN_POISON:
7398 assert(VT.isInteger() && VT == N1.getValueType() &&
7399 "Invalid ABS_MIN_POISON!");
7400 if (N1.isUndef())
7401 return getConstant(Val: 0, DL, VT);
7402 break;
7403 case ISD::BSWAP:
7404 assert(VT.isInteger() && VT == N1.getValueType() && "Invalid BSWAP!");
7405 assert((VT.getScalarSizeInBits() % 16 == 0) &&
7406 "BSWAP types must be a multiple of 16 bits!");
7407 if (N1.isUndef())
7408 return getUNDEF(VT);
7409 // bswap(bswap(X)) -> X.
7410 if (OpOpcode == ISD::BSWAP)
7411 return N1.getOperand(i: 0);
7412 break;
7413 case ISD::BITREVERSE:
7414 assert(VT.isInteger() && VT == N1.getValueType() && "Invalid BITREVERSE!");
7415 if (N1.isUndef())
7416 return getUNDEF(VT);
7417 break;
7418 case ISD::BITCAST:
7419 assert(VT.getSizeInBits() == N1.getValueSizeInBits() &&
7420 "Cannot BITCAST between types of different sizes!");
7421 if (VT == N1.getValueType()) return N1; // noop conversion.
7422 if (OpOpcode == ISD::BITCAST) // bitconv(bitconv(x)) -> bitconv(x)
7423 return getNode(Opcode: ISD::BITCAST, DL, VT, N1: N1.getOperand(i: 0));
7424 if (N1.isUndef())
7425 return getUNDEF(VT);
7426 break;
7427 case ISD::SCALAR_TO_VECTOR:
7428 assert(VT.isVector() && !N1.getValueType().isVector() &&
7429 (VT.getVectorElementType() == N1.getValueType() ||
7430 (VT.getVectorElementType().isInteger() &&
7431 N1.getValueType().isInteger() &&
7432 VT.getVectorElementType().bitsLE(N1.getValueType()))) &&
7433 "Illegal SCALAR_TO_VECTOR node!");
7434 if (N1.isUndef())
7435 return getUNDEF(VT);
7436 // scalar_to_vector(extract_vector_elt V, 0) -> V, top bits are undefined.
7437 if (OpOpcode == ISD::EXTRACT_VECTOR_ELT &&
7438 isa<ConstantSDNode>(Val: N1.getOperand(i: 1)) &&
7439 N1.getConstantOperandVal(i: 1) == 0 &&
7440 N1.getOperand(i: 0).getValueType() == VT)
7441 return N1.getOperand(i: 0);
7442 break;
7443 case ISD::FNEG:
7444 // Negation of an unknown bag of bits is still completely undefined.
7445 if (N1.isUndef())
7446 return getUNDEF(VT);
7447
7448 if (OpOpcode == ISD::FNEG) // --X -> X
7449 return N1.getOperand(i: 0);
7450 break;
7451 case ISD::FABS:
7452 if (OpOpcode == ISD::FNEG) // abs(-X) -> abs(X)
7453 return getNode(Opcode: ISD::FABS, DL, VT, N1: N1.getOperand(i: 0));
7454 break;
7455 case ISD::VSCALE:
7456 assert(VT == N1.getValueType() && "Unexpected VT!");
7457 break;
7458 case ISD::CTPOP:
7459 if (N1.getValueType().getScalarType() == MVT::i1)
7460 return N1;
7461 break;
7462 case ISD::CTLZ:
7463 case ISD::CTTZ:
7464 if (N1.getValueType().getScalarType() == MVT::i1)
7465 return getNOT(DL, Val: N1, VT: N1.getValueType());
7466 break;
7467 case ISD::CTLS:
7468 if (N1.getValueType().getScalarType() == MVT::i1)
7469 return getConstant(Val: 0, DL, VT);
7470 break;
7471 case ISD::VECREDUCE_ADD:
7472 if (N1.getValueType().getScalarType() == MVT::i1)
7473 return getNode(Opcode: ISD::VECREDUCE_XOR, DL, VT, N1);
7474 break;
7475 case ISD::VECREDUCE_SMIN:
7476 case ISD::VECREDUCE_UMAX:
7477 if (N1.getValueType().getScalarType() == MVT::i1)
7478 return getNode(Opcode: ISD::VECREDUCE_OR, DL, VT, N1);
7479 break;
7480 case ISD::VECREDUCE_SMAX:
7481 case ISD::VECREDUCE_UMIN:
7482 if (N1.getValueType().getScalarType() == MVT::i1)
7483 return getNode(Opcode: ISD::VECREDUCE_AND, DL, VT, N1);
7484 break;
7485 case ISD::VECTOR_REPEAT:
7486 assert(N1.getValueType().isFixedLengthVector() &&
7487 "VECTOR_REPEAT requires a fixed-length vector operand");
7488 assert(VT.isScalableVector() &&
7489 "VECTOR_REPEAT requires a scalable vector result");
7490 assert(N1.getValueType().getVectorNumElements() ==
7491 VT.getVectorMinNumElements() &&
7492 "VECTOR_REPEAT operand and result element counts must match");
7493 if (VT.getVectorMinNumElements() == 1)
7494 return getSplatVector(
7495 VT, DL, Op: getExtractVectorElt(DL, VT: VT.getVectorElementType(), Vec: N1, Idx: 0));
7496 break;
7497 case ISD::SPLAT_VECTOR:
7498 assert(VT.isVector() && "Wrong return type!");
7499 // FIXME: Hexagon uses i32 scalar for a floating point zero vector so allow
7500 // that for now.
7501 assert((VT.getVectorElementType() == N1.getValueType() ||
7502 (VT.isFloatingPoint() && N1.getValueType() == MVT::i32) ||
7503 (VT.getVectorElementType().isInteger() &&
7504 N1.getValueType().isInteger() &&
7505 VT.getVectorElementType().bitsLE(N1.getValueType()))) &&
7506 "Wrong operand type!");
7507 break;
7508 }
7509
7510 SDNode *N;
7511 SDVTList VTs = getVTList(VT);
7512 SDValue Ops[] = {N1};
7513 if (VT != MVT::Glue) { // Don't CSE glue producing nodes
7514 SDNodeKey ID(Opcode, VTs, Ops);
7515 FoldingSetInsertToken InsertToken;
7516 if (SDNode *E = lookupNode(Key: ID, DL, InsertToken)) {
7517 E->intersectFlagsWith(Flags);
7518 return SDValue(E, 0);
7519 }
7520
7521 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
7522 N->setFlags(Flags);
7523 createOperands(Node: N, Vals: Ops);
7524 CSEMap.insert(N, Token: InsertToken);
7525 } else {
7526 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
7527 createOperands(Node: N, Vals: Ops);
7528 }
7529
7530 InsertNode(N);
7531 SDValue V = SDValue(N, 0);
7532 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
7533 return V;
7534}
7535
7536static APInt getIntegerIdentity(unsigned Opcode, unsigned BitWidth) {
7537 switch (Opcode) {
7538 default:
7539 llvm_unreachable("Unexpected integer identity opcode");
7540 case ISD::ADD:
7541 case ISD::OR:
7542 case ISD::XOR:
7543 case ISD::UMAX:
7544 return APInt::getZero(numBits: BitWidth);
7545 case ISD::MUL:
7546 return APInt(BitWidth, 1);
7547 case ISD::AND:
7548 case ISD::UMIN:
7549 return APInt::getAllOnes(numBits: BitWidth);
7550 case ISD::SMAX:
7551 return APInt::getSignedMinValue(numBits: BitWidth);
7552 case ISD::SMIN:
7553 return APInt::getSignedMaxValue(numBits: BitWidth);
7554 }
7555}
7556
7557static std::optional<APInt> FoldValue(unsigned Opcode, const APInt &C1,
7558 const APInt &C2) {
7559 switch (Opcode) {
7560 case ISD::ADD: return C1 + C2;
7561 case ISD::SUB: return C1 - C2;
7562 case ISD::MUL: return C1 * C2;
7563 case ISD::AND: return C1 & C2;
7564 case ISD::OR: return C1 | C2;
7565 case ISD::XOR: return C1 ^ C2;
7566 case ISD::SHL: return C1 << C2;
7567 case ISD::SRL: return C1.lshr(ShiftAmt: C2);
7568 case ISD::SRA: return C1.ashr(ShiftAmt: C2);
7569 case ISD::ROTL: return C1.rotl(rotateAmt: C2);
7570 case ISD::ROTR: return C1.rotr(rotateAmt: C2);
7571 case ISD::SMIN: return C1.sle(RHS: C2) ? C1 : C2;
7572 case ISD::SMAX: return C1.sge(RHS: C2) ? C1 : C2;
7573 case ISD::UMIN: return C1.ule(RHS: C2) ? C1 : C2;
7574 case ISD::UMAX: return C1.uge(RHS: C2) ? C1 : C2;
7575 case ISD::SADDSAT: return C1.sadd_sat(RHS: C2);
7576 case ISD::UADDSAT: return C1.uadd_sat(RHS: C2);
7577 case ISD::SSUBSAT: return C1.ssub_sat(RHS: C2);
7578 case ISD::USUBSAT: return C1.usub_sat(RHS: C2);
7579 case ISD::SSHLSAT: return C1.sshl_sat(RHS: C2);
7580 case ISD::USHLSAT: return C1.ushl_sat(RHS: C2);
7581 case ISD::UDIV:
7582 if (!C2.getBoolValue())
7583 break;
7584 return C1.udiv(RHS: C2);
7585 case ISD::UREM:
7586 if (!C2.getBoolValue())
7587 break;
7588 return C1.urem(RHS: C2);
7589 case ISD::SDIV:
7590 if (!C2.getBoolValue())
7591 break;
7592 return C1.sdiv(RHS: C2);
7593 case ISD::SREM:
7594 if (!C2.getBoolValue())
7595 break;
7596 return C1.srem(RHS: C2);
7597 case ISD::AVGFLOORS:
7598 return APIntOps::avgFloorS(C1, C2);
7599 case ISD::AVGFLOORU:
7600 return APIntOps::avgFloorU(C1, C2);
7601 case ISD::AVGCEILS:
7602 return APIntOps::avgCeilS(C1, C2);
7603 case ISD::AVGCEILU:
7604 return APIntOps::avgCeilU(C1, C2);
7605 case ISD::ABDS:
7606 return APIntOps::abds(A: C1, B: C2);
7607 case ISD::ABDU:
7608 return APIntOps::abdu(A: C1, B: C2);
7609 case ISD::MULHS:
7610 return APIntOps::mulhs(C1, C2);
7611 case ISD::MULHU:
7612 return APIntOps::mulhu(C1, C2);
7613 case ISD::CLMUL:
7614 return APIntOps::clmul(LHS: C1, RHS: C2);
7615 case ISD::CLMULR:
7616 return APIntOps::clmulr(LHS: C1, RHS: C2);
7617 case ISD::CLMULH:
7618 return APIntOps::clmulh(LHS: C1, RHS: C2);
7619 case ISD::PEXT:
7620 return APIntOps::pext(Val: C1, Mask: C2);
7621 case ISD::PDEP:
7622 return APIntOps::pdep(Val: C1, Mask: C2);
7623 }
7624 return std::nullopt;
7625}
7626// Handle constant folding with UNDEF.
7627// TODO: Handle more cases.
7628static std::optional<APInt> FoldValueWithUndef(unsigned Opcode, const APInt &C1,
7629 bool IsUndef1, const APInt &C2,
7630 bool IsUndef2) {
7631 if (!(IsUndef1 || IsUndef2))
7632 return FoldValue(Opcode, C1, C2);
7633
7634 // Fold and(x, undef) -> 0
7635 // Fold mul(x, undef) -> 0
7636 if (Opcode == ISD::AND || Opcode == ISD::MUL)
7637 return APInt::getZero(numBits: C1.getBitWidth());
7638
7639 return std::nullopt;
7640}
7641
7642SDValue SelectionDAG::FoldSymbolOffset(unsigned Opcode, EVT VT,
7643 const GlobalAddressSDNode *GA,
7644 const SDNode *N2) {
7645 if (GA->getOpcode() != ISD::GlobalAddress)
7646 return SDValue();
7647 if (!TLI->isOffsetFoldingLegal(GA))
7648 return SDValue();
7649 auto *C2 = dyn_cast<ConstantSDNode>(Val: N2);
7650 if (!C2)
7651 return SDValue();
7652 int64_t Offset = C2->getSExtValue();
7653 switch (Opcode) {
7654 case ISD::ADD:
7655 case ISD::PTRADD:
7656 break;
7657 case ISD::SUB: Offset = -uint64_t(Offset); break;
7658 default: return SDValue();
7659 }
7660 return getGlobalAddress(GV: GA->getGlobal(), DL: SDLoc(C2), VT,
7661 Offset: GA->getOffset() + uint64_t(Offset));
7662}
7663
7664bool SelectionDAG::isUndef(unsigned Opcode, ArrayRef<SDValue> Ops) {
7665 switch (Opcode) {
7666 case ISD::SDIV:
7667 case ISD::UDIV:
7668 case ISD::SREM:
7669 case ISD::UREM: {
7670 // If a divisor is zero/undef or any element of a divisor vector is
7671 // zero/undef, the whole op is undef.
7672 assert(Ops.size() == 2 && "Div/rem should have 2 operands");
7673 SDValue Divisor = Ops[1];
7674 if (Divisor.isUndef() || isNullConstant(V: Divisor))
7675 return true;
7676
7677 return ISD::isBuildVectorOfConstantSDNodes(N: Divisor.getNode()) &&
7678 llvm::any_of(Range: Divisor->op_values(),
7679 P: [](SDValue V) { return V.isUndef() ||
7680 isNullConstant(V); });
7681 // TODO: Handle signed overflow.
7682 }
7683 // TODO: Handle oversized shifts.
7684 default:
7685 return false;
7686 }
7687}
7688
7689SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL,
7690 EVT VT, ArrayRef<SDValue> Ops,
7691 SDNodeFlags Flags) {
7692 // If the opcode is a target-specific ISD node, there's nothing we can
7693 // do here and the operand rules may not line up with the below, so
7694 // bail early.
7695 // We can't create a scalar CONCAT_VECTORS so skip it. It will break
7696 // for concats involving SPLAT_VECTOR. Concats of BUILD_VECTORS are handled by
7697 // foldCONCAT_VECTORS in getNode before this is called.
7698 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::CONCAT_VECTORS)
7699 return SDValue();
7700
7701 unsigned NumOps = Ops.size();
7702 if (NumOps == 0)
7703 return SDValue();
7704
7705 if (isUndef(Opcode, Ops))
7706 return getUNDEF(VT);
7707
7708 // Handle unary special cases.
7709 if (NumOps == 1) {
7710 SDValue N1 = Ops[0];
7711
7712 // Constant fold unary operations with an integer constant operand. Even
7713 // opaque constant will be folded, because the folding of unary operations
7714 // doesn't create new constants with different values. Nevertheless, the
7715 // opaque flag is preserved during folding to prevent future folding with
7716 // other constants.
7717 if (auto *C = dyn_cast<ConstantSDNode>(Val&: N1)) {
7718 const APInt &Val = C->getAPIntValue();
7719 switch (Opcode) {
7720 case ISD::SIGN_EXTEND:
7721 return getConstant(Val: Val.sextOrTrunc(width: VT.getSizeInBits()), DL, VT,
7722 isT: C->isTargetOpcode(), isO: C->isOpaque());
7723 case ISD::TRUNCATE:
7724 if (C->isOpaque())
7725 break;
7726 [[fallthrough]];
7727 case ISD::ZERO_EXTEND:
7728 return getConstant(Val: Val.zextOrTrunc(width: VT.getSizeInBits()), DL, VT,
7729 isT: C->isTargetOpcode(), isO: C->isOpaque());
7730 case ISD::ANY_EXTEND:
7731 // Some targets like RISCV prefer to sign extend some types.
7732 if (TLI->isSExtCheaperThanZExt(FromTy: N1.getValueType(), ToTy: VT))
7733 return getConstant(Val: Val.sextOrTrunc(width: VT.getSizeInBits()), DL, VT,
7734 isT: C->isTargetOpcode(), isO: C->isOpaque());
7735 return getConstant(Val: Val.zextOrTrunc(width: VT.getSizeInBits()), DL, VT,
7736 isT: C->isTargetOpcode(), isO: C->isOpaque());
7737 case ISD::ABS:
7738 return getConstant(Val: Val.abs(), DL, VT, isT: C->isTargetOpcode(),
7739 isO: C->isOpaque());
7740 case ISD::ABS_MIN_POISON:
7741 if (Val.isMinSignedValue())
7742 return getPOISON(VT);
7743 return getConstant(Val: Val.abs(), DL, VT, isT: C->isTargetOpcode(),
7744 isO: C->isOpaque());
7745 case ISD::BITREVERSE:
7746 return getConstant(Val: Val.reverseBits(), DL, VT, isT: C->isTargetOpcode(),
7747 isO: C->isOpaque());
7748 case ISD::BSWAP:
7749 return getConstant(Val: Val.byteSwap(), DL, VT, isT: C->isTargetOpcode(),
7750 isO: C->isOpaque());
7751 case ISD::CTPOP:
7752 return getConstant(Val: Val.popcount(), DL, VT, isT: C->isTargetOpcode(),
7753 isO: C->isOpaque());
7754 case ISD::PARITY:
7755 return getConstant(Val: Val.popcount() & 1, DL, VT, isT: C->isTargetOpcode(),
7756 isO: C->isOpaque());
7757 case ISD::CTLZ:
7758 case ISD::CTLZ_ZERO_POISON:
7759 return getConstant(Val: Val.countl_zero(), DL, VT, isT: C->isTargetOpcode(),
7760 isO: C->isOpaque());
7761 case ISD::CTTZ:
7762 case ISD::CTTZ_ZERO_POISON:
7763 return getConstant(Val: Val.countr_zero(), DL, VT, isT: C->isTargetOpcode(),
7764 isO: C->isOpaque());
7765 case ISD::CTLS:
7766 // CTLS returns the number of extra sign bits so subtract one.
7767 return getConstant(Val: Val.getNumSignBits() - 1, DL, VT,
7768 isT: C->isTargetOpcode(), isO: C->isOpaque());
7769 case ISD::UINT_TO_FP:
7770 case ISD::SINT_TO_FP: {
7771 APFloat FPV(VT.getFltSemantics(), APInt::getZero(numBits: VT.getSizeInBits()));
7772 (void)FPV.convertFromAPInt(Input: Val, IsSigned: Opcode == ISD::SINT_TO_FP,
7773 RM: APFloat::rmNearestTiesToEven);
7774 return getConstantFP(V: FPV, DL, VT);
7775 }
7776 case ISD::FP16_TO_FP:
7777 case ISD::BF16_TO_FP: {
7778 bool Ignored;
7779 APFloat FPV(Opcode == ISD::FP16_TO_FP ? APFloat::IEEEhalf()
7780 : APFloat::BFloat(),
7781 (Val.getBitWidth() == 16) ? Val : Val.trunc(width: 16));
7782
7783 // This can return overflow, underflow, or inexact; we don't care.
7784 // FIXME need to be more flexible about rounding mode.
7785 (void)FPV.convert(ToSemantics: VT.getFltSemantics(), RM: APFloat::rmNearestTiesToEven,
7786 losesInfo: &Ignored);
7787 return getConstantFP(V: FPV, DL, VT);
7788 }
7789 case ISD::STEP_VECTOR:
7790 if (SDValue V = FoldSTEP_VECTOR(DL, VT, Step: N1, DAG&: *this))
7791 return V;
7792 break;
7793 case ISD::BITCAST:
7794 if (VT == MVT::f16 && C->getValueType(ResNo: 0) == MVT::i16)
7795 return getConstantFP(V: APFloat(APFloat::IEEEhalf(), Val), DL, VT);
7796 if (VT == MVT::f32 && C->getValueType(ResNo: 0) == MVT::i32)
7797 return getConstantFP(V: APFloat(APFloat::IEEEsingle(), Val), DL, VT);
7798 if (VT == MVT::f64 && C->getValueType(ResNo: 0) == MVT::i64)
7799 return getConstantFP(V: APFloat(APFloat::IEEEdouble(), Val), DL, VT);
7800 if (VT == MVT::f128 && C->getValueType(ResNo: 0) == MVT::i128)
7801 return getConstantFP(V: APFloat(APFloat::IEEEquad(), Val), DL, VT);
7802 break;
7803 }
7804 }
7805
7806 // Constant fold unary operations with a floating point constant operand.
7807 if (auto *C = dyn_cast<ConstantFPSDNode>(Val&: N1)) {
7808 APFloat V = C->getValueAPF(); // make copy
7809 switch (Opcode) {
7810 case ISD::FNEG:
7811 V.changeSign();
7812 return getConstantFP(V, DL, VT);
7813 case ISD::FABS:
7814 V.clearSign();
7815 return getConstantFP(V, DL, VT);
7816 case ISD::FCEIL: {
7817 APFloat::opStatus fs = V.roundToIntegral(RM: APFloat::rmTowardPositive);
7818 if (fs == APFloat::opOK || fs == APFloat::opInexact)
7819 return getConstantFP(V, DL, VT);
7820 return SDValue();
7821 }
7822 case ISD::FTRUNC: {
7823 APFloat::opStatus fs = V.roundToIntegral(RM: APFloat::rmTowardZero);
7824 if (fs == APFloat::opOK || fs == APFloat::opInexact)
7825 return getConstantFP(V, DL, VT);
7826 return SDValue();
7827 }
7828 case ISD::FFLOOR: {
7829 APFloat::opStatus fs = V.roundToIntegral(RM: APFloat::rmTowardNegative);
7830 if (fs == APFloat::opOK || fs == APFloat::opInexact)
7831 return getConstantFP(V, DL, VT);
7832 return SDValue();
7833 }
7834 case ISD::FP_EXTEND: {
7835 bool ignored;
7836 // This can return overflow, underflow, or inexact; we don't care.
7837 // FIXME need to be more flexible about rounding mode.
7838 (void)V.convert(ToSemantics: VT.getFltSemantics(), RM: APFloat::rmNearestTiesToEven,
7839 losesInfo: &ignored);
7840 return getConstantFP(V, DL, VT);
7841 }
7842 case ISD::FP_TO_SINT:
7843 case ISD::FP_TO_UINT: {
7844 bool ignored;
7845 APSInt IntVal(VT.getSizeInBits(), Opcode == ISD::FP_TO_UINT);
7846 // FIXME need to be more flexible about rounding mode.
7847 APFloat::opStatus s =
7848 V.convertToInteger(Result&: IntVal, RM: APFloat::rmTowardZero, IsExact: &ignored);
7849 if (s == APFloat::opInvalidOp) // inexact is OK, in fact usual
7850 break;
7851 return getConstant(Val: IntVal, DL, VT);
7852 }
7853 case ISD::FP_TO_FP16:
7854 case ISD::FP_TO_BF16: {
7855 bool Ignored;
7856 // This can return overflow, underflow, or inexact; we don't care.
7857 // FIXME need to be more flexible about rounding mode.
7858 (void)V.convert(ToSemantics: Opcode == ISD::FP_TO_FP16 ? APFloat::IEEEhalf()
7859 : APFloat::BFloat(),
7860 RM: APFloat::rmNearestTiesToEven, losesInfo: &Ignored);
7861 return getConstant(Val: V.bitcastToAPInt().getZExtValue(), DL, VT);
7862 }
7863 case ISD::BITCAST:
7864 if (VT == MVT::i16 && C->getValueType(ResNo: 0) == MVT::f16)
7865 return getConstant(Val: (uint16_t)V.bitcastToAPInt().getZExtValue(), DL,
7866 VT);
7867 if (VT == MVT::i16 && C->getValueType(ResNo: 0) == MVT::bf16)
7868 return getConstant(Val: (uint16_t)V.bitcastToAPInt().getZExtValue(), DL,
7869 VT);
7870 if (VT == MVT::i32 && C->getValueType(ResNo: 0) == MVT::f32)
7871 return getConstant(Val: (uint32_t)V.bitcastToAPInt().getZExtValue(), DL,
7872 VT);
7873 if (VT == MVT::i64 && C->getValueType(ResNo: 0) == MVT::f64)
7874 return getConstant(Val: V.bitcastToAPInt().getZExtValue(), DL, VT);
7875 break;
7876 }
7877 }
7878
7879 // Early-out if we failed to constant fold a bitcast.
7880 if (Opcode == ISD::BITCAST)
7881 return SDValue();
7882
7883 // Constant fold integer vector reductions with constant BUILD_VECTORs.
7884 if ((Opcode == ISD::VECREDUCE_ADD || Opcode == ISD::VECREDUCE_SMAX ||
7885 Opcode == ISD::VECREDUCE_SMIN || Opcode == ISD::VECREDUCE_UMAX ||
7886 Opcode == ISD::VECREDUCE_UMIN || Opcode == ISD::VECREDUCE_MUL ||
7887 Opcode == ISD::VECREDUCE_OR || Opcode == ISD::VECREDUCE_XOR ||
7888 Opcode == ISD::VECREDUCE_AND) &&
7889 ISD::isBuildVectorOfConstantSDNodes(N: N1.getNode())) {
7890 unsigned EltBits = N1.getValueType().getScalarSizeInBits();
7891 unsigned BaseOpcode = ISD::getVecReduceBaseOpcode(VecReduceOpcode: Opcode);
7892 APInt Acc = getIntegerIdentity(Opcode: BaseOpcode, BitWidth: EltBits);
7893 for (SDValue Elt : N1->op_values()) {
7894 if (Elt.getOpcode() == ISD::POISON)
7895 return getPOISON(VT);
7896 if (Elt.isUndef() || cast<ConstantSDNode>(Val&: Elt)->isOpaque())
7897 return SDValue();
7898 APInt Value = cast<ConstantSDNode>(Val&: Elt)->getAPIntValue().trunc(width: EltBits);
7899 std::optional<APInt> Folded = FoldValue(Opcode: BaseOpcode, C1: Acc, C2: Value);
7900 assert(Folded &&
7901 "Expected vector reduction base opcode to be foldable");
7902 Acc = *Folded;
7903 }
7904 EVT EltVT = N1.getValueType().getScalarType();
7905 return getAnyExtOrTrunc(Op: getConstant(Val: Acc, DL, VT: EltVT), DL, VT);
7906 }
7907 }
7908
7909 // Handle binops special cases.
7910 if (NumOps == 2) {
7911 if (SDValue CFP = foldConstantFPMath(Opcode, DL, VT, Ops))
7912 return CFP;
7913
7914 if (auto *C1 = dyn_cast<ConstantSDNode>(Val: Ops[0])) {
7915 if (auto *C2 = dyn_cast<ConstantSDNode>(Val: Ops[1])) {
7916 if (C1->isOpaque() || C2->isOpaque())
7917 return SDValue();
7918
7919 std::optional<APInt> FoldAttempt =
7920 FoldValue(Opcode, C1: C1->getAPIntValue(), C2: C2->getAPIntValue());
7921 if (!FoldAttempt)
7922 return SDValue();
7923
7924 SDValue Folded = getConstant(Val: *FoldAttempt, DL, VT);
7925 assert((!Folded || !VT.isVector()) &&
7926 "Can't fold vectors ops with scalar operands");
7927 return Folded;
7928 }
7929 }
7930
7931 // fold (add Sym, c) -> Sym+c
7932 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Val: Ops[0]))
7933 return FoldSymbolOffset(Opcode, VT, GA, N2: Ops[1].getNode());
7934 if (TLI->isCommutativeBinOp(Opcode))
7935 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Val: Ops[1]))
7936 return FoldSymbolOffset(Opcode, VT, GA, N2: Ops[0].getNode());
7937
7938 // fold (sext_in_reg c1) -> c2
7939 if (Opcode == ISD::SIGN_EXTEND_INREG) {
7940 EVT EVT = cast<VTSDNode>(Val: Ops[1])->getVT();
7941
7942 auto SignExtendInReg = [&](APInt Val, llvm::EVT ConstantVT) {
7943 unsigned FromBits = EVT.getScalarSizeInBits();
7944 Val <<= Val.getBitWidth() - FromBits;
7945 Val.ashrInPlace(ShiftAmt: Val.getBitWidth() - FromBits);
7946 return getConstant(Val, DL, VT: ConstantVT);
7947 };
7948
7949 if (auto *C1 = dyn_cast<ConstantSDNode>(Val: Ops[0])) {
7950 const APInt &Val = C1->getAPIntValue();
7951 return SignExtendInReg(Val, VT);
7952 }
7953
7954 if (ISD::isBuildVectorOfConstantSDNodes(N: Ops[0].getNode())) {
7955 SmallVector<SDValue, 8> ScalarOps;
7956 llvm::EVT OpVT = Ops[0].getOperand(i: 0).getValueType();
7957 for (int I = 0, E = VT.getVectorNumElements(); I != E; ++I) {
7958 SDValue Op = Ops[0].getOperand(i: I);
7959 if (Op.isUndef()) {
7960 ScalarOps.push_back(Elt: getUNDEF(VT: OpVT));
7961 continue;
7962 }
7963 const APInt &Val = cast<ConstantSDNode>(Val&: Op)->getAPIntValue();
7964 ScalarOps.push_back(Elt: SignExtendInReg(Val, OpVT));
7965 }
7966 return getBuildVector(VT, DL, Ops: ScalarOps);
7967 }
7968
7969 if (Ops[0].getOpcode() == ISD::SPLAT_VECTOR &&
7970 isa<ConstantSDNode>(Val: Ops[0].getOperand(i: 0)))
7971 return getNode(Opcode: ISD::SPLAT_VECTOR, DL, VT,
7972 N1: SignExtendInReg(Ops[0].getConstantOperandAPInt(i: 0),
7973 Ops[0].getOperand(i: 0).getValueType()));
7974 }
7975 }
7976
7977 // Handle fshl/fshr special cases.
7978 if (Opcode == ISD::FSHL || Opcode == ISD::FSHR) {
7979 auto *C1 = dyn_cast<ConstantSDNode>(Val: Ops[0]);
7980 auto *C2 = dyn_cast<ConstantSDNode>(Val: Ops[1]);
7981 auto *C3 = dyn_cast<ConstantSDNode>(Val: Ops[2]);
7982
7983 if (C1 && C2 && C3) {
7984 if (C1->isOpaque() || C2->isOpaque() || C3->isOpaque())
7985 return SDValue();
7986 const APInt &V1 = C1->getAPIntValue(), &V2 = C2->getAPIntValue(),
7987 &V3 = C3->getAPIntValue();
7988
7989 APInt FoldedVal = Opcode == ISD::FSHL ? APIntOps::fshl(Hi: V1, Lo: V2, Shift: V3)
7990 : APIntOps::fshr(Hi: V1, Lo: V2, Shift: V3);
7991 return getConstant(Val: FoldedVal, DL, VT);
7992 }
7993 }
7994
7995 // Handle fma/fmad special cases.
7996 if (Opcode == ISD::FMA || Opcode == ISD::FMAD || Opcode == ISD::FMULADD) {
7997 assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
7998 assert(Ops[0].getValueType() == VT && Ops[1].getValueType() == VT &&
7999 Ops[2].getValueType() == VT && "FMA types must match!");
8000 ConstantFPSDNode *C1 = dyn_cast<ConstantFPSDNode>(Val: Ops[0]);
8001 ConstantFPSDNode *C2 = dyn_cast<ConstantFPSDNode>(Val: Ops[1]);
8002 ConstantFPSDNode *C3 = dyn_cast<ConstantFPSDNode>(Val: Ops[2]);
8003 if (C1 && C2 && C3) {
8004 APFloat V1 = C1->getValueAPF();
8005 const APFloat &V2 = C2->getValueAPF();
8006 const APFloat &V3 = C3->getValueAPF();
8007 if (Opcode == ISD::FMAD || Opcode == ISD::FMULADD) {
8008 V1.multiply(RHS: V2, RM: APFloat::rmNearestTiesToEven);
8009 V1.add(RHS: V3, RM: APFloat::rmNearestTiesToEven);
8010 } else
8011 V1.fusedMultiplyAdd(Multiplicand: V2, Addend: V3, RM: APFloat::rmNearestTiesToEven);
8012 return getConstantFP(V: V1, DL, VT);
8013 }
8014 }
8015
8016 // This is for vector folding only from here on.
8017 if (!VT.isVector())
8018 return SDValue();
8019
8020 // Constant fold integer partial reductions with constant BUILD_VECTOR
8021 // operands. The reduction order is deliberately unspecified. Use the same
8022 // subvector layout as TargetLowering::expandPartialReduceMLA(), where input
8023 // lane I contributes to accumulator lane I % NumAccElts.
8024 if (Opcode == ISD::PARTIAL_REDUCE_SMLA ||
8025 Opcode == ISD::PARTIAL_REDUCE_UMLA ||
8026 Opcode == ISD::PARTIAL_REDUCE_SUMLA) {
8027 // These nodes have no scalar form, so unsupported cases must not fall
8028 // through to generic per-lane vector folding.
8029 if (!llvm::all_of(Range&: Ops, P: [](SDValue Op) {
8030 return ISD::isBuildVectorOfConstantSDNodes(N: Op.getNode());
8031 }))
8032 return SDValue();
8033
8034 unsigned AccEltBits = VT.getScalarSizeInBits();
8035 unsigned InputEltBits = Ops[1].getScalarValueSizeInBits();
8036 unsigned NumAccElts = VT.getVectorNumElements();
8037 unsigned NumInputElts = Ops[1].getValueType().getVectorNumElements();
8038 SmallVector<APInt, 8> Results(NumAccElts, APInt::getZero(numBits: AccEltBits));
8039 BitVector PoisonElts(NumAccElts);
8040
8041 for (unsigned I = 0; I != NumAccElts; ++I) {
8042 SDValue Elt = Ops[0].getOperand(i: I);
8043 if (Elt.getOpcode() == ISD::POISON) {
8044 PoisonElts.set(I);
8045 continue;
8046 }
8047 auto *C = dyn_cast<ConstantSDNode>(Val&: Elt);
8048 if (!C || C->isOpaque())
8049 return SDValue();
8050 Results[I] = C->getAPIntValue().trunc(width: AccEltBits);
8051 }
8052
8053 bool IsLHSSigned = Opcode != ISD::PARTIAL_REDUCE_UMLA;
8054 bool IsRHSSigned = Opcode == ISD::PARTIAL_REDUCE_SMLA;
8055 for (unsigned I = 0; I != NumInputElts; ++I) {
8056 const unsigned AccIdx = I % NumAccElts;
8057 SDValue LHSElt = Ops[1].getOperand(i: I);
8058 SDValue RHSElt = Ops[2].getOperand(i: I);
8059 if (LHSElt.getOpcode() == ISD::POISON ||
8060 RHSElt.getOpcode() == ISD::POISON) {
8061 PoisonElts.set(AccIdx);
8062 continue;
8063 }
8064
8065 auto *LHS = dyn_cast<ConstantSDNode>(Val&: LHSElt);
8066 auto *RHS = dyn_cast<ConstantSDNode>(Val&: RHSElt);
8067 if (!LHS || !RHS || LHS->isOpaque() || RHS->isOpaque())
8068 return SDValue();
8069
8070 APInt LHSVal = LHS->getAPIntValue().trunc(width: InputEltBits);
8071 APInt RHSVal = RHS->getAPIntValue().trunc(width: InputEltBits);
8072 LHSVal = IsLHSSigned ? LHSVal.sext(width: AccEltBits) : LHSVal.zext(width: AccEltBits);
8073 RHSVal = IsRHSSigned ? RHSVal.sext(width: AccEltBits) : RHSVal.zext(width: AccEltBits);
8074 Results[AccIdx] += LHSVal * RHSVal;
8075 }
8076
8077 // After type legalization the vector element type may not be a legal
8078 // scalar type (e.g. i16 on AArch64). Create the folded constants in the
8079 // promoted legal scalar type instead, matching the generic per-lane path
8080 // below. Bail out if legalization would narrow the type, since the lane
8081 // value would not fit.
8082 EVT AccEltVT = VT.getVectorElementType();
8083 EVT LegalSVT = AccEltVT;
8084 if (NewNodesMustHaveLegalTypes && LegalSVT.isInteger()) {
8085 LegalSVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT: LegalSVT);
8086 if (LegalSVT.bitsLT(VT: AccEltVT))
8087 return SDValue();
8088 }
8089
8090 SmallVector<SDValue, 8> ResultOps;
8091 for (unsigned I = 0; I != NumAccElts; ++I)
8092 ResultOps.push_back(
8093 Elt: PoisonElts[I] ? getPOISON(VT: LegalSVT)
8094 : getConstant(Val: Results[I].sext(width: LegalSVT.getSizeInBits()),
8095 DL, VT: LegalSVT));
8096 return getBuildVector(VT, DL, Ops: ResultOps);
8097 }
8098
8099 ElementCount NumElts = VT.getVectorElementCount();
8100
8101 // See if we can fold through any bitcasted integer ops.
8102 if (NumOps == 2 && VT.isFixedLengthVector() && VT.isInteger() &&
8103 Ops[0].getValueType() == VT && Ops[1].getValueType() == VT &&
8104 (Ops[0].getOpcode() == ISD::BITCAST ||
8105 Ops[1].getOpcode() == ISD::BITCAST)) {
8106 SDValue N1 = peekThroughBitcasts(V: Ops[0]);
8107 SDValue N2 = peekThroughBitcasts(V: Ops[1]);
8108 auto *BV1 = dyn_cast<BuildVectorSDNode>(Val&: N1);
8109 auto *BV2 = dyn_cast<BuildVectorSDNode>(Val&: N2);
8110 if (BV1 && BV2 && N1.getValueType().isInteger() &&
8111 N2.getValueType().isInteger()) {
8112 bool IsLE = getDataLayout().isLittleEndian();
8113 unsigned EltBits = VT.getScalarSizeInBits();
8114 SmallVector<APInt> RawBits1, RawBits2;
8115 BitVector UndefElts1, UndefElts2;
8116 if (BV1->getConstantRawBits(IsLittleEndian: IsLE, DstEltSizeInBits: EltBits, RawBitElements&: RawBits1, UndefElements&: UndefElts1) &&
8117 BV2->getConstantRawBits(IsLittleEndian: IsLE, DstEltSizeInBits: EltBits, RawBitElements&: RawBits2, UndefElements&: UndefElts2)) {
8118 SmallVector<APInt> RawBits;
8119 for (unsigned I = 0, E = NumElts.getFixedValue(); I != E; ++I) {
8120 std::optional<APInt> Fold = FoldValueWithUndef(
8121 Opcode, C1: RawBits1[I], IsUndef1: UndefElts1[I], C2: RawBits2[I], IsUndef2: UndefElts2[I]);
8122 if (!Fold)
8123 break;
8124 RawBits.push_back(Elt: *Fold);
8125 }
8126 if (RawBits.size() == NumElts.getFixedValue()) {
8127 // We have constant folded, but we might need to cast this again back
8128 // to the original (possibly legalized) type.
8129 EVT BVVT, BVEltVT;
8130 if (N1.getValueType() == VT) {
8131 BVVT = N1.getValueType();
8132 BVEltVT = BV1->getOperand(Num: 0).getValueType();
8133 } else {
8134 BVVT = N2.getValueType();
8135 BVEltVT = BV2->getOperand(Num: 0).getValueType();
8136 }
8137 unsigned BVEltBits = BVEltVT.getSizeInBits();
8138 SmallVector<APInt> DstBits;
8139 BitVector DstUndefs;
8140 BuildVectorSDNode::recastRawBits(IsLittleEndian: IsLE, DstEltSizeInBits: BVVT.getScalarSizeInBits(),
8141 DstBitElements&: DstBits, SrcBitElements: RawBits, DstUndefElements&: DstUndefs,
8142 SrcUndefElements: BitVector(RawBits.size(), false));
8143 SmallVector<SDValue> Ops(DstBits.size(), getUNDEF(VT: BVEltVT));
8144 for (unsigned I = 0, E = DstBits.size(); I != E; ++I) {
8145 if (DstUndefs[I])
8146 continue;
8147 Ops[I] = getConstant(Val: DstBits[I].sext(width: BVEltBits), DL, VT: BVEltVT);
8148 }
8149 return getBitcast(VT, V: getBuildVector(VT: BVVT, DL, Ops));
8150 }
8151 }
8152 }
8153 // Logic ops can be folded from raw integer bits - mainly for AVX512 masks.
8154 if (ISD::isBitwiseLogicOp(Opcode) && isa<ConstantSDNode>(Val: N1) &&
8155 isa<ConstantSDNode>(Val: N2)) {
8156 if (SDValue Res = FoldConstantArithmetic(Opcode, DL, VT: N1.getValueType(),
8157 Ops: {N1, N2}, Flags))
8158 return getBitcast(VT, V: Res);
8159 }
8160 }
8161
8162 // Fold (mul step_vector(C0), C1) to (step_vector(C0 * C1)).
8163 // (shl step_vector(C0), C1) -> (step_vector(C0 << C1))
8164 if ((Opcode == ISD::MUL || Opcode == ISD::SHL) &&
8165 Ops[0].getOpcode() == ISD::STEP_VECTOR) {
8166 APInt RHSVal;
8167 if (ISD::isConstantSplatVector(N: Ops[1].getNode(), SplatVal&: RHSVal)) {
8168 APInt NewStep = Opcode == ISD::MUL
8169 ? Ops[0].getConstantOperandAPInt(i: 0) * RHSVal
8170 : Ops[0].getConstantOperandAPInt(i: 0) << RHSVal;
8171 return getStepVector(DL, ResVT: VT, StepVal: NewStep);
8172 }
8173 }
8174
8175 auto IsScalarOrSameVectorSize = [NumElts](const SDValue &Op) {
8176 return !Op.getValueType().isVector() ||
8177 Op.getValueType().getVectorElementCount() == NumElts;
8178 };
8179
8180 auto IsBuildVectorSplatVectorOrUndef = [](const SDValue &Op) {
8181 return Op.isUndef() || Op.getOpcode() == ISD::CONDCODE ||
8182 Op.getOpcode() == ISD::BUILD_VECTOR ||
8183 Op.getOpcode() == ISD::SPLAT_VECTOR;
8184 };
8185
8186 // All operands must be vector types with the same number of elements as
8187 // the result type and must be either UNDEF or a build/splat vector
8188 // or UNDEF scalars.
8189 if (!llvm::all_of(Range&: Ops, P: IsBuildVectorSplatVectorOrUndef) ||
8190 !llvm::all_of(Range&: Ops, P: IsScalarOrSameVectorSize))
8191 return SDValue();
8192
8193 // If we are comparing vectors, then the result needs to be a i1 boolean that
8194 // is then extended back to the legal result type depending on how booleans
8195 // are represented.
8196 EVT SVT = (Opcode == ISD::SETCC ? MVT::i1 : VT.getScalarType());
8197 ISD::NodeType ExtendCode =
8198 (Opcode == ISD::SETCC && SVT != VT.getScalarType())
8199 ? TargetLowering::getExtendForContent(Content: TLI->getBooleanContents(Type: VT))
8200 : ISD::SIGN_EXTEND;
8201
8202 // Find legal integer scalar type for constant promotion and
8203 // ensure that its scalar size is at least as large as source.
8204 EVT LegalSVT = VT.getScalarType();
8205 if (NewNodesMustHaveLegalTypes && LegalSVT.isInteger()) {
8206 LegalSVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT: LegalSVT);
8207 if (LegalSVT.bitsLT(VT: VT.getScalarType()))
8208 return SDValue();
8209 }
8210
8211 // For scalable vector types we know we're dealing with SPLAT_VECTORs. We
8212 // only have one operand to check. For fixed-length vector types we may have
8213 // a combination of BUILD_VECTOR and SPLAT_VECTOR.
8214 unsigned NumVectorElts = NumElts.isScalable() ? 1 : NumElts.getFixedValue();
8215
8216 // Constant fold each scalar lane separately.
8217 SmallVector<SDValue, 4> ScalarResults;
8218 for (unsigned I = 0; I != NumVectorElts; I++) {
8219 SmallVector<SDValue, 4> ScalarOps;
8220 for (SDValue Op : Ops) {
8221 EVT InSVT = Op.getValueType().getScalarType();
8222 if (Op.getOpcode() != ISD::BUILD_VECTOR &&
8223 Op.getOpcode() != ISD::SPLAT_VECTOR) {
8224 if (Op.isUndef())
8225 ScalarOps.push_back(Elt: getUNDEF(VT: InSVT));
8226 else
8227 ScalarOps.push_back(Elt: Op);
8228 continue;
8229 }
8230
8231 SDValue ScalarOp =
8232 Op.getOperand(i: Op.getOpcode() == ISD::SPLAT_VECTOR ? 0 : I);
8233 EVT ScalarVT = ScalarOp.getValueType();
8234
8235 // Build vector (integer) scalar operands may need implicit
8236 // truncation - do this before constant folding.
8237 if (ScalarVT.isInteger() && ScalarVT.bitsGT(VT: InSVT)) {
8238 // Don't create illegally-typed nodes unless they're constants or undef
8239 // - if we fail to constant fold we can't guarantee the (dead) nodes
8240 // we're creating will be cleaned up before being visited for
8241 // legalization.
8242 if (NewNodesMustHaveLegalTypes && !ScalarOp.isUndef() &&
8243 !isa<ConstantSDNode>(Val: ScalarOp) &&
8244 TLI->getTypeAction(Context&: *getContext(), VT: InSVT) !=
8245 TargetLowering::TypeLegal)
8246 return SDValue();
8247 ScalarOp = getNode(Opcode: ISD::TRUNCATE, DL, VT: InSVT, N1: ScalarOp);
8248 }
8249
8250 ScalarOps.push_back(Elt: ScalarOp);
8251 }
8252
8253 // Constant fold the scalar operands.
8254 SDValue ScalarResult = getNode(Opcode, DL, VT: SVT, Ops: ScalarOps, Flags);
8255
8256 // Scalar folding only succeeded if the result is a constant or UNDEF.
8257 if (!ScalarResult.isUndef() && ScalarResult.getOpcode() != ISD::Constant &&
8258 ScalarResult.getOpcode() != ISD::ConstantFP)
8259 return SDValue();
8260
8261 // Legalize the (integer) scalar constant if necessary. We only do
8262 // this once we know the folding succeeded, since otherwise we would
8263 // get a node with illegal type which has a user.
8264 if (LegalSVT != SVT)
8265 ScalarResult = getNode(Opcode: ExtendCode, DL, VT: LegalSVT, N1: ScalarResult);
8266
8267 ScalarResults.push_back(Elt: ScalarResult);
8268 }
8269
8270 SDValue V = NumElts.isScalable() ? getSplatVector(VT, DL, Op: ScalarResults[0])
8271 : getBuildVector(VT, DL, Ops: ScalarResults);
8272 NewSDValueDbgMsg(V, Msg: "New node fold constant vector: ", G: this);
8273 return V;
8274}
8275
8276SDValue SelectionDAG::foldConstantFPMath(unsigned Opcode, const SDLoc &DL,
8277 EVT VT, ArrayRef<SDValue> Ops) {
8278 // TODO: Add support for unary/ternary fp opcodes.
8279 if (Ops.size() != 2)
8280 return SDValue();
8281
8282 // TODO: We don't do any constant folding for strict FP opcodes here, but we
8283 // should. That will require dealing with a potentially non-default
8284 // rounding mode, checking the "opStatus" return value from the APFloat
8285 // math calculations, and possibly other variations.
8286 SDValue N1 = Ops[0];
8287 SDValue N2 = Ops[1];
8288 ConstantFPSDNode *N1CFP = isConstOrConstSplatFP(N: N1, /*AllowUndefs*/ false);
8289 ConstantFPSDNode *N2CFP = isConstOrConstSplatFP(N: N2, /*AllowUndefs*/ false);
8290 if (N1CFP && N2CFP) {
8291 APFloat C1 = N1CFP->getValueAPF(); // make copy
8292 const APFloat &C2 = N2CFP->getValueAPF();
8293 switch (Opcode) {
8294 case ISD::FADD:
8295 C1.add(RHS: C2, RM: APFloat::rmNearestTiesToEven);
8296 return getConstantFP(V: C1, DL, VT);
8297 case ISD::FSUB:
8298 C1.subtract(RHS: C2, RM: APFloat::rmNearestTiesToEven);
8299 return getConstantFP(V: C1, DL, VT);
8300 case ISD::FMUL:
8301 C1.multiply(RHS: C2, RM: APFloat::rmNearestTiesToEven);
8302 return getConstantFP(V: C1, DL, VT);
8303 case ISD::FDIV:
8304 C1.divide(RHS: C2, RM: APFloat::rmNearestTiesToEven);
8305 return getConstantFP(V: C1, DL, VT);
8306 case ISD::FREM:
8307 C1.mod(RHS: C2);
8308 return getConstantFP(V: C1, DL, VT);
8309 case ISD::FCOPYSIGN:
8310 C1.copySign(RHS: C2);
8311 return getConstantFP(V: C1, DL, VT);
8312 case ISD::FMINNUM:
8313 return getConstantFP(V: minnum(A: C1, B: C2), DL, VT);
8314 case ISD::FMAXNUM:
8315 return getConstantFP(V: maxnum(A: C1, B: C2), DL, VT);
8316 case ISD::FMINIMUM:
8317 return getConstantFP(V: minimum(A: C1, B: C2), DL, VT);
8318 case ISD::FMAXIMUM:
8319 return getConstantFP(V: maximum(A: C1, B: C2), DL, VT);
8320 case ISD::FMINIMUMNUM:
8321 return getConstantFP(V: minimumnum(A: C1, B: C2), DL, VT);
8322 case ISD::FMAXIMUMNUM:
8323 return getConstantFP(V: maximumnum(A: C1, B: C2), DL, VT);
8324 default: break;
8325 }
8326 }
8327 if (N1CFP && Opcode == ISD::FP_ROUND) {
8328 APFloat C1 = N1CFP->getValueAPF(); // make copy
8329 bool Unused;
8330 // This can return overflow, underflow, or inexact; we don't care.
8331 // FIXME need to be more flexible about rounding mode.
8332 (void)C1.convert(ToSemantics: VT.getFltSemantics(), RM: APFloat::rmNearestTiesToEven,
8333 losesInfo: &Unused);
8334 return getConstantFP(V: C1, DL, VT);
8335 }
8336
8337 switch (Opcode) {
8338 case ISD::FSUB:
8339 // -0.0 - undef --> undef (consistent with "fneg undef")
8340 if (ConstantFPSDNode *N1C = isConstOrConstSplatFP(N: N1, /*AllowUndefs*/ true))
8341 if (N1C && N1C->getValueAPF().isNegZero() && N2.isUndef())
8342 return getUNDEF(VT);
8343 [[fallthrough]];
8344
8345 case ISD::FADD:
8346 case ISD::FMUL:
8347 case ISD::FDIV:
8348 case ISD::FREM:
8349 // If both operands are undef, the result is undef. If 1 operand is undef,
8350 // the result is NaN. This should match the behavior of the IR optimizer.
8351 if (N1.isUndef() && N2.isUndef())
8352 return getUNDEF(VT);
8353 if (N1.isUndef() || N2.isUndef())
8354 return getConstantFP(V: APFloat::getNaN(Sem: VT.getFltSemantics()), DL, VT);
8355 }
8356 return SDValue();
8357}
8358
8359SDValue SelectionDAG::FoldConstantBuildVector(BuildVectorSDNode *BV,
8360 const SDLoc &DL, EVT DstEltVT) {
8361 EVT SrcEltVT = BV->getValueType(ResNo: 0).getVectorElementType();
8362
8363 // If this is already the right type, we're done.
8364 if (SrcEltVT == DstEltVT)
8365 return SDValue(BV, 0);
8366
8367 unsigned SrcBitSize = SrcEltVT.getSizeInBits();
8368 unsigned DstBitSize = DstEltVT.getSizeInBits();
8369
8370 // If this is a conversion of N elements of one type to N elements of another
8371 // type, convert each element. This handles FP<->INT cases.
8372 if (SrcBitSize == DstBitSize) {
8373 SmallVector<SDValue, 8> Ops;
8374 for (SDValue Op : BV->op_values()) {
8375 // If the vector element type is not legal, the BUILD_VECTOR operands
8376 // are promoted and implicitly truncated. Make that explicit here.
8377 if (Op.getValueType() != SrcEltVT)
8378 Op = getNode(Opcode: ISD::TRUNCATE, DL, VT: SrcEltVT, N1: Op);
8379 Ops.push_back(Elt: getBitcast(VT: DstEltVT, V: Op));
8380 }
8381 EVT VT = EVT::getVectorVT(Context&: *getContext(), VT: DstEltVT,
8382 NumElements: BV->getValueType(ResNo: 0).getVectorNumElements());
8383 return getBuildVector(VT, DL, Ops);
8384 }
8385
8386 // Otherwise, we're growing or shrinking the elements. To avoid having to
8387 // handle annoying details of growing/shrinking FP values, we convert them to
8388 // int first.
8389 if (SrcEltVT.isFloatingPoint()) {
8390 // Convert the input float vector to a int vector where the elements are the
8391 // same sizes.
8392 EVT IntEltVT = EVT::getIntegerVT(Context&: *getContext(), BitWidth: SrcEltVT.getSizeInBits());
8393 if (SDValue Tmp = FoldConstantBuildVector(BV, DL, DstEltVT: IntEltVT))
8394 return FoldConstantBuildVector(BV: cast<BuildVectorSDNode>(Val&: Tmp), DL,
8395 DstEltVT);
8396 return SDValue();
8397 }
8398
8399 // Now we know the input is an integer vector. If the output is a FP type,
8400 // convert to integer first, then to FP of the right size.
8401 if (DstEltVT.isFloatingPoint()) {
8402 EVT IntEltVT = EVT::getIntegerVT(Context&: *getContext(), BitWidth: DstEltVT.getSizeInBits());
8403 if (SDValue Tmp = FoldConstantBuildVector(BV, DL, DstEltVT: IntEltVT))
8404 return FoldConstantBuildVector(BV: cast<BuildVectorSDNode>(Val&: Tmp), DL,
8405 DstEltVT);
8406 return SDValue();
8407 }
8408
8409 // Okay, we know the src/dst types are both integers of differing types.
8410 assert(SrcEltVT.isInteger() && DstEltVT.isInteger());
8411
8412 // Extract the constant raw bit data.
8413 BitVector UndefElements;
8414 SmallVector<APInt> RawBits;
8415 bool IsLE = getDataLayout().isLittleEndian();
8416 if (!BV->getConstantRawBits(IsLittleEndian: IsLE, DstEltSizeInBits: DstBitSize, RawBitElements&: RawBits, UndefElements))
8417 return SDValue();
8418
8419 SmallVector<SDValue, 8> Ops;
8420 for (unsigned I = 0, E = RawBits.size(); I != E; ++I) {
8421 if (UndefElements[I])
8422 Ops.push_back(Elt: getUNDEF(VT: DstEltVT));
8423 else
8424 Ops.push_back(Elt: getConstant(Val: RawBits[I], DL, VT: DstEltVT));
8425 }
8426
8427 EVT VT = EVT::getVectorVT(Context&: *getContext(), VT: DstEltVT, NumElements: Ops.size());
8428 return getBuildVector(VT, DL, Ops);
8429}
8430
8431SDValue SelectionDAG::getAssertAlign(const SDLoc &DL, SDValue Val, Align A) {
8432 assert(Val.getValueType().isInteger() && "Invalid AssertAlign!");
8433
8434 // There's no need to assert on a byte-aligned pointer. All pointers are at
8435 // least byte aligned.
8436 if (A == Align(1))
8437 return Val;
8438
8439 SDVTList VTs = getVTList(VT: Val.getValueType());
8440 SDValue Ops[] = {Val};
8441 SDNodeKey ID(ISD::AssertAlign, VTs, Ops);
8442 ID.AddInteger(I: A.value());
8443
8444 FoldingSetInsertToken InsertToken;
8445 if (SDNode *E = lookupNode(Key: ID, DL, InsertToken))
8446 return SDValue(E, 0);
8447
8448 auto *N =
8449 newSDNode<AssertAlignSDNode>(Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs, Args&: A);
8450 createOperands(Node: N, Vals: {Val});
8451
8452 CSEMap.insert(N, Token: InsertToken);
8453 InsertNode(N);
8454
8455 SDValue V(N, 0);
8456 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
8457 return V;
8458}
8459
8460SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
8461 SDValue N1, SDValue N2) {
8462 SDNodeFlags Flags;
8463 if (Inserter)
8464 Flags = Inserter->getFlags();
8465 return getNode(Opcode, DL, VT, N1, N2, Flags);
8466}
8467
8468void SelectionDAG::canonicalizeCommutativeBinop(unsigned Opcode, SDValue &N1,
8469 SDValue &N2) const {
8470 if (!TLI->isCommutativeBinOp(Opcode))
8471 return;
8472
8473 // Canonicalize:
8474 // binop(const, nonconst) -> binop(nonconst, const)
8475 bool N1C = isConstantIntBuildVectorOrConstantInt(N: N1);
8476 bool N2C = isConstantIntBuildVectorOrConstantInt(N: N2);
8477 bool N1CFP = isConstantFPBuildVectorOrConstantFP(N: N1);
8478 bool N2CFP = isConstantFPBuildVectorOrConstantFP(N: N2);
8479 if ((N1C && !N2C) || (N1CFP && !N2CFP))
8480 std::swap(a&: N1, b&: N2);
8481
8482 // Canonicalize:
8483 // binop(splat(x), step_vector) -> binop(step_vector, splat(x))
8484 else if (N1.getOpcode() == ISD::SPLAT_VECTOR &&
8485 N2.getOpcode() == ISD::STEP_VECTOR)
8486 std::swap(a&: N1, b&: N2);
8487}
8488
8489SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
8490 SDValue N1, SDValue N2, const SDNodeFlags Flags) {
8491 assert(N1.getOpcode() != ISD::DELETED_NODE &&
8492 N2.getOpcode() != ISD::DELETED_NODE &&
8493 "Operand is DELETED_NODE!");
8494
8495 canonicalizeCommutativeBinop(Opcode, N1, N2);
8496
8497 auto *N1C = dyn_cast<ConstantSDNode>(Val&: N1);
8498 auto *N2C = dyn_cast<ConstantSDNode>(Val&: N2);
8499
8500 // Don't allow undefs in vector splats - we might be returning N2 when folding
8501 // to zero etc.
8502 ConstantSDNode *N2CV =
8503 isConstOrConstSplat(N: N2, /*AllowUndefs*/ false, /*AllowTruncation*/ true);
8504
8505 switch (Opcode) {
8506 default: break;
8507 case ISD::TokenFactor:
8508 assert(VT == MVT::Other && N1.getValueType() == MVT::Other &&
8509 N2.getValueType() == MVT::Other && "Invalid token factor!");
8510 // Fold trivial token factors.
8511 if (N1.getOpcode() == ISD::EntryToken) return N2;
8512 if (N2.getOpcode() == ISD::EntryToken) return N1;
8513 if (N1 == N2) return N1;
8514 break;
8515 case ISD::BUILD_VECTOR: {
8516 // Attempt to simplify BUILD_VECTOR.
8517 SDValue Ops[] = {N1, N2};
8518 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, DAG&: *this))
8519 return V;
8520 break;
8521 }
8522 case ISD::CONCAT_VECTORS: {
8523 SDValue Ops[] = {N1, N2};
8524 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, DAG&: *this))
8525 return V;
8526 break;
8527 }
8528 case ISD::AND:
8529 assert(VT.isInteger() && "This operator does not apply to FP types!");
8530 assert(N1.getValueType() == N2.getValueType() &&
8531 N1.getValueType() == VT && "Binary operator types must match!");
8532 // (X & 0) -> 0. This commonly occurs when legalizing i64 values, so it's
8533 // worth handling here.
8534 if (N2CV && N2CV->isZero())
8535 return N2;
8536 if (N2CV && N2CV->isAllOnes()) // X & -1 -> X
8537 return N1;
8538 break;
8539 case ISD::OR:
8540 case ISD::XOR:
8541 case ISD::ADD:
8542 case ISD::PTRADD:
8543 case ISD::SUB:
8544 assert(VT.isInteger() && "This operator does not apply to FP types!");
8545 assert(N1.getValueType() == N2.getValueType() &&
8546 N1.getValueType() == VT && "Binary operator types must match!");
8547 // The equal operand types requirement is unnecessarily strong for PTRADD.
8548 // However, the SelectionDAGBuilder does not generate PTRADDs with different
8549 // operand types, and we'd need to re-implement GEP's non-standard wrapping
8550 // logic everywhere where PTRADDs may be folded or combined to properly
8551 // support them. If/when we introduce pointer types to the SDAG, we will
8552 // need to relax this constraint.
8553
8554 // (X ^|+- 0) -> X. This commonly occurs when legalizing i64 values, so
8555 // it's worth handling here.
8556 if (N2CV && N2CV->isZero())
8557 return N1;
8558 if ((Opcode == ISD::ADD || Opcode == ISD::SUB) &&
8559 VT.getScalarType() == MVT::i1)
8560 return getNode(Opcode: ISD::XOR, DL, VT, N1, N2);
8561 // Fold (add (vscale * C0), (vscale * C1)) to (vscale * (C0 + C1)).
8562 if (Opcode == ISD::ADD && N1.getOpcode() == ISD::VSCALE &&
8563 N2.getOpcode() == ISD::VSCALE) {
8564 const APInt &C1 = N1->getConstantOperandAPInt(Num: 0);
8565 const APInt &C2 = N2->getConstantOperandAPInt(Num: 0);
8566 return getVScale(DL, VT, MulImm: C1 + C2);
8567 }
8568 break;
8569 case ISD::MUL:
8570 assert(VT.isInteger() && "This operator does not apply to FP types!");
8571 assert(N1.getValueType() == N2.getValueType() &&
8572 N1.getValueType() == VT && "Binary operator types must match!");
8573 if (VT.getScalarType() == MVT::i1)
8574 return getNode(Opcode: ISD::AND, DL, VT, N1, N2);
8575 if (N2CV && N2CV->isZero())
8576 return N2;
8577 if (N2C && (N1.getOpcode() == ISD::VSCALE) && Flags.hasNoSignedWrap()) {
8578 const APInt &MulImm = N1->getConstantOperandAPInt(Num: 0);
8579 const APInt &N2CImm = N2C->getAPIntValue();
8580 return getVScale(DL, VT, MulImm: MulImm * N2CImm);
8581 }
8582 break;
8583 case ISD::UDIV:
8584 case ISD::UREM:
8585 case ISD::MULHU:
8586 case ISD::MULHS:
8587 case ISD::SDIV:
8588 case ISD::SREM:
8589 case ISD::SADDSAT:
8590 case ISD::SSUBSAT:
8591 case ISD::UADDSAT:
8592 case ISD::USUBSAT:
8593 assert(VT.isInteger() && "This operator does not apply to FP types!");
8594 assert(N1.getValueType() == N2.getValueType() &&
8595 N1.getValueType() == VT && "Binary operator types must match!");
8596 if (VT.getScalarType() == MVT::i1) {
8597 // fold (add_sat x, y) -> (or x, y) for bool types.
8598 if (Opcode == ISD::SADDSAT || Opcode == ISD::UADDSAT)
8599 return getNode(Opcode: ISD::OR, DL, VT, N1, N2);
8600 // fold (sub_sat x, y) -> (and x, ~y) for bool types.
8601 if (Opcode == ISD::SSUBSAT || Opcode == ISD::USUBSAT)
8602 return getNode(Opcode: ISD::AND, DL, VT, N1, N2: getNOT(DL, Val: N2, VT));
8603 }
8604 break;
8605 case ISD::SCMP:
8606 case ISD::UCMP:
8607 assert(N1.getValueType() == N2.getValueType() &&
8608 "Types of operands of UCMP/SCMP must match");
8609 assert(N1.getValueType().isVector() == VT.isVector() &&
8610 "Operands and return type of must both be scalars or vectors");
8611 if (VT.isVector())
8612 assert(VT.getVectorElementCount() ==
8613 N1.getValueType().getVectorElementCount() &&
8614 "Result and operands must have the same number of elements");
8615 break;
8616 case ISD::AVGFLOORS:
8617 case ISD::AVGFLOORU:
8618 case ISD::AVGCEILS:
8619 case ISD::AVGCEILU:
8620 assert(VT.isInteger() && "This operator does not apply to FP types!");
8621 assert(N1.getValueType() == N2.getValueType() &&
8622 N1.getValueType() == VT && "Binary operator types must match!");
8623 break;
8624 case ISD::ABDS:
8625 case ISD::ABDU:
8626 assert(VT.isInteger() && "This operator does not apply to FP types!");
8627 assert(N1.getValueType() == N2.getValueType() &&
8628 N1.getValueType() == VT && "Binary operator types must match!");
8629 if (VT.getScalarType() == MVT::i1)
8630 return getNode(Opcode: ISD::XOR, DL, VT, N1, N2);
8631 break;
8632 case ISD::SMIN:
8633 case ISD::UMAX:
8634 assert(VT.isInteger() && "This operator does not apply to FP types!");
8635 assert(N1.getValueType() == N2.getValueType() &&
8636 N1.getValueType() == VT && "Binary operator types must match!");
8637 if (VT.getScalarType() == MVT::i1)
8638 return getNode(Opcode: ISD::OR, DL, VT, N1, N2);
8639 break;
8640 case ISD::SMAX:
8641 case ISD::UMIN:
8642 assert(VT.isInteger() && "This operator does not apply to FP types!");
8643 assert(N1.getValueType() == N2.getValueType() &&
8644 N1.getValueType() == VT && "Binary operator types must match!");
8645 if (VT.getScalarType() == MVT::i1)
8646 return getNode(Opcode: ISD::AND, DL, VT, N1, N2);
8647 break;
8648 case ISD::FADD:
8649 case ISD::FSUB:
8650 case ISD::FMUL:
8651 case ISD::FDIV:
8652 case ISD::FREM:
8653 assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
8654 assert(N1.getValueType() == N2.getValueType() &&
8655 N1.getValueType() == VT && "Binary operator types must match!");
8656 if (SDValue V = simplifyFPBinop(Opcode, X: N1, Y: N2, Flags))
8657 return V;
8658 break;
8659 case ISD::FCOPYSIGN: // N1 and result must match. N1/N2 need not match.
8660 assert(N1.getValueType() == VT &&
8661 N1.getValueType().isFloatingPoint() &&
8662 N2.getValueType().isFloatingPoint() &&
8663 "Invalid FCOPYSIGN!");
8664 break;
8665 case ISD::SHL:
8666 if (N2C && (N1.getOpcode() == ISD::VSCALE) && Flags.hasNoSignedWrap()) {
8667 const APInt &MulImm = N1->getConstantOperandAPInt(Num: 0);
8668 const APInt &ShiftImm = N2C->getAPIntValue();
8669 return getVScale(DL, VT, MulImm: MulImm << ShiftImm);
8670 }
8671 [[fallthrough]];
8672 case ISD::SRA:
8673 case ISD::SRL:
8674 if (SDValue V = simplifyShift(X: N1, Y: N2))
8675 return V;
8676 [[fallthrough]];
8677 case ISD::ROTL:
8678 case ISD::ROTR:
8679 case ISD::SSHLSAT:
8680 case ISD::USHLSAT:
8681 assert(VT == N1.getValueType() &&
8682 "Shift operators return type must be the same as their first arg");
8683 assert(VT.isInteger() && N2.getValueType().isInteger() &&
8684 "Shifts only work on integers");
8685 assert((!VT.isVector() || VT == N2.getValueType()) &&
8686 "Vector shift amounts must be in the same as their first arg");
8687 // Verify that the shift amount VT is big enough to hold valid shift
8688 // amounts. This catches things like trying to shift an i1024 value by an
8689 // i8, which is easy to fall into in generic code that uses
8690 // TLI.getShiftAmount().
8691 assert(N2.getValueType().getScalarSizeInBits() >=
8692 Log2_32_Ceil(VT.getScalarSizeInBits()) &&
8693 "Invalid use of small shift amount with oversized value!");
8694
8695 // Always fold shifts of i1 values so the code generator doesn't need to
8696 // handle them. Since we know the size of the shift has to be less than the
8697 // size of the value, the shift/rotate count is guaranteed to be zero.
8698 if (VT == MVT::i1)
8699 return N1;
8700 if (N2CV && N2CV->isZero())
8701 return N1;
8702 break;
8703 case ISD::FP_ROUND:
8704 assert(VT.isFloatingPoint() && N1.getValueType().isFloatingPoint() &&
8705 VT.bitsLE(N1.getValueType()) && N2C &&
8706 (N2C->getZExtValue() == 0 || N2C->getZExtValue() == 1) &&
8707 N2.getOpcode() == ISD::TargetConstant && "Invalid FP_ROUND!");
8708 if (N1.getValueType() == VT) return N1; // noop conversion.
8709 break;
8710 case ISD::IS_FPCLASS: {
8711 assert(N1.getValueType().isFloatingPoint() &&
8712 "IS_FPCLASS is used for a non-floating type");
8713 assert(isa<ConstantSDNode>(N2) && "FPClassTest is not Constant");
8714 // is.fpclass(poison, mask) -> poison
8715 if (N1.getOpcode() == ISD::POISON)
8716 return getPOISON(VT);
8717 FPClassTest Mask = static_cast<FPClassTest>(N2->getAsZExtVal());
8718 // If all tests are made, it doesn't matter what the value is.
8719 if ((Mask & fcAllFlags) == fcAllFlags)
8720 return getBoolConstant(V: true, DL, VT, OpVT: N1.getValueType());
8721 if ((Mask & fcAllFlags) == 0)
8722 return getBoolConstant(V: false, DL, VT, OpVT: N1.getValueType());
8723 break;
8724 }
8725 case ISD::AssertNoFPClass: {
8726 assert(N1.getValueType().isFloatingPoint() &&
8727 "AssertNoFPClass is used for a non-floating type");
8728 assert(isa<ConstantSDNode>(N2) && "NoFPClass is not Constant");
8729 FPClassTest NoFPClass = static_cast<FPClassTest>(N2->getAsZExtVal());
8730 assert(llvm::to_underlying(NoFPClass) <=
8731 BitmaskEnumDetail::Mask<FPClassTest>() &&
8732 "FPClassTest value too large");
8733 (void)NoFPClass;
8734 break;
8735 }
8736 case ISD::AssertSext:
8737 case ISD::AssertZext: {
8738 EVT EVT = cast<VTSDNode>(Val&: N2)->getVT();
8739 assert(VT == N1.getValueType() && "Not an inreg extend!");
8740 assert(VT.isInteger() && EVT.isInteger() &&
8741 "Cannot *_EXTEND_INREG FP types");
8742 assert(!EVT.isVector() &&
8743 "AssertSExt/AssertZExt type should be the vector element type "
8744 "rather than the vector type!");
8745 assert(EVT.bitsLE(VT.getScalarType()) && "Not extending!");
8746 if (VT.getScalarType() == EVT) return N1; // noop assertion.
8747 break;
8748 }
8749 case ISD::SIGN_EXTEND_INREG: {
8750 EVT EVT = cast<VTSDNode>(Val&: N2)->getVT();
8751 assert(VT == N1.getValueType() && "Not an inreg extend!");
8752 assert(VT.isInteger() && EVT.isInteger() &&
8753 "Cannot *_EXTEND_INREG FP types");
8754 assert(EVT.isVector() == VT.isVector() &&
8755 "SIGN_EXTEND_INREG type should be vector iff the operand "
8756 "type is vector!");
8757 assert((!EVT.isVector() ||
8758 EVT.getVectorElementCount() == VT.getVectorElementCount()) &&
8759 "Vector element counts must match in SIGN_EXTEND_INREG");
8760 assert(EVT.getScalarType().bitsLE(VT.getScalarType()) && "Not extending!");
8761 if (EVT == VT) return N1; // Not actually extending
8762 break;
8763 }
8764 case ISD::FP_TO_SINT_SAT:
8765 case ISD::FP_TO_UINT_SAT: {
8766 assert(VT.isInteger() && cast<VTSDNode>(N2)->getVT().isInteger() &&
8767 N1.getValueType().isFloatingPoint() && "Invalid FP_TO_*INT_SAT");
8768 assert(N1.getValueType().isVector() == VT.isVector() &&
8769 "FP_TO_*INT_SAT type should be vector iff the operand type is "
8770 "vector!");
8771 assert((!VT.isVector() || VT.getVectorElementCount() ==
8772 N1.getValueType().getVectorElementCount()) &&
8773 "Vector element counts must match in FP_TO_*INT_SAT");
8774 assert(!cast<VTSDNode>(N2)->getVT().isVector() &&
8775 "Type to saturate to must be a scalar.");
8776 assert(cast<VTSDNode>(N2)->getVT().bitsLE(VT.getScalarType()) &&
8777 "Not extending!");
8778 break;
8779 }
8780 case ISD::EXTRACT_VECTOR_ELT:
8781 assert(VT.getSizeInBits() >= N1.getValueType().getScalarSizeInBits() &&
8782 "The result of EXTRACT_VECTOR_ELT must be at least as wide as the \
8783 element type of the vector.");
8784
8785 // Extract from an undefined value or using an undefined index is undefined.
8786 if (N1.isUndef() || N2.isUndef())
8787 return getUNDEF(VT);
8788
8789 // EXTRACT_VECTOR_ELT of out-of-bounds element is POISON for fixed length
8790 // vectors. For scalable vectors we will provide appropriate support for
8791 // dealing with arbitrary indices.
8792 if (N2C && N1.getValueType().isFixedLengthVector() &&
8793 N2C->getAPIntValue().uge(RHS: N1.getValueType().getVectorNumElements()))
8794 return getPOISON(VT);
8795
8796 // EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is
8797 // expanding copies of large vectors from registers. This only works for
8798 // fixed length vectors, since we need to know the exact number of
8799 // elements.
8800 if (N2C && N1.getOpcode() == ISD::CONCAT_VECTORS &&
8801 N1.getOperand(i: 0).getValueType().isFixedLengthVector()) {
8802 unsigned Factor = N1.getOperand(i: 0).getValueType().getVectorNumElements();
8803 return getExtractVectorElt(DL, VT,
8804 Vec: N1.getOperand(i: N2C->getZExtValue() / Factor),
8805 Idx: N2C->getZExtValue() % Factor);
8806 }
8807
8808 // EXTRACT_VECTOR_ELT of BUILD_VECTOR or SPLAT_VECTOR is often formed while
8809 // lowering is expanding large vector constants.
8810 if (N2C && (N1.getOpcode() == ISD::BUILD_VECTOR ||
8811 N1.getOpcode() == ISD::SPLAT_VECTOR)) {
8812 assert((N1.getOpcode() != ISD::BUILD_VECTOR ||
8813 N1.getValueType().isFixedLengthVector()) &&
8814 "BUILD_VECTOR used for scalable vectors");
8815 unsigned Index =
8816 N1.getOpcode() == ISD::BUILD_VECTOR ? N2C->getZExtValue() : 0;
8817 SDValue Elt = N1.getOperand(i: Index);
8818
8819 if (VT != Elt.getValueType())
8820 // If the vector element type is not legal, the BUILD_VECTOR operands
8821 // are promoted and implicitly truncated, and the result implicitly
8822 // extended. Make that explicit here.
8823 Elt = getAnyExtOrTrunc(Op: Elt, DL, VT);
8824
8825 return Elt;
8826 }
8827
8828 // EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector
8829 // operations are lowered to scalars.
8830 if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT) {
8831 // If the indices are the same, return the inserted element else
8832 // if the indices are known different, extract the element from
8833 // the original vector.
8834 SDValue N1Op2 = N1.getOperand(i: 2);
8835 ConstantSDNode *N1Op2C = dyn_cast<ConstantSDNode>(Val&: N1Op2);
8836
8837 if (N1Op2C && N2C) {
8838 if (N1Op2C->getZExtValue() == N2C->getZExtValue()) {
8839 if (VT == N1.getOperand(i: 1).getValueType())
8840 return N1.getOperand(i: 1);
8841 if (VT.isFloatingPoint()) {
8842 assert(VT.getSizeInBits() > N1.getOperand(1).getValueType().getSizeInBits());
8843 return getFPExtendOrRound(Op: N1.getOperand(i: 1), DL, VT);
8844 }
8845 return getSExtOrTrunc(Op: N1.getOperand(i: 1), DL, VT);
8846 }
8847 return getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT, N1: N1.getOperand(i: 0), N2);
8848 }
8849 }
8850
8851 // EXTRACT_VECTOR_ELT of v1iX EXTRACT_SUBVECTOR could be formed
8852 // when vector types are scalarized and v1iX is legal.
8853 // vextract (v1iX extract_subvector(vNiX, Idx)) -> vextract(vNiX,Idx).
8854 // Here we are completely ignoring the extract element index (N2),
8855 // which is fine for fixed width vectors, since any index other than 0
8856 // is undefined anyway. However, this cannot be ignored for scalable
8857 // vectors - in theory we could support this, but we don't want to do this
8858 // without a profitability check.
8859 if (N1.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
8860 N1.getValueType().isFixedLengthVector() &&
8861 N1.getValueType().getVectorNumElements() == 1) {
8862 return getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT, N1: N1.getOperand(i: 0),
8863 N2: N1.getOperand(i: 1));
8864 }
8865 break;
8866 case ISD::EXTRACT_ELEMENT:
8867 assert(N2C && (unsigned)N2C->getZExtValue() < 2 && "Bad EXTRACT_ELEMENT!");
8868 assert(!N1.getValueType().isVector() && !VT.isVector() &&
8869 (N1.getValueType().isInteger() == VT.isInteger()) &&
8870 N1.getValueType() != VT &&
8871 "Wrong types for EXTRACT_ELEMENT!");
8872
8873 // EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding
8874 // 64-bit integers into 32-bit parts. Instead of building the extract of
8875 // the BUILD_PAIR, only to have legalize rip it apart, just do it now.
8876 if (N1.getOpcode() == ISD::BUILD_PAIR)
8877 return N1.getOperand(i: N2C->getZExtValue());
8878
8879 // EXTRACT_ELEMENT of a constant int is also very common.
8880 if (N1C) {
8881 unsigned ElementSize = VT.getSizeInBits();
8882 unsigned Shift = ElementSize * N2C->getZExtValue();
8883 const APInt &Val = N1C->getAPIntValue();
8884 return getConstant(Val: Val.extractBits(numBits: ElementSize, bitPosition: Shift), DL, VT);
8885 }
8886 break;
8887 case ISD::EXTRACT_SUBVECTOR: {
8888 EVT N1VT = N1.getValueType();
8889 assert(VT.isVector() && N1VT.isVector() &&
8890 "Extract subvector VTs must be vectors!");
8891 assert(VT.getVectorElementType() == N1VT.getVectorElementType() &&
8892 "Extract subvector VTs must have the same element type!");
8893 assert((VT.isFixedLengthVector() || N1VT.isScalableVector()) &&
8894 "Cannot extract a scalable vector from a fixed length vector!");
8895 assert((VT.isScalableVector() != N1VT.isScalableVector() ||
8896 VT.getVectorMinNumElements() <= N1VT.getVectorMinNumElements()) &&
8897 "Extract subvector must be from larger vector to smaller vector!");
8898 assert(N2C && "Extract subvector index must be a constant");
8899 assert((VT.isScalableVector() != N1VT.isScalableVector() ||
8900 (VT.getVectorMinNumElements() + N2C->getZExtValue()) <=
8901 N1VT.getVectorMinNumElements()) &&
8902 "Extract subvector overflow!");
8903 assert(N2C->getAPIntValue().getBitWidth() ==
8904 TLI->getVectorIdxWidth(getDataLayout()) &&
8905 "Constant index for EXTRACT_SUBVECTOR has an invalid size");
8906 assert(N2C->getZExtValue() % VT.getVectorMinNumElements() == 0 &&
8907 "Extract index is not a multiple of the output vector length");
8908
8909 // Trivial extraction.
8910 if (VT == N1VT)
8911 return N1;
8912
8913 // EXTRACT_SUBVECTOR of an UNDEF is an UNDEF.
8914 if (N1.isUndef())
8915 return getUNDEF(VT);
8916
8917 // EXTRACT_SUBVECTOR of CONCAT_VECTOR can be simplified if the pieces of
8918 // the concat have the same type as the extract.
8919 if (N1.getOpcode() == ISD::CONCAT_VECTORS &&
8920 VT == N1.getOperand(i: 0).getValueType()) {
8921 unsigned Factor = VT.getVectorMinNumElements();
8922 return N1.getOperand(i: N2C->getZExtValue() / Factor);
8923 }
8924
8925 // EXTRACT_SUBVECTOR of INSERT_SUBVECTOR is often created
8926 // during shuffle legalization.
8927 if (N1.getOpcode() == ISD::INSERT_SUBVECTOR && N2 == N1.getOperand(i: 2) &&
8928 VT == N1.getOperand(i: 1).getValueType())
8929 return N1.getOperand(i: 1);
8930 break;
8931 }
8932 }
8933
8934 if (N1.getOpcode() == ISD::POISON || N2.getOpcode() == ISD::POISON) {
8935 switch (Opcode) {
8936 case ISD::XOR:
8937 case ISD::ADD:
8938 case ISD::PTRADD:
8939 case ISD::SUB:
8940 case ISD::SIGN_EXTEND_INREG:
8941 case ISD::UDIV:
8942 case ISD::SDIV:
8943 case ISD::UREM:
8944 case ISD::SREM:
8945 case ISD::MUL:
8946 case ISD::AND:
8947 case ISD::SSUBSAT:
8948 case ISD::USUBSAT:
8949 case ISD::UMIN:
8950 case ISD::OR:
8951 case ISD::SADDSAT:
8952 case ISD::UADDSAT:
8953 case ISD::UMAX:
8954 case ISD::SMAX:
8955 case ISD::SMIN:
8956 // fold op(arg1, poison) -> poison, fold op(poison, arg2) -> poison.
8957 return N2.getOpcode() == ISD::POISON ? N2 : N1;
8958 }
8959 }
8960
8961 // Canonicalize an UNDEF to the RHS, even over a constant.
8962 if (N1.getOpcode() == ISD::UNDEF && N2.getOpcode() != ISD::UNDEF) {
8963 if (TLI->isCommutativeBinOp(Opcode)) {
8964 std::swap(a&: N1, b&: N2);
8965 } else {
8966 switch (Opcode) {
8967 case ISD::PTRADD:
8968 case ISD::SUB:
8969 // fold op(undef, non_undef_arg2) -> undef.
8970 return N1;
8971 case ISD::SIGN_EXTEND_INREG:
8972 case ISD::UDIV:
8973 case ISD::SDIV:
8974 case ISD::UREM:
8975 case ISD::SREM:
8976 case ISD::SSUBSAT:
8977 case ISD::USUBSAT:
8978 // fold op(undef, non_undef_arg2) -> 0.
8979 return getConstant(Val: 0, DL, VT);
8980 }
8981 }
8982 }
8983
8984 // Fold a bunch of operators when the RHS is undef.
8985 if (N2.getOpcode() == ISD::UNDEF) {
8986 switch (Opcode) {
8987 case ISD::XOR:
8988 if (N1.getOpcode() == ISD::UNDEF)
8989 // Handle undef ^ undef -> 0 special case. This is a common
8990 // idiom (misuse).
8991 return getConstant(Val: 0, DL, VT);
8992 [[fallthrough]];
8993 case ISD::ADD:
8994 case ISD::PTRADD:
8995 case ISD::SUB:
8996 // fold op(arg1, undef) -> undef.
8997 return N2;
8998 case ISD::UDIV:
8999 case ISD::SDIV:
9000 case ISD::UREM:
9001 case ISD::SREM:
9002 // fold op(arg1, undef) -> poison.
9003 return getPOISON(VT);
9004 case ISD::MUL:
9005 case ISD::AND:
9006 case ISD::SSUBSAT:
9007 case ISD::USUBSAT:
9008 case ISD::UMIN:
9009 // fold op(undef, undef) -> undef, fold op(arg1, undef) -> 0.
9010 return N1.getOpcode() == ISD::UNDEF ? N2 : getConstant(Val: 0, DL, VT);
9011 case ISD::OR:
9012 case ISD::SADDSAT:
9013 case ISD::UADDSAT:
9014 case ISD::UMAX:
9015 // fold op(undef, undef) -> undef, fold op(arg1, undef) -> -1.
9016 return N1.getOpcode() == ISD::UNDEF ? N2 : getAllOnesConstant(DL, VT);
9017 case ISD::SMAX:
9018 // fold op(undef, undef) -> undef, fold op(arg1, undef) -> MAX_INT.
9019 return N1.getOpcode() == ISD::UNDEF
9020 ? N2
9021 : getConstant(
9022 Val: APInt::getSignedMaxValue(numBits: VT.getScalarSizeInBits()), DL,
9023 VT);
9024 case ISD::SMIN:
9025 // fold op(undef, undef) -> undef, fold op(arg1, undef) -> MIN_INT.
9026 return N1.getOpcode() == ISD::UNDEF
9027 ? N2
9028 : getConstant(
9029 Val: APInt::getSignedMinValue(numBits: VT.getScalarSizeInBits()), DL,
9030 VT);
9031 }
9032 }
9033
9034 // Perform trivial constant folding.
9035 if (SDValue SV = FoldConstantArithmetic(Opcode, DL, VT, Ops: {N1, N2}, Flags))
9036 return SV;
9037
9038 // Memoize this node if possible.
9039 SDNode *N;
9040 SDVTList VTs = getVTList(VT);
9041 SDValue Ops[] = {N1, N2};
9042 if (VT != MVT::Glue) {
9043 SDNodeKey ID(Opcode, VTs, Ops);
9044 FoldingSetInsertToken InsertToken;
9045 if (SDNode *E = lookupNode(Key: ID, DL, InsertToken)) {
9046 E->intersectFlagsWith(Flags);
9047 return SDValue(E, 0);
9048 }
9049
9050 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
9051 N->setFlags(Flags);
9052 createOperands(Node: N, Vals: Ops);
9053 CSEMap.insert(N, Token: InsertToken);
9054 } else {
9055 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
9056 createOperands(Node: N, Vals: Ops);
9057 }
9058
9059 InsertNode(N);
9060 SDValue V = SDValue(N, 0);
9061 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
9062 return V;
9063}
9064
9065SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
9066 SDValue N1, SDValue N2, SDValue N3) {
9067 SDNodeFlags Flags;
9068 if (Inserter)
9069 Flags = Inserter->getFlags();
9070 return getNode(Opcode, DL, VT, N1, N2, N3, Flags);
9071}
9072
9073SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
9074 SDValue N1, SDValue N2, SDValue N3,
9075 const SDNodeFlags Flags) {
9076 assert(N1.getOpcode() != ISD::DELETED_NODE &&
9077 N2.getOpcode() != ISD::DELETED_NODE &&
9078 N3.getOpcode() != ISD::DELETED_NODE &&
9079 "Operand is DELETED_NODE!");
9080 // Perform various simplifications.
9081 switch (Opcode) {
9082 case ISD::BUILD_VECTOR: {
9083 // Attempt to simplify BUILD_VECTOR.
9084 SDValue Ops[] = {N1, N2, N3};
9085 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, DAG&: *this))
9086 return V;
9087 break;
9088 }
9089 case ISD::CONCAT_VECTORS: {
9090 SDValue Ops[] = {N1, N2, N3};
9091 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, DAG&: *this))
9092 return V;
9093 break;
9094 }
9095 case ISD::SETCC: {
9096 assert(VT.isInteger() && "SETCC result type must be an integer!");
9097 assert(N1.getValueType() == N2.getValueType() &&
9098 "SETCC operands must have the same type!");
9099 assert(VT.isVector() == N1.getValueType().isVector() &&
9100 "SETCC type should be vector iff the operand type is vector!");
9101 assert((!VT.isVector() || VT.getVectorElementCount() ==
9102 N1.getValueType().getVectorElementCount()) &&
9103 "SETCC vector element counts must match!");
9104 // Use FoldSetCC to simplify SETCC's.
9105 if (SDValue V =
9106 FoldSetCC(VT, N1, N2, Cond: cast<CondCodeSDNode>(Val&: N3)->get(), dl: DL, Flags))
9107 return V;
9108 break;
9109 }
9110 case ISD::SELECT:
9111 case ISD::VSELECT:
9112 if (SDValue V = simplifySelect(Cond: N1, TVal: N2, FVal: N3))
9113 return V;
9114 break;
9115 case ISD::VECTOR_SHUFFLE:
9116 llvm_unreachable("should use getVectorShuffle constructor!");
9117 case ISD::VECTOR_SPLICE_LEFT:
9118 if (isNullConstant(V: N3))
9119 return N1;
9120 break;
9121 case ISD::VECTOR_SPLICE_RIGHT:
9122 if (isNullConstant(V: N3))
9123 return N2;
9124 break;
9125 case ISD::INSERT_VECTOR_ELT: {
9126 assert(VT.isVector() && VT == N1.getValueType() &&
9127 "INSERT_VECTOR_ELT vector type mismatch");
9128 assert(VT.isFloatingPoint() == N2.getValueType().isFloatingPoint() &&
9129 "INSERT_VECTOR_ELT scalar fp/int mismatch");
9130 assert((!VT.isFloatingPoint() ||
9131 VT.getVectorElementType() == N2.getValueType()) &&
9132 "INSERT_VECTOR_ELT fp scalar type mismatch");
9133 assert((!VT.isInteger() ||
9134 VT.getScalarSizeInBits() <= N2.getScalarValueSizeInBits()) &&
9135 "INSERT_VECTOR_ELT int scalar size mismatch");
9136
9137 auto *N3C = dyn_cast<ConstantSDNode>(Val&: N3);
9138 // INSERT_VECTOR_ELT into out-of-bounds element is an UNDEF, except
9139 // for scalable vectors where we will generate appropriate code to
9140 // deal with out-of-bounds cases correctly.
9141 if (N3C && VT.isFixedLengthVector() &&
9142 N3C->getZExtValue() >= VT.getVectorNumElements())
9143 return getUNDEF(VT);
9144
9145 // Undefined index can be assumed out-of-bounds, so that's UNDEF too.
9146 if (N3.isUndef())
9147 return getUNDEF(VT);
9148
9149 // If inserting poison, just use the input vector.
9150 if (N2.getOpcode() == ISD::POISON)
9151 return N1;
9152
9153 // Inserting undef into undef/poison is still undef.
9154 if (N2.getOpcode() == ISD::UNDEF && N1.isUndef())
9155 return getUNDEF(VT);
9156
9157 // If the inserted element is an UNDEF, just use the input vector.
9158 // But not if skipping the insert could make the result more poisonous.
9159 if (N2.isUndef()) {
9160 if (N3C && VT.isFixedLengthVector()) {
9161 APInt EltMask =
9162 APInt::getOneBitSet(numBits: VT.getVectorNumElements(), BitNo: N3C->getZExtValue());
9163 if (isGuaranteedNotToBePoison(Op: N1, DemandedElts: EltMask))
9164 return N1;
9165 } else if (isGuaranteedNotToBePoison(Op: N1))
9166 return N1;
9167 }
9168 break;
9169 }
9170 case ISD::INSERT_SUBVECTOR: {
9171 // If inserting poison, just use the input vector,
9172 if (N2.getOpcode() == ISD::POISON)
9173 return N1;
9174
9175 // Inserting undef into undef/poison is still undef.
9176 if (N2.getOpcode() == ISD::UNDEF && N1.isUndef())
9177 return getUNDEF(VT);
9178
9179 EVT N2VT = N2.getValueType();
9180 assert(VT == N1.getValueType() &&
9181 "Dest and insert subvector source types must match!");
9182 assert(VT.isVector() && N2VT.isVector() &&
9183 "Insert subvector VTs must be vectors!");
9184 assert(VT.getVectorElementType() == N2VT.getVectorElementType() &&
9185 "Insert subvector VTs must have the same element type!");
9186 assert((VT.isScalableVector() || N2VT.isFixedLengthVector()) &&
9187 "Cannot insert a scalable vector into a fixed length vector!");
9188 assert((VT.isScalableVector() != N2VT.isScalableVector() ||
9189 VT.getVectorMinNumElements() >= N2VT.getVectorMinNumElements()) &&
9190 "Insert subvector must be from smaller vector to larger vector!");
9191 assert(isa<ConstantSDNode>(N3) &&
9192 "Insert subvector index must be constant");
9193 assert((VT.isScalableVector() != N2VT.isScalableVector() ||
9194 (N2VT.getVectorMinNumElements() + N3->getAsZExtVal()) <=
9195 VT.getVectorMinNumElements()) &&
9196 "Insert subvector overflow!");
9197 assert(N3->getAsAPIntVal().getBitWidth() ==
9198 TLI->getVectorIdxWidth(getDataLayout()) &&
9199 "Constant index for INSERT_SUBVECTOR has an invalid size");
9200
9201 // Trivial insertion.
9202 if (VT == N2VT)
9203 return N2;
9204
9205 // If this is an insert of an extracted vector into an undef/poison vector,
9206 // we can just use the input to the extract. But not if skipping the
9207 // extract+insert could make the result more poisonous.
9208 if (N1.isUndef() && N2.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
9209 N2.getOperand(i: 1) == N3 && N2.getOperand(i: 0).getValueType() == VT) {
9210 if (N1.getOpcode() == ISD::POISON)
9211 return N2.getOperand(i: 0);
9212 if (VT.isFixedLengthVector() && N2VT.isFixedLengthVector()) {
9213 unsigned LoBit = N3->getAsZExtVal();
9214 unsigned HiBit = LoBit + N2VT.getVectorNumElements();
9215 APInt EltMask =
9216 APInt::getBitsSet(numBits: VT.getVectorNumElements(), loBit: LoBit, hiBit: HiBit);
9217 if (isGuaranteedNotToBePoison(Op: N2.getOperand(i: 0), DemandedElts: ~EltMask))
9218 return N2.getOperand(i: 0);
9219 } else if (isGuaranteedNotToBePoison(Op: N2.getOperand(i: 0)))
9220 return N2.getOperand(i: 0);
9221 }
9222
9223 // If the inserted subvector is UNDEF, just use the input vector.
9224 // But not if skipping the insert could make the result more poisonous.
9225 if (N2.isUndef()) {
9226 if (VT.isFixedLengthVector()) {
9227 unsigned LoBit = N3->getAsZExtVal();
9228 unsigned HiBit = LoBit + N2VT.getVectorNumElements();
9229 APInt EltMask =
9230 APInt::getBitsSet(numBits: VT.getVectorNumElements(), loBit: LoBit, hiBit: HiBit);
9231 if (isGuaranteedNotToBePoison(Op: N1, DemandedElts: EltMask))
9232 return N1;
9233 } else if (isGuaranteedNotToBePoison(Op: N1))
9234 return N1;
9235 }
9236 break;
9237 }
9238 case ISD::BITCAST:
9239 // Fold bit_convert nodes from a type to themselves.
9240 if (N1.getValueType() == VT)
9241 return N1;
9242 break;
9243 case ISD::VECTOR_COMPRESS: {
9244 [[maybe_unused]] EVT VecVT = N1.getValueType();
9245 [[maybe_unused]] EVT MaskVT = N2.getValueType();
9246 [[maybe_unused]] EVT PassthruVT = N3.getValueType();
9247 assert(VT == VecVT && "Vector and result type don't match.");
9248 assert(VecVT.isVector() && MaskVT.isVector() && PassthruVT.isVector() &&
9249 "All inputs must be vectors.");
9250 assert(VecVT == PassthruVT && "Vector and passthru types don't match.");
9251 assert(VecVT.getVectorElementCount() == MaskVT.getVectorElementCount() &&
9252 "Vector and mask must have same number of elements.");
9253
9254 if (N1.isUndef() || N2.isUndef())
9255 return N3;
9256
9257 break;
9258 }
9259 case ISD::PARTIAL_REDUCE_UMLA:
9260 case ISD::PARTIAL_REDUCE_SMLA:
9261 case ISD::PARTIAL_REDUCE_SUMLA:
9262 case ISD::PARTIAL_REDUCE_FMLA: {
9263 [[maybe_unused]] EVT AccVT = N1.getValueType();
9264 [[maybe_unused]] EVT Input1VT = N2.getValueType();
9265 [[maybe_unused]] EVT Input2VT = N3.getValueType();
9266 assert(Input1VT.isVector() && Input1VT == Input2VT &&
9267 "Expected the second and third operands of the PARTIAL_REDUCE_MLA "
9268 "node to have the same type!");
9269 assert(VT.isVector() && VT == AccVT &&
9270 "Expected the first operand of the PARTIAL_REDUCE_MLA node to have "
9271 "the same type as its result!");
9272 assert(Input1VT.getVectorElementCount().hasKnownScalarFactor(
9273 AccVT.getVectorElementCount()) &&
9274 "Expected the element count of the second and third operands of the "
9275 "PARTIAL_REDUCE_MLA node to be a positive integer multiple of the "
9276 "element count of the first operand and the result!");
9277 assert(N2.getScalarValueSizeInBits() <= N1.getScalarValueSizeInBits() &&
9278 "Expected the second and third operands of the PARTIAL_REDUCE_MLA "
9279 "node to have an element type which is the same as or smaller than "
9280 "the element type of the first operand and result!");
9281 break;
9282 }
9283 }
9284
9285 // Perform trivial constant folding for arithmetic operators.
9286 switch (Opcode) {
9287 case ISD::PARTIAL_REDUCE_SMLA:
9288 case ISD::PARTIAL_REDUCE_UMLA:
9289 case ISD::PARTIAL_REDUCE_SUMLA:
9290 case ISD::FMA:
9291 case ISD::FMAD:
9292 case ISD::SETCC:
9293 case ISD::FSHL:
9294 case ISD::FSHR:
9295 if (SDValue SV =
9296 FoldConstantArithmetic(Opcode, DL, VT, Ops: {N1, N2, N3}, Flags))
9297 return SV;
9298 break;
9299 }
9300
9301 // Memoize node if it doesn't produce a glue result.
9302 SDNode *N;
9303 SDVTList VTs = getVTList(VT);
9304 SDValue Ops[] = {N1, N2, N3};
9305 if (VT != MVT::Glue) {
9306 SDNodeKey ID(Opcode, VTs, Ops);
9307 FoldingSetInsertToken InsertToken;
9308 if (SDNode *E = lookupNode(Key: ID, DL, InsertToken)) {
9309 E->intersectFlagsWith(Flags);
9310 return SDValue(E, 0);
9311 }
9312
9313 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
9314 N->setFlags(Flags);
9315 createOperands(Node: N, Vals: Ops);
9316 CSEMap.insert(N, Token: InsertToken);
9317 } else {
9318 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
9319 createOperands(Node: N, Vals: Ops);
9320 }
9321
9322 InsertNode(N);
9323 SDValue V = SDValue(N, 0);
9324 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
9325 return V;
9326}
9327
9328SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
9329 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
9330 const SDNodeFlags Flags) {
9331 SDValue Ops[] = { N1, N2, N3, N4 };
9332 return getNode(Opcode, DL, VT, Ops, Flags);
9333}
9334
9335SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
9336 SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
9337 SDNodeFlags Flags;
9338 if (Inserter)
9339 Flags = Inserter->getFlags();
9340 return getNode(Opcode, DL, VT, N1, N2, N3, N4, Flags);
9341}
9342
9343SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
9344 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
9345 SDValue N5, const SDNodeFlags Flags) {
9346 SDValue Ops[] = { N1, N2, N3, N4, N5 };
9347 return getNode(Opcode, DL, VT, Ops, Flags);
9348}
9349
9350SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
9351 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
9352 SDValue N5) {
9353 SDNodeFlags Flags;
9354 if (Inserter)
9355 Flags = Inserter->getFlags();
9356 return getNode(Opcode, DL, VT, N1, N2, N3, N4, N5, Flags);
9357}
9358
9359/// getStackArgumentTokenFactor - Compute a TokenFactor to force all
9360/// the incoming stack arguments to be loaded from the stack.
9361SDValue SelectionDAG::getStackArgumentTokenFactor(SDValue Chain) {
9362 SmallVector<SDValue, 8> ArgChains;
9363
9364 // Include the original chain at the beginning of the list. When this is
9365 // used by target LowerCall hooks, this helps legalize find the
9366 // CALLSEQ_BEGIN node.
9367 ArgChains.push_back(Elt: Chain);
9368
9369 // Add a chain value for each stack argument.
9370 for (SDNode *U : getEntryNode().getNode()->users())
9371 if (LoadSDNode *L = dyn_cast<LoadSDNode>(Val: U))
9372 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val: L->getBasePtr()))
9373 if (FI->getIndex() < 0)
9374 ArgChains.push_back(Elt: SDValue(L, 1));
9375
9376 // Build a tokenfactor for all the chains.
9377 return getNode(Opcode: ISD::TokenFactor, DL: SDLoc(Chain), VT: MVT::Other, Ops: ArgChains);
9378}
9379
9380/// getMemsetValue - Vectorized representation of the memset value
9381/// operand.
9382static SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG,
9383 const SDLoc &dl) {
9384 assert(!Value.isUndef());
9385
9386 unsigned NumBits = VT.getScalarSizeInBits();
9387 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: Value)) {
9388 assert(C->getAPIntValue().getBitWidth() == 8);
9389 APInt Val = APInt::getSplat(NewLen: NumBits, V: C->getAPIntValue());
9390 if (VT.isInteger()) {
9391 bool IsOpaque = VT.getSizeInBits() > 64 ||
9392 !DAG.getTargetLoweringInfo().isLegalStoreImmediate(Value: C->getSExtValue());
9393 return DAG.getConstant(Val, DL: dl, VT, isT: false, isO: IsOpaque);
9394 }
9395 return DAG.getConstantFP(V: APFloat(VT.getFltSemantics(), Val), DL: dl, VT);
9396 }
9397
9398 assert(Value.getValueType() == MVT::i8 && "memset with non-byte fill value?");
9399 EVT IntVT = VT.getScalarType();
9400 if (!IntVT.isInteger())
9401 IntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: IntVT.getSizeInBits());
9402
9403 Value = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: IntVT, N1: Value);
9404 if (NumBits > 8) {
9405 // Use a multiplication with 0x010101... to extend the input to the
9406 // required length.
9407 APInt Magic = APInt::getSplat(NewLen: NumBits, V: APInt(8, 0x01));
9408 Value = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: IntVT, N1: Value,
9409 N2: DAG.getConstant(Val: Magic, DL: dl, VT: IntVT));
9410 }
9411
9412 if (VT != Value.getValueType() && !VT.isInteger())
9413 Value = DAG.getBitcast(VT: VT.getScalarType(), V: Value);
9414 if (VT != Value.getValueType())
9415 Value = DAG.getSplatBuildVector(VT, DL: dl, Op: Value);
9416
9417 return Value;
9418}
9419
9420/// getMemsetStringVal - Similar to getMemsetValue. Except this is only
9421/// used when a memcpy is turned into a memset when the source is a constant
9422/// string ptr.
9423static SDValue getMemsetStringVal(EVT VT, const SDLoc &dl, SelectionDAG &DAG,
9424 const TargetLowering &TLI,
9425 const ConstantDataArraySlice &Slice) {
9426 // Handle vector with all elements zero.
9427 if (Slice.Array == nullptr) {
9428 if (VT.isInteger())
9429 return DAG.getConstant(Val: 0, DL: dl, VT);
9430 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT,
9431 N1: DAG.getConstant(Val: 0, DL: dl, VT: VT.changeTypeToInteger()));
9432 }
9433
9434 assert(!VT.isVector() && "Can't handle vector type here!");
9435 unsigned NumVTBits = VT.getSizeInBits();
9436 unsigned NumVTBytes = NumVTBits / 8;
9437 unsigned NumBytes = std::min(a: NumVTBytes, b: unsigned(Slice.Length));
9438
9439 APInt Val(NumVTBits, 0);
9440 if (DAG.getDataLayout().isLittleEndian()) {
9441 for (unsigned i = 0; i != NumBytes; ++i)
9442 Val |= (uint64_t)(unsigned char)Slice[i] << i*8;
9443 } else {
9444 for (unsigned i = 0; i != NumBytes; ++i)
9445 Val |= (uint64_t)(unsigned char)Slice[i] << (NumVTBytes-i-1)*8;
9446 }
9447
9448 // If the "cost" of materializing the integer immediate is less than the cost
9449 // of a load, then it is cost effective to turn the load into the immediate.
9450 Type *Ty = VT.getTypeForEVT(Context&: *DAG.getContext());
9451 if (TLI.shouldConvertConstantLoadToIntImm(Imm: Val, Ty))
9452 return DAG.getConstant(Val, DL: dl, VT);
9453 return SDValue();
9454}
9455
9456SDValue SelectionDAG::getMemBasePlusOffset(SDValue Base, TypeSize Offset,
9457 const SDLoc &DL,
9458 const SDNodeFlags Flags) {
9459 SDValue Index = getTypeSize(DL, VT: Base.getValueType(), TS: Offset);
9460 return getMemBasePlusOffset(Base, Offset: Index, DL, Flags);
9461}
9462
9463SDValue SelectionDAG::getMemBasePlusOffset(SDValue Ptr, SDValue Offset,
9464 const SDLoc &DL,
9465 const SDNodeFlags Flags) {
9466 assert(Offset.getValueType().isInteger());
9467 EVT BasePtrVT = Ptr.getValueType();
9468 if (TLI->shouldPreservePtrArith(F: this->getMachineFunction().getFunction(),
9469 PtrVT: BasePtrVT))
9470 return getNode(Opcode: ISD::PTRADD, DL, VT: BasePtrVT, N1: Ptr, N2: Offset, Flags);
9471 // InBounds only applies to PTRADD, don't set it if we generate ADD.
9472 SDNodeFlags AddFlags = Flags;
9473 AddFlags.setInBounds(false);
9474 return getNode(Opcode: ISD::ADD, DL, VT: BasePtrVT, N1: Ptr, N2: Offset, Flags: AddFlags);
9475}
9476
9477/// Returns true if memcpy source is constant data.
9478static bool isMemSrcFromConstant(SDValue Src, ConstantDataArraySlice &Slice) {
9479 uint64_t SrcDelta = 0;
9480 GlobalAddressSDNode *G = nullptr;
9481 if (Src.getOpcode() == ISD::GlobalAddress)
9482 G = cast<GlobalAddressSDNode>(Val&: Src);
9483 else if (Src->isAnyAdd() &&
9484 Src.getOperand(i: 0).getOpcode() == ISD::GlobalAddress &&
9485 Src.getOperand(i: 1).getOpcode() == ISD::Constant) {
9486 G = cast<GlobalAddressSDNode>(Val: Src.getOperand(i: 0));
9487 SrcDelta = Src.getConstantOperandVal(i: 1);
9488 }
9489 if (!G)
9490 return false;
9491
9492 return getConstantDataArrayInfo(V: G->getGlobal(), Slice, ElementSize: 8,
9493 Offset: SrcDelta + G->getOffset());
9494}
9495
9496static bool shouldLowerMemFuncForSize(const MachineFunction &MF,
9497 SelectionDAG &DAG) {
9498 // On Darwin, -Os means optimize for size without hurting performance, so
9499 // only really optimize for size when -Oz (MinSize) is used.
9500 if (MF.getTarget().getTargetTriple().isOSDarwin())
9501 return MF.getFunction().hasMinSize();
9502 return DAG.shouldOptForSize();
9503}
9504
9505static void chainLoadsAndStoresForMemcpy(SelectionDAG &DAG, const SDLoc &dl,
9506 SmallVector<SDValue, 32> &OutChains, unsigned From,
9507 unsigned To, SmallVector<SDValue, 16> &OutLoadChains,
9508 SmallVector<SDValue, 16> &OutStoreChains) {
9509 assert(OutLoadChains.size() && "Missing loads in memcpy inlining");
9510 assert(OutStoreChains.size() && "Missing stores in memcpy inlining");
9511 SmallVector<SDValue, 16> GluedLoadChains;
9512 for (unsigned i = From; i < To; ++i) {
9513 OutChains.push_back(Elt: OutLoadChains[i]);
9514 GluedLoadChains.push_back(Elt: OutLoadChains[i]);
9515 }
9516
9517 // Chain for all loads.
9518 SDValue LoadToken = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other,
9519 Ops: GluedLoadChains);
9520
9521 for (unsigned i = From; i < To; ++i) {
9522 StoreSDNode *ST = dyn_cast<StoreSDNode>(Val&: OutStoreChains[i]);
9523 SDValue NewStore = DAG.getTruncStore(Chain: LoadToken, dl, Val: ST->getValue(),
9524 Ptr: ST->getBasePtr(), SVT: ST->getMemoryVT(),
9525 MMO: ST->getMemOperand());
9526 OutChains.push_back(Elt: NewStore);
9527 }
9528}
9529
9530static SDValue
9531getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain,
9532 SDValue Dst, SDValue Src, uint64_t Size, Align DstAlign,
9533 Align SrcAlign, bool isVol, bool AlwaysInline,
9534 MachinePointerInfo DstPtrInfo,
9535 MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo,
9536 BatchAAResults *BatchAA, const MDNode *DstMemCacheHint,
9537 const MDNode *SrcMemCacheHint) {
9538 // Turn a memcpy of undef to nop.
9539 // FIXME: We need to honor volatile even is Src is undef.
9540 if (Src.isUndef())
9541 return Chain;
9542
9543 // Expand memcpy to a series of load and store ops if the size operand falls
9544 // below a certain threshold.
9545 // TODO: In the AlwaysInline case, if the size is big then generate a loop
9546 // rather than maybe a humongous number of loads and stores.
9547 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9548 const DataLayout &DL = DAG.getDataLayout();
9549 LLVMContext &C = *DAG.getContext();
9550 std::vector<EVT> MemOps;
9551 bool DstAlignCanChange = false;
9552 MachineFunction &MF = DAG.getMachineFunction();
9553 MachineFrameInfo &MFI = MF.getFrameInfo();
9554 bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
9555 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Dst);
9556 if (FI && !MFI.isFixedObjectIndex(ObjectIdx: FI->getIndex()))
9557 DstAlignCanChange = true;
9558 SrcAlign = std::max(a: SrcAlign, b: DAG.InferPtrAlign(Ptr: Src).valueOrOne());
9559 ConstantDataArraySlice Slice;
9560 // If marked as volatile, perform a copy even when marked as constant.
9561 bool CopyFromConstant = !isVol && isMemSrcFromConstant(Src, Slice);
9562 bool isZeroConstant = CopyFromConstant && Slice.Array == nullptr;
9563 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(OptSize);
9564 const MemOp Op = isZeroConstant
9565 ? MemOp::Set(Size, DstAlignCanChange, DstAlign,
9566 /*IsZeroMemset*/ true, IsVolatile: isVol)
9567 : MemOp::Copy(Size, DstAlignCanChange, DstAlign,
9568 SrcAlign, IsVolatile: isVol, MemcpyStrSrc: CopyFromConstant);
9569 if (!TLI.findOptimalMemOpLowering(
9570 Context&: C, MemOps, Limit, Op, DstAS: DstPtrInfo.getAddrSpace(),
9571 SrcAS: SrcPtrInfo.getAddrSpace(), FuncAttributes: MF.getFunction().getAttributes(), LargestVT: nullptr))
9572 return SDValue();
9573
9574 if (DstAlignCanChange) {
9575 Type *Ty = MemOps[0].getTypeForEVT(Context&: C);
9576 Align NewDstAlign = DL.getABITypeAlign(Ty);
9577
9578 // Don't promote to an alignment that would require dynamic stack
9579 // realignment which may conflict with optimizations such as tail call
9580 // optimization.
9581 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
9582 if (!TRI->hasStackRealignment(MF))
9583 if (MaybeAlign StackAlign = DL.getStackAlignment())
9584 NewDstAlign = std::min(a: NewDstAlign, b: *StackAlign);
9585
9586 if (NewDstAlign > DstAlign) {
9587 // Give the stack frame object a larger alignment if needed.
9588 if (MFI.getObjectAlign(ObjectIdx: FI->getIndex()) < NewDstAlign)
9589 MFI.setObjectAlignment(ObjectIdx: FI->getIndex(), Alignment: NewDstAlign);
9590 DstAlign = NewDstAlign;
9591 }
9592 }
9593
9594 // Prepare AAInfo for loads/stores after lowering this memcpy.
9595 AAMDNodes NewAAInfo = AAInfo;
9596 NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
9597
9598 const Value *SrcVal = dyn_cast_if_present<const Value *>(Val&: SrcPtrInfo.V);
9599 bool isConstant =
9600 BatchAA && SrcVal &&
9601 BatchAA->pointsToConstantMemory(Loc: MemoryLocation(SrcVal, Size, AAInfo));
9602
9603 MachineMemOperand::Flags MMOFlags =
9604 isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone;
9605 SmallVector<SDValue, 16> OutLoadChains;
9606 SmallVector<SDValue, 16> OutStoreChains;
9607 SmallVector<SDValue, 32> OutChains;
9608 unsigned NumMemOps = MemOps.size();
9609 uint64_t SrcOff = 0, DstOff = 0;
9610 for (unsigned i = 0; i != NumMemOps; ++i) {
9611 EVT VT = MemOps[i];
9612 unsigned VTSize = VT.getSizeInBits() / 8;
9613 SDValue Value, Store;
9614
9615 if (VTSize > Size) {
9616 // Issuing an unaligned load / store pair that overlaps with the previous
9617 // pair. Adjust the offset accordingly.
9618 assert(i == NumMemOps-1 && i != 0);
9619 SrcOff -= VTSize - Size;
9620 DstOff -= VTSize - Size;
9621 }
9622
9623 if (CopyFromConstant &&
9624 (isZeroConstant || (VT.isInteger() && !VT.isVector()))) {
9625 // It's unlikely a store of a vector immediate can be done in a single
9626 // instruction. It would require a load from a constantpool first.
9627 // We only handle zero vectors here.
9628 // FIXME: Handle other cases where store of vector immediate is done in
9629 // a single instruction.
9630 ConstantDataArraySlice SubSlice;
9631 if (SrcOff < Slice.Length) {
9632 SubSlice = Slice;
9633 SubSlice.move(Delta: SrcOff);
9634 } else {
9635 // This is an out-of-bounds access and hence UB. Pretend we read zero.
9636 SubSlice.Array = nullptr;
9637 SubSlice.Offset = 0;
9638 SubSlice.Length = VTSize;
9639 }
9640 Value = getMemsetStringVal(VT, dl, DAG, TLI, Slice: SubSlice);
9641 if (Value.getNode()) {
9642 Store = DAG.getStore(
9643 Chain, dl, Val: Value,
9644 Ptr: DAG.getObjectPtrOffset(SL: dl, Ptr: Dst, Offset: TypeSize::getFixed(ExactSize: DstOff)),
9645 PtrInfo: DstPtrInfo.getWithOffset(O: DstOff), Alignment: DstAlign, MMOFlags,
9646 Metadata: MMOMetadata(NewAAInfo, /*Ranges=*/nullptr, DstMemCacheHint));
9647 OutChains.push_back(Elt: Store);
9648 }
9649 }
9650
9651 if (!Store.getNode()) {
9652 // The type might not be legal for the target. This should only happen
9653 // if the type is smaller than a legal type, as on PPC, so the right
9654 // thing to do is generate a LoadExt/StoreTrunc pair. These simplify
9655 // to Load/Store if NVT==VT.
9656 // FIXME does the case above also need this?
9657 EVT NVT = TLI.getTypeToTransformTo(Context&: C, VT);
9658 assert(NVT.bitsGE(VT));
9659
9660 bool isDereferenceable =
9661 SrcPtrInfo.getWithOffset(O: SrcOff).isDereferenceable(Size: VTSize, C, DL);
9662 MachineMemOperand::Flags SrcMMOFlags = MMOFlags;
9663 if (isDereferenceable)
9664 SrcMMOFlags |= MachineMemOperand::MODereferenceable;
9665 if (isConstant)
9666 SrcMMOFlags |= MachineMemOperand::MOInvariant;
9667
9668 Value = DAG.getExtLoad(
9669 ExtType: ISD::EXTLOAD, dl, VT: NVT, Chain,
9670 Ptr: DAG.getObjectPtrOffset(SL: dl, Ptr: Src, Offset: TypeSize::getFixed(ExactSize: SrcOff)),
9671 PtrInfo: SrcPtrInfo.getWithOffset(O: SrcOff), MemVT: VT,
9672 Alignment: commonAlignment(A: SrcAlign, Offset: SrcOff), MMOFlags: SrcMMOFlags,
9673 Metadata: MMOMetadata(NewAAInfo, /*Ranges=*/nullptr, SrcMemCacheHint));
9674 OutLoadChains.push_back(Elt: Value.getValue(R: 1));
9675
9676 Store = DAG.getTruncStore(
9677 Chain, dl, Val: Value,
9678 Ptr: DAG.getObjectPtrOffset(SL: dl, Ptr: Dst, Offset: TypeSize::getFixed(ExactSize: DstOff)),
9679 PtrInfo: DstPtrInfo.getWithOffset(O: DstOff), SVT: VT, Alignment: DstAlign, MMOFlags,
9680 Metadata: MMOMetadata(NewAAInfo, /*Ranges=*/nullptr, DstMemCacheHint));
9681 OutStoreChains.push_back(Elt: Store);
9682 }
9683 SrcOff += VTSize;
9684 DstOff += VTSize;
9685 Size -= VTSize;
9686 }
9687
9688 unsigned GluedLdStLimit = MaxLdStGlue == 0 ?
9689 TLI.getMaxGluedStoresPerMemcpy() : MaxLdStGlue;
9690 unsigned NumLdStInMemcpy = OutStoreChains.size();
9691
9692 if (NumLdStInMemcpy) {
9693 // It may be that memcpy might be converted to memset if it's memcpy
9694 // of constants. In such a case, we won't have loads and stores, but
9695 // just stores. In the absence of loads, there is nothing to gang up.
9696 if ((GluedLdStLimit <= 1) || !EnableMemCpyDAGOpt) {
9697 // If target does not care, just leave as it.
9698 for (unsigned i = 0; i < NumLdStInMemcpy; ++i) {
9699 OutChains.push_back(Elt: OutLoadChains[i]);
9700 OutChains.push_back(Elt: OutStoreChains[i]);
9701 }
9702 } else {
9703 // Ld/St less than/equal limit set by target.
9704 if (NumLdStInMemcpy <= GluedLdStLimit) {
9705 chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, From: 0,
9706 To: NumLdStInMemcpy, OutLoadChains,
9707 OutStoreChains);
9708 } else {
9709 unsigned NumberLdChain = NumLdStInMemcpy / GluedLdStLimit;
9710 unsigned RemainingLdStInMemcpy = NumLdStInMemcpy % GluedLdStLimit;
9711 unsigned GlueIter = 0;
9712
9713 // Residual ld/st.
9714 if (RemainingLdStInMemcpy) {
9715 chainLoadsAndStoresForMemcpy(
9716 DAG, dl, OutChains, From: NumLdStInMemcpy - RemainingLdStInMemcpy,
9717 To: NumLdStInMemcpy, OutLoadChains, OutStoreChains);
9718 }
9719
9720 for (unsigned cnt = 0; cnt < NumberLdChain; ++cnt) {
9721 unsigned IndexFrom = NumLdStInMemcpy - RemainingLdStInMemcpy -
9722 GlueIter - GluedLdStLimit;
9723 unsigned IndexTo = NumLdStInMemcpy - RemainingLdStInMemcpy - GlueIter;
9724 chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, From: IndexFrom, To: IndexTo,
9725 OutLoadChains, OutStoreChains);
9726 GlueIter += GluedLdStLimit;
9727 }
9728 }
9729 }
9730 }
9731 return DAG.getTokenFactor(DL: dl, Vals&: OutChains);
9732}
9733
9734static SDValue getMemmoveLoadsAndStores(
9735 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
9736 uint64_t Size, Align DstAlign, Align SrcAlign, bool isVol,
9737 bool AlwaysInline, MachinePointerInfo DstPtrInfo,
9738 MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo) {
9739 // Turn a memmove of undef to nop.
9740 // FIXME: We need to honor volatile even is Src is undef.
9741 if (Src.isUndef())
9742 return Chain;
9743
9744 // Expand memmove to a series of load and store ops if the size operand falls
9745 // below a certain threshold.
9746 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9747 const DataLayout &DL = DAG.getDataLayout();
9748 LLVMContext &C = *DAG.getContext();
9749 std::vector<EVT> MemOps;
9750 bool DstAlignCanChange = false;
9751 MachineFunction &MF = DAG.getMachineFunction();
9752 MachineFrameInfo &MFI = MF.getFrameInfo();
9753 bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
9754 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Dst);
9755 if (FI && !MFI.isFixedObjectIndex(ObjectIdx: FI->getIndex()))
9756 DstAlignCanChange = true;
9757 SrcAlign = std::max(a: SrcAlign, b: DAG.InferPtrAlign(Ptr: Src).valueOrOne());
9758 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize);
9759 if (!TLI.findOptimalMemOpLowering(
9760 Context&: C, MemOps, Limit,
9761 Op: MemOp::Move(Size, DstAlignCanChange, DstAlign, SrcAlign, IsVolatile: isVol),
9762 DstAS: DstPtrInfo.getAddrSpace(), SrcAS: SrcPtrInfo.getAddrSpace(),
9763 FuncAttributes: MF.getFunction().getAttributes(), LargestVT: nullptr))
9764 return SDValue();
9765
9766 if (DstAlignCanChange) {
9767 Type *Ty = MemOps[0].getTypeForEVT(Context&: C);
9768 Align NewDstAlign = DL.getABITypeAlign(Ty);
9769
9770 // Don't promote to an alignment that would require dynamic stack
9771 // realignment which may conflict with optimizations such as tail call
9772 // optimization.
9773 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
9774 if (!TRI->hasStackRealignment(MF))
9775 if (MaybeAlign StackAlign = DL.getStackAlignment())
9776 NewDstAlign = std::min(a: NewDstAlign, b: *StackAlign);
9777
9778 if (NewDstAlign > DstAlign) {
9779 // Give the stack frame object a larger alignment if needed.
9780 if (MFI.getObjectAlign(ObjectIdx: FI->getIndex()) < NewDstAlign)
9781 MFI.setObjectAlignment(ObjectIdx: FI->getIndex(), Alignment: NewDstAlign);
9782 DstAlign = NewDstAlign;
9783 }
9784 }
9785
9786 // Prepare AAInfo for loads/stores after lowering this memmove.
9787 AAMDNodes NewAAInfo = AAInfo;
9788 NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
9789
9790 MachineMemOperand::Flags MMOFlags =
9791 isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone;
9792 uint64_t SrcOff = 0;
9793 SmallVector<SDValue, 8> LoadValues;
9794 SmallVector<SDValue, 8> LoadChains;
9795 SmallVector<SDValue, 8> OutChains;
9796 unsigned NumMemOps = MemOps.size();
9797 for (unsigned i = 0; i < NumMemOps; i++) {
9798 EVT VT = MemOps[i];
9799 unsigned VTSize = VT.getSizeInBits() / 8;
9800 SDValue Value;
9801 bool IsOverlapping = false;
9802
9803 if (i == NumMemOps - 1 && i != 0 && VTSize > Size - SrcOff) {
9804 // Issuing an unaligned load / store pair that overlaps with the previous
9805 // pair. Adjust the offset accordingly.
9806 SrcOff = Size - VTSize;
9807 IsOverlapping = true;
9808 }
9809
9810 // Calculate the actual alignment at the current offset. The alignment at
9811 // SrcOff may be lower than the base alignment, especially when using
9812 // overlapping loads.
9813 Align SrcAlignAtOffset = commonAlignment(A: SrcAlign, Offset: SrcOff);
9814 if (IsOverlapping) {
9815 // Verify that the target allows misaligned memory accesses at the
9816 // adjusted offset when using overlapping loads.
9817 unsigned Fast;
9818 if (!TLI.allowsMisalignedMemoryAccesses(VT, AddrSpace: SrcPtrInfo.getAddrSpace(),
9819 Alignment: SrcAlignAtOffset, Flags: MMOFlags,
9820 &Fast) ||
9821 !Fast) {
9822 // This should have been caught by findOptimalMemOpLowering, but verify
9823 // here for safety.
9824 return SDValue();
9825 }
9826 }
9827
9828 bool isDereferenceable =
9829 SrcPtrInfo.getWithOffset(O: SrcOff).isDereferenceable(Size: VTSize, C, DL);
9830 MachineMemOperand::Flags SrcMMOFlags = MMOFlags;
9831 if (isDereferenceable)
9832 SrcMMOFlags |= MachineMemOperand::MODereferenceable;
9833 Value =
9834 DAG.getLoad(VT, dl, Chain,
9835 Ptr: DAG.getObjectPtrOffset(SL: dl, Ptr: Src, Offset: TypeSize::getFixed(ExactSize: SrcOff)),
9836 PtrInfo: SrcPtrInfo.getWithOffset(O: SrcOff), Alignment: SrcAlignAtOffset,
9837 MMOFlags: SrcMMOFlags, Metadata: NewAAInfo);
9838 LoadValues.push_back(Elt: Value);
9839 LoadChains.push_back(Elt: Value.getValue(R: 1));
9840 SrcOff += VTSize;
9841 }
9842 Chain = DAG.getTokenFactor(DL: dl, Vals&: LoadChains);
9843 OutChains.clear();
9844 uint64_t DstOff = 0;
9845 for (unsigned i = 0; i < NumMemOps; i++) {
9846 EVT VT = MemOps[i];
9847 unsigned VTSize = VT.getSizeInBits() / 8;
9848 SDValue Store;
9849 bool IsOverlapping = false;
9850
9851 if (i == NumMemOps - 1 && i != 0 && VTSize > Size - DstOff) {
9852 // Issuing an unaligned load / store pair that overlaps with the previous
9853 // pair. Adjust the offset accordingly.
9854 DstOff = Size - VTSize;
9855 IsOverlapping = true;
9856 }
9857
9858 // Calculate the actual alignment at the current offset. The alignment at
9859 // DstOff may be lower than the base alignment, especially when using
9860 // overlapping stores.
9861 Align DstAlignAtOffset = commonAlignment(A: DstAlign, Offset: DstOff);
9862 if (IsOverlapping) {
9863 // Verify that the target allows misaligned memory accesses at the
9864 // adjusted offset when using overlapping stores.
9865 unsigned Fast;
9866 if (!TLI.allowsMisalignedMemoryAccesses(VT, AddrSpace: DstPtrInfo.getAddrSpace(),
9867 Alignment: DstAlignAtOffset, Flags: MMOFlags,
9868 &Fast) ||
9869 !Fast) {
9870 // This should have been caught by findOptimalMemOpLowering, but verify
9871 // here for safety.
9872 return SDValue();
9873 }
9874 }
9875 Store = DAG.getStore(
9876 Chain, dl, Val: LoadValues[i],
9877 Ptr: DAG.getObjectPtrOffset(SL: dl, Ptr: Dst, Offset: TypeSize::getFixed(ExactSize: DstOff)),
9878 PtrInfo: DstPtrInfo.getWithOffset(O: DstOff), Alignment: DstAlignAtOffset, MMOFlags,
9879 Metadata: NewAAInfo);
9880 OutChains.push_back(Elt: Store);
9881 DstOff += VTSize;
9882 }
9883
9884 return DAG.getTokenFactor(DL: dl, Vals&: OutChains);
9885}
9886
9887/// Lower the call to 'memset' intrinsic function into a series of store
9888/// operations.
9889///
9890/// \param DAG Selection DAG where lowered code is placed.
9891/// \param dl Link to corresponding IR location.
9892/// \param Chain Control flow dependency.
9893/// \param Dst Pointer to destination memory location.
9894/// \param Src Value of byte to write into the memory.
9895/// \param Size Number of bytes to write.
9896/// \param Alignment Alignment of the destination in bytes.
9897/// \param isVol True if destination is volatile.
9898/// \param AlwaysInline Makes sure no function call is generated.
9899/// \param DstPtrInfo IR information on the memory pointer.
9900/// \returns New head in the control flow, if lowering was successful, empty
9901/// SDValue otherwise.
9902///
9903/// The function tries to replace 'llvm.memset' intrinsic with several store
9904/// operations and value calculation code. This is usually profitable for small
9905/// memory size or when the semantic requires inlining.
9906static SDValue getMemsetStores(SelectionDAG &DAG, const SDLoc &dl,
9907 SDValue Chain, SDValue Dst, SDValue Src,
9908 uint64_t Size, Align Alignment, bool isVol,
9909 bool AlwaysInline, MachinePointerInfo DstPtrInfo,
9910 const AAMDNodes &AAInfo) {
9911 // Turn a memset of undef to nop.
9912 // FIXME: We need to honor volatile even is Src is undef.
9913 if (Src.isUndef())
9914 return Chain;
9915
9916 // Expand memset to a series of load/store ops if the size operand
9917 // falls below a certain threshold.
9918 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9919 std::vector<EVT> MemOps;
9920 bool DstAlignCanChange = false;
9921 LLVMContext &C = *DAG.getContext();
9922 MachineFunction &MF = DAG.getMachineFunction();
9923 MachineFrameInfo &MFI = MF.getFrameInfo();
9924 bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
9925 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Dst);
9926 if (FI && !MFI.isFixedObjectIndex(ObjectIdx: FI->getIndex()))
9927 DstAlignCanChange = true;
9928 bool IsZeroVal = isNullConstant(V: Src);
9929 unsigned Limit = AlwaysInline ? ~0 : TLI.getMaxStoresPerMemset(OptSize);
9930
9931 EVT LargestVT;
9932 if (!TLI.findOptimalMemOpLowering(
9933 Context&: C, MemOps, Limit,
9934 Op: MemOp::Set(Size, DstAlignCanChange, DstAlign: Alignment, IsZeroMemset: IsZeroVal, IsVolatile: isVol),
9935 DstAS: DstPtrInfo.getAddrSpace(), SrcAS: ~0u, FuncAttributes: MF.getFunction().getAttributes(),
9936 LargestVT: &LargestVT))
9937 return SDValue();
9938
9939 if (DstAlignCanChange) {
9940 Type *Ty = MemOps[0].getTypeForEVT(Context&: *DAG.getContext());
9941 const DataLayout &DL = DAG.getDataLayout();
9942 Align NewAlign = DL.getABITypeAlign(Ty);
9943
9944 // Don't promote to an alignment that would require dynamic stack
9945 // realignment which may conflict with optimizations such as tail call
9946 // optimization.
9947 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
9948 if (!TRI->hasStackRealignment(MF))
9949 if (MaybeAlign StackAlign = DL.getStackAlignment())
9950 NewAlign = std::min(a: NewAlign, b: *StackAlign);
9951
9952 if (NewAlign > Alignment) {
9953 // Give the stack frame object a larger alignment if needed.
9954 if (MFI.getObjectAlign(ObjectIdx: FI->getIndex()) < NewAlign)
9955 MFI.setObjectAlignment(ObjectIdx: FI->getIndex(), Alignment: NewAlign);
9956 Alignment = NewAlign;
9957 }
9958 }
9959
9960 SmallVector<SDValue, 8> OutChains;
9961 uint64_t DstOff = 0;
9962 unsigned NumMemOps = MemOps.size();
9963
9964 // Find the largest store and generate the bit pattern for it.
9965 // If target didn't set LargestVT, compute it from MemOps.
9966 if (!LargestVT.isSimple()) {
9967 LargestVT = MemOps[0];
9968 for (unsigned i = 1; i < NumMemOps; i++)
9969 if (MemOps[i].bitsGT(VT: LargestVT))
9970 LargestVT = MemOps[i];
9971 }
9972 SDValue MemSetValue = getMemsetValue(Value: Src, VT: LargestVT, DAG, dl);
9973
9974 // Prepare AAInfo for loads/stores after lowering this memset.
9975 AAMDNodes NewAAInfo = AAInfo;
9976 NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
9977
9978 for (unsigned i = 0; i < NumMemOps; i++) {
9979 EVT VT = MemOps[i];
9980 unsigned VTSize = VT.getSizeInBits() / 8;
9981 // The target should specify store types that exactly cover the memset size
9982 // (with the last store potentially being oversized for overlapping stores).
9983 assert(Size > 0 && "Target specified more stores than needed in "
9984 "findOptimalMemOpLowering");
9985 if (VTSize > Size) {
9986 // Issuing an unaligned load / store pair that overlaps with the previous
9987 // pair. Adjust the offset accordingly.
9988 assert(i == NumMemOps-1 && i != 0);
9989 DstOff -= VTSize - Size;
9990 }
9991
9992 // If this store is smaller than the largest store see whether we can get
9993 // the smaller value for free with a truncate or extract vector element and
9994 // then store.
9995 SDValue Value = MemSetValue;
9996 if (VT.bitsLT(VT: LargestVT)) {
9997 unsigned Index;
9998 unsigned NElts = LargestVT.getSizeInBits() / VT.getSizeInBits();
9999 EVT SVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: VT.getScalarType(), NumElements: NElts);
10000 if (!LargestVT.isVector() && !VT.isVector() &&
10001 TLI.isTruncateFree(FromVT: LargestVT, ToVT: VT))
10002 Value = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, N1: MemSetValue);
10003 else if (LargestVT.isVector() && !VT.isVector() &&
10004 TLI.shallExtractConstSplatVectorElementToStore(
10005 VectorTy: LargestVT.getTypeForEVT(Context&: *DAG.getContext()),
10006 ElemSizeInBits: VT.getSizeInBits(), Index) &&
10007 TLI.isTypeLegal(VT: SVT) &&
10008 LargestVT.getSizeInBits() == SVT.getSizeInBits()) {
10009 // Target which can combine store(extractelement VectorTy, Idx) can get
10010 // the smaller value for free.
10011 SDValue TailValue = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: SVT, N1: MemSetValue);
10012 Value = DAG.getExtractVectorElt(DL: dl, VT, Vec: TailValue, Idx: Index);
10013 } else
10014 Value = getMemsetValue(Value: Src, VT, DAG, dl);
10015 }
10016 assert(Value.getValueType() == VT && "Value with wrong type.");
10017 SDValue Store = DAG.getStore(
10018 Chain, dl, Val: Value,
10019 Ptr: DAG.getObjectPtrOffset(SL: dl, Ptr: Dst, Offset: TypeSize::getFixed(ExactSize: DstOff)),
10020 PtrInfo: DstPtrInfo.getWithOffset(O: DstOff), Alignment,
10021 MMOFlags: isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone,
10022 Metadata: NewAAInfo);
10023 OutChains.push_back(Elt: Store);
10024 DstOff += VT.getSizeInBits() / 8;
10025 // For oversized overlapping stores, only subtract the remaining bytes.
10026 // For normal stores, subtract the full store size.
10027 if (VTSize > Size) {
10028 Size = 0;
10029 } else {
10030 Size -= VTSize;
10031 }
10032 }
10033
10034 // After processing all stores, Size should be exactly 0. Any remaining bytes
10035 // indicate a bug in the target's findOptimalMemOpLowering implementation.
10036 assert(Size == 0 && "Target's findOptimalMemOpLowering did not specify "
10037 "stores that exactly cover the memset size");
10038
10039 return DAG.getTokenFactor(DL: dl, Vals&: OutChains);
10040}
10041
10042static void checkAddrSpaceIsValidForLibcall(const TargetLowering *TLI,
10043 const DataLayout &DL, unsigned AS) {
10044 // Lowering memcpy / memset / memmove intrinsics to calls is only valid if all
10045 // pointer operands can be losslessly bitcasted to pointers of address space 0
10046 if (AS != 0 && !TLI->getTargetMachine().isNoopAddrSpaceCast(DL, SrcAS: AS, DestAS: 0)) {
10047 report_fatal_error(reason: "cannot lower memory intrinsic in address space " +
10048 Twine(AS));
10049 }
10050}
10051
10052/// The length of a memory intrinsic (e.g. number of bytes to copy) is unsigned
10053/// and may have any integer type. We zero-extend or truncate it to the pointer
10054/// type of the narrower address space being accessed.
10055static SDValue getMemIntrinsicSize(SelectionDAG &DAG, const SDLoc &dl,
10056 SDValue Size, unsigned DstAS,
10057 unsigned SrcAS) {
10058 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
10059 MVT DstVT = TLI.getPointerTy(DL: DAG.getDataLayout(), AS: DstAS);
10060 MVT SrcVT = TLI.getPointerTy(DL: DAG.getDataLayout(), AS: SrcAS);
10061 return DAG.getZExtOrTrunc(Op: Size, DL: dl, VT: DstVT.bitsLT(VT: SrcVT) ? DstVT : SrcVT);
10062}
10063
10064static bool isInTailCallPositionWrapper(const CallInst *CI,
10065 const SelectionDAG *SelDAG,
10066 bool AllowReturnsFirstArg) {
10067 if (!CI || !CI->isTailCall())
10068 return false;
10069 // TODO: Fix "returns-first-arg" determination so it doesn't depend on which
10070 // helper symbol we lower to.
10071 return isInTailCallPosition(Call: *CI, TM: SelDAG->getTarget(),
10072 ReturnsFirstArg: AllowReturnsFirstArg &&
10073 funcReturnsFirstArgOfCall(CI: *CI));
10074}
10075
10076static std::pair<SDValue, SDValue>
10077getRuntimeCallSDValueHelper(SDValue Chain, const SDLoc &dl,
10078 TargetLowering::ArgListTy &&Args,
10079 const CallInst *CI, RTLIB::Libcall Call,
10080 SelectionDAG *DAG, const TargetLowering *TLI) {
10081 RTLIB::LibcallImpl LCImpl = DAG->getLibcalls().getLibcallImpl(Call);
10082
10083 if (LCImpl == RTLIB::Unsupported)
10084 return {};
10085
10086 TargetLowering::CallLoweringInfo CLI(*DAG);
10087 bool IsTailCall =
10088 isInTailCallPositionWrapper(CI, SelDAG: DAG, /*AllowReturnsFirstArg=*/true) &&
10089 // Lowering doesn't support tail calling inside a function with
10090 // a swifterror argument yet.
10091 !DAG->hasSwiftErrorArg();
10092 SDValue Callee =
10093 DAG->getExternalSymbol(Libcall: LCImpl, VT: TLI->getPointerTy(DL: DAG->getDataLayout()));
10094
10095 CLI.setDebugLoc(dl)
10096 .setChain(Chain)
10097 .setLibCallee(CC: DAG->getLibcalls().getLibcallImplCallingConv(Call: LCImpl),
10098 ResultType: CI->getType(), Target: Callee, ArgsList: std::move(Args))
10099 .setTailCall(IsTailCall);
10100
10101 return TLI->LowerCallTo(CLI);
10102}
10103
10104std::pair<SDValue, SDValue> SelectionDAG::getStrcmp(SDValue Chain,
10105 const SDLoc &dl, SDValue S1,
10106 SDValue S2,
10107 const CallInst *CI) {
10108 PointerType *PT = PointerType::getUnqual(C&: *getContext());
10109 TargetLowering::ArgListTy Args = {{S1, PT}, {S2, PT}};
10110 return getRuntimeCallSDValueHelper(Chain, dl, Args: std::move(Args), CI,
10111 Call: RTLIB::STRCMP, DAG: this, TLI);
10112}
10113
10114std::pair<SDValue, SDValue> SelectionDAG::getStrstr(SDValue Chain,
10115 const SDLoc &dl, SDValue S1,
10116 SDValue S2,
10117 const CallInst *CI) {
10118 PointerType *PT = PointerType::getUnqual(C&: *getContext());
10119 TargetLowering::ArgListTy Args = {{S1, PT}, {S2, PT}};
10120 return getRuntimeCallSDValueHelper(Chain, dl, Args: std::move(Args), CI,
10121 Call: RTLIB::STRSTR, DAG: this, TLI);
10122}
10123
10124std::pair<SDValue, SDValue> SelectionDAG::getMemccpy(SDValue Chain,
10125 const SDLoc &dl,
10126 SDValue Dst, SDValue Src,
10127 SDValue C, SDValue Size,
10128 const CallInst *CI) {
10129 PointerType *PT = PointerType::getUnqual(C&: *getContext());
10130
10131 TargetLowering::ArgListTy Args = {
10132 {Dst, PT},
10133 {Src, PT},
10134 {C, Type::getInt32Ty(C&: *getContext())},
10135 {Size, getDataLayout().getIntPtrType(C&: *getContext())}};
10136 return getRuntimeCallSDValueHelper(Chain, dl, Args: std::move(Args), CI,
10137 Call: RTLIB::MEMCCPY, DAG: this, TLI);
10138}
10139
10140std::pair<SDValue, SDValue>
10141SelectionDAG::getMemcmp(SDValue Chain, const SDLoc &dl, SDValue Mem0,
10142 SDValue Mem1, SDValue Size, const CallInst *CI) {
10143 PointerType *PT = PointerType::getUnqual(C&: *getContext());
10144 TargetLowering::ArgListTy Args = {
10145 {Mem0, PT},
10146 {Mem1, PT},
10147 {Size, getDataLayout().getIntPtrType(C&: *getContext())}};
10148 return getRuntimeCallSDValueHelper(Chain, dl, Args: std::move(Args), CI,
10149 Call: RTLIB::MEMCMP, DAG: this, TLI);
10150}
10151
10152std::pair<SDValue, SDValue> SelectionDAG::getStrcpy(SDValue Chain,
10153 const SDLoc &dl,
10154 SDValue Dst, SDValue Src,
10155 const CallInst *CI) {
10156 PointerType *PT = PointerType::getUnqual(C&: *getContext());
10157 TargetLowering::ArgListTy Args = {{Dst, PT}, {Src, PT}};
10158 return getRuntimeCallSDValueHelper(Chain, dl, Args: std::move(Args), CI,
10159 Call: RTLIB::STRCPY, DAG: this, TLI);
10160}
10161
10162std::pair<SDValue, SDValue> SelectionDAG::getStrlen(SDValue Chain,
10163 const SDLoc &dl,
10164 SDValue Src,
10165 const CallInst *CI) {
10166 // Emit a library call.
10167 TargetLowering::ArgListTy Args = {
10168 {Src, PointerType::getUnqual(C&: *getContext())}};
10169 return getRuntimeCallSDValueHelper(Chain, dl, Args: std::move(Args), CI,
10170 Call: RTLIB::STRLEN, DAG: this, TLI);
10171}
10172
10173bool SelectionDAG::hasSwiftErrorArg() const {
10174 return TLI->supportSwiftError() &&
10175 MF->getFunction().getAttributes().hasAttrSomewhere(
10176 Kind: Attribute::SwiftError);
10177}
10178
10179SDValue SelectionDAG::getMemcpy(
10180 SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size,
10181 Align DstAlign, Align SrcAlign, bool isVol, bool AlwaysInline,
10182 const CallInst *CI, std::optional<bool> OverrideTailCall,
10183 MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo,
10184 const AAMDNodes &AAInfo, BatchAAResults *BatchAA) {
10185 Size = getMemIntrinsicSize(DAG&: *this, dl, Size, DstAS: DstPtrInfo.getAddrSpace(),
10186 SrcAS: SrcPtrInfo.getAddrSpace());
10187 // Check to see if we should lower the memcpy to loads and stores first.
10188 // For cases within the target-specified limits, this is the best choice.
10189 const MDNode *DstMemCacheHint =
10190 CI ? getMemCacheHintMetadata(I: *CI, /*OperandNo=*/0) : nullptr;
10191 const MDNode *SrcMemCacheHint =
10192 CI ? getMemCacheHintMetadata(I: *CI, /*OperandNo=*/1) : nullptr;
10193
10194 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Val&: Size);
10195 if (ConstantSize) {
10196 // Memcpy with size zero? Just return the original chain.
10197 if (ConstantSize->isZero())
10198 return Chain;
10199
10200 SDValue Result = getMemcpyLoadsAndStores(
10201 DAG&: *this, dl, Chain, Dst, Src, Size: ConstantSize->getZExtValue(), DstAlign,
10202 SrcAlign, isVol, AlwaysInline: false, DstPtrInfo, SrcPtrInfo, AAInfo, BatchAA,
10203 DstMemCacheHint, SrcMemCacheHint);
10204 if (Result.getNode())
10205 return Result;
10206 }
10207
10208 // Then check to see if we should lower the memcpy with target-specific
10209 // code. If the target chooses to do this, this is the next best.
10210 if (TSI) {
10211 SDValue Result = TSI->EmitTargetCodeForMemcpy(
10212 DAG&: *this, dl, Chain, Op1: Dst, Op2: Src, Op3: Size, DstAlign, SrcAlign, isVolatile: isVol,
10213 AlwaysInline, DstPtrInfo, SrcPtrInfo);
10214 if (Result.getNode())
10215 return Result;
10216 }
10217
10218 // If we really need inline code and the target declined to provide it,
10219 // use a (potentially long) sequence of loads and stores.
10220 if (AlwaysInline) {
10221 assert(ConstantSize && "AlwaysInline requires a constant size!");
10222 return getMemcpyLoadsAndStores(
10223 DAG&: *this, dl, Chain, Dst, Src, Size: ConstantSize->getZExtValue(), DstAlign,
10224 SrcAlign, isVol, AlwaysInline: true, DstPtrInfo, SrcPtrInfo, AAInfo, BatchAA,
10225 DstMemCacheHint, SrcMemCacheHint);
10226 }
10227
10228 checkAddrSpaceIsValidForLibcall(TLI, DL: getDataLayout(),
10229 AS: DstPtrInfo.getAddrSpace());
10230 checkAddrSpaceIsValidForLibcall(TLI, DL: getDataLayout(),
10231 AS: SrcPtrInfo.getAddrSpace());
10232
10233 // FIXME: If the memcpy is volatile (isVol), lowering it to a plain libc
10234 // memcpy is not guaranteed to be safe. libc memcpys aren't required to
10235 // respect volatile, so they may do things like read or write memory
10236 // beyond the given memory regions. But fixing this isn't easy, and most
10237 // people don't care.
10238
10239 // Emit a library call.
10240 TargetLowering::ArgListTy Args;
10241 Type *PtrTy = PointerType::getUnqual(C&: *getContext());
10242 Args.emplace_back(args&: Dst, args&: PtrTy);
10243 Args.emplace_back(args&: Src, args&: PtrTy);
10244 Args.emplace_back(args&: Size, args: getDataLayout().getIntPtrType(C&: *getContext()));
10245 // FIXME: pass in SDLoc
10246 TargetLowering::CallLoweringInfo CLI(*this);
10247 bool IsTailCall = false;
10248 RTLIB::LibcallImpl MemCpyImpl = TLI->getMemcpyImpl();
10249
10250 if (OverrideTailCall.has_value()) {
10251 IsTailCall = *OverrideTailCall;
10252 } else {
10253 bool LowersToMemcpy = MemCpyImpl == RTLIB::impl_memcpy;
10254 IsTailCall = isInTailCallPositionWrapper(CI, SelDAG: this, AllowReturnsFirstArg: LowersToMemcpy);
10255 }
10256 // Lowering doesn't support tail calling inside a function with a
10257 // swifterror argument yet.
10258 IsTailCall &= !hasSwiftErrorArg();
10259
10260 CLI.setDebugLoc(dl)
10261 .setChain(Chain)
10262 .setLibCallee(
10263 CC: Libcalls->getLibcallImplCallingConv(Call: MemCpyImpl),
10264 ResultType: Dst.getValueType().getTypeForEVT(Context&: *getContext()),
10265 Target: getExternalSymbol(Libcall: MemCpyImpl, VT: TLI->getPointerTy(DL: getDataLayout())),
10266 ArgsList: std::move(Args))
10267 .setDiscardResult()
10268 .setTailCall(IsTailCall);
10269
10270 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
10271 return CallResult.second;
10272}
10273
10274SDValue SelectionDAG::getAtomicMemcpy(SDValue Chain, const SDLoc &dl,
10275 SDValue Dst, SDValue Src, SDValue Size,
10276 Type *SizeTy, unsigned ElemSz,
10277 bool isTailCall,
10278 MachinePointerInfo DstPtrInfo,
10279 MachinePointerInfo SrcPtrInfo) {
10280 // Lowering doesn't support tail calling inside a function with a
10281 // swifterror argument yet.
10282 isTailCall &= !hasSwiftErrorArg();
10283
10284 // Emit a library call.
10285 TargetLowering::ArgListTy Args;
10286 Type *ArgTy = getDataLayout().getIntPtrType(C&: *getContext());
10287 Args.emplace_back(args&: Dst, args&: ArgTy);
10288 Args.emplace_back(args&: Src, args&: ArgTy);
10289 Args.emplace_back(args&: Size, args&: SizeTy);
10290
10291 RTLIB::Libcall LibraryCall =
10292 RTLIB::getMEMCPY_ELEMENT_UNORDERED_ATOMIC(ElementSize: ElemSz);
10293 RTLIB::LibcallImpl LibcallImpl = Libcalls->getLibcallImpl(Call: LibraryCall);
10294 if (LibcallImpl == RTLIB::Unsupported)
10295 report_fatal_error(reason: "Unsupported element size");
10296
10297 TargetLowering::CallLoweringInfo CLI(*this);
10298 CLI.setDebugLoc(dl)
10299 .setChain(Chain)
10300 .setLibCallee(
10301 CC: Libcalls->getLibcallImplCallingConv(Call: LibcallImpl),
10302 ResultType: Type::getVoidTy(C&: *getContext()),
10303 Target: getExternalSymbol(Libcall: LibcallImpl, VT: TLI->getPointerTy(DL: getDataLayout())),
10304 ArgsList: std::move(Args))
10305 .setDiscardResult()
10306 .setTailCall(isTailCall);
10307
10308 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
10309 return CallResult.second;
10310}
10311
10312SDValue SelectionDAG::getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst,
10313 SDValue Src, SDValue Size, Align DstAlign,
10314 Align SrcAlign, bool isVol, const CallInst *CI,
10315 std::optional<bool> OverrideTailCall,
10316 MachinePointerInfo DstPtrInfo,
10317 MachinePointerInfo SrcPtrInfo,
10318 const AAMDNodes &AAInfo,
10319 BatchAAResults *BatchAA) {
10320 Size = getMemIntrinsicSize(DAG&: *this, dl, Size, DstAS: DstPtrInfo.getAddrSpace(),
10321 SrcAS: SrcPtrInfo.getAddrSpace());
10322 // Check to see if we should lower the memmove to loads and stores first.
10323 // For cases within the target-specified limits, this is the best choice.
10324 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Val&: Size);
10325 if (ConstantSize) {
10326 // Memmove with size zero? Just return the original chain.
10327 if (ConstantSize->isZero())
10328 return Chain;
10329
10330 SDValue Result = getMemmoveLoadsAndStores(
10331 DAG&: *this, dl, Chain, Dst, Src, Size: ConstantSize->getZExtValue(), DstAlign,
10332 SrcAlign, isVol, AlwaysInline: false, DstPtrInfo, SrcPtrInfo, AAInfo);
10333 if (Result.getNode())
10334 return Result;
10335 }
10336
10337 // Then check to see if we should lower the memmove with target-specific
10338 // code. If the target chooses to do this, this is the next best.
10339 if (TSI) {
10340 SDValue Result = TSI->EmitTargetCodeForMemmove(
10341 DAG&: *this, dl, Chain, Op1: Dst, Op2: Src, Op3: Size, DstAlign, SrcAlign, isVolatile: isVol, DstPtrInfo,
10342 SrcPtrInfo);
10343 if (Result.getNode())
10344 return Result;
10345 }
10346
10347 checkAddrSpaceIsValidForLibcall(TLI, DL: getDataLayout(),
10348 AS: DstPtrInfo.getAddrSpace());
10349 checkAddrSpaceIsValidForLibcall(TLI, DL: getDataLayout(),
10350 AS: SrcPtrInfo.getAddrSpace());
10351
10352 // FIXME: If the memmove is volatile, lowering it to plain libc memmove may
10353 // not be safe. See memcpy above for more details.
10354
10355 // Emit a library call.
10356 TargetLowering::ArgListTy Args;
10357 Type *PtrTy = PointerType::getUnqual(C&: *getContext());
10358 Args.emplace_back(args&: Dst, args&: PtrTy);
10359 Args.emplace_back(args&: Src, args&: PtrTy);
10360 Args.emplace_back(args&: Size, args: getDataLayout().getIntPtrType(C&: *getContext()));
10361 // FIXME: pass in SDLoc
10362 TargetLowering::CallLoweringInfo CLI(*this);
10363
10364 RTLIB::LibcallImpl MemmoveImpl = Libcalls->getLibcallImpl(Call: RTLIB::MEMMOVE);
10365
10366 bool IsTailCall = false;
10367 if (OverrideTailCall.has_value()) {
10368 IsTailCall = *OverrideTailCall;
10369 } else {
10370 bool LowersToMemmove = MemmoveImpl == RTLIB::impl_memmove;
10371 IsTailCall = isInTailCallPositionWrapper(CI, SelDAG: this, AllowReturnsFirstArg: LowersToMemmove);
10372 }
10373 // Lowering doesn't support tail calling inside a function with a
10374 // swifterror argument yet.
10375 IsTailCall &= !hasSwiftErrorArg();
10376
10377 CLI.setDebugLoc(dl)
10378 .setChain(Chain)
10379 .setLibCallee(
10380 CC: Libcalls->getLibcallImplCallingConv(Call: MemmoveImpl),
10381 ResultType: Dst.getValueType().getTypeForEVT(Context&: *getContext()),
10382 Target: getExternalSymbol(Libcall: MemmoveImpl, VT: TLI->getPointerTy(DL: getDataLayout())),
10383 ArgsList: std::move(Args))
10384 .setDiscardResult()
10385 .setTailCall(IsTailCall);
10386
10387 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
10388 return CallResult.second;
10389}
10390
10391SDValue SelectionDAG::getAtomicMemmove(SDValue Chain, const SDLoc &dl,
10392 SDValue Dst, SDValue Src, SDValue Size,
10393 Type *SizeTy, unsigned ElemSz,
10394 bool isTailCall,
10395 MachinePointerInfo DstPtrInfo,
10396 MachinePointerInfo SrcPtrInfo) {
10397 // Lowering doesn't support tail calling inside a function with a
10398 // swifterror argument yet.
10399 isTailCall &= !hasSwiftErrorArg();
10400
10401 // Emit a library call.
10402 TargetLowering::ArgListTy Args;
10403 Type *IntPtrTy = getDataLayout().getIntPtrType(C&: *getContext());
10404 Args.emplace_back(args&: Dst, args&: IntPtrTy);
10405 Args.emplace_back(args&: Src, args&: IntPtrTy);
10406 Args.emplace_back(args&: Size, args&: SizeTy);
10407
10408 RTLIB::Libcall LibraryCall =
10409 RTLIB::getMEMMOVE_ELEMENT_UNORDERED_ATOMIC(ElementSize: ElemSz);
10410 RTLIB::LibcallImpl LibcallImpl = Libcalls->getLibcallImpl(Call: LibraryCall);
10411 if (LibcallImpl == RTLIB::Unsupported)
10412 report_fatal_error(reason: "Unsupported element size");
10413
10414 TargetLowering::CallLoweringInfo CLI(*this);
10415 CLI.setDebugLoc(dl)
10416 .setChain(Chain)
10417 .setLibCallee(
10418 CC: Libcalls->getLibcallImplCallingConv(Call: LibcallImpl),
10419 ResultType: Type::getVoidTy(C&: *getContext()),
10420 Target: getExternalSymbol(Libcall: LibcallImpl, VT: TLI->getPointerTy(DL: getDataLayout())),
10421 ArgsList: std::move(Args))
10422 .setDiscardResult()
10423 .setTailCall(isTailCall);
10424
10425 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
10426 return CallResult.second;
10427}
10428
10429SDValue SelectionDAG::getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst,
10430 SDValue Src, SDValue Size, Align Alignment,
10431 bool isVol, bool AlwaysInline,
10432 const CallInst *CI,
10433 MachinePointerInfo DstPtrInfo,
10434 const AAMDNodes &AAInfo) {
10435 Size = getMemIntrinsicSize(DAG&: *this, dl, Size, DstAS: DstPtrInfo.getAddrSpace(),
10436 SrcAS: DstPtrInfo.getAddrSpace());
10437 // Check to see if we should lower the memset to stores first.
10438 // For cases within the target-specified limits, this is the best choice.
10439 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Val&: Size);
10440 if (ConstantSize) {
10441 // Memset with size zero? Just return the original chain.
10442 if (ConstantSize->isZero())
10443 return Chain;
10444
10445 SDValue Result = getMemsetStores(DAG&: *this, dl, Chain, Dst, Src,
10446 Size: ConstantSize->getZExtValue(), Alignment,
10447 isVol, AlwaysInline: false, DstPtrInfo, AAInfo);
10448
10449 if (Result.getNode())
10450 return Result;
10451 }
10452
10453 // Then check to see if we should lower the memset with target-specific
10454 // code. If the target chooses to do this, this is the next best.
10455 if (TSI) {
10456 SDValue Result = TSI->EmitTargetCodeForMemset(
10457 DAG&: *this, dl, Chain, Op1: Dst, Op2: Src, Op3: Size, Alignment, isVolatile: isVol, AlwaysInline, DstPtrInfo);
10458 if (Result.getNode())
10459 return Result;
10460 }
10461
10462 // If we really need inline code and the target declined to provide it,
10463 // use a (potentially long) sequence of loads and stores.
10464 if (AlwaysInline) {
10465 assert(ConstantSize && "AlwaysInline requires a constant size!");
10466 SDValue Result = getMemsetStores(DAG&: *this, dl, Chain, Dst, Src,
10467 Size: ConstantSize->getZExtValue(), Alignment,
10468 isVol, AlwaysInline: true, DstPtrInfo, AAInfo);
10469 assert(Result &&
10470 "getMemsetStores must return a valid sequence when AlwaysInline");
10471 return Result;
10472 }
10473
10474 checkAddrSpaceIsValidForLibcall(TLI, DL: getDataLayout(),
10475 AS: DstPtrInfo.getAddrSpace());
10476
10477 // Emit a library call.
10478 auto &Ctx = *getContext();
10479 const auto& DL = getDataLayout();
10480
10481 TargetLowering::CallLoweringInfo CLI(*this);
10482 // FIXME: pass in SDLoc
10483 CLI.setDebugLoc(dl).setChain(Chain);
10484
10485 RTLIB::LibcallImpl BzeroImpl = Libcalls->getLibcallImpl(Call: RTLIB::BZERO);
10486 bool UseBZero = BzeroImpl != RTLIB::Unsupported && isNullConstant(V: Src);
10487
10488 // If zeroing out and bzero is present, use it.
10489 if (UseBZero) {
10490 TargetLowering::ArgListTy Args;
10491 Args.emplace_back(args&: Dst, args: PointerType::getUnqual(C&: Ctx));
10492 Args.emplace_back(args&: Size, args: DL.getIntPtrType(C&: Ctx));
10493 CLI.setLibCallee(
10494 CC: Libcalls->getLibcallImplCallingConv(Call: BzeroImpl), ResultType: Type::getVoidTy(C&: Ctx),
10495 Target: getExternalSymbol(Libcall: BzeroImpl, VT: TLI->getPointerTy(DL)), ArgsList: std::move(Args));
10496 } else {
10497 RTLIB::LibcallImpl MemsetImpl = Libcalls->getLibcallImpl(Call: RTLIB::MEMSET);
10498
10499 MVT IntVT = MVT::getIntegerVT(BitWidth: LibInfo->getIntSize());
10500 TargetLowering::ArgListTy Args;
10501 Args.emplace_back(args&: Dst, args: PointerType::getUnqual(C&: Ctx));
10502 Args.emplace_back(args: getAnyExtOrTrunc(Op: Src, DL: dl, VT: IntVT),
10503 args: Type::getIntNTy(C&: Ctx, N: LibInfo->getIntSize()),
10504 args: LibInfo->getExtAttrForI32Param());
10505 Args.emplace_back(args&: Size, args: DL.getIntPtrType(C&: Ctx));
10506
10507 CLI.setLibCallee(CC: Libcalls->getLibcallImplCallingConv(Call: MemsetImpl),
10508 ResultType: Dst.getValueType().getTypeForEVT(Context&: Ctx),
10509 Target: getExternalSymbol(Libcall: MemsetImpl, VT: TLI->getPointerTy(DL)),
10510 ArgsList: std::move(Args));
10511 }
10512
10513 RTLIB::LibcallImpl MemsetImpl = Libcalls->getLibcallImpl(Call: RTLIB::MEMSET);
10514 bool LowersToMemset = MemsetImpl == RTLIB::impl_memset;
10515
10516 // If we're going to use bzero, make sure not to tail call unless the
10517 // subsequent return doesn't need a value, as bzero doesn't return the first
10518 // arg unlike memset.
10519 bool ReturnsFirstArg = CI && funcReturnsFirstArgOfCall(CI: *CI) && !UseBZero;
10520 bool IsTailCall = CI && CI->isTailCall() &&
10521 isInTailCallPosition(Call: *CI, TM: getTarget(),
10522 ReturnsFirstArg: ReturnsFirstArg && LowersToMemset) &&
10523 // Lowering doesn't support tail calling inside a function
10524 // with a swifterror argument yet.
10525 !hasSwiftErrorArg();
10526 CLI.setDiscardResult().setTailCall(IsTailCall);
10527
10528 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
10529 return CallResult.second;
10530}
10531
10532SDValue SelectionDAG::getAtomicMemset(SDValue Chain, const SDLoc &dl,
10533 SDValue Dst, SDValue Value, SDValue Size,
10534 Type *SizeTy, unsigned ElemSz,
10535 bool isTailCall,
10536 MachinePointerInfo DstPtrInfo) {
10537 // Lowering doesn't support tail calling inside a function with a
10538 // swifterror argument yet.
10539 isTailCall &= !hasSwiftErrorArg();
10540
10541 // Emit a library call.
10542 TargetLowering::ArgListTy Args;
10543 Args.emplace_back(args&: Dst, args: getDataLayout().getIntPtrType(C&: *getContext()));
10544 Args.emplace_back(args&: Value, args: Type::getInt8Ty(C&: *getContext()));
10545 Args.emplace_back(args&: Size, args&: SizeTy);
10546
10547 RTLIB::Libcall LibraryCall =
10548 RTLIB::getMEMSET_ELEMENT_UNORDERED_ATOMIC(ElementSize: ElemSz);
10549 RTLIB::LibcallImpl LibcallImpl = Libcalls->getLibcallImpl(Call: LibraryCall);
10550 if (LibcallImpl == RTLIB::Unsupported)
10551 report_fatal_error(reason: "Unsupported element size");
10552
10553 TargetLowering::CallLoweringInfo CLI(*this);
10554 CLI.setDebugLoc(dl)
10555 .setChain(Chain)
10556 .setLibCallee(
10557 CC: Libcalls->getLibcallImplCallingConv(Call: LibcallImpl),
10558 ResultType: Type::getVoidTy(C&: *getContext()),
10559 Target: getExternalSymbol(Libcall: LibcallImpl, VT: TLI->getPointerTy(DL: getDataLayout())),
10560 ArgsList: std::move(Args))
10561 .setDiscardResult()
10562 .setTailCall(isTailCall);
10563
10564 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
10565 return CallResult.second;
10566}
10567
10568SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
10569 SDVTList VTList, ArrayRef<SDValue> Ops,
10570 MachineMemOperand *MMO,
10571 ISD::LoadExtType ExtType) {
10572 SDNodeKey ID(Opcode, VTList, Ops);
10573 ID.AddInteger(I: MemVT.getRawBits());
10574 ID.AddInteger(I: getSyntheticNodeSubclassData<AtomicSDNode>(
10575 IROrder: dl.getIROrder(), Args&: Opcode, Args&: VTList, Args&: MemVT, Args&: MMO, Args&: ExtType));
10576 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10577 ID.AddInteger(I: MMO->getFlags());
10578 FoldingSetInsertToken InsertToken;
10579 if (auto *E = cast_or_null<AtomicSDNode>(Val: lookupNode(Key: ID, DL: dl, InsertToken))) {
10580 E->refineAlignment(NewMMO: MMO);
10581 E->refineMMOMetadata(NewMMO: MMO);
10582 return SDValue(E, 0);
10583 }
10584
10585 auto *N = newSDNode<AtomicSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: Opcode,
10586 Args&: VTList, Args&: MemVT, Args&: MMO, Args&: ExtType);
10587 createOperands(Node: N, Vals: Ops);
10588
10589 CSEMap.insert(N, Token: InsertToken);
10590 InsertNode(N);
10591 SDValue V(N, 0);
10592 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10593 return V;
10594}
10595
10596SDValue SelectionDAG::getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl,
10597 EVT MemVT, SDVTList VTs, SDValue Chain,
10598 SDValue Ptr, SDValue Cmp, SDValue Swp,
10599 MachineMemOperand *MMO) {
10600 assert(Opcode == ISD::ATOMIC_CMP_SWAP ||
10601 Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
10602 assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
10603
10604 SDValue Ops[] = {Chain, Ptr, Cmp, Swp};
10605 return getAtomic(Opcode, dl, MemVT, VTList: VTs, Ops, MMO);
10606}
10607
10608SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
10609 SDValue Chain, SDValue Ptr, SDValue Val,
10610 MachineMemOperand *MMO) {
10611 assert((Opcode == ISD::ATOMIC_LOAD_ADD || Opcode == ISD::ATOMIC_LOAD_SUB ||
10612 Opcode == ISD::ATOMIC_LOAD_AND || Opcode == ISD::ATOMIC_LOAD_CLR ||
10613 Opcode == ISD::ATOMIC_LOAD_OR || Opcode == ISD::ATOMIC_LOAD_XOR ||
10614 Opcode == ISD::ATOMIC_LOAD_NAND || Opcode == ISD::ATOMIC_LOAD_MIN ||
10615 Opcode == ISD::ATOMIC_LOAD_MAX || Opcode == ISD::ATOMIC_LOAD_UMIN ||
10616 Opcode == ISD::ATOMIC_LOAD_UMAX || Opcode == ISD::ATOMIC_LOAD_FADD ||
10617 Opcode == ISD::ATOMIC_LOAD_FSUB || Opcode == ISD::ATOMIC_LOAD_FMAX ||
10618 Opcode == ISD::ATOMIC_LOAD_FMIN ||
10619 Opcode == ISD::ATOMIC_LOAD_FMINIMUM ||
10620 Opcode == ISD::ATOMIC_LOAD_FMAXIMUM ||
10621 Opcode == ISD::ATOMIC_LOAD_UINC_WRAP ||
10622 Opcode == ISD::ATOMIC_LOAD_UDEC_WRAP ||
10623 Opcode == ISD::ATOMIC_LOAD_USUB_COND ||
10624 Opcode == ISD::ATOMIC_LOAD_USUB_SAT || Opcode == ISD::ATOMIC_SWAP ||
10625 Opcode == ISD::ATOMIC_STORE) &&
10626 "Invalid Atomic Op");
10627
10628 EVT VT = Val.getValueType();
10629
10630 SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(VT: MVT::Other) :
10631 getVTList(VT1: VT, VT2: MVT::Other);
10632 SDValue Ops[] = {Chain, Ptr, Val};
10633 return getAtomic(Opcode, dl, MemVT, VTList: VTs, Ops, MMO);
10634}
10635
10636SDValue SelectionDAG::getAtomicLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
10637 EVT MemVT, EVT VT, SDValue Chain,
10638 SDValue Ptr, MachineMemOperand *MMO) {
10639 SDVTList VTs = getVTList(VT1: VT, VT2: MVT::Other);
10640 SDValue Ops[] = {Chain, Ptr};
10641 return getAtomic(Opcode: ISD::ATOMIC_LOAD, dl, MemVT, VTList: VTs, Ops, MMO, ExtType);
10642}
10643
10644/// getMergeValues - Create a MERGE_VALUES node from the given operands.
10645SDValue SelectionDAG::getMergeValues(ArrayRef<SDValue> Ops, const SDLoc &dl) {
10646 if (Ops.size() == 1)
10647 return Ops[0];
10648
10649 SmallVector<EVT, 4> VTs;
10650 VTs.reserve(N: Ops.size());
10651 for (const SDValue &Op : Ops)
10652 VTs.push_back(Elt: Op.getValueType());
10653 return getNode(Opcode: ISD::MERGE_VALUES, DL: dl, VTList: getVTList(VTs), Ops);
10654}
10655
10656SDValue SelectionDAG::getErrorMergeValues(ArrayRef<EVT> ResultTypes,
10657 SDValue Chain, const SDLoc &dl) {
10658 SmallVector<SDValue, 4> RetValues;
10659 RetValues.reserve(N: ResultTypes.size());
10660 for (EVT VT : ResultTypes)
10661 RetValues.push_back(Elt: VT == MVT::Other ? Chain : getPOISON(VT));
10662 return getMergeValues(Ops: RetValues, dl);
10663}
10664
10665SDValue SelectionDAG::getMemIntrinsicNode(
10666 unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef<SDValue> Ops,
10667 EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment,
10668 MachineMemOperand::Flags Flags, LocationSize Size,
10669 const AAMDNodes &AAInfo) {
10670 if (Size.hasValue() && !Size.getValue())
10671 Size = LocationSize::precise(Value: MemVT.getStoreSize());
10672
10673 MachineFunction &MF = getMachineFunction();
10674 MachineMemOperand *MMO =
10675 MF.getMachineMemOperand(PtrInfo, F: Flags, Size, BaseAlignment: Alignment, Metadata: AAInfo);
10676
10677 return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, MMO);
10678}
10679
10680SDValue SelectionDAG::getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl,
10681 SDVTList VTList,
10682 ArrayRef<SDValue> Ops, EVT MemVT,
10683 MachineMemOperand *MMO) {
10684 return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, MMOs: ArrayRef(MMO));
10685}
10686
10687SDValue SelectionDAG::getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl,
10688 SDVTList VTList,
10689 ArrayRef<SDValue> Ops, EVT MemVT,
10690 ArrayRef<MachineMemOperand *> MMOs) {
10691 assert(!MMOs.empty() && "Must have at least one MMO");
10692 assert(
10693 (Opcode == ISD::INTRINSIC_VOID || Opcode == ISD::INTRINSIC_W_CHAIN ||
10694 Opcode == ISD::PREFETCH ||
10695 (Opcode <= (unsigned)std::numeric_limits<int>::max() &&
10696 Opcode >= ISD::BUILTIN_OP_END && TSI->isTargetMemoryOpcode(Opcode))) &&
10697 "Opcode is not a memory-accessing opcode!");
10698
10699 PointerUnion<MachineMemOperand *, MachineMemOperand **> MemRefs;
10700 if (MMOs.size() == 1) {
10701 MemRefs = MMOs[0];
10702 } else {
10703 // Allocate: [size_t count][MMO*][MMO*]...
10704 size_t AllocSize =
10705 sizeof(size_t) + MMOs.size() * sizeof(MachineMemOperand *);
10706 void *Buffer = Allocator.Allocate(Size: AllocSize, Alignment: alignof(size_t));
10707 size_t *CountPtr = static_cast<size_t *>(Buffer);
10708 *CountPtr = MMOs.size();
10709 MachineMemOperand **Array =
10710 reinterpret_cast<MachineMemOperand **>(CountPtr + 1);
10711 llvm::copy(Range&: MMOs, Out: Array);
10712 MemRefs = Array;
10713 }
10714
10715 // Memoize the node unless it returns a glue result.
10716 MemIntrinsicSDNode *N;
10717 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
10718 SDNodeKey ID(Opcode, VTList, Ops);
10719 ID.AddInteger(I: getSyntheticNodeSubclassData<MemIntrinsicSDNode>(
10720 Opc: Opcode, Order: dl.getIROrder(), VTs: VTList, MemoryVT: MemVT, MemRefs));
10721 ID.AddInteger(I: MemVT.getRawBits());
10722 for (const MachineMemOperand *MMO : MMOs) {
10723 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10724 ID.AddInteger(I: MMO->getFlags());
10725 }
10726 FoldingSetInsertToken InsertToken;
10727 if (SDNode *E = lookupNode(Key: ID, DL: dl, InsertToken)) {
10728 cast<MemIntrinsicSDNode>(Val: E)->refineAlignment(NewMMOs: MMOs);
10729 return SDValue(E, 0);
10730 }
10731
10732 N = newSDNode<MemIntrinsicSDNode>(Args&: Opcode, Args: dl.getIROrder(), Args: dl.getDebugLoc(),
10733 Args&: VTList, Args&: MemVT, Args&: MemRefs);
10734 createOperands(Node: N, Vals: Ops);
10735 CSEMap.insert(N, Token: InsertToken);
10736 } else {
10737 N = newSDNode<MemIntrinsicSDNode>(Args&: Opcode, Args: dl.getIROrder(), Args: dl.getDebugLoc(),
10738 Args&: VTList, Args&: MemVT, Args&: MemRefs);
10739 createOperands(Node: N, Vals: Ops);
10740 }
10741 InsertNode(N);
10742 SDValue V(N, 0);
10743 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10744 return V;
10745}
10746
10747SDValue SelectionDAG::getLifetimeNode(bool IsStart, const SDLoc &dl,
10748 SDValue Chain, int FrameIndex) {
10749 const unsigned Opcode = IsStart ? ISD::LIFETIME_START : ISD::LIFETIME_END;
10750 const auto VTs = getVTList(VT: MVT::Other);
10751 SDValue Ops[2] = {
10752 Chain,
10753 getFrameIndex(FI: FrameIndex,
10754 VT: getTargetLoweringInfo().getFrameIndexTy(DL: getDataLayout()),
10755 isTarget: true)};
10756
10757 SDNodeKey ID(Opcode, VTs, Ops);
10758 FoldingSetInsertToken InsertToken;
10759 if (SDNode *E = lookupNode(Key: ID, DL: dl, InsertToken))
10760 return SDValue(E, 0);
10761
10762 LifetimeSDNode *N =
10763 newSDNode<LifetimeSDNode>(Args: Opcode, Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args: VTs);
10764 createOperands(Node: N, Vals: Ops);
10765 CSEMap.insert(N, Token: InsertToken);
10766 InsertNode(N);
10767 SDValue V(N, 0);
10768 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10769 return V;
10770}
10771
10772SDValue SelectionDAG::getPseudoProbeNode(const SDLoc &Dl, SDValue Chain,
10773 uint64_t Guid, uint64_t Index,
10774 uint32_t Attr) {
10775 const unsigned Opcode = ISD::PSEUDO_PROBE;
10776 const auto VTs = getVTList(VT: MVT::Other);
10777 SDValue Ops[] = {Chain};
10778 SDNodeKey ID(Opcode, VTs, Ops);
10779 ID.AddInteger(I: Guid);
10780 ID.AddInteger(I: Index);
10781 ID.AddInteger(I: Attr);
10782 FoldingSetInsertToken InsertToken;
10783 if (SDNode *E = lookupNode(Key: ID, DL: Dl, InsertToken))
10784 return SDValue(E, 0);
10785
10786 auto *N = newSDNode<PseudoProbeSDNode>(
10787 Args: Opcode, Args: Dl.getIROrder(), Args: Dl.getDebugLoc(), Args: VTs, Args&: Guid, Args&: Index, Args&: Attr);
10788 createOperands(Node: N, Vals: Ops);
10789 CSEMap.insert(N, Token: InsertToken);
10790 InsertNode(N);
10791 SDValue V(N, 0);
10792 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10793 return V;
10794}
10795
10796/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
10797/// MachinePointerInfo record from it. This is particularly useful because the
10798/// code generator has many cases where it doesn't bother passing in a
10799/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
10800static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info,
10801 SelectionDAG &DAG, SDValue Ptr,
10802 int64_t Offset = 0) {
10803 // If this is FI+Offset, we can model it.
10804 if (const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Ptr))
10805 return MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(),
10806 FI: FI->getIndex(), Offset);
10807
10808 // If this is (FI+Offset1)+Offset2, we can model it.
10809 if (Ptr.getOpcode() != ISD::ADD ||
10810 !isa<ConstantSDNode>(Val: Ptr.getOperand(i: 1)) ||
10811 !isa<FrameIndexSDNode>(Val: Ptr.getOperand(i: 0)))
10812 return Info;
10813
10814 int FI = cast<FrameIndexSDNode>(Val: Ptr.getOperand(i: 0))->getIndex();
10815 return MachinePointerInfo::getFixedStack(
10816 MF&: DAG.getMachineFunction(), FI,
10817 Offset: Offset + cast<ConstantSDNode>(Val: Ptr.getOperand(i: 1))->getSExtValue());
10818}
10819
10820/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
10821/// MachinePointerInfo record from it. This is particularly useful because the
10822/// code generator has many cases where it doesn't bother passing in a
10823/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
10824static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info,
10825 SelectionDAG &DAG, SDValue Ptr,
10826 SDValue OffsetOp) {
10827 // If the 'Offset' value isn't a constant, we can't handle this.
10828 if (ConstantSDNode *OffsetNode = dyn_cast<ConstantSDNode>(Val&: OffsetOp))
10829 return InferPointerInfo(Info, DAG, Ptr, Offset: OffsetNode->getSExtValue());
10830 if (OffsetOp.isUndef())
10831 return InferPointerInfo(Info, DAG, Ptr);
10832 return Info;
10833}
10834
10835SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
10836 EVT VT, const SDLoc &dl, SDValue Chain,
10837 SDValue Ptr, SDValue Offset,
10838 MachinePointerInfo PtrInfo, EVT MemVT,
10839 Align Alignment,
10840 MachineMemOperand::Flags MMOFlags,
10841 const MMOMetadata &Metadata) {
10842 assert(Chain.getValueType() == MVT::Other &&
10843 "Invalid chain type");
10844
10845 MMOFlags |= MachineMemOperand::MOLoad;
10846 assert((MMOFlags & MachineMemOperand::MOStore) == 0);
10847 // If we don't have a PtrInfo, infer the trivial frame index case to simplify
10848 // clients.
10849 if (PtrInfo.V.isNull())
10850 PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr, OffsetOp: Offset);
10851
10852 TypeSize Size = MemVT.getStoreSize();
10853 MachineFunction &MF = getMachineFunction();
10854 MachineMemOperand *MMO =
10855 MF.getMachineMemOperand(PtrInfo, F: MMOFlags, Size, BaseAlignment: Alignment, Metadata);
10856 return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO);
10857}
10858
10859SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
10860 EVT VT, const SDLoc &dl, SDValue Chain,
10861 SDValue Ptr, SDValue Offset, EVT MemVT,
10862 MachineMemOperand *MMO) {
10863 if (VT == MemVT) {
10864 ExtType = ISD::NON_EXTLOAD;
10865 } else if (ExtType == ISD::NON_EXTLOAD) {
10866 assert(VT == MemVT && "Non-extending load from different memory type!");
10867 } else {
10868 // Extending load.
10869 assert(MemVT.getScalarType().bitsLT(VT.getScalarType()) &&
10870 "Should only be an extending load, not truncating!");
10871 assert(VT.isInteger() == MemVT.isInteger() &&
10872 "Cannot convert from FP to Int or Int -> FP!");
10873 assert(VT.isVector() == MemVT.isVector() &&
10874 "Cannot use an ext load to convert to or from a vector!");
10875 assert((!VT.isVector() ||
10876 VT.getVectorElementCount() == MemVT.getVectorElementCount()) &&
10877 "Cannot use an ext load to change the number of vector elements!");
10878 }
10879
10880 assert((!MMO->getRanges() ||
10881 (mdconst::extract<ConstantInt>(MMO->getRanges()->getOperand(0))
10882 ->getBitWidth() == MemVT.getScalarSizeInBits() &&
10883 MemVT.isInteger())) &&
10884 "Range metadata and load type must match!");
10885
10886 bool Indexed = AM != ISD::UNINDEXED;
10887 assert((Indexed || Offset.getOpcode() == ISD::POISON) &&
10888 "Unindexed load with an offset!");
10889
10890 SDVTList VTs = Indexed ?
10891 getVTList(VT1: VT, VT2: Ptr.getValueType(), VT3: MVT::Other) : getVTList(VT1: VT, VT2: MVT::Other);
10892 SDValue Ops[] = { Chain, Ptr, Offset };
10893 SDNodeKey ID(ISD::LOAD, VTs, Ops);
10894 ID.AddInteger(I: MemVT.getRawBits());
10895 ID.AddInteger(I: getSyntheticNodeSubclassData<LoadSDNode>(
10896 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: ExtType, Args&: MemVT, Args&: MMO));
10897 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10898 ID.AddInteger(I: MMO->getFlags());
10899 FoldingSetInsertToken InsertToken;
10900 if (auto *E = cast_or_null<LoadSDNode>(Val: lookupNode(Key: ID, DL: dl, InsertToken))) {
10901 E->refineAlignment(NewMMO: MMO);
10902 E->refineMMOMetadata(NewMMO: MMO);
10903 return SDValue(E, 0);
10904 }
10905 auto *N = newSDNode<LoadSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM,
10906 Args&: ExtType, Args&: MemVT, Args&: MMO);
10907 createOperands(Node: N, Vals: Ops);
10908
10909 CSEMap.insert(N, Token: InsertToken);
10910 InsertNode(N);
10911 SDValue V(N, 0);
10912 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10913 return V;
10914}
10915
10916SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain,
10917 SDValue Ptr, MachinePointerInfo PtrInfo,
10918 MaybeAlign Alignment,
10919 MachineMemOperand::Flags MMOFlags,
10920 const MMOMetadata &Metadata) {
10921 SDValue Undef = getPOISON(VT: Ptr.getValueType());
10922 return getLoad(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Offset: Undef,
10923 PtrInfo, MemVT: VT, Alignment, MMOFlags, Metadata);
10924}
10925
10926SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain,
10927 SDValue Ptr, MachineMemOperand *MMO) {
10928 SDValue Undef = getPOISON(VT: Ptr.getValueType());
10929 return getLoad(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Offset: Undef,
10930 MemVT: VT, MMO);
10931}
10932
10933SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
10934 EVT VT, SDValue Chain, SDValue Ptr,
10935 MachinePointerInfo PtrInfo, EVT MemVT,
10936 MaybeAlign Alignment,
10937 MachineMemOperand::Flags MMOFlags,
10938 const MMOMetadata &Metadata) {
10939 SDValue Undef = getPOISON(VT: Ptr.getValueType());
10940 return getLoad(AM: ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Offset: Undef, PtrInfo,
10941 MemVT, Alignment, MMOFlags, Metadata);
10942}
10943
10944SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
10945 EVT VT, SDValue Chain, SDValue Ptr, EVT MemVT,
10946 MachineMemOperand *MMO) {
10947 SDValue Undef = getPOISON(VT: Ptr.getValueType());
10948 return getLoad(AM: ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Offset: Undef,
10949 MemVT, MMO);
10950}
10951
10952SDValue SelectionDAG::getIndexedLoad(SDValue OrigLoad, const SDLoc &dl,
10953 SDValue Base, SDValue Offset,
10954 ISD::MemIndexedMode AM) {
10955 LoadSDNode *LD = cast<LoadSDNode>(Val&: OrigLoad);
10956 assert(LD->getOffset().getOpcode() == ISD::POISON &&
10957 "Load is already a indexed load!");
10958 // Don't propagate the invariant or dereferenceable flags.
10959 auto MMOFlags =
10960 LD->getMemOperand()->getFlags() &
10961 ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
10962 return getLoad(
10963 AM, ExtType: LD->getExtensionType(), VT: OrigLoad.getValueType(), dl, Chain: LD->getChain(),
10964 Ptr: Base, Offset, PtrInfo: LD->getPointerInfo(), MemVT: LD->getMemoryVT(), Alignment: LD->getAlign(),
10965 MMOFlags,
10966 Metadata: MMOMetadata(LD->getAAInfo(), LD->getRanges(), LD->getMemCacheHint()));
10967}
10968
10969SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
10970 SDValue Ptr, MachinePointerInfo PtrInfo,
10971 Align Alignment,
10972 MachineMemOperand::Flags MMOFlags,
10973 const MMOMetadata &Metadata) {
10974 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
10975
10976 MMOFlags |= MachineMemOperand::MOStore;
10977 assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
10978 assert(!Metadata.Ranges && "range metadata is invalid for stores");
10979
10980 if (PtrInfo.V.isNull())
10981 PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr);
10982
10983 MachineFunction &MF = getMachineFunction();
10984 TypeSize Size = Val.getValueType().getStoreSize();
10985 MachineMemOperand *MMO =
10986 MF.getMachineMemOperand(PtrInfo, F: MMOFlags, Size, BaseAlignment: Alignment, Metadata);
10987 return getStore(Chain, dl, Val, Ptr, MMO);
10988}
10989
10990SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
10991 SDValue Ptr, MachineMemOperand *MMO) {
10992 SDValue Undef = getPOISON(VT: Ptr.getValueType());
10993 return getStore(Chain, dl, Val, Ptr, Offset: Undef, SVT: Val.getValueType(), MMO,
10994 AM: ISD::UNINDEXED);
10995}
10996
10997SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
10998 SDValue Ptr, SDValue Offset, EVT SVT,
10999 MachineMemOperand *MMO, ISD::MemIndexedMode AM,
11000 bool IsTruncating) {
11001 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
11002 EVT VT = Val.getValueType();
11003 if (VT == SVT) {
11004 IsTruncating = false;
11005 } else if (!IsTruncating) {
11006 assert(VT == SVT && "No-truncating store from different memory type!");
11007 } else {
11008 assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&
11009 "Should only be a truncating store, not extending!");
11010 assert(VT.isInteger() == SVT.isInteger() && "Can't do FP-INT conversion!");
11011 assert(VT.isVector() == SVT.isVector() &&
11012 "Cannot use trunc store to convert to or from a vector!");
11013 assert((!VT.isVector() ||
11014 VT.getVectorElementCount() == SVT.getVectorElementCount()) &&
11015 "Cannot use trunc store to change the number of vector elements!");
11016 }
11017
11018 bool Indexed = AM != ISD::UNINDEXED;
11019 assert((Indexed || Offset.getOpcode() == ISD::POISON) &&
11020 "Unindexed store with an offset!");
11021 SDVTList VTs = Indexed ? getVTList(VT1: Ptr.getValueType(), VT2: MVT::Other)
11022 : getVTList(VT: MVT::Other);
11023 SDValue Ops[] = {Chain, Val, Ptr, Offset};
11024 SDNodeKey ID(ISD::STORE, VTs, Ops);
11025 ID.AddInteger(I: SVT.getRawBits());
11026 ID.AddInteger(I: getSyntheticNodeSubclassData<StoreSDNode>(
11027 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: IsTruncating, Args&: SVT, Args&: MMO));
11028 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11029 ID.AddInteger(I: MMO->getFlags());
11030 FoldingSetInsertToken InsertToken;
11031 if (SDNode *E = lookupNode(Key: ID, DL: dl, InsertToken)) {
11032 cast<StoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11033 cast<StoreSDNode>(Val: E)->refineMMOMetadata(NewMMO: MMO);
11034 return SDValue(E, 0);
11035 }
11036 auto *N = newSDNode<StoreSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM,
11037 Args&: IsTruncating, Args&: SVT, Args&: MMO);
11038 createOperands(Node: N, Vals: Ops);
11039
11040 CSEMap.insert(N, Token: InsertToken);
11041 InsertNode(N);
11042 SDValue V(N, 0);
11043 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11044 return V;
11045}
11046
11047SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
11048 SDValue Ptr, SDValue Offset,
11049 MachinePointerInfo PtrInfo, EVT SVT,
11050 Align Alignment,
11051 MachineMemOperand::Flags MMOFlags,
11052 const MMOMetadata &Metadata) {
11053 assert(Chain.getValueType() == MVT::Other &&
11054 "Invalid chain type");
11055
11056 MMOFlags |= MachineMemOperand::MOStore;
11057 assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
11058 assert(!Metadata.Ranges && "range metadata is invalid for stores");
11059
11060 if (PtrInfo.V.isNull())
11061 PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr);
11062
11063 MachineFunction &MF = getMachineFunction();
11064 MachineMemOperand *MMO = MF.getMachineMemOperand(
11065 PtrInfo, F: MMOFlags, Size: SVT.getStoreSize(), BaseAlignment: Alignment, Metadata);
11066 return getTruncStore(Chain, dl, Val, Ptr, Offset, SVT, MMO);
11067}
11068
11069SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
11070 SDValue Ptr, MachinePointerInfo PtrInfo,
11071 EVT SVT, Align Alignment,
11072 MachineMemOperand::Flags MMOFlags,
11073 const MMOMetadata &Metadata) {
11074 return getTruncStore(Chain, dl, Val, Ptr, Offset: getPOISON(VT: Ptr.getValueType()),
11075 PtrInfo, SVT, Alignment, MMOFlags, Metadata);
11076}
11077
11078SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
11079 SDValue Ptr, SDValue Offset, EVT SVT,
11080 MachineMemOperand *MMO) {
11081 return getStore(Chain, dl, Val, Ptr, Offset, SVT, MMO, AM: ISD::UNINDEXED, IsTruncating: true);
11082}
11083
11084SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
11085 SDValue Ptr, EVT SVT,
11086 MachineMemOperand *MMO) {
11087 return getStore(Chain, dl, Val, Ptr, Offset: getPOISON(VT: Ptr.getValueType()), SVT, MMO,
11088 AM: ISD::UNINDEXED, IsTruncating: true);
11089}
11090
11091SDValue SelectionDAG::getIndexedStore(SDValue OrigStore, const SDLoc &dl,
11092 SDValue Base, SDValue Offset,
11093 ISD::MemIndexedMode AM) {
11094 StoreSDNode *ST = cast<StoreSDNode>(Val&: OrigStore);
11095 assert(ST->getOffset().getOpcode() == ISD::POISON &&
11096 "Store is already a indexed store!");
11097 return getStore(Chain: ST->getChain(), dl, Val: ST->getValue(), Ptr: Base, Offset,
11098 SVT: ST->getMemoryVT(), MMO: ST->getMemOperand(), AM,
11099 IsTruncating: ST->isTruncatingStore());
11100}
11101
11102SDValue SelectionDAG::getLoadVP(
11103 ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &dl,
11104 SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL,
11105 MachinePointerInfo PtrInfo, EVT MemVT, Align Alignment,
11106 MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo,
11107 const MDNode *Ranges, bool IsExpanding) {
11108 MMOFlags |= MachineMemOperand::MOLoad;
11109 assert((MMOFlags & MachineMemOperand::MOStore) == 0);
11110 // If we don't have a PtrInfo, infer the trivial frame index case to simplify
11111 // clients.
11112 if (PtrInfo.V.isNull())
11113 PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr, OffsetOp: Offset);
11114
11115 TypeSize Size = MemVT.getStoreSize();
11116 MachineFunction &MF = getMachineFunction();
11117 MachineMemOperand *MMO = MF.getMachineMemOperand(
11118 PtrInfo, F: MMOFlags, Size, BaseAlignment: Alignment, Metadata: MMOMetadata(AAInfo, Ranges));
11119 return getLoadVP(AM, ExtType, VT, dl, Chain, Ptr, Offset, Mask, EVL, MemVT,
11120 MMO, IsExpanding);
11121}
11122
11123SDValue SelectionDAG::getLoadVP(ISD::MemIndexedMode AM,
11124 ISD::LoadExtType ExtType, EVT VT,
11125 const SDLoc &dl, SDValue Chain, SDValue Ptr,
11126 SDValue Offset, SDValue Mask, SDValue EVL,
11127 EVT MemVT, MachineMemOperand *MMO,
11128 bool IsExpanding) {
11129 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
11130 assert(Mask.getValueType().getVectorElementCount() ==
11131 VT.getVectorElementCount() &&
11132 "Vector width mismatch between mask and data");
11133
11134 bool Indexed = AM != ISD::UNINDEXED;
11135 assert((Indexed || Offset.getOpcode() == ISD::POISON) &&
11136 "Unindexed load with an offset!");
11137
11138 SDVTList VTs = Indexed ? getVTList(VT1: VT, VT2: Ptr.getValueType(), VT3: MVT::Other)
11139 : getVTList(VT1: VT, VT2: MVT::Other);
11140 SDValue Ops[] = {Chain, Ptr, Offset, Mask, EVL};
11141 SDNodeKey ID(ISD::VP_LOAD, VTs, Ops);
11142 ID.AddInteger(I: MemVT.getRawBits());
11143 ID.AddInteger(I: getSyntheticNodeSubclassData<VPLoadSDNode>(
11144 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: ExtType, Args&: IsExpanding, Args&: MemVT, Args&: MMO));
11145 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11146 ID.AddInteger(I: MMO->getFlags());
11147 FoldingSetInsertToken InsertToken;
11148 if (auto *E = cast_or_null<VPLoadSDNode>(Val: lookupNode(Key: ID, DL: dl, InsertToken))) {
11149 E->refineAlignment(NewMMO: MMO);
11150 E->refineMMOMetadata(NewMMO: MMO);
11151 return SDValue(E, 0);
11152 }
11153 auto *N = newSDNode<VPLoadSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM,
11154 Args&: ExtType, Args&: IsExpanding, Args&: MemVT, Args&: MMO);
11155 createOperands(Node: N, Vals: Ops);
11156
11157 CSEMap.insert(N, Token: InsertToken);
11158 InsertNode(N);
11159 SDValue V(N, 0);
11160 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11161 return V;
11162}
11163
11164SDValue SelectionDAG::getLoadVP(EVT VT, const SDLoc &dl, SDValue Chain,
11165 SDValue Ptr, SDValue Mask, SDValue EVL,
11166 MachinePointerInfo PtrInfo,
11167 MaybeAlign Alignment,
11168 MachineMemOperand::Flags MMOFlags,
11169 const AAMDNodes &AAInfo, const MDNode *Ranges,
11170 bool IsExpanding) {
11171 SDValue Undef = getPOISON(VT: Ptr.getValueType());
11172 return getLoadVP(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Offset: Undef,
11173 Mask, EVL, PtrInfo, MemVT: VT, Alignment, MMOFlags, AAInfo, Ranges,
11174 IsExpanding);
11175}
11176
11177SDValue SelectionDAG::getLoadVP(EVT VT, const SDLoc &dl, SDValue Chain,
11178 SDValue Ptr, SDValue Mask, SDValue EVL,
11179 MachineMemOperand *MMO, bool IsExpanding) {
11180 SDValue Undef = getPOISON(VT: Ptr.getValueType());
11181 return getLoadVP(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Offset: Undef,
11182 Mask, EVL, MemVT: VT, MMO, IsExpanding);
11183}
11184
11185SDValue SelectionDAG::getExtLoadVP(ISD::LoadExtType ExtType, const SDLoc &dl,
11186 EVT VT, SDValue Chain, SDValue Ptr,
11187 SDValue Mask, SDValue EVL,
11188 MachinePointerInfo PtrInfo, EVT MemVT,
11189 MaybeAlign Alignment,
11190 MachineMemOperand::Flags MMOFlags,
11191 const AAMDNodes &AAInfo, bool IsExpanding) {
11192 SDValue Undef = getPOISON(VT: Ptr.getValueType());
11193 return getLoadVP(AM: ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Offset: Undef, Mask,
11194 EVL, PtrInfo, MemVT, Alignment, MMOFlags, AAInfo, Ranges: nullptr,
11195 IsExpanding);
11196}
11197
11198SDValue SelectionDAG::getExtLoadVP(ISD::LoadExtType ExtType, const SDLoc &dl,
11199 EVT VT, SDValue Chain, SDValue Ptr,
11200 SDValue Mask, SDValue EVL, EVT MemVT,
11201 MachineMemOperand *MMO, bool IsExpanding) {
11202 SDValue Undef = getPOISON(VT: Ptr.getValueType());
11203 return getLoadVP(AM: ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Offset: Undef, Mask,
11204 EVL, MemVT, MMO, IsExpanding);
11205}
11206
11207SDValue SelectionDAG::getStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val,
11208 SDValue Ptr, SDValue Offset, SDValue Mask,
11209 SDValue EVL, EVT MemVT, MachineMemOperand *MMO,
11210 ISD::MemIndexedMode AM, bool IsTruncating,
11211 bool IsCompressing) {
11212 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
11213 assert(Mask.getValueType().getVectorElementCount() ==
11214 Val.getValueType().getVectorElementCount() &&
11215 "Vector width mismatch between mask and data");
11216
11217 bool Indexed = AM != ISD::UNINDEXED;
11218 assert((Indexed || Offset.getOpcode() == ISD::POISON) &&
11219 "Unindexed vp_store with an offset!");
11220 SDVTList VTs = Indexed ? getVTList(VT1: Ptr.getValueType(), VT2: MVT::Other)
11221 : getVTList(VT: MVT::Other);
11222 SDValue Ops[] = {Chain, Val, Ptr, Offset, Mask, EVL};
11223 SDNodeKey ID(ISD::VP_STORE, VTs, Ops);
11224 ID.AddInteger(I: MemVT.getRawBits());
11225 ID.AddInteger(I: getSyntheticNodeSubclassData<VPStoreSDNode>(
11226 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO));
11227 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11228 ID.AddInteger(I: MMO->getFlags());
11229 FoldingSetInsertToken InsertToken;
11230 if (SDNode *E = lookupNode(Key: ID, DL: dl, InsertToken)) {
11231 cast<VPStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11232 return SDValue(E, 0);
11233 }
11234 auto *N = newSDNode<VPStoreSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM,
11235 Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO);
11236 createOperands(Node: N, Vals: Ops);
11237
11238 CSEMap.insert(N, Token: InsertToken);
11239 InsertNode(N);
11240 SDValue V(N, 0);
11241 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11242 return V;
11243}
11244
11245SDValue SelectionDAG::getTruncStoreVP(SDValue Chain, const SDLoc &dl,
11246 SDValue Val, SDValue Ptr, SDValue Mask,
11247 SDValue EVL, MachinePointerInfo PtrInfo,
11248 EVT SVT, Align Alignment,
11249 MachineMemOperand::Flags MMOFlags,
11250 const AAMDNodes &AAInfo,
11251 bool IsCompressing) {
11252 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
11253
11254 MMOFlags |= MachineMemOperand::MOStore;
11255 assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
11256
11257 if (PtrInfo.V.isNull())
11258 PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr);
11259
11260 MachineFunction &MF = getMachineFunction();
11261 MachineMemOperand *MMO = MF.getMachineMemOperand(
11262 PtrInfo, F: MMOFlags, Size: SVT.getStoreSize(), BaseAlignment: Alignment, Metadata: AAInfo);
11263 return getTruncStoreVP(Chain, dl, Val, Ptr, Mask, EVL, SVT, MMO,
11264 IsCompressing);
11265}
11266
11267SDValue SelectionDAG::getTruncStoreVP(SDValue Chain, const SDLoc &dl,
11268 SDValue Val, SDValue Ptr, SDValue Mask,
11269 SDValue EVL, EVT SVT,
11270 MachineMemOperand *MMO,
11271 bool IsCompressing) {
11272 EVT VT = Val.getValueType();
11273
11274 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
11275 if (VT == SVT)
11276 return getStoreVP(Chain, dl, Val, Ptr, Offset: getPOISON(VT: Ptr.getValueType()), Mask,
11277 EVL, MemVT: VT, MMO, AM: ISD::UNINDEXED,
11278 /*IsTruncating*/ false, IsCompressing);
11279
11280 assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&
11281 "Should only be a truncating store, not extending!");
11282 assert(VT.isInteger() == SVT.isInteger() && "Can't do FP-INT conversion!");
11283 assert(VT.isVector() == SVT.isVector() &&
11284 "Cannot use trunc store to convert to or from a vector!");
11285 assert((!VT.isVector() ||
11286 VT.getVectorElementCount() == SVT.getVectorElementCount()) &&
11287 "Cannot use trunc store to change the number of vector elements!");
11288
11289 SDVTList VTs = getVTList(VT: MVT::Other);
11290 SDValue Undef = getPOISON(VT: Ptr.getValueType());
11291 SDValue Ops[] = {Chain, Val, Ptr, Undef, Mask, EVL};
11292 SDNodeKey ID(ISD::VP_STORE, VTs, Ops);
11293 ID.AddInteger(I: SVT.getRawBits());
11294 ID.AddInteger(I: getSyntheticNodeSubclassData<VPStoreSDNode>(
11295 IROrder: dl.getIROrder(), Args&: VTs, Args: ISD::UNINDEXED, Args: true, Args&: IsCompressing, Args&: SVT, Args&: MMO));
11296 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11297 ID.AddInteger(I: MMO->getFlags());
11298 FoldingSetInsertToken InsertToken;
11299 if (SDNode *E = lookupNode(Key: ID, DL: dl, InsertToken)) {
11300 cast<VPStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11301 return SDValue(E, 0);
11302 }
11303 auto *N =
11304 newSDNode<VPStoreSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs,
11305 Args: ISD::UNINDEXED, Args: true, Args&: IsCompressing, Args&: SVT, Args&: MMO);
11306 createOperands(Node: N, Vals: Ops);
11307
11308 CSEMap.insert(N, Token: InsertToken);
11309 InsertNode(N);
11310 SDValue V(N, 0);
11311 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11312 return V;
11313}
11314
11315SDValue SelectionDAG::getStridedLoadVP(
11316 ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &DL,
11317 SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask,
11318 SDValue EVL, EVT MemVT, MachineMemOperand *MMO, bool IsExpanding) {
11319 bool Indexed = AM != ISD::UNINDEXED;
11320 assert((Indexed || Offset.getOpcode() == ISD::POISON) &&
11321 "Unindexed load with an offset!");
11322
11323 SDValue Ops[] = {Chain, Ptr, Offset, Stride, Mask, EVL};
11324 SDVTList VTs = Indexed ? getVTList(VT1: VT, VT2: Ptr.getValueType(), VT3: MVT::Other)
11325 : getVTList(VT1: VT, VT2: MVT::Other);
11326 SDNodeKey ID(ISD::EXPERIMENTAL_VP_STRIDED_LOAD, VTs, Ops);
11327 ID.AddInteger(I: MemVT.getRawBits());
11328 ID.AddInteger(I: getSyntheticNodeSubclassData<VPStridedLoadSDNode>(
11329 IROrder: DL.getIROrder(), Args&: VTs, Args&: AM, Args&: ExtType, Args&: IsExpanding, Args&: MemVT, Args&: MMO));
11330 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11331
11332 FoldingSetInsertToken InsertToken;
11333 if (SDNode *E = lookupNode(Key: ID, DL, InsertToken)) {
11334 cast<VPStridedLoadSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11335 return SDValue(E, 0);
11336 }
11337
11338 auto *N =
11339 newSDNode<VPStridedLoadSDNode>(Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs, Args&: AM,
11340 Args&: ExtType, Args&: IsExpanding, Args&: MemVT, Args&: MMO);
11341 createOperands(Node: N, Vals: Ops);
11342 CSEMap.insert(N, Token: InsertToken);
11343 InsertNode(N);
11344 SDValue V(N, 0);
11345 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11346 return V;
11347}
11348
11349SDValue SelectionDAG::getStridedLoadVP(EVT VT, const SDLoc &DL, SDValue Chain,
11350 SDValue Ptr, SDValue Stride,
11351 SDValue Mask, SDValue EVL,
11352 MachineMemOperand *MMO,
11353 bool IsExpanding) {
11354 SDValue Undef = getPOISON(VT: Ptr.getValueType());
11355 return getStridedLoadVP(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, DL, Chain, Ptr,
11356 Offset: Undef, Stride, Mask, EVL, MemVT: VT, MMO, IsExpanding);
11357}
11358
11359SDValue SelectionDAG::getExtStridedLoadVP(
11360 ISD::LoadExtType ExtType, const SDLoc &DL, EVT VT, SDValue Chain,
11361 SDValue Ptr, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT,
11362 MachineMemOperand *MMO, bool IsExpanding) {
11363 SDValue Undef = getPOISON(VT: Ptr.getValueType());
11364 return getStridedLoadVP(AM: ISD::UNINDEXED, ExtType, VT, DL, Chain, Ptr, Offset: Undef,
11365 Stride, Mask, EVL, MemVT, MMO, IsExpanding);
11366}
11367
11368SDValue SelectionDAG::getStridedStoreVP(SDValue Chain, const SDLoc &DL,
11369 SDValue Val, SDValue Ptr,
11370 SDValue Offset, SDValue Stride,
11371 SDValue Mask, SDValue EVL, EVT MemVT,
11372 MachineMemOperand *MMO,
11373 ISD::MemIndexedMode AM,
11374 bool IsTruncating, bool IsCompressing) {
11375 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
11376 bool Indexed = AM != ISD::UNINDEXED;
11377 assert((Indexed || Offset.getOpcode() == ISD::POISON) &&
11378 "Unindexed vp_store with an offset!");
11379 SDVTList VTs = Indexed ? getVTList(VT1: Ptr.getValueType(), VT2: MVT::Other)
11380 : getVTList(VT: MVT::Other);
11381 SDValue Ops[] = {Chain, Val, Ptr, Offset, Stride, Mask, EVL};
11382 SDNodeKey ID(ISD::EXPERIMENTAL_VP_STRIDED_STORE, VTs, Ops);
11383 ID.AddInteger(I: MemVT.getRawBits());
11384 ID.AddInteger(I: getSyntheticNodeSubclassData<VPStridedStoreSDNode>(
11385 IROrder: DL.getIROrder(), Args&: VTs, Args&: AM, Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO));
11386 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11387 FoldingSetInsertToken InsertToken;
11388 if (SDNode *E = lookupNode(Key: ID, DL, InsertToken)) {
11389 cast<VPStridedStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11390 return SDValue(E, 0);
11391 }
11392 auto *N = newSDNode<VPStridedStoreSDNode>(Args: DL.getIROrder(), Args: DL.getDebugLoc(),
11393 Args&: VTs, Args&: AM, Args&: IsTruncating,
11394 Args&: IsCompressing, Args&: MemVT, Args&: MMO);
11395 createOperands(Node: N, Vals: Ops);
11396
11397 CSEMap.insert(N, Token: InsertToken);
11398 InsertNode(N);
11399 SDValue V(N, 0);
11400 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11401 return V;
11402}
11403
11404SDValue SelectionDAG::getGatherVP(SDVTList VTs, EVT VT, const SDLoc &dl,
11405 ArrayRef<SDValue> Ops, MachineMemOperand *MMO,
11406 ISD::MemIndexType IndexType) {
11407 assert(Ops.size() == 6 && "Incompatible number of operands");
11408
11409 SDNodeKey ID(ISD::VP_GATHER, VTs, Ops);
11410 ID.AddInteger(I: VT.getRawBits());
11411 ID.AddInteger(I: getSyntheticNodeSubclassData<VPGatherSDNode>(
11412 IROrder: dl.getIROrder(), Args&: VTs, Args&: VT, Args&: MMO, Args&: IndexType));
11413 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11414 ID.AddInteger(I: MMO->getFlags());
11415 FoldingSetInsertToken InsertToken;
11416 if (SDNode *E = lookupNode(Key: ID, DL: dl, InsertToken)) {
11417 cast<VPGatherSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11418 return SDValue(E, 0);
11419 }
11420
11421 auto *N = newSDNode<VPGatherSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs,
11422 Args&: VT, Args&: MMO, Args&: IndexType);
11423 createOperands(Node: N, Vals: Ops);
11424
11425 assert(N->getMask().getValueType().getVectorElementCount() ==
11426 N->getValueType(0).getVectorElementCount() &&
11427 "Vector width mismatch between mask and data");
11428 assert(N->getIndex().getValueType().getVectorElementCount().isScalable() ==
11429 N->getValueType(0).getVectorElementCount().isScalable() &&
11430 "Scalable flags of index and data do not match");
11431 assert(ElementCount::isKnownGE(
11432 N->getIndex().getValueType().getVectorElementCount(),
11433 N->getValueType(0).getVectorElementCount()) &&
11434 "Vector width mismatch between index and data");
11435 assert(isa<ConstantSDNode>(N->getScale()) &&
11436 N->getScale()->getAsAPIntVal().isPowerOf2() &&
11437 "Scale should be a constant power of 2");
11438
11439 CSEMap.insert(N, Token: InsertToken);
11440 InsertNode(N);
11441 SDValue V(N, 0);
11442 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11443 return V;
11444}
11445
11446SDValue SelectionDAG::getScatterVP(SDVTList VTs, EVT VT, const SDLoc &dl,
11447 ArrayRef<SDValue> Ops,
11448 MachineMemOperand *MMO,
11449 ISD::MemIndexType IndexType) {
11450 assert(Ops.size() == 7 && "Incompatible number of operands");
11451
11452 SDNodeKey ID(ISD::VP_SCATTER, VTs, Ops);
11453 ID.AddInteger(I: VT.getRawBits());
11454 ID.AddInteger(I: getSyntheticNodeSubclassData<VPScatterSDNode>(
11455 IROrder: dl.getIROrder(), Args&: VTs, Args&: VT, Args&: MMO, Args&: IndexType));
11456 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11457 ID.AddInteger(I: MMO->getFlags());
11458 FoldingSetInsertToken InsertToken;
11459 if (SDNode *E = lookupNode(Key: ID, DL: dl, InsertToken)) {
11460 cast<VPScatterSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11461 return SDValue(E, 0);
11462 }
11463 auto *N = newSDNode<VPScatterSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs,
11464 Args&: VT, Args&: MMO, Args&: IndexType);
11465 createOperands(Node: N, Vals: Ops);
11466
11467 assert(N->getMask().getValueType().getVectorElementCount() ==
11468 N->getValue().getValueType().getVectorElementCount() &&
11469 "Vector width mismatch between mask and data");
11470 assert(
11471 N->getIndex().getValueType().getVectorElementCount().isScalable() ==
11472 N->getValue().getValueType().getVectorElementCount().isScalable() &&
11473 "Scalable flags of index and data do not match");
11474 assert(ElementCount::isKnownGE(
11475 N->getIndex().getValueType().getVectorElementCount(),
11476 N->getValue().getValueType().getVectorElementCount()) &&
11477 "Vector width mismatch between index and data");
11478 assert(isa<ConstantSDNode>(N->getScale()) &&
11479 N->getScale()->getAsAPIntVal().isPowerOf2() &&
11480 "Scale should be a constant power of 2");
11481
11482 CSEMap.insert(N, Token: InsertToken);
11483 InsertNode(N);
11484 SDValue V(N, 0);
11485 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11486 return V;
11487}
11488
11489SDValue SelectionDAG::getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain,
11490 SDValue Base, SDValue Offset, SDValue Mask,
11491 SDValue PassThru, EVT MemVT,
11492 MachineMemOperand *MMO,
11493 ISD::MemIndexedMode AM,
11494 ISD::LoadExtType ExtTy, bool isExpanding) {
11495 bool Indexed = AM != ISD::UNINDEXED;
11496 assert((Indexed || Offset.getOpcode() == ISD::POISON) &&
11497 "Unindexed masked load with an offset!");
11498 SDVTList VTs = Indexed ? getVTList(VT1: VT, VT2: Base.getValueType(), VT3: MVT::Other)
11499 : getVTList(VT1: VT, VT2: MVT::Other);
11500 SDValue Ops[] = {Chain, Base, Offset, Mask, PassThru};
11501 SDNodeKey ID(ISD::MLOAD, VTs, Ops);
11502 ID.AddInteger(I: MemVT.getRawBits());
11503 ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedLoadSDNode>(
11504 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: ExtTy, Args&: isExpanding, Args&: MemVT, Args&: MMO));
11505 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11506 ID.AddInteger(I: MMO->getFlags());
11507 FoldingSetInsertToken InsertToken;
11508 if (SDNode *E = lookupNode(Key: ID, DL: dl, InsertToken)) {
11509 cast<MaskedLoadSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11510 return SDValue(E, 0);
11511 }
11512 auto *N = newSDNode<MaskedLoadSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs,
11513 Args&: AM, Args&: ExtTy, Args&: isExpanding, Args&: MemVT, Args&: MMO);
11514 createOperands(Node: N, Vals: Ops);
11515
11516 CSEMap.insert(N, Token: InsertToken);
11517 InsertNode(N);
11518 SDValue V(N, 0);
11519 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11520 return V;
11521}
11522
11523SDValue SelectionDAG::getIndexedMaskedLoad(SDValue OrigLoad, const SDLoc &dl,
11524 SDValue Base, SDValue Offset,
11525 ISD::MemIndexedMode AM) {
11526 MaskedLoadSDNode *LD = cast<MaskedLoadSDNode>(Val&: OrigLoad);
11527 assert(LD->getOffset().getOpcode() == ISD::POISON &&
11528 "Masked load is already a indexed load!");
11529 return getMaskedLoad(VT: OrigLoad.getValueType(), dl, Chain: LD->getChain(), Base,
11530 Offset, Mask: LD->getMask(), PassThru: LD->getPassThru(),
11531 MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand(), AM,
11532 ExtTy: LD->getExtensionType(), isExpanding: LD->isExpandingLoad());
11533}
11534
11535SDValue SelectionDAG::getMaskedStore(SDValue Chain, const SDLoc &dl,
11536 SDValue Val, SDValue Base, SDValue Offset,
11537 SDValue Mask, EVT MemVT,
11538 MachineMemOperand *MMO,
11539 ISD::MemIndexedMode AM, bool IsTruncating,
11540 bool IsCompressing) {
11541 assert(Chain.getValueType() == MVT::Other &&
11542 "Invalid chain type");
11543 bool Indexed = AM != ISD::UNINDEXED;
11544 assert((Indexed || Offset.getOpcode() == ISD::POISON) &&
11545 "Unindexed masked store with an offset!");
11546 SDVTList VTs = Indexed ? getVTList(VT1: Base.getValueType(), VT2: MVT::Other)
11547 : getVTList(VT: MVT::Other);
11548 SDValue Ops[] = {Chain, Val, Base, Offset, Mask};
11549 SDNodeKey ID(ISD::MSTORE, VTs, Ops);
11550 ID.AddInteger(I: MemVT.getRawBits());
11551 ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedStoreSDNode>(
11552 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO));
11553 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11554 ID.AddInteger(I: MMO->getFlags());
11555 FoldingSetInsertToken InsertToken;
11556 if (SDNode *E = lookupNode(Key: ID, DL: dl, InsertToken)) {
11557 cast<MaskedStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11558 return SDValue(E, 0);
11559 }
11560 auto *N =
11561 newSDNode<MaskedStoreSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM,
11562 Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO);
11563 createOperands(Node: N, Vals: Ops);
11564
11565 CSEMap.insert(N, Token: InsertToken);
11566 InsertNode(N);
11567 SDValue V(N, 0);
11568 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11569 return V;
11570}
11571
11572SDValue SelectionDAG::getIndexedMaskedStore(SDValue OrigStore, const SDLoc &dl,
11573 SDValue Base, SDValue Offset,
11574 ISD::MemIndexedMode AM) {
11575 MaskedStoreSDNode *ST = cast<MaskedStoreSDNode>(Val&: OrigStore);
11576 assert(ST->getOffset().getOpcode() == ISD::POISON &&
11577 "Masked store is already a indexed store!");
11578 return getMaskedStore(Chain: ST->getChain(), dl, Val: ST->getValue(), Base, Offset,
11579 Mask: ST->getMask(), MemVT: ST->getMemoryVT(), MMO: ST->getMemOperand(),
11580 AM, IsTruncating: ST->isTruncatingStore(), IsCompressing: ST->isCompressingStore());
11581}
11582
11583SDValue SelectionDAG::getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl,
11584 ArrayRef<SDValue> Ops,
11585 MachineMemOperand *MMO,
11586 ISD::MemIndexType IndexType,
11587 ISD::LoadExtType ExtTy) {
11588 assert(Ops.size() == 6 && "Incompatible number of operands");
11589
11590 SDNodeKey ID(ISD::MGATHER, VTs, Ops);
11591 ID.AddInteger(I: MemVT.getRawBits());
11592 ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedGatherSDNode>(
11593 IROrder: dl.getIROrder(), Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType, Args&: ExtTy));
11594 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11595 ID.AddInteger(I: MMO->getFlags());
11596 FoldingSetInsertToken InsertToken;
11597 if (SDNode *E = lookupNode(Key: ID, DL: dl, InsertToken)) {
11598 cast<MaskedGatherSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11599 return SDValue(E, 0);
11600 }
11601
11602 auto *N = newSDNode<MaskedGatherSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(),
11603 Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType, Args&: ExtTy);
11604 createOperands(Node: N, Vals: Ops);
11605
11606 assert(N->getPassThru().getValueType() == N->getValueType(0) &&
11607 "Incompatible type of the PassThru value in MaskedGatherSDNode");
11608 assert(N->getMask().getValueType().getVectorElementCount() ==
11609 N->getValueType(0).getVectorElementCount() &&
11610 "Vector width mismatch between mask and data");
11611 assert(N->getIndex().getValueType().getVectorElementCount().isScalable() ==
11612 N->getValueType(0).getVectorElementCount().isScalable() &&
11613 "Scalable flags of index and data do not match");
11614 assert(ElementCount::isKnownGE(
11615 N->getIndex().getValueType().getVectorElementCount(),
11616 N->getValueType(0).getVectorElementCount()) &&
11617 "Vector width mismatch between index and data");
11618 assert(isa<ConstantSDNode>(N->getScale()) &&
11619 N->getScale()->getAsAPIntVal().isPowerOf2() &&
11620 "Scale should be a constant power of 2");
11621
11622 CSEMap.insert(N, Token: InsertToken);
11623 InsertNode(N);
11624 SDValue V(N, 0);
11625 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11626 return V;
11627}
11628
11629SDValue SelectionDAG::getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl,
11630 ArrayRef<SDValue> Ops,
11631 MachineMemOperand *MMO,
11632 ISD::MemIndexType IndexType,
11633 bool IsTrunc) {
11634 assert(Ops.size() == 6 && "Incompatible number of operands");
11635
11636 SDNodeKey ID(ISD::MSCATTER, VTs, Ops);
11637 ID.AddInteger(I: MemVT.getRawBits());
11638 ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedScatterSDNode>(
11639 IROrder: dl.getIROrder(), Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType, Args&: IsTrunc));
11640 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11641 ID.AddInteger(I: MMO->getFlags());
11642 FoldingSetInsertToken InsertToken;
11643 if (SDNode *E = lookupNode(Key: ID, DL: dl, InsertToken)) {
11644 cast<MaskedScatterSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11645 return SDValue(E, 0);
11646 }
11647
11648 auto *N = newSDNode<MaskedScatterSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(),
11649 Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType, Args&: IsTrunc);
11650 createOperands(Node: N, Vals: Ops);
11651
11652 assert(N->getMask().getValueType().getVectorElementCount() ==
11653 N->getValue().getValueType().getVectorElementCount() &&
11654 "Vector width mismatch between mask and data");
11655 assert(
11656 N->getIndex().getValueType().getVectorElementCount().isScalable() ==
11657 N->getValue().getValueType().getVectorElementCount().isScalable() &&
11658 "Scalable flags of index and data do not match");
11659 assert(ElementCount::isKnownGE(
11660 N->getIndex().getValueType().getVectorElementCount(),
11661 N->getValue().getValueType().getVectorElementCount()) &&
11662 "Vector width mismatch between index and data");
11663 assert(isa<ConstantSDNode>(N->getScale()) &&
11664 N->getScale()->getAsAPIntVal().isPowerOf2() &&
11665 "Scale should be a constant power of 2");
11666
11667 CSEMap.insert(N, Token: InsertToken);
11668 InsertNode(N);
11669 SDValue V(N, 0);
11670 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11671 return V;
11672}
11673
11674SDValue SelectionDAG::getMaskedHistogram(SDVTList VTs, EVT MemVT,
11675 const SDLoc &dl, ArrayRef<SDValue> Ops,
11676 MachineMemOperand *MMO,
11677 ISD::MemIndexType IndexType) {
11678 assert(Ops.size() == 7 && "Incompatible number of operands");
11679
11680 SDNodeKey ID(ISD::EXPERIMENTAL_VECTOR_HISTOGRAM, VTs, Ops);
11681 ID.AddInteger(I: MemVT.getRawBits());
11682 ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedHistogramSDNode>(
11683 IROrder: dl.getIROrder(), Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType));
11684 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11685 ID.AddInteger(I: MMO->getFlags());
11686 FoldingSetInsertToken InsertToken;
11687 if (SDNode *E = lookupNode(Key: ID, DL: dl, InsertToken)) {
11688 cast<MaskedGatherSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11689 return SDValue(E, 0);
11690 }
11691
11692 auto *N = newSDNode<MaskedHistogramSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(),
11693 Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType);
11694 createOperands(Node: N, Vals: Ops);
11695
11696 assert(N->getMask().getValueType().getVectorElementCount() ==
11697 N->getIndex().getValueType().getVectorElementCount() &&
11698 "Vector width mismatch between mask and data");
11699 assert(isa<ConstantSDNode>(N->getScale()) &&
11700 N->getScale()->getAsAPIntVal().isPowerOf2() &&
11701 "Scale should be a constant power of 2");
11702 assert(N->getInc().getValueType().isInteger() && "Non integer update value");
11703
11704 CSEMap.insert(N, Token: InsertToken);
11705 InsertNode(N);
11706 SDValue V(N, 0);
11707 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11708 return V;
11709}
11710
11711SDValue SelectionDAG::getLoadFFVP(EVT VT, const SDLoc &DL, SDValue Chain,
11712 SDValue Ptr, SDValue Mask, SDValue EVL,
11713 MachineMemOperand *MMO) {
11714 SDVTList VTs = getVTList(VT1: VT, VT2: EVL.getValueType(), VT3: MVT::Other);
11715 SDValue Ops[] = {Chain, Ptr, Mask, EVL};
11716 SDNodeKey ID(ISD::VP_LOAD_FF, VTs, Ops);
11717 ID.AddInteger(I: VT.getRawBits());
11718 ID.AddInteger(I: getSyntheticNodeSubclassData<VPLoadFFSDNode>(IROrder: DL.getIROrder(),
11719 Args&: VTs, Args&: VT, Args&: MMO));
11720 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11721 ID.AddInteger(I: MMO->getFlags());
11722 FoldingSetInsertToken InsertToken;
11723 if (SDNode *E = lookupNode(Key: ID, DL, InsertToken)) {
11724 cast<VPLoadFFSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11725 return SDValue(E, 0);
11726 }
11727 auto *N = newSDNode<VPLoadFFSDNode>(Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs,
11728 Args&: VT, Args&: MMO);
11729 createOperands(Node: N, Vals: Ops);
11730
11731 CSEMap.insert(N, Token: InsertToken);
11732 InsertNode(N);
11733 SDValue V(N, 0);
11734 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11735 return V;
11736}
11737
11738SDValue SelectionDAG::getGetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr,
11739 EVT MemVT, MachineMemOperand *MMO) {
11740 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
11741 SDVTList VTs = getVTList(VT: MVT::Other);
11742 SDValue Ops[] = {Chain, Ptr};
11743 SDNodeKey ID(ISD::GET_FPENV_MEM, VTs, Ops);
11744 ID.AddInteger(I: MemVT.getRawBits());
11745 ID.AddInteger(I: getSyntheticNodeSubclassData<FPStateAccessSDNode>(
11746 Opc: ISD::GET_FPENV_MEM, Order: dl.getIROrder(), VTs, MemoryVT: MemVT, MMO));
11747 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11748 ID.AddInteger(I: MMO->getFlags());
11749 FoldingSetInsertToken InsertToken;
11750 if (SDNode *E = lookupNode(Key: ID, DL: dl, InsertToken))
11751 return SDValue(E, 0);
11752
11753 auto *N = newSDNode<FPStateAccessSDNode>(Args: ISD::GET_FPENV_MEM, Args: dl.getIROrder(),
11754 Args: dl.getDebugLoc(), Args&: VTs, Args&: MemVT, Args&: MMO);
11755 createOperands(Node: N, Vals: Ops);
11756
11757 CSEMap.insert(N, Token: InsertToken);
11758 InsertNode(N);
11759 SDValue V(N, 0);
11760 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11761 return V;
11762}
11763
11764SDValue SelectionDAG::getSetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr,
11765 EVT MemVT, MachineMemOperand *MMO) {
11766 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
11767 SDVTList VTs = getVTList(VT: MVT::Other);
11768 SDValue Ops[] = {Chain, Ptr};
11769 SDNodeKey ID(ISD::SET_FPENV_MEM, VTs, Ops);
11770 ID.AddInteger(I: MemVT.getRawBits());
11771 ID.AddInteger(I: getSyntheticNodeSubclassData<FPStateAccessSDNode>(
11772 Opc: ISD::SET_FPENV_MEM, Order: dl.getIROrder(), VTs, MemoryVT: MemVT, MMO));
11773 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11774 ID.AddInteger(I: MMO->getFlags());
11775 FoldingSetInsertToken InsertToken;
11776 if (SDNode *E = lookupNode(Key: ID, DL: dl, InsertToken))
11777 return SDValue(E, 0);
11778
11779 auto *N = newSDNode<FPStateAccessSDNode>(Args: ISD::SET_FPENV_MEM, Args: dl.getIROrder(),
11780 Args: dl.getDebugLoc(), Args&: VTs, Args&: MemVT, Args&: MMO);
11781 createOperands(Node: N, Vals: Ops);
11782
11783 CSEMap.insert(N, Token: InsertToken);
11784 InsertNode(N);
11785 SDValue V(N, 0);
11786 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11787 return V;
11788}
11789
11790SDValue SelectionDAG::simplifySelect(SDValue Cond, SDValue T, SDValue F) {
11791 // select undef, T, F --> T (if T is a constant), otherwise F
11792 // select, ?, undef, F --> F
11793 // select, ?, T, undef --> T
11794 if (Cond.isUndef())
11795 return isConstantValueOfAnyType(N: T) ? T : F;
11796 if (T.isUndef())
11797 return isGuaranteedNotToBePoison(Op: F) ? F : getFreeze(V: F);
11798 if (F.isUndef())
11799 return isGuaranteedNotToBePoison(Op: T) ? T : getFreeze(V: T);
11800
11801 // select true, T, F --> T
11802 // select false, T, F --> F
11803 if (auto C = isBoolConstant(N: Cond))
11804 return *C ? T : F;
11805
11806 // select ?, T, T --> T
11807 if (T == F)
11808 return T;
11809
11810 return SDValue();
11811}
11812
11813SDValue SelectionDAG::simplifyShift(SDValue X, SDValue Y) {
11814 // shift undef, Y --> 0 (can always assume that the undef value is 0)
11815 if (X.isUndef())
11816 return getConstant(Val: 0, DL: SDLoc(X.getNode()), VT: X.getValueType());
11817 // shift X, undef --> undef (because it may shift by the bitwidth)
11818 if (Y.isUndef())
11819 return getUNDEF(VT: X.getValueType());
11820
11821 // shift 0, Y --> 0
11822 // shift X, 0 --> X
11823 if (isNullOrNullSplat(V: X) || isNullOrNullSplat(V: Y))
11824 return X;
11825
11826 // shift X, C >= bitwidth(X) --> undef
11827 // All vector elements must be too big (or undef) to avoid partial undefs.
11828 auto isShiftTooBig = [X](ConstantSDNode *Val) {
11829 return !Val || Val->getAPIntValue().uge(RHS: X.getScalarValueSizeInBits());
11830 };
11831 if (ISD::matchUnaryPredicate(Op: Y, Match: isShiftTooBig, AllowUndefs: true))
11832 return getUNDEF(VT: X.getValueType());
11833
11834 // shift i1/vXi1 X, Y --> X (any non-zero shift amount is undefined).
11835 if (X.getValueType().getScalarType() == MVT::i1)
11836 return X;
11837
11838 return SDValue();
11839}
11840
11841SDValue SelectionDAG::simplifyFPBinop(unsigned Opcode, SDValue X, SDValue Y,
11842 SDNodeFlags Flags) {
11843 // If this operation has 'nnan' or 'ninf' and at least 1 disallowed operand
11844 // (an undef operand can be chosen to be Nan/Inf), then the result of this
11845 // operation is poison. That result can be relaxed to undef.
11846 ConstantFPSDNode *XC = isConstOrConstSplatFP(N: X, /* AllowUndefs */ true);
11847 ConstantFPSDNode *YC = isConstOrConstSplatFP(N: Y, /* AllowUndefs */ true);
11848 bool HasNan = (XC && XC->getValueAPF().isNaN()) ||
11849 (YC && YC->getValueAPF().isNaN());
11850 bool HasInf = (XC && XC->getValueAPF().isInfinity()) ||
11851 (YC && YC->getValueAPF().isInfinity());
11852
11853 if (Flags.hasNoNaNs() && (HasNan || X.isUndef() || Y.isUndef()))
11854 return getUNDEF(VT: X.getValueType());
11855
11856 if (Flags.hasNoInfs() && (HasInf || X.isUndef() || Y.isUndef()))
11857 return getUNDEF(VT: X.getValueType());
11858
11859 if (!YC)
11860 return SDValue();
11861
11862 // X + -0.0 --> X
11863 if (Opcode == ISD::FADD)
11864 if (YC->getValueAPF().isNegZero())
11865 return X;
11866
11867 // X - +0.0 --> X
11868 if (Opcode == ISD::FSUB)
11869 if (YC->getValueAPF().isPosZero())
11870 return X;
11871
11872 // X * 1.0 --> X
11873 // X / 1.0 --> X
11874 if (Opcode == ISD::FMUL || Opcode == ISD::FDIV)
11875 if (YC->getValueAPF().isOne())
11876 return X;
11877
11878 // X * 0.0 --> 0.0
11879 if (Opcode == ISD::FMUL && Flags.hasNoNaNs() && Flags.hasNoSignedZeros())
11880 if (YC->getValueAPF().isZero())
11881 return getConstantFP(Val: 0.0, DL: SDLoc(Y), VT: Y.getValueType());
11882
11883 return SDValue();
11884}
11885
11886SDValue SelectionDAG::getVAArg(EVT VT, const SDLoc &dl, SDValue Chain,
11887 SDValue Ptr, SDValue SV, unsigned Align) {
11888 SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Val: Align, DL: dl, VT: MVT::i32) };
11889 return getNode(Opcode: ISD::VAARG, DL: dl, VTList: getVTList(VT1: VT, VT2: MVT::Other), Ops);
11890}
11891
11892SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
11893 ArrayRef<SDUse> Ops) {
11894 switch (Ops.size()) {
11895 case 0: return getNode(Opcode, DL, VT);
11896 case 1: return getNode(Opcode, DL, VT, N1: Ops[0].get());
11897 case 2: return getNode(Opcode, DL, VT, N1: Ops[0], N2: Ops[1]);
11898 case 3: return getNode(Opcode, DL, VT, N1: Ops[0], N2: Ops[1], N3: Ops[2]);
11899 default: break;
11900 }
11901
11902 // Copy from an SDUse array into an SDValue array for use with
11903 // the regular getNode logic.
11904 SmallVector<SDValue, 8> NewOps(Ops);
11905 return getNode(Opcode, DL, VT, Ops: NewOps);
11906}
11907
11908SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
11909 ArrayRef<SDValue> Ops) {
11910 SDNodeFlags Flags;
11911 if (Inserter)
11912 Flags = Inserter->getFlags();
11913 return getNode(Opcode, DL, VT, Ops, Flags);
11914}
11915
11916SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
11917 ArrayRef<SDValue> Ops, const SDNodeFlags Flags) {
11918 unsigned NumOps = Ops.size();
11919 switch (NumOps) {
11920 case 0: return getNode(Opcode, DL, VT);
11921 case 1: return getNode(Opcode, DL, VT, N1: Ops[0], Flags);
11922 case 2: return getNode(Opcode, DL, VT, N1: Ops[0], N2: Ops[1], Flags);
11923 case 3: return getNode(Opcode, DL, VT, N1: Ops[0], N2: Ops[1], N3: Ops[2], Flags);
11924 default: break;
11925 }
11926
11927#ifndef NDEBUG
11928 for (const auto &Op : Ops)
11929 assert(Op.getOpcode() != ISD::DELETED_NODE &&
11930 "Operand is DELETED_NODE!");
11931#endif
11932
11933 switch (Opcode) {
11934 default: break;
11935 case ISD::BUILD_VECTOR:
11936 // Attempt to simplify BUILD_VECTOR.
11937 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, DAG&: *this))
11938 return V;
11939 break;
11940 case ISD::CONCAT_VECTORS:
11941 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, DAG&: *this))
11942 return V;
11943 break;
11944 case ISD::SELECT_CC:
11945 assert(NumOps == 5 && "SELECT_CC takes 5 operands!");
11946 assert(Ops[0].getValueType() == Ops[1].getValueType() &&
11947 "LHS and RHS of condition must have same type!");
11948 assert(Ops[2].getValueType() == Ops[3].getValueType() &&
11949 "True and False arms of SelectCC must have same type!");
11950 assert(Ops[2].getValueType() == VT &&
11951 "select_cc node must be of same type as true and false value!");
11952 assert((!Ops[0].getValueType().isVector() ||
11953 Ops[0].getValueType().getVectorElementCount() ==
11954 VT.getVectorElementCount()) &&
11955 "Expected select_cc with vector result to have the same sized "
11956 "comparison type!");
11957 break;
11958 case ISD::BR_CC:
11959 assert(NumOps == 5 && "BR_CC takes 5 operands!");
11960 assert(Ops[2].getValueType() == Ops[3].getValueType() &&
11961 "LHS/RHS of comparison should match types!");
11962 break;
11963 case ISD::VP_REDUCE_MUL:
11964 // If it is VP_REDUCE_MUL mask operation then turn it to VP_REDUCE_AND
11965 if (VT == MVT::i1)
11966 Opcode = ISD::VP_REDUCE_AND;
11967 break;
11968 case ISD::VP_REDUCE_ADD:
11969 // If it is VP_REDUCE_ADD mask operation then turn it to VP_REDUCE_XOR
11970 if (VT == MVT::i1)
11971 Opcode = ISD::VP_REDUCE_XOR;
11972 break;
11973 case ISD::VP_REDUCE_SMAX:
11974 case ISD::VP_REDUCE_UMIN:
11975 // If it is VP_REDUCE_SMAX/VP_REDUCE_UMIN mask operation then turn it to
11976 // VP_REDUCE_AND.
11977 if (VT == MVT::i1)
11978 Opcode = ISD::VP_REDUCE_AND;
11979 break;
11980 case ISD::VP_REDUCE_SMIN:
11981 case ISD::VP_REDUCE_UMAX:
11982 // If it is VP_REDUCE_SMIN/VP_REDUCE_UMAX mask operation then turn it to
11983 // VP_REDUCE_OR.
11984 if (VT == MVT::i1)
11985 Opcode = ISD::VP_REDUCE_OR;
11986 break;
11987 }
11988
11989 // Memoize nodes.
11990 SDNode *N;
11991 SDVTList VTs = getVTList(VT);
11992
11993 if (VT != MVT::Glue) {
11994 SDNodeKey ID(Opcode, VTs, Ops);
11995 FoldingSetInsertToken InsertToken;
11996
11997 if (SDNode *E = lookupNode(Key: ID, DL, InsertToken)) {
11998 E->intersectFlagsWith(Flags);
11999 return SDValue(E, 0);
12000 }
12001
12002 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
12003 createOperands(Node: N, Vals: Ops);
12004
12005 CSEMap.insert(N, Token: InsertToken);
12006 } else {
12007 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
12008 createOperands(Node: N, Vals: Ops);
12009 }
12010
12011 N->setFlags(Flags);
12012 InsertNode(N);
12013 SDValue V(N, 0);
12014 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
12015 return V;
12016}
12017
12018SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
12019 ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops) {
12020 SDNodeFlags Flags;
12021 if (Inserter)
12022 Flags = Inserter->getFlags();
12023 return getNode(Opcode, DL, VTList: getVTList(VTs: ResultTys), Ops, Flags);
12024}
12025
12026SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
12027 ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops,
12028 const SDNodeFlags Flags) {
12029 return getNode(Opcode, DL, VTList: getVTList(VTs: ResultTys), Ops, Flags);
12030}
12031
12032SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
12033 ArrayRef<SDValue> Ops) {
12034 SDNodeFlags Flags;
12035 if (Inserter)
12036 Flags = Inserter->getFlags();
12037 return getNode(Opcode, DL, VTList, Ops, Flags);
12038}
12039
12040SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
12041 ArrayRef<SDValue> Ops, const SDNodeFlags Flags) {
12042 if (VTList.NumVTs == 1)
12043 return getNode(Opcode, DL, VT: VTList.VTs[0], Ops, Flags);
12044
12045#ifndef NDEBUG
12046 for (const auto &Op : Ops)
12047 assert(Op.getOpcode() != ISD::DELETED_NODE &&
12048 "Operand is DELETED_NODE!");
12049#endif
12050
12051 switch (Opcode) {
12052 case ISD::SADDO:
12053 case ISD::UADDO:
12054 case ISD::SSUBO:
12055 case ISD::USUBO: {
12056 assert(VTList.NumVTs == 2 && Ops.size() == 2 &&
12057 "Invalid add/sub overflow op!");
12058 assert(VTList.VTs[0].isInteger() && VTList.VTs[1].isInteger() &&
12059 Ops[0].getValueType() == Ops[1].getValueType() &&
12060 Ops[0].getValueType() == VTList.VTs[0] &&
12061 "Binary operator types must match!");
12062 SDValue N1 = Ops[0], N2 = Ops[1];
12063 canonicalizeCommutativeBinop(Opcode, N1, N2);
12064
12065 // (X +- 0) -> X with zero-overflow.
12066 ConstantSDNode *N2CV = isConstOrConstSplat(N: N2, /*AllowUndefs*/ false,
12067 /*AllowTruncation*/ true);
12068 if (N2CV && N2CV->isZero()) {
12069 SDValue ZeroOverFlow = getConstant(Val: 0, DL, VT: VTList.VTs[1]);
12070 return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList, Ops: {N1, ZeroOverFlow}, Flags);
12071 }
12072
12073 if (VTList.VTs[0].getScalarType() == MVT::i1 &&
12074 VTList.VTs[1].getScalarType() == MVT::i1) {
12075 SDValue F1 = getFreeze(V: N1);
12076 SDValue F2 = getFreeze(V: N2);
12077 // {vXi1,vXi1} (u/s)addo(vXi1 x, vXi1y) -> {xor(x,y),and(x,y)}
12078 if (Opcode == ISD::UADDO || Opcode == ISD::SADDO)
12079 return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList,
12080 Ops: {getNode(Opcode: ISD::XOR, DL, VT: VTList.VTs[0], N1: F1, N2: F2),
12081 getNode(Opcode: ISD::AND, DL, VT: VTList.VTs[1], N1: F1, N2: F2)},
12082 Flags);
12083 // {vXi1,vXi1} (u/s)subo(vXi1 x, vXi1y) -> {xor(x,y),and(~x,y)}
12084 if (Opcode == ISD::USUBO || Opcode == ISD::SSUBO) {
12085 SDValue NotF1 = getNOT(DL, Val: F1, VT: VTList.VTs[0]);
12086 return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList,
12087 Ops: {getNode(Opcode: ISD::XOR, DL, VT: VTList.VTs[0], N1: F1, N2: F2),
12088 getNode(Opcode: ISD::AND, DL, VT: VTList.VTs[1], N1: NotF1, N2: F2)},
12089 Flags);
12090 }
12091 }
12092 break;
12093 }
12094 case ISD::SADDO_CARRY:
12095 case ISD::UADDO_CARRY:
12096 case ISD::SSUBO_CARRY:
12097 case ISD::USUBO_CARRY:
12098 assert(VTList.NumVTs == 2 && Ops.size() == 3 &&
12099 "Invalid add/sub overflow op!");
12100 assert(VTList.VTs[0].isInteger() && VTList.VTs[1].isInteger() &&
12101 Ops[0].getValueType() == Ops[1].getValueType() &&
12102 Ops[0].getValueType() == VTList.VTs[0] &&
12103 Ops[2].getValueType() == VTList.VTs[1] &&
12104 "Binary operator types must match!");
12105 break;
12106 case ISD::SMUL_LOHI:
12107 case ISD::UMUL_LOHI: {
12108 assert(VTList.NumVTs == 2 && Ops.size() == 2 && "Invalid mul lo/hi op!");
12109 assert(VTList.VTs[0].isInteger() && VTList.VTs[0] == VTList.VTs[1] &&
12110 VTList.VTs[0] == Ops[0].getValueType() &&
12111 VTList.VTs[0] == Ops[1].getValueType() &&
12112 "Binary operator types must match!");
12113 // Constant fold.
12114 ConstantSDNode *LHS = dyn_cast<ConstantSDNode>(Val: Ops[0]);
12115 ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Val: Ops[1]);
12116 if (LHS && RHS) {
12117 unsigned Width = VTList.VTs[0].getScalarSizeInBits();
12118 unsigned OutWidth = Width * 2;
12119 APInt Val = LHS->getAPIntValue();
12120 APInt Mul = RHS->getAPIntValue();
12121 if (Opcode == ISD::SMUL_LOHI) {
12122 Val = Val.sext(width: OutWidth);
12123 Mul = Mul.sext(width: OutWidth);
12124 } else {
12125 Val = Val.zext(width: OutWidth);
12126 Mul = Mul.zext(width: OutWidth);
12127 }
12128 Val *= Mul;
12129
12130 SDValue Hi =
12131 getConstant(Val: Val.extractBits(numBits: Width, bitPosition: Width), DL, VT: VTList.VTs[0]);
12132 SDValue Lo = getConstant(Val: Val.trunc(width: Width), DL, VT: VTList.VTs[0]);
12133 return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList, Ops: {Lo, Hi}, Flags);
12134 }
12135 break;
12136 }
12137 case ISD::FFREXP: {
12138 assert(VTList.NumVTs == 2 && Ops.size() == 1 && "Invalid ffrexp op!");
12139 assert(VTList.VTs[0].isFloatingPoint() && VTList.VTs[1].isInteger() &&
12140 VTList.VTs[0] == Ops[0].getValueType() && "frexp type mismatch");
12141
12142 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val: Ops[0])) {
12143 int FrexpExp;
12144 APFloat FrexpMant =
12145 frexp(X: C->getValueAPF(), Exp&: FrexpExp, RM: APFloat::rmNearestTiesToEven);
12146 SDValue Result0 = getConstantFP(V: FrexpMant, DL, VT: VTList.VTs[0]);
12147 SDValue Result1 = getSignedConstant(Val: FrexpMant.isFinite() ? FrexpExp : 0,
12148 DL, VT: VTList.VTs[1]);
12149 return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList, Ops: {Result0, Result1}, Flags);
12150 }
12151
12152 break;
12153 }
12154 case ISD::STRICT_FP_EXTEND:
12155 assert(VTList.NumVTs == 2 && Ops.size() == 2 &&
12156 "Invalid STRICT_FP_EXTEND!");
12157 assert(VTList.VTs[0].isFloatingPoint() &&
12158 Ops[1].getValueType().isFloatingPoint() && "Invalid FP cast!");
12159 assert(VTList.VTs[0].isVector() == Ops[1].getValueType().isVector() &&
12160 "STRICT_FP_EXTEND result type should be vector iff the operand "
12161 "type is vector!");
12162 assert((!VTList.VTs[0].isVector() ||
12163 VTList.VTs[0].getVectorElementCount() ==
12164 Ops[1].getValueType().getVectorElementCount()) &&
12165 "Vector element count mismatch!");
12166 assert(Ops[1].getValueType().bitsLT(VTList.VTs[0]) &&
12167 "Invalid fpext node, dst <= src!");
12168 break;
12169 case ISD::STRICT_FP_ROUND:
12170 assert(VTList.NumVTs == 2 && Ops.size() == 3 && "Invalid STRICT_FP_ROUND!");
12171 assert(VTList.VTs[0].isVector() == Ops[1].getValueType().isVector() &&
12172 "STRICT_FP_ROUND result type should be vector iff the operand "
12173 "type is vector!");
12174 assert((!VTList.VTs[0].isVector() ||
12175 VTList.VTs[0].getVectorElementCount() ==
12176 Ops[1].getValueType().getVectorElementCount()) &&
12177 "Vector element count mismatch!");
12178 assert(VTList.VTs[0].isFloatingPoint() &&
12179 Ops[1].getValueType().isFloatingPoint() &&
12180 VTList.VTs[0].bitsLT(Ops[1].getValueType()) &&
12181 Ops[2].getOpcode() == ISD::TargetConstant &&
12182 (Ops[2]->getAsZExtVal() == 0 || Ops[2]->getAsZExtVal() == 1) &&
12183 "Invalid STRICT_FP_ROUND!");
12184 break;
12185 }
12186
12187 // Memoize the node unless it returns a glue result.
12188 SDNode *N;
12189 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
12190 SDNodeKey ID(Opcode, VTList, Ops);
12191 FoldingSetInsertToken InsertToken;
12192 if (SDNode *E = lookupNode(Key: ID, DL, InsertToken)) {
12193 E->intersectFlagsWith(Flags);
12194 return SDValue(E, 0);
12195 }
12196
12197 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTList);
12198 createOperands(Node: N, Vals: Ops);
12199 CSEMap.insert(N, Token: InsertToken);
12200 } else {
12201 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTList);
12202 createOperands(Node: N, Vals: Ops);
12203 }
12204
12205 N->setFlags(Flags);
12206 InsertNode(N);
12207 SDValue V(N, 0);
12208 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
12209 return V;
12210}
12211
12212SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
12213 SDVTList VTList) {
12214 return getNode(Opcode, DL, VTList, Ops: ArrayRef<SDValue>());
12215}
12216
12217SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
12218 SDValue N1) {
12219 SDValue Ops[] = { N1 };
12220 return getNode(Opcode, DL, VTList, Ops);
12221}
12222
12223SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
12224 SDValue N1, SDValue N2) {
12225 SDValue Ops[] = { N1, N2 };
12226 return getNode(Opcode, DL, VTList, Ops);
12227}
12228
12229SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
12230 SDValue N1, SDValue N2, SDValue N3) {
12231 SDValue Ops[] = { N1, N2, N3 };
12232 return getNode(Opcode, DL, VTList, Ops);
12233}
12234
12235SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
12236 SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
12237 SDValue Ops[] = { N1, N2, N3, N4 };
12238 return getNode(Opcode, DL, VTList, Ops);
12239}
12240
12241SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
12242 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
12243 SDValue N5) {
12244 SDValue Ops[] = { N1, N2, N3, N4, N5 };
12245 return getNode(Opcode, DL, VTList, Ops);
12246}
12247
12248SDVTList SelectionDAG::getVTList(EVT VT) {
12249 if (!VT.isExtended())
12250 return makeVTList(VTs: SDNode::getValueTypeList(VT: VT.getSimpleVT()), NumVTs: 1);
12251
12252 EVT VTs[] = {VT};
12253 return getVTList(VTs);
12254}
12255
12256SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2) {
12257 EVT VTs[] = {VT1, VT2};
12258 return getVTList(VTs);
12259}
12260
12261SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3) {
12262 EVT VTs[] = {VT1, VT2, VT3};
12263 return getVTList(VTs);
12264}
12265
12266SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4) {
12267 EVT VTs[] = {VT1, VT2, VT3, VT4};
12268 return getVTList(VTs);
12269}
12270
12271SDVTList SelectionDAG::getVTList(ArrayRef<EVT> VTs) {
12272 auto It = VTLists.find(V: VTs);
12273 if (It == VTLists.end()) {
12274 EVT *Array = Allocator.Allocate<EVT>(Num: VTs.size());
12275 llvm::copy(Range&: VTs, Out: Array);
12276 It = VTLists.insert(V: ArrayRef(Array, VTs.size())).first;
12277 }
12278 return makeVTList(VTs: It->data(), NumVTs: It->size());
12279}
12280
12281/// UpdateNodeOperands - *Mutate* the specified node in-place to have the
12282/// specified operands. If the resultant node already exists in the DAG,
12283/// this does not modify the specified node, instead it returns the node that
12284/// already exists. If the resultant node does not exist in the DAG, the
12285/// input node is returned. As a degenerate case, if you specify the same
12286/// input operands as the node already has, the input node is returned.
12287SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op) {
12288 assert(N->getNumOperands() == 1 && "Update with wrong number of operands");
12289
12290 // Check to see if there is no change.
12291 if (Op == N->getOperand(Num: 0)) return N;
12292
12293 // See if the modified node already exists.
12294 FoldingSetInsertToken InsertToken;
12295 if (SDNode *Existing = FindModifiedNodeSlot(N, Op, InsertToken))
12296 return Existing;
12297
12298 // Nope it doesn't. Remove the node from its current place in the maps.
12299 if (InsertToken)
12300 if (!RemoveNodeFromCSEMaps(N))
12301 InsertToken = {};
12302
12303 // Now we update the operands.
12304 N->OperandList[0].set(Op);
12305
12306 updateDivergence(N);
12307 // If this gets put into a CSE map, add it.
12308 if (InsertToken)
12309 CSEMap.insert(N, Token: InsertToken);
12310 return N;
12311}
12312
12313SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2) {
12314 assert(N->getNumOperands() == 2 && "Update with wrong number of operands");
12315
12316 // Check to see if there is no change.
12317 if (Op1 == N->getOperand(Num: 0) && Op2 == N->getOperand(Num: 1))
12318 return N; // No operands changed, just return the input node.
12319
12320 // See if the modified node already exists.
12321 FoldingSetInsertToken InsertToken;
12322 if (SDNode *Existing = FindModifiedNodeSlot(N, Op1, Op2, InsertToken))
12323 return Existing;
12324
12325 // Nope it doesn't. Remove the node from its current place in the maps.
12326 if (InsertToken)
12327 if (!RemoveNodeFromCSEMaps(N))
12328 InsertToken = {};
12329
12330 // Now we update the operands.
12331 if (N->OperandList[0] != Op1)
12332 N->OperandList[0].set(Op1);
12333 if (N->OperandList[1] != Op2)
12334 N->OperandList[1].set(Op2);
12335
12336 updateDivergence(N);
12337 // If this gets put into a CSE map, add it.
12338 if (InsertToken)
12339 CSEMap.insert(N, Token: InsertToken);
12340 return N;
12341}
12342
12343SDNode *SelectionDAG::
12344UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, SDValue Op3) {
12345 SDValue Ops[] = { Op1, Op2, Op3 };
12346 return UpdateNodeOperands(N, Ops);
12347}
12348
12349SDNode *SelectionDAG::
12350UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
12351 SDValue Op3, SDValue Op4) {
12352 SDValue Ops[] = { Op1, Op2, Op3, Op4 };
12353 return UpdateNodeOperands(N, Ops);
12354}
12355
12356SDNode *SelectionDAG::
12357UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
12358 SDValue Op3, SDValue Op4, SDValue Op5) {
12359 SDValue Ops[] = { Op1, Op2, Op3, Op4, Op5 };
12360 return UpdateNodeOperands(N, Ops);
12361}
12362
12363SDNode *SelectionDAG::
12364UpdateNodeOperands(SDNode *N, ArrayRef<SDValue> Ops) {
12365 unsigned NumOps = Ops.size();
12366 assert(N->getNumOperands() == NumOps &&
12367 "Update with wrong number of operands");
12368
12369 // If no operands changed just return the input node.
12370 if (std::equal(first1: Ops.begin(), last1: Ops.end(), first2: N->op_begin()))
12371 return N;
12372
12373 // See if the modified node already exists.
12374 FoldingSetInsertToken InsertToken;
12375 if (SDNode *Existing = FindModifiedNodeSlot(N, Ops, InsertToken))
12376 return Existing;
12377
12378 // Nope it doesn't. Remove the node from its current place in the maps.
12379 if (InsertToken)
12380 if (!RemoveNodeFromCSEMaps(N))
12381 InsertToken = {};
12382
12383 // Now we update the operands.
12384 for (unsigned i = 0; i != NumOps; ++i)
12385 if (N->OperandList[i] != Ops[i])
12386 N->OperandList[i].set(Ops[i]);
12387
12388 updateDivergence(N);
12389 // If this gets put into a CSE map, add it.
12390 if (InsertToken)
12391 CSEMap.insert(N, Token: InsertToken);
12392 return N;
12393}
12394
12395/// DropOperands - Release the operands and set this node to have
12396/// zero operands.
12397void SDNode::DropOperands() {
12398 // Unlike the code in MorphNodeTo that does this, we don't need to
12399 // watch for dead nodes here.
12400 for (op_iterator I = op_begin(), E = op_end(); I != E; ) {
12401 SDUse &Use = *I++;
12402 Use.set(SDValue());
12403 }
12404}
12405
12406void SelectionDAG::setNodeMemRefs(MachineSDNode *N,
12407 ArrayRef<MachineMemOperand *> NewMemRefs) {
12408 if (NewMemRefs.empty()) {
12409 N->clearMemRefs();
12410 return;
12411 }
12412
12413 // Check if we can avoid allocating by storing a single reference directly.
12414 if (NewMemRefs.size() == 1) {
12415 N->MemRefs = NewMemRefs[0];
12416 N->NumMemRefs = 1;
12417 return;
12418 }
12419
12420 MachineMemOperand **MemRefsBuffer =
12421 Allocator.template Allocate<MachineMemOperand *>(Num: NewMemRefs.size());
12422 llvm::copy(Range&: NewMemRefs, Out: MemRefsBuffer);
12423 N->MemRefs = MemRefsBuffer;
12424 N->NumMemRefs = static_cast<int>(NewMemRefs.size());
12425}
12426
12427/// SelectNodeTo - These are wrappers around MorphNodeTo that accept a
12428/// machine opcode.
12429///
12430SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12431 EVT VT) {
12432 SDVTList VTs = getVTList(VT);
12433 return SelectNodeTo(N, MachineOpc, VTs, Ops: {});
12434}
12435
12436SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12437 EVT VT, SDValue Op1) {
12438 SDVTList VTs = getVTList(VT);
12439 SDValue Ops[] = { Op1 };
12440 return SelectNodeTo(N, MachineOpc, VTs, Ops);
12441}
12442
12443SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12444 EVT VT, SDValue Op1,
12445 SDValue Op2) {
12446 SDVTList VTs = getVTList(VT);
12447 SDValue Ops[] = { Op1, Op2 };
12448 return SelectNodeTo(N, MachineOpc, VTs, Ops);
12449}
12450
12451SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12452 EVT VT, SDValue Op1,
12453 SDValue Op2, SDValue Op3) {
12454 SDVTList VTs = getVTList(VT);
12455 SDValue Ops[] = { Op1, Op2, Op3 };
12456 return SelectNodeTo(N, MachineOpc, VTs, Ops);
12457}
12458
12459SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12460 EVT VT, ArrayRef<SDValue> Ops) {
12461 SDVTList VTs = getVTList(VT);
12462 return SelectNodeTo(N, MachineOpc, VTs, Ops);
12463}
12464
12465SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12466 EVT VT1, EVT VT2, ArrayRef<SDValue> Ops) {
12467 SDVTList VTs = getVTList(VT1, VT2);
12468 return SelectNodeTo(N, MachineOpc, VTs, Ops);
12469}
12470
12471SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12472 EVT VT1, EVT VT2) {
12473 SDVTList VTs = getVTList(VT1, VT2);
12474 return SelectNodeTo(N, MachineOpc, VTs, Ops: {});
12475}
12476
12477SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12478 EVT VT1, EVT VT2, EVT VT3,
12479 ArrayRef<SDValue> Ops) {
12480 SDVTList VTs = getVTList(VT1, VT2, VT3);
12481 return SelectNodeTo(N, MachineOpc, VTs, Ops);
12482}
12483
12484SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12485 EVT VT1, EVT VT2,
12486 SDValue Op1, SDValue Op2) {
12487 SDVTList VTs = getVTList(VT1, VT2);
12488 SDValue Ops[] = { Op1, Op2 };
12489 return SelectNodeTo(N, MachineOpc, VTs, Ops);
12490}
12491
12492SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12493 SDVTList VTs,ArrayRef<SDValue> Ops) {
12494 SDNode *New = MorphNodeTo(N, Opc: ~MachineOpc, VTs, Ops);
12495 // Reset the NodeID to -1.
12496 New->setNodeId(-1);
12497 if (New != N) {
12498 ReplaceAllUsesWith(From: N, To: New);
12499 RemoveDeadNode(N);
12500 }
12501 return New;
12502}
12503
12504/// UpdateSDLocOnMergeSDNode - If the opt level is -O0 then it throws away
12505/// the line number information on the merged node since it is not possible to
12506/// preserve the information that operation is associated with multiple lines.
12507/// This will make the debugger working better at -O0, were there is a higher
12508/// probability having other instructions associated with that line.
12509///
12510/// For IROrder, we keep the smaller of the two
12511SDNode *SelectionDAG::UpdateSDLocOnMergeSDNode(SDNode *N, const SDLoc &OLoc) {
12512 DebugLoc NLoc = N->getDebugLoc();
12513 if (NLoc && OptLevel == CodeGenOptLevel::None && OLoc.getDebugLoc() != NLoc) {
12514 N->setDebugLoc(DebugLoc());
12515 }
12516 unsigned Order = std::min(a: N->getIROrder(), b: OLoc.getIROrder());
12517 N->setIROrder(Order);
12518 return N;
12519}
12520
12521/// MorphNodeTo - This *mutates* the specified node to have the specified
12522/// return type, opcode, and operands.
12523///
12524/// Note that MorphNodeTo returns the resultant node. If there is already a
12525/// node of the specified opcode and operands, it returns that node instead of
12526/// the current one. Note that the SDLoc need not be the same.
12527///
12528/// Using MorphNodeTo is faster than creating a new node and swapping it in
12529/// with ReplaceAllUsesWith both because it often avoids allocating a new
12530/// node, and because it doesn't require CSE recalculation for any of
12531/// the node's users.
12532///
12533/// However, note that MorphNodeTo recursively deletes dead nodes from the DAG.
12534/// As a consequence it isn't appropriate to use from within the DAG combiner or
12535/// the legalizer which maintain worklists that would need to be updated when
12536/// deleting things.
12537SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
12538 SDVTList VTs, ArrayRef<SDValue> Ops) {
12539 // If an identical node already exists, use it.
12540 FoldingSetInsertToken InsertToken;
12541 if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) {
12542 SDNodeKey ID(Opc, VTs, Ops);
12543 AddNodeIDCustom(ID&: ID.Tail, N, Opc);
12544 if (SDNode *ON = lookupNode(Key: ID, DL: SDLoc(N), InsertToken))
12545 return UpdateSDLocOnMergeSDNode(N: ON, OLoc: SDLoc(N));
12546 }
12547
12548 if (!RemoveNodeFromCSEMaps(N))
12549 InsertToken = {};
12550
12551 // Start the morphing.
12552 N->NodeType = Opc;
12553 N->ValueList = VTs.VTs;
12554 N->NumValues = VTs.NumVTs;
12555
12556 // Clear the operands list, updating used nodes to remove this from their
12557 // use list. Keep track of any operands that become dead as a result.
12558 SmallPtrSet<SDNode*, 16> DeadNodeSet;
12559 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
12560 SDUse &Use = *I++;
12561 SDNode *Used = Use.getNode();
12562 Use.set(SDValue());
12563 if (Used->use_empty())
12564 DeadNodeSet.insert(Ptr: Used);
12565 }
12566
12567 // For MachineNode, initialize the memory references information.
12568 if (MachineSDNode *MN = dyn_cast<MachineSDNode>(Val: N))
12569 MN->clearMemRefs();
12570
12571 // Swap for an appropriately sized array from the recycler.
12572 removeOperands(Node: N);
12573 createOperands(Node: N, Vals: Ops);
12574
12575 // Delete any nodes that are still dead after adding the uses for the
12576 // new operands.
12577 if (!DeadNodeSet.empty()) {
12578 SmallVector<SDNode *, 16> DeadNodes;
12579 for (SDNode *N : DeadNodeSet)
12580 if (N->use_empty())
12581 DeadNodes.push_back(Elt: N);
12582 RemoveDeadNodes(DeadNodes);
12583 }
12584
12585 if (InsertToken)
12586 CSEMap.insert(N, Token: InsertToken); // Memoize the new node.
12587 return N;
12588}
12589
12590SDNode* SelectionDAG::mutateStrictFPToFP(SDNode *Node) {
12591 unsigned OrigOpc = Node->getOpcode();
12592 unsigned NewOpc;
12593 switch (OrigOpc) {
12594 default:
12595 llvm_unreachable("mutateStrictFPToFP called with unexpected opcode!");
12596#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
12597 case ISD::STRICT_##DAGN: NewOpc = ISD::DAGN; break;
12598#define CMP_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
12599 case ISD::STRICT_##DAGN: NewOpc = ISD::SETCC; break;
12600#include "llvm/IR/ConstrainedOps.def"
12601 }
12602
12603 assert(Node->getNumValues() == 2 && "Unexpected number of results!");
12604
12605 // We're taking this node out of the chain, so we need to re-link things.
12606 SDValue InputChain = Node->getOperand(Num: 0);
12607 SDValue OutputChain = SDValue(Node, 1);
12608 ReplaceAllUsesOfValueWith(From: OutputChain, To: InputChain);
12609
12610 SmallVector<SDValue, 3> Ops;
12611 for (unsigned i = 1, e = Node->getNumOperands(); i != e; ++i)
12612 Ops.push_back(Elt: Node->getOperand(Num: i));
12613
12614 SDVTList VTs = getVTList(VT: Node->getValueType(ResNo: 0));
12615 SDNode *Res = MorphNodeTo(N: Node, Opc: NewOpc, VTs, Ops);
12616
12617 // MorphNodeTo can operate in two ways: if an existing node with the
12618 // specified operands exists, it can just return it. Otherwise, it
12619 // updates the node in place to have the requested operands.
12620 if (Res == Node) {
12621 // If we updated the node in place, reset the node ID. To the isel,
12622 // this should be just like a newly allocated machine node.
12623 Res->setNodeId(-1);
12624 } else {
12625 ReplaceAllUsesWith(From: Node, To: Res);
12626 RemoveDeadNode(N: Node);
12627 }
12628
12629 return Res;
12630}
12631
12632/// getMachineNode - These are used for target selectors to create a new node
12633/// with specified return type(s), MachineInstr opcode, and operands.
12634///
12635/// Note that getMachineNode returns the resultant node. If there is already a
12636/// node of the specified opcode and operands, it returns that node instead of
12637/// the current one.
12638MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12639 EVT VT) {
12640 SDVTList VTs = getVTList(VT);
12641 return getMachineNode(Opcode, dl, VTs, Ops: {});
12642}
12643
12644MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12645 EVT VT, SDValue Op1) {
12646 SDVTList VTs = getVTList(VT);
12647 SDValue Ops[] = { Op1 };
12648 return getMachineNode(Opcode, dl, VTs, Ops);
12649}
12650
12651MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12652 EVT VT, SDValue Op1, SDValue Op2) {
12653 SDVTList VTs = getVTList(VT);
12654 SDValue Ops[] = { Op1, Op2 };
12655 return getMachineNode(Opcode, dl, VTs, Ops);
12656}
12657
12658MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12659 EVT VT, SDValue Op1, SDValue Op2,
12660 SDValue Op3) {
12661 SDVTList VTs = getVTList(VT);
12662 SDValue Ops[] = { Op1, Op2, Op3 };
12663 return getMachineNode(Opcode, dl, VTs, Ops);
12664}
12665
12666MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12667 EVT VT, ArrayRef<SDValue> Ops) {
12668 SDVTList VTs = getVTList(VT);
12669 return getMachineNode(Opcode, dl, VTs, Ops);
12670}
12671
12672MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12673 EVT VT1, EVT VT2, SDValue Op1,
12674 SDValue Op2) {
12675 SDVTList VTs = getVTList(VT1, VT2);
12676 SDValue Ops[] = { Op1, Op2 };
12677 return getMachineNode(Opcode, dl, VTs, Ops);
12678}
12679
12680MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12681 EVT VT1, EVT VT2, SDValue Op1,
12682 SDValue Op2, SDValue Op3) {
12683 SDVTList VTs = getVTList(VT1, VT2);
12684 SDValue Ops[] = { Op1, Op2, Op3 };
12685 return getMachineNode(Opcode, dl, VTs, Ops);
12686}
12687
12688MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12689 EVT VT1, EVT VT2,
12690 ArrayRef<SDValue> Ops) {
12691 SDVTList VTs = getVTList(VT1, VT2);
12692 return getMachineNode(Opcode, dl, VTs, Ops);
12693}
12694
12695MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12696 EVT VT1, EVT VT2, EVT VT3,
12697 SDValue Op1, SDValue Op2) {
12698 SDVTList VTs = getVTList(VT1, VT2, VT3);
12699 SDValue Ops[] = { Op1, Op2 };
12700 return getMachineNode(Opcode, dl, VTs, Ops);
12701}
12702
12703MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12704 EVT VT1, EVT VT2, EVT VT3,
12705 SDValue Op1, SDValue Op2,
12706 SDValue Op3) {
12707 SDVTList VTs = getVTList(VT1, VT2, VT3);
12708 SDValue Ops[] = { Op1, Op2, Op3 };
12709 return getMachineNode(Opcode, dl, VTs, Ops);
12710}
12711
12712MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12713 EVT VT1, EVT VT2, EVT VT3,
12714 ArrayRef<SDValue> Ops) {
12715 SDVTList VTs = getVTList(VT1, VT2, VT3);
12716 return getMachineNode(Opcode, dl, VTs, Ops);
12717}
12718
12719MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12720 ArrayRef<EVT> ResultTys,
12721 ArrayRef<SDValue> Ops) {
12722 SDVTList VTs = getVTList(VTs: ResultTys);
12723 return getMachineNode(Opcode, dl, VTs, Ops);
12724}
12725
12726MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &DL,
12727 SDVTList VTs,
12728 ArrayRef<SDValue> Ops) {
12729 bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue;
12730 MachineSDNode *N;
12731 FoldingSetInsertToken InsertToken;
12732
12733 if (DoCSE) {
12734 SDNodeKey ID(~Opcode, VTs, Ops);
12735 if (SDNode *E = lookupNode(Key: ID, DL, InsertToken)) {
12736 return cast<MachineSDNode>(Val: UpdateSDLocOnMergeSDNode(N: E, OLoc: DL));
12737 }
12738 }
12739
12740 // Allocate a new MachineSDNode.
12741 N = newSDNode<MachineSDNode>(Args: ~Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
12742 createOperands(Node: N, Vals: Ops);
12743
12744 if (DoCSE)
12745 CSEMap.insert(N, Token: InsertToken);
12746
12747 InsertNode(N);
12748 NewSDValueDbgMsg(V: SDValue(N, 0), Msg: "Creating new machine node: ", G: this);
12749 return N;
12750}
12751
12752/// getTargetExtractSubreg - A convenience function for creating
12753/// TargetOpcode::EXTRACT_SUBREG nodes.
12754SDValue SelectionDAG::getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT,
12755 SDValue Operand) {
12756 SDValue SRIdxVal = getTargetConstant(Val: SRIdx, DL, VT: MVT::i32);
12757 SDNode *Subreg = getMachineNode(Opcode: TargetOpcode::EXTRACT_SUBREG, dl: DL,
12758 VT, Op1: Operand, Op2: SRIdxVal);
12759 return SDValue(Subreg, 0);
12760}
12761
12762/// getTargetInsertSubreg - A convenience function for creating
12763/// TargetOpcode::INSERT_SUBREG nodes.
12764SDValue SelectionDAG::getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT,
12765 SDValue Operand, SDValue Subreg) {
12766 SDValue SRIdxVal = getTargetConstant(Val: SRIdx, DL, VT: MVT::i32);
12767 SDNode *Result = getMachineNode(Opcode: TargetOpcode::INSERT_SUBREG, dl: DL,
12768 VT, Op1: Operand, Op2: Subreg, Op3: SRIdxVal);
12769 return SDValue(Result, 0);
12770}
12771
12772/// getNodeIfExists - Get the specified node if it's already available, or
12773/// else return NULL.
12774SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,
12775 ArrayRef<SDValue> Ops,
12776 bool AllowCommute) {
12777 SDNodeFlags Flags;
12778 if (Inserter)
12779 Flags = Inserter->getFlags();
12780 return getNodeIfExists(Opcode, VTList, Ops, Flags, AllowCommute);
12781}
12782
12783SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,
12784 ArrayRef<SDValue> Ops,
12785 const SDNodeFlags Flags,
12786 bool AllowCommute) {
12787 if (VTList.VTs[VTList.NumVTs - 1] == MVT::Glue)
12788 return nullptr;
12789
12790 auto Lookup = [&](ArrayRef<SDValue> LookupOps) -> SDNode * {
12791 SDNodeKey ID(Opcode, VTList, LookupOps);
12792 FoldingSetInsertToken InsertToken;
12793 if (SDNode *E = lookupNode(Key: ID, InsertToken)) {
12794 E->intersectFlagsWith(Flags);
12795 return E;
12796 }
12797 return nullptr;
12798 };
12799
12800 if (SDNode *Existing = Lookup(Ops))
12801 return Existing;
12802
12803 if (AllowCommute && TLI->isCommutativeBinOp(Opcode))
12804 return Lookup({Ops[1], Ops[0]});
12805
12806 return nullptr;
12807}
12808
12809/// doesNodeExist - Check if a node exists without modifying its flags.
12810bool SelectionDAG::doesNodeExist(unsigned Opcode, SDVTList VTList,
12811 ArrayRef<SDValue> Ops) {
12812 if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) {
12813 SDNodeKey ID(Opcode, VTList, Ops);
12814 FoldingSetInsertToken InsertToken;
12815 if (lookupNode(Key: ID, DL: SDLoc(), InsertToken))
12816 return true;
12817 }
12818 return false;
12819}
12820
12821/// getDbgValue - Creates a SDDbgValue node.
12822///
12823/// SDNode
12824SDDbgValue *SelectionDAG::getDbgValue(DIVariable *Var, DIExpression *Expr,
12825 SDNode *N, unsigned R, bool IsIndirect,
12826 const DebugLoc &DL, unsigned O) {
12827 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
12828 "Expected inlined-at fields to agree");
12829 return new (DbgInfo->getAlloc())
12830 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromNode(Node: N, ResNo: R),
12831 {}, IsIndirect, DL, O,
12832 /*IsVariadic=*/false);
12833}
12834
12835/// Constant
12836SDDbgValue *SelectionDAG::getConstantDbgValue(DIVariable *Var,
12837 DIExpression *Expr,
12838 const Value *C,
12839 const DebugLoc &DL, unsigned O) {
12840 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
12841 "Expected inlined-at fields to agree");
12842 return new (DbgInfo->getAlloc())
12843 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromConst(Const: C), {},
12844 /*IsIndirect=*/false, DL, O,
12845 /*IsVariadic=*/false);
12846}
12847
12848/// FrameIndex
12849SDDbgValue *SelectionDAG::getFrameIndexDbgValue(DIVariable *Var,
12850 DIExpression *Expr, unsigned FI,
12851 bool IsIndirect,
12852 const DebugLoc &DL,
12853 unsigned O) {
12854 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
12855 "Expected inlined-at fields to agree");
12856 return getFrameIndexDbgValue(Var, Expr, FI, Dependencies: {}, IsIndirect, DL, O);
12857}
12858
12859/// FrameIndex with dependencies
12860SDDbgValue *SelectionDAG::getFrameIndexDbgValue(DIVariable *Var,
12861 DIExpression *Expr, unsigned FI,
12862 ArrayRef<SDNode *> Dependencies,
12863 bool IsIndirect,
12864 const DebugLoc &DL,
12865 unsigned O) {
12866 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
12867 "Expected inlined-at fields to agree");
12868 return new (DbgInfo->getAlloc())
12869 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromFrameIdx(FrameIdx: FI),
12870 Dependencies, IsIndirect, DL, O,
12871 /*IsVariadic=*/false);
12872}
12873
12874/// VReg
12875SDDbgValue *SelectionDAG::getVRegDbgValue(DIVariable *Var, DIExpression *Expr,
12876 Register VReg, bool IsIndirect,
12877 const DebugLoc &DL, unsigned O) {
12878 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
12879 "Expected inlined-at fields to agree");
12880 return new (DbgInfo->getAlloc())
12881 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromVReg(VReg),
12882 {}, IsIndirect, DL, O,
12883 /*IsVariadic=*/false);
12884}
12885
12886SDDbgValue *SelectionDAG::getDbgValueList(DIVariable *Var, DIExpression *Expr,
12887 ArrayRef<SDDbgOperand> Locs,
12888 ArrayRef<SDNode *> Dependencies,
12889 bool IsIndirect, const DebugLoc &DL,
12890 unsigned O, bool IsVariadic) {
12891 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
12892 "Expected inlined-at fields to agree");
12893 return new (DbgInfo->getAlloc())
12894 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, Locs, Dependencies, IsIndirect,
12895 DL, O, IsVariadic);
12896}
12897
12898void SelectionDAG::transferDbgValues(SDValue From, SDValue To,
12899 unsigned OffsetInBits, unsigned SizeInBits,
12900 bool InvalidateDbg) {
12901 SDNode *FromNode = From.getNode();
12902 SDNode *ToNode = To.getNode();
12903 assert(FromNode && ToNode && "Can't modify dbg values");
12904
12905 // PR35338
12906 // TODO: assert(From != To && "Redundant dbg value transfer");
12907 // TODO: assert(FromNode != ToNode && "Intranode dbg value transfer");
12908 if (From == To || FromNode == ToNode)
12909 return;
12910
12911 if (!FromNode->getHasDebugValue())
12912 return;
12913
12914 SDDbgOperand FromLocOp =
12915 SDDbgOperand::fromNode(Node: From.getNode(), ResNo: From.getResNo());
12916 SDDbgOperand ToLocOp = SDDbgOperand::fromNode(Node: To.getNode(), ResNo: To.getResNo());
12917
12918 SmallVector<SDDbgValue *, 2> ClonedDVs;
12919 for (SDDbgValue *Dbg : GetDbgValues(SD: FromNode)) {
12920 if (Dbg->isInvalidated())
12921 continue;
12922
12923 // TODO: assert(!Dbg->isInvalidated() && "Transfer of invalid dbg value");
12924
12925 // Create a new location ops vector that is equal to the old vector, but
12926 // with each instance of FromLocOp replaced with ToLocOp.
12927 bool Changed = false;
12928 auto NewLocOps = Dbg->copyLocationOps();
12929 std::replace_if(
12930 first: NewLocOps.begin(), last: NewLocOps.end(),
12931 pred: [&Changed, FromLocOp](const SDDbgOperand &Op) {
12932 bool Match = Op == FromLocOp;
12933 Changed |= Match;
12934 return Match;
12935 },
12936 new_value: ToLocOp);
12937 // Ignore this SDDbgValue if we didn't find a matching location.
12938 if (!Changed)
12939 continue;
12940
12941 DIVariable *Var = Dbg->getVariable();
12942 auto *Expr = Dbg->getExpression();
12943 // If a fragment is requested, update the expression.
12944 if (SizeInBits) {
12945 // When splitting a larger (e.g., sign-extended) value whose
12946 // lower bits are described with an SDDbgValue, do not attempt
12947 // to transfer the SDDbgValue to the upper bits.
12948 if (auto FI = Expr->getFragmentInfo())
12949 if (OffsetInBits + SizeInBits > FI->SizeInBits)
12950 continue;
12951 auto Fragment = DIExpression::createFragmentExpression(Expr, OffsetInBits,
12952 SizeInBits);
12953 if (!Fragment)
12954 continue;
12955 Expr = *Fragment;
12956 }
12957
12958 auto AdditionalDependencies = Dbg->getAdditionalDependencies();
12959 // Clone the SDDbgValue and move it to To.
12960 SDDbgValue *Clone = getDbgValueList(
12961 Var, Expr, Locs: NewLocOps, Dependencies: AdditionalDependencies, IsIndirect: Dbg->isIndirect(),
12962 DL: Dbg->getDebugLoc(), O: std::max(a: ToNode->getIROrder(), b: Dbg->getOrder()),
12963 IsVariadic: Dbg->isVariadic());
12964 ClonedDVs.push_back(Elt: Clone);
12965
12966 if (InvalidateDbg) {
12967 // Invalidate value and indicate the SDDbgValue should not be emitted.
12968 Dbg->setIsInvalidated();
12969 Dbg->setIsEmitted();
12970 }
12971 }
12972
12973 for (SDDbgValue *Dbg : ClonedDVs) {
12974 assert(is_contained(Dbg->getSDNodes(), ToNode) &&
12975 "Transferred DbgValues should depend on the new SDNode");
12976 AddDbgValue(DB: Dbg, isParameter: false);
12977 }
12978}
12979
12980void SelectionDAG::salvageDebugInfo(SDNode &N) {
12981 if (!N.getHasDebugValue())
12982 return;
12983
12984 auto GetLocationOperand = [](SDNode *Node, unsigned ResNo) {
12985 if (auto *FISDN = dyn_cast<FrameIndexSDNode>(Val: Node))
12986 return SDDbgOperand::fromFrameIdx(FrameIdx: FISDN->getIndex());
12987 return SDDbgOperand::fromNode(Node, ResNo);
12988 };
12989
12990 SmallVector<SDDbgValue *, 2> ClonedDVs;
12991 for (auto *DV : GetDbgValues(SD: &N)) {
12992 if (DV->isInvalidated())
12993 continue;
12994 switch (N.getOpcode()) {
12995 default:
12996 break;
12997 case ISD::ADD: {
12998 SDValue N0 = N.getOperand(Num: 0);
12999 SDValue N1 = N.getOperand(Num: 1);
13000 if (!isa<ConstantSDNode>(Val: N0)) {
13001 bool RHSConstant = isa<ConstantSDNode>(Val: N1);
13002 uint64_t Offset;
13003 if (RHSConstant)
13004 Offset = N.getConstantOperandVal(Num: 1);
13005 // We are not allowed to turn indirect debug values variadic, so
13006 // don't salvage those.
13007 if (!RHSConstant && DV->isIndirect())
13008 continue;
13009
13010 // Rewrite an ADD constant node into a DIExpression. Since we are
13011 // performing arithmetic to compute the variable's *value* in the
13012 // DIExpression, we need to mark the expression with a
13013 // DW_OP_stack_value.
13014 auto *DIExpr = DV->getExpression();
13015 auto NewLocOps = DV->copyLocationOps();
13016 bool Changed = false;
13017 size_t OrigLocOpsSize = NewLocOps.size();
13018 for (size_t i = 0; i < OrigLocOpsSize; ++i) {
13019 // We're not given a ResNo to compare against because the whole
13020 // node is going away. We know that any ISD::ADD only has one
13021 // result, so we can assume any node match is using the result.
13022 if (NewLocOps[i].getKind() != SDDbgOperand::SDNODE ||
13023 NewLocOps[i].getSDNode() != &N)
13024 continue;
13025 NewLocOps[i] = GetLocationOperand(N0.getNode(), N0.getResNo());
13026 if (RHSConstant) {
13027 SmallVector<uint64_t, 3> ExprOps;
13028 DIExpression::appendOffset(Ops&: ExprOps, Offset);
13029 DIExpr = DIExpression::appendOpsToArg(Expr: DIExpr, Ops: ExprOps, ArgNo: i, StackValue: true);
13030 } else {
13031 // Convert to a variadic expression (if not already).
13032 // convertToVariadicExpression() returns a const pointer, so we use
13033 // a temporary const variable here.
13034 const auto *TmpDIExpr =
13035 DIExpression::convertToVariadicExpression(Expr: DIExpr);
13036 SmallVector<uint64_t, 3> ExprOps;
13037 ExprOps.push_back(Elt: dwarf::DW_OP_LLVM_arg);
13038 ExprOps.push_back(Elt: NewLocOps.size());
13039 ExprOps.push_back(Elt: dwarf::DW_OP_plus);
13040 SDDbgOperand RHS =
13041 SDDbgOperand::fromNode(Node: N1.getNode(), ResNo: N1.getResNo());
13042 NewLocOps.push_back(Elt: RHS);
13043 DIExpr = DIExpression::appendOpsToArg(Expr: TmpDIExpr, Ops: ExprOps, ArgNo: i, StackValue: true);
13044 }
13045 Changed = true;
13046 }
13047 (void)Changed;
13048 assert(Changed && "Salvage target doesn't use N");
13049
13050 bool IsVariadic =
13051 DV->isVariadic() || OrigLocOpsSize != NewLocOps.size();
13052
13053 auto AdditionalDependencies = DV->getAdditionalDependencies();
13054 SDDbgValue *Clone = getDbgValueList(
13055 Var: DV->getVariable(), Expr: DIExpr, Locs: NewLocOps, Dependencies: AdditionalDependencies,
13056 IsIndirect: DV->isIndirect(), DL: DV->getDebugLoc(), O: DV->getOrder(), IsVariadic);
13057 ClonedDVs.push_back(Elt: Clone);
13058 DV->setIsInvalidated();
13059 DV->setIsEmitted();
13060 LLVM_DEBUG(dbgs() << "SALVAGE: Rewriting";
13061 N0.getNode()->dumprFull(this);
13062 dbgs() << " into " << *DIExpr << '\n');
13063 }
13064 break;
13065 }
13066 case ISD::TRUNCATE: {
13067 SDValue N0 = N.getOperand(Num: 0);
13068 TypeSize FromSize = N0.getValueSizeInBits();
13069 TypeSize ToSize = N.getValueSizeInBits(ResNo: 0);
13070
13071 DIExpression *DbgExpression = DV->getExpression();
13072 auto ExtOps = DIExpression::getExtOps(FromSize, ToSize, Signed: false);
13073 auto NewLocOps = DV->copyLocationOps();
13074 bool Changed = false;
13075 for (size_t i = 0; i < NewLocOps.size(); ++i) {
13076 if (NewLocOps[i].getKind() != SDDbgOperand::SDNODE ||
13077 NewLocOps[i].getSDNode() != &N)
13078 continue;
13079
13080 NewLocOps[i] = GetLocationOperand(N0.getNode(), N0.getResNo());
13081 DbgExpression = DIExpression::appendOpsToArg(Expr: DbgExpression, Ops: ExtOps, ArgNo: i);
13082 Changed = true;
13083 }
13084 assert(Changed && "Salvage target doesn't use N");
13085 (void)Changed;
13086
13087 SDDbgValue *Clone =
13088 getDbgValueList(Var: DV->getVariable(), Expr: DbgExpression, Locs: NewLocOps,
13089 Dependencies: DV->getAdditionalDependencies(), IsIndirect: DV->isIndirect(),
13090 DL: DV->getDebugLoc(), O: DV->getOrder(), IsVariadic: DV->isVariadic());
13091
13092 ClonedDVs.push_back(Elt: Clone);
13093 DV->setIsInvalidated();
13094 DV->setIsEmitted();
13095 LLVM_DEBUG(dbgs() << "SALVAGE: Rewriting"; N0.getNode()->dumprFull(this);
13096 dbgs() << " into " << *DbgExpression << '\n');
13097 break;
13098 }
13099 }
13100 }
13101
13102 for (SDDbgValue *Dbg : ClonedDVs) {
13103 assert((!Dbg->getSDNodes().empty() ||
13104 llvm::any_of(Dbg->getLocationOps(),
13105 [&](const SDDbgOperand &Op) {
13106 return Op.getKind() == SDDbgOperand::FRAMEIX;
13107 })) &&
13108 "Salvaged DbgValue should depend on a new SDNode");
13109 AddDbgValue(DB: Dbg, isParameter: false);
13110 }
13111}
13112
13113/// Creates a SDDbgLabel node.
13114SDDbgLabel *SelectionDAG::getDbgLabel(DILabel *Label,
13115 const DebugLoc &DL, unsigned O) {
13116 assert(cast<DILabel>(Label)->isValidLocationForIntrinsic(DL) &&
13117 "Expected inlined-at fields to agree");
13118 return new (DbgInfo->getAlloc()) SDDbgLabel(Label, DL, O);
13119}
13120
13121namespace {
13122
13123/// RAUWUpdateListener - Helper for ReplaceAllUsesWith - When the node
13124/// pointed to by a use iterator is deleted, increment the use iterator
13125/// so that it doesn't dangle.
13126///
13127class RAUWUpdateListener : public SelectionDAG::DAGUpdateListener {
13128 SDNode::use_iterator &UI;
13129 SDNode::use_iterator &UE;
13130
13131 void NodeDeleted(SDNode *N, SDNode *E) override {
13132 // Increment the iterator as needed.
13133 while (UI != UE && N == UI->getUser())
13134 ++UI;
13135 }
13136
13137public:
13138 RAUWUpdateListener(SelectionDAG &d,
13139 SDNode::use_iterator &ui,
13140 SDNode::use_iterator &ue)
13141 : SelectionDAG::DAGUpdateListener(d), UI(ui), UE(ue) {}
13142};
13143
13144} // end anonymous namespace
13145
13146/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
13147/// This can cause recursive merging of nodes in the DAG.
13148///
13149/// This version assumes From has a single result value.
13150///
13151void SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To) {
13152 SDNode *From = FromN.getNode();
13153 assert(From->getNumValues() == 1 && FromN.getResNo() == 0 &&
13154 "Cannot replace with this method!");
13155 assert(From != To.getNode() && "Cannot replace uses of with self");
13156
13157 // Preserve Debug Values
13158 transferDbgValues(From: FromN, To);
13159 // Preserve extra info.
13160 copyExtraInfo(From, To: To.getNode());
13161
13162 // Iterate over all the existing uses of From. New uses will be added
13163 // to the beginning of the use list, which we avoid visiting.
13164 // This specifically avoids visiting uses of From that arise while the
13165 // replacement is happening, because any such uses would be the result
13166 // of CSE: If an existing node looks like From after one of its operands
13167 // is replaced by To, we don't want to replace of all its users with To
13168 // too. See PR3018 for more info.
13169 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
13170 RAUWUpdateListener Listener(*this, UI, UE);
13171 while (UI != UE) {
13172 SDNode *User = UI->getUser();
13173
13174 // This node is about to morph, remove its old self from the CSE maps.
13175 RemoveNodeFromCSEMaps(N: User);
13176
13177 // A user can appear in a use list multiple times, and when this
13178 // happens the uses are usually next to each other in the list.
13179 // To help reduce the number of CSE recomputations, process all
13180 // the uses of this user that we can find this way.
13181 do {
13182 SDUse &Use = *UI;
13183 ++UI;
13184 Use.set(To);
13185 if (To->isDivergent() != From->isDivergent())
13186 updateDivergence(N: User);
13187 } while (UI != UE && UI->getUser() == User);
13188 // Now that we have modified User, add it back to the CSE maps. If it
13189 // already exists there, recursively merge the results together.
13190 AddModifiedNodeToCSEMaps(N: User);
13191 }
13192
13193 // If we just RAUW'd the root, take note.
13194 if (FromN == getRoot())
13195 setRoot(To);
13196}
13197
13198/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
13199/// This can cause recursive merging of nodes in the DAG.
13200///
13201/// This version assumes that for each value of From, there is a
13202/// corresponding value in To in the same position with the same type.
13203///
13204void SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To) {
13205#ifndef NDEBUG
13206 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
13207 assert((!From->hasAnyUseOfValue(i) ||
13208 From->getValueType(i) == To->getValueType(i)) &&
13209 "Cannot use this version of ReplaceAllUsesWith!");
13210#endif
13211
13212 // Handle the trivial case.
13213 if (From == To)
13214 return;
13215
13216 // Preserve Debug Info. Only do this if there's a use.
13217 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
13218 if (From->hasAnyUseOfValue(Value: i)) {
13219 assert((i < To->getNumValues()) && "Invalid To location");
13220 transferDbgValues(From: SDValue(From, i), To: SDValue(To, i));
13221 }
13222 // Preserve extra info.
13223 copyExtraInfo(From, To);
13224
13225 // Iterate over just the existing users of From. See the comments in
13226 // the ReplaceAllUsesWith above.
13227 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
13228 RAUWUpdateListener Listener(*this, UI, UE);
13229 while (UI != UE) {
13230 SDNode *User = UI->getUser();
13231
13232 // This node is about to morph, remove its old self from the CSE maps.
13233 RemoveNodeFromCSEMaps(N: User);
13234
13235 // A user can appear in a use list multiple times, and when this
13236 // happens the uses are usually next to each other in the list.
13237 // To help reduce the number of CSE recomputations, process all
13238 // the uses of this user that we can find this way.
13239 do {
13240 SDUse &Use = *UI;
13241 ++UI;
13242 Use.setNode(To);
13243 if (To->isDivergent() != From->isDivergent())
13244 updateDivergence(N: User);
13245 } while (UI != UE && UI->getUser() == User);
13246
13247 // Now that we have modified User, add it back to the CSE maps. If it
13248 // already exists there, recursively merge the results together.
13249 AddModifiedNodeToCSEMaps(N: User);
13250 }
13251
13252 // If we just RAUW'd the root, take note.
13253 if (From == getRoot().getNode())
13254 setRoot(SDValue(To, getRoot().getResNo()));
13255}
13256
13257/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
13258/// This can cause recursive merging of nodes in the DAG.
13259///
13260/// This version can replace From with any result values. To must match the
13261/// number and types of values returned by From.
13262void SelectionDAG::ReplaceAllUsesWith(SDNode *From, const SDValue *To) {
13263 if (From->getNumValues() == 1) // Handle the simple case efficiently.
13264 return ReplaceAllUsesWith(FromN: SDValue(From, 0), To: To[0]);
13265
13266 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i) {
13267 // Preserve Debug Info.
13268 transferDbgValues(From: SDValue(From, i), To: To[i]);
13269 // Preserve extra info.
13270 copyExtraInfo(From, To: To[i].getNode());
13271 }
13272
13273 // Iterate over just the existing users of From. See the comments in
13274 // the ReplaceAllUsesWith above.
13275 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
13276 RAUWUpdateListener Listener(*this, UI, UE);
13277 while (UI != UE) {
13278 SDNode *User = UI->getUser();
13279
13280 // This node is about to morph, remove its old self from the CSE maps.
13281 RemoveNodeFromCSEMaps(N: User);
13282
13283 // A user can appear in a use list multiple times, and when this happens the
13284 // uses are usually next to each other in the list. To help reduce the
13285 // number of CSE and divergence recomputations, process all the uses of this
13286 // user that we can find this way.
13287 bool To_IsDivergent = false;
13288 do {
13289 SDUse &Use = *UI;
13290 const SDValue &ToOp = To[Use.getResNo()];
13291 ++UI;
13292 Use.set(ToOp);
13293 if (ToOp.getValueType() != MVT::Other)
13294 To_IsDivergent |= ToOp->isDivergent();
13295 } while (UI != UE && UI->getUser() == User);
13296
13297 if (To_IsDivergent != From->isDivergent())
13298 updateDivergence(N: User);
13299
13300 // Now that we have modified User, add it back to the CSE maps. If it
13301 // already exists there, recursively merge the results together.
13302 AddModifiedNodeToCSEMaps(N: User);
13303 }
13304
13305 // If we just RAUW'd the root, take note.
13306 if (From == getRoot().getNode())
13307 setRoot(SDValue(To[getRoot().getResNo()]));
13308}
13309
13310/// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving
13311/// uses of other values produced by From.getNode() alone. The Deleted
13312/// vector is handled the same way as for ReplaceAllUsesWith.
13313void SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To){
13314 // Handle the really simple, really trivial case efficiently.
13315 if (From == To) return;
13316
13317 // Handle the simple, trivial, case efficiently.
13318 if (From.getNode()->getNumValues() == 1) {
13319 ReplaceAllUsesWith(FromN: From, To);
13320 return;
13321 }
13322
13323 // Preserve Debug Info.
13324 transferDbgValues(From, To);
13325 copyExtraInfo(From: From.getNode(), To: To.getNode());
13326
13327 // Iterate over just the existing users of From. See the comments in
13328 // the ReplaceAllUsesWith above.
13329 SDNode::use_iterator UI = From.getNode()->use_begin(),
13330 UE = From.getNode()->use_end();
13331 RAUWUpdateListener Listener(*this, UI, UE);
13332 while (UI != UE) {
13333 SDNode *User = UI->getUser();
13334 bool UserRemovedFromCSEMaps = false;
13335
13336 // A user can appear in a use list multiple times, and when this
13337 // happens the uses are usually next to each other in the list.
13338 // To help reduce the number of CSE recomputations, process all
13339 // the uses of this user that we can find this way.
13340 do {
13341 SDUse &Use = *UI;
13342
13343 // Skip uses of different values from the same node.
13344 if (Use.getResNo() != From.getResNo()) {
13345 ++UI;
13346 continue;
13347 }
13348
13349 // If this node hasn't been modified yet, it's still in the CSE maps,
13350 // so remove its old self from the CSE maps.
13351 if (!UserRemovedFromCSEMaps) {
13352 RemoveNodeFromCSEMaps(N: User);
13353 UserRemovedFromCSEMaps = true;
13354 }
13355
13356 ++UI;
13357 Use.set(To);
13358 if (To->isDivergent() != From->isDivergent())
13359 updateDivergence(N: User);
13360 } while (UI != UE && UI->getUser() == User);
13361 // We are iterating over all uses of the From node, so if a use
13362 // doesn't use the specific value, no changes are made.
13363 if (!UserRemovedFromCSEMaps)
13364 continue;
13365
13366 // Now that we have modified User, add it back to the CSE maps. If it
13367 // already exists there, recursively merge the results together.
13368 AddModifiedNodeToCSEMaps(N: User);
13369 }
13370
13371 // If we just RAUW'd the root, take note.
13372 if (From == getRoot())
13373 setRoot(To);
13374}
13375
13376namespace {
13377
13378/// UseMemo - This class is used by SelectionDAG::ReplaceAllUsesOfValuesWith
13379/// to record information about a use.
13380struct UseMemo {
13381 SDNode *User;
13382 unsigned Index;
13383 SDUse *Use;
13384};
13385
13386/// operator< - Sort Memos by User.
13387bool operator<(const UseMemo &L, const UseMemo &R) {
13388 return (intptr_t)L.User < (intptr_t)R.User;
13389}
13390
13391/// RAUOVWUpdateListener - Helper for ReplaceAllUsesOfValuesWith - When the node
13392/// pointed to by a UseMemo is deleted, set the User to nullptr to indicate that
13393/// the node already has been taken care of recursively.
13394class RAUOVWUpdateListener : public SelectionDAG::DAGUpdateListener {
13395 SmallVectorImpl<UseMemo> &Uses;
13396
13397 void NodeDeleted(SDNode *N, SDNode *E) override {
13398 for (UseMemo &Memo : Uses)
13399 if (Memo.User == N)
13400 Memo.User = nullptr;
13401 }
13402
13403public:
13404 RAUOVWUpdateListener(SelectionDAG &d, SmallVectorImpl<UseMemo> &uses)
13405 : SelectionDAG::DAGUpdateListener(d), Uses(uses) {}
13406};
13407
13408} // end anonymous namespace
13409
13410/// Return true if a glue output should propagate divergence information.
13411static bool gluePropagatesDivergence(const SDNode *Node) {
13412 switch (Node->getOpcode()) {
13413 case ISD::CopyFromReg:
13414 case ISD::CopyToReg:
13415 return false;
13416 default:
13417 return true;
13418 }
13419
13420 llvm_unreachable("covered opcode switch");
13421}
13422
13423bool SelectionDAG::calculateDivergence(SDNode *N) {
13424 if (TLI->isSDNodeAlwaysUniform(N)) {
13425 assert(!TLI->isSDNodeSourceOfDivergence(N, FLI, UA) &&
13426 "Conflicting divergence information!");
13427 return false;
13428 }
13429 if (TLI->isSDNodeSourceOfDivergence(N, FLI, UA))
13430 return true;
13431 for (const auto &Op : N->ops()) {
13432 EVT VT = Op.getValueType();
13433
13434 // Skip Chain. It does not carry divergence.
13435 if (VT != MVT::Other && Op.getNode()->isDivergent() &&
13436 (VT != MVT::Glue || gluePropagatesDivergence(Node: Op.getNode())))
13437 return true;
13438 }
13439 return false;
13440}
13441
13442void SelectionDAG::updateDivergence(SDNode *N) {
13443 SmallVector<SDNode *, 16> Worklist(1, N);
13444 do {
13445 N = Worklist.pop_back_val();
13446 bool IsDivergent = calculateDivergence(N);
13447 if (N->SDNodeBits.IsDivergent != IsDivergent) {
13448 N->SDNodeBits.IsDivergent = IsDivergent;
13449 llvm::append_range(C&: Worklist, R: N->users());
13450 }
13451 } while (!Worklist.empty());
13452}
13453
13454void SelectionDAG::CreateTopologicalOrder(std::vector<SDNode *> &Order) {
13455 DenseMap<SDNode *, unsigned> Degree;
13456 Order.reserve(n: AllNodes.size());
13457 for (auto &N : allnodes()) {
13458 unsigned NOps = N.getNumOperands();
13459 Degree[&N] = NOps;
13460 if (0 == NOps)
13461 Order.push_back(x: &N);
13462 }
13463 for (size_t I = 0; I != Order.size(); ++I) {
13464 SDNode *N = Order[I];
13465 for (auto *U : N->users()) {
13466 unsigned &UnsortedOps = Degree[U];
13467 if (0 == --UnsortedOps)
13468 Order.push_back(x: U);
13469 }
13470 }
13471}
13472
13473#if !defined(NDEBUG) && LLVM_ENABLE_ABI_BREAKING_CHECKS
13474void SelectionDAG::VerifyDAGDivergence() {
13475 std::vector<SDNode *> TopoOrder;
13476 CreateTopologicalOrder(TopoOrder);
13477 for (auto *N : TopoOrder) {
13478 assert(calculateDivergence(N) == N->isDivergent() &&
13479 "Divergence bit inconsistency detected");
13480 }
13481}
13482#endif
13483
13484/// ReplaceAllUsesOfValuesWith - Replace any uses of From with To, leaving
13485/// uses of other values produced by From.getNode() alone. The same value
13486/// may appear in both the From and To list. The Deleted vector is
13487/// handled the same way as for ReplaceAllUsesWith.
13488void SelectionDAG::ReplaceAllUsesOfValuesWith(const SDValue *From,
13489 const SDValue *To,
13490 unsigned Num){
13491 // Handle the simple, trivial case efficiently.
13492 if (Num == 1)
13493 return ReplaceAllUsesOfValueWith(From: *From, To: *To);
13494
13495 transferDbgValues(From: *From, To: *To);
13496 copyExtraInfo(From: From->getNode(), To: To->getNode());
13497
13498 // Read up all the uses and make records of them. This helps
13499 // processing new uses that are introduced during the
13500 // replacement process.
13501 SmallVector<UseMemo, 4> Uses;
13502 for (unsigned i = 0; i != Num; ++i) {
13503 unsigned FromResNo = From[i].getResNo();
13504 SDNode *FromNode = From[i].getNode();
13505 for (SDUse &Use : FromNode->uses()) {
13506 if (Use.getResNo() == FromResNo) {
13507 UseMemo Memo = {.User: Use.getUser(), .Index: i, .Use: &Use};
13508 Uses.push_back(Elt: Memo);
13509 }
13510 }
13511 }
13512
13513 // Sort the uses, so that all the uses from a given User are together.
13514 llvm::sort(C&: Uses);
13515 RAUOVWUpdateListener Listener(*this, Uses);
13516
13517 for (unsigned UseIndex = 0, UseIndexEnd = Uses.size();
13518 UseIndex != UseIndexEnd; ) {
13519 // We know that this user uses some value of From. If it is the right
13520 // value, update it.
13521 SDNode *User = Uses[UseIndex].User;
13522 // If the node has been deleted by recursive CSE updates when updating
13523 // another node, then just skip this entry.
13524 if (User == nullptr) {
13525 ++UseIndex;
13526 continue;
13527 }
13528
13529 // This node is about to morph, remove its old self from the CSE maps.
13530 RemoveNodeFromCSEMaps(N: User);
13531
13532 // The Uses array is sorted, so all the uses for a given User
13533 // are next to each other in the list.
13534 // To help reduce the number of CSE recomputations, process all
13535 // the uses of this user that we can find this way.
13536 do {
13537 unsigned i = Uses[UseIndex].Index;
13538 SDUse &Use = *Uses[UseIndex].Use;
13539 ++UseIndex;
13540
13541 Use.set(To[i]);
13542 } while (UseIndex != UseIndexEnd && Uses[UseIndex].User == User);
13543
13544 // Now that we have modified User, add it back to the CSE maps. If it
13545 // already exists there, recursively merge the results together.
13546 AddModifiedNodeToCSEMaps(N: User);
13547 }
13548}
13549
13550/// AssignTopologicalOrder - Assign a unique node id for each node in the DAG
13551/// based on their topological order. It returns the maximum id and a vector
13552/// of the SDNodes* in assigned order by reference.
13553unsigned SelectionDAG::AssignTopologicalOrder() {
13554 unsigned DAGSize = 0;
13555
13556 // SortedPos tracks the progress of the algorithm. Nodes before it are
13557 // sorted, nodes after it are unsorted. When the algorithm completes
13558 // it is at the end of the list.
13559 allnodes_iterator SortedPos = allnodes_begin();
13560
13561 // Visit all the nodes. Move nodes with no operands to the front of
13562 // the list immediately. Annotate nodes that do have operands with their
13563 // operand count. Before we do this, the Node Id fields of the nodes
13564 // may contain arbitrary values. After, the Node Id fields for nodes
13565 // before SortedPos will contain the topological sort index, and the
13566 // Node Id fields for nodes At SortedPos and after will contain the
13567 // count of outstanding operands.
13568 for (SDNode &N : llvm::make_early_inc_range(Range: allnodes())) {
13569 checkForCycles(N: &N, DAG: this);
13570 unsigned Degree = N.getNumOperands();
13571 if (Degree == 0) {
13572 // A node with no uses, add it to the result array immediately.
13573 N.setNodeId(DAGSize++);
13574 allnodes_iterator Q(&N);
13575 if (Q != SortedPos)
13576 SortedPos = AllNodes.insert(where: SortedPos, New: AllNodes.remove(IT&: Q));
13577 assert(SortedPos != AllNodes.end() && "Overran node list");
13578 ++SortedPos;
13579 } else {
13580 // Temporarily use the Node Id as scratch space for the degree count.
13581 N.setNodeId(Degree);
13582 }
13583 }
13584
13585 // Visit all the nodes. As we iterate, move nodes into sorted order,
13586 // such that by the time the end is reached all nodes will be sorted.
13587 for (SDNode &Node : allnodes()) {
13588 SDNode *N = &Node;
13589 checkForCycles(N, DAG: this);
13590 // N is in sorted position, so all its uses have one less operand
13591 // that needs to be sorted.
13592 for (SDNode *P : N->users()) {
13593 unsigned Degree = P->getNodeId();
13594 assert(Degree != 0 && "Invalid node degree");
13595 --Degree;
13596 if (Degree == 0) {
13597 // All of P's operands are sorted, so P may sorted now.
13598 P->setNodeId(DAGSize++);
13599 if (P->getIterator() != SortedPos)
13600 SortedPos = AllNodes.insert(where: SortedPos, New: AllNodes.remove(IT: P));
13601 assert(SortedPos != AllNodes.end() && "Overran node list");
13602 ++SortedPos;
13603 } else {
13604 // Update P's outstanding operand count.
13605 P->setNodeId(Degree);
13606 }
13607 }
13608 if (Node.getIterator() == SortedPos) {
13609#ifndef NDEBUG
13610 allnodes_iterator I(N);
13611 SDNode *S = &*++I;
13612 dbgs() << "Overran sorted position:\n";
13613 S->dumprFull(this); dbgs() << "\n";
13614 dbgs() << "Checking if this is due to cycles\n";
13615 checkForCycles(this, true);
13616#endif
13617 llvm_unreachable(nullptr);
13618 }
13619 }
13620
13621 assert(SortedPos == AllNodes.end() &&
13622 "Topological sort incomplete!");
13623 assert(AllNodes.front().getOpcode() == ISD::EntryToken &&
13624 "First node in topological sort is not the entry token!");
13625 assert(AllNodes.front().getNodeId() == 0 &&
13626 "First node in topological sort has non-zero id!");
13627 assert(AllNodes.front().getNumOperands() == 0 &&
13628 "First node in topological sort has operands!");
13629 assert(AllNodes.back().getNodeId() == (int)DAGSize-1 &&
13630 "Last node in topologic sort has unexpected id!");
13631 assert(AllNodes.back().use_empty() &&
13632 "Last node in topologic sort has users!");
13633 assert(DAGSize == allnodes_size() && "Node count mismatch!");
13634 return DAGSize;
13635}
13636
13637void SelectionDAG::getTopologicallyOrderedNodes(
13638 SmallVectorImpl<const SDNode *> &SortedNodes) const {
13639 SortedNodes.clear();
13640 // Node -> remaining number of outstanding operands.
13641 DenseMap<const SDNode *, unsigned> RemainingOperands;
13642
13643 // Put nodes without any operands into SortedNodes first.
13644 for (const SDNode &N : allnodes()) {
13645 checkForCycles(N: &N, DAG: this);
13646 unsigned NumOperands = N.getNumOperands();
13647 if (NumOperands == 0)
13648 SortedNodes.push_back(Elt: &N);
13649 else
13650 // Record their total number of outstanding operands.
13651 RemainingOperands[&N] = NumOperands;
13652 }
13653
13654 // A node is pushed into SortedNodes when all of its operands (predecessors in
13655 // the graph) are also in SortedNodes.
13656 for (unsigned i = 0U; i < SortedNodes.size(); ++i) {
13657 const SDNode *N = SortedNodes[i];
13658 for (const SDNode *U : N->users()) {
13659 // HandleSDNode is never part of a DAG and therefore has no entry in
13660 // RemainingOperands.
13661 if (U->getOpcode() == ISD::HANDLENODE)
13662 continue;
13663 unsigned &NumRemOperands = RemainingOperands[U];
13664 assert(NumRemOperands && "Invalid number of remaining operands");
13665 --NumRemOperands;
13666 if (!NumRemOperands)
13667 SortedNodes.push_back(Elt: U);
13668 }
13669 }
13670
13671 assert(SortedNodes.size() == AllNodes.size() && "Node count mismatch");
13672 assert(SortedNodes.front()->getOpcode() == ISD::EntryToken &&
13673 "First node in topological sort is not the entry token");
13674 assert(SortedNodes.front()->getNumOperands() == 0 &&
13675 "First node in topological sort has operands");
13676}
13677
13678/// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the
13679/// value is produced by SD.
13680void SelectionDAG::AddDbgValue(SDDbgValue *DB, bool isParameter) {
13681 for (SDNode *SD : DB->getSDNodes()) {
13682 if (!SD)
13683 continue;
13684 assert(DbgInfo->getSDDbgValues(SD).empty() || SD->getHasDebugValue());
13685 SD->setHasDebugValue(true);
13686 }
13687 DbgInfo->add(V: DB, isParameter);
13688}
13689
13690void SelectionDAG::AddDbgLabel(SDDbgLabel *DB) { DbgInfo->add(L: DB); }
13691
13692SDValue SelectionDAG::makeEquivalentMemoryOrdering(SDValue OldChain,
13693 SDValue NewMemOpChain) {
13694 assert(isa<MemSDNode>(NewMemOpChain) && "Expected a memop node");
13695 assert(NewMemOpChain.getValueType() == MVT::Other && "Expected a token VT");
13696 // The new memory operation must have the same position as the old load in
13697 // terms of memory dependency. Create a TokenFactor for the old load and new
13698 // memory operation and update uses of the old load's output chain to use that
13699 // TokenFactor.
13700 if (OldChain == NewMemOpChain || OldChain.use_empty())
13701 return NewMemOpChain;
13702
13703 SDValue TokenFactor = getNode(Opcode: ISD::TokenFactor, DL: SDLoc(OldChain), VT: MVT::Other,
13704 N1: OldChain, N2: NewMemOpChain);
13705 ReplaceAllUsesOfValueWith(From: OldChain, To: TokenFactor);
13706 UpdateNodeOperands(N: TokenFactor.getNode(), Op1: OldChain, Op2: NewMemOpChain);
13707 return TokenFactor;
13708}
13709
13710SDValue SelectionDAG::makeEquivalentMemoryOrdering(LoadSDNode *OldLoad,
13711 SDValue NewMemOp) {
13712 assert(isa<MemSDNode>(NewMemOp.getNode()) && "Expected a memop node");
13713 SDValue OldChain = SDValue(OldLoad, 1);
13714 SDValue NewMemOpChain = NewMemOp.getValue(R: 1);
13715 return makeEquivalentMemoryOrdering(OldChain, NewMemOpChain);
13716}
13717
13718SDValue SelectionDAG::getSymbolFunctionGlobalAddress(SDValue Op,
13719 Function **OutFunction) {
13720 assert(isa<ExternalSymbolSDNode>(Op) && "Node should be an ExternalSymbol");
13721
13722 auto *Symbol = cast<ExternalSymbolSDNode>(Val&: Op)->getSymbol();
13723 auto *Module = MF->getFunction().getParent();
13724 auto *Function = Module->getFunction(Name: Symbol);
13725
13726 if (OutFunction != nullptr)
13727 *OutFunction = Function;
13728
13729 if (Function != nullptr) {
13730 auto PtrTy = TLI->getPointerTy(DL: getDataLayout(), AS: Function->getAddressSpace());
13731 return getGlobalAddress(GV: Function, DL: SDLoc(Op), VT: PtrTy);
13732 }
13733
13734 std::string ErrorStr;
13735 raw_string_ostream ErrorFormatter(ErrorStr);
13736 ErrorFormatter << "Undefined external symbol ";
13737 ErrorFormatter << '"' << Symbol << '"';
13738 report_fatal_error(reason: Twine(ErrorStr));
13739}
13740
13741//===----------------------------------------------------------------------===//
13742// SDNode Class
13743//===----------------------------------------------------------------------===//
13744
13745bool llvm::isNullConstant(SDValue V) {
13746 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val&: V);
13747 return Const != nullptr && Const->isZero();
13748}
13749
13750bool llvm::isNullConstantOrUndef(SDValue V) {
13751 return V.isUndef() || isNullConstant(V);
13752}
13753
13754bool llvm::isNullFPConstant(SDValue V) {
13755 ConstantFPSDNode *Const = dyn_cast<ConstantFPSDNode>(Val&: V);
13756 return Const != nullptr && Const->isZero() && !Const->isNegative();
13757}
13758
13759bool llvm::isAllOnesConstant(SDValue V) {
13760 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val&: V);
13761 return Const != nullptr && Const->isAllOnes();
13762}
13763
13764bool llvm::isOneConstant(SDValue V) {
13765 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val&: V);
13766 return Const != nullptr && Const->isOne();
13767}
13768
13769bool llvm::isMinSignedConstant(SDValue V) {
13770 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val&: V);
13771 return Const != nullptr && Const->isMinSignedValue();
13772}
13773
13774bool SelectionDAG::isIdentityElement(unsigned Opcode, SDNodeFlags Flags,
13775 SDValue V, unsigned OperandNo,
13776 unsigned Depth) const {
13777 APInt DemandedElts = getDemandAllEltsMask(V);
13778 return isIdentityElement(Opc: Opcode, Flags, V, DemandedElts, OperandNo, Depth);
13779}
13780
13781bool SelectionDAG::isIdentityElement(unsigned Opcode, SDNodeFlags Flags,
13782 SDValue V, const APInt &DemandedElts,
13783 unsigned OperandNo, unsigned Depth) const {
13784 // NOTE: The cases should match with IR's ConstantExpr::getBinOpIdentity().
13785 // TODO: Target-specific opcodes could be added.
13786 if (V.getValueType().isInteger()) {
13787 KnownBits Known = computeKnownBits(Op: V, DemandedElts, Depth);
13788 if (Known.isConstant()) {
13789 const APInt &Const = Known.getConstant();
13790 switch (Opcode) {
13791 case ISD::ADD:
13792 case ISD::OR:
13793 case ISD::XOR:
13794 case ISD::UMAX:
13795 return Const.isZero();
13796 case ISD::MUL:
13797 return Const.isOne();
13798 case ISD::AND:
13799 case ISD::UMIN:
13800 return Const.isAllOnes();
13801 case ISD::SMAX:
13802 return Const.isMinSignedValue();
13803 case ISD::SMIN:
13804 return Const.isMaxSignedValue();
13805 case ISD::SUB:
13806 case ISD::SHL:
13807 case ISD::SRA:
13808 case ISD::SRL:
13809 return OperandNo == 1 && Const.isZero();
13810 case ISD::UDIV:
13811 case ISD::SDIV:
13812 return OperandNo == 1 && Const.isOne();
13813 }
13814 }
13815 } else if (auto *ConstFP = isConstOrConstSplatFP(N: V, DemandedElts)) {
13816 switch (Opcode) {
13817 case ISD::FADD:
13818 return ConstFP->isZero() &&
13819 (Flags.hasNoSignedZeros() || ConstFP->isNegative());
13820 case ISD::FSUB:
13821 return OperandNo == 1 && ConstFP->isZero() &&
13822 (Flags.hasNoSignedZeros() || !ConstFP->isNegative());
13823 case ISD::FMUL:
13824 return ConstFP->isOne();
13825 case ISD::FDIV:
13826 return OperandNo == 1 && ConstFP->isOne();
13827 case ISD::FMINNUM:
13828 case ISD::FMAXNUM:
13829 case ISD::FMINIMUMNUM:
13830 case ISD::FMAXIMUMNUM: {
13831 // Neutral element for fminnum/fminimumnum is NaN, Inf or FLT_MAX,
13832 // depending on fast-math flags (FMF).
13833 EVT VT = V.getValueType();
13834 const fltSemantics &Semantics = VT.getFltSemantics();
13835 APFloat NeutralAF = !Flags.hasNoNaNs() ? APFloat::getQNaN(Sem: Semantics)
13836 : !Flags.hasNoInfs() ? APFloat::getInf(Sem: Semantics)
13837 : APFloat::getLargest(Sem: Semantics);
13838 if (Opcode == ISD::FMAXNUM || Opcode == ISD::FMAXIMUMNUM)
13839 NeutralAF.changeSign();
13840
13841 return ConstFP->isExactlyValue(V: NeutralAF);
13842 }
13843 case ISD::FMINIMUM:
13844 case ISD::FMAXIMUM: {
13845 // Neutral element for fminimum is Inf or FLT_MAX, depending on FMF.
13846 const APFloat &VAPF = ConstFP->getValueAPF();
13847 bool NeutralNegative = (Opcode == ISD::FMAXIMUM);
13848 if (Flags.hasNoInfs())
13849 return VAPF.isLargest() && VAPF.isNegative() == NeutralNegative;
13850 return VAPF.isInfinity() && VAPF.isNegative() == NeutralNegative;
13851 }
13852 }
13853 }
13854 return false;
13855}
13856
13857SDValue llvm::peekThroughBitcasts(SDValue V) {
13858 while (V.getOpcode() == ISD::BITCAST)
13859 V = V.getOperand(i: 0);
13860 return V;
13861}
13862
13863SDValue llvm::peekThroughOneUseBitcasts(SDValue V) {
13864 while (V.getOpcode() == ISD::BITCAST && V.getOperand(i: 0).hasOneUse())
13865 V = V.getOperand(i: 0);
13866 return V;
13867}
13868
13869SDValue llvm::peekThroughExtractSubvectors(SDValue V) {
13870 while (V.getOpcode() == ISD::EXTRACT_SUBVECTOR)
13871 V = V.getOperand(i: 0);
13872 return V;
13873}
13874
13875SDValue llvm::peekThroughInsertVectorElt(SDValue V, const APInt &DemandedElts) {
13876 while (V.getOpcode() == ISD::INSERT_VECTOR_ELT) {
13877 SDValue InVec = V.getOperand(i: 0);
13878 SDValue EltNo = V.getOperand(i: 2);
13879 EVT VT = InVec.getValueType();
13880 auto *IndexC = dyn_cast<ConstantSDNode>(Val&: EltNo);
13881 if (IndexC && VT.isFixedLengthVector() &&
13882 IndexC->getAPIntValue().ult(RHS: VT.getVectorNumElements()) &&
13883 !DemandedElts[IndexC->getZExtValue()]) {
13884 V = InVec;
13885 continue;
13886 }
13887 break;
13888 }
13889 return V;
13890}
13891
13892SDValue llvm::peekThroughTruncates(SDValue V) {
13893 while (V.getOpcode() == ISD::TRUNCATE)
13894 V = V.getOperand(i: 0);
13895 return V;
13896}
13897
13898bool llvm::isBitwiseNot(SDValue V, bool AllowUndefs) {
13899 if (V.getOpcode() != ISD::XOR)
13900 return false;
13901 V = peekThroughBitcasts(V: V.getOperand(i: 1));
13902 unsigned NumBits = V.getScalarValueSizeInBits();
13903 ConstantSDNode *C =
13904 isConstOrConstSplat(N: V, AllowUndefs, /*AllowTruncation*/ true);
13905 return C && (C->getAPIntValue().countr_one() >= NumBits);
13906}
13907
13908ConstantSDNode *llvm::isConstOrConstSplat(SDValue N, bool AllowUndefs,
13909 bool AllowTruncation) {
13910 APInt DemandedElts = getDemandAllEltsMask(V: N);
13911 return isConstOrConstSplat(N, DemandedElts, AllowUndefs, AllowTruncation);
13912}
13913
13914ConstantSDNode *llvm::isConstOrConstSplat(SDValue N, const APInt &DemandedElts,
13915 bool AllowUndefs,
13916 bool AllowTruncation) {
13917 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val&: N))
13918 return CN;
13919
13920 // SplatVectors can truncate their operands. Ignore that case here unless
13921 // AllowTruncation is set.
13922 if (N->getOpcode() == ISD::SPLAT_VECTOR) {
13923 EVT VecEltVT = N->getValueType(ResNo: 0).getVectorElementType();
13924 if (auto *CN = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 0))) {
13925 EVT CVT = CN->getValueType(ResNo: 0);
13926 assert(CVT.bitsGE(VecEltVT) && "Illegal splat_vector element extension");
13927 if (AllowTruncation || CVT == VecEltVT)
13928 return CN;
13929 }
13930 }
13931
13932 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Val&: N)) {
13933 BitVector UndefElements;
13934 ConstantSDNode *CN = BV->getConstantSplatNode(DemandedElts, UndefElements: &UndefElements);
13935
13936 // BuildVectors can truncate their operands. Ignore that case here unless
13937 // AllowTruncation is set.
13938 // TODO: Look into whether we should allow UndefElements in non-DemandedElts
13939 if (CN && (UndefElements.none() || AllowUndefs)) {
13940 EVT CVT = CN->getValueType(ResNo: 0);
13941 EVT NSVT = N.getValueType().getScalarType();
13942 assert(CVT.bitsGE(NSVT) && "Illegal build vector element extension");
13943 if (AllowTruncation || (CVT == NSVT))
13944 return CN;
13945 }
13946 }
13947
13948 return nullptr;
13949}
13950
13951ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N, bool AllowUndefs) {
13952 APInt DemandedElts = getDemandAllEltsMask(V: N);
13953 return isConstOrConstSplatFP(N, DemandedElts, AllowUndefs);
13954}
13955
13956ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N,
13957 const APInt &DemandedElts,
13958 bool AllowUndefs) {
13959 if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(Val&: N))
13960 return CN;
13961
13962 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Val&: N)) {
13963 BitVector UndefElements;
13964 ConstantFPSDNode *CN =
13965 BV->getConstantFPSplatNode(DemandedElts, UndefElements: &UndefElements);
13966 // TODO: Look into whether we should allow UndefElements in non-DemandedElts
13967 if (CN && (UndefElements.none() || AllowUndefs))
13968 return CN;
13969 }
13970
13971 if (N.getOpcode() == ISD::SPLAT_VECTOR)
13972 if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(Val: N.getOperand(i: 0)))
13973 return CN;
13974
13975 return nullptr;
13976}
13977
13978bool llvm::isNullOrNullSplat(SDValue N, bool AllowUndefs) {
13979 // TODO: may want to use peekThroughBitcast() here.
13980 ConstantSDNode *C =
13981 isConstOrConstSplat(N, AllowUndefs, /*AllowTruncation=*/true);
13982 return C && C->isZero();
13983}
13984
13985bool llvm::isOneOrOneSplat(SDValue N, bool AllowUndefs) {
13986 ConstantSDNode *C =
13987 isConstOrConstSplat(N, AllowUndefs, /*AllowTruncation*/ true);
13988 return C && C->isOne();
13989}
13990
13991bool llvm::isOneOrOneSplatFP(SDValue N, bool AllowUndefs) {
13992 ConstantFPSDNode *C = isConstOrConstSplatFP(N, AllowUndefs);
13993 return C && C->isOne();
13994}
13995
13996bool llvm::isAllOnesOrAllOnesSplat(SDValue N, bool AllowUndefs) {
13997 N = peekThroughBitcasts(V: N);
13998 unsigned BitWidth = N.getScalarValueSizeInBits();
13999 ConstantSDNode *C =
14000 isConstOrConstSplat(N, AllowUndefs, /*AllowTruncation=*/true);
14001 return C && C->getAPIntValue().countTrailingOnes() >= BitWidth;
14002}
14003
14004bool llvm::isOnesOrOnesSplat(SDValue N, bool AllowUndefs) {
14005 ConstantSDNode *C = isConstOrConstSplat(N, AllowUndefs);
14006 return C && APInt::isSameValue(I1: C->getAPIntValue(),
14007 I2: APInt(C->getAPIntValue().getBitWidth(), 1));
14008}
14009
14010bool llvm::isZeroOrZeroSplat(SDValue N, bool AllowUndefs) {
14011 N = peekThroughBitcasts(V: N);
14012 ConstantSDNode *C = isConstOrConstSplat(N, AllowUndefs, AllowTruncation: true);
14013 return C && C->isZero();
14014}
14015
14016bool llvm::isZeroOrZeroSplatFP(SDValue N, bool AllowUndefs) {
14017 ConstantFPSDNode *C = isConstOrConstSplatFP(N, AllowUndefs);
14018 return C && C->isZero();
14019}
14020
14021HandleSDNode::~HandleSDNode() {
14022 DropOperands();
14023}
14024
14025MemSDNode::MemSDNode(
14026 unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTs, EVT memvt,
14027 PointerUnion<MachineMemOperand *, MachineMemOperand **> memrefs)
14028 : SDNode(Opc, Order, dl, VTs), MemoryVT(memvt), MemRefs(memrefs) {
14029 bool IsVolatile = false;
14030 bool IsNonTemporal = false;
14031 bool IsDereferenceable = true;
14032 bool IsInvariant = true;
14033 for (const MachineMemOperand *MMO : memoperands()) {
14034 IsVolatile |= MMO->isVolatile();
14035 IsNonTemporal |= MMO->isNonTemporal();
14036 IsDereferenceable &= MMO->isDereferenceable();
14037 IsInvariant &= MMO->isInvariant();
14038 }
14039 MemSDNodeBits.IsVolatile = IsVolatile;
14040 MemSDNodeBits.IsNonTemporal = IsNonTemporal;
14041 MemSDNodeBits.IsDereferenceable = IsDereferenceable;
14042 MemSDNodeBits.IsInvariant = IsInvariant;
14043
14044 // For the single-MMO case, we check here that the size of the memory operand
14045 // fits within the size of the MMO. This is because the MMO might indicate
14046 // only a possible address range instead of specifying the affected memory
14047 // addresses precisely.
14048 assert((getNumMemOperands() != 1 || !getMemOperand()->getType().isValid() ||
14049 TypeSize::isKnownLE(memvt.getStoreSize(),
14050 getMemOperand()->getSize().getValue())) &&
14051 "Size mismatch!");
14052}
14053
14054namespace {
14055
14056 struct EVTArray {
14057 std::vector<EVT> VTs;
14058
14059 EVTArray() {
14060 VTs.reserve(n: MVT::VALUETYPE_SIZE);
14061 for (unsigned i = 0; i < MVT::VALUETYPE_SIZE; ++i)
14062 VTs.push_back(x: MVT((MVT::SimpleValueType)i));
14063 }
14064 };
14065
14066} // end anonymous namespace
14067
14068/// getValueTypeList - Return a pointer to the specified value type.
14069///
14070const EVT *SDNode::getValueTypeList(MVT VT) {
14071 static EVTArray SimpleVTArray;
14072
14073 assert(VT < MVT::VALUETYPE_SIZE && "Value type out of range!");
14074 return &SimpleVTArray.VTs[VT.SimpleTy];
14075}
14076
14077/// hasAnyUseOfValue - Return true if there are any use of the indicated
14078/// value. This method ignores uses of other values defined by this operation.
14079bool SDNode::hasAnyUseOfValue(unsigned Value) const {
14080 assert(Value < getNumValues() && "Bad value!");
14081
14082 for (SDUse &U : uses())
14083 if (U.getResNo() == Value)
14084 return true;
14085
14086 return false;
14087}
14088
14089/// isOnlyUserOf - Return true if this node is the only use of N.
14090bool SDNode::isOnlyUserOf(const SDNode *N) const {
14091 bool Seen = false;
14092 for (const SDNode *User : N->users()) {
14093 if (User == this)
14094 Seen = true;
14095 else
14096 return false;
14097 }
14098
14099 return Seen;
14100}
14101
14102/// Return true if the only users of N are contained in Nodes.
14103bool SDNode::areOnlyUsersOf(ArrayRef<const SDNode *> Nodes, const SDNode *N) {
14104 bool Seen = false;
14105 for (const SDNode *User : N->users()) {
14106 if (llvm::is_contained(Range&: Nodes, Element: User))
14107 Seen = true;
14108 else
14109 return false;
14110 }
14111
14112 return Seen;
14113}
14114
14115/// Return true if the referenced return value is an operand of N.
14116bool SDValue::isOperandOf(const SDNode *N) const {
14117 return is_contained(Range: N->op_values(), Element: *this);
14118}
14119
14120bool SDNode::isOperandOf(const SDNode *N) const {
14121 return any_of(Range: N->op_values(),
14122 P: [this](SDValue Op) { return this == Op.getNode(); });
14123}
14124
14125/// reachesChainWithoutSideEffects - Return true if this operand (which must
14126/// be a chain) reaches the specified operand without crossing any
14127/// side-effecting instructions on any chain path. In practice, this looks
14128/// through token factors and non-volatile loads. In order to remain efficient,
14129/// this only looks a couple of nodes in, it does not do an exhaustive search.
14130///
14131/// Note that we only need to examine chains when we're searching for
14132/// side-effects; SelectionDAG requires that all side-effects are represented
14133/// by chains, even if another operand would force a specific ordering. This
14134/// constraint is necessary to allow transformations like splitting loads.
14135bool SDValue::reachesChainWithoutSideEffects(SDValue Dest,
14136 unsigned Depth) const {
14137 if (*this == Dest) return true;
14138
14139 // Don't search too deeply, we just want to be able to see through
14140 // TokenFactor's etc.
14141 if (Depth == 0) return false;
14142
14143 // If this is a token factor, all inputs to the TF happen in parallel.
14144 if (getOpcode() == ISD::TokenFactor) {
14145 // First, try a shallow search.
14146 if (is_contained(Range: (*this)->ops(), Element: Dest)) {
14147 // We found the chain we want as an operand of this TokenFactor.
14148 // Essentially, we reach the chain without side-effects if we could
14149 // serialize the TokenFactor into a simple chain of operations with
14150 // Dest as the last operation. This is automatically true if the
14151 // chain has one use: there are no other ordering constraints.
14152 // If the chain has more than one use, we give up: some other
14153 // use of Dest might force a side-effect between Dest and the current
14154 // node.
14155 if (Dest.hasOneUse())
14156 return true;
14157 }
14158 // Next, try a deep search: check whether every operand of the TokenFactor
14159 // reaches Dest.
14160 return llvm::all_of(Range: (*this)->ops(), P: [=](SDValue Op) {
14161 return Op.reachesChainWithoutSideEffects(Dest, Depth: Depth - 1);
14162 });
14163 }
14164
14165 // Loads don't have side effects, look through them.
14166 if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(Val: *this)) {
14167 if (Ld->isUnordered())
14168 return Ld->getChain().reachesChainWithoutSideEffects(Dest, Depth: Depth-1);
14169 }
14170 return false;
14171}
14172
14173bool SDNode::hasPredecessor(const SDNode *N) const {
14174 SmallPtrSet<const SDNode *, 32> Visited;
14175 SmallVector<const SDNode *, 16> Worklist;
14176 Worklist.push_back(Elt: this);
14177 return hasPredecessorHelper(N, Visited, Worklist);
14178}
14179
14180void SDNode::intersectFlagsWith(const SDNodeFlags Flags) {
14181 this->Flags &= Flags;
14182}
14183
14184SDValue
14185SelectionDAG::matchBinOpReduction(SDNode *Extract, ISD::NodeType &BinOp,
14186 ArrayRef<ISD::NodeType> CandidateBinOps,
14187 bool AllowPartials) {
14188 // The pattern must end in an extract from index 0.
14189 if (Extract->getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
14190 !isNullConstant(V: Extract->getOperand(Num: 1)))
14191 return SDValue();
14192
14193 // Match against one of the candidate binary ops.
14194 SDValue Op = Extract->getOperand(Num: 0);
14195 if (llvm::none_of(Range&: CandidateBinOps, P: [Op](ISD::NodeType BinOp) {
14196 return Op.getOpcode() == unsigned(BinOp);
14197 }))
14198 return SDValue();
14199
14200 // Floating-point reductions may require relaxed constraints on the final step
14201 // of the reduction because they may reorder intermediate operations.
14202 unsigned CandidateBinOp = Op.getOpcode();
14203 if (Op.getValueType().isFloatingPoint()) {
14204 SDNodeFlags Flags = Op->getFlags();
14205 switch (CandidateBinOp) {
14206 case ISD::FADD:
14207 if (!Flags.hasNoSignedZeros() || !Flags.hasAllowReassociation())
14208 return SDValue();
14209 break;
14210 default:
14211 llvm_unreachable("Unhandled FP opcode for binop reduction");
14212 }
14213 }
14214
14215 // Matching failed - attempt to see if we did enough stages that a partial
14216 // reduction from a subvector is possible.
14217 auto PartialReduction = [&](SDValue Op, unsigned NumSubElts) {
14218 if (!AllowPartials || !Op)
14219 return SDValue();
14220 EVT OpVT = Op.getValueType();
14221 EVT OpSVT = OpVT.getScalarType();
14222 EVT SubVT = EVT::getVectorVT(Context&: *getContext(), VT: OpSVT, NumElements: NumSubElts);
14223 if (TLI->getExtractSubvectorCost(ResVT: SubVT, SrcVT: OpVT, Index: 0) >
14224 TargetLowering::ExtractSubvectorCost::Cheap)
14225 return SDValue();
14226 BinOp = (ISD::NodeType)CandidateBinOp;
14227 return getExtractSubvector(DL: SDLoc(Op), VT: SubVT, Vec: Op, Idx: 0);
14228 };
14229
14230 // At each stage, we're looking for something that looks like:
14231 // %s = shufflevector <8 x i32> %op, <8 x i32> undef,
14232 // <8 x i32> <i32 2, i32 3, i32 undef, i32 undef,
14233 // i32 undef, i32 undef, i32 undef, i32 undef>
14234 // %a = binop <8 x i32> %op, %s
14235 // Where the mask changes according to the stage. E.g. for a 3-stage pyramid,
14236 // we expect something like:
14237 // <4,5,6,7,u,u,u,u>
14238 // <2,3,u,u,u,u,u,u>
14239 // <1,u,u,u,u,u,u,u>
14240 // While a partial reduction match would be:
14241 // <2,3,u,u,u,u,u,u>
14242 // <1,u,u,u,u,u,u,u>
14243 unsigned Stages = Log2_32(Value: Op.getValueType().getVectorNumElements());
14244 SDValue PrevOp;
14245 for (unsigned i = 0; i < Stages; ++i) {
14246 unsigned MaskEnd = (1 << i);
14247
14248 if (Op.getOpcode() != CandidateBinOp)
14249 return PartialReduction(PrevOp, MaskEnd);
14250
14251 SDValue Op0 = Op.getOperand(i: 0);
14252 SDValue Op1 = Op.getOperand(i: 1);
14253
14254 ShuffleVectorSDNode *Shuffle = dyn_cast<ShuffleVectorSDNode>(Val&: Op0);
14255 if (Shuffle) {
14256 Op = Op1;
14257 } else {
14258 Shuffle = dyn_cast<ShuffleVectorSDNode>(Val&: Op1);
14259 Op = Op0;
14260 }
14261
14262 // The first operand of the shuffle should be the same as the other operand
14263 // of the binop.
14264 if (!Shuffle || Shuffle->getOperand(Num: 0) != Op)
14265 return PartialReduction(PrevOp, MaskEnd);
14266
14267 // Verify the shuffle has the expected (at this stage of the pyramid) mask.
14268 for (int Index = 0; Index < (int)MaskEnd; ++Index)
14269 if (Shuffle->getMaskElt(Idx: Index) != (int)(MaskEnd + Index))
14270 return PartialReduction(PrevOp, MaskEnd);
14271
14272 PrevOp = Op;
14273 }
14274
14275 // Handle subvector reductions, which tend to appear after the shuffle
14276 // reduction stages.
14277 while (Op.getOpcode() == CandidateBinOp) {
14278 unsigned NumElts = Op.getValueType().getVectorNumElements();
14279 SDValue Op0 = Op.getOperand(i: 0);
14280 SDValue Op1 = Op.getOperand(i: 1);
14281 if (Op0.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
14282 Op1.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
14283 Op0.getOperand(i: 0) != Op1.getOperand(i: 0))
14284 break;
14285 SDValue Src = Op0.getOperand(i: 0);
14286 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
14287 if (NumSrcElts != (2 * NumElts))
14288 break;
14289 if (!(Op0.getConstantOperandAPInt(i: 1) == 0 &&
14290 Op1.getConstantOperandAPInt(i: 1) == NumElts) &&
14291 !(Op1.getConstantOperandAPInt(i: 1) == 0 &&
14292 Op0.getConstantOperandAPInt(i: 1) == NumElts))
14293 break;
14294 Op = Src;
14295 }
14296
14297 BinOp = (ISD::NodeType)CandidateBinOp;
14298 return Op;
14299}
14300
14301SDValue SelectionDAG::UnrollVectorOp(SDNode *N, unsigned ResNE) {
14302 EVT VT = N->getValueType(ResNo: 0);
14303 EVT EltVT = VT.getVectorElementType();
14304 unsigned NE = getMaxRuntimeNumElements(VT);
14305
14306 if (VT.isScalableVector() && (NE == 0 || ResNE != 0))
14307 reportFatalUsageError(reason: "Cannot unroll scalable vector!");
14308
14309 assert(NE && "Nothing to unroll!");
14310 SDLoc dl(N);
14311
14312 // If ResNE is 0, fully unroll the vector op.
14313 if (ResNE == 0)
14314 ResNE = NE;
14315 else if (NE > ResNE)
14316 NE = ResNE;
14317
14318 if (N->getNumValues() == 2) {
14319 SmallVector<SDValue, 8> Scalars0, Scalars1;
14320 SmallVector<SDValue, 4> Operands(N->getNumOperands());
14321 EVT VT1 = N->getValueType(ResNo: 1);
14322 EVT EltVT1 = VT1.getVectorElementType();
14323
14324 unsigned i;
14325 for (i = 0; i != NE; ++i) {
14326 for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) {
14327 SDValue Operand = N->getOperand(Num: j);
14328 EVT OperandVT = Operand.getValueType();
14329
14330 // A vector operand; extract a single element.
14331 EVT OperandEltVT = OperandVT.getVectorElementType();
14332 Operands[j] = getExtractVectorElt(DL: dl, VT: OperandEltVT, Vec: Operand, Idx: i);
14333 }
14334
14335 SDValue EltOp = getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {EltVT, EltVT1}, Ops: Operands);
14336 Scalars0.push_back(Elt: EltOp);
14337 Scalars1.push_back(Elt: EltOp.getValue(R: 1));
14338 }
14339
14340 for (; i < ResNE; ++i) {
14341 Scalars0.push_back(Elt: getUNDEF(VT: EltVT));
14342 Scalars1.push_back(Elt: getUNDEF(VT: EltVT1));
14343 }
14344
14345 ElementCount ResEC = VT.isScalableVector() ? VT.getVectorElementCount()
14346 : ElementCount::getFixed(MinVal: ResNE);
14347 EVT VecVT = EVT::getVectorVT(Context&: *getContext(), VT: EltVT, EC: ResEC);
14348 EVT VecVT1 = EVT::getVectorVT(Context&: *getContext(), VT: EltVT1, EC: ResEC);
14349 SDValue Vec0 = buildVectorFromUnrolledParts(VT: VecVT, DL: dl, Scalars: Scalars0);
14350 SDValue Vec1 = buildVectorFromUnrolledParts(VT: VecVT1, DL: dl, Scalars: Scalars1);
14351 return getMergeValues(Ops: {Vec0, Vec1}, dl);
14352 }
14353
14354 assert(N->getNumValues() == 1 &&
14355 "Can't unroll a vector with multiple results!");
14356
14357 SmallVector<SDValue, 8> Scalars;
14358 SmallVector<SDValue, 4> Operands(N->getNumOperands());
14359
14360 unsigned i;
14361 for (i= 0; i != NE; ++i) {
14362 for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) {
14363 SDValue Operand = N->getOperand(Num: j);
14364 EVT OperandVT = Operand.getValueType();
14365 if (OperandVT.isVector()) {
14366 // A vector operand; extract a single element.
14367 EVT OperandEltVT = OperandVT.getVectorElementType();
14368 Operands[j] = getExtractVectorElt(DL: dl, VT: OperandEltVT, Vec: Operand, Idx: i);
14369 } else {
14370 // A scalar operand; just use it as is.
14371 Operands[j] = Operand;
14372 }
14373 }
14374
14375 switch (N->getOpcode()) {
14376 default: {
14377 Scalars.push_back(Elt: getNode(Opcode: N->getOpcode(), DL: dl, VT: EltVT, Ops: Operands,
14378 Flags: N->getFlags()));
14379 break;
14380 }
14381 case ISD::VSELECT:
14382 Scalars.push_back(
14383 Elt: getNode(Opcode: ISD::SELECT, DL: dl, VT: EltVT, Ops: Operands, Flags: N->getFlags()));
14384 break;
14385 case ISD::SHL:
14386 case ISD::SRA:
14387 case ISD::SRL:
14388 case ISD::ROTL:
14389 case ISD::ROTR:
14390 Scalars.push_back(Elt: getNode(Opcode: N->getOpcode(), DL: dl, VT: EltVT, N1: Operands[0],
14391 N2: getShiftAmountOperand(LHSTy: Operands[0].getValueType(),
14392 Op: Operands[1])));
14393 break;
14394 case ISD::SIGN_EXTEND_INREG: {
14395 EVT ExtVT = cast<VTSDNode>(Val&: Operands[1])->getVT().getVectorElementType();
14396 Scalars.push_back(Elt: getNode(Opcode: N->getOpcode(), DL: dl, VT: EltVT,
14397 N1: Operands[0],
14398 N2: getValueType(VT: ExtVT)));
14399 break;
14400 }
14401 case ISD::ADDRSPACECAST: {
14402 const auto *ASC = cast<AddrSpaceCastSDNode>(Val: N);
14403 Scalars.push_back(
14404 Elt: getAddrSpaceCast(dl, VT: EltVT, Ptr: Operands[0], SrcAS: ASC->getSrcAddressSpace(),
14405 DestAS: ASC->getDestAddressSpace(), Flags: ASC->getFlags()));
14406 break;
14407 }
14408 }
14409 }
14410
14411 for (; i < ResNE; ++i)
14412 Scalars.push_back(Elt: getUNDEF(VT: EltVT));
14413
14414 EVT VecVT = VT.isScalableVector()
14415 ? VT
14416 : EVT::getVectorVT(Context&: *getContext(), VT: EltVT, NumElements: ResNE);
14417 return buildVectorFromUnrolledParts(VT: VecVT, DL: dl, Scalars);
14418}
14419
14420std::pair<SDValue, SDValue> SelectionDAG::UnrollVectorOverflowOp(
14421 SDNode *N, unsigned ResNE) {
14422 unsigned Opcode = N->getOpcode();
14423 assert((Opcode == ISD::UADDO || Opcode == ISD::SADDO ||
14424 Opcode == ISD::USUBO || Opcode == ISD::SSUBO ||
14425 Opcode == ISD::UMULO || Opcode == ISD::SMULO) &&
14426 "Expected an overflow opcode");
14427
14428 EVT ResVT = N->getValueType(ResNo: 0);
14429 EVT OvVT = N->getValueType(ResNo: 1);
14430 EVT ResEltVT = ResVT.getVectorElementType();
14431 EVT OvEltVT = OvVT.getVectorElementType();
14432 SDLoc dl(N);
14433
14434 // If ResNE is 0, fully unroll the vector op.
14435 unsigned NE = ResVT.getVectorNumElements();
14436 if (ResNE == 0)
14437 ResNE = NE;
14438 else if (NE > ResNE)
14439 NE = ResNE;
14440
14441 SmallVector<SDValue, 8> LHSScalars;
14442 SmallVector<SDValue, 8> RHSScalars;
14443 ExtractVectorElements(Op: N->getOperand(Num: 0), Args&: LHSScalars, Start: 0, Count: NE);
14444 ExtractVectorElements(Op: N->getOperand(Num: 1), Args&: RHSScalars, Start: 0, Count: NE);
14445
14446 EVT SVT = TLI->getSetCCResultType(DL: getDataLayout(), Context&: *getContext(), VT: ResEltVT);
14447 SDVTList VTs = getVTList(VT1: ResEltVT, VT2: SVT);
14448 SmallVector<SDValue, 8> ResScalars;
14449 SmallVector<SDValue, 8> OvScalars;
14450 for (unsigned i = 0; i < NE; ++i) {
14451 SDValue Res = getNode(Opcode, DL: dl, VTList: VTs, N1: LHSScalars[i], N2: RHSScalars[i]);
14452 SDValue Ov =
14453 getSelect(DL: dl, VT: OvEltVT, Cond: Res.getValue(R: 1),
14454 LHS: getBoolConstant(V: true, DL: dl, VT: OvEltVT, OpVT: ResVT),
14455 RHS: getConstant(Val: 0, DL: dl, VT: OvEltVT));
14456
14457 ResScalars.push_back(Elt: Res);
14458 OvScalars.push_back(Elt: Ov);
14459 }
14460
14461 ResScalars.append(NumInputs: ResNE - NE, Elt: getUNDEF(VT: ResEltVT));
14462 OvScalars.append(NumInputs: ResNE - NE, Elt: getUNDEF(VT: OvEltVT));
14463
14464 EVT NewResVT = EVT::getVectorVT(Context&: *getContext(), VT: ResEltVT, NumElements: ResNE);
14465 EVT NewOvVT = EVT::getVectorVT(Context&: *getContext(), VT: OvEltVT, NumElements: ResNE);
14466 return std::make_pair(x: getBuildVector(VT: NewResVT, DL: dl, Ops: ResScalars),
14467 y: getBuildVector(VT: NewOvVT, DL: dl, Ops: OvScalars));
14468}
14469
14470static bool areNonVolatileConsecutiveLoadsOrStores(LSBaseSDNode *LS,
14471 LSBaseSDNode *Base,
14472 unsigned Bytes, int Dist,
14473 const SelectionDAG &DAG) {
14474 if (LS->isVolatile() || Base->isVolatile())
14475 return false;
14476 // TODO: probably too restrictive for atomics, revisit
14477 if (!LS->isSimple())
14478 return false;
14479 if (LS->isIndexed() || Base->isIndexed())
14480 return false;
14481 if (LS->getChain() != Base->getChain())
14482 return false;
14483 EVT VT = LS->getMemoryVT();
14484 if (VT.getSizeInBits() / 8 != Bytes)
14485 return false;
14486
14487 auto BaseLocDecomp = BaseIndexOffset::match(N: Base, DAG);
14488 auto LocDecomp = BaseIndexOffset::match(N: LS, DAG);
14489
14490 int64_t Offset = 0;
14491 if (BaseLocDecomp.equalBaseIndex(Other: LocDecomp, DAG, Off&: Offset))
14492 return (Dist * (int64_t)Bytes == Offset);
14493 return false;
14494}
14495
14496bool SelectionDAG::areNonVolatileConsecutiveLoads(LoadSDNode *LD,
14497 LoadSDNode *Base,
14498 unsigned Bytes,
14499 int Dist) const {
14500 return areNonVolatileConsecutiveLoadsOrStores(LS: LD, Base, Bytes, Dist, DAG: *this);
14501}
14502
14503bool SelectionDAG::areNonVolatileConsecutiveStores(StoreSDNode *ST,
14504 StoreSDNode *Base,
14505 unsigned Bytes,
14506 int Dist) const {
14507 return areNonVolatileConsecutiveLoadsOrStores(LS: ST, Base, Bytes, Dist, DAG: *this);
14508}
14509
14510/// InferPtrAlignment - Infer alignment of a load / store address. Return
14511/// std::nullopt if it cannot be inferred.
14512MaybeAlign SelectionDAG::InferPtrAlign(SDValue Ptr) const {
14513 // If this is a GlobalAddress + cst, return the alignment.
14514 const GlobalValue *GV = nullptr;
14515 int64_t GVOffset = 0;
14516 if (TLI->isGAPlusOffset(N: Ptr.getNode(), GA&: GV, Offset&: GVOffset)) {
14517 unsigned PtrWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType());
14518 KnownBits Known(PtrWidth);
14519 llvm::computeKnownBits(V: GV, Known, DL: getDataLayout());
14520 unsigned AlignBits = Known.countMinTrailingZeros();
14521 if (AlignBits)
14522 return commonAlignment(A: Align(1ull << std::min(a: 31U, b: AlignBits)), Offset: GVOffset);
14523 }
14524
14525 // If this is a direct reference to a stack slot, use information about the
14526 // stack slot's alignment.
14527 int FrameIdx = INT_MIN;
14528 int64_t FrameOffset = 0;
14529 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Ptr)) {
14530 FrameIdx = FI->getIndex();
14531 } else if (isBaseWithConstantOffset(Op: Ptr) &&
14532 isa<FrameIndexSDNode>(Val: Ptr.getOperand(i: 0))) {
14533 // Handle FI+Cst
14534 FrameIdx = cast<FrameIndexSDNode>(Val: Ptr.getOperand(i: 0))->getIndex();
14535 FrameOffset = Ptr.getConstantOperandVal(i: 1);
14536 }
14537
14538 if (FrameIdx != INT_MIN) {
14539 const MachineFrameInfo &MFI = getMachineFunction().getFrameInfo();
14540 return commonAlignment(A: MFI.getObjectAlign(ObjectIdx: FrameIdx), Offset: FrameOffset);
14541 }
14542
14543 return std::nullopt;
14544}
14545
14546/// Split the scalar node with EXTRACT_ELEMENT using the provided
14547/// VTs and return the low/high part.
14548std::pair<SDValue, SDValue> SelectionDAG::SplitScalar(const SDValue &N,
14549 const SDLoc &DL,
14550 const EVT &LoVT,
14551 const EVT &HiVT) {
14552 assert(!LoVT.isVector() && !HiVT.isVector() && !N.getValueType().isVector() &&
14553 "Split node must be a scalar type");
14554 SDValue Lo =
14555 getNode(Opcode: ISD::EXTRACT_ELEMENT, DL, VT: LoVT, N1: N, N2: getIntPtrConstant(Val: 0, DL));
14556 SDValue Hi =
14557 getNode(Opcode: ISD::EXTRACT_ELEMENT, DL, VT: HiVT, N1: N, N2: getIntPtrConstant(Val: 1, DL));
14558 return std::make_pair(x&: Lo, y&: Hi);
14559}
14560
14561/// GetSplitDestVTs - Compute the VTs needed for the low/hi parts of a type
14562/// which is split (or expanded) into two not necessarily identical pieces.
14563std::pair<EVT, EVT> SelectionDAG::GetSplitDestVTs(const EVT &VT) const {
14564 // Currently all types are split in half.
14565 EVT LoVT, HiVT;
14566 if (!VT.isVector())
14567 LoVT = HiVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT);
14568 else
14569 LoVT = HiVT = VT.getHalfNumVectorElementsVT(Context&: *getContext());
14570
14571 return std::make_pair(x&: LoVT, y&: HiVT);
14572}
14573
14574/// GetDependentSplitDestVTs - Compute the VTs needed for the low/hi parts of a
14575/// type, dependent on an enveloping VT that has been split into two identical
14576/// pieces. Sets the HiIsEmpty flag when hi type has zero storage size.
14577std::pair<EVT, EVT>
14578SelectionDAG::GetDependentSplitDestVTs(const EVT &VT, const EVT &EnvVT,
14579 bool *HiIsEmpty) const {
14580 EVT EltTp = VT.getVectorElementType();
14581 // Examples:
14582 // custom VL=8 with enveloping VL=8/8 yields 8/0 (hi empty)
14583 // custom VL=9 with enveloping VL=8/8 yields 8/1
14584 // custom VL=10 with enveloping VL=8/8 yields 8/2
14585 // etc.
14586 ElementCount VTNumElts = VT.getVectorElementCount();
14587 ElementCount EnvNumElts = EnvVT.getVectorElementCount();
14588 assert(VTNumElts.isScalable() == EnvNumElts.isScalable() &&
14589 "Mixing fixed width and scalable vectors when enveloping a type");
14590 EVT LoVT, HiVT;
14591 if (VTNumElts.getKnownMinValue() > EnvNumElts.getKnownMinValue()) {
14592 LoVT = EVT::getVectorVT(Context&: *getContext(), VT: EltTp, EC: EnvNumElts);
14593 HiVT = EVT::getVectorVT(Context&: *getContext(), VT: EltTp, EC: VTNumElts - EnvNumElts);
14594 *HiIsEmpty = false;
14595 } else {
14596 // Flag that hi type has zero storage size, but return split envelop type
14597 // (this would be easier if vector types with zero elements were allowed).
14598 LoVT = EVT::getVectorVT(Context&: *getContext(), VT: EltTp, EC: VTNumElts);
14599 HiVT = EVT::getVectorVT(Context&: *getContext(), VT: EltTp, EC: EnvNumElts);
14600 *HiIsEmpty = true;
14601 }
14602 return std::make_pair(x&: LoVT, y&: HiVT);
14603}
14604
14605/// SplitVector - Split the vector with EXTRACT_SUBVECTOR and return the
14606/// low/high part.
14607std::pair<SDValue, SDValue>
14608SelectionDAG::SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT,
14609 const EVT &HiVT) {
14610 assert(LoVT.isScalableVector() == HiVT.isScalableVector() &&
14611 LoVT.isScalableVector() == N.getValueType().isScalableVector() &&
14612 "Splitting vector with an invalid mixture of fixed and scalable "
14613 "vector types");
14614 assert(LoVT.getVectorMinNumElements() + HiVT.getVectorMinNumElements() <=
14615 N.getValueType().getVectorMinNumElements() &&
14616 "More vector elements requested than available!");
14617 SDValue Lo, Hi;
14618 Lo = getExtractSubvector(DL, VT: LoVT, Vec: N, Idx: 0);
14619 // For scalable vectors it is safe to use LoVT.getVectorMinNumElements()
14620 // (rather than having to use ElementCount), because EXTRACT_SUBVECTOR scales
14621 // IDX with the runtime scaling factor of the result vector type. For
14622 // fixed-width result vectors, that runtime scaling factor is 1.
14623 Hi = getExtractSubvector(DL, VT: HiVT, Vec: N, Idx: LoVT.getVectorMinNumElements());
14624 return std::make_pair(x&: Lo, y&: Hi);
14625}
14626
14627std::pair<SDValue, SDValue> SelectionDAG::SplitEVL(SDValue N, EVT VecVT,
14628 const SDLoc &DL) {
14629 // Split the vector length parameter.
14630 // %evl -> umin(%evl, %halfnumelts) and usubsat(%evl - %halfnumelts).
14631 EVT VT = N.getValueType();
14632 assert(VecVT.getVectorElementCount().isKnownEven() &&
14633 "Expecting the mask to be an evenly-sized vector");
14634 SDValue HalfNumElts = getElementCount(
14635 DL, VT, EC: VecVT.getVectorElementCount().divideCoefficientBy(RHS: 2));
14636 SDValue Lo = getNode(Opcode: ISD::UMIN, DL, VT, N1: N, N2: HalfNumElts);
14637 SDValue Hi = getNode(Opcode: ISD::USUBSAT, DL, VT, N1: N, N2: HalfNumElts);
14638 return std::make_pair(x&: Lo, y&: Hi);
14639}
14640
14641/// Widen the vector up to the next power of two using INSERT_SUBVECTOR.
14642SDValue SelectionDAG::WidenVector(const SDValue &N, const SDLoc &DL) {
14643 EVT VT = N.getValueType();
14644 EVT WideVT = EVT::getVectorVT(Context&: *getContext(), VT: VT.getVectorElementType(),
14645 NumElements: NextPowerOf2(A: VT.getVectorNumElements()));
14646 return getInsertSubvector(DL, Vec: getPOISON(VT: WideVT), SubVec: N, Idx: 0);
14647}
14648
14649void SelectionDAG::ExtractVectorElements(SDValue Op,
14650 SmallVectorImpl<SDValue> &Args,
14651 unsigned Start, unsigned Count,
14652 EVT EltVT) {
14653 EVT VT = Op.getValueType();
14654 if (Count == 0)
14655 Count = VT.getVectorNumElements();
14656 if (EltVT == EVT())
14657 EltVT = VT.getVectorElementType();
14658 SDLoc SL(Op);
14659 for (unsigned i = Start, e = Start + Count; i != e; ++i) {
14660 Args.push_back(Elt: getExtractVectorElt(DL: SL, VT: EltVT, Vec: Op, Idx: i));
14661 }
14662}
14663
14664// getAddressSpace - Return the address space this GlobalAddress belongs to.
14665unsigned GlobalAddressSDNode::getAddressSpace() const {
14666 return getGlobal()->getType()->getAddressSpace();
14667}
14668
14669Type *ConstantPoolSDNode::getType() const {
14670 if (isMachineConstantPoolEntry())
14671 return Val.MachineCPVal->getType();
14672 return Val.ConstVal->getType();
14673}
14674
14675bool BuildVectorSDNode::isConstantSplat(APInt &SplatValue, APInt &SplatUndef,
14676 unsigned &SplatBitSize,
14677 bool &HasAnyUndefs,
14678 unsigned MinSplatBits,
14679 bool IsBigEndian) const {
14680 EVT VT = getValueType(ResNo: 0);
14681 assert(VT.isVector() && "Expected a vector type");
14682 unsigned VecWidth = VT.getSizeInBits();
14683 if (MinSplatBits > VecWidth)
14684 return false;
14685
14686 // FIXME: The widths are based on this node's type, but build vectors can
14687 // truncate their operands.
14688 SplatValue = APInt(VecWidth, 0);
14689 SplatUndef = APInt(VecWidth, 0);
14690
14691 // Get the bits. Bits with undefined values (when the corresponding element
14692 // of the vector is an ISD::UNDEF value) are set in SplatUndef and cleared
14693 // in SplatValue. If any of the values are not constant, give up and return
14694 // false.
14695 unsigned int NumOps = getNumOperands();
14696 assert(NumOps > 0 && "isConstantSplat has 0-size build vector");
14697 unsigned EltWidth = VT.getScalarSizeInBits();
14698
14699 for (unsigned j = 0; j < NumOps; ++j) {
14700 unsigned i = IsBigEndian ? NumOps - 1 - j : j;
14701 SDValue OpVal = getOperand(Num: i);
14702 unsigned BitPos = j * EltWidth;
14703
14704 if (OpVal.isUndef())
14705 SplatUndef.setBits(loBit: BitPos, hiBit: BitPos + EltWidth);
14706 else if (auto *CN = dyn_cast<ConstantSDNode>(Val&: OpVal))
14707 SplatValue.insertBits(SubBits: CN->getAPIntValue().zextOrTrunc(width: EltWidth), bitPosition: BitPos);
14708 else if (auto *CN = dyn_cast<ConstantFPSDNode>(Val&: OpVal))
14709 SplatValue.insertBits(SubBits: CN->getValueAPF().bitcastToAPInt(), bitPosition: BitPos);
14710 else
14711 return false;
14712 }
14713
14714 // The build_vector is all constants or undefs. Find the smallest element
14715 // size that splats the vector.
14716 HasAnyUndefs = (SplatUndef != 0);
14717
14718 // FIXME: This does not work for vectors with elements less than 8 bits.
14719 while (VecWidth > 8) {
14720 // If we can't split in half, stop here.
14721 if (VecWidth & 1)
14722 break;
14723
14724 unsigned HalfSize = VecWidth / 2;
14725 APInt HighValue = SplatValue.extractBits(numBits: HalfSize, bitPosition: HalfSize);
14726 APInt LowValue = SplatValue.extractBits(numBits: HalfSize, bitPosition: 0);
14727 APInt HighUndef = SplatUndef.extractBits(numBits: HalfSize, bitPosition: HalfSize);
14728 APInt LowUndef = SplatUndef.extractBits(numBits: HalfSize, bitPosition: 0);
14729
14730 // If the two halves do not match (ignoring undef bits), stop here.
14731 if ((HighValue & ~LowUndef) != (LowValue & ~HighUndef) ||
14732 MinSplatBits > HalfSize)
14733 break;
14734
14735 SplatValue = HighValue | LowValue;
14736 SplatUndef = HighUndef & LowUndef;
14737
14738 VecWidth = HalfSize;
14739 }
14740
14741 // FIXME: The loop above only tries to split in halves. But if the input
14742 // vector for example is <3 x i16> it wouldn't be able to detect a
14743 // SplatBitSize of 16. No idea if that is a design flaw currently limiting
14744 // optimizations. I guess that back in the days when this helper was created
14745 // vectors normally was power-of-2 sized.
14746
14747 SplatBitSize = VecWidth;
14748 return true;
14749}
14750
14751SDValue BuildVectorSDNode::getSplatValue(const APInt &DemandedElts,
14752 BitVector *UndefElements) const {
14753 unsigned NumOps = getNumOperands();
14754 if (UndefElements) {
14755 UndefElements->clear();
14756 UndefElements->resize(N: NumOps);
14757 }
14758 assert(NumOps == DemandedElts.getBitWidth() && "Unexpected vector size");
14759 if (!DemandedElts)
14760 return SDValue();
14761 SDValue Splatted;
14762 for (unsigned i = 0; i != NumOps; ++i) {
14763 if (!DemandedElts[i])
14764 continue;
14765 SDValue Op = getOperand(Num: i);
14766 if (Op.isUndef()) {
14767 if (UndefElements)
14768 (*UndefElements)[i] = true;
14769 } else if (!Splatted) {
14770 Splatted = Op;
14771 } else if (Splatted != Op) {
14772 return SDValue();
14773 }
14774 }
14775
14776 if (!Splatted) {
14777 unsigned FirstDemandedIdx = DemandedElts.countr_zero();
14778 assert(getOperand(FirstDemandedIdx).isUndef() &&
14779 "Can only have a splat without a constant for all undefs.");
14780 return getOperand(Num: FirstDemandedIdx);
14781 }
14782
14783 return Splatted;
14784}
14785
14786SDValue BuildVectorSDNode::getSplatValue(BitVector *UndefElements) const {
14787 APInt DemandedElts = APInt::getAllOnes(numBits: getNumOperands());
14788 return getSplatValue(DemandedElts, UndefElements);
14789}
14790
14791bool BuildVectorSDNode::getRepeatedSequence(const APInt &DemandedElts,
14792 SmallVectorImpl<SDValue> &Sequence,
14793 BitVector *UndefElements) const {
14794 unsigned NumOps = getNumOperands();
14795 Sequence.clear();
14796 if (UndefElements) {
14797 UndefElements->clear();
14798 UndefElements->resize(N: NumOps);
14799 }
14800 assert(NumOps == DemandedElts.getBitWidth() && "Unexpected vector size");
14801 if (!DemandedElts || NumOps < 2 || !isPowerOf2_32(Value: NumOps))
14802 return false;
14803
14804 // Set the undefs even if we don't find a sequence (like getSplatValue).
14805 if (UndefElements)
14806 for (unsigned I = 0; I != NumOps; ++I)
14807 if (DemandedElts[I] && getOperand(Num: I).isUndef())
14808 (*UndefElements)[I] = true;
14809
14810 // Iteratively widen the sequence length looking for repetitions.
14811 for (unsigned SeqLen = 1; SeqLen < NumOps; SeqLen *= 2) {
14812 Sequence.append(NumInputs: SeqLen, Elt: SDValue());
14813 for (unsigned I = 0; I != NumOps; ++I) {
14814 if (!DemandedElts[I])
14815 continue;
14816 SDValue &SeqOp = Sequence[I % SeqLen];
14817 SDValue Op = getOperand(Num: I);
14818 if (Op.isUndef()) {
14819 if (!SeqOp)
14820 SeqOp = Op;
14821 continue;
14822 }
14823 if (SeqOp && !SeqOp.isUndef() && SeqOp != Op) {
14824 Sequence.clear();
14825 break;
14826 }
14827 SeqOp = Op;
14828 }
14829 if (!Sequence.empty())
14830 return true;
14831 }
14832
14833 assert(Sequence.empty() && "Failed to empty non-repeating sequence pattern");
14834 return false;
14835}
14836
14837bool BuildVectorSDNode::getRepeatedSequence(SmallVectorImpl<SDValue> &Sequence,
14838 BitVector *UndefElements) const {
14839 APInt DemandedElts = APInt::getAllOnes(numBits: getNumOperands());
14840 return getRepeatedSequence(DemandedElts, Sequence, UndefElements);
14841}
14842
14843ConstantSDNode *
14844BuildVectorSDNode::getConstantSplatNode(const APInt &DemandedElts,
14845 BitVector *UndefElements) const {
14846 return dyn_cast_or_null<ConstantSDNode>(
14847 Val: getSplatValue(DemandedElts, UndefElements));
14848}
14849
14850ConstantSDNode *
14851BuildVectorSDNode::getConstantSplatNode(BitVector *UndefElements) const {
14852 return dyn_cast_or_null<ConstantSDNode>(Val: getSplatValue(UndefElements));
14853}
14854
14855ConstantFPSDNode *
14856BuildVectorSDNode::getConstantFPSplatNode(const APInt &DemandedElts,
14857 BitVector *UndefElements) const {
14858 return dyn_cast_or_null<ConstantFPSDNode>(
14859 Val: getSplatValue(DemandedElts, UndefElements));
14860}
14861
14862ConstantFPSDNode *
14863BuildVectorSDNode::getConstantFPSplatNode(BitVector *UndefElements) const {
14864 return dyn_cast_or_null<ConstantFPSDNode>(Val: getSplatValue(UndefElements));
14865}
14866
14867int32_t
14868BuildVectorSDNode::getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements,
14869 uint32_t BitWidth) const {
14870 if (ConstantFPSDNode *CN =
14871 dyn_cast_or_null<ConstantFPSDNode>(Val: getSplatValue(UndefElements))) {
14872 bool IsExact;
14873 APSInt IntVal(BitWidth);
14874 const APFloat &APF = CN->getValueAPF();
14875 if (APF.convertToInteger(Result&: IntVal, RM: APFloat::rmTowardZero, IsExact: &IsExact) !=
14876 APFloat::opOK ||
14877 !IsExact)
14878 return -1;
14879
14880 return IntVal.exactLogBase2();
14881 }
14882 return -1;
14883}
14884
14885bool BuildVectorSDNode::getConstantRawBits(
14886 bool IsLittleEndian, unsigned DstEltSizeInBits,
14887 SmallVectorImpl<APInt> &RawBitElements, BitVector &UndefElements) const {
14888 // Early-out if this contains anything but Undef/Constant/ConstantFP.
14889 if (!isConstant())
14890 return false;
14891
14892 unsigned NumSrcOps = getNumOperands();
14893 unsigned SrcEltSizeInBits = getValueType(ResNo: 0).getScalarSizeInBits();
14894 assert(((NumSrcOps * SrcEltSizeInBits) % DstEltSizeInBits) == 0 &&
14895 "Invalid bitcast scale");
14896
14897 // Extract raw src bits.
14898 SmallVector<APInt> SrcBitElements(NumSrcOps,
14899 APInt::getZero(numBits: SrcEltSizeInBits));
14900 BitVector SrcUndeElements(NumSrcOps, false);
14901
14902 for (unsigned I = 0; I != NumSrcOps; ++I) {
14903 SDValue Op = getOperand(Num: I);
14904 if (Op.isUndef()) {
14905 SrcUndeElements.set(I);
14906 continue;
14907 }
14908 auto *CInt = dyn_cast<ConstantSDNode>(Val&: Op);
14909 auto *CFP = dyn_cast<ConstantFPSDNode>(Val&: Op);
14910 assert((CInt || CFP) && "Unknown constant");
14911 SrcBitElements[I] = CInt ? CInt->getAPIntValue().trunc(width: SrcEltSizeInBits)
14912 : CFP->getValueAPF().bitcastToAPInt();
14913 }
14914
14915 // Recast to dst width.
14916 recastRawBits(IsLittleEndian, DstEltSizeInBits, DstBitElements&: RawBitElements,
14917 SrcBitElements, DstUndefElements&: UndefElements, SrcUndefElements: SrcUndeElements);
14918 return true;
14919}
14920
14921void BuildVectorSDNode::recastRawBits(bool IsLittleEndian,
14922 unsigned DstEltSizeInBits,
14923 SmallVectorImpl<APInt> &DstBitElements,
14924 ArrayRef<APInt> SrcBitElements,
14925 BitVector &DstUndefElements,
14926 const BitVector &SrcUndefElements) {
14927 unsigned NumSrcOps = SrcBitElements.size();
14928 unsigned SrcEltSizeInBits = SrcBitElements[0].getBitWidth();
14929 assert(((NumSrcOps * SrcEltSizeInBits) % DstEltSizeInBits) == 0 &&
14930 "Invalid bitcast scale");
14931 assert(NumSrcOps == SrcUndefElements.size() &&
14932 "Vector size mismatch");
14933
14934 unsigned NumDstOps = (NumSrcOps * SrcEltSizeInBits) / DstEltSizeInBits;
14935 DstUndefElements.clear();
14936 DstUndefElements.resize(N: NumDstOps, t: false);
14937 DstBitElements.assign(NumElts: NumDstOps, Elt: APInt::getZero(numBits: DstEltSizeInBits));
14938
14939 // Concatenate src elements constant bits together into dst element.
14940 if (SrcEltSizeInBits <= DstEltSizeInBits) {
14941 unsigned Scale = DstEltSizeInBits / SrcEltSizeInBits;
14942 for (unsigned I = 0; I != NumDstOps; ++I) {
14943 DstUndefElements.set(I);
14944 APInt &DstBits = DstBitElements[I];
14945 for (unsigned J = 0; J != Scale; ++J) {
14946 unsigned Idx = (I * Scale) + (IsLittleEndian ? J : (Scale - J - 1));
14947 if (SrcUndefElements[Idx])
14948 continue;
14949 DstUndefElements.reset(Idx: I);
14950 const APInt &SrcBits = SrcBitElements[Idx];
14951 assert(SrcBits.getBitWidth() == SrcEltSizeInBits &&
14952 "Illegal constant bitwidths");
14953 DstBits.insertBits(SubBits: SrcBits, bitPosition: J * SrcEltSizeInBits);
14954 }
14955 }
14956 return;
14957 }
14958
14959 // Split src element constant bits into dst elements.
14960 unsigned Scale = SrcEltSizeInBits / DstEltSizeInBits;
14961 for (unsigned I = 0; I != NumSrcOps; ++I) {
14962 if (SrcUndefElements[I]) {
14963 DstUndefElements.set(I: I * Scale, E: (I + 1) * Scale);
14964 continue;
14965 }
14966 const APInt &SrcBits = SrcBitElements[I];
14967 for (unsigned J = 0; J != Scale; ++J) {
14968 unsigned Idx = (I * Scale) + (IsLittleEndian ? J : (Scale - J - 1));
14969 APInt &DstBits = DstBitElements[Idx];
14970 DstBits = SrcBits.extractBits(numBits: DstEltSizeInBits, bitPosition: J * DstEltSizeInBits);
14971 }
14972 }
14973}
14974
14975bool BuildVectorSDNode::isConstant() const {
14976 for (const SDValue &Op : op_values()) {
14977 unsigned Opc = Op.getOpcode();
14978 if (!Op.isUndef() && Opc != ISD::Constant && Opc != ISD::ConstantFP)
14979 return false;
14980 }
14981 return true;
14982}
14983
14984std::optional<std::pair<APInt, APInt>>
14985BuildVectorSDNode::isArithmeticSequence() const {
14986 unsigned NumOps = getNumOperands();
14987 if (NumOps < 2)
14988 return std::nullopt;
14989
14990 unsigned EltSize = getValueType(ResNo: 0).getScalarSizeInBits();
14991 APInt Start, Stride;
14992 int FirstIdx = -1, SecondIdx = -1;
14993
14994 // Find the first two non-undef constant elements to determine Start and
14995 // Stride, then verify all remaining elements match the sequence.
14996 for (unsigned I = 0; I < NumOps; ++I) {
14997 SDValue Op = getOperand(Num: I);
14998 if (Op->isUndef())
14999 continue;
15000 if (!isa<ConstantSDNode>(Val: Op))
15001 return std::nullopt;
15002
15003 APInt Val = getConstantOperandAPInt(Num: I).trunc(width: EltSize);
15004 if (FirstIdx < 0) {
15005 FirstIdx = I;
15006 Start = Val;
15007 } else if (SecondIdx < 0) {
15008 SecondIdx = I;
15009 // Compute stride using modular arithmetic. Simple division would handle
15010 // common strides (1, 2, -1, etc.), but modular inverse maximizes matches.
15011 // Example: <0, poison, poison, 0xFF> has stride 0x55 since 3*0x55 = 0xFF
15012 // Note that modular arithmetic is agnostic to signed/unsigned.
15013 unsigned IdxDiff = I - FirstIdx;
15014 APInt ValDiff = Val - Start;
15015
15016 // Step 1: Factor out common powers of 2 from IdxDiff and ValDiff.
15017 unsigned CommonPow2Bits = llvm::countr_zero(Val: IdxDiff);
15018 if (ValDiff.countr_zero() < CommonPow2Bits)
15019 return std::nullopt; // ValDiff not divisible by 2^CommonPow2Bits
15020 IdxDiff >>= CommonPow2Bits;
15021 ValDiff.lshrInPlace(ShiftAmt: CommonPow2Bits);
15022
15023 // Step 2: IdxDiff is now odd, so its inverse mod 2^EltSize exists.
15024 // TODO: There are 2^CommonPow2Bits valid strides; currently we only try
15025 // one, but we could try all candidates to handle more cases.
15026 Stride = ValDiff * APInt(EltSize, IdxDiff).multiplicativeInverse();
15027 if (Stride.isZero())
15028 return std::nullopt;
15029
15030 // Step 3: Adjust Start based on the first defined element's index.
15031 Start -= Stride * FirstIdx;
15032 } else {
15033 // Verify this element matches the sequence.
15034 if (Val != Start + Stride * I)
15035 return std::nullopt;
15036 }
15037 }
15038
15039 // Need at least two defined elements.
15040 if (SecondIdx < 0)
15041 return std::nullopt;
15042
15043 return std::make_pair(x&: Start, y&: Stride);
15044}
15045
15046bool ShuffleVectorSDNode::isSplatMask(ArrayRef<int> Mask) {
15047 // Find the first non-undef value in the shuffle mask.
15048 unsigned i, e;
15049 for (i = 0, e = Mask.size(); i != e && Mask[i] < 0; ++i)
15050 /* search */;
15051
15052 // If all elements are undefined, this shuffle can be considered a splat
15053 // (although it should eventually get simplified away completely).
15054 if (i == e)
15055 return true;
15056
15057 // Make sure all remaining elements are either undef or the same as the first
15058 // non-undef value.
15059 for (int Idx = Mask[i]; i != e; ++i)
15060 if (Mask[i] >= 0 && Mask[i] != Idx)
15061 return false;
15062 return true;
15063}
15064
15065// Returns true if it is a constant integer BuildVector or constant integer,
15066// possibly hidden by a bitcast.
15067bool SelectionDAG::isConstantIntBuildVectorOrConstantInt(
15068 SDValue N, bool AllowOpaques) const {
15069 N = peekThroughBitcasts(V: N);
15070
15071 if (auto *C = dyn_cast<ConstantSDNode>(Val&: N))
15072 return AllowOpaques || !C->isOpaque();
15073
15074 if (ISD::isBuildVectorOfConstantSDNodes(N: N.getNode()))
15075 return true;
15076
15077 // Treat a GlobalAddress supporting constant offset folding as a
15078 // constant integer.
15079 if (auto *GA = dyn_cast<GlobalAddressSDNode>(Val&: N))
15080 if (GA->getOpcode() == ISD::GlobalAddress &&
15081 TLI->isOffsetFoldingLegal(GA))
15082 return true;
15083
15084 if ((N.getOpcode() == ISD::SPLAT_VECTOR) &&
15085 isa<ConstantSDNode>(Val: N.getOperand(i: 0)))
15086 return true;
15087 return false;
15088}
15089
15090// Returns true if it is a constant float BuildVector or constant float.
15091bool SelectionDAG::isConstantFPBuildVectorOrConstantFP(SDValue N) const {
15092 if (isa<ConstantFPSDNode>(Val: N))
15093 return true;
15094
15095 if (ISD::isBuildVectorOfConstantFPSDNodes(N: N.getNode()))
15096 return true;
15097
15098 if ((N.getOpcode() == ISD::SPLAT_VECTOR) &&
15099 isa<ConstantFPSDNode>(Val: N.getOperand(i: 0)))
15100 return true;
15101
15102 return false;
15103}
15104
15105std::optional<bool> SelectionDAG::isBoolConstant(SDValue N) const {
15106 ConstantSDNode *Const =
15107 isConstOrConstSplat(N, AllowUndefs: false, /*AllowTruncation=*/true);
15108 if (!Const)
15109 return std::nullopt;
15110
15111 EVT VT = N->getValueType(ResNo: 0);
15112 const APInt CVal = Const->getAPIntValue().trunc(width: VT.getScalarSizeInBits());
15113 switch (TLI->getBooleanContents(Type: N.getValueType())) {
15114 case TargetLowering::ZeroOrOneBooleanContent:
15115 if (CVal.isOne())
15116 return true;
15117 if (CVal.isZero())
15118 return false;
15119 return std::nullopt;
15120 case TargetLowering::ZeroOrNegativeOneBooleanContent:
15121 if (CVal.isAllOnes())
15122 return true;
15123 if (CVal.isZero())
15124 return false;
15125 return std::nullopt;
15126 case TargetLowering::UndefinedBooleanContent:
15127 return CVal[0];
15128 }
15129 llvm_unreachable("Unknown BooleanContent enum");
15130}
15131
15132void SelectionDAG::createOperands(SDNode *Node, ArrayRef<SDValue> Vals) {
15133 assert(!Node->OperandList && "Node already has operands");
15134 assert(SDNode::getMaxNumOperands() >= Vals.size() &&
15135 "too many operands to fit into SDNode");
15136 SDUse *Ops = OperandRecycler.allocate(
15137 Cap: ArrayRecycler<SDUse>::Capacity::get(N: Vals.size()), Allocator&: OperandAllocator);
15138
15139 bool IsDivergent = false;
15140 for (unsigned I = 0; I != Vals.size(); ++I) {
15141 Ops[I].setUser(Node);
15142 Ops[I].setInitial(Vals[I]);
15143 EVT VT = Ops[I].getValueType();
15144
15145 // Skip Chain. It does not carry divergence.
15146 if (VT != MVT::Other &&
15147 (VT != MVT::Glue || gluePropagatesDivergence(Node: Ops[I].getNode())) &&
15148 Ops[I].getNode()->isDivergent()) {
15149 IsDivergent = true;
15150 }
15151 }
15152 Node->NumOperands = Vals.size();
15153 Node->OperandList = Ops;
15154 if (!TLI->isSDNodeAlwaysUniform(N: Node)) {
15155 IsDivergent |= TLI->isSDNodeSourceOfDivergence(N: Node, FLI, UA);
15156 Node->SDNodeBits.IsDivergent = IsDivergent;
15157 }
15158 checkForCycles(N: Node);
15159}
15160
15161SDValue SelectionDAG::getTokenFactor(const SDLoc &DL,
15162 SmallVectorImpl<SDValue> &Vals) {
15163 size_t Limit = SDNode::getMaxNumOperands();
15164 while (Vals.size() > Limit) {
15165 unsigned SliceIdx = Vals.size() - Limit;
15166 auto ExtractedTFs = ArrayRef<SDValue>(Vals).slice(N: SliceIdx, M: Limit);
15167 SDValue NewTF = getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: ExtractedTFs);
15168 Vals.erase(CS: Vals.begin() + SliceIdx, CE: Vals.end());
15169 Vals.emplace_back(Args&: NewTF);
15170 }
15171 return getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: Vals);
15172}
15173
15174SDValue SelectionDAG::getIdentityElement(unsigned Opcode, const SDLoc &DL,
15175 EVT VT, SDNodeFlags Flags) {
15176 switch (Opcode) {
15177 default:
15178 return SDValue();
15179 case ISD::ADD:
15180 case ISD::OR:
15181 case ISD::XOR:
15182 case ISD::UMAX:
15183 case ISD::MUL:
15184 case ISD::AND:
15185 case ISD::UMIN:
15186 case ISD::SMAX:
15187 case ISD::SMIN:
15188 return getConstant(Val: getIntegerIdentity(Opcode, BitWidth: VT.getScalarSizeInBits()), DL,
15189 VT);
15190 case ISD::FADD:
15191 // If flags allow, prefer positive zero since it's generally cheaper
15192 // to materialize on most targets.
15193 return getConstantFP(Val: Flags.hasNoSignedZeros() ? 0.0 : -0.0, DL, VT);
15194 case ISD::FMUL:
15195 return getConstantFP(Val: 1.0, DL, VT);
15196 case ISD::FMINNUM:
15197 case ISD::FMAXNUM:
15198 case ISD::FMINIMUMNUM:
15199 case ISD::FMAXIMUMNUM: {
15200 // Neutral element for fminnum/fminimumnum is NaN, Inf or FLT_MAX,
15201 // depending on fast-math flags (FMF).
15202 const fltSemantics &Semantics = VT.getFltSemantics();
15203 APFloat NeutralAF = !Flags.hasNoNaNs() ? APFloat::getQNaN(Sem: Semantics) :
15204 !Flags.hasNoInfs() ? APFloat::getInf(Sem: Semantics) :
15205 APFloat::getLargest(Sem: Semantics);
15206 if (Opcode == ISD::FMAXNUM || Opcode == ISD::FMAXIMUMNUM)
15207 NeutralAF.changeSign();
15208
15209 return getConstantFP(V: NeutralAF, DL, VT);
15210 }
15211 case ISD::FMINIMUM:
15212 case ISD::FMAXIMUM: {
15213 // Neutral element for fminimum is Inf or FLT_MAX, depending on FMF.
15214 const fltSemantics &Semantics = VT.getFltSemantics();
15215 APFloat NeutralAF = !Flags.hasNoInfs() ? APFloat::getInf(Sem: Semantics)
15216 : APFloat::getLargest(Sem: Semantics);
15217 if (Opcode == ISD::FMAXIMUM)
15218 NeutralAF.changeSign();
15219
15220 return getConstantFP(V: NeutralAF, DL, VT);
15221 }
15222
15223 }
15224}
15225
15226SDValue SelectionDAG::getPartialReduceMLS(unsigned Opc, const SDLoc &DL,
15227 SDValue Acc, SDValue LHS,
15228 SDValue RHS) {
15229 EVT AccVT = Acc.getValueType();
15230 if (AccVT.isFloatingPoint()) {
15231 assert(Opc == ISD::PARTIAL_REDUCE_FMLA && "Unexpected opcode");
15232 SDValue NegRHS = getNode(Opcode: ISD::FNEG, DL, VT: RHS.getValueType(), N1: RHS);
15233 return getNode(Opcode: Opc, DL, VT: AccVT, N1: Acc, N2: LHS, N3: NegRHS);
15234 }
15235 assert((Opc == ISD::PARTIAL_REDUCE_UMLA || Opc == ISD::PARTIAL_REDUCE_SMLA ||
15236 Opc == ISD::PARTIAL_REDUCE_SUMLA) &&
15237 "Unexpected opcode");
15238 SDValue NegAcc = getNegative(Val: Acc, DL, VT: AccVT);
15239 SDValue MLA = getNode(Opcode: Opc, DL, VT: AccVT, N1: NegAcc, N2: LHS, N3: RHS);
15240 return getNegative(Val: MLA, DL, VT: AccVT);
15241}
15242
15243/// Helper used to make a call to a library function that has one argument of
15244/// pointer type.
15245///
15246/// Such functions include 'fegetmode', 'fesetenv' and some others, which are
15247/// used to get or set floating-point state. They have one argument of pointer
15248/// type, which points to the memory region containing bits of the
15249/// floating-point state. The value returned by such function is ignored in the
15250/// created call.
15251///
15252/// \param LibFunc Reference to library function (value of RTLIB::Libcall).
15253/// \param Ptr Pointer used to save/load state.
15254/// \param InChain Ingoing token chain.
15255/// \param Node Node being legalized
15256/// \returns Outgoing chain token.
15257SDValue SelectionDAG::makeStateFunctionCall(unsigned LibFunc, SDValue Ptr,
15258 SDValue InChain, SDNode *Node) {
15259 assert(InChain.getValueType() == MVT::Other && "Expected token chain");
15260 RTLIB::LibcallImpl LibcallImpl =
15261 Libcalls->getLibcallImpl(Call: static_cast<RTLIB::Libcall>(LibFunc));
15262 if (LibcallImpl == RTLIB::Unsupported) {
15263 getContext()->emitError(ErrorStr: Twine("no libcall available for ") +
15264 Node->getOperationName(G: this));
15265 return InChain;
15266 }
15267
15268 TargetLowering::ArgListTy Args;
15269 Args.emplace_back(args&: Ptr, args: Ptr.getValueType().getTypeForEVT(Context&: *getContext()));
15270 SDValue Callee =
15271 getExternalSymbol(Libcall: LibcallImpl, VT: TLI->getPointerTy(DL: getDataLayout()));
15272 TargetLowering::CallLoweringInfo CLI(*this);
15273 CLI.setDebugLoc(SDLoc(Node))
15274 .setChain(InChain)
15275 .setLibCallee(CC: Libcalls->getLibcallImplCallingConv(Call: LibcallImpl),
15276 ResultType: Type::getVoidTy(C&: *getContext()), Target: Callee, ArgsList: std::move(Args));
15277 return TLI->LowerCallTo(CLI).second;
15278}
15279
15280void SelectionDAG::copyExtraInfo(SDNode *From, SDNode *To) {
15281 assert(From && To && "Invalid SDNode; empty source SDValue?");
15282 auto I = SDEI.find(Val: From);
15283 if (I == SDEI.end())
15284 return;
15285
15286 // Use of operator[] on the DenseMap may cause an insertion, which invalidates
15287 // the iterator, hence the need to make a copy to prevent a use-after-free.
15288 NodeExtraInfo NEI = I->second;
15289 if (LLVM_LIKELY(!NEI.PCSections)) {
15290 // No deep copy required for the types of extra info set.
15291 //
15292 // FIXME: Investigate if other types of extra info also need deep copy. This
15293 // depends on the types of nodes they can be attached to: if some extra info
15294 // is only ever attached to nodes where a replacement To node is always the
15295 // node where later use and propagation of the extra info has the intended
15296 // semantics, no deep copy is required.
15297 SDEI[To] = std::move(NEI);
15298 return;
15299 }
15300
15301 const SDNode *EntrySDN = getEntryNode().getNode();
15302
15303 // We need to copy NodeExtraInfo to all _new_ nodes that are being introduced
15304 // through the replacement of From with To. Otherwise, replacements of a node
15305 // (From) with more complex nodes (To and its operands) may result in lost
15306 // extra info where the root node (To) is insignificant in further propagating
15307 // and using extra info when further lowering to MIR.
15308 //
15309 // In the first step pre-populate the visited set with the nodes reachable
15310 // from the old From node. This avoids copying NodeExtraInfo to parts of the
15311 // DAG that is not new and should be left untouched.
15312 SmallVector<const SDNode *> Leafs{From}; // Leafs reachable with VisitFrom.
15313 DenseSet<const SDNode *> FromReach; // The set of nodes reachable from From.
15314 auto VisitFrom = [&](auto &&Self, const SDNode *N, int MaxDepth) {
15315 if (MaxDepth == 0) {
15316 // Remember this node in case we need to increase MaxDepth and continue
15317 // populating FromReach from this node.
15318 Leafs.emplace_back(Args&: N);
15319 return;
15320 }
15321 if (!FromReach.insert(V: N).second)
15322 return;
15323 for (const SDValue &Op : N->op_values())
15324 Self(Self, Op.getNode(), MaxDepth - 1);
15325 };
15326
15327 // Copy extra info to To and all its transitive operands (that are new).
15328 SmallPtrSet<const SDNode *, 8> Visited;
15329 auto DeepCopyTo = [&](auto &&Self, const SDNode *N) {
15330 if (FromReach.contains(V: N))
15331 return true;
15332 if (!Visited.insert(Ptr: N).second)
15333 return true;
15334 if (EntrySDN == N)
15335 return false;
15336 for (const SDValue &Op : N->op_values()) {
15337 if (N == To && Op.getNode() == EntrySDN) {
15338 // Special case: New node's operand is the entry node; just need to
15339 // copy extra info to new node.
15340 break;
15341 }
15342 if (!Self(Self, Op.getNode()))
15343 return false;
15344 }
15345 // Copy only if entry node was not reached.
15346 SDEI[N] = std::move(NEI);
15347 return true;
15348 };
15349
15350 // We first try with a lower MaxDepth, assuming that the path to common
15351 // operands between From and To is relatively short. This significantly
15352 // improves performance in the common case. The initial MaxDepth is big
15353 // enough to avoid retry in the common case; the last MaxDepth is large
15354 // enough to avoid having to use the fallback below (and protects from
15355 // potential stack exhaustion from recursion).
15356 for (int PrevDepth = 0, MaxDepth = 16; MaxDepth <= 1024;
15357 PrevDepth = MaxDepth, MaxDepth *= 2, Visited.clear()) {
15358 // StartFrom is the previous (or initial) set of leafs reachable at the
15359 // previous maximum depth.
15360 SmallVector<const SDNode *> StartFrom;
15361 std::swap(LHS&: StartFrom, RHS&: Leafs);
15362 for (const SDNode *N : StartFrom)
15363 VisitFrom(VisitFrom, N, MaxDepth - PrevDepth);
15364 if (LLVM_LIKELY(DeepCopyTo(DeepCopyTo, To)))
15365 return;
15366 // This should happen very rarely (reached the entry node).
15367 LLVM_DEBUG(dbgs() << __func__ << ": MaxDepth=" << MaxDepth << " too low\n");
15368 assert(!Leafs.empty());
15369 }
15370
15371 // This should not happen - but if it did, that means the subgraph reachable
15372 // from From has depth greater or equal to maximum MaxDepth, and VisitFrom()
15373 // could not visit all reachable common operands. Consequently, we were able
15374 // to reach the entry node.
15375 errs() << "warning: incomplete propagation of SelectionDAG::NodeExtraInfo\n";
15376 assert(false && "From subgraph too complex - increase max. MaxDepth?");
15377 // Best-effort fallback if assertions disabled.
15378 SDEI[To] = std::move(NEI);
15379}
15380
15381unsigned SelectionDAG::getMaxRuntimeNumElements(EVT VT) const {
15382 assert(VT.isVector() && "Can only unroll vector types!");
15383 if (VT.isFixedLengthVector())
15384 return VT.getVectorNumElements();
15385
15386 const MachineFunction &MF = getMachineFunction();
15387 const Function &F = MF.getFunction();
15388
15389 APInt MaxVScale = getVScaleRange(F: &F, BitWidth: sizeof(unsigned) * 8).getUnsignedMax();
15390 if (MaxVScale.ugt(RHS: VScaleUnrollLimit))
15391 return 0;
15392
15393 bool Overflow;
15394 APInt MinNElts(sizeof(unsigned) * 8, VT.getVectorMinNumElements());
15395 APInt MaxNElts = MinNElts.umul_ov(RHS: MaxVScale, Overflow);
15396 if (Overflow)
15397 return 0;
15398
15399 return MaxNElts.getZExtValue();
15400}
15401
15402SDValue SelectionDAG::buildVectorFromUnrolledParts(EVT VT, const SDLoc &DL,
15403 ArrayRef<SDValue> Scalars) {
15404 assert(Scalars.size() == getMaxRuntimeNumElements(VT) &&
15405 "Element count mismatch!");
15406 if (VT.isFixedLengthVector())
15407 return getBuildVector(VT, DL, Ops: Scalars);
15408
15409 SDValue Vec = getPOISON(VT);
15410 // Iterate in reverse so result remains poison until we encounter a lane that
15411 // exists, after which all lower-numbered lanes must also exist.
15412 for (unsigned IdxVal : reverse(C: seq(Size: Scalars.size())))
15413 Vec = getInsertVectorElt(DL, Vec, Elt: Scalars[IdxVal], Idx: IdxVal);
15414
15415 return Vec;
15416}
15417
15418#ifndef NDEBUG
15419static void checkForCyclesHelper(const SDNode *N,
15420 SmallPtrSetImpl<const SDNode*> &Visited,
15421 SmallPtrSetImpl<const SDNode*> &Checked,
15422 const llvm::SelectionDAG *DAG) {
15423 // If this node has already been checked, don't check it again.
15424 if (Checked.count(N))
15425 return;
15426
15427 // If a node has already been visited on this depth-first walk, reject it as
15428 // a cycle.
15429 if (!Visited.insert(N).second) {
15430 errs() << "Detected cycle in SelectionDAG\n";
15431 dbgs() << "Offending node:\n";
15432 N->dumprFull(DAG); dbgs() << "\n";
15433 abort();
15434 }
15435
15436 for (const SDValue &Op : N->op_values())
15437 checkForCyclesHelper(Op.getNode(), Visited, Checked, DAG);
15438
15439 Checked.insert(N);
15440 Visited.erase(N);
15441}
15442#endif
15443
15444void llvm::checkForCycles(const llvm::SDNode *N,
15445 const llvm::SelectionDAG *DAG,
15446 bool force) {
15447#ifndef NDEBUG
15448 bool check = force;
15449#ifdef EXPENSIVE_CHECKS
15450 check = true;
15451#endif // EXPENSIVE_CHECKS
15452 if (check) {
15453 assert(N && "Checking nonexistent SDNode");
15454 SmallPtrSet<const SDNode*, 32> visited;
15455 SmallPtrSet<const SDNode*, 32> checked;
15456 checkForCyclesHelper(N, visited, checked, DAG);
15457 }
15458#endif // !NDEBUG
15459}
15460
15461void llvm::checkForCycles(const llvm::SelectionDAG *DAG, bool force) {
15462 checkForCycles(N: DAG->getRoot().getNode(), DAG, force);
15463}
15464