1//===- SelectionDAG.cpp - Implement the SelectionDAG data structures ------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements the SelectionDAG class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/CodeGen/SelectionDAG.h"
14#include "SDNodeDbgValue.h"
15#include "llvm/ADT/APFloat.h"
16#include "llvm/ADT/APInt.h"
17#include "llvm/ADT/APSInt.h"
18#include "llvm/ADT/ArrayRef.h"
19#include "llvm/ADT/BitVector.h"
20#include "llvm/ADT/DenseSet.h"
21#include "llvm/ADT/FoldingSet.h"
22#include "llvm/ADT/STLExtras.h"
23#include "llvm/ADT/SmallPtrSet.h"
24#include "llvm/ADT/SmallVector.h"
25#include "llvm/ADT/Twine.h"
26#include "llvm/Analysis/AliasAnalysis.h"
27#include "llvm/Analysis/MemoryLocation.h"
28#include "llvm/Analysis/TargetLibraryInfo.h"
29#include "llvm/Analysis/ValueTracking.h"
30#include "llvm/Analysis/VectorUtils.h"
31#include "llvm/BinaryFormat/Dwarf.h"
32#include "llvm/CodeGen/Analysis.h"
33#include "llvm/CodeGen/CodeGenCommonISel.h"
34#include "llvm/CodeGen/FunctionLoweringInfo.h"
35#include "llvm/CodeGen/ISDOpcodes.h"
36#include "llvm/CodeGen/MachineBasicBlock.h"
37#include "llvm/CodeGen/MachineConstantPool.h"
38#include "llvm/CodeGen/MachineFrameInfo.h"
39#include "llvm/CodeGen/MachineFunction.h"
40#include "llvm/CodeGen/MachineMemOperand.h"
41#include "llvm/CodeGen/RuntimeLibcallUtil.h"
42#include "llvm/CodeGen/SDPatternMatch.h"
43#include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
44#include "llvm/CodeGen/SelectionDAGNodes.h"
45#include "llvm/CodeGen/SelectionDAGTargetInfo.h"
46#include "llvm/CodeGen/TargetFrameLowering.h"
47#include "llvm/CodeGen/TargetLowering.h"
48#include "llvm/CodeGen/TargetRegisterInfo.h"
49#include "llvm/CodeGen/TargetSubtargetInfo.h"
50#include "llvm/CodeGen/ValueTypes.h"
51#include "llvm/CodeGenTypes/MachineValueType.h"
52#include "llvm/IR/Constant.h"
53#include "llvm/IR/Constants.h"
54#include "llvm/IR/DataLayout.h"
55#include "llvm/IR/DebugInfoMetadata.h"
56#include "llvm/IR/DebugLoc.h"
57#include "llvm/IR/DerivedTypes.h"
58#include "llvm/IR/Function.h"
59#include "llvm/IR/GlobalValue.h"
60#include "llvm/IR/Metadata.h"
61#include "llvm/IR/Type.h"
62#include "llvm/Support/Casting.h"
63#include "llvm/Support/CodeGen.h"
64#include "llvm/Support/Compiler.h"
65#include "llvm/Support/Debug.h"
66#include "llvm/Support/ErrorHandling.h"
67#include "llvm/Support/KnownBits.h"
68#include "llvm/Support/KnownFPClass.h"
69#include "llvm/Support/MathExtras.h"
70#include "llvm/Support/raw_ostream.h"
71#include "llvm/Target/TargetMachine.h"
72#include "llvm/Target/TargetOptions.h"
73#include "llvm/TargetParser/Triple.h"
74#include "llvm/Transforms/Utils/SizeOpts.h"
75#include <algorithm>
76#include <cassert>
77#include <cstdint>
78#include <cstdlib>
79#include <limits>
80#include <optional>
81#include <string>
82#include <utility>
83#include <vector>
84
85using namespace llvm;
86using namespace llvm::SDPatternMatch;
87
88/// makeVTList - Return an instance of the SDVTList struct initialized with the
89/// specified members.
90static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) {
91 SDVTList Res = {.VTs: VTs, .NumVTs: NumVTs};
92 return Res;
93}
94
95// Default null implementations of the callbacks.
96void SelectionDAG::DAGUpdateListener::NodeDeleted(SDNode*, SDNode*) {}
97void SelectionDAG::DAGUpdateListener::NodeUpdated(SDNode*) {}
98void SelectionDAG::DAGUpdateListener::NodeInserted(SDNode *) {}
99
100void SelectionDAG::DAGNodeDeletedListener::anchor() {}
101void SelectionDAG::DAGNodeInsertedListener::anchor() {}
102
103#define DEBUG_TYPE "selectiondag"
104
105static cl::opt<bool> EnableMemCpyDAGOpt("enable-memcpy-dag-opt",
106 cl::Hidden, cl::init(Val: true),
107 cl::desc("Gang up loads and stores generated by inlining of memcpy"));
108
109static cl::opt<int> MaxLdStGlue("ldstmemcpy-glue-max",
110 cl::desc("Number limit for gluing ld/st of memcpy."),
111 cl::Hidden, cl::init(Val: 0));
112
113static cl::opt<unsigned>
114 MaxSteps("has-predecessor-max-steps", cl::Hidden, cl::init(Val: 8192),
115 cl::desc("DAG combiner limit number of steps when searching DAG "
116 "for predecessor nodes"));
117
118static cl::opt<int> VScaleUnrollLimit(
119 "vscale-unroll-limit",
120 cl::desc("Maximum vscale for which vector unrolling is allowed."),
121 cl::Hidden, cl::init(Val: 64));
122
123static void NewSDValueDbgMsg(SDValue V, StringRef Msg, SelectionDAG *G) {
124 LLVM_DEBUG(dbgs() << Msg; V.getNode()->dump(G););
125}
126
127unsigned SelectionDAG::getHasPredecessorMaxSteps() { return MaxSteps; }
128
129//===----------------------------------------------------------------------===//
130// ConstantFPSDNode Class
131//===----------------------------------------------------------------------===//
132
133/// isExactlyValue - We don't rely on operator== working on double values, as
134/// it returns true for things that are clearly not equal, like -0.0 and 0.0.
135/// As such, this method can be used to do an exact bit-for-bit comparison of
136/// two floating point values.
137bool ConstantFPSDNode::isExactlyValue(const APFloat& V) const {
138 return getValueAPF().bitwiseIsEqual(RHS: V);
139}
140
141bool ConstantFPSDNode::isValueValidForType(EVT VT,
142 const APFloat& Val) {
143 assert(VT.isFloatingPoint() && "Can only convert between FP types");
144
145 // convert modifies in place, so make a copy.
146 APFloat Val2 = APFloat(Val);
147 bool losesInfo;
148 (void)Val2.convert(ToSemantics: VT.getFltSemantics(), RM: APFloat::rmNearestTiesToEven,
149 losesInfo: &losesInfo);
150 return !losesInfo;
151}
152
153//===----------------------------------------------------------------------===//
154// ISD Namespace
155//===----------------------------------------------------------------------===//
156
157bool ISD::isConstantSplatVector(const SDNode *N, APInt &SplatVal) {
158 if (N->getOpcode() == ISD::SPLAT_VECTOR) {
159 if (auto OptAPInt = N->getOperand(Num: 0)->bitcastToAPInt()) {
160 unsigned EltSize =
161 N->getValueType(ResNo: 0).getVectorElementType().getSizeInBits();
162 SplatVal = OptAPInt->trunc(width: EltSize);
163 return true;
164 }
165 }
166
167 auto *BV = dyn_cast<BuildVectorSDNode>(Val: N);
168 if (!BV)
169 return false;
170
171 APInt SplatUndef;
172 unsigned SplatBitSize;
173 bool HasUndefs;
174 unsigned EltSize = N->getValueType(ResNo: 0).getVectorElementType().getSizeInBits();
175 // Endianness does not matter here. We are checking for a splat given the
176 // element size of the vector, and if we find such a splat for little endian
177 // layout, then that should be valid also for big endian (as the full vector
178 // size is known to be a multiple of the element size).
179 const bool IsBigEndian = false;
180 return BV->isConstantSplat(SplatValue&: SplatVal, SplatUndef, SplatBitSize, HasAnyUndefs&: HasUndefs,
181 MinSplatBits: EltSize, isBigEndian: IsBigEndian) &&
182 EltSize == SplatBitSize;
183}
184
185// FIXME: AllOnes and AllZeros duplicate a lot of code. Could these be
186// specializations of the more general isConstantSplatVector()?
187
188bool ISD::isConstantSplatVectorAllOnes(const SDNode *N, bool BuildVectorOnly) {
189 // Look through a bit convert.
190 while (N->getOpcode() == ISD::BITCAST)
191 N = N->getOperand(Num: 0).getNode();
192
193 if (!BuildVectorOnly && N->getOpcode() == ISD::SPLAT_VECTOR) {
194 APInt SplatVal;
195 return isConstantSplatVector(N, SplatVal) && SplatVal.isAllOnes();
196 }
197
198 if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
199
200 unsigned i = 0, e = N->getNumOperands();
201
202 // Skip over all of the undef values.
203 while (i != e && N->getOperand(Num: i).isUndef())
204 ++i;
205
206 // Do not accept an all-undef vector.
207 if (i == e) return false;
208
209 // Do not accept build_vectors that aren't all constants or which have non-~0
210 // elements. We have to be a bit careful here, as the type of the constant
211 // may not be the same as the type of the vector elements due to type
212 // legalization (the elements are promoted to a legal type for the target and
213 // a vector of a type may be legal when the base element type is not).
214 // We only want to check enough bits to cover the vector elements, because
215 // we care if the resultant vector is all ones, not whether the individual
216 // constants are.
217 SDValue NotZero = N->getOperand(Num: i);
218 if (auto OptAPInt = NotZero->bitcastToAPInt()) {
219 unsigned EltSize = N->getValueType(ResNo: 0).getScalarSizeInBits();
220 if (OptAPInt->countr_one() < EltSize)
221 return false;
222 } else
223 return false;
224
225 // Okay, we have at least one ~0 value, check to see if the rest match or are
226 // undefs. Even with the above element type twiddling, this should be OK, as
227 // the same type legalization should have applied to all the elements.
228 for (++i; i != e; ++i)
229 if (N->getOperand(Num: i) != NotZero && !N->getOperand(Num: i).isUndef())
230 return false;
231 return true;
232}
233
234bool ISD::isConstantSplatVectorAllZeros(const SDNode *N, bool BuildVectorOnly) {
235 // Look through a bit convert.
236 while (N->getOpcode() == ISD::BITCAST)
237 N = N->getOperand(Num: 0).getNode();
238
239 if (!BuildVectorOnly && N->getOpcode() == ISD::SPLAT_VECTOR) {
240 APInt SplatVal;
241 return isConstantSplatVector(N, SplatVal) && SplatVal.isZero();
242 }
243
244 if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
245
246 bool IsAllUndef = true;
247 for (const SDValue &Op : N->op_values()) {
248 if (Op.isUndef())
249 continue;
250 IsAllUndef = false;
251 // Do not accept build_vectors that aren't all constants or which have non-0
252 // elements. We have to be a bit careful here, as the type of the constant
253 // may not be the same as the type of the vector elements due to type
254 // legalization (the elements are promoted to a legal type for the target
255 // and a vector of a type may be legal when the base element type is not).
256 // We only want to check enough bits to cover the vector elements, because
257 // we care if the resultant vector is all zeros, not whether the individual
258 // constants are.
259 if (auto OptAPInt = Op->bitcastToAPInt()) {
260 unsigned EltSize = N->getValueType(ResNo: 0).getScalarSizeInBits();
261 if (OptAPInt->countr_zero() < EltSize)
262 return false;
263 } else
264 return false;
265 }
266
267 // Do not accept an all-undef vector.
268 if (IsAllUndef)
269 return false;
270 return true;
271}
272
273bool ISD::isBuildVectorAllOnes(const SDNode *N) {
274 return isConstantSplatVectorAllOnes(N, /*BuildVectorOnly*/ true);
275}
276
277bool ISD::isBuildVectorAllZeros(const SDNode *N) {
278 return isConstantSplatVectorAllZeros(N, /*BuildVectorOnly*/ true);
279}
280
281bool ISD::isBuildVectorOfConstantSDNodes(const SDNode *N) {
282 if (N->getOpcode() != ISD::BUILD_VECTOR)
283 return false;
284
285 for (const SDValue &Op : N->op_values()) {
286 if (Op.isUndef())
287 continue;
288 if (!isa<ConstantSDNode>(Val: Op))
289 return false;
290 }
291 return true;
292}
293
294bool ISD::isBuildVectorOfConstantFPSDNodes(const SDNode *N) {
295 if (N->getOpcode() != ISD::BUILD_VECTOR)
296 return false;
297
298 for (const SDValue &Op : N->op_values()) {
299 if (Op.isUndef())
300 continue;
301 if (!isa<ConstantFPSDNode>(Val: Op))
302 return false;
303 }
304 return true;
305}
306
307bool ISD::isVectorShrinkable(const SDNode *N, unsigned NewEltSize,
308 bool Signed) {
309 assert(N->getValueType(0).isVector() && "Expected a vector!");
310
311 unsigned EltSize = N->getValueType(ResNo: 0).getScalarSizeInBits();
312 if (EltSize <= NewEltSize)
313 return false;
314
315 if (N->getOpcode() == ISD::ZERO_EXTEND) {
316 return (N->getOperand(Num: 0).getValueType().getScalarSizeInBits() <=
317 NewEltSize) &&
318 !Signed;
319 }
320 if (N->getOpcode() == ISD::SIGN_EXTEND) {
321 return (N->getOperand(Num: 0).getValueType().getScalarSizeInBits() <=
322 NewEltSize) &&
323 Signed;
324 }
325 if (N->getOpcode() != ISD::BUILD_VECTOR)
326 return false;
327
328 for (const SDValue &Op : N->op_values()) {
329 if (Op.isUndef())
330 continue;
331 if (!isa<ConstantSDNode>(Val: Op))
332 return false;
333
334 APInt C = Op->getAsAPIntVal().trunc(width: EltSize);
335 if (Signed && C.trunc(width: NewEltSize).sext(width: EltSize) != C)
336 return false;
337 if (!Signed && C.trunc(width: NewEltSize).zext(width: EltSize) != C)
338 return false;
339 }
340
341 return true;
342}
343
344bool ISD::allOperandsUndef(const SDNode *N) {
345 // Return false if the node has no operands.
346 // This is "logically inconsistent" with the definition of "all" but
347 // is probably the desired behavior.
348 if (N->getNumOperands() == 0)
349 return false;
350 return all_of(Range: N->op_values(), P: [](SDValue Op) { return Op.isUndef(); });
351}
352
353bool ISD::isFreezeUndef(const SDNode *N) {
354 return N->getOpcode() == ISD::FREEZE && N->getOperand(Num: 0).isUndef();
355}
356
357template <typename ConstNodeType>
358bool ISD::matchUnaryPredicateImpl(SDValue Op, const APInt &DemandedElts,
359 std::function<bool(ConstNodeType *)> Match,
360 bool AllowUndefs, bool AllowTruncation) {
361 // FIXME: Add support for scalar UNDEF cases?
362 if (auto *C = dyn_cast<ConstNodeType>(Op))
363 return Match(C);
364
365 // FIXME: Add support for vector UNDEF cases?
366 if (ISD::BUILD_VECTOR != Op.getOpcode() &&
367 ISD::SPLAT_VECTOR != Op.getOpcode())
368 return false;
369
370 if (ISD::SPLAT_VECTOR == Op.getOpcode() && !DemandedElts)
371 return true;
372
373 EVT SVT = Op.getValueType().getScalarType();
374 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
375 if (ISD::SPLAT_VECTOR != Op.getOpcode() && !DemandedElts[i])
376 continue;
377
378 if (AllowUndefs && Op.getOperand(i).isUndef()) {
379 if (!Match(nullptr))
380 return false;
381 continue;
382 }
383
384 auto *Cst = dyn_cast<ConstNodeType>(Op.getOperand(i));
385 if (!Cst || (!AllowTruncation && Cst->getValueType(0) != SVT) ||
386 !Match(Cst))
387 return false;
388 }
389 return true;
390}
391// Build used template types.
392template bool ISD::matchUnaryPredicateImpl<ConstantSDNode>(
393 SDValue, const APInt &, std::function<bool(ConstantSDNode *)>, bool, bool);
394template bool ISD::matchUnaryPredicateImpl<ConstantFPSDNode>(
395 SDValue, const APInt &, std::function<bool(ConstantFPSDNode *)>, bool,
396 bool);
397
398bool ISD::matchBinaryPredicate(
399 SDValue LHS, SDValue RHS, const APInt &DemandedElts,
400 std::function<bool(ConstantSDNode *, ConstantSDNode *)> Match,
401 bool AllowUndefs, bool AllowTypeMismatch) {
402 if (!AllowTypeMismatch && LHS.getValueType() != RHS.getValueType())
403 return false;
404
405 // TODO: Add support for scalar UNDEF cases?
406 if (auto *LHSCst = dyn_cast<ConstantSDNode>(Val&: LHS))
407 if (auto *RHSCst = dyn_cast<ConstantSDNode>(Val&: RHS))
408 return Match(LHSCst, RHSCst);
409
410 // TODO: Add support for vector UNDEF cases?
411 if (LHS.getOpcode() != RHS.getOpcode() ||
412 (LHS.getOpcode() != ISD::BUILD_VECTOR &&
413 LHS.getOpcode() != ISD::SPLAT_VECTOR))
414 return false;
415
416 if (ISD::SPLAT_VECTOR == LHS.getOpcode() && !DemandedElts)
417 return true;
418
419 EVT SVT = LHS.getValueType().getScalarType();
420 for (unsigned i = 0, e = LHS.getNumOperands(); i != e; ++i) {
421 if (ISD::SPLAT_VECTOR != LHS.getOpcode() && !DemandedElts[i])
422 continue;
423 SDValue LHSOp = LHS.getOperand(i);
424 SDValue RHSOp = RHS.getOperand(i);
425 bool LHSUndef = AllowUndefs && LHSOp.isUndef();
426 bool RHSUndef = AllowUndefs && RHSOp.isUndef();
427 auto *LHSCst = dyn_cast<ConstantSDNode>(Val&: LHSOp);
428 auto *RHSCst = dyn_cast<ConstantSDNode>(Val&: RHSOp);
429 if ((!LHSCst && !LHSUndef) || (!RHSCst && !RHSUndef))
430 return false;
431 if (!AllowTypeMismatch && (LHSOp.getValueType() != SVT ||
432 LHSOp.getValueType() != RHSOp.getValueType()))
433 return false;
434 if (!Match(LHSCst, RHSCst))
435 return false;
436 }
437 return true;
438}
439
440ISD::NodeType ISD::getInverseMinMaxOpcode(unsigned MinMaxOpc) {
441 switch (MinMaxOpc) {
442 default:
443 llvm_unreachable("unrecognized opcode");
444 case ISD::UMIN:
445 return ISD::UMAX;
446 case ISD::UMAX:
447 return ISD::UMIN;
448 case ISD::SMIN:
449 return ISD::SMAX;
450 case ISD::SMAX:
451 return ISD::SMIN;
452 }
453}
454
455ISD::NodeType ISD::getOppositeSignednessMinMaxOpcode(unsigned MinMaxOpc) {
456 switch (MinMaxOpc) {
457 default:
458 llvm_unreachable("unrecognized min/max opcode");
459 case ISD::SMIN:
460 return ISD::UMIN;
461 case ISD::SMAX:
462 return ISD::UMAX;
463 case ISD::UMIN:
464 return ISD::SMIN;
465 case ISD::UMAX:
466 return ISD::SMAX;
467 }
468}
469
470ISD::NodeType ISD::getVecReduceBaseOpcode(unsigned VecReduceOpcode) {
471 switch (VecReduceOpcode) {
472 default:
473 llvm_unreachable("Expected VECREDUCE opcode");
474 case ISD::VECREDUCE_FADD:
475 case ISD::VECREDUCE_SEQ_FADD:
476 case ISD::VP_REDUCE_FADD:
477 case ISD::VP_REDUCE_SEQ_FADD:
478 return ISD::FADD;
479 case ISD::VECREDUCE_FMUL:
480 case ISD::VECREDUCE_SEQ_FMUL:
481 case ISD::VP_REDUCE_FMUL:
482 case ISD::VP_REDUCE_SEQ_FMUL:
483 return ISD::FMUL;
484 case ISD::VECREDUCE_ADD:
485 case ISD::VP_REDUCE_ADD:
486 return ISD::ADD;
487 case ISD::VECREDUCE_MUL:
488 case ISD::VP_REDUCE_MUL:
489 return ISD::MUL;
490 case ISD::VECREDUCE_AND:
491 case ISD::VP_REDUCE_AND:
492 return ISD::AND;
493 case ISD::VECREDUCE_OR:
494 case ISD::VP_REDUCE_OR:
495 return ISD::OR;
496 case ISD::VECREDUCE_XOR:
497 case ISD::VP_REDUCE_XOR:
498 return ISD::XOR;
499 case ISD::VECREDUCE_SMAX:
500 case ISD::VP_REDUCE_SMAX:
501 return ISD::SMAX;
502 case ISD::VECREDUCE_SMIN:
503 case ISD::VP_REDUCE_SMIN:
504 return ISD::SMIN;
505 case ISD::VECREDUCE_UMAX:
506 case ISD::VP_REDUCE_UMAX:
507 return ISD::UMAX;
508 case ISD::VECREDUCE_UMIN:
509 case ISD::VP_REDUCE_UMIN:
510 return ISD::UMIN;
511 case ISD::VECREDUCE_FMAX:
512 case ISD::VP_REDUCE_FMAX:
513 return ISD::FMAXNUM;
514 case ISD::VECREDUCE_FMIN:
515 case ISD::VP_REDUCE_FMIN:
516 return ISD::FMINNUM;
517 case ISD::VECREDUCE_FMAXIMUM:
518 case ISD::VP_REDUCE_FMAXIMUM:
519 return ISD::FMAXIMUM;
520 case ISD::VECREDUCE_FMINIMUM:
521 case ISD::VP_REDUCE_FMINIMUM:
522 return ISD::FMINIMUM;
523 case ISD::VECREDUCE_FMAXIMUMNUM:
524 return ISD::FMAXIMUMNUM;
525 case ISD::VECREDUCE_FMINIMUMNUM:
526 return ISD::FMINIMUMNUM;
527 }
528}
529
530ISD::NodeType ISD::getUnmaskedBinOpOpcode(unsigned MaskedOpc) {
531 switch (MaskedOpc) {
532 case ISD::MASKED_UDIV:
533 return ISD::UDIV;
534 case ISD::MASKED_SDIV:
535 return ISD::SDIV;
536 case ISD::MASKED_UREM:
537 return ISD::UREM;
538 case ISD::MASKED_SREM:
539 return ISD::SREM;
540 default:
541 llvm_unreachable("Expected masked binop opcode");
542 }
543}
544
545bool ISD::isVPOpcode(unsigned Opcode) {
546 switch (Opcode) {
547 default:
548 return false;
549#define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) \
550 case ISD::VPSD: \
551 return true;
552#include "llvm/IR/VPIntrinsics.def"
553 }
554}
555
556bool ISD::isVPBinaryOp(unsigned Opcode) {
557 switch (Opcode) {
558 default:
559 break;
560#define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) case ISD::VPSD:
561#define VP_PROPERTY_BINARYOP return true;
562#define END_REGISTER_VP_SDNODE(VPSD) break;
563#include "llvm/IR/VPIntrinsics.def"
564 }
565 return false;
566}
567
568bool ISD::isVPReduction(unsigned Opcode) {
569 switch (Opcode) {
570 default:
571 return false;
572 case ISD::VP_REDUCE_ADD:
573 case ISD::VP_REDUCE_MUL:
574 case ISD::VP_REDUCE_AND:
575 case ISD::VP_REDUCE_OR:
576 case ISD::VP_REDUCE_XOR:
577 case ISD::VP_REDUCE_SMAX:
578 case ISD::VP_REDUCE_SMIN:
579 case ISD::VP_REDUCE_UMAX:
580 case ISD::VP_REDUCE_UMIN:
581 case ISD::VP_REDUCE_FMAX:
582 case ISD::VP_REDUCE_FMIN:
583 case ISD::VP_REDUCE_FMAXIMUM:
584 case ISD::VP_REDUCE_FMINIMUM:
585 case ISD::VP_REDUCE_FADD:
586 case ISD::VP_REDUCE_FMUL:
587 case ISD::VP_REDUCE_SEQ_FADD:
588 case ISD::VP_REDUCE_SEQ_FMUL:
589 return true;
590 }
591}
592
593/// The operand position of the vector mask.
594std::optional<unsigned> ISD::getVPMaskIdx(unsigned Opcode) {
595 switch (Opcode) {
596 default:
597 return std::nullopt;
598#define BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, TDNAME, MASKPOS, ...) \
599 case ISD::VPSD: \
600 return MASKPOS;
601#include "llvm/IR/VPIntrinsics.def"
602 }
603}
604
605/// The operand position of the explicit vector length parameter.
606std::optional<unsigned> ISD::getVPExplicitVectorLengthIdx(unsigned Opcode) {
607 switch (Opcode) {
608 default:
609 return std::nullopt;
610#define BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, TDNAME, MASKPOS, EVLPOS) \
611 case ISD::VPSD: \
612 return EVLPOS;
613#include "llvm/IR/VPIntrinsics.def"
614 }
615}
616
617std::optional<unsigned> ISD::getBaseOpcodeForVP(unsigned VPOpcode,
618 bool hasFPExcept) {
619 // FIXME: Return strict opcodes in case of fp exceptions.
620 switch (VPOpcode) {
621 default:
622 return std::nullopt;
623#define BEGIN_REGISTER_VP_SDNODE(VPOPC, ...) case ISD::VPOPC:
624#define VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) return ISD::SDOPC;
625#define END_REGISTER_VP_SDNODE(VPOPC) break;
626#include "llvm/IR/VPIntrinsics.def"
627 }
628 return std::nullopt;
629}
630
631std::optional<unsigned> ISD::getVPForBaseOpcode(unsigned Opcode) {
632 switch (Opcode) {
633 default:
634 return std::nullopt;
635#define BEGIN_REGISTER_VP_SDNODE(VPOPC, ...) break;
636#define VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) case ISD::SDOPC:
637#define END_REGISTER_VP_SDNODE(VPOPC) return ISD::VPOPC;
638#include "llvm/IR/VPIntrinsics.def"
639 }
640}
641
642ISD::NodeType ISD::getExtForLoadExtType(bool IsFP, ISD::LoadExtType ExtType) {
643 switch (ExtType) {
644 case ISD::EXTLOAD:
645 return IsFP ? ISD::FP_EXTEND : ISD::ANY_EXTEND;
646 case ISD::SEXTLOAD:
647 return ISD::SIGN_EXTEND;
648 case ISD::ZEXTLOAD:
649 return ISD::ZERO_EXTEND;
650 default:
651 break;
652 }
653
654 llvm_unreachable("Invalid LoadExtType");
655}
656
657ISD::CondCode ISD::getSetCCSwappedOperands(ISD::CondCode Operation) {
658 // To perform this operation, we just need to swap the L and G bits of the
659 // operation.
660 unsigned OldL = (Operation >> 2) & 1;
661 unsigned OldG = (Operation >> 1) & 1;
662 return ISD::CondCode((Operation & ~6) | // Keep the N, U, E bits
663 (OldL << 1) | // New G bit
664 (OldG << 2)); // New L bit.
665}
666
667static ISD::CondCode getSetCCInverseImpl(ISD::CondCode Op, bool isIntegerLike) {
668 unsigned Operation = Op;
669 if (isIntegerLike)
670 Operation ^= 7; // Flip L, G, E bits, but not U.
671 else
672 Operation ^= 15; // Flip all of the condition bits.
673
674 if (Operation > ISD::SETTRUE2)
675 Operation &= ~8; // Don't let N and U bits get set.
676
677 return ISD::CondCode(Operation);
678}
679
680ISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, EVT Type) {
681 return getSetCCInverseImpl(Op, isIntegerLike: Type.isInteger());
682}
683
684ISD::CondCode ISD::GlobalISel::getSetCCInverse(ISD::CondCode Op,
685 bool isIntegerLike) {
686 return getSetCCInverseImpl(Op, isIntegerLike);
687}
688
689/// For an integer comparison, return 1 if the comparison is a signed operation
690/// and 2 if the result is an unsigned comparison. Return zero if the operation
691/// does not depend on the sign of the input (setne and seteq).
692static int isSignedOp(ISD::CondCode Opcode) {
693 switch (Opcode) {
694 default: llvm_unreachable("Illegal integer setcc operation!");
695 case ISD::SETEQ:
696 case ISD::SETNE: return 0;
697 case ISD::SETLT:
698 case ISD::SETLE:
699 case ISD::SETGT:
700 case ISD::SETGE: return 1;
701 case ISD::SETULT:
702 case ISD::SETULE:
703 case ISD::SETUGT:
704 case ISD::SETUGE: return 2;
705 }
706}
707
708ISD::CondCode ISD::getSetCCOrOperation(ISD::CondCode Op1, ISD::CondCode Op2,
709 EVT Type) {
710 bool IsInteger = Type.isInteger();
711 if (IsInteger && (isSignedOp(Opcode: Op1) | isSignedOp(Opcode: Op2)) == 3)
712 // Cannot fold a signed integer setcc with an unsigned integer setcc.
713 return ISD::SETCC_INVALID;
714
715 unsigned Op = Op1 | Op2; // Combine all of the condition bits.
716
717 // If the N and U bits get set, then the resultant comparison DOES suddenly
718 // care about orderedness, and it is true when ordered.
719 if (Op > ISD::SETTRUE2)
720 Op &= ~16; // Clear the U bit if the N bit is set.
721
722 // Canonicalize illegal integer setcc's.
723 if (IsInteger && Op == ISD::SETUNE) // e.g. SETUGT | SETULT
724 Op = ISD::SETNE;
725
726 return ISD::CondCode(Op);
727}
728
729ISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2,
730 EVT Type) {
731 bool IsInteger = Type.isInteger();
732 if (IsInteger && (isSignedOp(Opcode: Op1) | isSignedOp(Opcode: Op2)) == 3)
733 // Cannot fold a signed setcc with an unsigned setcc.
734 return ISD::SETCC_INVALID;
735
736 // Combine all of the condition bits.
737 ISD::CondCode Result = ISD::CondCode(Op1 & Op2);
738
739 // Canonicalize illegal integer setcc's.
740 if (IsInteger) {
741 switch (Result) {
742 default: break;
743 case ISD::SETUO : Result = ISD::SETFALSE; break; // SETUGT & SETULT
744 case ISD::SETOEQ: // SETEQ & SETU[LG]E
745 case ISD::SETUEQ: Result = ISD::SETEQ ; break; // SETUGE & SETULE
746 case ISD::SETOLT: Result = ISD::SETULT ; break; // SETULT & SETNE
747 case ISD::SETOGT: Result = ISD::SETUGT ; break; // SETUGT & SETNE
748 }
749 }
750
751 return Result;
752}
753
754//===----------------------------------------------------------------------===//
755// SDNode Key Support
756//===----------------------------------------------------------------------===//
757
758/// If this is an SDNode with special info, add this info to the NodeID data.
759/// MorphNodeTo passes the opcode \p N is becoming, so \p Opc may differ from
760/// N->getOpcode() and a case may only cast to a class \p N already is.
761static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N,
762 unsigned Opc) {
763 // Machine nodes are uniqued by opcode, value types and operands alone, as in
764 // getMachineNode; MorphNodeTo's clearMemRefs overwrites the fields below.
765 if (SDNode::isMachineOpcode(Opc))
766 return;
767
768 switch (Opc) {
769 case ISD::TargetExternalSymbol:
770 case ISD::ExternalSymbol:
771 case ISD::MCSymbol:
772 llvm_unreachable("Should only be used on nodes with operands");
773 default: break; // Normal nodes don't need extra info.
774 case ISD::TargetConstant:
775 case ISD::Constant: {
776 const ConstantSDNode *C = cast<ConstantSDNode>(Val: N);
777 ID.AddPointer(Ptr: C->getConstantIntValue());
778 ID.AddBoolean(B: C->isOpaque());
779 break;
780 }
781 case ISD::TargetConstantFP:
782 case ISD::ConstantFP:
783 ID.AddPointer(Ptr: cast<ConstantFPSDNode>(Val: N)->getConstantFPValue());
784 break;
785 case ISD::TargetGlobalAddress:
786 case ISD::GlobalAddress:
787 case ISD::TargetGlobalTLSAddress:
788 case ISD::GlobalTLSAddress: {
789 const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Val: N);
790 ID.AddPointer(Ptr: GA->getGlobal());
791 ID.AddInteger(I: GA->getOffset());
792 ID.AddInteger(I: GA->getTargetFlags());
793 break;
794 }
795 case ISD::BasicBlock:
796 ID.AddPointer(Ptr: cast<BasicBlockSDNode>(Val: N)->getBasicBlock());
797 break;
798 case ISD::EH_LABEL:
799 case ISD::ANNOTATION_LABEL:
800 ID.AddPointer(Ptr: cast<LabelSDNode>(Val: N)->getLabel());
801 break;
802 case ISD::DEACTIVATION_SYMBOL:
803 ID.AddPointer(Ptr: cast<DeactivationSymbolSDNode>(Val: N)->getGlobal());
804 break;
805 case ISD::Register:
806 ID.AddInteger(I: cast<RegisterSDNode>(Val: N)->getReg().id());
807 break;
808 case ISD::RegisterMask:
809 ID.AddPointer(Ptr: cast<RegisterMaskSDNode>(Val: N)->getRegMask());
810 break;
811 case ISD::SRCVALUE:
812 ID.AddPointer(Ptr: cast<SrcValueSDNode>(Val: N)->getValue());
813 break;
814 case ISD::FrameIndex:
815 case ISD::TargetFrameIndex:
816 ID.AddInteger(I: cast<FrameIndexSDNode>(Val: N)->getIndex());
817 break;
818 case ISD::PSEUDO_PROBE:
819 ID.AddInteger(I: cast<PseudoProbeSDNode>(Val: N)->getGuid());
820 ID.AddInteger(I: cast<PseudoProbeSDNode>(Val: N)->getIndex());
821 ID.AddInteger(I: cast<PseudoProbeSDNode>(Val: N)->getAttributes());
822 break;
823 case ISD::JumpTable:
824 case ISD::TargetJumpTable:
825 ID.AddInteger(I: cast<JumpTableSDNode>(Val: N)->getIndex());
826 ID.AddInteger(I: cast<JumpTableSDNode>(Val: N)->getTargetFlags());
827 break;
828 case ISD::ConstantPool:
829 case ISD::TargetConstantPool: {
830 const ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Val: N);
831 ID.AddInteger(I: CP->getAlign().value());
832 ID.AddInteger(I: CP->getOffset());
833 if (CP->isMachineConstantPoolEntry())
834 CP->getMachineCPVal()->addSelectionDAGCSEId(ID);
835 else
836 ID.AddPointer(Ptr: CP->getConstVal());
837 ID.AddInteger(I: CP->getTargetFlags());
838 break;
839 }
840 case ISD::TargetIndex: {
841 const TargetIndexSDNode *TI = cast<TargetIndexSDNode>(Val: N);
842 ID.AddInteger(I: TI->getIndex());
843 ID.AddInteger(I: TI->getOffset());
844 ID.AddInteger(I: TI->getTargetFlags());
845 break;
846 }
847 case ISD::LOAD: {
848 const LoadSDNode *LD = cast<LoadSDNode>(Val: N);
849 ID.AddInteger(I: LD->getMemoryVT().getRawBits());
850 ID.AddInteger(I: LD->getRawSubclassData());
851 ID.AddInteger(I: LD->getPointerInfo().getAddrSpace());
852 ID.AddInteger(I: LD->getMemOperand()->getFlags());
853 break;
854 }
855 case ISD::STORE: {
856 const StoreSDNode *ST = cast<StoreSDNode>(Val: N);
857 ID.AddInteger(I: ST->getMemoryVT().getRawBits());
858 ID.AddInteger(I: ST->getRawSubclassData());
859 ID.AddInteger(I: ST->getPointerInfo().getAddrSpace());
860 ID.AddInteger(I: ST->getMemOperand()->getFlags());
861 break;
862 }
863 case ISD::VP_LOAD: {
864 const VPLoadSDNode *ELD = cast<VPLoadSDNode>(Val: N);
865 ID.AddInteger(I: ELD->getMemoryVT().getRawBits());
866 ID.AddInteger(I: ELD->getRawSubclassData());
867 ID.AddInteger(I: ELD->getPointerInfo().getAddrSpace());
868 ID.AddInteger(I: ELD->getMemOperand()->getFlags());
869 break;
870 }
871 case ISD::VP_LOAD_FF: {
872 const auto *LD = cast<VPLoadFFSDNode>(Val: N);
873 ID.AddInteger(I: LD->getMemoryVT().getRawBits());
874 ID.AddInteger(I: LD->getRawSubclassData());
875 ID.AddInteger(I: LD->getPointerInfo().getAddrSpace());
876 ID.AddInteger(I: LD->getMemOperand()->getFlags());
877 break;
878 }
879 case ISD::VP_STORE: {
880 const VPStoreSDNode *EST = cast<VPStoreSDNode>(Val: N);
881 ID.AddInteger(I: EST->getMemoryVT().getRawBits());
882 ID.AddInteger(I: EST->getRawSubclassData());
883 ID.AddInteger(I: EST->getPointerInfo().getAddrSpace());
884 ID.AddInteger(I: EST->getMemOperand()->getFlags());
885 break;
886 }
887 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD: {
888 const VPStridedLoadSDNode *SLD = cast<VPStridedLoadSDNode>(Val: N);
889 ID.AddInteger(I: SLD->getMemoryVT().getRawBits());
890 ID.AddInteger(I: SLD->getRawSubclassData());
891 ID.AddInteger(I: SLD->getPointerInfo().getAddrSpace());
892 break;
893 }
894 case ISD::EXPERIMENTAL_VP_STRIDED_STORE: {
895 const VPStridedStoreSDNode *SST = cast<VPStridedStoreSDNode>(Val: N);
896 ID.AddInteger(I: SST->getMemoryVT().getRawBits());
897 ID.AddInteger(I: SST->getRawSubclassData());
898 ID.AddInteger(I: SST->getPointerInfo().getAddrSpace());
899 break;
900 }
901 case ISD::VP_GATHER: {
902 const VPGatherSDNode *EG = cast<VPGatherSDNode>(Val: N);
903 ID.AddInteger(I: EG->getMemoryVT().getRawBits());
904 ID.AddInteger(I: EG->getRawSubclassData());
905 ID.AddInteger(I: EG->getPointerInfo().getAddrSpace());
906 ID.AddInteger(I: EG->getMemOperand()->getFlags());
907 break;
908 }
909 case ISD::VP_SCATTER: {
910 const VPScatterSDNode *ES = cast<VPScatterSDNode>(Val: N);
911 ID.AddInteger(I: ES->getMemoryVT().getRawBits());
912 ID.AddInteger(I: ES->getRawSubclassData());
913 ID.AddInteger(I: ES->getPointerInfo().getAddrSpace());
914 ID.AddInteger(I: ES->getMemOperand()->getFlags());
915 break;
916 }
917 case ISD::MLOAD: {
918 const MaskedLoadSDNode *MLD = cast<MaskedLoadSDNode>(Val: N);
919 ID.AddInteger(I: MLD->getMemoryVT().getRawBits());
920 ID.AddInteger(I: MLD->getRawSubclassData());
921 ID.AddInteger(I: MLD->getPointerInfo().getAddrSpace());
922 ID.AddInteger(I: MLD->getMemOperand()->getFlags());
923 break;
924 }
925 case ISD::MSTORE: {
926 const MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(Val: N);
927 ID.AddInteger(I: MST->getMemoryVT().getRawBits());
928 ID.AddInteger(I: MST->getRawSubclassData());
929 ID.AddInteger(I: MST->getPointerInfo().getAddrSpace());
930 ID.AddInteger(I: MST->getMemOperand()->getFlags());
931 break;
932 }
933 case ISD::MGATHER: {
934 const MaskedGatherSDNode *MG = cast<MaskedGatherSDNode>(Val: N);
935 ID.AddInteger(I: MG->getMemoryVT().getRawBits());
936 ID.AddInteger(I: MG->getRawSubclassData());
937 ID.AddInteger(I: MG->getPointerInfo().getAddrSpace());
938 ID.AddInteger(I: MG->getMemOperand()->getFlags());
939 break;
940 }
941 case ISD::MSCATTER:
942 case ISD::GET_FPENV_MEM:
943 case ISD::SET_FPENV_MEM:
944 case ISD::EXPERIMENTAL_VECTOR_HISTOGRAM: {
945 const MemSDNode *M = cast<MemSDNode>(Val: N);
946 ID.AddInteger(I: M->getMemoryVT().getRawBits());
947 ID.AddInteger(I: M->getRawSubclassData());
948 ID.AddInteger(I: M->getPointerInfo().getAddrSpace());
949 ID.AddInteger(I: M->getMemOperand()->getFlags());
950 break;
951 }
952 case ISD::VECTOR_SHUFFLE: {
953 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Val: N)->getMask();
954 for (int M : Mask)
955 ID.AddInteger(I: M);
956 break;
957 }
958 case ISD::ADDRSPACECAST: {
959 const AddrSpaceCastSDNode *ASC = cast<AddrSpaceCastSDNode>(Val: N);
960 ID.AddInteger(I: ASC->getSrcAddressSpace());
961 ID.AddInteger(I: ASC->getDestAddressSpace());
962 break;
963 }
964 case ISD::TargetBlockAddress:
965 case ISD::BlockAddress: {
966 const BlockAddressSDNode *BA = cast<BlockAddressSDNode>(Val: N);
967 ID.AddPointer(Ptr: BA->getBlockAddress());
968 ID.AddInteger(I: BA->getOffset());
969 ID.AddInteger(I: BA->getTargetFlags());
970 break;
971 }
972 case ISD::AssertAlign:
973 ID.AddInteger(I: cast<AssertAlignSDNode>(Val: N)->getAlign().value());
974 break;
975 case ISD::PREFETCH:
976 case ISD::INTRINSIC_VOID:
977 case ISD::INTRINSIC_W_CHAIN:
978 // Handled by MemIntrinsicSDNode check after the switch.
979 break;
980 case ISD::MDNODE_SDNODE:
981 ID.AddPointer(Ptr: cast<MDNodeSDNode>(Val: N)->getMD());
982 break;
983 } // end switch (Opc)
984
985 if (auto *AT = dyn_cast<AtomicSDNode>(Val: N)) {
986 ID.AddInteger(I: AT->getMemoryVT().getRawBits());
987 ID.AddInteger(I: AT->getRawSubclassData());
988 ID.AddInteger(I: AT->getPointerInfo().getAddrSpace());
989 ID.AddInteger(I: AT->getMemOperand()->getFlags());
990 }
991
992 // MemIntrinsic nodes could also have subclass data, address spaces, and flags
993 // to check.
994 if (auto *MN = dyn_cast<MemIntrinsicSDNode>(Val: N)) {
995 ID.AddInteger(I: MN->getRawSubclassData());
996 ID.AddInteger(I: MN->getMemoryVT().getRawBits());
997 for (const MachineMemOperand *MMO : MN->memoperands()) {
998 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
999 ID.AddInteger(I: MMO->getFlags());
1000 }
1001 }
1002}
1003
1004SDNodeKey::SDNodeKey(const SDNode &N)
1005 : Opcode(N.getOpcode()), VTs(N.getVTList().VTs),
1006 OpStorage(N.op_begin(), N.op_end()) {
1007 Ops = OpStorage;
1008 AddNodeIDCustom(ID&: Tail, N: &N, Opc: N.getOpcode());
1009}
1010
1011#ifndef NDEBUG
1012/// Serialize a node the way SDNodes were uniqued before SDNodeKey, to
1013/// cross-check the typed comparison against. Deliberately reads the node
1014/// rather than going through SDNodeKey, so the key constructor is checked too
1015/// and not used as its own oracle.
1016static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) {
1017 ID.AddInteger(N->getOpcode());
1018 ID.AddPointer(N->getVTList().VTs);
1019 for (const SDUse &Op : N->ops()) {
1020 ID.AddPointer(Op.getNode());
1021 ID.AddInteger(Op.getResNo());
1022 }
1023 AddNodeIDCustom(ID, N, N->getOpcode());
1024}
1025
1026static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNodeKey &Key) {
1027 ID.AddInteger(Key.Opcode);
1028 ID.AddPointer(Key.VTs);
1029 for (const SDValue &Op : Key.Ops) {
1030 ID.AddPointer(Op.getNode());
1031 ID.AddInteger(Op.getResNo());
1032 }
1033 ID.AddNodeID(Key.Tail);
1034}
1035#endif
1036
1037static bool keyMatches(const SDNodeKey &Key, const SDNode &N) {
1038 if (N.getOpcode() != Key.Opcode || N.getVTList().VTs != Key.VTs)
1039 return false;
1040 if (!llvm::equal(LRange: N.ops(), RRange: Key.Ops))
1041 return false;
1042 // Build only what AddNodeIDCustom would have added, which is nothing for
1043 // most opcodes, rather than the whole profile.
1044 FoldingSetNodeID Tail;
1045 AddNodeIDCustom(ID&: Tail, N: &N, Opc: N.getOpcode());
1046 return Tail == Key.Tail;
1047}
1048
1049bool SDNodeKeyInfo::isEqual(const SDNodeKey &Key, const SDNode &N) {
1050 bool Result = keyMatches(Key, N);
1051
1052#ifndef NDEBUG
1053 FoldingSetNodeID NodeProfile, KeyProfile;
1054 AddNodeIDNode(NodeProfile, &N);
1055 AddNodeIDNode(KeyProfile, Key);
1056 assert((NodeProfile == KeyProfile) == Result &&
1057 "SDNodeKey equality disagrees with profile");
1058#endif
1059 return Result;
1060}
1061
1062//===----------------------------------------------------------------------===//
1063// SelectionDAG Class
1064//===----------------------------------------------------------------------===//
1065
1066/// doNotCSE - Return true if CSE should not be performed for this node.
1067static bool doNotCSE(SDNode *N) {
1068 if (N->getValueType(ResNo: 0) == MVT::Glue)
1069 return true; // Never CSE anything that produces a glue result.
1070
1071 switch (N->getOpcode()) {
1072 default: break;
1073 case ISD::HANDLENODE:
1074 case ISD::EH_LABEL:
1075 return true; // Never CSE these nodes.
1076 }
1077
1078 // Check that remaining values produced are not flags.
1079 for (unsigned i = 1, e = N->getNumValues(); i != e; ++i)
1080 if (N->getValueType(ResNo: i) == MVT::Glue)
1081 return true; // Never CSE anything that produces a glue result.
1082
1083 return false;
1084}
1085
1086/// Construct a DemandedElts mask which demands all elements of \p V.
1087/// If \p V is not a fixed-length vector, then this will return a single bit.
1088static APInt getDemandAllEltsMask(SDValue V) {
1089 EVT VT = V.getValueType();
1090 // Since the number of lanes in a scalable vector is unknown at compile time,
1091 // we track one bit which is implicitly broadcast to all lanes. This means
1092 // that all lanes in a scalable vector are considered demanded.
1093 return VT.isFixedLengthVector() ? APInt::getAllOnes(numBits: VT.getVectorNumElements())
1094 : APInt(1, 1);
1095}
1096
1097/// RemoveDeadNodes - This method deletes all unreachable nodes in the
1098/// SelectionDAG.
1099void SelectionDAG::RemoveDeadNodes() {
1100 // Create a dummy node (which is not added to allnodes), that adds a reference
1101 // to the root node, preventing it from being deleted.
1102 HandleSDNode Dummy(getRoot());
1103
1104 SmallVector<SDNode*, 128> DeadNodes;
1105
1106 // Add all obviously-dead nodes to the DeadNodes worklist.
1107 for (SDNode &Node : allnodes())
1108 if (Node.use_empty())
1109 DeadNodes.push_back(Elt: &Node);
1110
1111 RemoveDeadNodes(DeadNodes);
1112
1113 // If the root changed (e.g. it was a dead load, update the root).
1114 setRoot(Dummy.getValue());
1115}
1116
1117/// RemoveDeadNodes - This method deletes the unreachable nodes in the
1118/// given list, and any nodes that become unreachable as a result.
1119void SelectionDAG::RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes) {
1120
1121 // Process the worklist, deleting the nodes and adding their uses to the
1122 // worklist.
1123 while (!DeadNodes.empty()) {
1124 SDNode *N = DeadNodes.pop_back_val();
1125 // Skip to next node if we've already managed to delete the node. This could
1126 // happen if replacing a node causes a node previously added to the node to
1127 // be deleted.
1128 if (N->getOpcode() == ISD::DELETED_NODE)
1129 continue;
1130
1131 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1132 DUL->NodeDeleted(N, nullptr);
1133
1134 // Take the node out of the appropriate CSE map.
1135 RemoveNodeFromCSEMaps(N);
1136
1137 // Next, brutally remove the operand list. This is safe to do, as there are
1138 // no cycles in the graph.
1139 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
1140 SDUse &Use = *I++;
1141 SDNode *Operand = Use.getNode();
1142 Use.set(SDValue());
1143
1144 // Now that we removed this operand, see if there are no uses of it left.
1145 if (Operand->use_empty())
1146 DeadNodes.push_back(Elt: Operand);
1147 }
1148
1149 DeallocateNode(N);
1150 }
1151}
1152
1153void SelectionDAG::RemoveDeadNode(SDNode *N){
1154 SmallVector<SDNode*, 16> DeadNodes(1, N);
1155
1156 // Create a dummy node that adds a reference to the root node, preventing
1157 // it from being deleted. (This matters if the root is an operand of the
1158 // dead node.)
1159 HandleSDNode Dummy(getRoot());
1160
1161 RemoveDeadNodes(DeadNodes);
1162}
1163
1164void SelectionDAG::DeleteNode(SDNode *N) {
1165 // First take this out of the appropriate CSE map.
1166 RemoveNodeFromCSEMaps(N);
1167
1168 // Finally, remove uses due to operands of this node, remove from the
1169 // AllNodes list, and delete the node.
1170 DeleteNodeNotInCSEMaps(N);
1171}
1172
1173void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) {
1174 assert(N->getIterator() != AllNodes.begin() &&
1175 "Cannot delete the entry node!");
1176 assert(N->use_empty() && "Cannot delete a node that is not dead!");
1177
1178 // Drop all of the operands and decrement used node's use counts.
1179 N->DropOperands();
1180
1181 DeallocateNode(N);
1182}
1183
1184void SDDbgInfo::add(SDDbgValue *V, bool isParameter) {
1185 assert(!(V->isVariadic() && isParameter));
1186 if (isParameter)
1187 ByvalParmDbgValues.push_back(Elt: V);
1188 else
1189 DbgValues.push_back(Elt: V);
1190 for (const SDNode *Node : V->getSDNodes())
1191 if (Node)
1192 DbgValMap[Node].push_back(Elt: V);
1193}
1194
1195void SDDbgInfo::erase(const SDNode *Node) {
1196 DbgValMapType::iterator I = DbgValMap.find(Val: Node);
1197 if (I == DbgValMap.end())
1198 return;
1199 for (auto &Val: I->second)
1200 Val->setIsInvalidated();
1201 DbgValMap.erase(I);
1202}
1203
1204void SelectionDAG::DeallocateNode(SDNode *N) {
1205 // If we have operands, deallocate them.
1206 removeOperands(Node: N);
1207
1208 NodeAllocator.Deallocate(E: AllNodes.remove(IT: N));
1209
1210 // Set the opcode to DELETED_NODE to help catch bugs when node
1211 // memory is reallocated.
1212 // FIXME: There are places in SDag that have grown a dependency on the opcode
1213 // value in the released node.
1214 __asan_unpoison_memory_region(&N->NodeType, sizeof(N->NodeType));
1215 N->NodeType = ISD::DELETED_NODE;
1216
1217 // If any of the SDDbgValue nodes refer to this SDNode, invalidate
1218 // them and forget about that node.
1219 DbgInfo->erase(Node: N);
1220
1221 // Invalidate extra info.
1222 SDEI.erase(Val: N);
1223}
1224
1225#ifndef NDEBUG
1226/// VerifySDNode - Check the given SDNode. Aborts if it is invalid.
1227void SelectionDAG::verifyNode(SDNode *N) const {
1228 switch (N->getOpcode()) {
1229 default:
1230 if (N->isTargetOpcode())
1231 getSelectionDAGInfo().verifyTargetNode(*this, N);
1232 break;
1233 case ISD::BUILD_PAIR: {
1234 EVT VT = N->getValueType(0);
1235 assert(N->getNumValues() == 1 && "Too many results!");
1236 assert(!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) &&
1237 "Wrong return type!");
1238 assert(N->getNumOperands() == 2 && "Wrong number of operands!");
1239 assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() &&
1240 "Mismatched operand types!");
1241 assert(N->getOperand(0).getValueType().isInteger() == VT.isInteger() &&
1242 "Wrong operand type!");
1243 assert(VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() &&
1244 "Wrong return type size");
1245 break;
1246 }
1247 case ISD::BUILD_VECTOR: {
1248 assert(N->getNumValues() == 1 && "Too many results!");
1249 assert(N->getValueType(0).isVector() && "Wrong return type!");
1250 assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() &&
1251 "Wrong number of operands!");
1252 EVT EltVT = N->getValueType(0).getVectorElementType();
1253 for (const SDUse &Op : N->ops()) {
1254 assert((Op.getValueType() == EltVT ||
1255 (EltVT.isInteger() && Op.getValueType().isInteger() &&
1256 EltVT.bitsLE(Op.getValueType()))) &&
1257 "Wrong operand type!");
1258 assert(Op.getValueType() == N->getOperand(0).getValueType() &&
1259 "Operands must all have the same type");
1260 }
1261 break;
1262 }
1263 case ISD::SADDO:
1264 case ISD::UADDO:
1265 case ISD::SSUBO:
1266 case ISD::USUBO:
1267 assert(N->getNumValues() == 2 && "Wrong number of results!");
1268 assert(N->getVTList().NumVTs == 2 && N->getNumOperands() == 2 &&
1269 "Invalid add/sub overflow op!");
1270 assert(N->getVTList().VTs[0].isInteger() &&
1271 N->getVTList().VTs[1].isInteger() &&
1272 N->getOperand(0).getValueType() == N->getOperand(1).getValueType() &&
1273 N->getOperand(0).getValueType() == N->getVTList().VTs[0] &&
1274 "Binary operator types must match!");
1275 break;
1276 }
1277}
1278#endif // NDEBUG
1279
1280/// Insert a newly allocated node into the DAG.
1281///
1282/// Handles insertion into the all nodes list and CSE map, as well as
1283/// verification and other common operations when a new node is allocated.
1284void SelectionDAG::InsertNode(SDNode *N) {
1285 AllNodes.push_back(val: N);
1286#ifndef NDEBUG
1287 N->PersistentId = NextPersistentId++;
1288 verifyNode(N);
1289#endif
1290 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1291 DUL->NodeInserted(N);
1292}
1293
1294/// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that
1295/// correspond to it. This is useful when we're about to delete or repurpose
1296/// the node. We don't want future request for structurally identical nodes
1297/// to return N anymore.
1298bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) {
1299 bool Erased = false;
1300 switch (N->getOpcode()) {
1301 case ISD::HANDLENODE: return false; // noop.
1302 case ISD::CONDCODE:
1303 assert(CondCodeNodes[cast<CondCodeSDNode>(N)->get()] &&
1304 "Cond code doesn't exist!");
1305 Erased = CondCodeNodes[cast<CondCodeSDNode>(Val: N)->get()] != nullptr;
1306 CondCodeNodes[cast<CondCodeSDNode>(Val: N)->get()] = nullptr;
1307 break;
1308 case ISD::ExternalSymbol:
1309 Erased = ExternalSymbols.erase(Key: cast<ExternalSymbolSDNode>(Val: N)->getSymbol());
1310 break;
1311 case ISD::TargetExternalSymbol: {
1312 ExternalSymbolSDNode *ESN = cast<ExternalSymbolSDNode>(Val: N);
1313 Erased = TargetExternalSymbols.erase(x: std::pair<std::string, unsigned>(
1314 ESN->getSymbol(), ESN->getTargetFlags()));
1315 break;
1316 }
1317 case ISD::MCSymbol: {
1318 auto *MCSN = cast<MCSymbolSDNode>(Val: N);
1319 Erased = MCSymbols.erase(Val: MCSN->getMCSymbol());
1320 break;
1321 }
1322 case ISD::VALUETYPE: {
1323 EVT VT = cast<VTSDNode>(Val: N)->getVT();
1324 if (VT.isExtended()) {
1325 Erased = ExtendedValueTypeNodes.erase(x: VT);
1326 } else {
1327 Erased = ValueTypeNodes[VT.getSimpleVT().SimpleTy] != nullptr;
1328 ValueTypeNodes[VT.getSimpleVT().SimpleTy] = nullptr;
1329 }
1330 break;
1331 }
1332 default:
1333 // Remove it from the CSE Map.
1334 assert(N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!");
1335 assert(N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!");
1336 Erased = CSEMap.erase(N);
1337 break;
1338 }
1339#ifndef NDEBUG
1340 // Verify that the node was actually in one of the CSE maps, unless it has a
1341 // glue result (which cannot be CSE'd) or is one of the special cases that are
1342 // not subject to CSE.
1343 if (!Erased && N->getValueType(N->getNumValues()-1) != MVT::Glue &&
1344 !N->isMachineOpcode() && !doNotCSE(N)) {
1345 N->dump(this);
1346 dbgs() << "\n";
1347 llvm_unreachable("Node is not in map!");
1348 }
1349#endif
1350 return Erased;
1351}
1352
1353/// AddModifiedNodeToCSEMaps - The specified node has been removed from the CSE
1354/// maps and modified in place. Add it back to the CSE maps, unless an identical
1355/// node already exists, in which case transfer all its users to the existing
1356/// node. This transfer can potentially trigger recursive merging.
1357void
1358SelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N) {
1359 // For node types that aren't CSE'd, just act as if no identical node
1360 // already exists.
1361 if (!doNotCSE(N)) {
1362 SDNode *Existing = CSEMap.getOrInsert(N);
1363 if (Existing != N) {
1364 // If there was already an existing matching node, use ReplaceAllUsesWith
1365 // to replace the dead one with the existing one. This can cause
1366 // recursive merging of other unrelated nodes down the line.
1367 Existing->intersectFlagsWith(Flags: N->getFlags());
1368 if (auto *MemNode = dyn_cast<MemSDNode>(Val: Existing)) {
1369 ArrayRef<MachineMemOperand *> NewMMOs =
1370 cast<MemSDNode>(Val: N)->memoperands();
1371 // Range and cache hint metadata are not part of the DAG CSE key because
1372 // we prefer to CSE even when metadata does not match. Merge potentially
1373 // differing metadata conservatively.
1374 MemNode->refineMMOMetadata(NewMMOs);
1375 }
1376 ReplaceAllUsesWith(From: N, To: Existing);
1377
1378 // N is now dead. Inform the listeners and delete it.
1379 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1380 DUL->NodeDeleted(N, Existing);
1381 DeleteNodeNotInCSEMaps(N);
1382 return;
1383 }
1384 }
1385
1386 // If the node doesn't already exist, we updated it. Inform listeners.
1387 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1388 DUL->NodeUpdated(N);
1389}
1390
1391/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1392/// were replaced with those specified. If this node is never memoized,
1393/// return null, otherwise return a pointer to the slot it would take. If a
1394/// node already exists with these operands, the slot will be non-null.
1395SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op,
1396 FoldingSetInsertToken &InsertToken) {
1397 if (doNotCSE(N))
1398 return nullptr;
1399
1400 SDValue Ops[] = { Op };
1401 SDNodeKey ID(N->getOpcode(), N->getVTList(), Ops);
1402 AddNodeIDCustom(ID&: ID.Tail, N, Opc: N->getOpcode());
1403 SDNode *Node = lookupNode(Key: ID, DL: SDLoc(N), InsertToken);
1404 if (Node)
1405 Node->intersectFlagsWith(Flags: N->getFlags());
1406 return Node;
1407}
1408
1409/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1410/// were replaced with those specified. If this node is never memoized,
1411/// return null, otherwise return a pointer to the slot it would take. If a
1412/// node already exists with these operands, the slot will be non-null.
1413SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op1, SDValue Op2,
1414 FoldingSetInsertToken &InsertToken) {
1415 if (doNotCSE(N))
1416 return nullptr;
1417
1418 SDValue Ops[] = { Op1, Op2 };
1419 SDNodeKey ID(N->getOpcode(), N->getVTList(), Ops);
1420 AddNodeIDCustom(ID&: ID.Tail, N, Opc: N->getOpcode());
1421 SDNode *Node = lookupNode(Key: ID, DL: SDLoc(N), InsertToken);
1422 if (Node)
1423 Node->intersectFlagsWith(Flags: N->getFlags());
1424 return Node;
1425}
1426
1427/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1428/// were replaced with those specified. If this node is never memoized,
1429/// return null, otherwise return a pointer to the slot it would take. If a
1430/// node already exists with these operands, the slot will be non-null.
1431SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, ArrayRef<SDValue> Ops,
1432 FoldingSetInsertToken &InsertToken) {
1433 if (doNotCSE(N))
1434 return nullptr;
1435
1436 SDNodeKey ID(N->getOpcode(), N->getVTList(), Ops);
1437 AddNodeIDCustom(ID&: ID.Tail, N, Opc: N->getOpcode());
1438 SDNode *Node = lookupNode(Key: ID, DL: SDLoc(N), InsertToken);
1439 if (Node)
1440 Node->intersectFlagsWith(Flags: N->getFlags());
1441 return Node;
1442}
1443
1444Align SelectionDAG::getEVTAlign(EVT VT) const {
1445 Type *Ty = VT == MVT::iPTR ? PointerType::get(C&: *getContext(), AddressSpace: 0)
1446 : VT.getTypeForEVT(Context&: *getContext());
1447
1448 return getDataLayout().getABITypeAlign(Ty);
1449}
1450
1451// EntryNode could meaningfully have debug info if we can find it...
1452SelectionDAG::SelectionDAG(const TargetMachine &tm, CodeGenOptLevel OL)
1453 : TM(tm), OptLevel(OL), EntryNode(ISD::EntryToken, 0, DebugLoc(),
1454 getVTList(VT1: MVT::Other, VT2: MVT::Glue)),
1455 Root(getEntryNode()) {
1456 InsertNode(N: &EntryNode);
1457 DbgInfo = new SDDbgInfo();
1458}
1459
1460void SelectionDAG::init(MachineFunction &NewMF,
1461 const TargetLibraryInfo *LibraryInfo,
1462 const LibcallLoweringInfo *LibcallsInfo,
1463 UniformityInfo *NewUA, ProfileSummaryInfo *PSIin,
1464 BlockFrequencyInfo *BFIin,
1465 FunctionVarLocs const *VarLocs) {
1466 MF = &NewMF;
1467 TLI = getSubtarget().getTargetLowering();
1468 TSI = getSubtarget().getSelectionDAGInfo();
1469 LibInfo = LibraryInfo;
1470 Libcalls = LibcallsInfo;
1471 Context = &MF->getFunction().getContext();
1472 UA = NewUA;
1473 PSI = PSIin;
1474 BFI = BFIin;
1475 FnVarLocs = VarLocs;
1476}
1477
1478SelectionDAG::~SelectionDAG() {
1479 assert(!UpdateListeners && "Dangling registered DAGUpdateListeners");
1480 allnodes_clear();
1481 OperandRecycler.clear(OperandAllocator);
1482 delete DbgInfo;
1483}
1484
1485bool SelectionDAG::shouldOptForSize() const {
1486 return llvm::shouldOptimizeForSize(BB: FLI->MBB->getBasicBlock(), PSI, BFI);
1487}
1488
1489const BasicBlock *SelectionDAG::getBasicBlock() const {
1490 return FLI->MBB->getBasicBlock();
1491}
1492
1493void SelectionDAG::allnodes_clear() {
1494 assert(&*AllNodes.begin() == &EntryNode);
1495 AllNodes.remove(IT: AllNodes.begin());
1496 while (!AllNodes.empty())
1497 DeallocateNode(N: &AllNodes.front());
1498#ifndef NDEBUG
1499 NextPersistentId = 0;
1500#endif
1501}
1502
1503SDNode *SelectionDAG::lookupNode(const SDNodeKey &Key,
1504 FoldingSetInsertToken &InsertToken) {
1505 SDNode *N = CSEMap.lookup(Key, Token&: InsertToken);
1506 if (N) {
1507 switch (N->getOpcode()) {
1508 default: break;
1509 case ISD::Constant:
1510 case ISD::ConstantFP:
1511 llvm_unreachable("Querying for Constant and ConstantFP nodes requires "
1512 "debug location. Use another overload.");
1513 }
1514 }
1515 return N;
1516}
1517
1518SDNode *SelectionDAG::lookupNode(const SDNodeKey &Key, const SDLoc &DL,
1519 FoldingSetInsertToken &InsertToken) {
1520 SDNode *N = CSEMap.lookup(Key, Token&: InsertToken);
1521 if (N) {
1522 switch (N->getOpcode()) {
1523 case ISD::Constant:
1524 case ISD::ConstantFP:
1525 // Erase debug location from the node if the node is used at several
1526 // different places. Do not propagate one location to all uses as it
1527 // will cause a worse single stepping debugging experience.
1528 if (N->getDebugLoc() != DL.getDebugLoc())
1529 N->setDebugLoc(DebugLoc());
1530 break;
1531 default:
1532 // When the node's point of use is located earlier in the instruction
1533 // sequence than its prior point of use, update its debug info to the
1534 // earlier location.
1535 if (DL.getIROrder() && DL.getIROrder() < N->getIROrder())
1536 N->setDebugLoc(DL.getDebugLoc());
1537 break;
1538 }
1539 }
1540 return N;
1541}
1542
1543void SelectionDAG::clear() {
1544 allnodes_clear();
1545 OperandRecycler.clear(OperandAllocator);
1546 OperandAllocator.Reset();
1547 CSEMap.clear();
1548
1549 ExtendedValueTypeNodes.clear();
1550 ExternalSymbols.clear();
1551 TargetExternalSymbols.clear();
1552 MCSymbols.clear();
1553 SDEI.clear();
1554 llvm::fill(Range&: CondCodeNodes, Value: nullptr);
1555 llvm::fill(Range&: ValueTypeNodes, Value: nullptr);
1556
1557 EntryNode.UseList = nullptr;
1558 InsertNode(N: &EntryNode);
1559 Root = getEntryNode();
1560 DbgInfo->clear();
1561}
1562
1563SDValue SelectionDAG::getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT) {
1564 return VT.bitsGT(VT: Op.getValueType())
1565 ? getNode(Opcode: ISD::FP_EXTEND, DL, VT, Operand: Op)
1566 : getNode(Opcode: ISD::FP_ROUND, DL, VT, N1: Op,
1567 N2: getIntPtrConstant(Val: 0, DL, /*isTarget=*/true));
1568}
1569
1570std::pair<SDValue, SDValue>
1571SelectionDAG::getStrictFPExtendOrRound(SDValue Op, SDValue Chain,
1572 const SDLoc &DL, EVT VT) {
1573 assert(!VT.bitsEq(Op.getValueType()) &&
1574 "Strict no-op FP extend/round not allowed.");
1575 SDValue Res =
1576 VT.bitsGT(VT: Op.getValueType())
1577 ? getNode(Opcode: ISD::STRICT_FP_EXTEND, DL, ResultTys: {VT, MVT::Other}, Ops: {Chain, Op})
1578 : getNode(Opcode: ISD::STRICT_FP_ROUND, DL, ResultTys: {VT, MVT::Other},
1579 Ops: {Chain, Op, getIntPtrConstant(Val: 0, DL, /*isTarget=*/true)});
1580
1581 return std::pair<SDValue, SDValue>(Res, SDValue(Res.getNode(), 1));
1582}
1583
1584SDValue SelectionDAG::getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1585 return VT.bitsGT(VT: Op.getValueType()) ?
1586 getNode(Opcode: ISD::ANY_EXTEND, DL, VT, Operand: Op) :
1587 getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Op);
1588}
1589
1590SDValue SelectionDAG::getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1591 return VT.bitsGT(VT: Op.getValueType()) ?
1592 getNode(Opcode: ISD::SIGN_EXTEND, DL, VT, Operand: Op) :
1593 getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Op);
1594}
1595
1596SDValue SelectionDAG::getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1597 return VT.bitsGT(VT: Op.getValueType()) ?
1598 getNode(Opcode: ISD::ZERO_EXTEND, DL, VT, Operand: Op) :
1599 getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Op);
1600}
1601
1602SDValue SelectionDAG::getBitcastedAnyExtOrTrunc(SDValue Op, const SDLoc &DL,
1603 EVT VT) {
1604 assert(!VT.isVector());
1605 auto Type = Op.getValueType();
1606 SDValue DestOp;
1607 if (Type == VT)
1608 return Op;
1609 auto Size = Op.getValueSizeInBits();
1610 DestOp = getBitcast(VT: EVT::getIntegerVT(Context&: *Context, BitWidth: Size), V: Op);
1611 if (DestOp.getValueType() == VT)
1612 return DestOp;
1613
1614 return getAnyExtOrTrunc(Op: DestOp, DL, VT);
1615}
1616
1617SDValue SelectionDAG::getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT,
1618 EVT OpVT) {
1619 if (VT.bitsLE(VT: Op.getValueType()))
1620 return getNode(Opcode: ISD::TRUNCATE, DL: SL, VT, Operand: Op);
1621
1622 TargetLowering::BooleanContent BType = TLI->getBooleanContents(Type: OpVT);
1623 return getNode(Opcode: TLI->getExtendForContent(Content: BType), DL: SL, VT, Operand: Op);
1624}
1625
1626SDValue SelectionDAG::getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) {
1627 EVT OpVT = Op.getValueType();
1628 assert(VT.isInteger() && OpVT.isInteger() &&
1629 "Cannot getZeroExtendInReg FP types");
1630 assert(VT.isVector() == OpVT.isVector() &&
1631 "getZeroExtendInReg type should be vector iff the operand "
1632 "type is vector!");
1633 assert((!VT.isVector() ||
1634 VT.getVectorElementCount() == OpVT.getVectorElementCount()) &&
1635 "Vector element counts must match in getZeroExtendInReg");
1636 assert(VT.getScalarType().bitsLE(OpVT.getScalarType()) && "Not extending!");
1637 if (OpVT == VT)
1638 return Op;
1639 // TODO: Use computeKnownBits instead of AssertZext.
1640 if (Op.getOpcode() == ISD::AssertZext && cast<VTSDNode>(Val: Op.getOperand(i: 1))
1641 ->getVT()
1642 .getScalarType()
1643 .bitsLE(VT: VT.getScalarType()))
1644 return Op;
1645 APInt Imm = APInt::getLowBitsSet(numBits: OpVT.getScalarSizeInBits(),
1646 loBitsSet: VT.getScalarSizeInBits());
1647 return getNode(Opcode: ISD::AND, DL, VT: OpVT, N1: Op, N2: getConstant(Val: Imm, DL, VT: OpVT));
1648}
1649
1650SDValue SelectionDAG::getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1651 // Only unsigned pointer semantics are supported right now. In the future this
1652 // might delegate to TLI to check pointer signedness.
1653 return getZExtOrTrunc(Op, DL, VT);
1654}
1655
1656SDValue SelectionDAG::getPtrExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) {
1657 // Only unsigned pointer semantics are supported right now. In the future this
1658 // might delegate to TLI to check pointer signedness.
1659 return getZeroExtendInReg(Op, DL, VT);
1660}
1661
1662SDValue SelectionDAG::getNegative(SDValue Val, const SDLoc &DL, EVT VT) {
1663 return getNode(Opcode: ISD::SUB, DL, VT, N1: getConstant(Val: 0, DL, VT), N2: Val);
1664}
1665
1666/// getNOT - Create a bitwise NOT operation as (XOR Val, -1).
1667SDValue SelectionDAG::getNOT(const SDLoc &DL, SDValue Val, EVT VT) {
1668 return getNode(Opcode: ISD::XOR, DL, VT, N1: Val, N2: getAllOnesConstant(DL, VT));
1669}
1670
1671SDValue SelectionDAG::getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT) {
1672 SDValue TrueValue = getBoolConstant(V: true, DL, VT, OpVT: VT);
1673 return getNode(Opcode: ISD::XOR, DL, VT, N1: Val, N2: TrueValue);
1674}
1675
1676SDValue SelectionDAG::getBoolConstant(bool V, const SDLoc &DL, EVT VT,
1677 EVT OpVT) {
1678 if (!V)
1679 return getConstant(Val: 0, DL, VT);
1680
1681 switch (TLI->getBooleanContents(Type: OpVT)) {
1682 case TargetLowering::ZeroOrOneBooleanContent:
1683 case TargetLowering::UndefinedBooleanContent:
1684 return getConstant(Val: 1, DL, VT);
1685 case TargetLowering::ZeroOrNegativeOneBooleanContent:
1686 return getAllOnesConstant(DL, VT);
1687 }
1688 llvm_unreachable("Unexpected boolean content enum!");
1689}
1690
1691SDValue SelectionDAG::getConstant(uint64_t Val, const SDLoc &DL, EVT VT,
1692 bool isT, bool isO) {
1693 return getConstant(Val: APInt(VT.getScalarSizeInBits(), Val, /*isSigned=*/false),
1694 DL, VT, isTarget: isT, isOpaque: isO);
1695}
1696
1697SDValue SelectionDAG::getConstant(const APInt &Val, const SDLoc &DL, EVT VT,
1698 bool isT, bool isO) {
1699 return getConstant(Val: *ConstantInt::get(Context&: *Context, V: Val), DL, VT, isTarget: isT, isOpaque: isO);
1700}
1701
1702SDValue SelectionDAG::getConstant(const ConstantInt &Val, const SDLoc &DL,
1703 EVT VT, bool isT, bool isO) {
1704 assert(VT.isInteger() && "Cannot create FP integer constant!");
1705
1706 EVT EltVT = VT.getScalarType();
1707 const ConstantInt *Elt = &Val;
1708
1709 // Vector splats are explicit within the DAG, with ConstantSDNode holding the
1710 // to-be-splatted scalar ConstantInt.
1711 if (isa<VectorType>(Val: Elt->getType()))
1712 Elt = ConstantInt::get(Context&: *getContext(), V: Elt->getValue());
1713
1714 // In some cases the vector type is legal but the element type is illegal and
1715 // needs to be promoted, for example v8i8 on ARM. In this case, promote the
1716 // inserted value (the type does not need to match the vector element type).
1717 // Any extra bits introduced will be truncated away.
1718 if (VT.isVector() && TLI->getTypeAction(Context&: *getContext(), VT: EltVT) ==
1719 TargetLowering::TypePromoteInteger) {
1720 EltVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT: EltVT);
1721 APInt NewVal;
1722 if (TLI->isSExtCheaperThanZExt(FromTy: VT.getScalarType(), ToTy: EltVT))
1723 NewVal = Elt->getValue().sextOrTrunc(width: EltVT.getSizeInBits());
1724 else
1725 NewVal = Elt->getValue().zextOrTrunc(width: EltVT.getSizeInBits());
1726 Elt = ConstantInt::get(Context&: *getContext(), V: NewVal);
1727 }
1728 // In other cases the element type is illegal and needs to be expanded, for
1729 // example v2i64 on MIPS32. In this case, find the nearest legal type, split
1730 // the value into n parts and use a vector type with n-times the elements.
1731 // Then bitcast to the type requested.
1732 // Legalizing constants too early makes the DAGCombiner's job harder so we
1733 // only legalize if the DAG tells us we must produce legal types.
1734 else if (NewNodesMustHaveLegalTypes && VT.isVector() &&
1735 TLI->getTypeAction(Context&: *getContext(), VT: EltVT) ==
1736 TargetLowering::TypeExpandInteger) {
1737 const APInt &NewVal = Elt->getValue();
1738 EVT ViaEltVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT: EltVT);
1739 unsigned ViaEltSizeInBits = ViaEltVT.getSizeInBits();
1740
1741 // For scalable vectors, try to use a SPLAT_VECTOR_PARTS node.
1742 if (VT.isScalableVector() ||
1743 TLI->isOperationLegal(Op: ISD::SPLAT_VECTOR, VT)) {
1744 assert(EltVT.getSizeInBits() % ViaEltSizeInBits == 0 &&
1745 "Can only handle an even split!");
1746 unsigned Parts = EltVT.getSizeInBits() / ViaEltSizeInBits;
1747
1748 SmallVector<SDValue, 2> ScalarParts;
1749 for (unsigned i = 0; i != Parts; ++i)
1750 ScalarParts.push_back(Elt: getConstant(
1751 Val: NewVal.extractBits(numBits: ViaEltSizeInBits, bitPosition: i * ViaEltSizeInBits), DL,
1752 VT: ViaEltVT, isT, isO));
1753
1754 return getNode(Opcode: ISD::SPLAT_VECTOR_PARTS, DL, VT, Ops: ScalarParts);
1755 }
1756
1757 unsigned ViaVecNumElts = VT.getSizeInBits() / ViaEltSizeInBits;
1758 EVT ViaVecVT = EVT::getVectorVT(Context&: *getContext(), VT: ViaEltVT, NumElements: ViaVecNumElts);
1759
1760 // Check the temporary vector is the correct size. If this fails then
1761 // getTypeToTransformTo() probably returned a type whose size (in bits)
1762 // isn't a power-of-2 factor of the requested type size.
1763 assert(ViaVecVT.getSizeInBits() == VT.getSizeInBits());
1764
1765 SmallVector<SDValue, 2> EltParts;
1766 for (unsigned i = 0; i < ViaVecNumElts / VT.getVectorNumElements(); ++i)
1767 EltParts.push_back(Elt: getConstant(
1768 Val: NewVal.extractBits(numBits: ViaEltSizeInBits, bitPosition: i * ViaEltSizeInBits), DL,
1769 VT: ViaEltVT, isT, isO));
1770
1771 // EltParts is currently in little endian order. If we actually want
1772 // big-endian order then reverse it now.
1773 if (getDataLayout().isBigEndian())
1774 std::reverse(first: EltParts.begin(), last: EltParts.end());
1775
1776 // The elements must be reversed when the element order is different
1777 // to the endianness of the elements (because the BITCAST is itself a
1778 // vector shuffle in this situation). However, we do not need any code to
1779 // perform this reversal because getConstant() is producing a vector
1780 // splat.
1781 // This situation occurs in MIPS MSA.
1782
1783 SmallVector<SDValue, 8> Ops;
1784 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i)
1785 llvm::append_range(C&: Ops, R&: EltParts);
1786
1787 SDValue V =
1788 getNode(Opcode: ISD::BITCAST, DL, VT, Operand: getBuildVector(VT: ViaVecVT, DL, Ops));
1789 return V;
1790 }
1791
1792 assert(Elt->getBitWidth() == EltVT.getSizeInBits() &&
1793 "APInt size does not match type size!");
1794 unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant;
1795 SDVTList VTs = getVTList(VT: EltVT);
1796 SDNodeKey ID(Opc, VTs, {});
1797 ID.AddPointer(P: Elt);
1798 ID.AddBoolean(B: isO);
1799 FoldingSetInsertToken InsertToken;
1800 SDNode *N = nullptr;
1801 if ((N = lookupNode(Key: ID, DL, InsertToken)))
1802 if (!VT.isVector())
1803 return SDValue(N, 0);
1804
1805 if (!N) {
1806 N = newSDNode<ConstantSDNode>(Args&: isT, Args&: isO, Args&: Elt, Args&: VTs);
1807 if (!isT)
1808 N->setDebugLoc(DL.getDebugLoc());
1809 CSEMap.insert(N, Token: InsertToken);
1810 InsertNode(N);
1811 NewSDValueDbgMsg(V: SDValue(N, 0), Msg: "Creating constant: ", G: this);
1812 }
1813
1814 SDValue Result(N, 0);
1815 if (VT.isVector())
1816 Result = getSplat(VT, DL, Op: Result);
1817 return Result;
1818}
1819
1820SDValue SelectionDAG::getSignedConstant(int64_t Val, const SDLoc &DL, EVT VT,
1821 bool isT, bool isO) {
1822 unsigned Size = VT.getScalarSizeInBits();
1823 return getConstant(Val: APInt(Size, Val, /*isSigned=*/true), DL, VT, isT, isO);
1824}
1825
1826SDValue SelectionDAG::getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget,
1827 bool IsOpaque) {
1828 return getConstant(Val: APInt::getAllOnes(numBits: VT.getScalarSizeInBits()), DL, VT,
1829 isT: IsTarget, isO: IsOpaque);
1830}
1831
1832SDValue SelectionDAG::getIntPtrConstant(uint64_t Val, const SDLoc &DL,
1833 bool isTarget) {
1834 return getConstant(Val, DL, VT: TLI->getPointerTy(DL: getDataLayout()), isT: isTarget);
1835}
1836
1837SDValue SelectionDAG::getShiftAmountConstant(uint64_t Val, EVT VT,
1838 const SDLoc &DL) {
1839 assert(VT.isInteger() && "Shift amount is not an integer type!");
1840 EVT ShiftVT = TLI->getShiftAmountTy(LHSTy: VT, DL: getDataLayout());
1841 return getConstant(Val, DL, VT: ShiftVT);
1842}
1843
1844SDValue SelectionDAG::getShiftAmountConstant(const APInt &Val, EVT VT,
1845 const SDLoc &DL) {
1846 assert(Val.ult(VT.getScalarSizeInBits()) && "Out of range shift");
1847 return getShiftAmountConstant(Val: Val.getZExtValue(), VT, DL);
1848}
1849
1850SDValue SelectionDAG::getVectorIdxConstant(uint64_t Val, const SDLoc &DL,
1851 bool isTarget) {
1852 return getConstant(Val, DL, VT: TLI->getVectorIdxTy(DL: getDataLayout()), isT: isTarget);
1853}
1854
1855SDValue SelectionDAG::getConstantFP(const APFloat &V, const SDLoc &DL, EVT VT,
1856 bool isTarget) {
1857 return getConstantFP(V: *ConstantFP::get(Context&: *getContext(), V), DL, VT, isTarget);
1858}
1859
1860SDValue SelectionDAG::getConstantFP(const ConstantFP &V, const SDLoc &DL,
1861 EVT VT, bool isTarget) {
1862 assert(VT.isFloatingPoint() && "Cannot create integer FP constant!");
1863
1864 EVT EltVT = VT.getScalarType();
1865 const ConstantFP *Elt = &V;
1866
1867 // Vector splats are explicit within the DAG, with ConstantFPSDNode holding
1868 // the to-be-splatted scalar ConstantFP.
1869 if (isa<VectorType>(Val: Elt->getType()))
1870 Elt = ConstantFP::get(Context&: *getContext(), V: Elt->getValue());
1871
1872 // Do the map lookup using the actual bit pattern for the floating point
1873 // value, so that we don't have problems with 0.0 comparing equal to -0.0, and
1874 // we don't have issues with SNANs.
1875 unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP;
1876 SDVTList VTs = getVTList(VT: EltVT);
1877 SDNodeKey ID(Opc, VTs, {});
1878 ID.AddPointer(P: Elt);
1879 FoldingSetInsertToken InsertToken;
1880 SDNode *N = nullptr;
1881 if ((N = lookupNode(Key: ID, DL, InsertToken)))
1882 if (!VT.isVector())
1883 return SDValue(N, 0);
1884
1885 if (!N) {
1886 N = newSDNode<ConstantFPSDNode>(Args&: isTarget, Args&: Elt, Args&: VTs);
1887 CSEMap.insert(N, Token: InsertToken);
1888 InsertNode(N);
1889 }
1890
1891 SDValue Result(N, 0);
1892 if (VT.isVector())
1893 Result = getSplat(VT, DL, Op: Result);
1894 NewSDValueDbgMsg(V: Result, Msg: "Creating fp constant: ", G: this);
1895 return Result;
1896}
1897
1898SDValue SelectionDAG::getConstantFP(double Val, const SDLoc &DL, EVT VT,
1899 bool isTarget) {
1900 EVT EltVT = VT.getScalarType();
1901 if (EltVT == MVT::f32)
1902 return getConstantFP(V: APFloat((float)Val), DL, VT, isTarget);
1903 if (EltVT == MVT::f64)
1904 return getConstantFP(V: APFloat(Val), DL, VT, isTarget);
1905 if (EltVT == MVT::f80 || EltVT == MVT::f128 || EltVT == MVT::ppcf128 ||
1906 EltVT == MVT::f16 || EltVT == MVT::bf16) {
1907 bool Ignored;
1908 APFloat APF = APFloat(Val);
1909 APF.convert(ToSemantics: EltVT.getFltSemantics(), RM: APFloat::rmNearestTiesToEven,
1910 losesInfo: &Ignored);
1911 return getConstantFP(V: APF, DL, VT, isTarget);
1912 }
1913 llvm_unreachable("Unsupported type in getConstantFP");
1914}
1915
1916SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, const SDLoc &DL,
1917 EVT VT, int64_t Offset, bool isTargetGA,
1918 unsigned TargetFlags) {
1919 assert((TargetFlags == 0 || isTargetGA) &&
1920 "Cannot set target flags on target-independent globals");
1921
1922 // Truncate (with sign-extension) the offset value to the pointer size.
1923 unsigned BitWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType());
1924 if (BitWidth < 64)
1925 Offset = SignExtend64(X: Offset, B: BitWidth);
1926
1927 unsigned Opc;
1928 if (GV->isThreadLocal())
1929 Opc = isTargetGA ? ISD::TargetGlobalTLSAddress : ISD::GlobalTLSAddress;
1930 else
1931 Opc = isTargetGA ? ISD::TargetGlobalAddress : ISD::GlobalAddress;
1932
1933 SDVTList VTs = getVTList(VT);
1934 SDNodeKey ID(Opc, VTs, {});
1935 ID.AddPointer(P: GV);
1936 ID.AddInteger(I: Offset);
1937 ID.AddInteger(I: TargetFlags);
1938 FoldingSetInsertToken InsertToken;
1939 if (SDNode *E = lookupNode(Key: ID, DL, InsertToken))
1940 return SDValue(E, 0);
1941
1942 auto *N = newSDNode<GlobalAddressSDNode>(
1943 Args&: Opc, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: GV, Args&: VTs, Args&: Offset, Args&: TargetFlags);
1944 CSEMap.insert(N, Token: InsertToken);
1945 InsertNode(N);
1946 return SDValue(N, 0);
1947}
1948
1949SDValue SelectionDAG::getDeactivationSymbol(const GlobalValue *GV) {
1950 SDVTList VTs = getVTList(VT: MVT::Untyped);
1951 SDNodeKey ID(ISD::DEACTIVATION_SYMBOL, VTs, {});
1952 ID.AddPointer(P: GV);
1953 FoldingSetInsertToken InsertToken;
1954 if (SDNode *E = lookupNode(Key: ID, DL: SDLoc(), InsertToken))
1955 return SDValue(E, 0);
1956
1957 auto *N = newSDNode<DeactivationSymbolSDNode>(Args&: GV, Args&: VTs);
1958 CSEMap.insert(N, Token: InsertToken);
1959 InsertNode(N);
1960 return SDValue(N, 0);
1961}
1962
1963SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) {
1964 unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex;
1965 SDVTList VTs = getVTList(VT);
1966 SDNodeKey ID(Opc, VTs, {});
1967 ID.AddInteger(I: FI);
1968 FoldingSetInsertToken InsertToken;
1969 if (SDNode *E = lookupNode(Key: ID, InsertToken))
1970 return SDValue(E, 0);
1971
1972 auto *N = newSDNode<FrameIndexSDNode>(Args&: FI, Args&: VTs, Args&: isTarget);
1973 CSEMap.insert(N, Token: InsertToken);
1974 InsertNode(N);
1975 return SDValue(N, 0);
1976}
1977
1978SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget,
1979 unsigned TargetFlags) {
1980 assert((TargetFlags == 0 || isTarget) &&
1981 "Cannot set target flags on target-independent jump tables");
1982 unsigned Opc = isTarget ? ISD::TargetJumpTable : ISD::JumpTable;
1983 SDVTList VTs = getVTList(VT);
1984 SDNodeKey ID(Opc, VTs, {});
1985 ID.AddInteger(I: JTI);
1986 ID.AddInteger(I: TargetFlags);
1987 FoldingSetInsertToken InsertToken;
1988 if (SDNode *E = lookupNode(Key: ID, InsertToken))
1989 return SDValue(E, 0);
1990
1991 auto *N = newSDNode<JumpTableSDNode>(Args&: JTI, Args&: VTs, Args&: isTarget, Args&: TargetFlags);
1992 CSEMap.insert(N, Token: InsertToken);
1993 InsertNode(N);
1994 return SDValue(N, 0);
1995}
1996
1997SDValue SelectionDAG::getJumpTableDebugInfo(int JTI, SDValue Chain,
1998 const SDLoc &DL) {
1999 EVT PTy = getTargetLoweringInfo().getPointerTy(DL: getDataLayout());
2000 return getNode(Opcode: ISD::JUMP_TABLE_DEBUG_INFO, DL, VT: MVT::Other, N1: Chain,
2001 N2: getTargetConstant(Val: static_cast<uint64_t>(JTI), DL, VT: PTy, isOpaque: true));
2002}
2003
2004SDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT,
2005 MaybeAlign Alignment, int Offset,
2006 bool isTarget, unsigned TargetFlags) {
2007 assert((TargetFlags == 0 || isTarget) &&
2008 "Cannot set target flags on target-independent globals");
2009 if (!Alignment)
2010 Alignment = shouldOptForSize()
2011 ? getDataLayout().getABITypeAlign(Ty: C->getType())
2012 : getDataLayout().getPrefTypeAlign(Ty: C->getType());
2013 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
2014 SDVTList VTs = getVTList(VT);
2015 SDNodeKey ID(Opc, VTs, {});
2016 ID.AddInteger(I: Alignment->value());
2017 ID.AddInteger(I: Offset);
2018 ID.AddPointer(P: C);
2019 ID.AddInteger(I: TargetFlags);
2020 FoldingSetInsertToken InsertToken;
2021 if (SDNode *E = lookupNode(Key: ID, InsertToken))
2022 return SDValue(E, 0);
2023
2024 auto *N = newSDNode<ConstantPoolSDNode>(Args&: isTarget, Args&: C, Args&: VTs, Args&: Offset, Args&: *Alignment,
2025 Args&: TargetFlags);
2026 CSEMap.insert(N, Token: InsertToken);
2027 InsertNode(N);
2028 SDValue V = SDValue(N, 0);
2029 NewSDValueDbgMsg(V, Msg: "Creating new constant pool: ", G: this);
2030 return V;
2031}
2032
2033SDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT,
2034 MaybeAlign Alignment, int Offset,
2035 bool isTarget, unsigned TargetFlags) {
2036 assert((TargetFlags == 0 || isTarget) &&
2037 "Cannot set target flags on target-independent globals");
2038 if (!Alignment)
2039 Alignment = getDataLayout().getPrefTypeAlign(Ty: C->getType());
2040 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
2041 SDVTList VTs = getVTList(VT);
2042 SDNodeKey ID(Opc, VTs, {});
2043 ID.AddInteger(I: Alignment->value());
2044 ID.AddInteger(I: Offset);
2045 C->addSelectionDAGCSEId(ID&: ID.Tail);
2046 ID.AddInteger(I: TargetFlags);
2047 FoldingSetInsertToken InsertToken;
2048 if (SDNode *E = lookupNode(Key: ID, InsertToken))
2049 return SDValue(E, 0);
2050
2051 auto *N = newSDNode<ConstantPoolSDNode>(Args&: isTarget, Args&: C, Args&: VTs, Args&: Offset, Args&: *Alignment,
2052 Args&: TargetFlags);
2053 CSEMap.insert(N, Token: InsertToken);
2054 InsertNode(N);
2055 return SDValue(N, 0);
2056}
2057
2058SDValue SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) {
2059 SDNodeKey ID(ISD::BasicBlock, getVTList(VT: MVT::Other), {});
2060 ID.AddPointer(P: MBB);
2061 FoldingSetInsertToken InsertToken;
2062 if (SDNode *E = lookupNode(Key: ID, InsertToken))
2063 return SDValue(E, 0);
2064
2065 auto *N = newSDNode<BasicBlockSDNode>(Args&: MBB);
2066 CSEMap.insert(N, Token: InsertToken);
2067 InsertNode(N);
2068 return SDValue(N, 0);
2069}
2070
2071SDValue SelectionDAG::getValueType(EVT VT) {
2072 if (VT.isSimple() && (unsigned)VT.getSimpleVT().SimpleTy >=
2073 ValueTypeNodes.size())
2074 ValueTypeNodes.resize(new_size: VT.getSimpleVT().SimpleTy+1);
2075
2076 SDNode *&N = VT.isExtended() ?
2077 ExtendedValueTypeNodes[VT] : ValueTypeNodes[VT.getSimpleVT().SimpleTy];
2078
2079 if (N) return SDValue(N, 0);
2080 N = newSDNode<VTSDNode>(Args&: VT);
2081 InsertNode(N);
2082 return SDValue(N, 0);
2083}
2084
2085SDValue SelectionDAG::getExternalSymbol(const char *Sym, EVT VT) {
2086 SDNode *&N = ExternalSymbols[Sym];
2087 if (N) return SDValue(N, 0);
2088 N = newSDNode<ExternalSymbolSDNode>(Args: false, Args&: Sym, Args: 0, Args: getVTList(VT));
2089 InsertNode(N);
2090 return SDValue(N, 0);
2091}
2092
2093SDValue SelectionDAG::getExternalSymbol(RTLIB::LibcallImpl Libcall, EVT VT) {
2094 StringRef SymName = RTLIB::RuntimeLibcallsInfo::getLibcallImplName(CallImpl: Libcall);
2095 return getExternalSymbol(Sym: SymName.data(), VT);
2096}
2097
2098SDValue SelectionDAG::getMCSymbol(MCSymbol *Sym, EVT VT) {
2099 SDNode *&N = MCSymbols[Sym];
2100 if (N)
2101 return SDValue(N, 0);
2102 N = newSDNode<MCSymbolSDNode>(Args&: Sym, Args: getVTList(VT));
2103 InsertNode(N);
2104 return SDValue(N, 0);
2105}
2106
2107SDValue SelectionDAG::getTargetExternalSymbol(const char *Sym, EVT VT,
2108 unsigned TargetFlags) {
2109 SDNode *&N =
2110 TargetExternalSymbols[std::pair<std::string, unsigned>(Sym, TargetFlags)];
2111 if (N) return SDValue(N, 0);
2112 N = newSDNode<ExternalSymbolSDNode>(Args: true, Args&: Sym, Args&: TargetFlags, Args: getVTList(VT));
2113 InsertNode(N);
2114 return SDValue(N, 0);
2115}
2116
2117SDValue SelectionDAG::getTargetExternalSymbol(RTLIB::LibcallImpl Libcall,
2118 EVT VT, unsigned TargetFlags) {
2119 StringRef SymName = RTLIB::RuntimeLibcallsInfo::getLibcallImplName(CallImpl: Libcall);
2120 return getTargetExternalSymbol(Sym: SymName.data(), VT, TargetFlags);
2121}
2122
2123SDValue SelectionDAG::getCondCode(ISD::CondCode Cond) {
2124 if ((unsigned)Cond >= CondCodeNodes.size())
2125 CondCodeNodes.resize(new_size: Cond+1);
2126
2127 if (!CondCodeNodes[Cond]) {
2128 auto *N = newSDNode<CondCodeSDNode>(Args&: Cond);
2129 CondCodeNodes[Cond] = N;
2130 InsertNode(N);
2131 }
2132
2133 return SDValue(CondCodeNodes[Cond], 0);
2134}
2135
2136SDValue SelectionDAG::getVScale(const SDLoc &DL, EVT VT, APInt MulImm) {
2137 assert(MulImm.getBitWidth() == VT.getSizeInBits() &&
2138 "APInt size does not match type size!");
2139
2140 if (MulImm == 0)
2141 return getConstant(Val: 0, DL, VT);
2142
2143 const MachineFunction &MF = getMachineFunction();
2144 const Function &F = MF.getFunction();
2145 ConstantRange CR = getVScaleRange(F: &F, BitWidth: 64);
2146 if (const APInt *C = CR.getSingleElement())
2147 return getConstant(Val: MulImm * C->getZExtValue(), DL, VT);
2148
2149 return getNode(Opcode: ISD::VSCALE, DL, VT, Operand: getConstant(Val: MulImm, DL, VT));
2150}
2151
2152/// \returns a value of type \p VT that represents the runtime value of \p
2153/// Quantity, i.e. scaled by vscale if it's scalable, or a fixed constant
2154/// otherwise. Quantity should be a FixedOrScalableQuantity, i.e. ElementCount
2155/// or TypeSize.
2156template <typename Ty>
2157static SDValue getFixedOrScalableQuantity(SelectionDAG &DAG, const SDLoc &DL,
2158 EVT VT, Ty Quantity) {
2159 if (Quantity.isScalable())
2160 return DAG.getVScale(
2161 DL, VT, MulImm: APInt(VT.getSizeInBits(), Quantity.getKnownMinValue()));
2162
2163 return DAG.getConstant(Quantity.getKnownMinValue(), DL, VT);
2164}
2165
2166SDValue SelectionDAG::getElementCount(const SDLoc &DL, EVT VT,
2167 ElementCount EC) {
2168 return getFixedOrScalableQuantity(DAG&: *this, DL, VT, Quantity: EC);
2169}
2170
2171SDValue SelectionDAG::getTypeSize(const SDLoc &DL, EVT VT, TypeSize TS) {
2172 return getFixedOrScalableQuantity(DAG&: *this, DL, VT, Quantity: TS);
2173}
2174
2175SDValue SelectionDAG::getMaskFromElementCount(const SDLoc &DL, EVT DataVT,
2176 ElementCount EC) {
2177 EVT IdxVT = TLI->getVectorIdxTy(DL: getDataLayout());
2178 EVT MaskVT = TLI->getSetCCResultType(DL: getDataLayout(), Context&: *getContext(), VT: DataVT);
2179 return getNode(Opcode: ISD::GET_ACTIVE_LANE_MASK, DL, VT: MaskVT,
2180 N1: getConstant(Val: 0, DL, VT: IdxVT), N2: getElementCount(DL, VT: IdxVT, EC));
2181}
2182
2183SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT) {
2184 APInt One(ResVT.getScalarSizeInBits(), 1);
2185 return getStepVector(DL, ResVT, StepVal: One);
2186}
2187
2188SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT,
2189 const APInt &StepVal) {
2190 assert(ResVT.getScalarSizeInBits() == StepVal.getBitWidth());
2191 if (ResVT.isScalableVector())
2192 return getNode(
2193 Opcode: ISD::STEP_VECTOR, DL, VT: ResVT,
2194 Operand: getTargetConstant(Val: StepVal, DL, VT: ResVT.getVectorElementType()));
2195
2196 SmallVector<SDValue, 16> OpsStepConstants;
2197 for (uint64_t i = 0; i < ResVT.getVectorNumElements(); i++)
2198 OpsStepConstants.push_back(
2199 Elt: getConstant(Val: StepVal * i, DL, VT: ResVT.getVectorElementType()));
2200 return getBuildVector(VT: ResVT, DL, Ops: OpsStepConstants);
2201}
2202
2203/// Swaps the values of N1 and N2. Swaps all indices in the shuffle mask M that
2204/// point at N1 to point at N2 and indices that point at N2 to point at N1.
2205static void commuteShuffle(SDValue &N1, SDValue &N2, MutableArrayRef<int> M) {
2206 std::swap(a&: N1, b&: N2);
2207 ShuffleVectorSDNode::commuteMask(Mask: M);
2208}
2209
2210SDValue SelectionDAG::getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1,
2211 SDValue N2, ArrayRef<int> Mask) {
2212 assert(VT.getVectorNumElements() == Mask.size() &&
2213 "Must have the same number of vector elements as mask elements!");
2214 assert(VT == N1.getValueType() && VT == N2.getValueType() &&
2215 "Invalid VECTOR_SHUFFLE");
2216
2217 // Canonicalize shuffle undef, undef -> undef
2218 if (N1.isUndef() && N2.isUndef()) {
2219 if (N1.getOpcode() == ISD::POISON && N2.getOpcode() == ISD::POISON)
2220 return getPOISON(VT);
2221 return getUNDEF(VT);
2222 }
2223
2224 // Validate that all indices in Mask are within the range of the elements
2225 // input to the shuffle.
2226 int NElts = Mask.size();
2227 assert(llvm::all_of(Mask,
2228 [&](int M) { return M < (NElts * 2) && M >= -1; }) &&
2229 "Index out of range");
2230
2231 // Copy the mask so we can do any needed cleanup.
2232 SmallVector<int, 8> MaskVec(Mask);
2233
2234 // Canonicalize shuffle v, v -> v, poison
2235 if (N1 == N2) {
2236 N2 = getPOISON(VT);
2237 for (int i = 0; i != NElts; ++i)
2238 if (MaskVec[i] >= NElts) MaskVec[i] -= NElts;
2239 }
2240
2241 // Canonicalize shuffle undef, v -> v, undef. Commute the shuffle mask.
2242 if (N1.isUndef())
2243 commuteShuffle(N1, N2, M: MaskVec);
2244
2245 if (TLI->hasVectorBlend()) {
2246 // If shuffling a splat, try to blend the splat instead. We do this here so
2247 // that even when this arises during lowering we don't have to re-handle it.
2248 auto BlendSplat = [&](BuildVectorSDNode *BV, int Offset) {
2249 BitVector UndefElements;
2250 SDValue Splat = BV->getSplatValue(UndefElements: &UndefElements);
2251 if (!Splat)
2252 return;
2253
2254 for (int i = 0; i < NElts; ++i) {
2255 if (MaskVec[i] < Offset || MaskVec[i] >= (Offset + NElts))
2256 continue;
2257
2258 // If this input comes from undef, mark it as such.
2259 if (UndefElements[MaskVec[i] - Offset]) {
2260 MaskVec[i] = -1;
2261 continue;
2262 }
2263
2264 // If we can blend a non-undef lane, use that instead.
2265 if (!UndefElements[i])
2266 MaskVec[i] = i + Offset;
2267 }
2268 };
2269 if (auto *N1BV = dyn_cast<BuildVectorSDNode>(Val&: N1))
2270 BlendSplat(N1BV, 0);
2271 if (auto *N2BV = dyn_cast<BuildVectorSDNode>(Val&: N2))
2272 BlendSplat(N2BV, NElts);
2273 }
2274
2275 // Canonicalize all index into lhs, -> shuffle lhs, poison
2276 // Canonicalize all index into rhs, -> shuffle rhs, poison
2277 bool AllLHS = true, AllRHS = true;
2278 bool N2Undef = N2.isUndef();
2279 for (int i = 0; i != NElts; ++i) {
2280 if (MaskVec[i] >= NElts) {
2281 if (N2Undef)
2282 MaskVec[i] = -1;
2283 else
2284 AllLHS = false;
2285 } else if (MaskVec[i] >= 0) {
2286 AllRHS = false;
2287 }
2288 }
2289 if (AllLHS && AllRHS)
2290 return getPOISON(VT);
2291 if (AllLHS && !N2Undef)
2292 N2 = getPOISON(VT);
2293 if (AllRHS) {
2294 N1 = getPOISON(VT);
2295 commuteShuffle(N1, N2, M: MaskVec);
2296 }
2297 // Reset our undef status after accounting for the mask.
2298 N2Undef = N2.isUndef();
2299 // Re-check whether both sides ended up undef.
2300 if (N1.isUndef() && N2Undef) {
2301 if (N1.getOpcode() == ISD::POISON && N2.getOpcode() == ISD::POISON)
2302 return getPOISON(VT);
2303 return getUNDEF(VT);
2304 }
2305
2306 // If Identity shuffle return that node.
2307 bool Identity = true, AllSame = true;
2308 for (int i = 0; i != NElts; ++i) {
2309 if (MaskVec[i] >= 0 && MaskVec[i] != i) Identity = false;
2310 if (MaskVec[i] != MaskVec[0]) AllSame = false;
2311 }
2312 if (Identity && NElts)
2313 return N1;
2314
2315 // Shuffling a constant splat doesn't change the result.
2316 if (N2Undef) {
2317 SDValue V = N1;
2318
2319 // Look through any bitcasts. We check that these don't change the number
2320 // (and size) of elements and just changes their types.
2321 while (V.getOpcode() == ISD::BITCAST)
2322 V = V->getOperand(Num: 0);
2323
2324 // A splat should always show up as a build vector node.
2325 if (auto *BV = dyn_cast<BuildVectorSDNode>(Val&: V)) {
2326 BitVector UndefElements;
2327 SDValue Splat = BV->getSplatValue(UndefElements: &UndefElements);
2328 // If this is a splat of an undef, shuffling it is also undef.
2329 if (Splat && Splat.isUndef())
2330 return Splat.getOpcode() == ISD::POISON ? getPOISON(VT) : getUNDEF(VT);
2331
2332 bool SameNumElts =
2333 V.getValueType().getVectorNumElements() == VT.getVectorNumElements();
2334
2335 // We only have a splat which can skip shuffles if there is a splatted
2336 // value and no undef lanes rearranged by the shuffle.
2337 if (Splat && UndefElements.none()) {
2338 // Splat of <x, x, ..., x>, return <x, x, ..., x>, provided that the
2339 // number of elements match or the value splatted is a zero constant.
2340 if (SameNumElts || isNullConstant(V: Splat))
2341 return N1;
2342 }
2343
2344 // If the shuffle itself creates a splat, build the vector directly.
2345 if (AllSame && SameNumElts) {
2346 EVT BuildVT = BV->getValueType(ResNo: 0);
2347 const SDValue &Splatted = BV->getOperand(Num: MaskVec[0]);
2348 SDValue NewBV = getSplatBuildVector(VT: BuildVT, DL: dl, Op: Splatted);
2349
2350 // We may have jumped through bitcasts, so the type of the
2351 // BUILD_VECTOR may not match the type of the shuffle.
2352 if (BuildVT != VT)
2353 NewBV = getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: NewBV);
2354 return NewBV;
2355 }
2356 }
2357 }
2358
2359 SDVTList VTs = getVTList(VT);
2360 SDValue Ops[2] = { N1, N2 };
2361 SDNodeKey ID(ISD::VECTOR_SHUFFLE, VTs, Ops);
2362 for (int i = 0; i != NElts; ++i)
2363 ID.AddInteger(I: MaskVec[i]);
2364
2365 FoldingSetInsertToken InsertToken;
2366 if (SDNode *E = lookupNode(Key: ID, DL: dl, InsertToken))
2367 return SDValue(E, 0);
2368
2369 // Allocate the mask array for the node out of the BumpPtrAllocator, since
2370 // SDNode doesn't have access to it. This memory will be "leaked" when
2371 // the node is deallocated, but recovered when the NodeAllocator is released.
2372 int *MaskAlloc = OperandAllocator.Allocate<int>(Num: NElts);
2373 llvm::copy(Range&: MaskVec, Out: MaskAlloc);
2374
2375 auto *N = newSDNode<ShuffleVectorSDNode>(Args&: VTs, Args: dl.getIROrder(),
2376 Args: dl.getDebugLoc(), Args&: MaskAlloc);
2377 createOperands(Node: N, Vals: Ops);
2378
2379 CSEMap.insert(N, Token: InsertToken);
2380 InsertNode(N);
2381 SDValue V = SDValue(N, 0);
2382 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
2383 return V;
2384}
2385
2386SDValue SelectionDAG::getCommutedVectorShuffle(const ShuffleVectorSDNode &SV) {
2387 EVT VT = SV.getValueType(ResNo: 0);
2388 SmallVector<int, 8> MaskVec(SV.getMask());
2389 ShuffleVectorSDNode::commuteMask(Mask: MaskVec);
2390
2391 SDValue Op0 = SV.getOperand(Num: 0);
2392 SDValue Op1 = SV.getOperand(Num: 1);
2393 return getVectorShuffle(VT, dl: SDLoc(&SV), N1: Op1, N2: Op0, Mask: MaskVec);
2394}
2395
2396SDValue SelectionDAG::getRegister(Register Reg, EVT VT) {
2397 SDVTList VTs = getVTList(VT);
2398 SDNodeKey ID(ISD::Register, VTs, {});
2399 ID.AddInteger(I: Reg.id());
2400 FoldingSetInsertToken InsertToken;
2401 if (SDNode *E = lookupNode(Key: ID, InsertToken))
2402 return SDValue(E, 0);
2403
2404 auto *N = newSDNode<RegisterSDNode>(Args&: Reg, Args&: VTs);
2405 N->SDNodeBits.IsDivergent = TLI->isSDNodeSourceOfDivergence(N, FLI, UA);
2406 CSEMap.insert(N, Token: InsertToken);
2407 InsertNode(N);
2408 return SDValue(N, 0);
2409}
2410
2411SDValue SelectionDAG::getRegisterMask(const uint32_t *RegMask) {
2412 SDNodeKey ID(ISD::RegisterMask, getVTList(VT: MVT::Untyped), {});
2413 ID.AddPointer(P: RegMask);
2414 FoldingSetInsertToken InsertToken;
2415 if (SDNode *E = lookupNode(Key: ID, InsertToken))
2416 return SDValue(E, 0);
2417
2418 auto *N = newSDNode<RegisterMaskSDNode>(Args&: RegMask);
2419 CSEMap.insert(N, Token: InsertToken);
2420 InsertNode(N);
2421 return SDValue(N, 0);
2422}
2423
2424SDValue SelectionDAG::getEHLabel(const SDLoc &dl, SDValue Root,
2425 MCSymbol *Label) {
2426 return getLabelNode(Opcode: ISD::EH_LABEL, dl, Root, Label);
2427}
2428
2429SDValue SelectionDAG::getLabelNode(unsigned Opcode, const SDLoc &dl,
2430 SDValue Root, MCSymbol *Label) {
2431 SDValue Ops[] = { Root };
2432 SDNodeKey ID(Opcode, getVTList(VT: MVT::Other), Ops);
2433 ID.AddPointer(P: Label);
2434 FoldingSetInsertToken InsertToken;
2435 if (SDNode *E = lookupNode(Key: ID, InsertToken))
2436 return SDValue(E, 0);
2437
2438 auto *N =
2439 newSDNode<LabelSDNode>(Args&: Opcode, Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: Label);
2440 createOperands(Node: N, Vals: Ops);
2441
2442 CSEMap.insert(N, Token: InsertToken);
2443 InsertNode(N);
2444 return SDValue(N, 0);
2445}
2446
2447SDValue SelectionDAG::getBlockAddress(const BlockAddress *BA, EVT VT,
2448 int64_t Offset, bool isTarget,
2449 unsigned TargetFlags) {
2450 unsigned Opc = isTarget ? ISD::TargetBlockAddress : ISD::BlockAddress;
2451 SDVTList VTs = getVTList(VT);
2452
2453 SDNodeKey ID(Opc, VTs, {});
2454 ID.AddPointer(P: BA);
2455 ID.AddInteger(I: Offset);
2456 ID.AddInteger(I: TargetFlags);
2457 FoldingSetInsertToken InsertToken;
2458 if (SDNode *E = lookupNode(Key: ID, InsertToken))
2459 return SDValue(E, 0);
2460
2461 auto *N = newSDNode<BlockAddressSDNode>(Args&: Opc, Args&: VTs, Args&: BA, Args&: Offset, Args&: TargetFlags);
2462 CSEMap.insert(N, Token: InsertToken);
2463 InsertNode(N);
2464 return SDValue(N, 0);
2465}
2466
2467SDValue SelectionDAG::getSrcValue(const Value *V) {
2468 SDNodeKey ID(ISD::SRCVALUE, getVTList(VT: MVT::Other), {});
2469 ID.AddPointer(P: V);
2470
2471 FoldingSetInsertToken InsertToken;
2472 if (SDNode *E = lookupNode(Key: ID, InsertToken))
2473 return SDValue(E, 0);
2474
2475 auto *N = newSDNode<SrcValueSDNode>(Args&: V);
2476 CSEMap.insert(N, Token: InsertToken);
2477 InsertNode(N);
2478 return SDValue(N, 0);
2479}
2480
2481SDValue SelectionDAG::getMDNode(const MDNode *MD) {
2482 SDNodeKey ID(ISD::MDNODE_SDNODE, getVTList(VT: MVT::Other), {});
2483 ID.AddPointer(P: MD);
2484
2485 FoldingSetInsertToken InsertToken;
2486 if (SDNode *E = lookupNode(Key: ID, InsertToken))
2487 return SDValue(E, 0);
2488
2489 auto *N = newSDNode<MDNodeSDNode>(Args&: MD);
2490 CSEMap.insert(N, Token: InsertToken);
2491 InsertNode(N);
2492 return SDValue(N, 0);
2493}
2494
2495SDValue SelectionDAG::getBitcast(EVT VT, SDValue V) {
2496 if (VT == V.getValueType())
2497 return V;
2498
2499 return getNode(Opcode: ISD::BITCAST, DL: SDLoc(V), VT, Operand: V);
2500}
2501
2502SDValue SelectionDAG::getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr,
2503 unsigned SrcAS, unsigned DestAS,
2504 const SDNodeFlags Flags) {
2505 SDVTList VTs = getVTList(VT);
2506 SDValue Ops[] = {Ptr};
2507 SDNodeKey ID(ISD::ADDRSPACECAST, VTs, Ops);
2508 ID.AddInteger(I: SrcAS);
2509 ID.AddInteger(I: DestAS);
2510
2511 FoldingSetInsertToken InsertToken;
2512 if (SDNode *E = lookupNode(Key: ID, DL: dl, InsertToken)) {
2513 E->intersectFlagsWith(Flags);
2514 return SDValue(E, 0);
2515 }
2516
2517 auto *N = newSDNode<AddrSpaceCastSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(),
2518 Args&: VTs, Args&: SrcAS, Args&: DestAS);
2519 N->setFlags(Flags);
2520 createOperands(Node: N, Vals: Ops);
2521
2522 CSEMap.insert(N, Token: InsertToken);
2523 InsertNode(N);
2524 return SDValue(N, 0);
2525}
2526
2527SDValue SelectionDAG::getFreeze(SDValue V) {
2528 return getNode(Opcode: ISD::FREEZE, DL: SDLoc(V), VT: V.getValueType(), Operand: V);
2529}
2530
2531SDValue SelectionDAG::getFreeze(SDValue V, const APInt &DemandedElts,
2532 UndefPoisonKind Kind) {
2533 if (isGuaranteedNotToBeUndefOrPoison(Op: V, DemandedElts, Kind))
2534 return V;
2535 return getFreeze(V);
2536}
2537
2538/// getShiftAmountOperand - Return the specified value casted to
2539/// the target's desired shift amount type.
2540SDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) {
2541 EVT OpTy = Op.getValueType();
2542 EVT ShTy = TLI->getShiftAmountTy(LHSTy, DL: getDataLayout());
2543 if (OpTy == ShTy || OpTy.isVector()) return Op;
2544
2545 return getZExtOrTrunc(Op, DL: SDLoc(Op), VT: ShTy);
2546}
2547
2548SDValue SelectionDAG::expandVAArg(SDNode *Node) {
2549 SDLoc dl(Node);
2550 const TargetLowering &TLI = getTargetLoweringInfo();
2551 const Value *V = cast<SrcValueSDNode>(Val: Node->getOperand(Num: 2))->getValue();
2552 EVT VT = Node->getValueType(ResNo: 0);
2553 SDValue Tmp1 = Node->getOperand(Num: 0);
2554 SDValue Tmp2 = Node->getOperand(Num: 1);
2555 const MaybeAlign MA(Node->getConstantOperandVal(Num: 3));
2556
2557 SDValue VAListLoad = getLoad(VT: TLI.getPointerTy(DL: getDataLayout()), dl, Chain: Tmp1,
2558 Ptr: Tmp2, PtrInfo: MachinePointerInfo(V));
2559 SDValue VAList = VAListLoad;
2560
2561 if (MA && *MA > TLI.getMinStackArgumentAlignment()) {
2562 VAList = getNode(Opcode: ISD::ADD, DL: dl, VT: VAList.getValueType(), N1: VAList,
2563 N2: getConstant(Val: MA->value() - 1, DL: dl, VT: VAList.getValueType()));
2564
2565 VAList = getNode(
2566 Opcode: ISD::AND, DL: dl, VT: VAList.getValueType(), N1: VAList,
2567 N2: getSignedConstant(Val: -(int64_t)MA->value(), DL: dl, VT: VAList.getValueType()));
2568 }
2569
2570 // Increment the pointer, VAList, to the next vaarg
2571 Tmp1 = getNode(Opcode: ISD::ADD, DL: dl, VT: VAList.getValueType(), N1: VAList,
2572 N2: getConstant(Val: getDataLayout().getTypeAllocSize(
2573 Ty: VT.getTypeForEVT(Context&: *getContext())),
2574 DL: dl, VT: VAList.getValueType()));
2575 // Store the incremented VAList to the legalized pointer
2576 Tmp1 =
2577 getStore(Chain: VAListLoad.getValue(R: 1), dl, Val: Tmp1, Ptr: Tmp2, PtrInfo: MachinePointerInfo(V));
2578 // Load the actual argument out of the pointer VAList
2579 return getLoad(VT, dl, Chain: Tmp1, Ptr: VAList, PtrInfo: MachinePointerInfo());
2580}
2581
2582SDValue SelectionDAG::expandVACopy(SDNode *Node) {
2583 SDLoc dl(Node);
2584 const TargetLowering &TLI = getTargetLoweringInfo();
2585 // This defaults to loading a pointer from the input and storing it to the
2586 // output, returning the chain.
2587 const Value *VD = cast<SrcValueSDNode>(Val: Node->getOperand(Num: 3))->getValue();
2588 const Value *VS = cast<SrcValueSDNode>(Val: Node->getOperand(Num: 4))->getValue();
2589 SDValue Tmp1 =
2590 getLoad(VT: TLI.getPointerTy(DL: getDataLayout()), dl, Chain: Node->getOperand(Num: 0),
2591 Ptr: Node->getOperand(Num: 2), PtrInfo: MachinePointerInfo(VS));
2592 return getStore(Chain: Tmp1.getValue(R: 1), dl, Val: Tmp1, Ptr: Node->getOperand(Num: 1),
2593 PtrInfo: MachinePointerInfo(VD));
2594}
2595
2596Align SelectionDAG::getReducedAlign(EVT VT, bool UseABI) {
2597 const DataLayout &DL = getDataLayout();
2598 Type *Ty = VT.getTypeForEVT(Context&: *getContext());
2599 Align RedAlign = UseABI ? DL.getABITypeAlign(Ty) : DL.getPrefTypeAlign(Ty);
2600
2601 if (TLI->isTypeLegal(VT) || !VT.isVector())
2602 return RedAlign;
2603
2604 const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering();
2605 const Align StackAlign = TFI->getStackAlign();
2606
2607 // See if we can choose a smaller ABI alignment in cases where it's an
2608 // illegal vector type that will get broken down.
2609 if (RedAlign > StackAlign) {
2610 EVT IntermediateVT;
2611 MVT RegisterVT;
2612 unsigned NumIntermediates;
2613 TLI->getVectorTypeBreakdown(Context&: *getContext(), VT, IntermediateVT,
2614 NumIntermediates, RegisterVT);
2615 Ty = IntermediateVT.getTypeForEVT(Context&: *getContext());
2616 Align RedAlign2 = UseABI ? DL.getABITypeAlign(Ty) : DL.getPrefTypeAlign(Ty);
2617 if (RedAlign2 < RedAlign)
2618 RedAlign = RedAlign2;
2619
2620 if (!getMachineFunction().getFrameInfo().isStackRealignable())
2621 // If the stack is not realignable, the alignment should be limited to the
2622 // StackAlignment
2623 RedAlign = std::min(a: RedAlign, b: StackAlign);
2624 }
2625
2626 return RedAlign;
2627}
2628
2629SDValue SelectionDAG::CreateStackTemporary(TypeSize Bytes, Align Alignment) {
2630 MachineFrameInfo &MFI = MF->getFrameInfo();
2631 const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering();
2632 int StackID = 0;
2633 if (Bytes.isScalable())
2634 StackID = TFI->getStackIDForScalableVectors();
2635 // The stack id gives an indication of whether the object is scalable or
2636 // not, so it's safe to pass in the minimum size here.
2637 int FrameIdx = MFI.CreateStackObject(Size: Bytes.getKnownMinValue(), Alignment,
2638 isSpillSlot: false, Alloca: nullptr, ID: StackID);
2639 return getFrameIndex(FI: FrameIdx, VT: TLI->getFrameIndexTy(DL: getDataLayout()));
2640}
2641
2642SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) {
2643 Type *Ty = VT.getTypeForEVT(Context&: *getContext());
2644 Align StackAlign =
2645 std::max(a: getDataLayout().getPrefTypeAlign(Ty), b: Align(minAlign));
2646 return CreateStackTemporary(Bytes: VT.getStoreSize(), Alignment: StackAlign);
2647}
2648
2649SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) {
2650 TypeSize VT1Size = VT1.getStoreSize();
2651 TypeSize VT2Size = VT2.getStoreSize();
2652 assert(VT1Size.isScalable() == VT2Size.isScalable() &&
2653 "Don't know how to choose the maximum size when creating a stack "
2654 "temporary");
2655 TypeSize Bytes = VT1Size.getKnownMinValue() > VT2Size.getKnownMinValue()
2656 ? VT1Size
2657 : VT2Size;
2658
2659 Type *Ty1 = VT1.getTypeForEVT(Context&: *getContext());
2660 Type *Ty2 = VT2.getTypeForEVT(Context&: *getContext());
2661 const DataLayout &DL = getDataLayout();
2662 Align Align = std::max(a: DL.getPrefTypeAlign(Ty: Ty1), b: DL.getPrefTypeAlign(Ty: Ty2));
2663 return CreateStackTemporary(Bytes, Alignment: Align);
2664}
2665
2666SDValue SelectionDAG::emitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
2667 const SDLoc &DL, SDValue Chain) {
2668 EVT SrcVT = SrcOp.getValueType();
2669 Type *DestType = DestVT.getTypeForEVT(Context&: *getContext());
2670 Align DestAlign = getDataLayout().getPrefTypeAlign(Ty: DestType);
2671
2672 // Create the stack frame object.
2673 Align SrcAlign =
2674 getDataLayout().getPrefTypeAlign(Ty: SrcVT.getTypeForEVT(Context&: *getContext()));
2675 SDValue FIPtr = CreateStackTemporary(Bytes: SlotVT.getStoreSize(), Alignment: SrcAlign);
2676
2677 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(Val&: FIPtr);
2678 int SPFI = StackPtrFI->getIndex();
2679 MachinePointerInfo PtrInfo =
2680 MachinePointerInfo::getFixedStack(MF&: getMachineFunction(), FI: SPFI);
2681
2682 // Emit a store to the stack slot. Use a truncstore if the input value is
2683 // later than DestVT.
2684 SDValue Store;
2685
2686 if (SrcVT.bitsGT(VT: SlotVT))
2687 Store = getTruncStore(Chain, dl: DL, Val: SrcOp, Ptr: FIPtr, PtrInfo, SVT: SlotVT, Alignment: SrcAlign);
2688 else {
2689 assert(SrcVT.bitsEq(SlotVT) && "Invalid store");
2690 Store = getStore(Chain, dl: DL, Val: SrcOp, Ptr: FIPtr, PtrInfo, Alignment: SrcAlign);
2691 }
2692
2693 // Result is a load from the stack slot.
2694 if (SlotVT.bitsEq(VT: DestVT))
2695 return getLoad(VT: DestVT, dl: DL, Chain: Store, Ptr: FIPtr, PtrInfo, Alignment: DestAlign);
2696
2697 assert(SlotVT.bitsLT(DestVT) && "Unknown extension!");
2698 return getExtLoad(ExtType: ISD::EXTLOAD, dl: DL, VT: DestVT, Chain: Store, Ptr: FIPtr, PtrInfo, MemVT: SlotVT,
2699 Alignment: DestAlign);
2700}
2701
2702SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1, SDValue N2,
2703 ISD::CondCode Cond, const SDLoc &dl,
2704 SDNodeFlags Flags) {
2705 EVT OpVT = N1.getValueType();
2706
2707 auto GetUndefBooleanConstant = [&]() {
2708 if (VT.getScalarType() == MVT::i1 ||
2709 TLI->getBooleanContents(Type: OpVT) ==
2710 TargetLowering::UndefinedBooleanContent)
2711 return getUNDEF(VT);
2712 // ZeroOrOne / ZeroOrNegative require specific values for the high bits,
2713 // so we cannot use getUNDEF(). Return zero instead.
2714 return getConstant(Val: 0, DL: dl, VT);
2715 };
2716
2717 // These setcc operations always fold.
2718 switch (Cond) {
2719 default: break;
2720 case ISD::SETFALSE:
2721 case ISD::SETFALSE2: return getBoolConstant(V: false, DL: dl, VT, OpVT);
2722 case ISD::SETTRUE:
2723 case ISD::SETTRUE2: return getBoolConstant(V: true, DL: dl, VT, OpVT);
2724
2725 case ISD::SETOEQ:
2726 case ISD::SETOGT:
2727 case ISD::SETOGE:
2728 case ISD::SETOLT:
2729 case ISD::SETOLE:
2730 case ISD::SETONE:
2731 case ISD::SETO:
2732 case ISD::SETUO:
2733 case ISD::SETUEQ:
2734 case ISD::SETUNE:
2735 assert(!OpVT.isInteger() && "Illegal setcc for integer!");
2736 break;
2737 }
2738
2739 if (OpVT.isInteger()) {
2740 // For EQ and NE, we can always pick a value for the undef to make the
2741 // predicate pass or fail, so we can return undef.
2742 // Matches behavior in llvm::ConstantFoldCompareInstruction.
2743 // icmp eq/ne X, undef -> undef.
2744 if ((N1.isUndef() || N2.isUndef()) &&
2745 (Cond == ISD::SETEQ || Cond == ISD::SETNE))
2746 return GetUndefBooleanConstant();
2747
2748 // If both operands are undef, we can return undef for int comparison.
2749 // icmp undef, undef -> undef.
2750 if (N1.isUndef() && N2.isUndef())
2751 return GetUndefBooleanConstant();
2752
2753 // icmp X, X -> true/false
2754 // icmp X, undef -> true/false because undef could be X.
2755 if (N1.isUndef() || N2.isUndef() || N1 == N2)
2756 return getBoolConstant(V: ISD::isTrueWhenEqual(Cond), DL: dl, VT, OpVT);
2757 }
2758
2759 if (ConstantSDNode *N2C = isConstOrConstSplat(N: N2)) {
2760 const APInt &C2 = N2C->getAPIntValue();
2761 if (ConstantSDNode *N1C = isConstOrConstSplat(N: N1)) {
2762 const APInt &C1 = N1C->getAPIntValue();
2763
2764 return getBoolConstant(V: ICmpInst::compare(LHS: C1, RHS: C2, Pred: getICmpCondCode(Pred: Cond)),
2765 DL: dl, VT, OpVT);
2766 }
2767 }
2768
2769 auto *N1CFP = dyn_cast<ConstantFPSDNode>(Val&: N1);
2770 auto *N2CFP = dyn_cast<ConstantFPSDNode>(Val&: N2);
2771
2772 if (N1CFP && N2CFP) {
2773 APFloat::cmpResult R = N1CFP->getValueAPF().compare(RHS: N2CFP->getValueAPF());
2774 switch (Cond) {
2775 default: break;
2776 case ISD::SETEQ: if (R==APFloat::cmpUnordered)
2777 return GetUndefBooleanConstant();
2778 [[fallthrough]];
2779 case ISD::SETOEQ: return getBoolConstant(V: R==APFloat::cmpEqual, DL: dl, VT,
2780 OpVT);
2781 case ISD::SETNE: if (R==APFloat::cmpUnordered)
2782 return GetUndefBooleanConstant();
2783 [[fallthrough]];
2784 case ISD::SETONE: return getBoolConstant(V: R==APFloat::cmpGreaterThan ||
2785 R==APFloat::cmpLessThan, DL: dl, VT,
2786 OpVT);
2787 case ISD::SETLT: if (R==APFloat::cmpUnordered)
2788 return GetUndefBooleanConstant();
2789 [[fallthrough]];
2790 case ISD::SETOLT: return getBoolConstant(V: R==APFloat::cmpLessThan, DL: dl, VT,
2791 OpVT);
2792 case ISD::SETGT: if (R==APFloat::cmpUnordered)
2793 return GetUndefBooleanConstant();
2794 [[fallthrough]];
2795 case ISD::SETOGT: return getBoolConstant(V: R==APFloat::cmpGreaterThan, DL: dl,
2796 VT, OpVT);
2797 case ISD::SETLE: if (R==APFloat::cmpUnordered)
2798 return GetUndefBooleanConstant();
2799 [[fallthrough]];
2800 case ISD::SETOLE: return getBoolConstant(V: R==APFloat::cmpLessThan ||
2801 R==APFloat::cmpEqual, DL: dl, VT,
2802 OpVT);
2803 case ISD::SETGE: if (R==APFloat::cmpUnordered)
2804 return GetUndefBooleanConstant();
2805 [[fallthrough]];
2806 case ISD::SETOGE: return getBoolConstant(V: R==APFloat::cmpGreaterThan ||
2807 R==APFloat::cmpEqual, DL: dl, VT, OpVT);
2808 case ISD::SETO: return getBoolConstant(V: R!=APFloat::cmpUnordered, DL: dl, VT,
2809 OpVT);
2810 case ISD::SETUO: return getBoolConstant(V: R==APFloat::cmpUnordered, DL: dl, VT,
2811 OpVT);
2812 case ISD::SETUEQ: return getBoolConstant(V: R==APFloat::cmpUnordered ||
2813 R==APFloat::cmpEqual, DL: dl, VT,
2814 OpVT);
2815 case ISD::SETUNE: return getBoolConstant(V: R!=APFloat::cmpEqual, DL: dl, VT,
2816 OpVT);
2817 case ISD::SETULT: return getBoolConstant(V: R==APFloat::cmpUnordered ||
2818 R==APFloat::cmpLessThan, DL: dl, VT,
2819 OpVT);
2820 case ISD::SETUGT: return getBoolConstant(V: R==APFloat::cmpGreaterThan ||
2821 R==APFloat::cmpUnordered, DL: dl, VT,
2822 OpVT);
2823 case ISD::SETULE: return getBoolConstant(V: R!=APFloat::cmpGreaterThan, DL: dl,
2824 VT, OpVT);
2825 case ISD::SETUGE: return getBoolConstant(V: R!=APFloat::cmpLessThan, DL: dl, VT,
2826 OpVT);
2827 }
2828 } else if (N1CFP && OpVT.isSimple() && !N2.isUndef()) {
2829 // Ensure that the constant occurs on the RHS.
2830 ISD::CondCode SwappedCond = ISD::getSetCCSwappedOperands(Operation: Cond);
2831 if (!TLI->isCondCodeLegal(CC: SwappedCond, VT: OpVT.getSimpleVT()))
2832 return SDValue();
2833 return getSetCC(DL: dl, VT, LHS: N2, RHS: N1, Cond: SwappedCond, /*Chain=*/{},
2834 /*IsSignaling=*/false, Flags);
2835 } else if ((N2CFP && N2CFP->getValueAPF().isNaN()) ||
2836 (OpVT.isFloatingPoint() && (N1.isUndef() || N2.isUndef()))) {
2837 // If an operand is known to be a nan (or undef that could be a nan), we can
2838 // fold it.
2839 // Choosing NaN for the undef will always make unordered comparison succeed
2840 // and ordered comparison fails.
2841 // Matches behavior in llvm::ConstantFoldCompareInstruction.
2842 switch (ISD::getUnorderedFlavor(Cond)) {
2843 default:
2844 llvm_unreachable("Unknown flavor!");
2845 case 0: // Known false.
2846 return getBoolConstant(V: false, DL: dl, VT, OpVT);
2847 case 1: // Known true.
2848 return getBoolConstant(V: true, DL: dl, VT, OpVT);
2849 case 2: // Undefined.
2850 return GetUndefBooleanConstant();
2851 }
2852 }
2853
2854 // Could not fold it.
2855 return SDValue();
2856}
2857
2858/// SignBitIsZero - Return true if the sign bit of Op is known to be zero. We
2859/// use this predicate to simplify operations downstream.
2860bool SelectionDAG::SignBitIsZero(SDValue Op, unsigned Depth) const {
2861 unsigned BitWidth = Op.getScalarValueSizeInBits();
2862 return MaskedValueIsZero(Op, Mask: APInt::getSignMask(BitWidth), Depth);
2863}
2864
2865// TODO: Should have argument to specify if sign bit of nan is ignorable.
2866bool SelectionDAG::SignBitIsZeroFP(SDValue Op, unsigned Depth) const {
2867 if (Depth >= MaxRecursionDepth)
2868 return false; // Limit search depth.
2869
2870 unsigned Opc = Op.getOpcode();
2871 switch (Opc) {
2872 case ISD::FABS:
2873 return true;
2874 case ISD::AssertNoFPClass: {
2875 FPClassTest NoFPClass =
2876 static_cast<FPClassTest>(Op.getConstantOperandVal(i: 1));
2877
2878 const FPClassTest TestMask = fcNan | fcNegative;
2879 return (NoFPClass & TestMask) == TestMask;
2880 }
2881 case ISD::ARITH_FENCE:
2882 return SignBitIsZeroFP(Op: Op.getOperand(i: 0), Depth: Depth + 1);
2883 case ISD::FEXP:
2884 case ISD::FEXP2:
2885 case ISD::FEXP10:
2886 return Op->getFlags().hasNoNaNs();
2887 case ISD::FMINNUM:
2888 case ISD::FMINNUM_IEEE:
2889 case ISD::FMINIMUM:
2890 case ISD::FMINIMUMNUM:
2891 return SignBitIsZeroFP(Op: Op.getOperand(i: 1), Depth: Depth + 1) &&
2892 SignBitIsZeroFP(Op: Op.getOperand(i: 0), Depth: Depth + 1);
2893 case ISD::FMAXNUM:
2894 case ISD::FMAXNUM_IEEE:
2895 case ISD::FMAXIMUM:
2896 case ISD::FMAXIMUMNUM:
2897 // TODO: If we can ignore the sign bit of nans, only one side being known 0
2898 // is sufficient.
2899 return SignBitIsZeroFP(Op: Op.getOperand(i: 1), Depth: Depth + 1) &&
2900 SignBitIsZeroFP(Op: Op.getOperand(i: 0), Depth: Depth + 1);
2901 default:
2902 return false;
2903 }
2904
2905 llvm_unreachable("covered opcode switch");
2906}
2907
2908/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero. We use
2909/// this predicate to simplify operations downstream. Mask is known to be zero
2910/// for bits that V cannot have.
2911bool SelectionDAG::MaskedValueIsZero(SDValue V, const APInt &Mask,
2912 unsigned Depth) const {
2913 return Mask.isSubsetOf(RHS: computeKnownBits(Op: V, Depth).Zero);
2914}
2915
2916/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero in
2917/// DemandedElts. We use this predicate to simplify operations downstream.
2918/// Mask is known to be zero for bits that V cannot have.
2919bool SelectionDAG::MaskedValueIsZero(SDValue V, const APInt &Mask,
2920 const APInt &DemandedElts,
2921 unsigned Depth) const {
2922 return Mask.isSubsetOf(RHS: computeKnownBits(Op: V, DemandedElts, Depth).Zero);
2923}
2924
2925/// MaskedVectorIsZero - Return true if 'Op' is known to be zero in
2926/// DemandedElts. We use this predicate to simplify operations downstream.
2927bool SelectionDAG::MaskedVectorIsZero(SDValue V, const APInt &DemandedElts,
2928 unsigned Depth /* = 0 */) const {
2929 return computeKnownBits(Op: V, DemandedElts, Depth).isZero();
2930}
2931
2932/// MaskedValueIsAllOnes - Return true if '(Op & Mask) == Mask'.
2933bool SelectionDAG::MaskedValueIsAllOnes(SDValue V, const APInt &Mask,
2934 unsigned Depth) const {
2935 return Mask.isSubsetOf(RHS: computeKnownBits(Op: V, Depth).One);
2936}
2937
2938APInt SelectionDAG::computeVectorKnownZeroElements(SDValue Op,
2939 const APInt &DemandedElts,
2940 unsigned Depth) const {
2941 EVT VT = Op.getValueType();
2942 assert(VT.isVector() && !VT.isScalableVector() && "Only for fixed vectors!");
2943
2944 unsigned NumElts = VT.getVectorNumElements();
2945 assert(DemandedElts.getBitWidth() == NumElts && "Unexpected demanded mask.");
2946
2947 APInt KnownZeroElements = APInt::getZero(numBits: NumElts);
2948 for (unsigned EltIdx = 0; EltIdx != NumElts; ++EltIdx) {
2949 if (!DemandedElts[EltIdx])
2950 continue; // Don't query elements that are not demanded.
2951 APInt Mask = APInt::getOneBitSet(numBits: NumElts, BitNo: EltIdx);
2952 if (MaskedVectorIsZero(V: Op, DemandedElts: Mask, Depth))
2953 KnownZeroElements.setBit(EltIdx);
2954 }
2955 return KnownZeroElements;
2956}
2957
2958/// isSplatValue - Return true if the vector V has the same value
2959/// across all DemandedElts. For scalable vectors, we don't know the
2960/// number of lanes at compile time. Instead, we use a 1 bit APInt
2961/// to represent a conservative value for all lanes; that is, that
2962/// one bit value is implicitly splatted across all lanes.
2963bool SelectionDAG::isSplatValue(SDValue V, const APInt &DemandedElts,
2964 APInt &UndefElts, unsigned Depth) const {
2965 unsigned Opcode = V.getOpcode();
2966 EVT VT = V.getValueType();
2967 assert(VT.isVector() && "Vector type expected");
2968 assert((!VT.isScalableVector() || DemandedElts.getBitWidth() == 1) &&
2969 "scalable demanded bits are ignored");
2970
2971 if (!DemandedElts)
2972 return false; // No demanded elts, better to assume we don't know anything.
2973
2974 if (Depth >= MaxRecursionDepth)
2975 return false; // Limit search depth.
2976
2977 // Deal with some common cases here that work for both fixed and scalable
2978 // vector types.
2979 switch (Opcode) {
2980 case ISD::SPLAT_VECTOR:
2981 UndefElts = V.getOperand(i: 0).isUndef()
2982 ? APInt::getAllOnes(numBits: DemandedElts.getBitWidth())
2983 : APInt(DemandedElts.getBitWidth(), 0);
2984 return true;
2985 case ISD::ADD:
2986 case ISD::SUB:
2987 case ISD::AND:
2988 case ISD::XOR:
2989 case ISD::OR: {
2990 APInt UndefLHS, UndefRHS;
2991 SDValue LHS = V.getOperand(i: 0);
2992 SDValue RHS = V.getOperand(i: 1);
2993 // Only recognize splats with the same demanded undef elements for both
2994 // operands, otherwise we might fail to handle binop-specific undef
2995 // handling.
2996 // e.g. (and undef, 0) -> 0 etc.
2997 if (isSplatValue(V: LHS, DemandedElts, UndefElts&: UndefLHS, Depth: Depth + 1) &&
2998 isSplatValue(V: RHS, DemandedElts, UndefElts&: UndefRHS, Depth: Depth + 1) &&
2999 (DemandedElts & UndefLHS) == (DemandedElts & UndefRHS)) {
3000 UndefElts = UndefLHS | UndefRHS;
3001 return true;
3002 }
3003 return false;
3004 }
3005 case ISD::ABS:
3006 case ISD::ABS_MIN_POISON:
3007 case ISD::TRUNCATE:
3008 case ISD::SIGN_EXTEND:
3009 case ISD::ZERO_EXTEND:
3010 return isSplatValue(V: V.getOperand(i: 0), DemandedElts, UndefElts, Depth: Depth + 1);
3011 default:
3012 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
3013 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)
3014 return TLI->isSplatValueForTargetNode(Op: V, DemandedElts, UndefElts, DAG: *this,
3015 Depth);
3016 break;
3017 }
3018
3019 // We don't support other cases than those above for scalable vectors at
3020 // the moment.
3021 if (VT.isScalableVector())
3022 return false;
3023
3024 unsigned NumElts = VT.getVectorNumElements();
3025 assert(NumElts == DemandedElts.getBitWidth() && "Vector size mismatch");
3026 UndefElts = APInt::getZero(numBits: NumElts);
3027
3028 switch (Opcode) {
3029 case ISD::BUILD_VECTOR: {
3030 SDValue Scl;
3031 for (unsigned i = 0; i != NumElts; ++i) {
3032 SDValue Op = V.getOperand(i);
3033 if (Op.isUndef()) {
3034 UndefElts.setBit(i);
3035 continue;
3036 }
3037 if (!DemandedElts[i])
3038 continue;
3039 if (Scl && Scl != Op)
3040 return false;
3041 Scl = Op;
3042 }
3043 return true;
3044 }
3045 case ISD::VECTOR_SHUFFLE: {
3046 // Check if this is a shuffle node doing a splat or a shuffle of a splat.
3047 APInt DemandedLHS = APInt::getZero(numBits: NumElts);
3048 APInt DemandedRHS = APInt::getZero(numBits: NumElts);
3049 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Val&: V)->getMask();
3050 for (int i = 0; i != (int)NumElts; ++i) {
3051 int M = Mask[i];
3052 if (M < 0) {
3053 UndefElts.setBit(i);
3054 continue;
3055 }
3056 if (!DemandedElts[i])
3057 continue;
3058 if (M < (int)NumElts)
3059 DemandedLHS.setBit(M);
3060 else
3061 DemandedRHS.setBit(M - NumElts);
3062 }
3063
3064 // If we aren't demanding either op, assume there's no splat.
3065 // If we are demanding both ops, assume there's no splat.
3066 if ((DemandedLHS.isZero() && DemandedRHS.isZero()) ||
3067 (!DemandedLHS.isZero() && !DemandedRHS.isZero()))
3068 return false;
3069
3070 // See if the demanded elts of the source op is a splat or we only demand
3071 // one element, which should always be a splat.
3072 // TODO: Handle source ops splats with undefs.
3073 auto CheckSplatSrc = [&](SDValue Src, const APInt &SrcElts) {
3074 APInt SrcUndefs;
3075 return (SrcElts.popcount() == 1) ||
3076 (isSplatValue(V: Src, DemandedElts: SrcElts, UndefElts&: SrcUndefs, Depth: Depth + 1) &&
3077 (SrcElts & SrcUndefs).isZero());
3078 };
3079 if (!DemandedLHS.isZero())
3080 return CheckSplatSrc(V.getOperand(i: 0), DemandedLHS);
3081 return CheckSplatSrc(V.getOperand(i: 1), DemandedRHS);
3082 }
3083 case ISD::EXTRACT_SUBVECTOR: {
3084 // Offset the demanded elts by the subvector index.
3085 SDValue Src = V.getOperand(i: 0);
3086 // We don't support scalable vectors at the moment.
3087 if (Src.getValueType().isScalableVector())
3088 return false;
3089 uint64_t Idx = V.getConstantOperandVal(i: 1);
3090 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3091 APInt UndefSrcElts;
3092 APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
3093 if (isSplatValue(V: Src, DemandedElts: DemandedSrcElts, UndefElts&: UndefSrcElts, Depth: Depth + 1)) {
3094 UndefElts = UndefSrcElts.extractBits(numBits: NumElts, bitPosition: Idx);
3095 return true;
3096 }
3097 break;
3098 }
3099 case ISD::ANY_EXTEND_VECTOR_INREG:
3100 case ISD::SIGN_EXTEND_VECTOR_INREG:
3101 case ISD::ZERO_EXTEND_VECTOR_INREG: {
3102 // Widen the demanded elts by the src element count.
3103 SDValue Src = V.getOperand(i: 0);
3104 // We don't support scalable vectors at the moment.
3105 if (Src.getValueType().isScalableVector())
3106 return false;
3107 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3108 APInt UndefSrcElts;
3109 APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts);
3110 if (isSplatValue(V: Src, DemandedElts: DemandedSrcElts, UndefElts&: UndefSrcElts, Depth: Depth + 1)) {
3111 UndefElts = UndefSrcElts.trunc(width: NumElts);
3112 return true;
3113 }
3114 break;
3115 }
3116 case ISD::BITCAST: {
3117 SDValue Src = V.getOperand(i: 0);
3118 EVT SrcVT = Src.getValueType();
3119 unsigned SrcBitWidth = SrcVT.getScalarSizeInBits();
3120 unsigned BitWidth = VT.getScalarSizeInBits();
3121
3122 // Ignore bitcasts from unsupported types.
3123 // TODO: Add fp support?
3124 if (!SrcVT.isVector() || !SrcVT.isInteger() || !VT.isInteger())
3125 break;
3126
3127 // Bitcast 'small element' vector to 'large element' vector.
3128 if ((BitWidth % SrcBitWidth) == 0) {
3129 // See if each sub element is a splat.
3130 unsigned Scale = BitWidth / SrcBitWidth;
3131 unsigned NumSrcElts = SrcVT.getVectorNumElements();
3132 APInt ScaledDemandedElts =
3133 APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumSrcElts);
3134 for (unsigned I = 0; I != Scale; ++I) {
3135 APInt SubUndefElts;
3136 APInt SubDemandedElt = APInt::getOneBitSet(numBits: Scale, BitNo: I);
3137 APInt SubDemandedElts = APInt::getSplat(NewLen: NumSrcElts, V: SubDemandedElt);
3138 SubDemandedElts &= ScaledDemandedElts;
3139 if (!isSplatValue(V: Src, DemandedElts: SubDemandedElts, UndefElts&: SubUndefElts, Depth: Depth + 1))
3140 return false;
3141 // TODO: Add support for merging sub undef elements.
3142 if (!SubUndefElts.isZero())
3143 return false;
3144 }
3145 return true;
3146 }
3147 break;
3148 }
3149 }
3150
3151 return false;
3152}
3153
3154/// Helper wrapper to main isSplatValue function.
3155bool SelectionDAG::isSplatValue(SDValue V, bool AllowUndefs) const {
3156 EVT VT = V.getValueType();
3157 assert(VT.isVector() && "Vector type expected");
3158
3159 APInt UndefElts;
3160 // Since the number of lanes in a scalable vector is unknown at compile time,
3161 // we track one bit which is implicitly broadcast to all lanes. This means
3162 // that all lanes in a scalable vector are considered demanded.
3163 APInt DemandedElts
3164 = APInt::getAllOnes(numBits: VT.isScalableVector() ? 1 : VT.getVectorNumElements());
3165 return isSplatValue(V, DemandedElts, UndefElts) &&
3166 (AllowUndefs || !UndefElts);
3167}
3168
3169SDValue SelectionDAG::getSplatSourceVector(SDValue V, int &SplatIdx) {
3170 V = peekThroughExtractSubvectors(V);
3171
3172 EVT VT = V.getValueType();
3173 unsigned Opcode = V.getOpcode();
3174 switch (Opcode) {
3175 default: {
3176 APInt UndefElts;
3177 // Since the number of lanes in a scalable vector is unknown at compile time,
3178 // we track one bit which is implicitly broadcast to all lanes. This means
3179 // that all lanes in a scalable vector are considered demanded.
3180 APInt DemandedElts
3181 = APInt::getAllOnes(numBits: VT.isScalableVector() ? 1 : VT.getVectorNumElements());
3182
3183 if (isSplatValue(V, DemandedElts, UndefElts)) {
3184 if (VT.isScalableVector()) {
3185 // DemandedElts and UndefElts are ignored for scalable vectors, since
3186 // the only supported cases are SPLAT_VECTOR nodes.
3187 SplatIdx = 0;
3188 } else {
3189 // Handle case where all demanded elements are UNDEF.
3190 if (DemandedElts.isSubsetOf(RHS: UndefElts)) {
3191 SplatIdx = 0;
3192 return getUNDEF(VT);
3193 }
3194 SplatIdx = (UndefElts & DemandedElts).countr_one();
3195 }
3196 return V;
3197 }
3198 break;
3199 }
3200 case ISD::SPLAT_VECTOR:
3201 SplatIdx = 0;
3202 return V;
3203 case ISD::VECTOR_SHUFFLE: {
3204 assert(!VT.isScalableVector());
3205 // Check if this is a shuffle node doing a splat.
3206 // TODO - remove this and rely purely on SelectionDAG::isSplatValue,
3207 // getTargetVShiftNode currently struggles without the splat source.
3208 auto *SVN = cast<ShuffleVectorSDNode>(Val&: V);
3209 if (!SVN->isSplat())
3210 break;
3211 int Idx = SVN->getSplatIndex();
3212 int NumElts = V.getValueType().getVectorNumElements();
3213 SplatIdx = Idx % NumElts;
3214 return V.getOperand(i: Idx / NumElts);
3215 }
3216 }
3217
3218 return SDValue();
3219}
3220
3221SDValue SelectionDAG::getSplatValue(SDValue V, bool LegalTypes) {
3222 int SplatIdx;
3223 if (SDValue SrcVector = getSplatSourceVector(V, SplatIdx)) {
3224 EVT SVT = SrcVector.getValueType().getScalarType();
3225 EVT LegalSVT = SVT;
3226 if (LegalTypes && !TLI->isTypeLegal(VT: SVT)) {
3227 if (!SVT.isInteger())
3228 return SDValue();
3229 LegalSVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT: LegalSVT);
3230 if (LegalSVT.bitsLT(VT: SVT))
3231 return SDValue();
3232 }
3233 return getExtractVectorElt(DL: SDLoc(V), VT: LegalSVT, Vec: SrcVector, Idx: SplatIdx);
3234 }
3235 return SDValue();
3236}
3237
3238std::optional<ConstantRange>
3239SelectionDAG::getValidShiftAmountRange(SDValue V, const APInt &DemandedElts,
3240 unsigned Depth) const {
3241 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
3242 V.getOpcode() == ISD::SRA) &&
3243 "Unknown shift node");
3244 // Shifting more than the bitwidth is not valid.
3245 unsigned BitWidth = V.getScalarValueSizeInBits();
3246
3247 if (auto *Cst = dyn_cast<ConstantSDNode>(Val: V.getOperand(i: 1))) {
3248 const APInt &ShAmt = Cst->getAPIntValue();
3249 if (ShAmt.uge(RHS: BitWidth))
3250 return std::nullopt;
3251 return ConstantRange(ShAmt);
3252 }
3253
3254 if (auto *BV = dyn_cast<BuildVectorSDNode>(Val: V.getOperand(i: 1))) {
3255 const APInt *MinAmt = nullptr, *MaxAmt = nullptr;
3256 for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) {
3257 if (!DemandedElts[i])
3258 continue;
3259 auto *SA = dyn_cast<ConstantSDNode>(Val: BV->getOperand(Num: i));
3260 if (!SA) {
3261 MinAmt = MaxAmt = nullptr;
3262 break;
3263 }
3264 const APInt &ShAmt = SA->getAPIntValue();
3265 if (ShAmt.uge(RHS: BitWidth))
3266 return std::nullopt;
3267 if (!MinAmt || MinAmt->ugt(RHS: ShAmt))
3268 MinAmt = &ShAmt;
3269 if (!MaxAmt || MaxAmt->ult(RHS: ShAmt))
3270 MaxAmt = &ShAmt;
3271 }
3272 assert(((!MinAmt && !MaxAmt) || (MinAmt && MaxAmt)) &&
3273 "Failed to find matching min/max shift amounts");
3274 if (MinAmt && MaxAmt)
3275 return ConstantRange(*MinAmt, *MaxAmt + 1);
3276 }
3277
3278 // Use computeKnownBits to find a hidden constant/knownbits (usually type
3279 // legalized). e.g. Hidden behind multiple bitcasts/build_vector/casts etc.
3280 KnownBits KnownAmt = computeKnownBits(Op: V.getOperand(i: 1), DemandedElts, Depth);
3281 if (KnownAmt.getMaxValue().ult(RHS: BitWidth))
3282 return ConstantRange::fromKnownBits(Known: KnownAmt, /*IsSigned=*/false);
3283
3284 return std::nullopt;
3285}
3286
3287std::optional<unsigned>
3288SelectionDAG::getValidShiftAmount(SDValue V, const APInt &DemandedElts,
3289 unsigned Depth) const {
3290 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
3291 V.getOpcode() == ISD::SRA) &&
3292 "Unknown shift node");
3293 if (std::optional<ConstantRange> AmtRange =
3294 getValidShiftAmountRange(V, DemandedElts, Depth))
3295 if (const APInt *ShAmt = AmtRange->getSingleElement())
3296 return ShAmt->getZExtValue();
3297 return std::nullopt;
3298}
3299
3300std::optional<unsigned>
3301SelectionDAG::getValidShiftAmount(SDValue V, unsigned Depth) const {
3302 APInt DemandedElts = getDemandAllEltsMask(V);
3303 return getValidShiftAmount(V, DemandedElts, Depth);
3304}
3305
3306std::optional<unsigned>
3307SelectionDAG::getValidMinimumShiftAmount(SDValue V, const APInt &DemandedElts,
3308 unsigned Depth) const {
3309 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
3310 V.getOpcode() == ISD::SRA) &&
3311 "Unknown shift node");
3312 if (std::optional<ConstantRange> AmtRange =
3313 getValidShiftAmountRange(V, DemandedElts, Depth))
3314 return AmtRange->getUnsignedMin().getZExtValue();
3315 return std::nullopt;
3316}
3317
3318std::optional<unsigned>
3319SelectionDAG::getValidMinimumShiftAmount(SDValue V, unsigned Depth) const {
3320 APInt DemandedElts = getDemandAllEltsMask(V);
3321 return getValidMinimumShiftAmount(V, DemandedElts, Depth);
3322}
3323
3324std::optional<unsigned>
3325SelectionDAG::getValidMaximumShiftAmount(SDValue V, const APInt &DemandedElts,
3326 unsigned Depth) const {
3327 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
3328 V.getOpcode() == ISD::SRA) &&
3329 "Unknown shift node");
3330 if (std::optional<ConstantRange> AmtRange =
3331 getValidShiftAmountRange(V, DemandedElts, Depth))
3332 return AmtRange->getUnsignedMax().getZExtValue();
3333 return std::nullopt;
3334}
3335
3336std::optional<unsigned>
3337SelectionDAG::getValidMaximumShiftAmount(SDValue V, unsigned Depth) const {
3338 APInt DemandedElts = getDemandAllEltsMask(V);
3339 return getValidMaximumShiftAmount(V, DemandedElts, Depth);
3340}
3341
3342/// Determine which bits of Op are known to be either zero or one and return
3343/// them in Known. For vectors, the known bits are those that are shared by
3344/// every vector element.
3345KnownBits SelectionDAG::computeKnownBits(SDValue Op, unsigned Depth) const {
3346 APInt DemandedElts = getDemandAllEltsMask(V: Op);
3347 return computeKnownBits(Op, DemandedElts, Depth);
3348}
3349
3350/// Determine which bits of Op are known to be either zero or one and return
3351/// them in Known. The DemandedElts argument allows us to only collect the known
3352/// bits that are shared by the requested vector elements.
3353KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts,
3354 unsigned Depth) const {
3355 unsigned BitWidth = Op.getScalarValueSizeInBits();
3356
3357 KnownBits Known(BitWidth); // Don't know anything.
3358
3359 if (auto OptAPInt = Op->bitcastToAPInt()) {
3360 // We know all of the bits for a constant!
3361 APInt V = *std::move(OptAPInt);
3362
3363 // Swap the low-order and high-order double of a ppc_fp128 when casting to
3364 // i128, see #44482.
3365 //
3366 // A ppc_fp128 is two doubles, with the high-order double stored at the
3367 // lower address. Reading that as an integer therefore puts the high-order
3368 // double in the high 64 bits on big-endian targets and in the low 64 bits
3369 // on little-endian targets.
3370 //
3371 // But APFloat::bitcastToAPInt is endianness-agnostic and always places the
3372 // high-order double in the low 64 bits. Hence the two doubles must be
3373 // flipped on big-endian targets.
3374 if (getDataLayout().isBigEndian() && Op.getValueType() == MVT::ppcf128)
3375 V = V.rotl(rotateAmt: 64);
3376
3377 return KnownBits::makeConstant(C: V);
3378 }
3379
3380 if (Depth >= MaxRecursionDepth)
3381 return Known; // Limit search depth.
3382
3383 KnownBits Known2;
3384 unsigned NumElts = DemandedElts.getBitWidth();
3385 assert((!Op.getValueType().isScalableVector() || NumElts == 1) &&
3386 "DemandedElts for scalable vectors must be 1 to represent all lanes");
3387 assert((!Op.getValueType().isFixedLengthVector() ||
3388 NumElts == Op.getValueType().getVectorNumElements()) &&
3389 "Unexpected vector size");
3390
3391 if (!DemandedElts)
3392 return Known; // No demanded elts, better to assume we don't know anything.
3393
3394 unsigned Opcode = Op.getOpcode();
3395 switch (Opcode) {
3396 case ISD::FREEZE: {
3397 if (isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 0), DemandedElts,
3398 Kind: UndefPoisonKind::UndefOrPoison))
3399 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3400 break;
3401 }
3402 case ISD::MERGE_VALUES:
3403 return computeKnownBits(Op: Op.getOperand(i: Op.getResNo()), DemandedElts,
3404 Depth: Depth + 1);
3405 case ISD::SPLAT_VECTOR: {
3406 SDValue SrcOp = Op.getOperand(i: 0);
3407 assert(SrcOp.getValueSizeInBits() >= BitWidth &&
3408 "Expected SPLAT_VECTOR implicit truncation");
3409 // Implicitly truncate the bits to match the official semantics of
3410 // SPLAT_VECTOR.
3411 Known = computeKnownBits(Op: SrcOp, Depth: Depth + 1).trunc(BitWidth);
3412 break;
3413 }
3414 case ISD::SPLAT_VECTOR_PARTS: {
3415 unsigned ScalarSize = Op.getOperand(i: 0).getScalarValueSizeInBits();
3416 assert(ScalarSize * Op.getNumOperands() == BitWidth &&
3417 "Expected SPLAT_VECTOR_PARTS scalars to cover element width");
3418 for (auto [I, SrcOp] : enumerate(First: Op->ops())) {
3419 Known.insertBits(SubBits: computeKnownBits(Op: SrcOp, Depth: Depth + 1), BitPosition: ScalarSize * I);
3420 }
3421 break;
3422 }
3423 case ISD::STEP_VECTOR: {
3424 const APInt &Step = Op.getConstantOperandAPInt(i: 0);
3425
3426 if (Step.isPowerOf2())
3427 Known.Zero.setLowBits(Step.logBase2());
3428
3429 const Function &F = getMachineFunction().getFunction();
3430
3431 if (!isUIntN(N: BitWidth, x: Op.getValueType().getVectorMinNumElements()))
3432 break;
3433 const APInt MinNumElts =
3434 APInt(BitWidth, Op.getValueType().getVectorMinNumElements());
3435
3436 bool Overflow;
3437 const APInt MaxNumElts = getVScaleRange(F: &F, BitWidth)
3438 .getUnsignedMax()
3439 .umul_ov(RHS: MinNumElts, Overflow);
3440 if (Overflow)
3441 break;
3442
3443 const APInt MaxValue = (MaxNumElts - 1).umul_ov(RHS: Step, Overflow);
3444 if (Overflow)
3445 break;
3446
3447 Known.Zero.setHighBits(MaxValue.countl_zero());
3448 break;
3449 }
3450 case ISD::BUILD_VECTOR:
3451 assert(!Op.getValueType().isScalableVector());
3452 // Collect the known bits that are shared by every demanded vector element.
3453 Known.setAllConflict();
3454 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
3455 if (!DemandedElts[i])
3456 continue;
3457
3458 SDValue SrcOp = Op.getOperand(i);
3459 if (SrcOp.getOpcode() == ISD::POISON)
3460 continue;
3461
3462 Known2 = computeKnownBits(Op: SrcOp, Depth: Depth + 1);
3463
3464 // BUILD_VECTOR can implicitly truncate sources, we must handle this.
3465 if (SrcOp.getValueSizeInBits() != BitWidth) {
3466 assert(SrcOp.getValueSizeInBits() > BitWidth &&
3467 "Expected BUILD_VECTOR implicit truncation");
3468 Known2 = Known2.trunc(BitWidth);
3469 }
3470
3471 // Known bits are the values that are shared by every demanded element.
3472 Known = Known.intersectWith(RHS: Known2);
3473
3474 // If we don't know any bits, early out.
3475 if (Known.isUnknown())
3476 break;
3477 }
3478
3479 // If every demanded element was poison, we know nothing.
3480 if (Known.hasConflict())
3481 Known.resetAll();
3482 break;
3483 case ISD::VECTOR_COMPRESS: {
3484 SDValue Vec = Op.getOperand(i: 0);
3485 SDValue PassThru = Op.getOperand(i: 2);
3486 Known = computeKnownBits(Op: PassThru, DemandedElts, Depth: Depth + 1);
3487 // If we don't know any bits, early out.
3488 if (Known.isUnknown())
3489 break;
3490 Known2 = computeKnownBits(Op: Vec, Depth: Depth + 1);
3491 Known = Known.intersectWith(RHS: Known2);
3492 break;
3493 }
3494 case ISD::VECTOR_SHUFFLE: {
3495 assert(!Op.getValueType().isScalableVector());
3496 // Collect the known bits that are shared by every vector element referenced
3497 // by the shuffle.
3498 APInt DemandedLHS, DemandedRHS;
3499 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
3500 assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
3501 if (!getShuffleDemandedElts(SrcWidth: NumElts, Mask: SVN->getMask(), DemandedElts,
3502 DemandedLHS, DemandedRHS))
3503 break;
3504
3505 // Known bits are the values that are shared by every demanded element.
3506 Known.setAllConflict();
3507 if (!!DemandedLHS) {
3508 SDValue LHS = Op.getOperand(i: 0);
3509 Known2 = computeKnownBits(Op: LHS, DemandedElts: DemandedLHS, Depth: Depth + 1);
3510 Known = Known.intersectWith(RHS: Known2);
3511 }
3512 // If we don't know any bits, early out.
3513 if (Known.isUnknown())
3514 break;
3515 if (!!DemandedRHS) {
3516 SDValue RHS = Op.getOperand(i: 1);
3517 Known2 = computeKnownBits(Op: RHS, DemandedElts: DemandedRHS, Depth: Depth + 1);
3518 Known = Known.intersectWith(RHS: Known2);
3519 }
3520 break;
3521 }
3522 case ISD::VSCALE: {
3523 const Function &F = getMachineFunction().getFunction();
3524 const APInt &Multiplier = Op.getConstantOperandAPInt(i: 0);
3525 Known = getVScaleRange(F: &F, BitWidth).multiply(Other: Multiplier).toKnownBits();
3526 break;
3527 }
3528 case ISD::CONCAT_VECTORS: {
3529 if (Op.getValueType().isScalableVector())
3530 break;
3531 // Split DemandedElts and test each of the demanded subvectors.
3532 Known.setAllConflict();
3533 EVT SubVectorVT = Op.getOperand(i: 0).getValueType();
3534 unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements();
3535 unsigned NumSubVectors = Op.getNumOperands();
3536 for (unsigned i = 0; i != NumSubVectors; ++i) {
3537 APInt DemandedSub =
3538 DemandedElts.extractBits(numBits: NumSubVectorElts, bitPosition: i * NumSubVectorElts);
3539 if (!!DemandedSub) {
3540 SDValue Sub = Op.getOperand(i);
3541 Known2 = computeKnownBits(Op: Sub, DemandedElts: DemandedSub, Depth: Depth + 1);
3542 Known = Known.intersectWith(RHS: Known2);
3543 }
3544 // If we don't know any bits, early out.
3545 if (Known.isUnknown())
3546 break;
3547 }
3548 break;
3549 }
3550 case ISD::INSERT_SUBVECTOR: {
3551 if (Op.getValueType().isScalableVector())
3552 break;
3553 // Demand any elements from the subvector and the remainder from the src its
3554 // inserted into.
3555 SDValue Src = Op.getOperand(i: 0);
3556 SDValue Sub = Op.getOperand(i: 1);
3557 uint64_t Idx = Op.getConstantOperandVal(i: 2);
3558 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
3559 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
3560 APInt DemandedSrcElts = DemandedElts;
3561 DemandedSrcElts.clearBits(LoBit: Idx, HiBit: Idx + NumSubElts);
3562
3563 Known.setAllConflict();
3564 if (!!DemandedSubElts) {
3565 Known = computeKnownBits(Op: Sub, DemandedElts: DemandedSubElts, Depth: Depth + 1);
3566 if (Known.isUnknown())
3567 break; // early-out.
3568 }
3569 if (!!DemandedSrcElts) {
3570 Known2 = computeKnownBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
3571 Known = Known.intersectWith(RHS: Known2);
3572 }
3573 break;
3574 }
3575 case ISD::EXTRACT_SUBVECTOR: {
3576 // Offset the demanded elts by the subvector index.
3577 SDValue Src = Op.getOperand(i: 0);
3578
3579 APInt DemandedSrcElts;
3580 if (Src.getValueType().isScalableVector())
3581 DemandedSrcElts = APInt(1, 1); // <=> 'demand all elements'
3582 else {
3583 uint64_t Idx = Op.getConstantOperandVal(i: 1);
3584 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3585 DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
3586 }
3587 Known = computeKnownBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
3588 break;
3589 }
3590 case ISD::SCALAR_TO_VECTOR: {
3591 if (Op.getValueType().isScalableVector())
3592 break;
3593 // We know about scalar_to_vector as much as we know about it source,
3594 // which becomes the first element of otherwise unknown vector.
3595 if (DemandedElts != 1)
3596 break;
3597
3598 SDValue N0 = Op.getOperand(i: 0);
3599 Known = computeKnownBits(Op: N0, Depth: Depth + 1);
3600 if (N0.getValueSizeInBits() != BitWidth)
3601 Known = Known.trunc(BitWidth);
3602
3603 break;
3604 }
3605 case ISD::BITCAST: {
3606 if (Op.getValueType().isScalableVector())
3607 break;
3608
3609 SDValue N0 = Op.getOperand(i: 0);
3610 EVT SubVT = N0.getValueType();
3611 unsigned SubBitWidth = SubVT.getScalarSizeInBits();
3612
3613 // Ignore bitcasts from unsupported types.
3614 if (!(SubVT.isInteger() || SubVT.isFloatingPoint()))
3615 break;
3616
3617 // Fast handling of 'identity' bitcasts.
3618 if (BitWidth == SubBitWidth) {
3619 Known = computeKnownBits(Op: N0, DemandedElts, Depth: Depth + 1);
3620 break;
3621 }
3622
3623 bool IsLE = getDataLayout().isLittleEndian();
3624
3625 // Bitcast 'small element' vector to 'large element' scalar/vector.
3626 if ((BitWidth % SubBitWidth) == 0) {
3627 assert(N0.getValueType().isVector() && "Expected bitcast from vector");
3628
3629 // Collect known bits for the (larger) output by collecting the known
3630 // bits from each set of sub elements and shift these into place.
3631 // We need to separately call computeKnownBits for each set of
3632 // sub elements as the knownbits for each is likely to be different.
3633 unsigned SubScale = BitWidth / SubBitWidth;
3634 APInt SubDemandedElts(NumElts * SubScale, 0);
3635 for (unsigned i = 0; i != NumElts; ++i)
3636 if (DemandedElts[i])
3637 SubDemandedElts.setBit(i * SubScale);
3638
3639 for (unsigned i = 0; i != SubScale; ++i) {
3640 Known2 = computeKnownBits(Op: N0, DemandedElts: SubDemandedElts.shl(shiftAmt: i),
3641 Depth: Depth + 1);
3642 unsigned Shifts = IsLE ? i : SubScale - 1 - i;
3643 Known.insertBits(SubBits: Known2, BitPosition: SubBitWidth * Shifts);
3644 }
3645 }
3646
3647 // Bitcast 'large element' scalar/vector to 'small element' vector.
3648 if ((SubBitWidth % BitWidth) == 0) {
3649 assert(Op.getValueType().isVector() && "Expected bitcast to vector");
3650
3651 // Collect known bits for the (smaller) output by collecting the known
3652 // bits from the overlapping larger input elements and extracting the
3653 // sub sections we actually care about.
3654 unsigned SubScale = SubBitWidth / BitWidth;
3655 APInt SubDemandedElts =
3656 APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumElts / SubScale);
3657 Known2 = computeKnownBits(Op: N0, DemandedElts: SubDemandedElts, Depth: Depth + 1);
3658
3659 Known.setAllConflict();
3660 for (unsigned i = 0; i != NumElts; ++i)
3661 if (DemandedElts[i]) {
3662 unsigned Shifts = IsLE ? i : NumElts - 1 - i;
3663 unsigned Offset = (Shifts % SubScale) * BitWidth;
3664 Known = Known.intersectWith(RHS: Known2.extractBits(NumBits: BitWidth, BitPosition: Offset));
3665 // If we don't know any bits, early out.
3666 if (Known.isUnknown())
3667 break;
3668 }
3669 }
3670 break;
3671 }
3672 case ISD::AND:
3673 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3674 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3675
3676 Known &= Known2;
3677 break;
3678 case ISD::OR:
3679 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3680 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3681
3682 Known |= Known2;
3683 break;
3684 case ISD::XOR:
3685 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3686 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3687
3688 Known ^= Known2;
3689 break;
3690 case ISD::MUL: {
3691 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3692 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3693 bool SelfMultiply = Op.getOperand(i: 0) == Op.getOperand(i: 1);
3694 // TODO: SelfMultiply can be poison, but not undef.
3695 if (SelfMultiply)
3696 SelfMultiply &= isGuaranteedNotToBeUndefOrPoison(
3697 Op: Op.getOperand(i: 0), DemandedElts, Kind: UndefPoisonKind::UndefOrPoison,
3698 Depth: Depth + 1);
3699 Known = KnownBits::mul(LHS: Known, RHS: Known2, NoUndefSelfMultiply: SelfMultiply);
3700
3701 // If the multiplication is known not to overflow, the product of a number
3702 // with itself is non-negative. Only do this if we didn't already computed
3703 // the opposite value for the sign bit.
3704 if (Op->getFlags().hasNoSignedWrap() &&
3705 Op.getOperand(i: 0) == Op.getOperand(i: 1) &&
3706 !Known.isNegative())
3707 Known.makeNonNegative();
3708 break;
3709 }
3710 case ISD::MULHU: {
3711 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3712 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3713 Known = KnownBits::mulhu(LHS: Known, RHS: Known2);
3714 break;
3715 }
3716 case ISD::MULHS: {
3717 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3718 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3719 Known = KnownBits::mulhs(LHS: Known, RHS: Known2);
3720 break;
3721 }
3722 case ISD::ABDU: {
3723 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3724 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3725 Known = KnownBits::abdu(LHS: Known, RHS: Known2);
3726 break;
3727 }
3728 case ISD::ABDS: {
3729 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3730 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3731 Known = KnownBits::abds(LHS: Known, RHS: Known2);
3732 unsigned SignBits1 =
3733 ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3734 if (SignBits1 == 1)
3735 break;
3736 unsigned SignBits0 =
3737 ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3738 Known.Zero.setHighBits(std::min(a: SignBits0, b: SignBits1) - 1);
3739 break;
3740 }
3741 case ISD::UMUL_LOHI: {
3742 assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result");
3743 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3744 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3745 bool SelfMultiply = Op.getOperand(i: 0) == Op.getOperand(i: 1);
3746 if (Op.getResNo() == 0)
3747 Known = KnownBits::mul(LHS: Known, RHS: Known2, NoUndefSelfMultiply: SelfMultiply);
3748 else
3749 Known = KnownBits::mulhu(LHS: Known, RHS: Known2);
3750 break;
3751 }
3752 case ISD::SMUL_LOHI: {
3753 assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result");
3754 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3755 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3756 bool SelfMultiply = Op.getOperand(i: 0) == Op.getOperand(i: 1);
3757 if (Op.getResNo() == 0)
3758 Known = KnownBits::mul(LHS: Known, RHS: Known2, NoUndefSelfMultiply: SelfMultiply);
3759 else
3760 Known = KnownBits::mulhs(LHS: Known, RHS: Known2);
3761 break;
3762 }
3763 case ISD::AVGFLOORU: {
3764 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3765 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3766 Known = KnownBits::avgFloorU(LHS: Known, RHS: Known2);
3767 break;
3768 }
3769 case ISD::AVGCEILU: {
3770 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3771 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3772 Known = KnownBits::avgCeilU(LHS: Known, RHS: Known2);
3773 break;
3774 }
3775 case ISD::AVGFLOORS: {
3776 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3777 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3778 Known = KnownBits::avgFloorS(LHS: Known, RHS: Known2);
3779 break;
3780 }
3781 case ISD::AVGCEILS: {
3782 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3783 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3784 Known = KnownBits::avgCeilS(LHS: Known, RHS: Known2);
3785 break;
3786 }
3787 case ISD::AVGSUBFLOORU:
3788 case ISD::AVGSUBFLOORS: {
3789 bool IsSigned = Op.getOpcode() == ISD::AVGSUBFLOORS;
3790 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3791 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3792 // Subtract in i[N+1] so the difference can't overflow, then halve.
3793 Known = IsSigned ? Known.sext(BitWidth: BitWidth + 1) : Known.zext(BitWidth: BitWidth + 1);
3794 Known2 = IsSigned ? Known2.sext(BitWidth: BitWidth + 1) : Known2.zext(BitWidth: BitWidth + 1);
3795 Known = KnownBits::sub(LHS: Known, RHS: Known2);
3796 // The shifted in top bit is unknown, but it is truncated away.
3797 Known >>= 1;
3798 Known = Known.trunc(BitWidth);
3799 break;
3800 }
3801 case ISD::SELECT:
3802 case ISD::VSELECT:
3803 Known = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth+1);
3804 // If we don't know any bits, early out.
3805 if (Known.isUnknown())
3806 break;
3807 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth+1);
3808
3809 // Only known if known in both the LHS and RHS.
3810 Known = Known.intersectWith(RHS: Known2);
3811 break;
3812 case ISD::SELECT_CC:
3813 Known = computeKnownBits(Op: Op.getOperand(i: 3), DemandedElts, Depth: Depth+1);
3814 // If we don't know any bits, early out.
3815 if (Known.isUnknown())
3816 break;
3817 Known2 = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth+1);
3818
3819 // Only known if known in both the LHS and RHS.
3820 Known = Known.intersectWith(RHS: Known2);
3821 break;
3822 case ISD::SMULO:
3823 case ISD::UMULO:
3824 if (Op.getResNo() != 1)
3825 break;
3826 // The boolean result conforms to getBooleanContents.
3827 // If we know the result of a setcc has the top bits zero, use this info.
3828 // We know that we have an integer-based boolean since these operations
3829 // are only available for integer.
3830 if (TLI->getBooleanContents(isVec: Op.getValueType().isVector(), isFloat: false) ==
3831 TargetLowering::ZeroOrOneBooleanContent &&
3832 BitWidth > 1)
3833 Known.Zero.setBitsFrom(1);
3834 break;
3835 case ISD::SETCC:
3836 case ISD::SETCCCARRY:
3837 case ISD::STRICT_FSETCC:
3838 case ISD::STRICT_FSETCCS: {
3839 unsigned OpNo = Op->isStrictFPOpcode() ? 1 : 0;
3840 // If we know the result of a setcc has the top bits zero, use this info.
3841 if (TLI->getBooleanContents(Type: Op.getOperand(i: OpNo).getValueType()) ==
3842 TargetLowering::ZeroOrOneBooleanContent &&
3843 BitWidth > 1)
3844 Known.Zero.setBitsFrom(1);
3845 break;
3846 }
3847 case ISD::SHL: {
3848 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3849 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3850
3851 bool NUW = Op->getFlags().hasNoUnsignedWrap();
3852 bool NSW = Op->getFlags().hasNoSignedWrap();
3853
3854 bool ShAmtNonZero = Known2.isNonZero();
3855
3856 Known = KnownBits::shl(LHS: Known, RHS: Known2, NUW, NSW, ShAmtNonZero);
3857
3858 // Minimum shift low bits are known zero.
3859 if (std::optional<unsigned> ShMinAmt =
3860 getValidMinimumShiftAmount(V: Op, DemandedElts, Depth: Depth + 1))
3861 Known.Zero.setLowBits(*ShMinAmt);
3862 break;
3863 }
3864 case ISD::SRL:
3865 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3866 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3867 Known = KnownBits::lshr(LHS: Known, RHS: Known2, /*ShAmtNonZero=*/false,
3868 Exact: Op->getFlags().hasExact());
3869
3870 // Minimum shift high bits are known zero.
3871 if (std::optional<unsigned> ShMinAmt =
3872 getValidMinimumShiftAmount(V: Op, DemandedElts, Depth: Depth + 1))
3873 Known.Zero.setHighBits(*ShMinAmt);
3874 break;
3875 case ISD::SRA:
3876 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3877 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3878 Known = KnownBits::ashr(LHS: Known, RHS: Known2, /*ShAmtNonZero=*/false,
3879 Exact: Op->getFlags().hasExact());
3880 break;
3881 case ISD::ROTL:
3882 case ISD::ROTR:
3883 if (ConstantSDNode *C =
3884 isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts)) {
3885 unsigned Amt = C->getAPIntValue().urem(RHS: BitWidth);
3886
3887 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3888
3889 // Canonicalize to ROTR.
3890 if (Opcode == ISD::ROTL && Amt != 0)
3891 Amt = BitWidth - Amt;
3892
3893 Known.Zero = Known.Zero.rotr(rotateAmt: Amt);
3894 Known.One = Known.One.rotr(rotateAmt: Amt);
3895 }
3896 break;
3897 case ISD::FSHL:
3898 case ISD::FSHR:
3899 if (ConstantSDNode *C = isConstOrConstSplat(N: Op.getOperand(i: 2), DemandedElts)) {
3900 unsigned Amt = C->getAPIntValue().urem(RHS: BitWidth);
3901
3902 // For fshl, 0-shift returns the 1st arg.
3903 // For fshr, 0-shift returns the 2nd arg.
3904 if (Amt == 0) {
3905 Known = computeKnownBits(Op: Op.getOperand(i: Opcode == ISD::FSHL ? 0 : 1),
3906 DemandedElts, Depth: Depth + 1);
3907 break;
3908 }
3909
3910 // fshl: (X << (Z % BW)) | (Y >> (BW - (Z % BW)))
3911 // fshr: (X << (BW - (Z % BW))) | (Y >> (Z % BW))
3912 const APInt ShAmt(BitWidth, Amt);
3913 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3914 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3915 Known = Opcode == ISD::FSHL ? KnownBits::fshl(LHS: Known, RHS: Known2, Amt: ShAmt)
3916 : KnownBits::fshr(LHS: Known, RHS: Known2, Amt: ShAmt);
3917 }
3918 break;
3919 case ISD::SHL_PARTS:
3920 case ISD::SRA_PARTS:
3921 case ISD::SRL_PARTS: {
3922 assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result");
3923
3924 // Collect lo/hi source values and concatenate.
3925 unsigned LoBits = Op.getOperand(i: 0).getScalarValueSizeInBits();
3926 unsigned HiBits = Op.getOperand(i: 1).getScalarValueSizeInBits();
3927 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3928 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3929 Known = Known2.concat(Lo: Known);
3930
3931 // Collect shift amount.
3932 Known2 = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth + 1);
3933
3934 if (Opcode == ISD::SHL_PARTS)
3935 Known = KnownBits::shl(LHS: Known, RHS: Known2);
3936 else if (Opcode == ISD::SRA_PARTS)
3937 Known = KnownBits::ashr(LHS: Known, RHS: Known2);
3938 else // if (Opcode == ISD::SRL_PARTS)
3939 Known = KnownBits::lshr(LHS: Known, RHS: Known2);
3940
3941 // TODO: Minimum shift low/high bits are known zero.
3942
3943 if (Op.getResNo() == 0)
3944 Known = Known.extractBits(NumBits: LoBits, BitPosition: 0);
3945 else
3946 Known = Known.extractBits(NumBits: HiBits, BitPosition: LoBits);
3947 break;
3948 }
3949 case ISD::SIGN_EXTEND_INREG: {
3950 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3951 EVT EVT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
3952 Known = Known.sextInReg(SrcBitWidth: EVT.getScalarSizeInBits());
3953 break;
3954 }
3955 case ISD::CTTZ:
3956 case ISD::CTTZ_ZERO_POISON: {
3957 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3958 // If we have a known 1, its position is our upper bound.
3959 unsigned PossibleTZ = Known2.countMaxTrailingZeros();
3960 unsigned LowBits = llvm::bit_width(Value: PossibleTZ);
3961 Known.Zero.setBitsFrom(LowBits);
3962 break;
3963 }
3964 case ISD::CTLZ:
3965 case ISD::CTLZ_ZERO_POISON: {
3966 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3967 // If we have a known 1, its position is our upper bound.
3968 unsigned PossibleLZ = Known2.countMaxLeadingZeros();
3969 unsigned LowBits = llvm::bit_width(Value: PossibleLZ);
3970 Known.Zero.setBitsFrom(LowBits);
3971 break;
3972 }
3973 case ISD::CTLS: {
3974 unsigned MinRedundantSignBits =
3975 ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1) - 1;
3976 ConstantRange Range(APInt(BitWidth, MinRedundantSignBits),
3977 APInt(BitWidth, BitWidth));
3978 Known = Range.toKnownBits();
3979 break;
3980 }
3981 case ISD::CTPOP: {
3982 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3983 // If we know some of the bits are zero, they can't be one.
3984 unsigned PossibleOnes = Known2.countMaxPopulation();
3985 Known.Zero.setBitsFrom(llvm::bit_width(Value: PossibleOnes));
3986 break;
3987 }
3988 case ISD::PARITY: {
3989 // Parity returns 0 everywhere but the LSB.
3990 Known.Zero.setBitsFrom(1);
3991 break;
3992 }
3993 case ISD::PDEP: {
3994 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3995 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3996 Known = KnownBits::pdep(Val: Known2, Mask: Known);
3997 break;
3998 }
3999 case ISD::PEXT: {
4000 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4001 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4002 Known = KnownBits::pext(Val: Known2, Mask: Known);
4003 break;
4004 }
4005 case ISD::CLMUL: {
4006 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4007 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4008 Known = KnownBits::clmul(LHS: Known, RHS: Known2);
4009 break;
4010 }
4011 case ISD::MGATHER:
4012 case ISD::MLOAD: {
4013 ISD::LoadExtType ETy =
4014 (Opcode == ISD::MGATHER)
4015 ? cast<MaskedGatherSDNode>(Val&: Op)->getExtensionType()
4016 : cast<MaskedLoadSDNode>(Val&: Op)->getExtensionType();
4017 if (ETy == ISD::ZEXTLOAD) {
4018 EVT MemVT = cast<MemSDNode>(Val&: Op)->getMemoryVT();
4019 KnownBits Known0(MemVT.getScalarSizeInBits());
4020 return Known0.zext(BitWidth);
4021 }
4022 break;
4023 }
4024 case ISD::LOAD: {
4025 LoadSDNode *LD = cast<LoadSDNode>(Val&: Op);
4026 const Constant *Cst = TLI->getTargetConstantFromLoad(LD);
4027 if (ISD::isNON_EXTLoad(N: LD) && Cst) {
4028 // Determine any common known bits from the loaded constant pool value.
4029 Type *CstTy = Cst->getType();
4030 if ((NumElts * BitWidth) == CstTy->getPrimitiveSizeInBits() &&
4031 !Op.getValueType().isScalableVector()) {
4032 // If its a vector splat, then we can (quickly) reuse the scalar path.
4033 // NOTE: We assume all elements match and none are UNDEF.
4034 if (CstTy->isVectorTy()) {
4035 if (const Constant *Splat = Cst->getSplatValue()) {
4036 Cst = Splat;
4037 CstTy = Cst->getType();
4038 }
4039 }
4040 // TODO - do we need to handle different bitwidths?
4041 if (CstTy->isVectorTy() && BitWidth == CstTy->getScalarSizeInBits()) {
4042 // Iterate across all vector elements finding common known bits.
4043 Known.setAllConflict();
4044 for (unsigned i = 0; i != NumElts; ++i) {
4045 if (!DemandedElts[i])
4046 continue;
4047 if (Constant *Elt = Cst->getAggregateElement(Elt: i)) {
4048 if (auto *CInt = dyn_cast<ConstantInt>(Val: Elt)) {
4049 const APInt &Value = CInt->getValue();
4050 Known.One &= Value;
4051 Known.Zero &= ~Value;
4052 continue;
4053 }
4054 if (auto *CFP = dyn_cast<ConstantFP>(Val: Elt)) {
4055 APInt Value = CFP->getValueAPF().bitcastToAPInt();
4056 Known.One &= Value;
4057 Known.Zero &= ~Value;
4058 continue;
4059 }
4060 }
4061 Known.One.clearAllBits();
4062 Known.Zero.clearAllBits();
4063 break;
4064 }
4065 } else if (BitWidth == CstTy->getPrimitiveSizeInBits()) {
4066 if (auto *CInt = dyn_cast<ConstantInt>(Val: Cst)) {
4067 Known = KnownBits::makeConstant(C: CInt->getValue());
4068 } else if (auto *CFP = dyn_cast<ConstantFP>(Val: Cst)) {
4069 Known =
4070 KnownBits::makeConstant(C: CFP->getValueAPF().bitcastToAPInt());
4071 }
4072 }
4073 }
4074 } else if (Op.getResNo() == 0) {
4075 unsigned ScalarMemorySize = LD->getMemoryVT().getScalarSizeInBits();
4076 KnownBits KnownScalarMemory(ScalarMemorySize);
4077 if (const MDNode *MD = LD->getRanges())
4078 computeKnownBitsFromRangeMetadata(Ranges: *MD, Known&: KnownScalarMemory);
4079
4080 // Extend the Known bits from memory to the size of the scalar result.
4081 if (ISD::isZEXTLoad(N: Op.getNode()))
4082 Known = KnownScalarMemory.zext(BitWidth);
4083 else if (ISD::isSEXTLoad(N: Op.getNode()))
4084 Known = KnownScalarMemory.sext(BitWidth);
4085 else if (ISD::isEXTLoad(N: Op.getNode()))
4086 Known = KnownScalarMemory.anyext(BitWidth);
4087 else
4088 Known = KnownScalarMemory;
4089 assert(Known.getBitWidth() == BitWidth);
4090 return Known;
4091 }
4092 break;
4093 }
4094 case ISD::ZERO_EXTEND_VECTOR_INREG: {
4095 if (Op.getValueType().isScalableVector())
4096 break;
4097 EVT InVT = Op.getOperand(i: 0).getValueType();
4098 APInt InDemandedElts = DemandedElts.zext(width: InVT.getVectorNumElements());
4099 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts: InDemandedElts, Depth: Depth + 1);
4100 Known = Known.zext(BitWidth);
4101 break;
4102 }
4103 case ISD::ZERO_EXTEND: {
4104 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4105 Known = Known.zext(BitWidth);
4106 break;
4107 }
4108 case ISD::SIGN_EXTEND_VECTOR_INREG: {
4109 if (Op.getValueType().isScalableVector())
4110 break;
4111 EVT InVT = Op.getOperand(i: 0).getValueType();
4112 APInt InDemandedElts = DemandedElts.zext(width: InVT.getVectorNumElements());
4113 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts: InDemandedElts, Depth: Depth + 1);
4114 // If the sign bit is known to be zero or one, then sext will extend
4115 // it to the top bits, else it will just zext.
4116 Known = Known.sext(BitWidth);
4117 break;
4118 }
4119 case ISD::SIGN_EXTEND: {
4120 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4121 // If the sign bit is known to be zero or one, then sext will extend
4122 // it to the top bits, else it will just zext.
4123 Known = Known.sext(BitWidth);
4124 break;
4125 }
4126 case ISD::ANY_EXTEND_VECTOR_INREG: {
4127 if (Op.getValueType().isScalableVector())
4128 break;
4129 EVT InVT = Op.getOperand(i: 0).getValueType();
4130 APInt InDemandedElts = DemandedElts.zext(width: InVT.getVectorNumElements());
4131 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts: InDemandedElts, Depth: Depth + 1);
4132 Known = Known.anyext(BitWidth);
4133 break;
4134 }
4135 case ISD::ANY_EXTEND: {
4136 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4137 Known = Known.anyext(BitWidth);
4138 break;
4139 }
4140 case ISD::TRUNCATE: {
4141 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4142 Known = Known.trunc(BitWidth);
4143 break;
4144 }
4145 case ISD::TRUNCATE_SSAT_S: {
4146 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4147 Known = Known.truncSSat(BitWidth);
4148 break;
4149 }
4150 case ISD::TRUNCATE_SSAT_U: {
4151 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4152 Known = Known.truncSSatU(BitWidth);
4153 break;
4154 }
4155 case ISD::TRUNCATE_USAT_U: {
4156 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4157 Known = Known.truncUSat(BitWidth);
4158 break;
4159 }
4160 case ISD::AssertZext: {
4161 EVT VT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
4162 APInt InMask = APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: VT.getSizeInBits());
4163 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4164 Known.Zero |= (~InMask);
4165 Known.One &= (~Known.Zero);
4166 break;
4167 }
4168 case ISD::AssertAlign: {
4169 unsigned LogOfAlign = Log2(A: cast<AssertAlignSDNode>(Val&: Op)->getAlign());
4170 assert(LogOfAlign != 0);
4171
4172 // TODO: Should use maximum with source
4173 // If a node is guaranteed to be aligned, set low zero bits accordingly as
4174 // well as clearing one bits.
4175 Known.Zero.setLowBits(LogOfAlign);
4176 Known.One.clearLowBits(loBits: LogOfAlign);
4177 break;
4178 }
4179 case ISD::AssertNoFPClass: {
4180 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4181
4182 FPClassTest NoFPClass =
4183 static_cast<FPClassTest>(Op.getConstantOperandVal(i: 1));
4184 const FPClassTest NegativeTestMask = fcNan | fcNegative;
4185 if ((NoFPClass & NegativeTestMask) == NegativeTestMask) {
4186 // Cannot be negative.
4187 Known.makeNonNegative();
4188 }
4189
4190 const FPClassTest PositiveTestMask = fcNan | fcPositive;
4191 if ((NoFPClass & PositiveTestMask) == PositiveTestMask) {
4192 // Cannot be positive.
4193 Known.makeNegative();
4194 }
4195
4196 break;
4197 }
4198 case ISD::FABS:
4199 // fabs clears the sign bit
4200 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4201 Known.Zero.setSignBit();
4202 Known.One.clearSignBit();
4203 break;
4204 case ISD::FGETSIGN:
4205 // All bits are zero except the low bit.
4206 Known.Zero.setBitsFrom(1);
4207 break;
4208 case ISD::ADD: {
4209 SDNodeFlags Flags = Op.getNode()->getFlags();
4210 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4211 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4212 bool SelfAdd = Op.getOperand(i: 0) == Op.getOperand(i: 1) &&
4213 isGuaranteedNotToBeUndefOrPoison(
4214 Op: Op.getOperand(i: 0), DemandedElts,
4215 Kind: UndefPoisonKind::UndefOrPoison, Depth: Depth + 1);
4216 Known = KnownBits::add(LHS: Known, RHS: Known2, NSW: Flags.hasNoSignedWrap(),
4217 NUW: Flags.hasNoUnsignedWrap(), SelfAdd);
4218 break;
4219 }
4220 case ISD::SUB: {
4221 SDNodeFlags Flags = Op.getNode()->getFlags();
4222 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4223 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4224 Known = KnownBits::sub(LHS: Known, RHS: Known2, NSW: Flags.hasNoSignedWrap(),
4225 NUW: Flags.hasNoUnsignedWrap());
4226 break;
4227 }
4228 case ISD::USUBO:
4229 case ISD::SSUBO:
4230 case ISD::USUBO_CARRY:
4231 case ISD::SSUBO_CARRY:
4232 if (Op.getResNo() == 1) {
4233 // If we know the result of a setcc has the top bits zero, use this info.
4234 if (TLI->getBooleanContents(Type: Op.getOperand(i: 0).getValueType()) ==
4235 TargetLowering::ZeroOrOneBooleanContent &&
4236 BitWidth > 1)
4237 Known.Zero.setBitsFrom(1);
4238 break;
4239 }
4240 [[fallthrough]];
4241 case ISD::SUBC: {
4242 assert(Op.getResNo() == 0 &&
4243 "We only compute knownbits for the difference here.");
4244
4245 // With USUBO_CARRY and SSUBO_CARRY a borrow bit may be added in.
4246 KnownBits Borrow(1);
4247 if (Opcode == ISD::USUBO_CARRY || Opcode == ISD::SSUBO_CARRY) {
4248 Borrow = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth + 1);
4249 // Borrow has bit width 1
4250 Borrow = Borrow.trunc(BitWidth: 1);
4251 } else {
4252 Borrow.setAllZero();
4253 }
4254
4255 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4256 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4257 Known = KnownBits::computeForSubBorrow(LHS: Known, RHS: Known2, Borrow);
4258 break;
4259 }
4260 case ISD::UADDO:
4261 case ISD::SADDO:
4262 case ISD::UADDO_CARRY:
4263 case ISD::SADDO_CARRY:
4264 if (Op.getResNo() == 1) {
4265 // If we know the result of a setcc has the top bits zero, use this info.
4266 if (TLI->getBooleanContents(Type: Op.getOperand(i: 0).getValueType()) ==
4267 TargetLowering::ZeroOrOneBooleanContent &&
4268 BitWidth > 1)
4269 Known.Zero.setBitsFrom(1);
4270 break;
4271 }
4272 [[fallthrough]];
4273 case ISD::ADDC:
4274 case ISD::ADDE: {
4275 assert(Op.getResNo() == 0 && "We only compute knownbits for the sum here.");
4276
4277 // With ADDE and UADDO_CARRY, a carry bit may be added in.
4278 KnownBits Carry(1);
4279 if (Opcode == ISD::ADDE)
4280 // Can't track carry from glue, set carry to unknown.
4281 Carry.resetAll();
4282 else if (Opcode == ISD::UADDO_CARRY || Opcode == ISD::SADDO_CARRY) {
4283 Carry = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth + 1);
4284 // Carry has bit width 1
4285 Carry = Carry.trunc(BitWidth: 1);
4286 } else {
4287 Carry.setAllZero();
4288 }
4289
4290 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4291 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4292 Known = KnownBits::computeForAddCarry(LHS: Known, RHS: Known2, Carry);
4293 break;
4294 }
4295 case ISD::UDIV: {
4296 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4297 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4298 Known = KnownBits::udiv(LHS: Known, RHS: Known2, Exact: Op->getFlags().hasExact());
4299 break;
4300 }
4301 case ISD::SDIV: {
4302 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4303 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4304 Known = KnownBits::sdiv(LHS: Known, RHS: Known2, Exact: Op->getFlags().hasExact());
4305 break;
4306 }
4307 case ISD::SREM: {
4308 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4309 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4310 Known = KnownBits::srem(LHS: Known, RHS: Known2);
4311 break;
4312 }
4313 case ISD::UREM: {
4314 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4315 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4316 Known = KnownBits::urem(LHS: Known, RHS: Known2);
4317 break;
4318 }
4319 case ISD::EXTRACT_ELEMENT: {
4320 Known = computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth+1);
4321 const unsigned Index = Op.getConstantOperandVal(i: 1);
4322 const unsigned EltBitWidth = Op.getValueSizeInBits();
4323
4324 Known = Known.extractBits(NumBits: EltBitWidth, BitPosition: Index * EltBitWidth);
4325 break;
4326 }
4327 case ISD::EXTRACT_VECTOR_ELT: {
4328 SDValue InVec = Op.getOperand(i: 0);
4329 SDValue EltNo = Op.getOperand(i: 1);
4330 EVT VecVT = InVec.getValueType();
4331 // computeKnownBits not yet implemented for scalable vectors.
4332 if (VecVT.isScalableVector())
4333 break;
4334 const unsigned EltBitWidth = VecVT.getScalarSizeInBits();
4335 const unsigned NumSrcElts = VecVT.getVectorNumElements();
4336
4337 // If BitWidth > EltBitWidth the value is anyext:ed. So we do not know
4338 // anything about the extended bits.
4339 if (BitWidth > EltBitWidth)
4340 Known = Known.trunc(BitWidth: EltBitWidth);
4341
4342 // If we know the element index, just demand that vector element, else for
4343 // an unknown element index, ignore DemandedElts and demand them all.
4344 APInt DemandedSrcElts = APInt::getAllOnes(numBits: NumSrcElts);
4345 auto *ConstEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
4346 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(RHS: NumSrcElts))
4347 DemandedSrcElts =
4348 APInt::getOneBitSet(numBits: NumSrcElts, BitNo: ConstEltNo->getZExtValue());
4349
4350 Known = computeKnownBits(Op: InVec, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
4351 if (BitWidth > EltBitWidth)
4352 Known = Known.anyext(BitWidth);
4353 break;
4354 }
4355 case ISD::BUILD_PAIR: {
4356 // Operand 0 is the low half and operand 1 the high half,
4357 // KnownBits::concat places its argument in the low bits.
4358 Known = computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
4359 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), Depth: Depth + 1);
4360 Known = Known2.concat(Lo: Known);
4361 break;
4362 }
4363 case ISD::INSERT_VECTOR_ELT: {
4364 if (Op.getValueType().isScalableVector())
4365 break;
4366
4367 // If we know the element index, split the demand between the
4368 // source vector and the inserted element, otherwise assume we need
4369 // the original demanded vector elements and the value.
4370 SDValue InVec = Op.getOperand(i: 0);
4371 SDValue InVal = Op.getOperand(i: 1);
4372 SDValue EltNo = Op.getOperand(i: 2);
4373 bool DemandedVal = true;
4374 APInt DemandedVecElts = DemandedElts;
4375 auto *CEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
4376 if (CEltNo && CEltNo->getAPIntValue().ult(RHS: NumElts)) {
4377 unsigned EltIdx = CEltNo->getZExtValue();
4378 DemandedVal = !!DemandedElts[EltIdx];
4379 DemandedVecElts.clearBit(BitPosition: EltIdx);
4380 }
4381 Known.setAllConflict();
4382 if (DemandedVal) {
4383 Known2 = computeKnownBits(Op: InVal, Depth: Depth + 1);
4384 Known = Known.intersectWith(RHS: Known2.zextOrTrunc(BitWidth));
4385 }
4386 if (!!DemandedVecElts) {
4387 Known2 = computeKnownBits(Op: InVec, DemandedElts: DemandedVecElts, Depth: Depth + 1);
4388 Known = Known.intersectWith(RHS: Known2);
4389 }
4390 break;
4391 }
4392 case ISD::BITREVERSE: {
4393 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4394 Known = Known2.reverseBits();
4395 break;
4396 }
4397 case ISD::BSWAP: {
4398 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4399 Known = Known2.byteSwap();
4400 break;
4401 }
4402 case ISD::ABS:
4403 case ISD::ABS_MIN_POISON: {
4404 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4405 Known = Known2.abs();
4406 Known.Zero.setHighBits(
4407 ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1) - 1);
4408 break;
4409 }
4410 case ISD::USUBSAT: {
4411 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4412 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4413 Known = KnownBits::usub_sat(LHS: Known, RHS: Known2);
4414 break;
4415 }
4416 case ISD::UMIN: {
4417 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4418 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4419 Known = KnownBits::umin(LHS: Known, RHS: Known2);
4420 break;
4421 }
4422 case ISD::UMAX: {
4423 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4424 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4425 Known = KnownBits::umax(LHS: Known, RHS: Known2);
4426 break;
4427 }
4428 case ISD::SMIN:
4429 case ISD::SMAX: {
4430 // If we have a clamp pattern, we know that the number of sign bits will be
4431 // the minimum of the clamp min/max range.
4432 bool IsMax = (Opcode == ISD::SMAX);
4433 ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr;
4434 if ((CstLow = isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts)))
4435 if (Op.getOperand(i: 0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX))
4436 CstHigh =
4437 isConstOrConstSplat(N: Op.getOperand(i: 0).getOperand(i: 1), DemandedElts);
4438 if (CstLow && CstHigh) {
4439 if (!IsMax)
4440 std::swap(a&: CstLow, b&: CstHigh);
4441
4442 const APInt &ValueLow = CstLow->getAPIntValue();
4443 const APInt &ValueHigh = CstHigh->getAPIntValue();
4444 if (ValueLow.sle(RHS: ValueHigh)) {
4445 unsigned LowSignBits = ValueLow.getNumSignBits();
4446 unsigned HighSignBits = ValueHigh.getNumSignBits();
4447 unsigned MinSignBits = std::min(a: LowSignBits, b: HighSignBits);
4448 if (ValueLow.isNegative() && ValueHigh.isNegative()) {
4449 Known.One.setHighBits(MinSignBits);
4450 break;
4451 }
4452 if (ValueLow.isNonNegative() && ValueHigh.isNonNegative()) {
4453 Known.Zero.setHighBits(MinSignBits);
4454 break;
4455 }
4456 }
4457 }
4458
4459 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4460 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4461 if (IsMax)
4462 Known = KnownBits::smax(LHS: Known, RHS: Known2);
4463 else
4464 Known = KnownBits::smin(LHS: Known, RHS: Known2);
4465
4466 // For SMAX, if CstLow is non-negative we know the result will be
4467 // non-negative and thus all sign bits are 0.
4468 // TODO: There's an equivalent of this for smin with negative constant for
4469 // known ones.
4470 if (IsMax && CstLow) {
4471 const APInt &ValueLow = CstLow->getAPIntValue();
4472 if (ValueLow.isNonNegative()) {
4473 unsigned SignBits = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
4474 Known.Zero.setHighBits(std::min(a: SignBits, b: ValueLow.getNumSignBits()));
4475 }
4476 }
4477
4478 break;
4479 }
4480 case ISD::UINT_TO_FP: {
4481 Known.makeNonNegative();
4482 break;
4483 }
4484 case ISD::SINT_TO_FP: {
4485 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4486 if (Known2.isNonNegative())
4487 Known.makeNonNegative();
4488 else if (Known2.isNegative())
4489 Known.makeNegative();
4490 break;
4491 }
4492 case ISD::FP_TO_UINT_SAT: {
4493 // FP_TO_UINT_SAT produces an unsigned value that fits in the saturating VT.
4494 EVT VT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
4495 Known.Zero |= APInt::getBitsSetFrom(numBits: BitWidth, loBit: VT.getScalarSizeInBits());
4496 break;
4497 }
4498 case ISD::ATOMIC_LOAD: {
4499 // If we are looking at the loaded value.
4500 if (Op.getResNo() == 0) {
4501 auto *AT = cast<AtomicSDNode>(Val&: Op);
4502 unsigned ScalarMemorySize = AT->getMemoryVT().getScalarSizeInBits();
4503 KnownBits KnownScalarMemory(ScalarMemorySize);
4504 if (const MDNode *MD = AT->getRanges())
4505 computeKnownBitsFromRangeMetadata(Ranges: *MD, Known&: KnownScalarMemory);
4506
4507 switch (AT->getExtensionType()) {
4508 case ISD::ZEXTLOAD:
4509 Known = KnownScalarMemory.zext(BitWidth);
4510 break;
4511 case ISD::SEXTLOAD:
4512 Known = KnownScalarMemory.sext(BitWidth);
4513 break;
4514 case ISD::EXTLOAD:
4515 switch (TLI->getExtendForAtomicOps()) {
4516 case ISD::ZERO_EXTEND:
4517 Known = KnownScalarMemory.zext(BitWidth);
4518 break;
4519 case ISD::SIGN_EXTEND:
4520 Known = KnownScalarMemory.sext(BitWidth);
4521 break;
4522 default:
4523 Known = KnownScalarMemory.anyext(BitWidth);
4524 break;
4525 }
4526 break;
4527 case ISD::NON_EXTLOAD:
4528 Known = KnownScalarMemory;
4529 break;
4530 }
4531 assert(Known.getBitWidth() == BitWidth);
4532 }
4533 break;
4534 }
4535 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
4536 if (Op.getResNo() == 1) {
4537 // The boolean result conforms to getBooleanContents.
4538 // If we know the result of a setcc has the top bits zero, use this info.
4539 // We know that we have an integer-based boolean since these operations
4540 // are only available for integer.
4541 if (TLI->getBooleanContents(isVec: Op.getValueType().isVector(), isFloat: false) ==
4542 TargetLowering::ZeroOrOneBooleanContent &&
4543 BitWidth > 1)
4544 Known.Zero.setBitsFrom(1);
4545 break;
4546 }
4547 [[fallthrough]];
4548 case ISD::ATOMIC_CMP_SWAP:
4549 case ISD::ATOMIC_SWAP:
4550 case ISD::ATOMIC_LOAD_ADD:
4551 case ISD::ATOMIC_LOAD_SUB:
4552 case ISD::ATOMIC_LOAD_AND:
4553 case ISD::ATOMIC_LOAD_CLR:
4554 case ISD::ATOMIC_LOAD_OR:
4555 case ISD::ATOMIC_LOAD_XOR:
4556 case ISD::ATOMIC_LOAD_NAND:
4557 case ISD::ATOMIC_LOAD_MIN:
4558 case ISD::ATOMIC_LOAD_MAX:
4559 case ISD::ATOMIC_LOAD_UMIN:
4560 case ISD::ATOMIC_LOAD_UMAX: {
4561 // If we are looking at the loaded value.
4562 if (Op.getResNo() == 0) {
4563 auto *AT = cast<AtomicSDNode>(Val&: Op);
4564 unsigned MemBits = AT->getMemoryVT().getScalarSizeInBits();
4565
4566 if (TLI->getExtendForAtomicOps() == ISD::ZERO_EXTEND)
4567 Known.Zero.setBitsFrom(MemBits);
4568 }
4569 break;
4570 }
4571 case ISD::FrameIndex:
4572 case ISD::TargetFrameIndex: {
4573 const MachineFunction &MF = getMachineFunction();
4574 int FrameIdx = cast<FrameIndexSDNode>(Val&: Op)->getIndex();
4575 TLI->computeKnownBitsForStackObjectPointer(
4576 Known, MF, Alignment: MF.getFrameInfo().getObjectAlign(ObjectIdx: FrameIdx));
4577 break;
4578 }
4579 case ISD::VP_LOAD_FF: {
4580 if (Op.getResNo() != 1)
4581 break;
4582 // The second result of vp.load.ff is an unsigned value that is less than or
4583 // equal to the EVL operand.
4584 KnownBits VLKB =
4585 computeKnownBits(Op: Op.getOperand(i: 3), DemandedElts, Depth: Depth + 1);
4586 // The new VL is also bounded by the largest vector length.
4587 EVT ResVT = Op->getValueType(ResNo: 0);
4588 auto ResKB = KnownBits::makeConstant(
4589 C: APInt(BitWidth, ResVT.getVectorMinNumElements()));
4590 if (ResVT.isScalableVector()) {
4591 const Function &F = getMachineFunction().getFunction();
4592 ResKB = KnownBits::mul(LHS: getVScaleRange(F: &F, BitWidth).toKnownBits(), RHS: ResKB);
4593 }
4594 Known.Zero.setHighBits(KnownBits::umin(LHS: VLKB, RHS: ResKB).countMinLeadingZeros());
4595 break;
4596 }
4597
4598 default:
4599 if (Opcode < ISD::BUILTIN_OP_END)
4600 break;
4601 [[fallthrough]];
4602 case ISD::INTRINSIC_WO_CHAIN:
4603 case ISD::INTRINSIC_W_CHAIN:
4604 case ISD::INTRINSIC_VOID:
4605 // Allow the target to implement this method for its nodes.
4606 TLI->computeKnownBitsForTargetNode(Op, Known, DemandedElts, DAG: *this, Depth);
4607 break;
4608 }
4609
4610 return Known;
4611}
4612
4613/// Convert ConstantRange OverflowResult into SelectionDAG::OverflowKind.
4614static SelectionDAG::OverflowKind mapOverflowResult(ConstantRange::OverflowResult OR) {
4615 switch (OR) {
4616 case ConstantRange::OverflowResult::MayOverflow:
4617 return SelectionDAG::OFK_Sometime;
4618 case ConstantRange::OverflowResult::AlwaysOverflowsLow:
4619 case ConstantRange::OverflowResult::AlwaysOverflowsHigh:
4620 return SelectionDAG::OFK_Always;
4621 case ConstantRange::OverflowResult::NeverOverflows:
4622 return SelectionDAG::OFK_Never;
4623 }
4624 llvm_unreachable("Unknown OverflowResult");
4625}
4626
4627SelectionDAG::OverflowKind
4628SelectionDAG::computeOverflowForSignedAdd(SDValue N0, SDValue N1) const {
4629 // X + 0 never overflow
4630 if (isNullConstant(V: N1))
4631 return OFK_Never;
4632
4633 // If both operands each have at least two sign bits, the addition
4634 // cannot overflow.
4635 if (ComputeNumSignBits(Op: N0) > 1 && ComputeNumSignBits(Op: N1) > 1)
4636 return OFK_Never;
4637
4638 // TODO: Add ConstantRange::signedAddMayOverflow handling.
4639 return OFK_Sometime;
4640}
4641
4642SelectionDAG::OverflowKind
4643SelectionDAG::computeOverflowForUnsignedAdd(SDValue N0, SDValue N1) const {
4644 // X + 0 never overflow
4645 if (isNullConstant(V: N1))
4646 return OFK_Never;
4647
4648 // mulhi + 1 never overflow
4649 KnownBits N1Known = computeKnownBits(Op: N1);
4650 if (N0.getOpcode() == ISD::UMUL_LOHI && N0.getResNo() == 1 &&
4651 N1Known.getMaxValue().ult(RHS: 2))
4652 return OFK_Never;
4653
4654 KnownBits N0Known = computeKnownBits(Op: N0);
4655 if (N1.getOpcode() == ISD::UMUL_LOHI && N1.getResNo() == 1 &&
4656 N0Known.getMaxValue().ult(RHS: 2))
4657 return OFK_Never;
4658
4659 // Fallback to ConstantRange::unsignedAddMayOverflow handling.
4660 ConstantRange N0Range = ConstantRange::fromKnownBits(Known: N0Known, IsSigned: false);
4661 ConstantRange N1Range = ConstantRange::fromKnownBits(Known: N1Known, IsSigned: false);
4662 return mapOverflowResult(OR: N0Range.unsignedAddMayOverflow(Other: N1Range));
4663}
4664
4665SelectionDAG::OverflowKind
4666SelectionDAG::computeOverflowForSignedSub(SDValue N0, SDValue N1) const {
4667 // X - 0 never overflow
4668 if (isNullConstant(V: N1))
4669 return OFK_Never;
4670
4671 // If both operands each have at least two sign bits, the subtraction
4672 // cannot overflow.
4673 if (ComputeNumSignBits(Op: N0) > 1 && ComputeNumSignBits(Op: N1) > 1)
4674 return OFK_Never;
4675
4676 KnownBits N0Known = computeKnownBits(Op: N0);
4677 KnownBits N1Known = computeKnownBits(Op: N1);
4678 ConstantRange N0Range = ConstantRange::fromKnownBits(Known: N0Known, IsSigned: true);
4679 ConstantRange N1Range = ConstantRange::fromKnownBits(Known: N1Known, IsSigned: true);
4680 return mapOverflowResult(OR: N0Range.signedSubMayOverflow(Other: N1Range));
4681}
4682
4683SelectionDAG::OverflowKind
4684SelectionDAG::computeOverflowForUnsignedSub(SDValue N0, SDValue N1) const {
4685 // X - 0 never overflow
4686 if (isNullConstant(V: N1))
4687 return OFK_Never;
4688
4689 ConstantRange N0Range =
4690 computeConstantRangeIncludingKnownBits(Op: N0, /*ForSigned=*/false);
4691 ConstantRange N1Range =
4692 computeConstantRangeIncludingKnownBits(Op: N1, /*ForSigned=*/false);
4693 return mapOverflowResult(OR: N0Range.unsignedSubMayOverflow(Other: N1Range));
4694}
4695
4696SelectionDAG::OverflowKind
4697SelectionDAG::computeOverflowForUnsignedMul(SDValue N0, SDValue N1) const {
4698 // X * 0 and X * 1 never overflow.
4699 if (isNullConstant(V: N1) || isOneConstant(V: N1))
4700 return OFK_Never;
4701
4702 ConstantRange N0Range = computeConstantRangeIncludingKnownBits(Op: N0, ForSigned: false);
4703 ConstantRange N1Range = computeConstantRangeIncludingKnownBits(Op: N1, ForSigned: false);
4704 return mapOverflowResult(OR: N0Range.unsignedMulMayOverflow(Other: N1Range));
4705}
4706
4707SelectionDAG::OverflowKind
4708SelectionDAG::computeOverflowForSignedMul(SDValue N0, SDValue N1) const {
4709 // X * 0 and X * 1 never overflow.
4710 if (isNullConstant(V: N1) || isOneConstant(V: N1))
4711 return OFK_Never;
4712
4713 // Get the size of the result.
4714 unsigned BitWidth = N0.getScalarValueSizeInBits();
4715
4716 // Sum of the sign bits.
4717 unsigned SignBits = ComputeNumSignBits(Op: N0) + ComputeNumSignBits(Op: N1);
4718
4719 // If we have enough sign bits, then there's no overflow.
4720 if (SignBits > BitWidth + 1)
4721 return OFK_Never;
4722
4723 if (SignBits == BitWidth + 1) {
4724 // The overflow occurs when the true multiplication of the
4725 // the operands is the minimum negative number.
4726 KnownBits N0Known = computeKnownBits(Op: N0);
4727 KnownBits N1Known = computeKnownBits(Op: N1);
4728 // If one of the operands is non-negative, then there's no
4729 // overflow.
4730 if (N0Known.isNonNegative() || N1Known.isNonNegative())
4731 return OFK_Never;
4732 }
4733
4734 return OFK_Sometime;
4735}
4736
4737ConstantRange SelectionDAG::computeConstantRange(SDValue Op, bool ForSigned,
4738 unsigned Depth) const {
4739 APInt DemandedElts = getDemandAllEltsMask(V: Op);
4740 return computeConstantRange(Op, DemandedElts, ForSigned, Depth);
4741}
4742
4743ConstantRange SelectionDAG::computeConstantRange(SDValue Op,
4744 const APInt &DemandedElts,
4745 bool ForSigned,
4746 unsigned Depth) const {
4747 EVT VT = Op.getValueType();
4748 unsigned BitWidth = VT.getScalarSizeInBits();
4749
4750 if (Depth >= MaxRecursionDepth)
4751 return ConstantRange::getFull(BitWidth);
4752
4753 if (ConstantSDNode *C = isConstOrConstSplat(N: Op, DemandedElts))
4754 return ConstantRange(C->getAPIntValue());
4755
4756 unsigned Opcode = Op.getOpcode();
4757 switch (Opcode) {
4758 case ISD::VSCALE: {
4759 const Function &F = getMachineFunction().getFunction();
4760 const APInt &Multiplier = Op.getConstantOperandAPInt(i: 0);
4761 return getVScaleRange(F: &F, BitWidth).multiply(Other: Multiplier);
4762 }
4763 default:
4764 break;
4765 }
4766
4767 return ConstantRange::getFull(BitWidth);
4768}
4769
4770ConstantRange
4771SelectionDAG::computeConstantRangeIncludingKnownBits(SDValue Op, bool ForSigned,
4772 unsigned Depth) const {
4773 APInt DemandedElts = getDemandAllEltsMask(V: Op);
4774 return computeConstantRangeIncludingKnownBits(Op, DemandedElts, ForSigned,
4775 Depth);
4776}
4777
4778ConstantRange SelectionDAG::computeConstantRangeIncludingKnownBits(
4779 SDValue Op, const APInt &DemandedElts, bool ForSigned,
4780 unsigned Depth) const {
4781 KnownBits Known = computeKnownBits(Op, DemandedElts, Depth);
4782 ConstantRange CR1 = ConstantRange::fromKnownBits(Known, IsSigned: ForSigned);
4783 ConstantRange CR2 = computeConstantRange(Op, DemandedElts, ForSigned, Depth);
4784 ConstantRange::PreferredRangeType RangeType =
4785 ForSigned ? ConstantRange::Signed : ConstantRange::Unsigned;
4786 return CR1.intersectWith(CR: CR2, Type: RangeType);
4787}
4788
4789bool SelectionDAG::isKnownToBeAPowerOfTwo(SDValue Val, bool OrZero,
4790 unsigned Depth) const {
4791 APInt DemandedElts = getDemandAllEltsMask(V: Val);
4792 return isKnownToBeAPowerOfTwo(Val, DemandedElts, OrZero, Depth);
4793}
4794
4795bool SelectionDAG::isKnownToBeAPowerOfTwo(SDValue Val,
4796 const APInt &DemandedElts,
4797 bool OrZero, unsigned Depth) const {
4798 if (Depth >= MaxRecursionDepth)
4799 return false; // Limit search depth.
4800
4801 EVT OpVT = Val.getValueType();
4802 unsigned BitWidth = OpVT.getScalarSizeInBits();
4803 [[maybe_unused]] unsigned NumElts = DemandedElts.getBitWidth();
4804 assert((!OpVT.isScalableVector() || NumElts == 1) &&
4805 "DemandedElts for scalable vectors must be 1 to represent all lanes");
4806 assert(
4807 (!OpVT.isFixedLengthVector() || NumElts == OpVT.getVectorNumElements()) &&
4808 "Unexpected vector size");
4809
4810 auto IsPowerOfTwoOrZero = [BitWidth, OrZero](const ConstantSDNode *C) {
4811 APInt V = C->getAPIntValue().zextOrTrunc(width: BitWidth);
4812 return (OrZero && V.isZero()) || V.isPowerOf2();
4813 };
4814
4815 // Is the constant a known power of 2 or zero?
4816 if (ISD::matchUnaryPredicate(Op: Val, DemandedElts, Match: IsPowerOfTwoOrZero,
4817 /*AllowUndefs=*/false, /*AllowTruncation=*/true))
4818 return true;
4819
4820 switch (Val.getOpcode()) {
4821 case ISD::EXTRACT_VECTOR_ELT: {
4822 SDValue InVec = Val.getOperand(i: 0);
4823 SDValue EltNo = Val.getOperand(i: 1);
4824 EVT VecVT = InVec.getValueType();
4825
4826 // Skip scalable vectors or implicit extensions.
4827 if (VecVT.isScalableVector() ||
4828 OpVT.getScalarSizeInBits() != VecVT.getScalarSizeInBits())
4829 break;
4830
4831 // If we know the element index, just demand that vector element, else for
4832 // an unknown element index, ignore DemandedElts and demand them all.
4833 const unsigned NumSrcElts = VecVT.getVectorNumElements();
4834 auto *ConstEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
4835 APInt DemandedSrcElts =
4836 ConstEltNo && ConstEltNo->getAPIntValue().ult(RHS: NumSrcElts)
4837 ? APInt::getOneBitSet(numBits: NumSrcElts, BitNo: ConstEltNo->getZExtValue())
4838 : APInt::getAllOnes(numBits: NumSrcElts);
4839 return isKnownToBeAPowerOfTwo(Val: InVec, DemandedElts: DemandedSrcElts, OrZero, Depth: Depth + 1);
4840 }
4841
4842 case ISD::AND: {
4843 // Looking for `x & -x` pattern:
4844 // If x == 0:
4845 // x & -x -> 0
4846 // If x != 0:
4847 // x & -x -> non-zero pow2
4848 // so if we find the pattern return whether we know `x` is non-zero.
4849 SDValue X, Z;
4850 if (sd_match(N: Val, P: m_And(L: m_Value(N&: X), R: m_Neg(V: m_Deferred(V&: X)))) ||
4851 (sd_match(N: Val, P: m_And(L: m_Value(N&: X), R: m_Sub(L: m_Value(N&: Z), R: m_Deferred(V&: X)))) &&
4852 MaskedVectorIsZero(V: Z, DemandedElts, Depth: Depth + 1)))
4853 return OrZero || isKnownNeverZero(Op: X, DemandedElts, Depth);
4854 break;
4855 }
4856
4857 case ISD::SHL: {
4858 // A left-shift of a constant one will have exactly one bit set because
4859 // shifting the bit off the end is undefined.
4860 auto *C = isConstOrConstSplat(N: Val.getOperand(i: 0), DemandedElts);
4861 if (C && C->getAPIntValue() == 1)
4862 return true;
4863 return (OrZero || isKnownNeverZero(Op: Val, DemandedElts, Depth)) &&
4864 isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), DemandedElts, OrZero,
4865 Depth: Depth + 1);
4866 }
4867
4868 case ISD::SRL: {
4869 // A logical right-shift of a constant sign-bit will have exactly
4870 // one bit set.
4871 auto *C = isConstOrConstSplat(N: Val.getOperand(i: 0), DemandedElts);
4872 if (C && C->getAPIntValue().isSignMask())
4873 return true;
4874 return (OrZero || isKnownNeverZero(Op: Val, DemandedElts, Depth)) &&
4875 isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), DemandedElts, OrZero,
4876 Depth: Depth + 1);
4877 }
4878
4879 case ISD::TRUNCATE:
4880 return (OrZero || isKnownNeverZero(Op: Val, DemandedElts, Depth)) &&
4881 isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), DemandedElts, OrZero,
4882 Depth: Depth + 1);
4883
4884 case ISD::ROTL:
4885 case ISD::ROTR:
4886 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), DemandedElts, OrZero,
4887 Depth: Depth + 1);
4888 case ISD::BSWAP:
4889 case ISD::BITREVERSE:
4890 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), DemandedElts, OrZero,
4891 Depth: Depth + 1);
4892
4893 case ISD::SMIN:
4894 case ISD::SMAX:
4895 case ISD::UMIN:
4896 case ISD::UMAX:
4897 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 1), DemandedElts, OrZero,
4898 Depth: Depth + 1) &&
4899 isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), DemandedElts, OrZero,
4900 Depth: Depth + 1);
4901
4902 case ISD::SELECT:
4903 case ISD::VSELECT:
4904 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 2), DemandedElts, OrZero,
4905 Depth: Depth + 1) &&
4906 isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 1), DemandedElts, OrZero,
4907 Depth: Depth + 1);
4908
4909 case ISD::ZERO_EXTEND:
4910 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), DemandedElts, OrZero,
4911 Depth: Depth + 1);
4912
4913 case ISD::VSCALE:
4914 // vscale(power-of-two) is a power-of-two
4915 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), /*OrZero=*/false,
4916 Depth: Depth + 1);
4917
4918 case ISD::VECTOR_SHUFFLE: {
4919 assert(!Val.getValueType().isScalableVector());
4920 // Demanded elements with undef shuffle mask elements are unknown
4921 // - we cannot guarantee they are a power of two, so return false.
4922 APInt DemandedLHS, DemandedRHS;
4923 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val);
4924 assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
4925 if (!getShuffleDemandedElts(SrcWidth: NumElts, Mask: SVN->getMask(), DemandedElts,
4926 DemandedLHS, DemandedRHS))
4927 return false;
4928
4929 // All demanded elements from LHS must be known power of two.
4930 if (!!DemandedLHS && !isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), DemandedElts: DemandedLHS,
4931 OrZero, Depth: Depth + 1))
4932 return false;
4933
4934 // All demanded elements from RHS must be known power of two.
4935 if (!!DemandedRHS && !isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 1), DemandedElts: DemandedRHS,
4936 OrZero, Depth: Depth + 1))
4937 return false;
4938
4939 return true;
4940 }
4941 }
4942
4943 // More could be done here, though the above checks are enough
4944 // to handle some common cases.
4945 return false;
4946}
4947
4948bool SelectionDAG::isKnownToBeAPowerOfTwoFP(SDValue Val, unsigned Depth) const {
4949 if (ConstantFPSDNode *C1 = isConstOrConstSplatFP(N: Val, AllowUndefs: true))
4950 return C1->getValueAPF().getExactLog2Abs() >= 0;
4951
4952 if (Val.getOpcode() == ISD::UINT_TO_FP || Val.getOpcode() == ISD::SINT_TO_FP)
4953 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), OrZero: Depth + 1);
4954
4955 return false;
4956}
4957
4958unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const {
4959 APInt DemandedElts = getDemandAllEltsMask(V: Op);
4960 return ComputeNumSignBits(Op, DemandedElts, Depth);
4961}
4962
4963unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, const APInt &DemandedElts,
4964 unsigned Depth) const {
4965 EVT VT = Op.getValueType();
4966 assert((VT.isInteger() || VT.isFloatingPoint()) && "Invalid VT!");
4967 unsigned VTBits = VT.getScalarSizeInBits();
4968 unsigned NumElts = DemandedElts.getBitWidth();
4969 unsigned Tmp, Tmp2;
4970 unsigned FirstAnswer = 1;
4971
4972 assert((!VT.isScalableVector() || NumElts == 1) &&
4973 "DemandedElts for scalable vectors must be 1 to represent all lanes");
4974
4975 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op)) {
4976 const APInt &Val = C->getAPIntValue();
4977 return Val.getNumSignBits();
4978 }
4979
4980 if (Depth >= MaxRecursionDepth)
4981 return 1; // Limit search depth.
4982
4983 if (!DemandedElts)
4984 return 1; // No demanded elts, better to assume we don't know anything.
4985
4986 unsigned Opcode = Op.getOpcode();
4987 switch (Opcode) {
4988 default: break;
4989 case ISD::AssertSext:
4990 Tmp = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT().getSizeInBits();
4991 return VTBits-Tmp+1;
4992 case ISD::AssertZext:
4993 Tmp = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT().getSizeInBits();
4994 return VTBits-Tmp;
4995 case ISD::FREEZE:
4996 if (isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 0), DemandedElts,
4997 Kind: UndefPoisonKind::UndefOrPoison))
4998 return ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4999 break;
5000 case ISD::MERGE_VALUES:
5001 return ComputeNumSignBits(Op: Op.getOperand(i: Op.getResNo()), DemandedElts,
5002 Depth: Depth + 1);
5003 case ISD::SPLAT_VECTOR: {
5004 // Check if the sign bits of source go down as far as the truncated value.
5005 unsigned NumSrcBits = Op.getOperand(i: 0).getValueSizeInBits();
5006 unsigned NumSrcSignBits = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
5007 if (NumSrcSignBits > (NumSrcBits - VTBits))
5008 return NumSrcSignBits - (NumSrcBits - VTBits);
5009 break;
5010 }
5011 case ISD::BUILD_VECTOR:
5012 assert(!VT.isScalableVector());
5013 Tmp = VTBits;
5014 for (unsigned i = 0, e = Op.getNumOperands(); (i < e) && (Tmp > 1); ++i) {
5015 if (!DemandedElts[i])
5016 continue;
5017
5018 SDValue SrcOp = Op.getOperand(i);
5019 // BUILD_VECTOR can implicitly truncate sources, we handle this specially
5020 // for constant nodes to ensure we only look at the sign bits.
5021 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: SrcOp)) {
5022 APInt T = C->getAPIntValue().trunc(width: VTBits);
5023 Tmp2 = T.getNumSignBits();
5024 } else if (SrcOp.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
5025 SrcOp.getOperand(i: 0).getScalarValueSizeInBits() >= VTBits) {
5026 // EXTRACT_VECTOR_ELT can extend the value with high bits undefined. If
5027 // this BUILD_VECTOR truncates those undefined bits we can just look
5028 // through the SrcOp and query the vector directly.
5029 SDValue InVec = SrcOp.getOperand(i: 0);
5030 EVT InVecVT = InVec.getValueType();
5031
5032 APInt DemandedSrcElts;
5033 if (InVecVT.isScalableVector())
5034 // Demand all elements.
5035 DemandedSrcElts = APInt(1, 1);
5036 else {
5037 unsigned NumSrcElts = InVecVT.getVectorNumElements();
5038 auto *ConstEltNo = dyn_cast<ConstantSDNode>(Val: SrcOp.getOperand(i: 1));
5039 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(RHS: NumSrcElts))
5040 DemandedSrcElts =
5041 APInt::getOneBitSet(numBits: NumSrcElts, BitNo: ConstEltNo->getZExtValue());
5042 else
5043 DemandedSrcElts = APInt::getAllOnes(numBits: NumSrcElts);
5044 }
5045
5046 Tmp2 = ComputeNumSignBits(Op: InVec, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
5047 unsigned ExtraBits = InVec.getScalarValueSizeInBits() - VTBits;
5048 if (ExtraBits)
5049 Tmp2 = (Tmp2 > ExtraBits ? Tmp2 - ExtraBits : 1);
5050 } else {
5051 Tmp2 = ComputeNumSignBits(Op: SrcOp, Depth: Depth + 1);
5052
5053 if (SrcOp.getValueSizeInBits() != VTBits) {
5054 assert(SrcOp.getValueSizeInBits() > VTBits &&
5055 "Expected BUILD_VECTOR implicit truncation");
5056 unsigned ExtraBits = SrcOp.getValueSizeInBits() - VTBits;
5057 Tmp2 = (Tmp2 > ExtraBits ? Tmp2 - ExtraBits : 1);
5058 }
5059 }
5060 Tmp = std::min(a: Tmp, b: Tmp2);
5061 }
5062 return Tmp;
5063
5064 case ISD::VECTOR_COMPRESS: {
5065 SDValue Vec = Op.getOperand(i: 0);
5066 SDValue PassThru = Op.getOperand(i: 2);
5067 Tmp = ComputeNumSignBits(Op: PassThru, DemandedElts, Depth: Depth + 1);
5068 if (Tmp == 1)
5069 return 1;
5070 Tmp2 = ComputeNumSignBits(Op: Vec, Depth: Depth + 1);
5071 Tmp = std::min(a: Tmp, b: Tmp2);
5072 return Tmp;
5073 }
5074
5075 case ISD::VECTOR_SHUFFLE: {
5076 // Collect the minimum number of sign bits that are shared by every vector
5077 // element referenced by the shuffle.
5078 APInt DemandedLHS, DemandedRHS;
5079 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
5080 assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
5081 if (!getShuffleDemandedElts(SrcWidth: NumElts, Mask: SVN->getMask(), DemandedElts,
5082 DemandedLHS, DemandedRHS))
5083 return 1;
5084
5085 Tmp = std::numeric_limits<unsigned>::max();
5086 if (!!DemandedLHS)
5087 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts: DemandedLHS, Depth: Depth + 1);
5088 if (!!DemandedRHS) {
5089 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts: DemandedRHS, Depth: Depth + 1);
5090 Tmp = std::min(a: Tmp, b: Tmp2);
5091 }
5092 // If we don't know anything, early out and try computeKnownBits fall-back.
5093 if (Tmp == 1)
5094 break;
5095 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
5096 return Tmp;
5097 }
5098
5099 case ISD::BITCAST: {
5100 if (VT.isScalableVector())
5101 break;
5102 SDValue N0 = Op.getOperand(i: 0);
5103 EVT SrcVT = N0.getValueType();
5104 unsigned SrcBits = SrcVT.getScalarSizeInBits();
5105
5106 // Ignore bitcasts from unsupported types..
5107 if (!(SrcVT.isInteger() || SrcVT.isFloatingPoint()))
5108 break;
5109
5110 // Fast handling of 'identity' bitcasts.
5111 if (VTBits == SrcBits)
5112 return ComputeNumSignBits(Op: N0, DemandedElts, Depth: Depth + 1);
5113
5114 bool IsLE = getDataLayout().isLittleEndian();
5115
5116 // Bitcast 'large element' scalar/vector to 'small element' vector.
5117 if ((SrcBits % VTBits) == 0) {
5118 assert(VT.isVector() && "Expected bitcast to vector");
5119
5120 unsigned Scale = SrcBits / VTBits;
5121 APInt SrcDemandedElts =
5122 APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumElts / Scale);
5123
5124 // Fast case - sign splat can be simply split across the small elements.
5125 Tmp = ComputeNumSignBits(Op: N0, DemandedElts: SrcDemandedElts, Depth: Depth + 1);
5126 if (Tmp == SrcBits)
5127 return VTBits;
5128
5129 // Slow case - determine how far the sign extends into each sub-element.
5130 Tmp2 = VTBits;
5131 for (unsigned i = 0; i != NumElts; ++i)
5132 if (DemandedElts[i]) {
5133 unsigned SubOffset = i % Scale;
5134 SubOffset = (IsLE ? ((Scale - 1) - SubOffset) : SubOffset);
5135 SubOffset = SubOffset * VTBits;
5136 if (Tmp <= SubOffset)
5137 return 1;
5138 Tmp2 = std::min(a: Tmp2, b: Tmp - SubOffset);
5139 }
5140 return Tmp2;
5141 }
5142 break;
5143 }
5144
5145 case ISD::FP_TO_SINT_SAT:
5146 // FP_TO_SINT_SAT produces a signed value that fits in the saturating VT.
5147 Tmp = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT().getScalarSizeInBits();
5148 return VTBits - Tmp + 1;
5149 case ISD::SIGN_EXTEND:
5150 Tmp = VTBits - Op.getOperand(i: 0).getScalarValueSizeInBits();
5151 return ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth+1) + Tmp;
5152 case ISD::SIGN_EXTEND_INREG:
5153 // Max of the input and what this extends.
5154 Tmp = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT().getScalarSizeInBits();
5155 Tmp = VTBits-Tmp+1;
5156 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth+1);
5157 return std::max(a: Tmp, b: Tmp2);
5158 case ISD::SIGN_EXTEND_VECTOR_INREG: {
5159 if (VT.isScalableVector())
5160 break;
5161 SDValue Src = Op.getOperand(i: 0);
5162 EVT SrcVT = Src.getValueType();
5163 APInt DemandedSrcElts = DemandedElts.zext(width: SrcVT.getVectorNumElements());
5164 Tmp = VTBits - SrcVT.getScalarSizeInBits();
5165 return ComputeNumSignBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth+1) + Tmp;
5166 }
5167 case ISD::SRA:
5168 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5169 // SRA X, C -> adds C sign bits.
5170 if (std::optional<unsigned> ShAmt =
5171 getValidMinimumShiftAmount(V: Op, DemandedElts, Depth: Depth + 1))
5172 Tmp = std::min(a: Tmp + *ShAmt, b: VTBits);
5173 return Tmp;
5174 case ISD::SHL:
5175 if (std::optional<ConstantRange> ShAmtRange =
5176 getValidShiftAmountRange(V: Op, DemandedElts, Depth: Depth + 1)) {
5177 unsigned MaxShAmt = ShAmtRange->getUnsignedMax().getZExtValue();
5178 unsigned MinShAmt = ShAmtRange->getUnsignedMin().getZExtValue();
5179 // Try to look through ZERO/SIGN/ANY_EXTEND. If all extended bits are
5180 // shifted out, then we can compute the number of sign bits for the
5181 // operand being extended. A future improvement could be to pass along the
5182 // "shifted left by" information in the recursive calls to
5183 // ComputeKnownSignBits. Allowing us to handle this more generically.
5184 if (ISD::isExtOpcode(Opcode: Op.getOperand(i: 0).getOpcode())) {
5185 SDValue Ext = Op.getOperand(i: 0);
5186 EVT ExtVT = Ext.getValueType();
5187 SDValue Extendee = Ext.getOperand(i: 0);
5188 EVT ExtendeeVT = Extendee.getValueType();
5189 unsigned SizeDifference =
5190 ExtVT.getScalarSizeInBits() - ExtendeeVT.getScalarSizeInBits();
5191 if (SizeDifference <= MinShAmt) {
5192 Tmp = SizeDifference +
5193 ComputeNumSignBits(Op: Extendee, DemandedElts, Depth: Depth + 1);
5194 if (MaxShAmt < Tmp)
5195 return Tmp - MaxShAmt;
5196 }
5197 }
5198 // shl destroys sign bits, ensure it doesn't shift out all sign bits.
5199 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5200 if (MaxShAmt < Tmp)
5201 return Tmp - MaxShAmt;
5202 }
5203 break;
5204 case ISD::AND:
5205 case ISD::OR:
5206 case ISD::XOR: // NOT is handled here.
5207 // Logical binary ops preserve the number of sign bits at the worst.
5208 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth+1);
5209 if (Tmp != 1) {
5210 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth+1);
5211 FirstAnswer = std::min(a: Tmp, b: Tmp2);
5212 // We computed what we know about the sign bits as our first
5213 // answer. Now proceed to the generic code that uses
5214 // computeKnownBits, and pick whichever answer is better.
5215 }
5216 break;
5217
5218 case ISD::SELECT:
5219 case ISD::VSELECT:
5220 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth+1);
5221 if (Tmp == 1) return 1; // Early out.
5222 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth+1);
5223 return std::min(a: Tmp, b: Tmp2);
5224 case ISD::SELECT_CC:
5225 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth+1);
5226 if (Tmp == 1) return 1; // Early out.
5227 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 3), DemandedElts, Depth: Depth+1);
5228 return std::min(a: Tmp, b: Tmp2);
5229
5230 case ISD::SMIN:
5231 case ISD::SMAX: {
5232 // If we have a clamp pattern, we know that the number of sign bits will be
5233 // the minimum of the clamp min/max range.
5234 bool IsMax = (Opcode == ISD::SMAX);
5235 ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr;
5236 if ((CstLow = isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts)))
5237 if (Op.getOperand(i: 0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX))
5238 CstHigh =
5239 isConstOrConstSplat(N: Op.getOperand(i: 0).getOperand(i: 1), DemandedElts);
5240 if (CstLow && CstHigh) {
5241 if (!IsMax)
5242 std::swap(a&: CstLow, b&: CstHigh);
5243 if (CstLow->getAPIntValue().sle(RHS: CstHigh->getAPIntValue())) {
5244 Tmp = CstLow->getAPIntValue().getNumSignBits();
5245 Tmp2 = CstHigh->getAPIntValue().getNumSignBits();
5246 return std::min(a: Tmp, b: Tmp2);
5247 }
5248 }
5249
5250 // Fallback - just get the minimum number of sign bits of the operands.
5251 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5252 if (Tmp == 1)
5253 return 1; // Early out.
5254 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5255 return std::min(a: Tmp, b: Tmp2);
5256 }
5257 case ISD::UMIN:
5258 case ISD::UMAX:
5259 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5260 if (Tmp == 1)
5261 return 1; // Early out.
5262 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5263 return std::min(a: Tmp, b: Tmp2);
5264 case ISD::SSUBO_CARRY:
5265 case ISD::USUBO_CARRY:
5266 // sub_carry(x,x,c) -> 0/-1 (sext carry)
5267 if (Op.getResNo() == 0 && Op.getOperand(i: 0) == Op.getOperand(i: 1))
5268 return VTBits;
5269 [[fallthrough]];
5270 case ISD::SADDO:
5271 case ISD::UADDO:
5272 case ISD::SADDO_CARRY:
5273 case ISD::UADDO_CARRY:
5274 case ISD::SSUBO:
5275 case ISD::USUBO:
5276 case ISD::SMULO:
5277 case ISD::UMULO:
5278 if (Op.getResNo() != 1)
5279 break;
5280 // The boolean result conforms to getBooleanContents. Fall through.
5281 // If setcc returns 0/-1, all bits are sign bits.
5282 // We know that we have an integer-based boolean since these operations
5283 // are only available for integer.
5284 if (TLI->getBooleanContents(isVec: VT.isVector(), isFloat: false) ==
5285 TargetLowering::ZeroOrNegativeOneBooleanContent)
5286 return VTBits;
5287 break;
5288 case ISD::SETCC:
5289 case ISD::SETCCCARRY:
5290 case ISD::STRICT_FSETCC:
5291 case ISD::STRICT_FSETCCS: {
5292 unsigned OpNo = Op->isStrictFPOpcode() ? 1 : 0;
5293 // If setcc returns 0/-1, all bits are sign bits.
5294 if (TLI->getBooleanContents(Type: Op.getOperand(i: OpNo).getValueType()) ==
5295 TargetLowering::ZeroOrNegativeOneBooleanContent)
5296 return VTBits;
5297 break;
5298 }
5299 case ISD::GET_ACTIVE_LANE_MASK:
5300 // Semantically similar to icmp ult.
5301 if (TLI->getBooleanContents(isVec: VT.isVector(), /*isFloat=*/false) ==
5302 TargetLowering::ZeroOrNegativeOneBooleanContent)
5303 return VTBits;
5304 break;
5305 case ISD::ROTL:
5306 case ISD::ROTR: {
5307 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5308 ConstantSDNode *C = isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts);
5309 FirstAnswer = SignBitsOps::rot(
5310 SrcSignBits: Tmp, BitWidth: VTBits, RotAmt: C ? std::optional(C->getAPIntValue()) : std::nullopt,
5311 IsRotateRight: Opcode == ISD::ROTR);
5312 break;
5313 }
5314 case ISD::ADD:
5315 case ISD::ADDC:
5316 // TODO: Move Operand 1 check before Operand 0 check
5317 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5318 if (Tmp == 1) return 1; // Early out.
5319
5320 // Special case decrementing a value (ADD X, -1):
5321 if (ConstantSDNode *CRHS =
5322 isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts))
5323 if (CRHS->isAllOnes()) {
5324 KnownBits Known =
5325 computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5326
5327 // If the input is known to be 0 or 1, the output is 0/-1, which is all
5328 // sign bits set.
5329 if ((Known.Zero | 1).isAllOnes())
5330 return VTBits;
5331
5332 // If we are subtracting one from a positive number, there is no carry
5333 // out of the result.
5334 if (Known.isNonNegative())
5335 return Tmp;
5336 }
5337
5338 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5339 if (Tmp2 == 1) return 1; // Early out.
5340
5341 // Add can have at most one carry bit. Thus we know that the output
5342 // is, at worst, one more bit than the inputs.
5343 return std::min(a: Tmp, b: Tmp2) - 1;
5344 case ISD::SUB:
5345 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5346 if (Tmp2 == 1) return 1; // Early out.
5347
5348 // Handle NEG.
5349 if (ConstantSDNode *CLHS =
5350 isConstOrConstSplat(N: Op.getOperand(i: 0), DemandedElts))
5351 if (CLHS->isZero()) {
5352 KnownBits Known =
5353 computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5354 // If the input is known to be 0 or 1, the output is 0/-1, which is all
5355 // sign bits set.
5356 if ((Known.Zero | 1).isAllOnes())
5357 return VTBits;
5358
5359 // If the input is known to be positive (the sign bit is known clear),
5360 // the output of the NEG has the same number of sign bits as the input.
5361 if (Known.isNonNegative())
5362 return Tmp2;
5363
5364 // Otherwise, we treat this like a SUB.
5365 }
5366
5367 // Sub can have at most one carry bit. Thus we know that the output
5368 // is, at worst, one more bit than the inputs.
5369 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5370 if (Tmp == 1) return 1; // Early out.
5371 return std::min(a: Tmp, b: Tmp2) - 1;
5372 case ISD::MUL: {
5373 // The output of the Mul can be at most twice the valid bits in the inputs.
5374 unsigned SignBitsOp0 =
5375 ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5376 if (SignBitsOp0 == 1)
5377 break;
5378 unsigned SignBitsOp1 =
5379 ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5380 if (SignBitsOp1 == 1)
5381 break;
5382 unsigned OutValidBits =
5383 (VTBits - SignBitsOp0 + 1) + (VTBits - SignBitsOp1 + 1);
5384 return OutValidBits > VTBits ? 1 : VTBits - OutValidBits + 1;
5385 }
5386 case ISD::AVGCEILS:
5387 case ISD::AVGFLOORS:
5388 case ISD::AVGSUBFLOORS:
5389 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5390 if (Tmp == 1)
5391 return 1; // Early out.
5392 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5393 return std::min(a: Tmp, b: Tmp2);
5394 case ISD::SREM:
5395 // The sign bit is the LHS's sign bit, except when the result of the
5396 // remainder is zero. The magnitude of the result should be less than or
5397 // equal to the magnitude of the LHS. Therefore, the result should have
5398 // at least as many sign bits as the left hand side.
5399 return ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5400 case ISD::TRUNCATE: {
5401 // Check if the sign bits of source go down as far as the truncated value.
5402 unsigned NumSrcBits = Op.getOperand(i: 0).getScalarValueSizeInBits();
5403 unsigned NumSrcSignBits =
5404 ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5405 if (NumSrcSignBits > (NumSrcBits - VTBits))
5406 return NumSrcSignBits - (NumSrcBits - VTBits);
5407 break;
5408 }
5409 case ISD::EXTRACT_ELEMENT: {
5410 if (VT.isScalableVector())
5411 break;
5412 const int KnownSign = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth+1);
5413 const int BitWidth = Op.getValueSizeInBits();
5414 const int Items = Op.getOperand(i: 0).getValueSizeInBits() / BitWidth;
5415
5416 // Get reverse index (starting from 1), Op1 value indexes elements from
5417 // little end. Sign starts at big end.
5418 const int rIndex = Items - 1 - Op.getConstantOperandVal(i: 1);
5419
5420 // If the sign portion ends in our element the subtraction gives correct
5421 // result. Otherwise it gives either negative or > bitwidth result
5422 return std::clamp(val: KnownSign - rIndex * BitWidth, lo: 1, hi: BitWidth);
5423 }
5424 case ISD::INSERT_VECTOR_ELT: {
5425 if (VT.isScalableVector())
5426 break;
5427 // If we know the element index, split the demand between the
5428 // source vector and the inserted element, otherwise assume we need
5429 // the original demanded vector elements and the value.
5430 SDValue InVec = Op.getOperand(i: 0);
5431 SDValue InVal = Op.getOperand(i: 1);
5432 SDValue EltNo = Op.getOperand(i: 2);
5433 bool DemandedVal = true;
5434 APInt DemandedVecElts = DemandedElts;
5435 auto *CEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
5436 if (CEltNo && CEltNo->getAPIntValue().ult(RHS: NumElts)) {
5437 unsigned EltIdx = CEltNo->getZExtValue();
5438 DemandedVal = !!DemandedElts[EltIdx];
5439 DemandedVecElts.clearBit(BitPosition: EltIdx);
5440 }
5441 Tmp = std::numeric_limits<unsigned>::max();
5442 if (DemandedVal) {
5443 // TODO - handle implicit truncation of inserted elements.
5444 if (InVal.getScalarValueSizeInBits() != VTBits)
5445 break;
5446 Tmp2 = ComputeNumSignBits(Op: InVal, Depth: Depth + 1);
5447 Tmp = std::min(a: Tmp, b: Tmp2);
5448 }
5449 if (!!DemandedVecElts) {
5450 Tmp2 = ComputeNumSignBits(Op: InVec, DemandedElts: DemandedVecElts, Depth: Depth + 1);
5451 Tmp = std::min(a: Tmp, b: Tmp2);
5452 }
5453 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
5454 return Tmp;
5455 }
5456 case ISD::EXTRACT_VECTOR_ELT: {
5457 SDValue InVec = Op.getOperand(i: 0);
5458 SDValue EltNo = Op.getOperand(i: 1);
5459 EVT VecVT = InVec.getValueType();
5460 // ComputeNumSignBits not yet implemented for scalable vectors.
5461 if (VecVT.isScalableVector())
5462 break;
5463 const unsigned BitWidth = Op.getValueSizeInBits();
5464 const unsigned EltBitWidth = Op.getOperand(i: 0).getScalarValueSizeInBits();
5465 const unsigned NumSrcElts = VecVT.getVectorNumElements();
5466
5467 // If BitWidth > EltBitWidth the value is anyext:ed, and we do not know
5468 // anything about sign bits. But if the sizes match we can derive knowledge
5469 // about sign bits from the vector operand.
5470 if (BitWidth != EltBitWidth)
5471 break;
5472
5473 // If we know the element index, just demand that vector element, else for
5474 // an unknown element index, ignore DemandedElts and demand them all.
5475 APInt DemandedSrcElts = APInt::getAllOnes(numBits: NumSrcElts);
5476 auto *ConstEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
5477 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(RHS: NumSrcElts))
5478 DemandedSrcElts =
5479 APInt::getOneBitSet(numBits: NumSrcElts, BitNo: ConstEltNo->getZExtValue());
5480
5481 return ComputeNumSignBits(Op: InVec, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
5482 }
5483 case ISD::EXTRACT_SUBVECTOR: {
5484 // Offset the demanded elts by the subvector index.
5485 SDValue Src = Op.getOperand(i: 0);
5486
5487 APInt DemandedSrcElts;
5488 if (Src.getValueType().isScalableVector())
5489 DemandedSrcElts = APInt(1, 1);
5490 else {
5491 uint64_t Idx = Op.getConstantOperandVal(i: 1);
5492 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
5493 DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
5494 }
5495 return ComputeNumSignBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
5496 }
5497 case ISD::CONCAT_VECTORS: {
5498 if (VT.isScalableVector())
5499 break;
5500 // Determine the minimum number of sign bits across all demanded
5501 // elts of the input vectors. Early out if the result is already 1.
5502 Tmp = std::numeric_limits<unsigned>::max();
5503 EVT SubVectorVT = Op.getOperand(i: 0).getValueType();
5504 unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements();
5505 unsigned NumSubVectors = Op.getNumOperands();
5506 for (unsigned i = 0; (i < NumSubVectors) && (Tmp > 1); ++i) {
5507 APInt DemandedSub =
5508 DemandedElts.extractBits(numBits: NumSubVectorElts, bitPosition: i * NumSubVectorElts);
5509 if (!DemandedSub)
5510 continue;
5511 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i), DemandedElts: DemandedSub, Depth: Depth + 1);
5512 Tmp = std::min(a: Tmp, b: Tmp2);
5513 }
5514 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
5515 return Tmp;
5516 }
5517 case ISD::INSERT_SUBVECTOR: {
5518 SDValue Src = Op.getOperand(i: 0);
5519 SDValue Sub = Op.getOperand(i: 1);
5520 if (VT.isScalableVector()) {
5521 Tmp = ComputeNumSignBits(Op: Sub, Depth: Depth + 1);
5522 Tmp = std::min(a: Tmp, b: ComputeNumSignBits(Op: Src, Depth: Depth + 1));
5523 return Tmp;
5524 }
5525 // Demand any elements from the subvector and the remainder from the src its
5526 // inserted into.
5527 uint64_t Idx = Op.getConstantOperandVal(i: 2);
5528 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
5529 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
5530 APInt DemandedSrcElts = DemandedElts;
5531 DemandedSrcElts.clearBits(LoBit: Idx, HiBit: Idx + NumSubElts);
5532
5533 Tmp = std::numeric_limits<unsigned>::max();
5534 if (!!DemandedSubElts) {
5535 Tmp = ComputeNumSignBits(Op: Sub, DemandedElts: DemandedSubElts, Depth: Depth + 1);
5536 if (Tmp == 1)
5537 return 1; // early-out
5538 }
5539 if (!!DemandedSrcElts) {
5540 Tmp2 = ComputeNumSignBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
5541 Tmp = std::min(a: Tmp, b: Tmp2);
5542 }
5543 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
5544 return Tmp;
5545 }
5546 case ISD::LOAD: {
5547 // If we are looking at the loaded value of the SDNode.
5548 if (Op.getResNo() != 0)
5549 break;
5550
5551 LoadSDNode *LD = cast<LoadSDNode>(Val&: Op);
5552 if (const MDNode *Ranges = LD->getRanges()) {
5553 if (DemandedElts != 1)
5554 break;
5555
5556 ConstantRange CR = getConstantRangeFromMetadata(RangeMD: *Ranges);
5557 if (VTBits > CR.getBitWidth()) {
5558 switch (LD->getExtensionType()) {
5559 case ISD::SEXTLOAD:
5560 CR = CR.signExtend(BitWidth: VTBits);
5561 break;
5562 case ISD::ZEXTLOAD:
5563 CR = CR.zeroExtend(BitWidth: VTBits);
5564 break;
5565 default:
5566 break;
5567 }
5568 }
5569
5570 if (VTBits != CR.getBitWidth())
5571 break;
5572 return std::min(a: CR.getSignedMin().getNumSignBits(),
5573 b: CR.getSignedMax().getNumSignBits());
5574 }
5575
5576 unsigned ExtType = LD->getExtensionType();
5577 switch (ExtType) {
5578 default:
5579 break;
5580 case ISD::SEXTLOAD: // e.g. i16->i32 = '17' bits known.
5581 Tmp = LD->getMemoryVT().getScalarSizeInBits();
5582 return VTBits - Tmp + 1;
5583 case ISD::ZEXTLOAD: // e.g. i16->i32 = '16' bits known.
5584 Tmp = LD->getMemoryVT().getScalarSizeInBits();
5585 return VTBits - Tmp;
5586 case ISD::NON_EXTLOAD:
5587 if (const Constant *Cst = TLI->getTargetConstantFromLoad(LD)) {
5588 // We only need to handle vectors - computeKnownBits should handle
5589 // scalar cases.
5590 Type *CstTy = Cst->getType();
5591 if (CstTy->isVectorTy() && !VT.isScalableVector() &&
5592 (NumElts * VTBits) == CstTy->getPrimitiveSizeInBits() &&
5593 VTBits == CstTy->getScalarSizeInBits()) {
5594 Tmp = VTBits;
5595 for (unsigned i = 0; i != NumElts; ++i) {
5596 if (!DemandedElts[i])
5597 continue;
5598 if (Constant *Elt = Cst->getAggregateElement(Elt: i)) {
5599 if (auto *CInt = dyn_cast<ConstantInt>(Val: Elt)) {
5600 const APInt &Value = CInt->getValue();
5601 Tmp = std::min(a: Tmp, b: Value.getNumSignBits());
5602 continue;
5603 }
5604 if (auto *CFP = dyn_cast<ConstantFP>(Val: Elt)) {
5605 APInt Value = CFP->getValueAPF().bitcastToAPInt();
5606 Tmp = std::min(a: Tmp, b: Value.getNumSignBits());
5607 continue;
5608 }
5609 }
5610 // Unknown type. Conservatively assume no bits match sign bit.
5611 return 1;
5612 }
5613 return Tmp;
5614 }
5615 }
5616 break;
5617 }
5618
5619 break;
5620 }
5621 case ISD::ATOMIC_CMP_SWAP:
5622 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
5623 case ISD::ATOMIC_SWAP:
5624 case ISD::ATOMIC_LOAD_ADD:
5625 case ISD::ATOMIC_LOAD_SUB:
5626 case ISD::ATOMIC_LOAD_AND:
5627 case ISD::ATOMIC_LOAD_CLR:
5628 case ISD::ATOMIC_LOAD_OR:
5629 case ISD::ATOMIC_LOAD_XOR:
5630 case ISD::ATOMIC_LOAD_NAND:
5631 case ISD::ATOMIC_LOAD_MIN:
5632 case ISD::ATOMIC_LOAD_MAX:
5633 case ISD::ATOMIC_LOAD_UMIN:
5634 case ISD::ATOMIC_LOAD_UMAX:
5635 case ISD::ATOMIC_LOAD: {
5636 auto *AT = cast<AtomicSDNode>(Val&: Op);
5637 // If we are looking at the loaded value.
5638 if (Op.getResNo() == 0) {
5639 Tmp = AT->getMemoryVT().getScalarSizeInBits();
5640 if (Tmp == VTBits)
5641 return 1; // early-out
5642
5643 // For atomic_load, prefer to use the extension type.
5644 if (Op->getOpcode() == ISD::ATOMIC_LOAD) {
5645 switch (AT->getExtensionType()) {
5646 default:
5647 break;
5648 case ISD::SEXTLOAD:
5649 return VTBits - Tmp + 1;
5650 case ISD::ZEXTLOAD:
5651 return VTBits - Tmp;
5652 }
5653 }
5654
5655 if (TLI->getExtendForAtomicOps() == ISD::SIGN_EXTEND)
5656 return VTBits - Tmp + 1;
5657 if (TLI->getExtendForAtomicOps() == ISD::ZERO_EXTEND)
5658 return VTBits - Tmp;
5659 }
5660 break;
5661 }
5662 }
5663
5664 // Allow the target to implement this method for its nodes.
5665 if (Opcode >= ISD::BUILTIN_OP_END ||
5666 Opcode == ISD::INTRINSIC_WO_CHAIN ||
5667 Opcode == ISD::INTRINSIC_W_CHAIN ||
5668 Opcode == ISD::INTRINSIC_VOID) {
5669 // TODO: This can probably be removed once target code is audited. This
5670 // is here purely to reduce patch size and review complexity.
5671 if (!VT.isScalableVector()) {
5672 unsigned NumBits =
5673 TLI->ComputeNumSignBitsForTargetNode(Op, DemandedElts, DAG: *this, Depth);
5674 if (NumBits > 1)
5675 FirstAnswer = std::max(a: FirstAnswer, b: NumBits);
5676 }
5677 }
5678
5679 // Finally, if we can prove that the top bits of the result are 0's or 1's,
5680 // use this information.
5681 KnownBits Known = computeKnownBits(Op, DemandedElts, Depth);
5682 return std::max(a: FirstAnswer, b: Known.countMinSignBits());
5683}
5684
5685unsigned SelectionDAG::ComputeMaxSignificantBits(SDValue Op,
5686 unsigned Depth) const {
5687 unsigned SignBits = ComputeNumSignBits(Op, Depth);
5688 return Op.getScalarValueSizeInBits() - SignBits + 1;
5689}
5690
5691unsigned SelectionDAG::ComputeMaxSignificantBits(SDValue Op,
5692 const APInt &DemandedElts,
5693 unsigned Depth) const {
5694 unsigned SignBits = ComputeNumSignBits(Op, DemandedElts, Depth);
5695 return Op.getScalarValueSizeInBits() - SignBits + 1;
5696}
5697
5698bool SelectionDAG::isGuaranteedNotToBeUndefOrPoison(SDValue Op,
5699 UndefPoisonKind Kind,
5700 unsigned Depth) const {
5701 // Early out for FREEZE.
5702 if (Op.getOpcode() == ISD::FREEZE)
5703 return true;
5704
5705 APInt DemandedElts = getDemandAllEltsMask(V: Op);
5706 return isGuaranteedNotToBeUndefOrPoison(Op, DemandedElts, Kind, Depth);
5707}
5708
5709bool SelectionDAG::isGuaranteedNotToBeUndefOrPoison(SDValue Op,
5710 const APInt &DemandedElts,
5711 UndefPoisonKind Kind,
5712 unsigned Depth) const {
5713 unsigned Opcode = Op.getOpcode();
5714
5715 // Early out for FREEZE.
5716 if (Opcode == ISD::FREEZE)
5717 return true;
5718
5719 if (Depth >= MaxRecursionDepth)
5720 return false; // Limit search depth.
5721
5722 if (isIntOrFPConstant(V: Op))
5723 return true;
5724
5725 switch (Opcode) {
5726 case ISD::CONDCODE:
5727 case ISD::VALUETYPE:
5728 case ISD::FrameIndex:
5729 case ISD::TargetFrameIndex:
5730 case ISD::CopyFromReg:
5731 return true;
5732
5733 case ISD::POISON:
5734 return !includesPoison(Kind);
5735
5736 case ISD::UNDEF:
5737 return !includesUndef(Kind);
5738
5739 case ISD::BITCAST: {
5740 SDValue Src = Op.getOperand(i: 0);
5741 EVT SrcVT = Src.getValueType();
5742 EVT DstVT = Op.getValueType();
5743
5744 if (!SrcVT.isVector() || !DstVT.isVector())
5745 return isGuaranteedNotToBeUndefOrPoison(Op: Src, Kind, Depth: Depth + 1);
5746
5747 unsigned SrcEltBits = SrcVT.getScalarSizeInBits();
5748 unsigned DstEltBits = DstVT.getScalarSizeInBits();
5749 ElementCount NumSrcElts = SrcVT.getVectorElementCount();
5750 [[maybe_unused]] ElementCount NumDstElts = DstVT.getVectorElementCount();
5751
5752 if (SrcEltBits == DstEltBits)
5753 return isGuaranteedNotToBeUndefOrPoison(Op: Src, DemandedElts, Kind,
5754 Depth: Depth + 1);
5755
5756 if (SrcEltBits < DstEltBits) {
5757 if (DstEltBits % SrcEltBits != 0)
5758 return isGuaranteedNotToBeUndefOrPoison(Op: Src, Kind, Depth: Depth + 1);
5759
5760 assert(NumSrcElts == NumDstElts * (DstEltBits / SrcEltBits) &&
5761 "Unexpected vector bitcast");
5762 APInt DemandedSrcElts =
5763 APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumSrcElts.getKnownMinValue());
5764 return isGuaranteedNotToBeUndefOrPoison(Op: Src, DemandedElts: DemandedSrcElts, Kind,
5765 Depth: Depth + 1);
5766 }
5767
5768 if (SrcEltBits % DstEltBits != 0)
5769 return isGuaranteedNotToBeUndefOrPoison(Op: Src, Kind, Depth: Depth + 1);
5770
5771 assert(NumDstElts == NumSrcElts * (SrcEltBits / DstEltBits) &&
5772 "Unexpected vector bitcast");
5773 APInt DemandedSrcElts =
5774 APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumSrcElts.getKnownMinValue());
5775 return isGuaranteedNotToBeUndefOrPoison(Op: Src, DemandedElts: DemandedSrcElts, Kind,
5776 Depth: Depth + 1);
5777 }
5778
5779 case ISD::BUILD_VECTOR:
5780 // NOTE: BUILD_VECTOR has implicit truncation of wider scalar elements -
5781 // this shouldn't affect the result.
5782 for (unsigned i = 0, e = Op.getNumOperands(); i < e; ++i) {
5783 if (!DemandedElts[i])
5784 continue;
5785 if (!isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i), Kind, Depth: Depth + 1))
5786 return false;
5787 }
5788 return true;
5789
5790 case ISD::CONCAT_VECTORS: {
5791 EVT VT = Op.getValueType();
5792 if (!VT.isFixedLengthVector())
5793 break;
5794
5795 EVT SubVT = Op.getOperand(i: 0).getValueType();
5796 unsigned NumSubElts = SubVT.getVectorNumElements();
5797 for (unsigned I = 0, E = Op.getNumOperands(); I != E; ++I) {
5798 APInt DemandedSubElts =
5799 DemandedElts.extractBits(numBits: NumSubElts, bitPosition: I * NumSubElts);
5800 if (!!DemandedSubElts &&
5801 !isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: I), DemandedElts: DemandedSubElts,
5802 Kind, Depth: Depth + 1))
5803 return false;
5804 }
5805 return true;
5806 }
5807
5808 case ISD::EXTRACT_SUBVECTOR: {
5809 SDValue Src = Op.getOperand(i: 0);
5810 if (Src.getValueType().isScalableVector())
5811 break;
5812 uint64_t Idx = Op.getConstantOperandVal(i: 1);
5813 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
5814 APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
5815 return isGuaranteedNotToBeUndefOrPoison(Op: Src, DemandedElts: DemandedSrcElts, Kind,
5816 Depth: Depth + 1);
5817 }
5818
5819 case ISD::INSERT_SUBVECTOR: {
5820 if (Op.getValueType().isScalableVector())
5821 break;
5822 SDValue Src = Op.getOperand(i: 0);
5823 SDValue Sub = Op.getOperand(i: 1);
5824 uint64_t Idx = Op.getConstantOperandVal(i: 2);
5825 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
5826 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
5827 APInt DemandedSrcElts = DemandedElts;
5828 DemandedSrcElts.clearBits(LoBit: Idx, HiBit: Idx + NumSubElts);
5829
5830 if (!!DemandedSubElts && !isGuaranteedNotToBeUndefOrPoison(
5831 Op: Sub, DemandedElts: DemandedSubElts, Kind, Depth: Depth + 1))
5832 return false;
5833 if (!!DemandedSrcElts && !isGuaranteedNotToBeUndefOrPoison(
5834 Op: Src, DemandedElts: DemandedSrcElts, Kind, Depth: Depth + 1))
5835 return false;
5836 return true;
5837 }
5838
5839 case ISD::EXTRACT_VECTOR_ELT: {
5840 SDValue Src = Op.getOperand(i: 0);
5841 auto *IndexC = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1));
5842 EVT SrcVT = Src.getValueType();
5843 if (SrcVT.isFixedLengthVector() && IndexC &&
5844 IndexC->getAPIntValue().ult(RHS: SrcVT.getVectorNumElements())) {
5845 APInt DemandedSrcElts = APInt::getOneBitSet(numBits: SrcVT.getVectorNumElements(),
5846 BitNo: IndexC->getZExtValue());
5847 return isGuaranteedNotToBeUndefOrPoison(Op: Src, DemandedElts: DemandedSrcElts, Kind,
5848 Depth: Depth + 1);
5849 }
5850 break;
5851 }
5852
5853 case ISD::INSERT_VECTOR_ELT: {
5854 SDValue InVec = Op.getOperand(i: 0);
5855 SDValue InVal = Op.getOperand(i: 1);
5856 SDValue EltNo = Op.getOperand(i: 2);
5857 EVT VT = InVec.getValueType();
5858 auto *IndexC = dyn_cast<ConstantSDNode>(Val&: EltNo);
5859 if (IndexC && VT.isFixedLengthVector() &&
5860 IndexC->getAPIntValue().ult(RHS: VT.getVectorNumElements())) {
5861 if (DemandedElts[IndexC->getZExtValue()] &&
5862 !isGuaranteedNotToBeUndefOrPoison(Op: InVal, Kind, Depth: Depth + 1))
5863 return false;
5864 APInt InVecDemandedElts = DemandedElts;
5865 InVecDemandedElts.clearBit(BitPosition: IndexC->getZExtValue());
5866 if (!!InVecDemandedElts &&
5867 !isGuaranteedNotToBeUndefOrPoison(
5868 Op: peekThroughInsertVectorElt(V: InVec, DemandedElts: InVecDemandedElts),
5869 DemandedElts: InVecDemandedElts, Kind, Depth: Depth + 1))
5870 return false;
5871 return true;
5872 }
5873 break;
5874 }
5875
5876 case ISD::SCALAR_TO_VECTOR:
5877 // Check upper (known poison) elements.
5878 if (DemandedElts.ugt(RHS: 1) && includesPoison(Kind))
5879 return false;
5880 // Check element zero.
5881 if (DemandedElts[0] &&
5882 !isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 0), Kind, Depth: Depth + 1))
5883 return false;
5884 return true;
5885
5886 case ISD::SPLAT_VECTOR:
5887 return isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 0), Kind, Depth: Depth + 1);
5888
5889 case ISD::SELECT: {
5890 return !canCreateUndefOrPoison(Op, DemandedElts, Kind,
5891 /*ConsiderFlags*/ true, Depth) &&
5892 isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 0), Kind,
5893 Depth: Depth + 1) &&
5894 isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 1), DemandedElts,
5895 Kind, Depth: Depth + 1) &&
5896 isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 2), DemandedElts,
5897 Kind, Depth: Depth + 1);
5898 }
5899
5900 case ISD::VECTOR_SHUFFLE: {
5901 APInt DemandedLHS, DemandedRHS;
5902 auto *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
5903 if (!getShuffleDemandedElts(SrcWidth: DemandedElts.getBitWidth(), Mask: SVN->getMask(),
5904 DemandedElts, DemandedLHS, DemandedRHS,
5905 /*AllowUndefElts=*/false))
5906 return false;
5907 if (!DemandedLHS.isZero() &&
5908 !isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 0), DemandedElts: DemandedLHS, Kind,
5909 Depth: Depth + 1))
5910 return false;
5911 if (!DemandedRHS.isZero() &&
5912 !isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 1), DemandedElts: DemandedRHS, Kind,
5913 Depth: Depth + 1))
5914 return false;
5915 return true;
5916 }
5917
5918 case ISD::SHL:
5919 case ISD::SRL:
5920 case ISD::SRA:
5921 // Shift amount operand is checked by canCreateUndefOrPoison. So it is
5922 // enough to check operand 0 if Op can't create undef/poison.
5923 return !canCreateUndefOrPoison(Op, DemandedElts, Kind,
5924 /*ConsiderFlags*/ true, Depth) &&
5925 isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 0), DemandedElts,
5926 Kind, Depth: Depth + 1);
5927
5928 case ISD::BSWAP:
5929 case ISD::CTPOP:
5930 case ISD::BITREVERSE:
5931 case ISD::AND:
5932 case ISD::OR:
5933 case ISD::XOR:
5934 case ISD::ADD:
5935 case ISD::SUB:
5936 case ISD::MUL:
5937 case ISD::SADDSAT:
5938 case ISD::UADDSAT:
5939 case ISD::SSUBSAT:
5940 case ISD::USUBSAT:
5941 case ISD::SSHLSAT:
5942 case ISD::USHLSAT:
5943 case ISD::SMIN:
5944 case ISD::SMAX:
5945 case ISD::UMIN:
5946 case ISD::UMAX:
5947 case ISD::ZERO_EXTEND:
5948 case ISD::SIGN_EXTEND:
5949 case ISD::ANY_EXTEND:
5950 case ISD::TRUNCATE:
5951 case ISD::VSELECT: {
5952 // If Op can't create undef/poison and none of its operands are undef/poison
5953 // then Op is never undef/poison. A difference from the more common check
5954 // below, outside the switch, is that we handle elementwise operations for
5955 // which the DemandedElts mask is valid for all operands here.
5956 return !canCreateUndefOrPoison(Op, DemandedElts, Kind,
5957 /*ConsiderFlags*/ true, Depth) &&
5958 all_of(Range: Op->ops(), P: [&](SDValue V) {
5959 return isGuaranteedNotToBeUndefOrPoison(Op: V, DemandedElts, Kind,
5960 Depth: Depth + 1);
5961 });
5962 }
5963
5964 // TODO: Search for noundef attributes from library functions.
5965
5966 // TODO: Pointers dereferenced by ISD::LOAD/STORE ops are noundef.
5967
5968 default:
5969 // Allow the target to implement this method for its nodes.
5970 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
5971 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)
5972 return TLI->isGuaranteedNotToBeUndefOrPoisonForTargetNode(
5973 Op, DemandedElts, DAG: *this, Kind, Depth);
5974 break;
5975 }
5976
5977 // If Op can't create undef/poison and none of its operands are undef/poison
5978 // then Op is never undef/poison.
5979 // NOTE: TargetNodes can handle this in themselves in
5980 // isGuaranteedNotToBeUndefOrPoisonForTargetNode or let
5981 // TargetLowering::isGuaranteedNotToBeUndefOrPoisonForTargetNode handle it.
5982 return !canCreateUndefOrPoison(Op, Kind, /*ConsiderFlags*/ true, Depth) &&
5983 all_of(Range: Op->ops(), P: [&](SDValue V) {
5984 return isGuaranteedNotToBeUndefOrPoison(Op: V, Kind, Depth: Depth + 1);
5985 });
5986}
5987
5988bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, UndefPoisonKind Kind,
5989 bool ConsiderFlags,
5990 unsigned Depth) const {
5991 APInt DemandedElts = getDemandAllEltsMask(V: Op);
5992 return canCreateUndefOrPoison(Op, DemandedElts, Kind, ConsiderFlags, Depth);
5993}
5994
5995bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, const APInt &DemandedElts,
5996 UndefPoisonKind Kind,
5997 bool ConsiderFlags,
5998 unsigned Depth) const {
5999 if (ConsiderFlags && includesPoison(Kind) && Op->hasPoisonGeneratingFlags())
6000 return true;
6001
6002 unsigned Opcode = Op.getOpcode();
6003 switch (Opcode) {
6004 case ISD::AssertSext:
6005 case ISD::AssertZext:
6006 case ISD::AssertAlign:
6007 case ISD::AssertNoFPClass:
6008 // Assertion nodes can create poison if the assertion fails.
6009 return includesPoison(Kind);
6010
6011 case ISD::FREEZE:
6012 case ISD::CONCAT_VECTORS:
6013 case ISD::INSERT_SUBVECTOR:
6014 case ISD::EXTRACT_SUBVECTOR:
6015 case ISD::SADDSAT:
6016 case ISD::UADDSAT:
6017 case ISD::SSUBSAT:
6018 case ISD::USUBSAT:
6019 case ISD::MULHU:
6020 case ISD::MULHS:
6021 case ISD::AVGFLOORS:
6022 case ISD::AVGFLOORU:
6023 case ISD::AVGCEILS:
6024 case ISD::AVGCEILU:
6025 case ISD::AVGSUBFLOORS:
6026 case ISD::AVGSUBFLOORU:
6027 case ISD::ABDU:
6028 case ISD::ABDS:
6029 case ISD::SMIN:
6030 case ISD::SMAX:
6031 case ISD::SCMP:
6032 case ISD::UMIN:
6033 case ISD::UMAX:
6034 case ISD::UCMP:
6035 case ISD::AND:
6036 case ISD::XOR:
6037 case ISD::ROTL:
6038 case ISD::ROTR:
6039 case ISD::FSHL:
6040 case ISD::FSHR:
6041 case ISD::BSWAP:
6042 case ISD::CTTZ:
6043 case ISD::CTLZ:
6044 case ISD::CTLS:
6045 case ISD::CTPOP:
6046 case ISD::BITREVERSE:
6047 case ISD::PARITY:
6048 case ISD::SIGN_EXTEND:
6049 case ISD::TRUNCATE:
6050 case ISD::SIGN_EXTEND_INREG:
6051 case ISD::SIGN_EXTEND_VECTOR_INREG:
6052 case ISD::ZERO_EXTEND_VECTOR_INREG:
6053 case ISD::BITCAST:
6054 case ISD::BUILD_VECTOR:
6055 case ISD::BUILD_PAIR:
6056 case ISD::SPLAT_VECTOR:
6057 case ISD::FABS:
6058 case ISD::FCEIL:
6059 case ISD::FFLOOR:
6060 case ISD::FTRUNC:
6061 case ISD::FRINT:
6062 case ISD::FNEARBYINT:
6063 case ISD::FROUND:
6064 case ISD::FROUNDEVEN:
6065 return false;
6066
6067 case ISD::ABS:
6068 // ISD::ABS defines abs(INT_MIN) -> INT_MIN and never generates poison.
6069 // Different to Intrinsic::abs.
6070 return false;
6071 case ISD::ABS_MIN_POISON:
6072 // ABS_MIN_POISON may produce poison if the input is INT_MIN.
6073 return ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1) <= 1;
6074
6075 case ISD::ADDC:
6076 case ISD::SUBC:
6077 case ISD::ADDE:
6078 case ISD::SUBE:
6079 case ISD::SADDO:
6080 case ISD::SSUBO:
6081 case ISD::SMULO:
6082 case ISD::SADDO_CARRY:
6083 case ISD::SSUBO_CARRY:
6084 case ISD::UADDO:
6085 case ISD::USUBO:
6086 case ISD::UMULO:
6087 case ISD::UADDO_CARRY:
6088 case ISD::USUBO_CARRY:
6089 // No poison on result or overflow flags.
6090 return false;
6091
6092 case ISD::SELECT_CC:
6093 case ISD::SETCC: {
6094 // Integer setcc cannot create undef or poison.
6095 if (Op.getOperand(i: 0).getValueType().isInteger())
6096 return false;
6097
6098 // FP compares are more complicated. They can create poison for nan/infinity
6099 // based on options and flags. The options and flags also cause special
6100 // nonan condition codes to be used. Those condition codes may be preserved
6101 // even if the nonan flag is dropped somewhere.
6102 unsigned CCOp = Opcode == ISD::SETCC ? 2 : 4;
6103 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: Op.getOperand(i: CCOp))->get();
6104 return (unsigned)CCCode & 0x10U;
6105 }
6106
6107 case ISD::OR:
6108 case ISD::ZERO_EXTEND:
6109 case ISD::SELECT:
6110 case ISD::VSELECT:
6111 case ISD::ADD:
6112 case ISD::SUB:
6113 case ISD::MUL:
6114 case ISD::FNEG:
6115 case ISD::FADD:
6116 case ISD::FSUB:
6117 case ISD::FMUL:
6118 case ISD::FDIV:
6119 case ISD::FREM:
6120 case ISD::FCOPYSIGN:
6121 case ISD::FMA:
6122 case ISD::FMAD:
6123 case ISD::FMULADD:
6124 case ISD::FP_EXTEND:
6125 case ISD::FMINNUM:
6126 case ISD::FMAXNUM:
6127 case ISD::FMINNUM_IEEE:
6128 case ISD::FMAXNUM_IEEE:
6129 case ISD::FMINIMUM:
6130 case ISD::FMAXIMUM:
6131 case ISD::FMINIMUMNUM:
6132 case ISD::FMAXIMUMNUM:
6133 case ISD::FP_TO_SINT_SAT:
6134 case ISD::FP_TO_UINT_SAT:
6135 case ISD::TRUNCATE_SSAT_S:
6136 case ISD::TRUNCATE_SSAT_U:
6137 case ISD::TRUNCATE_USAT_U:
6138 // No poison except from flags (which is handled above)
6139 return false;
6140
6141 case ISD::SHL:
6142 case ISD::SRL:
6143 case ISD::SRA:
6144 // If the max shift amount isn't in range, then the shift can
6145 // create poison.
6146 return includesPoison(Kind) &&
6147 !getValidMaximumShiftAmount(V: Op, DemandedElts, Depth: Depth + 1);
6148
6149 case ISD::CTTZ_ZERO_POISON:
6150 case ISD::CTLZ_ZERO_POISON:
6151 // If the amount is zero then the result will be poison.
6152 // TODO: Add isKnownNeverZero DemandedElts handling.
6153 return includesPoison(Kind) &&
6154 !isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6155
6156 case ISD::SCALAR_TO_VECTOR:
6157 // Check if we demand any upper (poison) elements.
6158 return includesPoison(Kind) && DemandedElts.ugt(RHS: 1);
6159
6160 case ISD::INSERT_VECTOR_ELT:
6161 case ISD::EXTRACT_VECTOR_ELT: {
6162 // Ensure that the element index is in bounds.
6163 if (includesPoison(Kind)) {
6164 EVT VecVT = Op.getOperand(i: 0).getValueType();
6165 SDValue Idx = Op.getOperand(i: Opcode == ISD::INSERT_VECTOR_ELT ? 2 : 1);
6166 KnownBits KnownIdx = computeKnownBits(Op: Idx, Depth: Depth + 1);
6167 return KnownIdx.getMaxValue().uge(RHS: VecVT.getVectorMinNumElements());
6168 }
6169 return false;
6170 }
6171
6172 case ISD::VECTOR_SHUFFLE: {
6173 // Check for any demanded shuffle element that is undef.
6174 auto *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
6175 for (auto [Idx, Elt] : enumerate(First: SVN->getMask()))
6176 if (Elt < 0 && DemandedElts[Idx])
6177 return true;
6178 return false;
6179 }
6180
6181 case ISD::VECTOR_COMPRESS:
6182 return false;
6183
6184 default:
6185 // Allow the target to implement this method for its nodes.
6186 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
6187 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)
6188 return TLI->canCreateUndefOrPoisonForTargetNode(
6189 Op, DemandedElts, DAG: *this, Kind, ConsiderFlags, Depth);
6190 break;
6191 }
6192
6193 // Be conservative and return true.
6194 return true;
6195}
6196
6197bool SelectionDAG::isADDLike(SDValue Op, bool NoWrap) const {
6198 unsigned Opcode = Op.getOpcode();
6199 if (Opcode == ISD::OR)
6200 return Op->getFlags().hasDisjoint() ||
6201 haveNoCommonBitsSet(A: Op.getOperand(i: 0), B: Op.getOperand(i: 1));
6202 if (Opcode == ISD::XOR)
6203 return !NoWrap && isMinSignedConstant(V: Op.getOperand(i: 1));
6204 return false;
6205}
6206
6207bool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const {
6208 return Op.getNumOperands() == 2 && isa<ConstantSDNode>(Val: Op.getOperand(i: 1)) &&
6209 (Op.isAnyAdd() || isADDLike(Op));
6210}
6211
6212KnownFPClass SelectionDAG::computeKnownFPClass(SDValue Op,
6213 FPClassTest InterestedClasses,
6214 unsigned Depth) const {
6215 APInt DemandedElts = getDemandAllEltsMask(V: Op);
6216 return computeKnownFPClass(Op, DemandedElts, InterestedClasses, Depth);
6217}
6218
6219KnownFPClass SelectionDAG::computeKnownFPClass(SDValue Op,
6220 const APInt &DemandedElts,
6221 FPClassTest InterestedClasses,
6222 unsigned Depth) const {
6223 KnownFPClass Known;
6224
6225 if (const auto *CFP = dyn_cast<ConstantFPSDNode>(Val&: Op))
6226 return KnownFPClass(CFP->getValueAPF());
6227
6228 if (Depth >= MaxRecursionDepth)
6229 return Known;
6230
6231 if (Op.getOpcode() == ISD::UNDEF)
6232 return Known;
6233
6234 EVT VT = Op.getValueType();
6235 assert(VT.isFloatingPoint() && "Computing KnownFPClass on non-FP op!");
6236 assert((!VT.isFixedLengthVector() ||
6237 DemandedElts.getBitWidth() == VT.getVectorNumElements()) &&
6238 "Unexpected vector size");
6239
6240 if (!DemandedElts)
6241 return Known;
6242
6243 unsigned Opcode = Op.getOpcode();
6244 switch (Opcode) {
6245 case ISD::POISON: {
6246 Known.setKnownFPClasses(fcNone);
6247 Known.setSignBit(false);
6248 break;
6249 }
6250 case ISD::FNEG: {
6251 Known = computeKnownFPClass(Op: Op.getOperand(i: 0), DemandedElts,
6252 InterestedClasses, Depth: Depth + 1);
6253 Known.fneg();
6254 break;
6255 }
6256 case ISD::BUILD_VECTOR: {
6257 assert(!VT.isScalableVector());
6258 bool First = true;
6259 for (unsigned I = 0, E = Op.getNumOperands(); I != E; ++I) {
6260 if (!DemandedElts[I])
6261 continue;
6262
6263 if (First) {
6264 Known =
6265 computeKnownFPClass(Op: Op.getOperand(i: I), InterestedClasses, Depth: Depth + 1);
6266 First = false;
6267 } else {
6268 Known |=
6269 computeKnownFPClass(Op: Op.getOperand(i: I), InterestedClasses, Depth: Depth + 1);
6270 }
6271
6272 if (Known.isUnknown())
6273 break;
6274 }
6275 break;
6276 }
6277 case ISD::EXTRACT_VECTOR_ELT: {
6278 SDValue Src = Op.getOperand(i: 0);
6279 auto *CIdx = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1));
6280 EVT SrcVT = Src.getValueType();
6281 if (SrcVT.isFixedLengthVector() && CIdx) {
6282 if (CIdx->getAPIntValue().ult(RHS: SrcVT.getVectorNumElements())) {
6283 APInt DemandedSrcElts = APInt::getOneBitSet(
6284 numBits: SrcVT.getVectorNumElements(), BitNo: CIdx->getZExtValue());
6285 Known = computeKnownFPClass(Op: Src, DemandedElts: DemandedSrcElts, InterestedClasses,
6286 Depth: Depth + 1);
6287 } else {
6288 // Out of bounds index is poison.
6289 Known.setKnownFPClasses(fcNone);
6290 }
6291 } else {
6292 Known = computeKnownFPClass(Op: Src, InterestedClasses, Depth: Depth + 1);
6293 }
6294 break;
6295 }
6296 case ISD::SPLAT_VECTOR: {
6297 Known = computeKnownFPClass(Op: Op.getOperand(i: 0), InterestedClasses, Depth: Depth + 1);
6298 break;
6299 }
6300 case ISD::BITCAST: {
6301 // FIXME: It should not be necessary to check for an elementwise bitcast.
6302 // If a bitcast is not elementwise between vector / scalar types,
6303 // computeKnownBits already splices the known bits of the source elements
6304 // appropriately so as to line up with the bits of the result's demanded
6305 // elements.
6306 EVT SrcVT = Op.getOperand(i: 0).getValueType();
6307 if (VT.isScalableVector() || SrcVT.isScalableVector())
6308 break;
6309 unsigned VTNumElts = VT.isVector() ? VT.getVectorNumElements() : 1;
6310 unsigned SrcVTNumElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
6311 if (VTNumElts != SrcVTNumElts)
6312 break;
6313
6314 KnownBits Bits = computeKnownBits(Op, DemandedElts, Depth: Depth + 1);
6315 Known = KnownFPClass::bitcast(FltSemantics: VT.getFltSemantics(), Bits);
6316 break;
6317 }
6318 case ISD::FABS: {
6319 Known = computeKnownFPClass(Op: Op.getOperand(i: 0), DemandedElts,
6320 InterestedClasses, Depth: Depth + 1);
6321 Known.fabs();
6322 break;
6323 }
6324 case ISD::FCOPYSIGN: {
6325 Known = computeKnownFPClass(Op: Op.getOperand(i: 0), DemandedElts,
6326 InterestedClasses, Depth: Depth + 1);
6327 KnownFPClass KnownSign = computeKnownFPClass(Op: Op.getOperand(i: 1), DemandedElts,
6328 InterestedClasses, Depth: Depth + 1);
6329 Known.copysign(Sign: KnownSign);
6330 break;
6331 }
6332 case ISD::AssertNoFPClass: {
6333 Known = computeKnownFPClass(Op: Op.getOperand(i: 0), DemandedElts,
6334 InterestedClasses, Depth: Depth + 1);
6335 FPClassTest AssertedClasses =
6336 static_cast<FPClassTest>(Op->getConstantOperandVal(Num: 1));
6337 Known.setKnownFPClasses(Known.getKnownFPClasses() & ~AssertedClasses);
6338 break;
6339 }
6340 case ISD::EXTRACT_SUBVECTOR: {
6341 SDValue Src = Op.getOperand(i: 0);
6342 EVT SrcVT = Src.getValueType();
6343 if (SrcVT.isFixedLengthVector()) {
6344 unsigned Idx = Op.getConstantOperandVal(i: 1);
6345 unsigned NumSrcElts = SrcVT.getVectorNumElements();
6346
6347 APInt DemandedSrcElts = DemandedElts.zextOrTrunc(width: NumSrcElts).shl(shiftAmt: Idx);
6348 Known = computeKnownFPClass(Op: Src, DemandedElts: DemandedSrcElts, InterestedClasses,
6349 Depth: Depth + 1);
6350 } else {
6351 Known = computeKnownFPClass(Op: Src, InterestedClasses, Depth: Depth + 1);
6352 }
6353 break;
6354 }
6355 case ISD::INSERT_SUBVECTOR: {
6356 SDValue BaseVector = Op.getOperand(i: 0);
6357 SDValue SubVector = Op.getOperand(i: 1);
6358 EVT BaseVT = BaseVector.getValueType();
6359 if (BaseVT.isFixedLengthVector()) {
6360 unsigned Idx = Op.getConstantOperandVal(i: 2);
6361 unsigned NumBaseElts = BaseVT.getVectorNumElements();
6362 unsigned NumSubElts = SubVector.getValueType().getVectorNumElements();
6363
6364 APInt DemandedMask =
6365 APInt::getBitsSet(numBits: NumBaseElts, loBit: Idx, hiBit: Idx + NumSubElts);
6366 APInt DemandedSrcElts = DemandedElts & ~DemandedMask;
6367 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
6368
6369 if (!DemandedSrcElts.isZero())
6370 Known = computeKnownFPClass(Op: BaseVector, DemandedElts: DemandedSrcElts,
6371 InterestedClasses, Depth: Depth + 1);
6372 if (!DemandedSubElts.isZero()) {
6373 KnownFPClass SubKnown = computeKnownFPClass(
6374 Op: SubVector, DemandedElts: DemandedSubElts, InterestedClasses, Depth: Depth + 1);
6375 Known = DemandedSrcElts.isZero() ? SubKnown : (Known | SubKnown);
6376 }
6377 } else {
6378 Known = computeKnownFPClass(Op: SubVector, InterestedClasses, Depth: Depth + 1);
6379 if (!Known.isUnknown())
6380 Known |= computeKnownFPClass(Op: BaseVector, InterestedClasses, Depth: Depth + 1);
6381 }
6382 break;
6383 }
6384 case ISD::SELECT:
6385 case ISD::VSELECT: {
6386 // TODO: Add adjustKnownFPClassForSelectArm clamp recognition as in
6387 // IR-level ValueTracking.
6388 KnownFPClass KnownFalseClass = computeKnownFPClass(
6389 Op: Op.getOperand(i: 2), DemandedElts, InterestedClasses, Depth: Depth + 1);
6390 if (KnownFalseClass.isUnknown())
6391 break;
6392 KnownFPClass KnownTrueClass = computeKnownFPClass(
6393 Op: Op.getOperand(i: 1), DemandedElts, InterestedClasses, Depth: Depth + 1);
6394 Known = KnownTrueClass.intersectWith(RHS: KnownFalseClass);
6395 break;
6396 }
6397 default:
6398 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
6399 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID) {
6400 TLI->computeKnownFPClassForTargetNode(Op, Known, DemandedElts, DAG: *this,
6401 Depth);
6402 }
6403 break;
6404 }
6405
6406 return Known;
6407}
6408
6409bool SelectionDAG::isKnownNeverNaN(SDValue Op, bool SNaN,
6410 unsigned Depth) const {
6411 APInt DemandedElts = getDemandAllEltsMask(V: Op);
6412 return isKnownNeverNaN(Op, DemandedElts, SNaN, Depth);
6413}
6414
6415bool SelectionDAG::isKnownNeverNaN(SDValue Op, const APInt &DemandedElts,
6416 bool SNaN, unsigned Depth) const {
6417 assert(!DemandedElts.isZero() && "No demanded elements");
6418
6419 // If we're told that NaNs won't happen, assume they won't.
6420 if (Op->getFlags().hasNoNaNs())
6421 return true;
6422
6423 if (Depth >= MaxRecursionDepth)
6424 return false; // Limit search depth.
6425
6426 unsigned Opcode = Op.getOpcode();
6427 switch (Opcode) {
6428 case ISD::FADD:
6429 case ISD::FSUB:
6430 case ISD::FMUL:
6431 case ISD::FDIV:
6432 case ISD::FREM:
6433 case ISD::FSIN:
6434 case ISD::FCOS:
6435 case ISD::FTAN:
6436 case ISD::FASIN:
6437 case ISD::FACOS:
6438 case ISD::FATAN:
6439 case ISD::FATAN2:
6440 case ISD::FSINH:
6441 case ISD::FCOSH:
6442 case ISD::FTANH:
6443 case ISD::FMA:
6444 case ISD::FMULADD:
6445 case ISD::FMAD: {
6446 if (SNaN)
6447 return true;
6448 // TODO: Need isKnownNeverInfinity
6449 return false;
6450 }
6451 case ISD::FCANONICALIZE:
6452 case ISD::FEXP:
6453 case ISD::FEXP2:
6454 case ISD::FEXP10:
6455 case ISD::FTRUNC:
6456 case ISD::FFLOOR:
6457 case ISD::FCEIL:
6458 case ISD::FROUND:
6459 case ISD::FROUNDEVEN:
6460 case ISD::LROUND:
6461 case ISD::LLROUND:
6462 case ISD::FRINT:
6463 case ISD::LRINT:
6464 case ISD::LLRINT:
6465 case ISD::FNEARBYINT:
6466 case ISD::FLDEXP: {
6467 if (SNaN)
6468 return true;
6469 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1);
6470 }
6471 case ISD::FABS:
6472 case ISD::FNEG:
6473 case ISD::FCOPYSIGN: {
6474 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1);
6475 }
6476 case ISD::SELECT:
6477 return isKnownNeverNaN(Op: Op.getOperand(i: 1), DemandedElts, SNaN, Depth: Depth + 1) &&
6478 isKnownNeverNaN(Op: Op.getOperand(i: 2), DemandedElts, SNaN, Depth: Depth + 1);
6479 case ISD::FP_EXTEND:
6480 case ISD::FP_ROUND: {
6481 if (SNaN)
6482 return true;
6483 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1);
6484 }
6485 case ISD::SINT_TO_FP:
6486 case ISD::UINT_TO_FP:
6487 return true;
6488 case ISD::FSQRT: // Need is known positive
6489 case ISD::FLOG:
6490 case ISD::FLOG2:
6491 case ISD::FLOG10:
6492 case ISD::FPOWI:
6493 case ISD::FPOW: {
6494 if (SNaN)
6495 return true;
6496 // TODO: Refine on operand
6497 return false;
6498 }
6499 case ISD::FMINNUM:
6500 case ISD::FMAXNUM:
6501 case ISD::FMINIMUMNUM:
6502 case ISD::FMAXIMUMNUM: {
6503 // Only one needs to be known not-nan, since it will be returned if the
6504 // other ends up being one.
6505 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1) ||
6506 isKnownNeverNaN(Op: Op.getOperand(i: 1), DemandedElts, SNaN, Depth: Depth + 1);
6507 }
6508 case ISD::FMINNUM_IEEE:
6509 case ISD::FMAXNUM_IEEE: {
6510 if (SNaN)
6511 return true;
6512 // This can return a NaN if either operand is an sNaN, or if both operands
6513 // are NaN.
6514 return (isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN: false, Depth: Depth + 1) &&
6515 isKnownNeverSNaN(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1)) ||
6516 (isKnownNeverNaN(Op: Op.getOperand(i: 1), DemandedElts, SNaN: false, Depth: Depth + 1) &&
6517 isKnownNeverSNaN(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1));
6518 }
6519 case ISD::FMINIMUM:
6520 case ISD::FMAXIMUM: {
6521 // TODO: Does this quiet or return the origina NaN as-is?
6522 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1) &&
6523 isKnownNeverNaN(Op: Op.getOperand(i: 1), DemandedElts, SNaN, Depth: Depth + 1);
6524 }
6525 case ISD::EXTRACT_VECTOR_ELT: {
6526 SDValue Src = Op.getOperand(i: 0);
6527 auto *Idx = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1));
6528 EVT SrcVT = Src.getValueType();
6529 if (SrcVT.isFixedLengthVector() && Idx &&
6530 Idx->getAPIntValue().ult(RHS: SrcVT.getVectorNumElements())) {
6531 APInt DemandedSrcElts = APInt::getOneBitSet(numBits: SrcVT.getVectorNumElements(),
6532 BitNo: Idx->getZExtValue());
6533 return isKnownNeverNaN(Op: Src, DemandedElts: DemandedSrcElts, SNaN, Depth: Depth + 1);
6534 }
6535 return isKnownNeverNaN(Op: Src, SNaN, Depth: Depth + 1);
6536 }
6537 case ISD::EXTRACT_SUBVECTOR: {
6538 SDValue Src = Op.getOperand(i: 0);
6539 if (Src.getValueType().isFixedLengthVector()) {
6540 unsigned Idx = Op.getConstantOperandVal(i: 1);
6541 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
6542 APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
6543 return isKnownNeverNaN(Op: Src, DemandedElts: DemandedSrcElts, SNaN, Depth: Depth + 1);
6544 }
6545 return isKnownNeverNaN(Op: Src, SNaN, Depth: Depth + 1);
6546 }
6547 case ISD::INSERT_SUBVECTOR: {
6548 SDValue BaseVector = Op.getOperand(i: 0);
6549 SDValue SubVector = Op.getOperand(i: 1);
6550 EVT BaseVectorVT = BaseVector.getValueType();
6551 if (BaseVectorVT.isFixedLengthVector()) {
6552 unsigned Idx = Op.getConstantOperandVal(i: 2);
6553 unsigned NumBaseElts = BaseVectorVT.getVectorNumElements();
6554 unsigned NumSubElts = SubVector.getValueType().getVectorNumElements();
6555
6556 // Clear/Extract the bits at the position where the subvector will be
6557 // inserted.
6558 APInt DemandedMask =
6559 APInt::getBitsSet(numBits: NumBaseElts, loBit: Idx, hiBit: Idx + NumSubElts);
6560 APInt DemandedSrcElts = DemandedElts & ~DemandedMask;
6561 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
6562
6563 bool NeverNaN = true;
6564 if (!DemandedSrcElts.isZero())
6565 NeverNaN &=
6566 isKnownNeverNaN(Op: BaseVector, DemandedElts: DemandedSrcElts, SNaN, Depth: Depth + 1);
6567 if (NeverNaN && !DemandedSubElts.isZero())
6568 NeverNaN &=
6569 isKnownNeverNaN(Op: SubVector, DemandedElts: DemandedSubElts, SNaN, Depth: Depth + 1);
6570 return NeverNaN;
6571 }
6572 return isKnownNeverNaN(Op: BaseVector, SNaN, Depth: Depth + 1) &&
6573 isKnownNeverNaN(Op: SubVector, SNaN, Depth: Depth + 1);
6574 }
6575 case ISD::BUILD_VECTOR: {
6576 unsigned NumElts = Op.getNumOperands();
6577 for (unsigned I = 0; I != NumElts; ++I)
6578 if (DemandedElts[I] &&
6579 !isKnownNeverNaN(Op: Op.getOperand(i: I), SNaN, Depth: Depth + 1))
6580 return false;
6581 return true;
6582 }
6583 case ISD::SPLAT_VECTOR:
6584 return isKnownNeverNaN(Op: Op.getOperand(i: 0), SNaN, Depth: Depth + 1);
6585 case ISD::AssertNoFPClass: {
6586 FPClassTest NoFPClass =
6587 static_cast<FPClassTest>(Op.getConstantOperandVal(i: 1));
6588 if ((NoFPClass & fcNan) == fcNan)
6589 return true;
6590 if (SNaN && (NoFPClass & fcSNan) == fcSNan)
6591 return true;
6592 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1);
6593 }
6594 default:
6595 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
6596 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID) {
6597 return TLI->isKnownNeverNaNForTargetNode(Op, DemandedElts, DAG: *this, SNaN,
6598 Depth);
6599 }
6600 break;
6601 }
6602
6603 FPClassTest NanMask = SNaN ? fcSNan : fcNan;
6604 KnownFPClass Known = computeKnownFPClass(Op, DemandedElts, InterestedClasses: NanMask, Depth);
6605 return Known.isKnownNever(Mask: NanMask);
6606}
6607
6608bool SelectionDAG::isKnownNeverLogicalZero(SDValue Op, unsigned Depth) const {
6609 APInt DemandedElts = getDemandAllEltsMask(V: Op);
6610 return isKnownNeverLogicalZero(Op, DemandedElts, Depth);
6611}
6612
6613bool SelectionDAG::isKnownNeverLogicalZero(SDValue Op,
6614 const APInt &DemandedElts,
6615 unsigned Depth) const {
6616 assert(!DemandedElts.isZero() && "No demanded elements");
6617 EVT VT = Op.getValueType();
6618 KnownFPClass Known =
6619 computeKnownFPClass(Op, DemandedElts, InterestedClasses: fcZero | fcSubnormal, Depth);
6620 return Known.isKnownNeverLogicalZero(Mode: getDenormalMode(VT));
6621}
6622
6623bool SelectionDAG::isKnownNeverZero(SDValue Op, unsigned Depth) const {
6624 APInt DemandedElts = getDemandAllEltsMask(V: Op);
6625 return isKnownNeverZero(Op, DemandedElts, Depth);
6626}
6627
6628bool SelectionDAG::isKnownNeverZero(SDValue Op, const APInt &DemandedElts,
6629 unsigned Depth) const {
6630 if (Depth >= MaxRecursionDepth)
6631 return false; // Limit search depth.
6632
6633 EVT OpVT = Op.getValueType();
6634 unsigned BitWidth = OpVT.getScalarSizeInBits();
6635
6636 assert(!Op.getValueType().isFloatingPoint() &&
6637 "Floating point types unsupported - use isKnownNeverLogicalZero");
6638
6639 // If the value is a constant, we can obviously see if it is a zero or not.
6640 auto IsNeverZero = [BitWidth](const ConstantSDNode *C) {
6641 APInt V = C->getAPIntValue().zextOrTrunc(width: BitWidth);
6642 return !V.isZero();
6643 };
6644
6645 if (ISD::matchUnaryPredicate(Op, DemandedElts, Match: IsNeverZero,
6646 /*AllowUndefs=*/false, /*AllowTruncation=*/true))
6647 return true;
6648
6649 // TODO: Recognize more cases here. Most of the cases are also incomplete to
6650 // some degree.
6651 switch (Op.getOpcode()) {
6652 default:
6653 break;
6654
6655 case ISD::EXTRACT_VECTOR_ELT: {
6656 SDValue InVec = Op.getOperand(i: 0);
6657 SDValue EltNo = Op.getOperand(i: 1);
6658 EVT VecVT = InVec.getValueType();
6659
6660 // Skip scalable vectors or implicit extensions.
6661 if (VecVT.isScalableVector() ||
6662 OpVT.getScalarSizeInBits() != VecVT.getScalarSizeInBits())
6663 break;
6664
6665 // If we know the element index, just demand that vector element, else for
6666 // an unknown element index, ignore DemandedElts and demand them all.
6667 const unsigned NumSrcElts = VecVT.getVectorNumElements();
6668 APInt DemandedSrcElts = APInt::getAllOnes(numBits: NumSrcElts);
6669 auto *ConstEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
6670 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(RHS: NumSrcElts))
6671 DemandedSrcElts =
6672 APInt::getOneBitSet(numBits: NumSrcElts, BitNo: ConstEltNo->getZExtValue());
6673
6674 return isKnownNeverZero(Op: InVec, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
6675 }
6676
6677 case ISD::OR:
6678 return isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1) ||
6679 isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6680
6681 case ISD::VSELECT:
6682 case ISD::SELECT:
6683 return isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1) &&
6684 isKnownNeverZero(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth + 1);
6685
6686 case ISD::SHL: {
6687 if (Op->getFlags().hasNoSignedWrap() || Op->getFlags().hasNoUnsignedWrap())
6688 return isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6689 KnownBits ValKnown =
6690 computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6691 // 1 << X is never zero.
6692 if (ValKnown.One[0])
6693 return true;
6694 // If max shift cnt of known ones is non-zero, result is non-zero.
6695 APInt MaxCnt = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1)
6696 .getMaxValue();
6697 if (MaxCnt.ult(RHS: ValKnown.getBitWidth()) &&
6698 !ValKnown.One.shl(ShiftAmt: MaxCnt).isZero())
6699 return true;
6700 break;
6701 }
6702
6703 case ISD::VECTOR_SHUFFLE: {
6704 if (Op.getValueType().isScalableVector())
6705 return false;
6706
6707 unsigned NumElts = DemandedElts.getBitWidth();
6708
6709 // All demanded elements from LHS and RHS must be known non-zero.
6710 // Demanded elements with undef shuffle mask elements are unknown.
6711
6712 APInt DemandedLHS, DemandedRHS;
6713 auto *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
6714 assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
6715 if (!getShuffleDemandedElts(SrcWidth: NumElts, Mask: SVN->getMask(), DemandedElts,
6716 DemandedLHS, DemandedRHS))
6717 return false;
6718
6719 return (!DemandedLHS ||
6720 isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts: DemandedLHS, Depth: Depth + 1)) &&
6721 (!DemandedRHS ||
6722 isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts: DemandedRHS, Depth: Depth + 1));
6723 }
6724
6725 case ISD::UADDSAT:
6726 case ISD::UMAX:
6727 return isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1) ||
6728 isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6729
6730 case ISD::UMIN:
6731 return isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1) &&
6732 isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6733
6734 // For smin/smax: If either operand is known negative/positive
6735 // respectively we don't need the other to be known at all.
6736 case ISD::SMAX: {
6737 KnownBits Op1 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
6738 if (Op1.isStrictlyPositive())
6739 return true;
6740
6741 KnownBits Op0 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6742 if (Op0.isStrictlyPositive())
6743 return true;
6744
6745 if (Op1.isNonZero() && Op0.isNonZero())
6746 return true;
6747
6748 return isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1) &&
6749 isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6750 }
6751 case ISD::SMIN: {
6752 KnownBits Op1 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
6753 if (Op1.isNegative())
6754 return true;
6755
6756 KnownBits Op0 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6757 if (Op0.isNegative())
6758 return true;
6759
6760 if (Op1.isNonZero() && Op0.isNonZero())
6761 return true;
6762
6763 return isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1) &&
6764 isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6765 }
6766
6767 case ISD::ROTL:
6768 case ISD::ROTR:
6769 case ISD::BITREVERSE:
6770 case ISD::BSWAP:
6771 case ISD::CTPOP:
6772 case ISD::ABS:
6773 case ISD::ABS_MIN_POISON:
6774 return isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6775
6776 case ISD::SRA:
6777 case ISD::SRL: {
6778 if (Op->getFlags().hasExact())
6779 return isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6780 KnownBits ValKnown =
6781 computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6782 if (ValKnown.isNegative())
6783 return true;
6784 // If max shift cnt of known ones is non-zero, result is non-zero.
6785 APInt MaxCnt = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1)
6786 .getMaxValue();
6787 if (MaxCnt.ult(RHS: ValKnown.getBitWidth()) &&
6788 !ValKnown.One.lshr(ShiftAmt: MaxCnt).isZero())
6789 return true;
6790 break;
6791 }
6792 case ISD::UDIV:
6793 case ISD::SDIV:
6794 // div exact can only produce a zero if the dividend is zero.
6795 // TODO: For udiv this is also true if Op1 u<= Op0
6796 if (Op->getFlags().hasExact())
6797 return isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6798 break;
6799
6800 case ISD::ADD:
6801 if (Op->getFlags().hasNoUnsignedWrap())
6802 if (isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1) ||
6803 isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1))
6804 return true;
6805 // TODO: There are a lot more cases we can prove for add.
6806 break;
6807
6808 case ISD::SUB: {
6809 if (isNullConstant(V: Op.getOperand(i: 0)))
6810 return isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
6811
6812 std::optional<bool> ne = KnownBits::ne(
6813 LHS: computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1),
6814 RHS: computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1));
6815 return ne && *ne;
6816 }
6817
6818 case ISD::MUL:
6819 if (Op->getFlags().hasNoSignedWrap() || Op->getFlags().hasNoUnsignedWrap())
6820 if (isKnownNeverZero(Op: Op.getOperand(i: 1), Depth: Depth + 1) &&
6821 isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1))
6822 return true;
6823 break;
6824
6825 case ISD::ZERO_EXTEND:
6826 case ISD::SIGN_EXTEND:
6827 return isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6828 case ISD::VSCALE: {
6829 const Function &F = getMachineFunction().getFunction();
6830 const APInt &Multiplier = Op.getConstantOperandAPInt(i: 0);
6831 ConstantRange CR =
6832 getVScaleRange(F: &F, BitWidth: Op.getScalarValueSizeInBits()).multiply(Other: Multiplier);
6833 if (!CR.contains(Val: APInt(CR.getBitWidth(), 0)))
6834 return true;
6835 break;
6836 }
6837 }
6838
6839 return computeKnownBits(Op, DemandedElts, Depth).isNonZero();
6840}
6841
6842bool SelectionDAG::cannotBeOrderedNegativeFP(SDValue Op) const {
6843 if (ConstantFPSDNode *C1 = isConstOrConstSplatFP(N: Op, AllowUndefs: true))
6844 return !C1->isNegative();
6845
6846 switch (Op.getOpcode()) {
6847 case ISD::FABS:
6848 case ISD::FEXP:
6849 case ISD::FEXP2:
6850 case ISD::FEXP10:
6851 return true;
6852 default:
6853 return false;
6854 }
6855
6856 llvm_unreachable("covered opcode switch");
6857}
6858
6859bool SelectionDAG::canIgnoreSignBitOfZero(const SDUse &Use) const {
6860 assert(Use.getValueType().isFloatingPoint());
6861 const SDNode *User = Use.getUser();
6862 if (User->getFlags().hasNoSignedZeros())
6863 return true;
6864
6865 unsigned OperandNo = Use.getOperandNo();
6866 // Check if this use is insensitive to the sign of zero
6867 switch (User->getOpcode()) {
6868 case ISD::SETCC:
6869 // Comparisons: IEEE-754 specifies +0.0 == -0.0.
6870 case ISD::FABS:
6871 // fabs always produces +0.0.
6872 return true;
6873 case ISD::FCOPYSIGN:
6874 // copysign overwrites the sign bit of the first operand.
6875 return OperandNo == 0;
6876 case ISD::FADD:
6877 case ISD::FSUB: {
6878 // Arithmetic with non-zero constants fixes the uncertainty around the
6879 // sign bit.
6880 SDValue Other = User->getOperand(Num: 1 - OperandNo);
6881 return isKnownNeverLogicalZero(Op: Other);
6882 }
6883 case ISD::FP_TO_SINT:
6884 case ISD::FP_TO_UINT:
6885 // fp-to-int conversions normalize signed zeros.
6886 return true;
6887 default:
6888 return false;
6889 }
6890}
6891
6892bool SelectionDAG::canIgnoreSignBitOfZero(SDValue Op) const {
6893 if (Op->getFlags().hasNoSignedZeros())
6894 return true;
6895 // FIXME: Limit the amount of checked uses to not introduce a compile-time
6896 // regression. Ideally, this should be implemented as a demanded-bits
6897 // optimization that stems from the users.
6898 if (Op->use_size() > 2)
6899 return false;
6900 return all_of(Range: Op->uses(),
6901 P: [&](const SDUse &Use) { return canIgnoreSignBitOfZero(Use); });
6902}
6903
6904bool SelectionDAG::isEqualTo(SDValue A, SDValue B) const {
6905 // Check the obvious case.
6906 if (A == B) return true;
6907
6908 // For negative and positive zero.
6909 if (const ConstantFPSDNode *CA = dyn_cast<ConstantFPSDNode>(Val&: A))
6910 if (const ConstantFPSDNode *CB = dyn_cast<ConstantFPSDNode>(Val&: B))
6911 if (CA->isZero() && CB->isZero()) return true;
6912
6913 // Otherwise they may not be equal.
6914 return false;
6915}
6916
6917// Only bits set in Mask must be negated, other bits may be arbitrary.
6918SDValue llvm::getBitwiseNotOperand(SDValue V, SDValue Mask, bool AllowUndefs) {
6919 if (isBitwiseNot(V, AllowUndefs))
6920 return V.getOperand(i: 0);
6921
6922 // Handle any_extend (not (truncate X)) pattern, where Mask only sets
6923 // bits in the non-extended part.
6924 ConstantSDNode *MaskC = isConstOrConstSplat(N: Mask);
6925 if (!MaskC || V.getOpcode() != ISD::ANY_EXTEND)
6926 return SDValue();
6927 SDValue ExtArg = V.getOperand(i: 0);
6928 if (ExtArg.getScalarValueSizeInBits() >=
6929 MaskC->getAPIntValue().getActiveBits() &&
6930 isBitwiseNot(V: ExtArg, AllowUndefs) &&
6931 ExtArg.getOperand(i: 0).getOpcode() == ISD::TRUNCATE &&
6932 ExtArg.getOperand(i: 0).getOperand(i: 0).getValueType() == V.getValueType())
6933 return ExtArg.getOperand(i: 0).getOperand(i: 0);
6934 return SDValue();
6935}
6936
6937static bool haveNoCommonBitsSetCommutative(SDValue A, SDValue B) {
6938 // Match masked merge pattern (X & ~M) op (Y & M)
6939 // Including degenerate case (X & ~M) op M
6940 auto MatchNoCommonBitsPattern = [&](SDValue Not, SDValue Mask,
6941 SDValue Other) {
6942 if (SDValue NotOperand =
6943 getBitwiseNotOperand(V: Not, Mask, /* AllowUndefs */ true)) {
6944 if (NotOperand->getOpcode() == ISD::ZERO_EXTEND ||
6945 NotOperand->getOpcode() == ISD::TRUNCATE)
6946 NotOperand = NotOperand->getOperand(Num: 0);
6947
6948 if (Other == NotOperand)
6949 return true;
6950 if (Other->getOpcode() == ISD::AND)
6951 return NotOperand == Other->getOperand(Num: 0) ||
6952 NotOperand == Other->getOperand(Num: 1);
6953 }
6954 return false;
6955 };
6956
6957 if (A->getOpcode() == ISD::ZERO_EXTEND || A->getOpcode() == ISD::TRUNCATE)
6958 A = A->getOperand(Num: 0);
6959
6960 if (B->getOpcode() == ISD::ZERO_EXTEND || B->getOpcode() == ISD::TRUNCATE)
6961 B = B->getOperand(Num: 0);
6962
6963 if (A->getOpcode() == ISD::AND)
6964 return MatchNoCommonBitsPattern(A->getOperand(Num: 0), A->getOperand(Num: 1), B) ||
6965 MatchNoCommonBitsPattern(A->getOperand(Num: 1), A->getOperand(Num: 0), B);
6966 return false;
6967}
6968
6969// FIXME: unify with llvm::haveNoCommonBitsSet.
6970bool SelectionDAG::haveNoCommonBitsSet(SDValue A, SDValue B) const {
6971 assert(A.getValueType() == B.getValueType() &&
6972 "Values must have the same type");
6973 if (haveNoCommonBitsSetCommutative(A, B) ||
6974 haveNoCommonBitsSetCommutative(A: B, B: A))
6975 return true;
6976 return KnownBits::haveNoCommonBitsSet(LHS: computeKnownBits(Op: A),
6977 RHS: computeKnownBits(Op: B));
6978}
6979
6980static SDValue FoldSTEP_VECTOR(const SDLoc &DL, EVT VT, SDValue Step,
6981 SelectionDAG &DAG) {
6982 if (cast<ConstantSDNode>(Val&: Step)->isZero())
6983 return DAG.getConstant(Val: 0, DL, VT);
6984
6985 return SDValue();
6986}
6987
6988static SDValue FoldBUILD_VECTOR(const SDLoc &DL, EVT VT,
6989 ArrayRef<SDValue> Ops,
6990 SelectionDAG &DAG) {
6991 int NumOps = Ops.size();
6992 assert(NumOps != 0 && "Can't build an empty vector!");
6993 assert(!VT.isScalableVector() &&
6994 "BUILD_VECTOR cannot be used with scalable types");
6995 assert(VT.getVectorNumElements() == (unsigned)NumOps &&
6996 "Incorrect element count in BUILD_VECTOR!");
6997
6998 // BUILD_VECTOR of UNDEFs is UNDEF.
6999 bool AllPoison = true;
7000 if (llvm::all_of(Range&: Ops, P: [&AllPoison](SDValue Op) {
7001 AllPoison &= Op.getOpcode() == ISD::POISON;
7002 return Op.isUndef();
7003 }))
7004 return AllPoison ? DAG.getPOISON(VT) : DAG.getUNDEF(VT);
7005
7006 // BUILD_VECTOR of seq extract/insert from the same vector + type is Identity.
7007 SDValue IdentitySrc;
7008 bool IsIdentity = true;
7009 for (int i = 0; i != NumOps; ++i) {
7010 if (Ops[i].getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
7011 Ops[i].getOperand(i: 0).getValueType() != VT ||
7012 (IdentitySrc && Ops[i].getOperand(i: 0) != IdentitySrc) ||
7013 !isa<ConstantSDNode>(Val: Ops[i].getOperand(i: 1)) ||
7014 Ops[i].getConstantOperandAPInt(i: 1) != i) {
7015 IsIdentity = false;
7016 break;
7017 }
7018 IdentitySrc = Ops[i].getOperand(i: 0);
7019 }
7020 if (IsIdentity)
7021 return IdentitySrc;
7022
7023 return SDValue();
7024}
7025
7026/// Try to simplify vector concatenation to an input value, undef, or build
7027/// vector.
7028static SDValue foldCONCAT_VECTORS(const SDLoc &DL, EVT VT,
7029 ArrayRef<SDValue> Ops,
7030 SelectionDAG &DAG) {
7031 assert(!Ops.empty() && "Can't concatenate an empty list of vectors!");
7032 assert(llvm::all_of(Ops,
7033 [Ops](SDValue Op) {
7034 return Ops[0].getValueType() == Op.getValueType();
7035 }) &&
7036 "Concatenation of vectors with inconsistent value types!");
7037 assert((Ops[0].getValueType().getVectorElementCount() * Ops.size()) ==
7038 VT.getVectorElementCount() &&
7039 "Incorrect element count in vector concatenation!");
7040
7041 if (Ops.size() == 1)
7042 return Ops[0];
7043
7044 // Concat of UNDEFs is UNDEF.
7045 bool AllPoison = true;
7046 if (llvm::all_of(Range&: Ops, P: [&AllPoison](SDValue Op) {
7047 AllPoison &= Op.getOpcode() == ISD::POISON;
7048 return Op.isUndef();
7049 }))
7050 return AllPoison ? DAG.getPOISON(VT) : DAG.getUNDEF(VT);
7051
7052 // Scan the operands and look for extract operations from a single source
7053 // that correspond to insertion at the same location via this concatenation:
7054 // concat (extract X, 0*subvec_elts), (extract X, 1*subvec_elts), ...
7055 SDValue IdentitySrc;
7056 bool IsIdentity = true;
7057 for (unsigned i = 0, e = Ops.size(); i != e; ++i) {
7058 SDValue Op = Ops[i];
7059 unsigned IdentityIndex = i * Op.getValueType().getVectorMinNumElements();
7060 if (Op.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
7061 Op.getOperand(i: 0).getValueType() != VT ||
7062 (IdentitySrc && Op.getOperand(i: 0) != IdentitySrc) ||
7063 Op.getConstantOperandVal(i: 1) != IdentityIndex) {
7064 IsIdentity = false;
7065 break;
7066 }
7067 assert((!IdentitySrc || IdentitySrc == Op.getOperand(0)) &&
7068 "Unexpected identity source vector for concat of extracts");
7069 IdentitySrc = Op.getOperand(i: 0);
7070 }
7071 if (IsIdentity) {
7072 assert(IdentitySrc && "Failed to set source vector of extracts");
7073 return IdentitySrc;
7074 }
7075
7076 // The code below this point is only designed to work for fixed width
7077 // vectors, so we bail out for now.
7078 if (VT.isScalableVector())
7079 return SDValue();
7080
7081 // A CONCAT_VECTOR of scalar sources, such as UNDEF, BUILD_VECTOR and
7082 // single-element INSERT_VECTOR_ELT operands can be simplified to one big
7083 // BUILD_VECTOR.
7084 // FIXME: Add support for SCALAR_TO_VECTOR as well.
7085 EVT SVT = VT.getScalarType();
7086 SmallVector<SDValue, 16> Elts;
7087 for (SDValue Op : Ops) {
7088 EVT OpVT = Op.getValueType();
7089 if (Op.getOpcode() == ISD::POISON)
7090 Elts.append(NumInputs: OpVT.getVectorNumElements(), Elt: DAG.getPOISON(VT: SVT));
7091 else if (Op.getOpcode() == ISD::UNDEF)
7092 Elts.append(NumInputs: OpVT.getVectorNumElements(), Elt: DAG.getUNDEF(VT: SVT));
7093 else if (Op.getOpcode() == ISD::BUILD_VECTOR)
7094 Elts.append(in_start: Op->op_begin(), in_end: Op->op_end());
7095 else if (Op.getOpcode() == ISD::INSERT_VECTOR_ELT &&
7096 OpVT.getVectorNumElements() == 1 &&
7097 isNullConstant(V: Op.getOperand(i: 2)))
7098 Elts.push_back(Elt: Op.getOperand(i: 1));
7099 else
7100 return SDValue();
7101 }
7102
7103 // BUILD_VECTOR requires all inputs to be of the same type, find the
7104 // maximum type and extend them all.
7105 for (SDValue Op : Elts)
7106 SVT = (SVT.bitsLT(VT: Op.getValueType()) ? Op.getValueType() : SVT);
7107
7108 if (SVT.bitsGT(VT: VT.getScalarType())) {
7109 for (SDValue &Op : Elts) {
7110 if (Op.getOpcode() == ISD::POISON)
7111 Op = DAG.getPOISON(VT: SVT);
7112 else if (Op.getOpcode() == ISD::UNDEF)
7113 Op = DAG.getUNDEF(VT: SVT);
7114 else
7115 Op = DAG.getTargetLoweringInfo().isZExtFree(FromTy: Op.getValueType(), ToTy: SVT)
7116 ? DAG.getZExtOrTrunc(Op, DL, VT: SVT)
7117 : DAG.getSExtOrTrunc(Op, DL, VT: SVT);
7118 }
7119 }
7120
7121 SDValue V = DAG.getBuildVector(VT, DL, Ops: Elts);
7122 NewSDValueDbgMsg(V, Msg: "New node fold concat vectors: ", G: &DAG);
7123 return V;
7124}
7125
7126/// Gets or creates the specified node.
7127SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT) {
7128 SDVTList VTs = getVTList(VT);
7129 SDNodeKey ID(Opcode, VTs, {});
7130 FoldingSetInsertToken InsertToken;
7131 if (SDNode *E = lookupNode(Key: ID, DL, InsertToken))
7132 return SDValue(E, 0);
7133
7134 auto *N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
7135 CSEMap.insert(N, Token: InsertToken);
7136
7137 InsertNode(N);
7138 SDValue V = SDValue(N, 0);
7139 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
7140 return V;
7141}
7142
7143SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7144 SDValue N1) {
7145 SDNodeFlags Flags;
7146 if (Inserter)
7147 Flags = Inserter->getFlags();
7148 return getNode(Opcode, DL, VT, Operand: N1, Flags);
7149}
7150
7151SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7152 SDValue N1, const SDNodeFlags Flags) {
7153 assert(N1.getOpcode() != ISD::DELETED_NODE && "Operand is DELETED_NODE!");
7154
7155 // Constant fold unary operations with a vector integer or float operand.
7156 switch (Opcode) {
7157 default:
7158 // FIXME: Entirely reasonable to perform folding of other unary
7159 // operations here as the need arises.
7160 break;
7161 case ISD::FNEG:
7162 case ISD::FABS:
7163 case ISD::FCEIL:
7164 case ISD::FTRUNC:
7165 case ISD::FFLOOR:
7166 case ISD::FP_EXTEND:
7167 case ISD::FP_TO_SINT:
7168 case ISD::FP_TO_UINT:
7169 case ISD::FP_TO_FP16:
7170 case ISD::FP_TO_BF16:
7171 case ISD::TRUNCATE:
7172 case ISD::ANY_EXTEND:
7173 case ISD::ZERO_EXTEND:
7174 case ISD::SIGN_EXTEND:
7175 case ISD::UINT_TO_FP:
7176 case ISD::SINT_TO_FP:
7177 case ISD::FP16_TO_FP:
7178 case ISD::BF16_TO_FP:
7179 case ISD::BITCAST:
7180 case ISD::ABS:
7181 case ISD::ABS_MIN_POISON:
7182 case ISD::BITREVERSE:
7183 case ISD::BSWAP:
7184 case ISD::CTLZ:
7185 case ISD::CTLZ_ZERO_POISON:
7186 case ISD::CTTZ:
7187 case ISD::CTTZ_ZERO_POISON:
7188 case ISD::CTPOP:
7189 case ISD::PARITY:
7190 case ISD::CTLS:
7191 case ISD::VECREDUCE_ADD:
7192 case ISD::VECREDUCE_SMAX:
7193 case ISD::VECREDUCE_SMIN:
7194 case ISD::VECREDUCE_UMAX:
7195 case ISD::VECREDUCE_UMIN:
7196 case ISD::VECREDUCE_MUL:
7197 case ISD::VECREDUCE_AND:
7198 case ISD::VECREDUCE_OR:
7199 case ISD::VECREDUCE_XOR:
7200 case ISD::STEP_VECTOR: {
7201 SDValue Ops = {N1};
7202 if (SDValue Fold = FoldConstantArithmetic(Opcode, DL, VT, Ops))
7203 return Fold;
7204 }
7205 }
7206
7207 unsigned OpOpcode = N1.getNode()->getOpcode();
7208 switch (Opcode) {
7209 case ISD::STEP_VECTOR:
7210 assert(VT.isScalableVector() &&
7211 "STEP_VECTOR can only be used with scalable types");
7212 assert(OpOpcode == ISD::TargetConstant &&
7213 VT.getVectorElementType() == N1.getValueType() &&
7214 "Unexpected step operand");
7215 break;
7216 case ISD::FREEZE:
7217 assert(VT == N1.getValueType() && "Unexpected VT!");
7218 if (isGuaranteedNotToBeUndefOrPoison(Op: N1, Kind: UndefPoisonKind::UndefOrPoison))
7219 return N1;
7220 break;
7221 case ISD::TokenFactor:
7222 case ISD::MERGE_VALUES:
7223 case ISD::CONCAT_VECTORS:
7224 return N1; // Factor, merge or concat of one node? No need.
7225 case ISD::BUILD_VECTOR: {
7226 // Attempt to simplify BUILD_VECTOR.
7227 SDValue Ops[] = {N1};
7228 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, DAG&: *this))
7229 return V;
7230 break;
7231 }
7232 case ISD::FP_ROUND: llvm_unreachable("Invalid method to make FP_ROUND node");
7233 case ISD::FP_EXTEND:
7234 assert(VT.isFloatingPoint() && N1.getValueType().isFloatingPoint() &&
7235 "Invalid FP cast!");
7236 if (N1.getValueType() == VT) return N1; // noop conversion.
7237 assert((!VT.isVector() || VT.getVectorElementCount() ==
7238 N1.getValueType().getVectorElementCount()) &&
7239 "Vector element count mismatch!");
7240 assert(N1.getValueType().bitsLT(VT) && "Invalid fpext node, dst < src!");
7241 if (N1.isUndef())
7242 return getUNDEF(VT);
7243 break;
7244 case ISD::FP_TO_SINT:
7245 case ISD::FP_TO_UINT:
7246 if (N1.isUndef())
7247 return getUNDEF(VT);
7248 break;
7249 case ISD::SINT_TO_FP:
7250 case ISD::UINT_TO_FP:
7251 // [us]itofp(undef) = 0, because the result value is bounded.
7252 if (N1.isUndef())
7253 return getConstantFP(Val: 0.0, DL, VT);
7254 break;
7255 case ISD::SIGN_EXTEND:
7256 assert(VT.isInteger() && N1.getValueType().isInteger() &&
7257 "Invalid SIGN_EXTEND!");
7258 assert(VT.isVector() == N1.getValueType().isVector() &&
7259 "SIGN_EXTEND result type type should be vector iff the operand "
7260 "type is vector!");
7261 if (N1.getValueType() == VT) return N1; // noop extension
7262 assert((!VT.isVector() || VT.getVectorElementCount() ==
7263 N1.getValueType().getVectorElementCount()) &&
7264 "Vector element count mismatch!");
7265 assert(N1.getValueType().bitsLT(VT) && "Invalid sext node, dst < src!");
7266 if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND) {
7267 SDNodeFlags Flags;
7268 if (OpOpcode == ISD::ZERO_EXTEND)
7269 Flags.setNonNeg(N1->getFlags().hasNonNeg());
7270 SDValue NewVal = getNode(Opcode: OpOpcode, DL, VT, N1: N1.getOperand(i: 0), Flags);
7271 transferDbgValues(From: N1, To: NewVal);
7272 return NewVal;
7273 }
7274
7275 if (OpOpcode == ISD::POISON)
7276 return getPOISON(VT);
7277
7278 if (N1.isUndef())
7279 // sext(undef) = 0, because the top bits will all be the same.
7280 return getConstant(Val: 0, DL, VT);
7281
7282 // Skip unnecessary sext_inreg pattern:
7283 // (sext (trunc x)) -> x iff the upper bits are all signbits.
7284 if (OpOpcode == ISD::TRUNCATE) {
7285 SDValue OpOp = N1.getOperand(i: 0);
7286 if (OpOp.getValueType() == VT) {
7287 unsigned NumSignExtBits =
7288 VT.getScalarSizeInBits() - N1.getScalarValueSizeInBits();
7289 if (ComputeNumSignBits(Op: OpOp) > NumSignExtBits) {
7290 transferDbgValues(From: N1, To: OpOp);
7291 return OpOp;
7292 }
7293 }
7294 }
7295 break;
7296 case ISD::ZERO_EXTEND:
7297 assert(VT.isInteger() && N1.getValueType().isInteger() &&
7298 "Invalid ZERO_EXTEND!");
7299 assert(VT.isVector() == N1.getValueType().isVector() &&
7300 "ZERO_EXTEND result type type should be vector iff the operand "
7301 "type is vector!");
7302 if (N1.getValueType() == VT) return N1; // noop extension
7303 assert((!VT.isVector() || VT.getVectorElementCount() ==
7304 N1.getValueType().getVectorElementCount()) &&
7305 "Vector element count mismatch!");
7306 assert(N1.getValueType().bitsLT(VT) && "Invalid zext node, dst < src!");
7307 if (OpOpcode == ISD::ZERO_EXTEND) { // (zext (zext x)) -> (zext x)
7308 SDNodeFlags Flags;
7309 Flags.setNonNeg(N1->getFlags().hasNonNeg());
7310 SDValue NewVal =
7311 getNode(Opcode: ISD::ZERO_EXTEND, DL, VT, N1: N1.getOperand(i: 0), Flags);
7312 transferDbgValues(From: N1, To: NewVal);
7313 return NewVal;
7314 }
7315
7316 if (OpOpcode == ISD::POISON)
7317 return getPOISON(VT);
7318
7319 if (N1.isUndef())
7320 // zext(undef) = 0, because the top bits will be zero.
7321 return getConstant(Val: 0, DL, VT);
7322
7323 // Skip unnecessary zext_inreg pattern:
7324 // (zext (trunc x)) -> x iff the upper bits are known zero.
7325 // TODO: Remove (zext (trunc (and x, c))) exception which some targets
7326 // use to recognise zext_inreg patterns.
7327 if (OpOpcode == ISD::TRUNCATE) {
7328 SDValue OpOp = N1.getOperand(i: 0);
7329 if (OpOp.getValueType() == VT) {
7330 if (OpOp.getOpcode() != ISD::AND) {
7331 APInt HiBits = APInt::getBitsSetFrom(numBits: VT.getScalarSizeInBits(),
7332 loBit: N1.getScalarValueSizeInBits());
7333 if (MaskedValueIsZero(V: OpOp, Mask: HiBits)) {
7334 transferDbgValues(From: N1, To: OpOp);
7335 return OpOp;
7336 }
7337 }
7338 }
7339 }
7340 break;
7341 case ISD::ANY_EXTEND:
7342 assert(VT.isInteger() && N1.getValueType().isInteger() &&
7343 "Invalid ANY_EXTEND!");
7344 assert(VT.isVector() == N1.getValueType().isVector() &&
7345 "ANY_EXTEND result type type should be vector iff the operand "
7346 "type is vector!");
7347 if (N1.getValueType() == VT) return N1; // noop extension
7348 assert((!VT.isVector() || VT.getVectorElementCount() ==
7349 N1.getValueType().getVectorElementCount()) &&
7350 "Vector element count mismatch!");
7351 assert(N1.getValueType().bitsLT(VT) && "Invalid anyext node, dst < src!");
7352
7353 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
7354 OpOpcode == ISD::ANY_EXTEND) {
7355 SDNodeFlags Flags;
7356 if (OpOpcode == ISD::ZERO_EXTEND)
7357 Flags.setNonNeg(N1->getFlags().hasNonNeg());
7358 // (ext (zext x)) -> (zext x) and (ext (sext x)) -> (sext x)
7359 return getNode(Opcode: OpOpcode, DL, VT, N1: N1.getOperand(i: 0), Flags);
7360 }
7361 if (N1.isUndef())
7362 return getUNDEF(VT);
7363
7364 // (ext (trunc x)) -> x
7365 if (OpOpcode == ISD::TRUNCATE) {
7366 SDValue OpOp = N1.getOperand(i: 0);
7367 if (OpOp.getValueType() == VT) {
7368 transferDbgValues(From: N1, To: OpOp);
7369 return OpOp;
7370 }
7371 }
7372 break;
7373 case ISD::TRUNCATE:
7374 assert(VT.isInteger() && N1.getValueType().isInteger() &&
7375 "Invalid TRUNCATE!");
7376 assert(VT.isVector() == N1.getValueType().isVector() &&
7377 "TRUNCATE result type type should be vector iff the operand "
7378 "type is vector!");
7379 if (N1.getValueType() == VT) return N1; // noop truncate
7380 assert((!VT.isVector() || VT.getVectorElementCount() ==
7381 N1.getValueType().getVectorElementCount()) &&
7382 "Vector element count mismatch!");
7383 assert(N1.getValueType().bitsGT(VT) && "Invalid truncate node, src < dst!");
7384 if (OpOpcode == ISD::TRUNCATE)
7385 return getNode(Opcode: ISD::TRUNCATE, DL, VT, N1: N1.getOperand(i: 0));
7386 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
7387 OpOpcode == ISD::ANY_EXTEND) {
7388 // If the source is smaller than the dest, we still need an extend.
7389 if (N1.getOperand(i: 0).getValueType().getScalarType().bitsLT(
7390 VT: VT.getScalarType())) {
7391 SDNodeFlags Flags;
7392 if (OpOpcode == ISD::ZERO_EXTEND)
7393 Flags.setNonNeg(N1->getFlags().hasNonNeg());
7394 return getNode(Opcode: OpOpcode, DL, VT, N1: N1.getOperand(i: 0), Flags);
7395 }
7396 if (N1.getOperand(i: 0).getValueType().bitsGT(VT))
7397 return getNode(Opcode: ISD::TRUNCATE, DL, VT, N1: N1.getOperand(i: 0));
7398 return N1.getOperand(i: 0);
7399 }
7400 if (N1.isUndef())
7401 return getUNDEF(VT);
7402 if (OpOpcode == ISD::VSCALE && !NewNodesMustHaveLegalTypes)
7403 return getVScale(DL, VT,
7404 MulImm: N1.getConstantOperandAPInt(i: 0).trunc(width: VT.getSizeInBits()));
7405 break;
7406 case ISD::ANY_EXTEND_VECTOR_INREG:
7407 case ISD::ZERO_EXTEND_VECTOR_INREG:
7408 case ISD::SIGN_EXTEND_VECTOR_INREG:
7409 assert(VT.isVector() && "This DAG node is restricted to vector types.");
7410 assert(N1.getValueType().bitsLE(VT) &&
7411 "The input must be the same size or smaller than the result.");
7412 assert(VT.getVectorMinNumElements() <
7413 N1.getValueType().getVectorMinNumElements() &&
7414 "The destination vector type must have fewer lanes than the input.");
7415 break;
7416 case ISD::ABS:
7417 assert(VT.isInteger() && VT == N1.getValueType() && "Invalid ABS!");
7418 if (N1.isUndef())
7419 return getConstant(Val: 0, DL, VT);
7420 break;
7421 case ISD::ABS_MIN_POISON:
7422 assert(VT.isInteger() && VT == N1.getValueType() &&
7423 "Invalid ABS_MIN_POISON!");
7424 if (N1.isUndef())
7425 return getConstant(Val: 0, DL, VT);
7426 break;
7427 case ISD::BSWAP:
7428 assert(VT.isInteger() && VT == N1.getValueType() && "Invalid BSWAP!");
7429 assert((VT.getScalarSizeInBits() % 16 == 0) &&
7430 "BSWAP types must be a multiple of 16 bits!");
7431 if (N1.isUndef())
7432 return getUNDEF(VT);
7433 // bswap(bswap(X)) -> X.
7434 if (OpOpcode == ISD::BSWAP)
7435 return N1.getOperand(i: 0);
7436 break;
7437 case ISD::BITREVERSE:
7438 assert(VT.isInteger() && VT == N1.getValueType() && "Invalid BITREVERSE!");
7439 if (N1.isUndef())
7440 return getUNDEF(VT);
7441 break;
7442 case ISD::BITCAST:
7443 assert(VT.getSizeInBits() == N1.getValueSizeInBits() &&
7444 "Cannot BITCAST between types of different sizes!");
7445 if (VT == N1.getValueType()) return N1; // noop conversion.
7446 if (OpOpcode == ISD::BITCAST) // bitconv(bitconv(x)) -> bitconv(x)
7447 return getNode(Opcode: ISD::BITCAST, DL, VT, N1: N1.getOperand(i: 0));
7448 if (N1.isUndef())
7449 return getUNDEF(VT);
7450 break;
7451 case ISD::SCALAR_TO_VECTOR:
7452 assert(VT.isVector() && !N1.getValueType().isVector() &&
7453 (VT.getVectorElementType() == N1.getValueType() ||
7454 (VT.getVectorElementType().isInteger() &&
7455 N1.getValueType().isInteger() &&
7456 VT.getVectorElementType().bitsLE(N1.getValueType()))) &&
7457 "Illegal SCALAR_TO_VECTOR node!");
7458 if (N1.isUndef())
7459 return getUNDEF(VT);
7460 // scalar_to_vector(extract_vector_elt V, 0) -> V, top bits are undefined.
7461 if (OpOpcode == ISD::EXTRACT_VECTOR_ELT &&
7462 isa<ConstantSDNode>(Val: N1.getOperand(i: 1)) &&
7463 N1.getConstantOperandVal(i: 1) == 0 &&
7464 N1.getOperand(i: 0).getValueType() == VT)
7465 return N1.getOperand(i: 0);
7466 break;
7467 case ISD::FNEG:
7468 // Negation of an unknown bag of bits is still completely undefined.
7469 if (N1.isUndef())
7470 return getUNDEF(VT);
7471
7472 if (OpOpcode == ISD::FNEG) // --X -> X
7473 return N1.getOperand(i: 0);
7474 break;
7475 case ISD::FABS:
7476 if (OpOpcode == ISD::FNEG) // abs(-X) -> abs(X)
7477 return getNode(Opcode: ISD::FABS, DL, VT, N1: N1.getOperand(i: 0));
7478 break;
7479 case ISD::VSCALE:
7480 assert(VT == N1.getValueType() && "Unexpected VT!");
7481 break;
7482 case ISD::CTPOP:
7483 if (N1.getValueType().getScalarType() == MVT::i1)
7484 return N1;
7485 break;
7486 case ISD::CTLZ:
7487 case ISD::CTTZ:
7488 if (N1.getValueType().getScalarType() == MVT::i1)
7489 return getNOT(DL, Val: N1, VT: N1.getValueType());
7490 break;
7491 case ISD::CTLS:
7492 if (N1.getValueType().getScalarType() == MVT::i1)
7493 return getConstant(Val: 0, DL, VT);
7494 break;
7495 case ISD::VECREDUCE_ADD:
7496 if (N1.getValueType().getScalarType() == MVT::i1)
7497 return getNode(Opcode: ISD::VECREDUCE_XOR, DL, VT, N1);
7498 break;
7499 case ISD::VECREDUCE_SMIN:
7500 case ISD::VECREDUCE_UMAX:
7501 if (N1.getValueType().getScalarType() == MVT::i1)
7502 return getNode(Opcode: ISD::VECREDUCE_OR, DL, VT, N1);
7503 break;
7504 case ISD::VECREDUCE_SMAX:
7505 case ISD::VECREDUCE_UMIN:
7506 if (N1.getValueType().getScalarType() == MVT::i1)
7507 return getNode(Opcode: ISD::VECREDUCE_AND, DL, VT, N1);
7508 break;
7509 case ISD::MASK_BEFOREFIRST:
7510 // With a single lane, the result is true iff that lane is false.
7511 if (VT.getVectorElementCount().isScalar())
7512 return getLogicalNOT(DL, Val: N1, VT);
7513 break;
7514 case ISD::VECTOR_REPEAT:
7515 assert(N1.getValueType().isFixedLengthVector() &&
7516 "VECTOR_REPEAT requires a fixed-length vector operand");
7517 assert(VT.isScalableVector() &&
7518 "VECTOR_REPEAT requires a scalable vector result");
7519 assert(N1.getValueType().getVectorNumElements() ==
7520 VT.getVectorMinNumElements() &&
7521 "VECTOR_REPEAT operand and result element counts must match");
7522 if (VT.getVectorMinNumElements() == 1)
7523 return getSplatVector(
7524 VT, DL, Op: getExtractVectorElt(DL, VT: VT.getVectorElementType(), Vec: N1, Idx: 0));
7525 break;
7526 case ISD::SPLAT_VECTOR:
7527 assert(VT.isVector() && "Wrong return type!");
7528 // FIXME: Hexagon uses i32 scalar for a floating point zero vector so allow
7529 // that for now.
7530 assert((VT.getVectorElementType() == N1.getValueType() ||
7531 (VT.isFloatingPoint() && N1.getValueType() == MVT::i32) ||
7532 (VT.getVectorElementType().isInteger() &&
7533 N1.getValueType().isInteger() &&
7534 VT.getVectorElementType().bitsLE(N1.getValueType()))) &&
7535 "Wrong operand type!");
7536 break;
7537 }
7538
7539 SDNode *N;
7540 SDVTList VTs = getVTList(VT);
7541 SDValue Ops[] = {N1};
7542 if (VT != MVT::Glue) { // Don't CSE glue producing nodes
7543 SDNodeKey ID(Opcode, VTs, Ops);
7544 FoldingSetInsertToken InsertToken;
7545 if (SDNode *E = lookupNode(Key: ID, DL, InsertToken)) {
7546 E->intersectFlagsWith(Flags);
7547 return SDValue(E, 0);
7548 }
7549
7550 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
7551 N->setFlags(Flags);
7552 createOperands(Node: N, Vals: Ops);
7553 CSEMap.insert(N, Token: InsertToken);
7554 } else {
7555 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
7556 createOperands(Node: N, Vals: Ops);
7557 }
7558
7559 InsertNode(N);
7560 SDValue V = SDValue(N, 0);
7561 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
7562 return V;
7563}
7564
7565static APInt getIntegerIdentity(unsigned Opcode, unsigned BitWidth) {
7566 switch (Opcode) {
7567 default:
7568 llvm_unreachable("Unexpected integer identity opcode");
7569 case ISD::ADD:
7570 case ISD::OR:
7571 case ISD::XOR:
7572 case ISD::UMAX:
7573 return APInt::getZero(numBits: BitWidth);
7574 case ISD::MUL:
7575 return APInt(BitWidth, 1);
7576 case ISD::AND:
7577 case ISD::UMIN:
7578 return APInt::getAllOnes(numBits: BitWidth);
7579 case ISD::SMAX:
7580 return APInt::getSignedMinValue(numBits: BitWidth);
7581 case ISD::SMIN:
7582 return APInt::getSignedMaxValue(numBits: BitWidth);
7583 }
7584}
7585
7586static std::optional<APInt> FoldValue(unsigned Opcode, const APInt &C1,
7587 const APInt &C2) {
7588 switch (Opcode) {
7589 case ISD::ADD: return C1 + C2;
7590 case ISD::SUB: return C1 - C2;
7591 case ISD::MUL: return C1 * C2;
7592 case ISD::AND: return C1 & C2;
7593 case ISD::OR: return C1 | C2;
7594 case ISD::XOR: return C1 ^ C2;
7595 case ISD::SHL: return C1 << C2;
7596 case ISD::SRL: return C1.lshr(ShiftAmt: C2);
7597 case ISD::SRA: return C1.ashr(ShiftAmt: C2);
7598 case ISD::ROTL: return C1.rotl(rotateAmt: C2);
7599 case ISD::ROTR: return C1.rotr(rotateAmt: C2);
7600 case ISD::SMIN: return C1.sle(RHS: C2) ? C1 : C2;
7601 case ISD::SMAX: return C1.sge(RHS: C2) ? C1 : C2;
7602 case ISD::UMIN: return C1.ule(RHS: C2) ? C1 : C2;
7603 case ISD::UMAX: return C1.uge(RHS: C2) ? C1 : C2;
7604 case ISD::SADDSAT: return C1.sadd_sat(RHS: C2);
7605 case ISD::UADDSAT: return C1.uadd_sat(RHS: C2);
7606 case ISD::SSUBSAT: return C1.ssub_sat(RHS: C2);
7607 case ISD::USUBSAT: return C1.usub_sat(RHS: C2);
7608 case ISD::SSHLSAT: return C1.sshl_sat(RHS: C2);
7609 case ISD::USHLSAT: return C1.ushl_sat(RHS: C2);
7610 case ISD::UDIV:
7611 if (!C2.getBoolValue())
7612 break;
7613 return C1.udiv(RHS: C2);
7614 case ISD::UREM:
7615 if (!C2.getBoolValue())
7616 break;
7617 return C1.urem(RHS: C2);
7618 case ISD::SDIV:
7619 if (!C2.getBoolValue())
7620 break;
7621 return C1.sdiv(RHS: C2);
7622 case ISD::SREM:
7623 if (!C2.getBoolValue())
7624 break;
7625 return C1.srem(RHS: C2);
7626 case ISD::AVGFLOORS:
7627 return APIntOps::avgFloorS(C1, C2);
7628 case ISD::AVGFLOORU:
7629 return APIntOps::avgFloorU(C1, C2);
7630 case ISD::AVGCEILS:
7631 return APIntOps::avgCeilS(C1, C2);
7632 case ISD::AVGCEILU:
7633 return APIntOps::avgCeilU(C1, C2);
7634 case ISD::AVGSUBFLOORS:
7635 return APIntOps::avgSubFloorS(C1, C2);
7636 case ISD::AVGSUBFLOORU:
7637 return APIntOps::avgSubFloorU(C1, C2);
7638 case ISD::ABDS:
7639 return APIntOps::abds(A: C1, B: C2);
7640 case ISD::ABDU:
7641 return APIntOps::abdu(A: C1, B: C2);
7642 case ISD::MULHS:
7643 return APIntOps::mulhs(C1, C2);
7644 case ISD::MULHU:
7645 return APIntOps::mulhu(C1, C2);
7646 case ISD::CLMUL:
7647 return APIntOps::clmul(LHS: C1, RHS: C2);
7648 case ISD::CLMULR:
7649 return APIntOps::clmulr(LHS: C1, RHS: C2);
7650 case ISD::CLMULH:
7651 return APIntOps::clmulh(LHS: C1, RHS: C2);
7652 case ISD::PEXT:
7653 return APIntOps::pext(Val: C1, Mask: C2);
7654 case ISD::PDEP:
7655 return APIntOps::pdep(Val: C1, Mask: C2);
7656 }
7657 return std::nullopt;
7658}
7659// Handle constant folding with UNDEF.
7660// TODO: Handle more cases.
7661static std::optional<APInt> FoldValueWithUndef(unsigned Opcode, const APInt &C1,
7662 bool IsUndef1, const APInt &C2,
7663 bool IsUndef2) {
7664 if (!(IsUndef1 || IsUndef2))
7665 return FoldValue(Opcode, C1, C2);
7666
7667 // Fold and(x, undef) -> 0
7668 // Fold mul(x, undef) -> 0
7669 if (Opcode == ISD::AND || Opcode == ISD::MUL)
7670 return APInt::getZero(numBits: C1.getBitWidth());
7671
7672 return std::nullopt;
7673}
7674
7675SDValue SelectionDAG::FoldSymbolOffset(unsigned Opcode, EVT VT,
7676 const GlobalAddressSDNode *GA,
7677 const SDNode *N2) {
7678 if (GA->getOpcode() != ISD::GlobalAddress)
7679 return SDValue();
7680 if (!TLI->isOffsetFoldingLegal(GA))
7681 return SDValue();
7682 auto *C2 = dyn_cast<ConstantSDNode>(Val: N2);
7683 if (!C2)
7684 return SDValue();
7685 int64_t Offset = C2->getSExtValue();
7686 switch (Opcode) {
7687 case ISD::ADD:
7688 case ISD::PTRADD:
7689 break;
7690 case ISD::SUB: Offset = -uint64_t(Offset); break;
7691 default: return SDValue();
7692 }
7693 return getGlobalAddress(GV: GA->getGlobal(), DL: SDLoc(C2), VT,
7694 Offset: GA->getOffset() + uint64_t(Offset));
7695}
7696
7697bool SelectionDAG::isUndef(unsigned Opcode, ArrayRef<SDValue> Ops) {
7698 switch (Opcode) {
7699 case ISD::SDIV:
7700 case ISD::UDIV:
7701 case ISD::SREM:
7702 case ISD::UREM: {
7703 // If a divisor is zero/undef or any element of a divisor vector is
7704 // zero/undef, the whole op is undef.
7705 assert(Ops.size() == 2 && "Div/rem should have 2 operands");
7706 SDValue Divisor = Ops[1];
7707 if (Divisor.isUndef() || isNullConstant(V: Divisor))
7708 return true;
7709
7710 return ISD::isBuildVectorOfConstantSDNodes(N: Divisor.getNode()) &&
7711 llvm::any_of(Range: Divisor->op_values(),
7712 P: [](SDValue V) { return V.isUndef() ||
7713 isNullConstant(V); });
7714 // TODO: Handle signed overflow.
7715 }
7716 // TODO: Handle oversized shifts.
7717 default:
7718 return false;
7719 }
7720}
7721
7722SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL,
7723 EVT VT, ArrayRef<SDValue> Ops,
7724 SDNodeFlags Flags) {
7725 // If the opcode is a target-specific ISD node, there's nothing we can
7726 // do here and the operand rules may not line up with the below, so
7727 // bail early.
7728 // We can't create a scalar CONCAT_VECTORS so skip it. It will break
7729 // for concats involving SPLAT_VECTOR. Concats of BUILD_VECTORS are handled by
7730 // foldCONCAT_VECTORS in getNode before this is called.
7731 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::CONCAT_VECTORS)
7732 return SDValue();
7733
7734 unsigned NumOps = Ops.size();
7735 if (NumOps == 0)
7736 return SDValue();
7737
7738 if (isUndef(Opcode, Ops))
7739 return getUNDEF(VT);
7740
7741 // Handle unary special cases.
7742 if (NumOps == 1) {
7743 SDValue N1 = Ops[0];
7744
7745 // Constant fold unary operations with an integer constant operand. Even
7746 // opaque constant will be folded, because the folding of unary operations
7747 // doesn't create new constants with different values. Nevertheless, the
7748 // opaque flag is preserved during folding to prevent future folding with
7749 // other constants.
7750 if (auto *C = dyn_cast<ConstantSDNode>(Val&: N1)) {
7751 const APInt &Val = C->getAPIntValue();
7752 switch (Opcode) {
7753 case ISD::SIGN_EXTEND:
7754 return getConstant(Val: Val.sextOrTrunc(width: VT.getSizeInBits()), DL, VT,
7755 isT: C->isTargetOpcode(), isO: C->isOpaque());
7756 case ISD::TRUNCATE:
7757 if (C->isOpaque())
7758 break;
7759 [[fallthrough]];
7760 case ISD::ZERO_EXTEND:
7761 return getConstant(Val: Val.zextOrTrunc(width: VT.getSizeInBits()), DL, VT,
7762 isT: C->isTargetOpcode(), isO: C->isOpaque());
7763 case ISD::ANY_EXTEND:
7764 // Some targets like RISCV prefer to sign extend some types.
7765 if (TLI->isSExtCheaperThanZExt(FromTy: N1.getValueType(), ToTy: VT))
7766 return getConstant(Val: Val.sextOrTrunc(width: VT.getSizeInBits()), DL, VT,
7767 isT: C->isTargetOpcode(), isO: C->isOpaque());
7768 return getConstant(Val: Val.zextOrTrunc(width: VT.getSizeInBits()), DL, VT,
7769 isT: C->isTargetOpcode(), isO: C->isOpaque());
7770 case ISD::ABS:
7771 return getConstant(Val: Val.abs(), DL, VT, isT: C->isTargetOpcode(),
7772 isO: C->isOpaque());
7773 case ISD::ABS_MIN_POISON:
7774 if (Val.isMinSignedValue())
7775 return getPOISON(VT);
7776 return getConstant(Val: Val.abs(), DL, VT, isT: C->isTargetOpcode(),
7777 isO: C->isOpaque());
7778 case ISD::BITREVERSE:
7779 return getConstant(Val: Val.reverseBits(), DL, VT, isT: C->isTargetOpcode(),
7780 isO: C->isOpaque());
7781 case ISD::BSWAP:
7782 return getConstant(Val: Val.byteSwap(), DL, VT, isT: C->isTargetOpcode(),
7783 isO: C->isOpaque());
7784 case ISD::CTPOP:
7785 return getConstant(Val: Val.popcount(), DL, VT, isT: C->isTargetOpcode(),
7786 isO: C->isOpaque());
7787 case ISD::PARITY:
7788 return getConstant(Val: Val.popcount() & 1, DL, VT, isT: C->isTargetOpcode(),
7789 isO: C->isOpaque());
7790 case ISD::CTLZ:
7791 case ISD::CTLZ_ZERO_POISON:
7792 return getConstant(Val: Val.countl_zero(), DL, VT, isT: C->isTargetOpcode(),
7793 isO: C->isOpaque());
7794 case ISD::CTTZ:
7795 case ISD::CTTZ_ZERO_POISON:
7796 return getConstant(Val: Val.countr_zero(), DL, VT, isT: C->isTargetOpcode(),
7797 isO: C->isOpaque());
7798 case ISD::CTLS:
7799 // CTLS returns the number of extra sign bits so subtract one.
7800 return getConstant(Val: Val.getNumSignBits() - 1, DL, VT,
7801 isT: C->isTargetOpcode(), isO: C->isOpaque());
7802 case ISD::UINT_TO_FP:
7803 case ISD::SINT_TO_FP: {
7804 APFloat FPV(VT.getFltSemantics(), APInt::getZero(numBits: VT.getSizeInBits()));
7805 (void)FPV.convertFromAPInt(Input: Val, IsSigned: Opcode == ISD::SINT_TO_FP,
7806 RM: APFloat::rmNearestTiesToEven);
7807 return getConstantFP(V: FPV, DL, VT);
7808 }
7809 case ISD::FP16_TO_FP:
7810 case ISD::BF16_TO_FP: {
7811 bool Ignored;
7812 APFloat FPV(Opcode == ISD::FP16_TO_FP ? APFloat::IEEEhalf()
7813 : APFloat::BFloat(),
7814 (Val.getBitWidth() == 16) ? Val : Val.trunc(width: 16));
7815
7816 // This can return overflow, underflow, or inexact; we don't care.
7817 // FIXME need to be more flexible about rounding mode.
7818 (void)FPV.convert(ToSemantics: VT.getFltSemantics(), RM: APFloat::rmNearestTiesToEven,
7819 losesInfo: &Ignored);
7820 return getConstantFP(V: FPV, DL, VT);
7821 }
7822 case ISD::STEP_VECTOR:
7823 if (SDValue V = FoldSTEP_VECTOR(DL, VT, Step: N1, DAG&: *this))
7824 return V;
7825 break;
7826 case ISD::BITCAST:
7827 if (C->isOpaque())
7828 break;
7829 if (VT == MVT::f16 && C->getValueType(ResNo: 0) == MVT::i16)
7830 return getConstantFP(V: APFloat(APFloat::IEEEhalf(), Val), DL, VT);
7831 if (VT == MVT::f32 && C->getValueType(ResNo: 0) == MVT::i32)
7832 return getConstantFP(V: APFloat(APFloat::IEEEsingle(), Val), DL, VT);
7833 if (VT == MVT::f64 && C->getValueType(ResNo: 0) == MVT::i64)
7834 return getConstantFP(V: APFloat(APFloat::IEEEdouble(), Val), DL, VT);
7835 if (VT == MVT::f128 && C->getValueType(ResNo: 0) == MVT::i128)
7836 return getConstantFP(V: APFloat(APFloat::IEEEquad(), Val), DL, VT);
7837 break;
7838 }
7839 }
7840
7841 // Constant fold unary operations with a floating point constant operand.
7842 if (auto *C = dyn_cast<ConstantFPSDNode>(Val&: N1)) {
7843 APFloat V = C->getValueAPF(); // make copy
7844 switch (Opcode) {
7845 case ISD::FNEG:
7846 V.changeSign();
7847 return getConstantFP(V, DL, VT);
7848 case ISD::FABS:
7849 V.clearSign();
7850 return getConstantFP(V, DL, VT);
7851 case ISD::FCEIL: {
7852 APFloat::opStatus fs = V.roundToIntegral(RM: APFloat::rmTowardPositive);
7853 if (fs == APFloat::opOK || fs == APFloat::opInexact)
7854 return getConstantFP(V, DL, VT);
7855 return SDValue();
7856 }
7857 case ISD::FTRUNC: {
7858 APFloat::opStatus fs = V.roundToIntegral(RM: APFloat::rmTowardZero);
7859 if (fs == APFloat::opOK || fs == APFloat::opInexact)
7860 return getConstantFP(V, DL, VT);
7861 return SDValue();
7862 }
7863 case ISD::FFLOOR: {
7864 APFloat::opStatus fs = V.roundToIntegral(RM: APFloat::rmTowardNegative);
7865 if (fs == APFloat::opOK || fs == APFloat::opInexact)
7866 return getConstantFP(V, DL, VT);
7867 return SDValue();
7868 }
7869 case ISD::FP_EXTEND: {
7870 bool ignored;
7871 // This can return overflow, underflow, or inexact; we don't care.
7872 // FIXME need to be more flexible about rounding mode.
7873 (void)V.convert(ToSemantics: VT.getFltSemantics(), RM: APFloat::rmNearestTiesToEven,
7874 losesInfo: &ignored);
7875 return getConstantFP(V, DL, VT);
7876 }
7877 case ISD::FP_TO_SINT:
7878 case ISD::FP_TO_UINT: {
7879 bool ignored;
7880 APSInt IntVal(VT.getSizeInBits(), Opcode == ISD::FP_TO_UINT);
7881 // FIXME need to be more flexible about rounding mode.
7882 APFloat::opStatus s =
7883 V.convertToInteger(Result&: IntVal, RM: APFloat::rmTowardZero, IsExact: &ignored);
7884 if (s == APFloat::opInvalidOp) // inexact is OK, in fact usual
7885 break;
7886 return getConstant(Val: IntVal, DL, VT);
7887 }
7888 case ISD::FP_TO_FP16:
7889 case ISD::FP_TO_BF16: {
7890 bool Ignored;
7891 // This can return overflow, underflow, or inexact; we don't care.
7892 // FIXME need to be more flexible about rounding mode.
7893 (void)V.convert(ToSemantics: Opcode == ISD::FP_TO_FP16 ? APFloat::IEEEhalf()
7894 : APFloat::BFloat(),
7895 RM: APFloat::rmNearestTiesToEven, losesInfo: &Ignored);
7896 return getConstant(Val: V.bitcastToAPInt().getZExtValue(), DL, VT);
7897 }
7898 case ISD::BITCAST:
7899 if (VT == MVT::i16 && C->getValueType(ResNo: 0) == MVT::f16)
7900 return getConstant(Val: (uint16_t)V.bitcastToAPInt().getZExtValue(), DL,
7901 VT);
7902 if (VT == MVT::i16 && C->getValueType(ResNo: 0) == MVT::bf16)
7903 return getConstant(Val: (uint16_t)V.bitcastToAPInt().getZExtValue(), DL,
7904 VT);
7905 if (VT == MVT::i32 && C->getValueType(ResNo: 0) == MVT::f32)
7906 return getConstant(Val: (uint32_t)V.bitcastToAPInt().getZExtValue(), DL,
7907 VT);
7908 if (VT == MVT::i64 && C->getValueType(ResNo: 0) == MVT::f64)
7909 return getConstant(Val: V.bitcastToAPInt().getZExtValue(), DL, VT);
7910 break;
7911 }
7912 }
7913
7914 // Early-out if we failed to constant fold a bitcast.
7915 if (Opcode == ISD::BITCAST)
7916 return SDValue();
7917
7918 // Constant fold integer vector reductions with constant BUILD_VECTORs.
7919 if ((Opcode == ISD::VECREDUCE_ADD || Opcode == ISD::VECREDUCE_SMAX ||
7920 Opcode == ISD::VECREDUCE_SMIN || Opcode == ISD::VECREDUCE_UMAX ||
7921 Opcode == ISD::VECREDUCE_UMIN || Opcode == ISD::VECREDUCE_MUL ||
7922 Opcode == ISD::VECREDUCE_OR || Opcode == ISD::VECREDUCE_XOR ||
7923 Opcode == ISD::VECREDUCE_AND) &&
7924 ISD::isBuildVectorOfConstantSDNodes(N: N1.getNode())) {
7925 unsigned EltBits = N1.getValueType().getScalarSizeInBits();
7926 unsigned BaseOpcode = ISD::getVecReduceBaseOpcode(VecReduceOpcode: Opcode);
7927 APInt Acc = getIntegerIdentity(Opcode: BaseOpcode, BitWidth: EltBits);
7928 for (SDValue Elt : N1->op_values()) {
7929 if (Elt.getOpcode() == ISD::POISON)
7930 return getPOISON(VT);
7931 if (Elt.isUndef() || cast<ConstantSDNode>(Val&: Elt)->isOpaque())
7932 return SDValue();
7933 APInt Value = cast<ConstantSDNode>(Val&: Elt)->getAPIntValue().trunc(width: EltBits);
7934 std::optional<APInt> Folded = FoldValue(Opcode: BaseOpcode, C1: Acc, C2: Value);
7935 assert(Folded &&
7936 "Expected vector reduction base opcode to be foldable");
7937 Acc = *Folded;
7938 }
7939 EVT EltVT = N1.getValueType().getScalarType();
7940 return getAnyExtOrTrunc(Op: getConstant(Val: Acc, DL, VT: EltVT), DL, VT);
7941 }
7942 }
7943
7944 // Handle binops special cases.
7945 if (NumOps == 2) {
7946 if (SDValue CFP = foldConstantFPMath(Opcode, DL, VT, Ops))
7947 return CFP;
7948
7949 if (auto *C1 = dyn_cast<ConstantSDNode>(Val: Ops[0])) {
7950 if (auto *C2 = dyn_cast<ConstantSDNode>(Val: Ops[1])) {
7951 if (C1->isOpaque() || C2->isOpaque())
7952 return SDValue();
7953
7954 std::optional<APInt> FoldAttempt =
7955 FoldValue(Opcode, C1: C1->getAPIntValue(), C2: C2->getAPIntValue());
7956 if (!FoldAttempt)
7957 return SDValue();
7958
7959 SDValue Folded = getConstant(Val: *FoldAttempt, DL, VT);
7960 assert((!Folded || !VT.isVector()) &&
7961 "Can't fold vectors ops with scalar operands");
7962 return Folded;
7963 }
7964 }
7965
7966 // fold (add Sym, c) -> Sym+c
7967 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Val: Ops[0]))
7968 return FoldSymbolOffset(Opcode, VT, GA, N2: Ops[1].getNode());
7969 if (TLI->isCommutativeBinOp(Opcode))
7970 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Val: Ops[1]))
7971 return FoldSymbolOffset(Opcode, VT, GA, N2: Ops[0].getNode());
7972
7973 // fold (sext_in_reg c1) -> c2
7974 if (Opcode == ISD::SIGN_EXTEND_INREG) {
7975 EVT EVT = cast<VTSDNode>(Val: Ops[1])->getVT();
7976
7977 auto SignExtendInReg = [&](APInt Val, llvm::EVT ConstantVT) {
7978 unsigned FromBits = EVT.getScalarSizeInBits();
7979 Val <<= Val.getBitWidth() - FromBits;
7980 Val.ashrInPlace(ShiftAmt: Val.getBitWidth() - FromBits);
7981 return getConstant(Val, DL, VT: ConstantVT);
7982 };
7983
7984 if (auto *C1 = dyn_cast<ConstantSDNode>(Val: Ops[0])) {
7985 const APInt &Val = C1->getAPIntValue();
7986 return SignExtendInReg(Val, VT);
7987 }
7988
7989 if (ISD::isBuildVectorOfConstantSDNodes(N: Ops[0].getNode())) {
7990 SmallVector<SDValue, 8> ScalarOps;
7991 llvm::EVT OpVT = Ops[0].getOperand(i: 0).getValueType();
7992 for (int I = 0, E = VT.getVectorNumElements(); I != E; ++I) {
7993 SDValue Op = Ops[0].getOperand(i: I);
7994 if (Op.isUndef()) {
7995 ScalarOps.push_back(Elt: getUNDEF(VT: OpVT));
7996 continue;
7997 }
7998 const APInt &Val = cast<ConstantSDNode>(Val&: Op)->getAPIntValue();
7999 ScalarOps.push_back(Elt: SignExtendInReg(Val, OpVT));
8000 }
8001 return getBuildVector(VT, DL, Ops: ScalarOps);
8002 }
8003
8004 if (Ops[0].getOpcode() == ISD::SPLAT_VECTOR &&
8005 isa<ConstantSDNode>(Val: Ops[0].getOperand(i: 0)))
8006 return getNode(Opcode: ISD::SPLAT_VECTOR, DL, VT,
8007 N1: SignExtendInReg(Ops[0].getConstantOperandAPInt(i: 0),
8008 Ops[0].getOperand(i: 0).getValueType()));
8009 }
8010 }
8011
8012 // Handle fshl/fshr special cases.
8013 if (Opcode == ISD::FSHL || Opcode == ISD::FSHR) {
8014 auto *C1 = dyn_cast<ConstantSDNode>(Val: Ops[0]);
8015 auto *C2 = dyn_cast<ConstantSDNode>(Val: Ops[1]);
8016 auto *C3 = dyn_cast<ConstantSDNode>(Val: Ops[2]);
8017
8018 if (C1 && C2 && C3) {
8019 if (C1->isOpaque() || C2->isOpaque() || C3->isOpaque())
8020 return SDValue();
8021 const APInt &V1 = C1->getAPIntValue(), &V2 = C2->getAPIntValue(),
8022 &V3 = C3->getAPIntValue();
8023
8024 APInt FoldedVal = Opcode == ISD::FSHL ? APIntOps::fshl(Hi: V1, Lo: V2, Shift: V3)
8025 : APIntOps::fshr(Hi: V1, Lo: V2, Shift: V3);
8026 return getConstant(Val: FoldedVal, DL, VT);
8027 }
8028 }
8029
8030 // Handle fma/fmad special cases.
8031 if (Opcode == ISD::FMA || Opcode == ISD::FMAD || Opcode == ISD::FMULADD) {
8032 assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
8033 assert(Ops[0].getValueType() == VT && Ops[1].getValueType() == VT &&
8034 Ops[2].getValueType() == VT && "FMA types must match!");
8035 ConstantFPSDNode *C1 = dyn_cast<ConstantFPSDNode>(Val: Ops[0]);
8036 ConstantFPSDNode *C2 = dyn_cast<ConstantFPSDNode>(Val: Ops[1]);
8037 ConstantFPSDNode *C3 = dyn_cast<ConstantFPSDNode>(Val: Ops[2]);
8038 if (C1 && C2 && C3) {
8039 APFloat V1 = C1->getValueAPF();
8040 const APFloat &V2 = C2->getValueAPF();
8041 const APFloat &V3 = C3->getValueAPF();
8042 if (Opcode == ISD::FMAD || Opcode == ISD::FMULADD) {
8043 V1.multiply(RHS: V2, RM: APFloat::rmNearestTiesToEven);
8044 V1.add(RHS: V3, RM: APFloat::rmNearestTiesToEven);
8045 } else
8046 V1.fusedMultiplyAdd(Multiplicand: V2, Addend: V3, RM: APFloat::rmNearestTiesToEven);
8047 return getConstantFP(V: V1, DL, VT);
8048 }
8049 }
8050
8051 // This is for vector folding only from here on.
8052 if (!VT.isVector())
8053 return SDValue();
8054
8055 // Constant fold integer partial reductions with constant BUILD_VECTOR
8056 // operands. The reduction order is deliberately unspecified. Use the same
8057 // subvector layout as TargetLowering::expandPartialReduceMLA(), where input
8058 // lane I contributes to accumulator lane I % NumAccElts.
8059 if (Opcode == ISD::PARTIAL_REDUCE_SMLA ||
8060 Opcode == ISD::PARTIAL_REDUCE_UMLA ||
8061 Opcode == ISD::PARTIAL_REDUCE_SUMLA) {
8062 // These nodes have no scalar form, so unsupported cases must not fall
8063 // through to generic per-lane vector folding.
8064 if (!llvm::all_of(Range&: Ops, P: [](SDValue Op) {
8065 return ISD::isBuildVectorOfConstantSDNodes(N: Op.getNode());
8066 }))
8067 return SDValue();
8068
8069 unsigned AccEltBits = VT.getScalarSizeInBits();
8070 unsigned InputEltBits = Ops[1].getScalarValueSizeInBits();
8071 unsigned NumAccElts = VT.getVectorNumElements();
8072 unsigned NumInputElts = Ops[1].getValueType().getVectorNumElements();
8073 SmallVector<APInt, 8> Results(NumAccElts, APInt::getZero(numBits: AccEltBits));
8074 BitVector PoisonElts(NumAccElts);
8075
8076 for (unsigned I = 0; I != NumAccElts; ++I) {
8077 SDValue Elt = Ops[0].getOperand(i: I);
8078 if (Elt.getOpcode() == ISD::POISON) {
8079 PoisonElts.set(I);
8080 continue;
8081 }
8082 auto *C = dyn_cast<ConstantSDNode>(Val&: Elt);
8083 if (!C || C->isOpaque())
8084 return SDValue();
8085 Results[I] = C->getAPIntValue().trunc(width: AccEltBits);
8086 }
8087
8088 bool IsLHSSigned = Opcode != ISD::PARTIAL_REDUCE_UMLA;
8089 bool IsRHSSigned = Opcode == ISD::PARTIAL_REDUCE_SMLA;
8090 for (unsigned I = 0; I != NumInputElts; ++I) {
8091 const unsigned AccIdx = I % NumAccElts;
8092 SDValue LHSElt = Ops[1].getOperand(i: I);
8093 SDValue RHSElt = Ops[2].getOperand(i: I);
8094 if (LHSElt.getOpcode() == ISD::POISON ||
8095 RHSElt.getOpcode() == ISD::POISON) {
8096 PoisonElts.set(AccIdx);
8097 continue;
8098 }
8099
8100 auto *LHS = dyn_cast<ConstantSDNode>(Val&: LHSElt);
8101 auto *RHS = dyn_cast<ConstantSDNode>(Val&: RHSElt);
8102 if (!LHS || !RHS || LHS->isOpaque() || RHS->isOpaque())
8103 return SDValue();
8104
8105 APInt LHSVal = LHS->getAPIntValue().trunc(width: InputEltBits);
8106 APInt RHSVal = RHS->getAPIntValue().trunc(width: InputEltBits);
8107 LHSVal = IsLHSSigned ? LHSVal.sext(width: AccEltBits) : LHSVal.zext(width: AccEltBits);
8108 RHSVal = IsRHSSigned ? RHSVal.sext(width: AccEltBits) : RHSVal.zext(width: AccEltBits);
8109 Results[AccIdx] += LHSVal * RHSVal;
8110 }
8111
8112 // After type legalization the vector element type may not be a legal
8113 // scalar type (e.g. i16 on AArch64). Create the folded constants in the
8114 // promoted legal scalar type instead, matching the generic per-lane path
8115 // below. Bail out if legalization would narrow the type, since the lane
8116 // value would not fit.
8117 EVT AccEltVT = VT.getVectorElementType();
8118 EVT LegalSVT = AccEltVT;
8119 if (NewNodesMustHaveLegalTypes && LegalSVT.isInteger()) {
8120 LegalSVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT: LegalSVT);
8121 if (LegalSVT.bitsLT(VT: AccEltVT))
8122 return SDValue();
8123 }
8124
8125 SmallVector<SDValue, 8> ResultOps;
8126 for (unsigned I = 0; I != NumAccElts; ++I)
8127 ResultOps.push_back(
8128 Elt: PoisonElts[I] ? getPOISON(VT: LegalSVT)
8129 : getConstant(Val: Results[I].sext(width: LegalSVT.getSizeInBits()),
8130 DL, VT: LegalSVT));
8131 return getBuildVector(VT, DL, Ops: ResultOps);
8132 }
8133
8134 ElementCount NumElts = VT.getVectorElementCount();
8135
8136 // See if we can fold through any bitcasted integer ops.
8137 if (NumOps == 2 && VT.isFixedLengthVector() && VT.isInteger() &&
8138 Ops[0].getValueType() == VT && Ops[1].getValueType() == VT &&
8139 (Ops[0].getOpcode() == ISD::BITCAST ||
8140 Ops[1].getOpcode() == ISD::BITCAST)) {
8141 SDValue N1 = peekThroughBitcasts(V: Ops[0]);
8142 SDValue N2 = peekThroughBitcasts(V: Ops[1]);
8143 auto *BV1 = dyn_cast<BuildVectorSDNode>(Val&: N1);
8144 auto *BV2 = dyn_cast<BuildVectorSDNode>(Val&: N2);
8145 if (BV1 && BV2 && N1.getValueType().isInteger() &&
8146 N2.getValueType().isInteger()) {
8147 bool IsLE = getDataLayout().isLittleEndian();
8148 unsigned EltBits = VT.getScalarSizeInBits();
8149 SmallVector<APInt> RawBits1, RawBits2;
8150 BitVector UndefElts1, UndefElts2;
8151 if (BV1->getConstantRawBits(IsLittleEndian: IsLE, DstEltSizeInBits: EltBits, RawBitElements&: RawBits1, UndefElements&: UndefElts1) &&
8152 BV2->getConstantRawBits(IsLittleEndian: IsLE, DstEltSizeInBits: EltBits, RawBitElements&: RawBits2, UndefElements&: UndefElts2)) {
8153 SmallVector<APInt> RawBits;
8154 for (unsigned I = 0, E = NumElts.getFixedValue(); I != E; ++I) {
8155 std::optional<APInt> Fold = FoldValueWithUndef(
8156 Opcode, C1: RawBits1[I], IsUndef1: UndefElts1[I], C2: RawBits2[I], IsUndef2: UndefElts2[I]);
8157 if (!Fold)
8158 break;
8159 RawBits.push_back(Elt: *Fold);
8160 }
8161 if (RawBits.size() == NumElts.getFixedValue()) {
8162 // We have constant folded, but we might need to cast this again back
8163 // to the original (possibly legalized) type.
8164 EVT BVVT, BVEltVT;
8165 if (N1.getValueType() == VT) {
8166 BVVT = N1.getValueType();
8167 BVEltVT = BV1->getOperand(Num: 0).getValueType();
8168 } else {
8169 BVVT = N2.getValueType();
8170 BVEltVT = BV2->getOperand(Num: 0).getValueType();
8171 }
8172 unsigned BVEltBits = BVEltVT.getSizeInBits();
8173 SmallVector<APInt> DstBits;
8174 BitVector DstUndefs;
8175 BuildVectorSDNode::recastRawBits(IsLittleEndian: IsLE, DstEltSizeInBits: BVVT.getScalarSizeInBits(),
8176 DstBitElements&: DstBits, SrcBitElements: RawBits, DstUndefElements&: DstUndefs,
8177 SrcUndefElements: BitVector(RawBits.size(), false));
8178 SmallVector<SDValue> Ops(DstBits.size(), getUNDEF(VT: BVEltVT));
8179 for (unsigned I = 0, E = DstBits.size(); I != E; ++I) {
8180 if (DstUndefs[I])
8181 continue;
8182 Ops[I] = getConstant(Val: DstBits[I].sext(width: BVEltBits), DL, VT: BVEltVT);
8183 }
8184 return getBitcast(VT, V: getBuildVector(VT: BVVT, DL, Ops));
8185 }
8186 }
8187 }
8188 // Logic ops can be folded from raw integer bits - mainly for AVX512 masks.
8189 if (ISD::isBitwiseLogicOp(Opcode) && isa<ConstantSDNode>(Val: N1) &&
8190 isa<ConstantSDNode>(Val: N2)) {
8191 if (SDValue Res = FoldConstantArithmetic(Opcode, DL, VT: N1.getValueType(),
8192 Ops: {N1, N2}, Flags))
8193 return getBitcast(VT, V: Res);
8194 }
8195 }
8196
8197 // Fold (mul step_vector(C0), C1) to (step_vector(C0 * C1)).
8198 // (shl step_vector(C0), C1) -> (step_vector(C0 << C1))
8199 if ((Opcode == ISD::MUL || Opcode == ISD::SHL) &&
8200 Ops[0].getOpcode() == ISD::STEP_VECTOR) {
8201 APInt RHSVal;
8202 if (ISD::isConstantSplatVector(N: Ops[1].getNode(), SplatVal&: RHSVal)) {
8203 APInt NewStep = Opcode == ISD::MUL
8204 ? Ops[0].getConstantOperandAPInt(i: 0) * RHSVal
8205 : Ops[0].getConstantOperandAPInt(i: 0) << RHSVal;
8206 return getStepVector(DL, ResVT: VT, StepVal: NewStep);
8207 }
8208 }
8209
8210 auto IsScalarOrSameVectorSize = [NumElts](const SDValue &Op) {
8211 return !Op.getValueType().isVector() ||
8212 Op.getValueType().getVectorElementCount() == NumElts;
8213 };
8214
8215 auto IsBuildVectorSplatVectorOrUndef = [](const SDValue &Op) {
8216 return Op.isUndef() || Op.getOpcode() == ISD::CONDCODE ||
8217 Op.getOpcode() == ISD::BUILD_VECTOR ||
8218 Op.getOpcode() == ISD::SPLAT_VECTOR;
8219 };
8220
8221 // All operands must be vector types with the same number of elements as
8222 // the result type and must be either UNDEF or a build/splat vector
8223 // or UNDEF scalars.
8224 if (!llvm::all_of(Range&: Ops, P: IsBuildVectorSplatVectorOrUndef) ||
8225 !llvm::all_of(Range&: Ops, P: IsScalarOrSameVectorSize))
8226 return SDValue();
8227
8228 // If we are comparing vectors, then the result needs to be a i1 boolean that
8229 // is then extended back to the legal result type depending on how booleans
8230 // are represented.
8231 EVT SVT = (Opcode == ISD::SETCC ? MVT::i1 : VT.getScalarType());
8232 ISD::NodeType ExtendCode =
8233 (Opcode == ISD::SETCC && SVT != VT.getScalarType())
8234 ? TargetLowering::getExtendForContent(Content: TLI->getBooleanContents(Type: VT))
8235 : ISD::SIGN_EXTEND;
8236
8237 // Find legal integer scalar type for constant promotion and
8238 // ensure that its scalar size is at least as large as source.
8239 EVT LegalSVT = VT.getScalarType();
8240 if (NewNodesMustHaveLegalTypes && LegalSVT.isInteger()) {
8241 LegalSVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT: LegalSVT);
8242 if (LegalSVT.bitsLT(VT: VT.getScalarType()))
8243 return SDValue();
8244 }
8245
8246 // For scalable vector types we know we're dealing with SPLAT_VECTORs. We
8247 // only have one operand to check. For fixed-length vector types we may have
8248 // a combination of BUILD_VECTOR and SPLAT_VECTOR.
8249 unsigned NumVectorElts = NumElts.isScalable() ? 1 : NumElts.getFixedValue();
8250
8251 // Constant fold each scalar lane separately.
8252 SmallVector<SDValue, 4> ScalarResults;
8253 for (unsigned I = 0; I != NumVectorElts; I++) {
8254 SmallVector<SDValue, 4> ScalarOps;
8255 for (SDValue Op : Ops) {
8256 EVT InSVT = Op.getValueType().getScalarType();
8257 if (Op.getOpcode() != ISD::BUILD_VECTOR &&
8258 Op.getOpcode() != ISD::SPLAT_VECTOR) {
8259 if (Op.isUndef())
8260 ScalarOps.push_back(Elt: getUNDEF(VT: InSVT));
8261 else
8262 ScalarOps.push_back(Elt: Op);
8263 continue;
8264 }
8265
8266 SDValue ScalarOp =
8267 Op.getOperand(i: Op.getOpcode() == ISD::SPLAT_VECTOR ? 0 : I);
8268 EVT ScalarVT = ScalarOp.getValueType();
8269
8270 // Build vector (integer) scalar operands may need implicit
8271 // truncation - do this before constant folding.
8272 if (ScalarVT.isInteger() && ScalarVT.bitsGT(VT: InSVT)) {
8273 // Don't create illegally-typed nodes unless they're constants or undef
8274 // - if we fail to constant fold we can't guarantee the (dead) nodes
8275 // we're creating will be cleaned up before being visited for
8276 // legalization.
8277 if (NewNodesMustHaveLegalTypes && !ScalarOp.isUndef() &&
8278 !isa<ConstantSDNode>(Val: ScalarOp) &&
8279 TLI->getTypeAction(Context&: *getContext(), VT: InSVT) !=
8280 TargetLowering::TypeLegal)
8281 return SDValue();
8282 ScalarOp = getNode(Opcode: ISD::TRUNCATE, DL, VT: InSVT, N1: ScalarOp);
8283 }
8284
8285 ScalarOps.push_back(Elt: ScalarOp);
8286 }
8287
8288 // Constant fold the scalar operands.
8289 SDValue ScalarResult = getNode(Opcode, DL, VT: SVT, Ops: ScalarOps, Flags);
8290
8291 // Scalar folding only succeeded if the result is a constant or UNDEF.
8292 if (!ScalarResult.isUndef() && ScalarResult.getOpcode() != ISD::Constant &&
8293 ScalarResult.getOpcode() != ISD::ConstantFP)
8294 return SDValue();
8295
8296 // Legalize the (integer) scalar constant if necessary. We only do
8297 // this once we know the folding succeeded, since otherwise we would
8298 // get a node with illegal type which has a user.
8299 if (LegalSVT != SVT)
8300 ScalarResult = getNode(Opcode: ExtendCode, DL, VT: LegalSVT, N1: ScalarResult);
8301
8302 ScalarResults.push_back(Elt: ScalarResult);
8303 }
8304
8305 SDValue V = NumElts.isScalable() ? getSplatVector(VT, DL, Op: ScalarResults[0])
8306 : getBuildVector(VT, DL, Ops: ScalarResults);
8307 NewSDValueDbgMsg(V, Msg: "New node fold constant vector: ", G: this);
8308 return V;
8309}
8310
8311SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL,
8312 SDVTList VTList,
8313 ArrayRef<SDValue> Ops,
8314 SDNodeFlags Flags) {
8315 if (VTList.NumVTs == 1)
8316 return FoldConstantArithmetic(Opcode, DL, VT: VTList.VTs[0], Ops, Flags);
8317
8318 switch (Opcode) {
8319 case ISD::SADDO:
8320 case ISD::UADDO:
8321 case ISD::SSUBO:
8322 case ISD::USUBO:
8323 case ISD::SMULO:
8324 case ISD::UMULO: {
8325 // TODO: Support non-splat vector constants.
8326 ConstantSDNode *C1 = isConstOrConstSplat(N: Ops[0]);
8327 ConstantSDNode *C2 = isConstOrConstSplat(N: Ops[1]);
8328 if (!C1 || !C2 || C1->isOpaque() || C2->isOpaque())
8329 return SDValue();
8330
8331 const APInt &V1 = C1->getAPIntValue(), &V2 = C2->getAPIntValue();
8332 bool Overflow;
8333 APInt Result;
8334 switch (Opcode) {
8335 default:
8336 llvm_unreachable("Unexpected overflow opcode");
8337 case ISD::SADDO:
8338 Result = V1.sadd_ov(RHS: V2, Overflow);
8339 break;
8340 case ISD::UADDO:
8341 Result = V1.uadd_ov(RHS: V2, Overflow);
8342 break;
8343 case ISD::SSUBO:
8344 Result = V1.ssub_ov(RHS: V2, Overflow);
8345 break;
8346 case ISD::USUBO:
8347 Result = V1.usub_ov(RHS: V2, Overflow);
8348 break;
8349 case ISD::SMULO:
8350 Result = V1.smul_ov(RHS: V2, Overflow);
8351 break;
8352 case ISD::UMULO:
8353 Result = V1.umul_ov(RHS: V2, Overflow);
8354 break;
8355 }
8356
8357 EVT VT = VTList.VTs[0];
8358 return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList,
8359 Ops: {getConstant(Val: Result, DL, VT),
8360 getBoolConstant(V: Overflow, DL, VT: VTList.VTs[1], OpVT: VT)},
8361 Flags);
8362 }
8363 }
8364 return SDValue();
8365}
8366
8367SDValue SelectionDAG::foldConstantFPMath(unsigned Opcode, const SDLoc &DL,
8368 EVT VT, ArrayRef<SDValue> Ops) {
8369 // TODO: Add support for unary/ternary fp opcodes.
8370 if (Ops.size() != 2)
8371 return SDValue();
8372
8373 // TODO: We don't do any constant folding for strict FP opcodes here, but we
8374 // should. That will require dealing with a potentially non-default
8375 // rounding mode, checking the "opStatus" return value from the APFloat
8376 // math calculations, and possibly other variations.
8377 SDValue N1 = Ops[0];
8378 SDValue N2 = Ops[1];
8379 ConstantFPSDNode *N1CFP = isConstOrConstSplatFP(N: N1, /*AllowUndefs*/ false);
8380 ConstantFPSDNode *N2CFP = isConstOrConstSplatFP(N: N2, /*AllowUndefs*/ false);
8381 if (N1CFP && N2CFP) {
8382 APFloat C1 = N1CFP->getValueAPF(); // make copy
8383 const APFloat &C2 = N2CFP->getValueAPF();
8384 switch (Opcode) {
8385 case ISD::FADD:
8386 C1.add(RHS: C2, RM: APFloat::rmNearestTiesToEven);
8387 return getConstantFP(V: C1, DL, VT);
8388 case ISD::FSUB:
8389 C1.subtract(RHS: C2, RM: APFloat::rmNearestTiesToEven);
8390 return getConstantFP(V: C1, DL, VT);
8391 case ISD::FMUL:
8392 C1.multiply(RHS: C2, RM: APFloat::rmNearestTiesToEven);
8393 return getConstantFP(V: C1, DL, VT);
8394 case ISD::FDIV:
8395 C1.divide(RHS: C2, RM: APFloat::rmNearestTiesToEven);
8396 return getConstantFP(V: C1, DL, VT);
8397 case ISD::FREM:
8398 C1.mod(RHS: C2);
8399 return getConstantFP(V: C1, DL, VT);
8400 case ISD::FCOPYSIGN:
8401 C1.copySign(RHS: C2);
8402 return getConstantFP(V: C1, DL, VT);
8403 case ISD::FMINNUM:
8404 return getConstantFP(V: minnum(A: C1, B: C2), DL, VT);
8405 case ISD::FMAXNUM:
8406 return getConstantFP(V: maxnum(A: C1, B: C2), DL, VT);
8407 case ISD::FMINIMUM:
8408 return getConstantFP(V: minimum(A: C1, B: C2), DL, VT);
8409 case ISD::FMAXIMUM:
8410 return getConstantFP(V: maximum(A: C1, B: C2), DL, VT);
8411 case ISD::FMINIMUMNUM:
8412 return getConstantFP(V: minimumnum(A: C1, B: C2), DL, VT);
8413 case ISD::FMAXIMUMNUM:
8414 return getConstantFP(V: maximumnum(A: C1, B: C2), DL, VT);
8415 default: break;
8416 }
8417 }
8418 if (N1CFP && Opcode == ISD::FP_ROUND) {
8419 APFloat C1 = N1CFP->getValueAPF(); // make copy
8420 bool Unused;
8421 // This can return overflow, underflow, or inexact; we don't care.
8422 // FIXME need to be more flexible about rounding mode.
8423 (void)C1.convert(ToSemantics: VT.getFltSemantics(), RM: APFloat::rmNearestTiesToEven,
8424 losesInfo: &Unused);
8425 return getConstantFP(V: C1, DL, VT);
8426 }
8427
8428 switch (Opcode) {
8429 case ISD::FSUB:
8430 // -0.0 - undef --> undef (consistent with "fneg undef")
8431 if (ConstantFPSDNode *N1C = isConstOrConstSplatFP(N: N1, /*AllowUndefs*/ true))
8432 if (N1C && N1C->getValueAPF().isNegZero() && N2.isUndef())
8433 return getUNDEF(VT);
8434 [[fallthrough]];
8435
8436 case ISD::FADD:
8437 case ISD::FMUL:
8438 case ISD::FDIV:
8439 case ISD::FREM:
8440 // If both operands are undef, the result is undef. If 1 operand is undef,
8441 // the result is NaN. This should match the behavior of the IR optimizer.
8442 if (N1.isUndef() && N2.isUndef())
8443 return getUNDEF(VT);
8444 if (N1.isUndef() || N2.isUndef())
8445 return getConstantFP(V: APFloat::getNaN(Sem: VT.getFltSemantics()), DL, VT);
8446 }
8447 return SDValue();
8448}
8449
8450SDValue SelectionDAG::FoldConstantBuildVector(BuildVectorSDNode *BV,
8451 const SDLoc &DL, EVT DstEltVT) {
8452 EVT SrcEltVT = BV->getValueType(ResNo: 0).getVectorElementType();
8453
8454 // If this is already the right type, we're done.
8455 if (SrcEltVT == DstEltVT)
8456 return SDValue(BV, 0);
8457
8458 unsigned SrcBitSize = SrcEltVT.getSizeInBits();
8459 unsigned DstBitSize = DstEltVT.getSizeInBits();
8460
8461 // If this is a conversion of N elements of one type to N elements of another
8462 // type, convert each element. This handles FP<->INT cases.
8463 if (SrcBitSize == DstBitSize) {
8464 SmallVector<SDValue, 8> Ops;
8465 for (SDValue Op : BV->op_values()) {
8466 // If the vector element type is not legal, the BUILD_VECTOR operands
8467 // are promoted and implicitly truncated. Make that explicit here.
8468 if (Op.getValueType() != SrcEltVT)
8469 Op = getNode(Opcode: ISD::TRUNCATE, DL, VT: SrcEltVT, N1: Op);
8470 Ops.push_back(Elt: getBitcast(VT: DstEltVT, V: Op));
8471 }
8472 EVT VT = EVT::getVectorVT(Context&: *getContext(), VT: DstEltVT,
8473 NumElements: BV->getValueType(ResNo: 0).getVectorNumElements());
8474 return getBuildVector(VT, DL, Ops);
8475 }
8476
8477 // Otherwise, we're growing or shrinking the elements. To avoid having to
8478 // handle annoying details of growing/shrinking FP values, we convert them to
8479 // int first.
8480 if (SrcEltVT.isFloatingPoint()) {
8481 // Convert the input float vector to a int vector where the elements are the
8482 // same sizes.
8483 EVT IntEltVT = EVT::getIntegerVT(Context&: *getContext(), BitWidth: SrcEltVT.getSizeInBits());
8484 if (SDValue Tmp = FoldConstantBuildVector(BV, DL, DstEltVT: IntEltVT))
8485 return FoldConstantBuildVector(BV: cast<BuildVectorSDNode>(Val&: Tmp), DL,
8486 DstEltVT);
8487 return SDValue();
8488 }
8489
8490 // Now we know the input is an integer vector. If the output is a FP type,
8491 // convert to integer first, then to FP of the right size.
8492 if (DstEltVT.isFloatingPoint()) {
8493 EVT IntEltVT = EVT::getIntegerVT(Context&: *getContext(), BitWidth: DstEltVT.getSizeInBits());
8494 if (SDValue Tmp = FoldConstantBuildVector(BV, DL, DstEltVT: IntEltVT))
8495 return FoldConstantBuildVector(BV: cast<BuildVectorSDNode>(Val&: Tmp), DL,
8496 DstEltVT);
8497 return SDValue();
8498 }
8499
8500 // Okay, we know the src/dst types are both integers of differing types.
8501 assert(SrcEltVT.isInteger() && DstEltVT.isInteger());
8502
8503 // Extract the constant raw bit data.
8504 BitVector UndefElements;
8505 SmallVector<APInt> RawBits;
8506 bool IsLE = getDataLayout().isLittleEndian();
8507 if (!BV->getConstantRawBits(IsLittleEndian: IsLE, DstEltSizeInBits: DstBitSize, RawBitElements&: RawBits, UndefElements))
8508 return SDValue();
8509
8510 SmallVector<SDValue, 8> Ops;
8511 for (unsigned I = 0, E = RawBits.size(); I != E; ++I) {
8512 if (UndefElements[I])
8513 Ops.push_back(Elt: getUNDEF(VT: DstEltVT));
8514 else
8515 Ops.push_back(Elt: getConstant(Val: RawBits[I], DL, VT: DstEltVT));
8516 }
8517
8518 EVT VT = EVT::getVectorVT(Context&: *getContext(), VT: DstEltVT, NumElements: Ops.size());
8519 return getBuildVector(VT, DL, Ops);
8520}
8521
8522SDValue SelectionDAG::getAssertAlign(const SDLoc &DL, SDValue Val, Align A) {
8523 assert(Val.getValueType().isInteger() && "Invalid AssertAlign!");
8524
8525 // There's no need to assert on a byte-aligned pointer. All pointers are at
8526 // least byte aligned.
8527 if (A == Align(1))
8528 return Val;
8529
8530 SDVTList VTs = getVTList(VT: Val.getValueType());
8531 SDValue Ops[] = {Val};
8532 SDNodeKey ID(ISD::AssertAlign, VTs, Ops);
8533 ID.AddInteger(I: A.value());
8534
8535 FoldingSetInsertToken InsertToken;
8536 if (SDNode *E = lookupNode(Key: ID, DL, InsertToken))
8537 return SDValue(E, 0);
8538
8539 auto *N =
8540 newSDNode<AssertAlignSDNode>(Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs, Args&: A);
8541 createOperands(Node: N, Vals: {Val});
8542
8543 CSEMap.insert(N, Token: InsertToken);
8544 InsertNode(N);
8545
8546 SDValue V(N, 0);
8547 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
8548 return V;
8549}
8550
8551SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
8552 SDValue N1, SDValue N2) {
8553 SDNodeFlags Flags;
8554 if (Inserter)
8555 Flags = Inserter->getFlags();
8556 return getNode(Opcode, DL, VT, N1, N2, Flags);
8557}
8558
8559void SelectionDAG::canonicalizeCommutativeBinop(unsigned Opcode, SDValue &N1,
8560 SDValue &N2) const {
8561 if (!TLI->isCommutativeBinOp(Opcode))
8562 return;
8563
8564 // Canonicalize:
8565 // binop(const, nonconst) -> binop(nonconst, const)
8566 bool N1C = isConstantIntBuildVectorOrConstantInt(N: N1);
8567 bool N2C = isConstantIntBuildVectorOrConstantInt(N: N2);
8568 bool N1CFP = isConstantFPBuildVectorOrConstantFP(N: N1);
8569 bool N2CFP = isConstantFPBuildVectorOrConstantFP(N: N2);
8570 if ((N1C && !N2C) || (N1CFP && !N2CFP))
8571 std::swap(a&: N1, b&: N2);
8572
8573 // Canonicalize:
8574 // binop(splat(x), step_vector) -> binop(step_vector, splat(x))
8575 else if (N1.getOpcode() == ISD::SPLAT_VECTOR &&
8576 N2.getOpcode() == ISD::STEP_VECTOR)
8577 std::swap(a&: N1, b&: N2);
8578}
8579
8580SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
8581 SDValue N1, SDValue N2, const SDNodeFlags Flags) {
8582 assert(N1.getOpcode() != ISD::DELETED_NODE &&
8583 N2.getOpcode() != ISD::DELETED_NODE &&
8584 "Operand is DELETED_NODE!");
8585
8586 canonicalizeCommutativeBinop(Opcode, N1, N2);
8587
8588 auto *N1C = dyn_cast<ConstantSDNode>(Val&: N1);
8589 auto *N2C = dyn_cast<ConstantSDNode>(Val&: N2);
8590
8591 // Don't allow undefs in vector splats - we might be returning N2 when folding
8592 // to zero etc.
8593 ConstantSDNode *N2CV =
8594 isConstOrConstSplat(N: N2, /*AllowUndefs*/ false, /*AllowTruncation*/ true);
8595
8596 switch (Opcode) {
8597 default: break;
8598 case ISD::TokenFactor:
8599 assert(VT == MVT::Other && N1.getValueType() == MVT::Other &&
8600 N2.getValueType() == MVT::Other && "Invalid token factor!");
8601 // Fold trivial token factors.
8602 if (N1.getOpcode() == ISD::EntryToken) return N2;
8603 if (N2.getOpcode() == ISD::EntryToken) return N1;
8604 if (N1 == N2) return N1;
8605 break;
8606 case ISD::BUILD_VECTOR: {
8607 // Attempt to simplify BUILD_VECTOR.
8608 SDValue Ops[] = {N1, N2};
8609 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, DAG&: *this))
8610 return V;
8611 break;
8612 }
8613 case ISD::CONCAT_VECTORS: {
8614 SDValue Ops[] = {N1, N2};
8615 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, DAG&: *this))
8616 return V;
8617 break;
8618 }
8619 case ISD::AND:
8620 assert(VT.isInteger() && "This operator does not apply to FP types!");
8621 assert(N1.getValueType() == N2.getValueType() &&
8622 N1.getValueType() == VT && "Binary operator types must match!");
8623 // (X & 0) -> 0. This commonly occurs when legalizing i64 values, so it's
8624 // worth handling here.
8625 if (N2CV && N2CV->isZero())
8626 return N2;
8627 if (N2CV && N2CV->isAllOnes()) // X & -1 -> X
8628 return N1;
8629 break;
8630 case ISD::OR:
8631 case ISD::XOR:
8632 case ISD::ADD:
8633 case ISD::PTRADD:
8634 case ISD::SUB:
8635 assert(VT.isInteger() && "This operator does not apply to FP types!");
8636 assert(N1.getValueType() == N2.getValueType() &&
8637 N1.getValueType() == VT && "Binary operator types must match!");
8638 // The equal operand types requirement is unnecessarily strong for PTRADD.
8639 // However, the SelectionDAGBuilder does not generate PTRADDs with different
8640 // operand types, and we'd need to re-implement GEP's non-standard wrapping
8641 // logic everywhere where PTRADDs may be folded or combined to properly
8642 // support them. If/when we introduce pointer types to the SDAG, we will
8643 // need to relax this constraint.
8644
8645 // (X ^|+- 0) -> X. This commonly occurs when legalizing i64 values, so
8646 // it's worth handling here.
8647 if (N2CV && N2CV->isZero())
8648 return N1;
8649 if ((Opcode == ISD::ADD || Opcode == ISD::SUB) &&
8650 VT.getScalarType() == MVT::i1)
8651 return getNode(Opcode: ISD::XOR, DL, VT, N1, N2);
8652 // Fold (add (vscale * C0), (vscale * C1)) to (vscale * (C0 + C1)).
8653 if (Opcode == ISD::ADD && N1.getOpcode() == ISD::VSCALE &&
8654 N2.getOpcode() == ISD::VSCALE) {
8655 const APInt &C1 = N1->getConstantOperandAPInt(Num: 0);
8656 const APInt &C2 = N2->getConstantOperandAPInt(Num: 0);
8657 return getVScale(DL, VT, MulImm: C1 + C2);
8658 }
8659 break;
8660 case ISD::MUL:
8661 assert(VT.isInteger() && "This operator does not apply to FP types!");
8662 assert(N1.getValueType() == N2.getValueType() &&
8663 N1.getValueType() == VT && "Binary operator types must match!");
8664 if (VT.getScalarType() == MVT::i1)
8665 return getNode(Opcode: ISD::AND, DL, VT, N1, N2);
8666 if (N2CV && N2CV->isZero())
8667 return N2;
8668 if (N2C && (N1.getOpcode() == ISD::VSCALE) && Flags.hasNoSignedWrap()) {
8669 const APInt &MulImm = N1->getConstantOperandAPInt(Num: 0);
8670 const APInt &N2CImm = N2C->getAPIntValue();
8671 return getVScale(DL, VT, MulImm: MulImm * N2CImm);
8672 }
8673 break;
8674 case ISD::UDIV:
8675 case ISD::UREM:
8676 case ISD::MULHU:
8677 case ISD::MULHS:
8678 case ISD::SDIV:
8679 case ISD::SREM:
8680 case ISD::SADDSAT:
8681 case ISD::SSUBSAT:
8682 case ISD::UADDSAT:
8683 case ISD::USUBSAT:
8684 assert(VT.isInteger() && "This operator does not apply to FP types!");
8685 assert(N1.getValueType() == N2.getValueType() &&
8686 N1.getValueType() == VT && "Binary operator types must match!");
8687 if (VT.getScalarType() == MVT::i1) {
8688 // fold (add_sat x, y) -> (or x, y) for bool types.
8689 if (Opcode == ISD::SADDSAT || Opcode == ISD::UADDSAT)
8690 return getNode(Opcode: ISD::OR, DL, VT, N1, N2);
8691 // fold (sub_sat x, y) -> (and x, ~y) for bool types.
8692 if (Opcode == ISD::SSUBSAT || Opcode == ISD::USUBSAT)
8693 return getNode(Opcode: ISD::AND, DL, VT, N1, N2: getNOT(DL, Val: N2, VT));
8694 }
8695 break;
8696 case ISD::SCMP:
8697 case ISD::UCMP:
8698 assert(N1.getValueType() == N2.getValueType() &&
8699 "Types of operands of UCMP/SCMP must match");
8700 assert(N1.getValueType().isVector() == VT.isVector() &&
8701 "Operands and return type of must both be scalars or vectors");
8702 if (VT.isVector())
8703 assert(VT.getVectorElementCount() ==
8704 N1.getValueType().getVectorElementCount() &&
8705 "Result and operands must have the same number of elements");
8706 break;
8707 case ISD::AVGFLOORS:
8708 case ISD::AVGFLOORU:
8709 case ISD::AVGCEILS:
8710 case ISD::AVGCEILU:
8711 case ISD::AVGSUBFLOORS:
8712 case ISD::AVGSUBFLOORU:
8713 assert(VT.isInteger() && "This operator does not apply to FP types!");
8714 assert(N1.getValueType() == N2.getValueType() &&
8715 N1.getValueType() == VT && "Binary operator types must match!");
8716 break;
8717 case ISD::ABDS:
8718 case ISD::ABDU:
8719 assert(VT.isInteger() && "This operator does not apply to FP types!");
8720 assert(N1.getValueType() == N2.getValueType() &&
8721 N1.getValueType() == VT && "Binary operator types must match!");
8722 if (VT.getScalarType() == MVT::i1)
8723 return getNode(Opcode: ISD::XOR, DL, VT, N1, N2);
8724 break;
8725 case ISD::SMIN:
8726 case ISD::UMAX:
8727 assert(VT.isInteger() && "This operator does not apply to FP types!");
8728 assert(N1.getValueType() == N2.getValueType() &&
8729 N1.getValueType() == VT && "Binary operator types must match!");
8730 if (VT.getScalarType() == MVT::i1)
8731 return getNode(Opcode: ISD::OR, DL, VT, N1, N2);
8732 break;
8733 case ISD::SMAX:
8734 case ISD::UMIN:
8735 assert(VT.isInteger() && "This operator does not apply to FP types!");
8736 assert(N1.getValueType() == N2.getValueType() &&
8737 N1.getValueType() == VT && "Binary operator types must match!");
8738 if (VT.getScalarType() == MVT::i1)
8739 return getNode(Opcode: ISD::AND, DL, VT, N1, N2);
8740 break;
8741 case ISD::FADD:
8742 case ISD::FSUB:
8743 case ISD::FMUL:
8744 case ISD::FDIV:
8745 case ISD::FREM:
8746 assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
8747 assert(N1.getValueType() == N2.getValueType() &&
8748 N1.getValueType() == VT && "Binary operator types must match!");
8749 if (SDValue V = simplifyFPBinop(Opcode, X: N1, Y: N2, Flags))
8750 return V;
8751 break;
8752 case ISD::FCOPYSIGN: // N1 and result must match. N1/N2 need not match.
8753 assert(N1.getValueType() == VT &&
8754 N1.getValueType().isFloatingPoint() &&
8755 N2.getValueType().isFloatingPoint() &&
8756 "Invalid FCOPYSIGN!");
8757 break;
8758 case ISD::SHL:
8759 if (N2C && (N1.getOpcode() == ISD::VSCALE) && Flags.hasNoSignedWrap()) {
8760 const APInt &MulImm = N1->getConstantOperandAPInt(Num: 0);
8761 const APInt &ShiftImm = N2C->getAPIntValue();
8762 return getVScale(DL, VT, MulImm: MulImm << ShiftImm);
8763 }
8764 [[fallthrough]];
8765 case ISD::SRA:
8766 case ISD::SRL:
8767 if (SDValue V = simplifyShift(X: N1, Y: N2))
8768 return V;
8769 [[fallthrough]];
8770 case ISD::ROTL:
8771 case ISD::ROTR:
8772 case ISD::SSHLSAT:
8773 case ISD::USHLSAT:
8774 assert(VT == N1.getValueType() &&
8775 "Shift operators return type must be the same as their first arg");
8776 assert(VT.isInteger() && N2.getValueType().isInteger() &&
8777 "Shifts only work on integers");
8778 assert((!VT.isVector() || VT == N2.getValueType()) &&
8779 "Vector shift amounts must be in the same as their first arg");
8780 // Verify that the shift amount VT is big enough to hold valid shift
8781 // amounts. This catches things like trying to shift an i1024 value by an
8782 // i8, which is easy to fall into in generic code that uses
8783 // TLI.getShiftAmount().
8784 assert(N2.getValueType().getScalarSizeInBits() >=
8785 Log2_32_Ceil(VT.getScalarSizeInBits()) &&
8786 "Invalid use of small shift amount with oversized value!");
8787
8788 // Always fold shifts of i1 values so the code generator doesn't need to
8789 // handle them. Since we know the size of the shift has to be less than the
8790 // size of the value, the shift/rotate count is guaranteed to be zero.
8791 if (VT == MVT::i1)
8792 return N1;
8793 if (N2CV && N2CV->isZero())
8794 return N1;
8795 break;
8796 case ISD::FP_ROUND:
8797 assert(VT.isFloatingPoint() && N1.getValueType().isFloatingPoint() &&
8798 VT.bitsLE(N1.getValueType()) && N2C &&
8799 (N2C->getZExtValue() == 0 || N2C->getZExtValue() == 1) &&
8800 N2.getOpcode() == ISD::TargetConstant && "Invalid FP_ROUND!");
8801 if (N1.getValueType() == VT) return N1; // noop conversion.
8802 break;
8803 case ISD::IS_FPCLASS: {
8804 assert(N1.getValueType().isFloatingPoint() &&
8805 "IS_FPCLASS is used for a non-floating type");
8806 assert(isa<ConstantSDNode>(N2) && "FPClassTest is not Constant");
8807 // is.fpclass(poison, mask) -> poison
8808 if (N1.getOpcode() == ISD::POISON)
8809 return getPOISON(VT);
8810 FPClassTest Mask = static_cast<FPClassTest>(N2->getAsZExtVal());
8811 // If all tests are made, it doesn't matter what the value is.
8812 if ((Mask & fcAllFlags) == fcAllFlags)
8813 return getBoolConstant(V: true, DL, VT, OpVT: N1.getValueType());
8814 if ((Mask & fcAllFlags) == 0)
8815 return getBoolConstant(V: false, DL, VT, OpVT: N1.getValueType());
8816 break;
8817 }
8818 case ISD::AssertNoFPClass: {
8819 assert(N1.getValueType().isFloatingPoint() &&
8820 "AssertNoFPClass is used for a non-floating type");
8821 assert(isa<ConstantSDNode>(N2) && "NoFPClass is not Constant");
8822 FPClassTest NoFPClass = static_cast<FPClassTest>(N2->getAsZExtVal());
8823 assert(llvm::to_underlying(NoFPClass) <=
8824 BitmaskEnumDetail::Mask<FPClassTest>() &&
8825 "FPClassTest value too large");
8826 (void)NoFPClass;
8827 break;
8828 }
8829 case ISD::AssertSext:
8830 case ISD::AssertZext: {
8831 EVT EVT = cast<VTSDNode>(Val&: N2)->getVT();
8832 assert(VT == N1.getValueType() && "Not an inreg extend!");
8833 assert(VT.isInteger() && EVT.isInteger() &&
8834 "Cannot *_EXTEND_INREG FP types");
8835 assert(!EVT.isVector() &&
8836 "AssertSExt/AssertZExt type should be the vector element type "
8837 "rather than the vector type!");
8838 assert(EVT.bitsLE(VT.getScalarType()) && "Not extending!");
8839 if (VT.getScalarType() == EVT) return N1; // noop assertion.
8840 break;
8841 }
8842 case ISD::SIGN_EXTEND_INREG: {
8843 EVT EVT = cast<VTSDNode>(Val&: N2)->getVT();
8844 assert(VT == N1.getValueType() && "Not an inreg extend!");
8845 assert(VT.isInteger() && EVT.isInteger() &&
8846 "Cannot *_EXTEND_INREG FP types");
8847 assert(EVT.isVector() == VT.isVector() &&
8848 "SIGN_EXTEND_INREG type should be vector iff the operand "
8849 "type is vector!");
8850 assert((!EVT.isVector() ||
8851 EVT.getVectorElementCount() == VT.getVectorElementCount()) &&
8852 "Vector element counts must match in SIGN_EXTEND_INREG");
8853 assert(EVT.getScalarType().bitsLE(VT.getScalarType()) && "Not extending!");
8854 if (EVT == VT) return N1; // Not actually extending
8855 break;
8856 }
8857 case ISD::FP_TO_SINT_SAT:
8858 case ISD::FP_TO_UINT_SAT: {
8859 assert(VT.isInteger() && cast<VTSDNode>(N2)->getVT().isInteger() &&
8860 N1.getValueType().isFloatingPoint() && "Invalid FP_TO_*INT_SAT");
8861 assert(N1.getValueType().isVector() == VT.isVector() &&
8862 "FP_TO_*INT_SAT type should be vector iff the operand type is "
8863 "vector!");
8864 assert((!VT.isVector() || VT.getVectorElementCount() ==
8865 N1.getValueType().getVectorElementCount()) &&
8866 "Vector element counts must match in FP_TO_*INT_SAT");
8867 assert(!cast<VTSDNode>(N2)->getVT().isVector() &&
8868 "Type to saturate to must be a scalar.");
8869 assert(cast<VTSDNode>(N2)->getVT().bitsLE(VT.getScalarType()) &&
8870 "Not extending!");
8871 break;
8872 }
8873 case ISD::EXTRACT_VECTOR_ELT:
8874 assert(VT.getSizeInBits() >= N1.getValueType().getScalarSizeInBits() &&
8875 "The result of EXTRACT_VECTOR_ELT must be at least as wide as the \
8876 element type of the vector.");
8877
8878 // Extract from an undefined value or using an undefined index is undefined.
8879 if (N1.isUndef() || N2.isUndef())
8880 return getUNDEF(VT);
8881
8882 // EXTRACT_VECTOR_ELT of out-of-bounds element is POISON for fixed length
8883 // vectors. For scalable vectors we will provide appropriate support for
8884 // dealing with arbitrary indices.
8885 if (N2C && N1.getValueType().isFixedLengthVector() &&
8886 N2C->getAPIntValue().uge(RHS: N1.getValueType().getVectorNumElements()))
8887 return getPOISON(VT);
8888
8889 // EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is
8890 // expanding copies of large vectors from registers. This only works for
8891 // fixed length vectors, since we need to know the exact number of
8892 // elements.
8893 if (N2C && N1.getOpcode() == ISD::CONCAT_VECTORS &&
8894 N1.getOperand(i: 0).getValueType().isFixedLengthVector()) {
8895 unsigned Factor = N1.getOperand(i: 0).getValueType().getVectorNumElements();
8896 return getExtractVectorElt(DL, VT,
8897 Vec: N1.getOperand(i: N2C->getZExtValue() / Factor),
8898 Idx: N2C->getZExtValue() % Factor);
8899 }
8900
8901 // EXTRACT_VECTOR_ELT of BUILD_VECTOR or SPLAT_VECTOR is often formed while
8902 // lowering is expanding large vector constants.
8903 if (N2C && (N1.getOpcode() == ISD::BUILD_VECTOR ||
8904 N1.getOpcode() == ISD::SPLAT_VECTOR)) {
8905 assert((N1.getOpcode() != ISD::BUILD_VECTOR ||
8906 N1.getValueType().isFixedLengthVector()) &&
8907 "BUILD_VECTOR used for scalable vectors");
8908 unsigned Index =
8909 N1.getOpcode() == ISD::BUILD_VECTOR ? N2C->getZExtValue() : 0;
8910 SDValue Elt = N1.getOperand(i: Index);
8911
8912 if (VT != Elt.getValueType())
8913 // If the vector element type is not legal, the BUILD_VECTOR operands
8914 // are promoted and implicitly truncated, and the result implicitly
8915 // extended. Make that explicit here.
8916 Elt = getAnyExtOrTrunc(Op: Elt, DL, VT);
8917
8918 return Elt;
8919 }
8920
8921 // EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector
8922 // operations are lowered to scalars.
8923 if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT) {
8924 // If the indices are the same, return the inserted element else
8925 // if the indices are known different, extract the element from
8926 // the original vector.
8927 SDValue N1Op2 = N1.getOperand(i: 2);
8928 ConstantSDNode *N1Op2C = dyn_cast<ConstantSDNode>(Val&: N1Op2);
8929
8930 if (N1Op2C && N2C) {
8931 if (N1Op2C->getZExtValue() == N2C->getZExtValue()) {
8932 if (VT == N1.getOperand(i: 1).getValueType())
8933 return N1.getOperand(i: 1);
8934 if (VT.isFloatingPoint()) {
8935 assert(VT.getSizeInBits() > N1.getOperand(1).getValueType().getSizeInBits());
8936 return getFPExtendOrRound(Op: N1.getOperand(i: 1), DL, VT);
8937 }
8938 return getSExtOrTrunc(Op: N1.getOperand(i: 1), DL, VT);
8939 }
8940 return getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT, N1: N1.getOperand(i: 0), N2);
8941 }
8942 }
8943
8944 // EXTRACT_VECTOR_ELT of v1iX EXTRACT_SUBVECTOR could be formed
8945 // when vector types are scalarized and v1iX is legal.
8946 // vextract (v1iX extract_subvector(vNiX, Idx)) -> vextract(vNiX,Idx).
8947 // Here we are completely ignoring the extract element index (N2),
8948 // which is fine for fixed width vectors, since any index other than 0
8949 // is undefined anyway. However, this cannot be ignored for scalable
8950 // vectors - in theory we could support this, but we don't want to do this
8951 // without a profitability check.
8952 if (N1.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
8953 N1.getValueType().isFixedLengthVector() &&
8954 N1.getValueType().getVectorNumElements() == 1) {
8955 return getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT, N1: N1.getOperand(i: 0),
8956 N2: N1.getOperand(i: 1));
8957 }
8958 break;
8959 case ISD::EXTRACT_ELEMENT:
8960 assert(N2C && (unsigned)N2C->getZExtValue() < 2 && "Bad EXTRACT_ELEMENT!");
8961 assert(!N1.getValueType().isVector() && !VT.isVector() &&
8962 (N1.getValueType().isInteger() == VT.isInteger()) &&
8963 N1.getValueType() != VT &&
8964 "Wrong types for EXTRACT_ELEMENT!");
8965
8966 // EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding
8967 // 64-bit integers into 32-bit parts. Instead of building the extract of
8968 // the BUILD_PAIR, only to have legalize rip it apart, just do it now.
8969 if (N1.getOpcode() == ISD::BUILD_PAIR)
8970 return N1.getOperand(i: N2C->getZExtValue());
8971
8972 // EXTRACT_ELEMENT of a constant int is also very common.
8973 if (N1C) {
8974 unsigned ElementSize = VT.getSizeInBits();
8975 unsigned Shift = ElementSize * N2C->getZExtValue();
8976 const APInt &Val = N1C->getAPIntValue();
8977 return getConstant(Val: Val.extractBits(numBits: ElementSize, bitPosition: Shift), DL, VT);
8978 }
8979 break;
8980 case ISD::EXTRACT_SUBVECTOR: {
8981 EVT N1VT = N1.getValueType();
8982 assert(VT.isVector() && N1VT.isVector() &&
8983 "Extract subvector VTs must be vectors!");
8984 assert(VT.getVectorElementType() == N1VT.getVectorElementType() &&
8985 "Extract subvector VTs must have the same element type!");
8986 assert((VT.isFixedLengthVector() || N1VT.isScalableVector()) &&
8987 "Cannot extract a scalable vector from a fixed length vector!");
8988 assert((VT.isScalableVector() != N1VT.isScalableVector() ||
8989 VT.getVectorMinNumElements() <= N1VT.getVectorMinNumElements()) &&
8990 "Extract subvector must be from larger vector to smaller vector!");
8991 assert(N2C && "Extract subvector index must be a constant");
8992 assert((VT.isScalableVector() != N1VT.isScalableVector() ||
8993 (VT.getVectorMinNumElements() + N2C->getZExtValue()) <=
8994 N1VT.getVectorMinNumElements()) &&
8995 "Extract subvector overflow!");
8996 assert(N2C->getAPIntValue().getBitWidth() ==
8997 TLI->getVectorIdxWidth(getDataLayout()) &&
8998 "Constant index for EXTRACT_SUBVECTOR has an invalid size");
8999 assert(N2C->getZExtValue() % VT.getVectorMinNumElements() == 0 &&
9000 "Extract index is not a multiple of the output vector length");
9001
9002 // Trivial extraction.
9003 if (VT == N1VT)
9004 return N1;
9005
9006 // EXTRACT_SUBVECTOR of an UNDEF is an UNDEF.
9007 if (N1.isUndef())
9008 return getUNDEF(VT);
9009
9010 // EXTRACT_SUBVECTOR of CONCAT_VECTOR can be simplified if the pieces of
9011 // the concat have the same type as the extract.
9012 if (N1.getOpcode() == ISD::CONCAT_VECTORS &&
9013 VT == N1.getOperand(i: 0).getValueType()) {
9014 unsigned Factor = VT.getVectorMinNumElements();
9015 return N1.getOperand(i: N2C->getZExtValue() / Factor);
9016 }
9017
9018 // EXTRACT_SUBVECTOR of INSERT_SUBVECTOR is often created
9019 // during shuffle legalization.
9020 if (N1.getOpcode() == ISD::INSERT_SUBVECTOR && N2 == N1.getOperand(i: 2) &&
9021 VT == N1.getOperand(i: 1).getValueType())
9022 return N1.getOperand(i: 1);
9023 break;
9024 }
9025 }
9026
9027 if (N1.getOpcode() == ISD::POISON || N2.getOpcode() == ISD::POISON) {
9028 switch (Opcode) {
9029 case ISD::XOR:
9030 case ISD::ADD:
9031 case ISD::PTRADD:
9032 case ISD::SUB:
9033 case ISD::SIGN_EXTEND_INREG:
9034 case ISD::UDIV:
9035 case ISD::SDIV:
9036 case ISD::UREM:
9037 case ISD::SREM:
9038 case ISD::MUL:
9039 case ISD::AND:
9040 case ISD::SSUBSAT:
9041 case ISD::USUBSAT:
9042 case ISD::UMIN:
9043 case ISD::OR:
9044 case ISD::SADDSAT:
9045 case ISD::UADDSAT:
9046 case ISD::UMAX:
9047 case ISD::SMAX:
9048 case ISD::SMIN:
9049 // fold op(arg1, poison) -> poison, fold op(poison, arg2) -> poison.
9050 return N2.getOpcode() == ISD::POISON ? N2 : N1;
9051 }
9052 }
9053
9054 // Canonicalize an UNDEF to the RHS, even over a constant.
9055 if (N1.getOpcode() == ISD::UNDEF && N2.getOpcode() != ISD::UNDEF) {
9056 if (TLI->isCommutativeBinOp(Opcode)) {
9057 std::swap(a&: N1, b&: N2);
9058 } else {
9059 switch (Opcode) {
9060 case ISD::PTRADD:
9061 case ISD::SUB:
9062 // fold op(undef, non_undef_arg2) -> undef.
9063 return N1;
9064 case ISD::SIGN_EXTEND_INREG:
9065 case ISD::UDIV:
9066 case ISD::SDIV:
9067 case ISD::UREM:
9068 case ISD::SREM:
9069 case ISD::SSUBSAT:
9070 case ISD::USUBSAT:
9071 // fold op(undef, non_undef_arg2) -> 0.
9072 return getConstant(Val: 0, DL, VT);
9073 }
9074 }
9075 }
9076
9077 // Fold a bunch of operators when the RHS is undef.
9078 if (N2.getOpcode() == ISD::UNDEF) {
9079 switch (Opcode) {
9080 case ISD::XOR:
9081 if (N1.getOpcode() == ISD::UNDEF)
9082 // Handle undef ^ undef -> 0 special case. This is a common
9083 // idiom (misuse).
9084 return getConstant(Val: 0, DL, VT);
9085 [[fallthrough]];
9086 case ISD::ADD:
9087 case ISD::PTRADD:
9088 case ISD::SUB:
9089 // fold op(arg1, undef) -> undef.
9090 return N2;
9091 case ISD::UDIV:
9092 case ISD::SDIV:
9093 case ISD::UREM:
9094 case ISD::SREM:
9095 // fold op(arg1, undef) -> poison.
9096 return getPOISON(VT);
9097 case ISD::MUL:
9098 case ISD::AND:
9099 case ISD::SSUBSAT:
9100 case ISD::USUBSAT:
9101 case ISD::UMIN:
9102 // fold op(undef, undef) -> undef, fold op(arg1, undef) -> 0.
9103 return N1.getOpcode() == ISD::UNDEF ? N2 : getConstant(Val: 0, DL, VT);
9104 case ISD::OR:
9105 case ISD::SADDSAT:
9106 case ISD::UADDSAT:
9107 case ISD::UMAX:
9108 // fold op(undef, undef) -> undef, fold op(arg1, undef) -> -1.
9109 return N1.getOpcode() == ISD::UNDEF ? N2 : getAllOnesConstant(DL, VT);
9110 case ISD::SMAX:
9111 // fold op(undef, undef) -> undef, fold op(arg1, undef) -> MAX_INT.
9112 return N1.getOpcode() == ISD::UNDEF
9113 ? N2
9114 : getConstant(
9115 Val: APInt::getSignedMaxValue(numBits: VT.getScalarSizeInBits()), DL,
9116 VT);
9117 case ISD::SMIN:
9118 // fold op(undef, undef) -> undef, fold op(arg1, undef) -> MIN_INT.
9119 return N1.getOpcode() == ISD::UNDEF
9120 ? N2
9121 : getConstant(
9122 Val: APInt::getSignedMinValue(numBits: VT.getScalarSizeInBits()), DL,
9123 VT);
9124 }
9125 }
9126
9127 // Perform trivial constant folding.
9128 if (SDValue SV = FoldConstantArithmetic(Opcode, DL, VT, Ops: {N1, N2}, Flags))
9129 return SV;
9130
9131 // Memoize this node if possible.
9132 SDNode *N;
9133 SDVTList VTs = getVTList(VT);
9134 SDValue Ops[] = {N1, N2};
9135 if (VT != MVT::Glue) {
9136 SDNodeKey ID(Opcode, VTs, Ops);
9137 FoldingSetInsertToken InsertToken;
9138 if (SDNode *E = lookupNode(Key: ID, DL, InsertToken)) {
9139 E->intersectFlagsWith(Flags);
9140 return SDValue(E, 0);
9141 }
9142
9143 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
9144 N->setFlags(Flags);
9145 createOperands(Node: N, Vals: Ops);
9146 CSEMap.insert(N, Token: InsertToken);
9147 } else {
9148 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
9149 createOperands(Node: N, Vals: Ops);
9150 }
9151
9152 InsertNode(N);
9153 SDValue V = SDValue(N, 0);
9154 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
9155 return V;
9156}
9157
9158SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
9159 SDValue N1, SDValue N2, SDValue N3) {
9160 SDNodeFlags Flags;
9161 if (Inserter)
9162 Flags = Inserter->getFlags();
9163 return getNode(Opcode, DL, VT, N1, N2, N3, Flags);
9164}
9165
9166SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
9167 SDValue N1, SDValue N2, SDValue N3,
9168 const SDNodeFlags Flags) {
9169 assert(N1.getOpcode() != ISD::DELETED_NODE &&
9170 N2.getOpcode() != ISD::DELETED_NODE &&
9171 N3.getOpcode() != ISD::DELETED_NODE &&
9172 "Operand is DELETED_NODE!");
9173 // Perform various simplifications.
9174 switch (Opcode) {
9175 case ISD::BUILD_VECTOR: {
9176 // Attempt to simplify BUILD_VECTOR.
9177 SDValue Ops[] = {N1, N2, N3};
9178 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, DAG&: *this))
9179 return V;
9180 break;
9181 }
9182 case ISD::CONCAT_VECTORS: {
9183 SDValue Ops[] = {N1, N2, N3};
9184 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, DAG&: *this))
9185 return V;
9186 break;
9187 }
9188 case ISD::SETCC: {
9189 assert(VT.isInteger() && "SETCC result type must be an integer!");
9190 assert(N1.getValueType() == N2.getValueType() &&
9191 "SETCC operands must have the same type!");
9192 assert(VT.isVector() == N1.getValueType().isVector() &&
9193 "SETCC type should be vector iff the operand type is vector!");
9194 assert((!VT.isVector() || VT.getVectorElementCount() ==
9195 N1.getValueType().getVectorElementCount()) &&
9196 "SETCC vector element counts must match!");
9197 // Use FoldSetCC to simplify SETCC's.
9198 if (SDValue V =
9199 FoldSetCC(VT, N1, N2, Cond: cast<CondCodeSDNode>(Val&: N3)->get(), dl: DL, Flags))
9200 return V;
9201 break;
9202 }
9203 case ISD::SELECT:
9204 case ISD::VSELECT:
9205 if (SDValue V = simplifySelect(Cond: N1, TVal: N2, FVal: N3))
9206 return V;
9207 break;
9208 case ISD::VECTOR_SHUFFLE:
9209 llvm_unreachable("should use getVectorShuffle constructor!");
9210 case ISD::VECTOR_SPLICE_LEFT:
9211 if (isNullConstant(V: N3))
9212 return N1;
9213 break;
9214 case ISD::VECTOR_SPLICE_RIGHT:
9215 if (isNullConstant(V: N3))
9216 return N2;
9217 break;
9218 case ISD::INSERT_VECTOR_ELT: {
9219 assert(VT.isVector() && VT == N1.getValueType() &&
9220 "INSERT_VECTOR_ELT vector type mismatch");
9221 assert(VT.isFloatingPoint() == N2.getValueType().isFloatingPoint() &&
9222 "INSERT_VECTOR_ELT scalar fp/int mismatch");
9223 assert((!VT.isFloatingPoint() ||
9224 VT.getVectorElementType() == N2.getValueType()) &&
9225 "INSERT_VECTOR_ELT fp scalar type mismatch");
9226 assert((!VT.isInteger() ||
9227 VT.getScalarSizeInBits() <= N2.getScalarValueSizeInBits()) &&
9228 "INSERT_VECTOR_ELT int scalar size mismatch");
9229
9230 auto *N3C = dyn_cast<ConstantSDNode>(Val&: N3);
9231 // INSERT_VECTOR_ELT into out-of-bounds element is an UNDEF, except
9232 // for scalable vectors where we will generate appropriate code to
9233 // deal with out-of-bounds cases correctly.
9234 if (N3C && VT.isFixedLengthVector() &&
9235 N3C->getZExtValue() >= VT.getVectorNumElements())
9236 return getUNDEF(VT);
9237
9238 // Undefined index can be assumed out-of-bounds, so that's UNDEF too.
9239 if (N3.isUndef())
9240 return getUNDEF(VT);
9241
9242 // If inserting poison, just use the input vector.
9243 if (N2.getOpcode() == ISD::POISON)
9244 return N1;
9245
9246 // Inserting undef into undef/poison is still undef.
9247 if (N2.getOpcode() == ISD::UNDEF && N1.isUndef())
9248 return getUNDEF(VT);
9249
9250 // If the inserted element is an UNDEF, just use the input vector.
9251 // But not if skipping the insert could make the result more poisonous.
9252 if (N2.isUndef()) {
9253 if (N3C && VT.isFixedLengthVector()) {
9254 APInt EltMask =
9255 APInt::getOneBitSet(numBits: VT.getVectorNumElements(), BitNo: N3C->getZExtValue());
9256 if (isGuaranteedNotToBePoison(Op: N1, DemandedElts: EltMask))
9257 return N1;
9258 } else if (isGuaranteedNotToBePoison(Op: N1))
9259 return N1;
9260 }
9261 break;
9262 }
9263 case ISD::INSERT_SUBVECTOR: {
9264 // If inserting poison, just use the input vector,
9265 if (N2.getOpcode() == ISD::POISON)
9266 return N1;
9267
9268 // Inserting undef into undef/poison is still undef.
9269 if (N2.getOpcode() == ISD::UNDEF && N1.isUndef())
9270 return getUNDEF(VT);
9271
9272 EVT N2VT = N2.getValueType();
9273 assert(VT == N1.getValueType() &&
9274 "Dest and insert subvector source types must match!");
9275 assert(VT.isVector() && N2VT.isVector() &&
9276 "Insert subvector VTs must be vectors!");
9277 assert(VT.getVectorElementType() == N2VT.getVectorElementType() &&
9278 "Insert subvector VTs must have the same element type!");
9279 assert((VT.isScalableVector() || N2VT.isFixedLengthVector()) &&
9280 "Cannot insert a scalable vector into a fixed length vector!");
9281 assert((VT.isScalableVector() != N2VT.isScalableVector() ||
9282 VT.getVectorMinNumElements() >= N2VT.getVectorMinNumElements()) &&
9283 "Insert subvector must be from smaller vector to larger vector!");
9284 assert(isa<ConstantSDNode>(N3) &&
9285 "Insert subvector index must be constant");
9286 assert((VT.isScalableVector() != N2VT.isScalableVector() ||
9287 (N2VT.getVectorMinNumElements() + N3->getAsZExtVal()) <=
9288 VT.getVectorMinNumElements()) &&
9289 "Insert subvector overflow!");
9290 assert(N3->getAsAPIntVal().getBitWidth() ==
9291 TLI->getVectorIdxWidth(getDataLayout()) &&
9292 "Constant index for INSERT_SUBVECTOR has an invalid size");
9293
9294 // Trivial insertion.
9295 if (VT == N2VT)
9296 return N2;
9297
9298 // If this is an insert of an extracted vector into an undef/poison vector,
9299 // we can just use the input to the extract. But not if skipping the
9300 // extract+insert could make the result more poisonous.
9301 if (N1.isUndef() && N2.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
9302 N2.getOperand(i: 1) == N3 && N2.getOperand(i: 0).getValueType() == VT) {
9303 if (N1.getOpcode() == ISD::POISON)
9304 return N2.getOperand(i: 0);
9305 if (VT.isFixedLengthVector() && N2VT.isFixedLengthVector()) {
9306 unsigned LoBit = N3->getAsZExtVal();
9307 unsigned HiBit = LoBit + N2VT.getVectorNumElements();
9308 APInt EltMask =
9309 APInt::getBitsSet(numBits: VT.getVectorNumElements(), loBit: LoBit, hiBit: HiBit);
9310 if (isGuaranteedNotToBePoison(Op: N2.getOperand(i: 0), DemandedElts: ~EltMask))
9311 return N2.getOperand(i: 0);
9312 } else if (isGuaranteedNotToBePoison(Op: N2.getOperand(i: 0)))
9313 return N2.getOperand(i: 0);
9314 }
9315
9316 // If the inserted subvector is UNDEF, just use the input vector.
9317 // But not if skipping the insert could make the result more poisonous.
9318 if (N2.isUndef()) {
9319 if (VT.isFixedLengthVector()) {
9320 unsigned LoBit = N3->getAsZExtVal();
9321 unsigned HiBit = LoBit + N2VT.getVectorNumElements();
9322 APInt EltMask =
9323 APInt::getBitsSet(numBits: VT.getVectorNumElements(), loBit: LoBit, hiBit: HiBit);
9324 if (isGuaranteedNotToBePoison(Op: N1, DemandedElts: EltMask))
9325 return N1;
9326 } else if (isGuaranteedNotToBePoison(Op: N1))
9327 return N1;
9328 }
9329 break;
9330 }
9331 case ISD::BITCAST:
9332 // Fold bit_convert nodes from a type to themselves.
9333 if (N1.getValueType() == VT)
9334 return N1;
9335 break;
9336 case ISD::VECTOR_COMPRESS: {
9337 [[maybe_unused]] EVT VecVT = N1.getValueType();
9338 [[maybe_unused]] EVT MaskVT = N2.getValueType();
9339 [[maybe_unused]] EVT PassthruVT = N3.getValueType();
9340 assert(VT == VecVT && "Vector and result type don't match.");
9341 assert(VecVT.isVector() && MaskVT.isVector() && PassthruVT.isVector() &&
9342 "All inputs must be vectors.");
9343 assert(VecVT == PassthruVT && "Vector and passthru types don't match.");
9344 assert(VecVT.getVectorElementCount() == MaskVT.getVectorElementCount() &&
9345 "Vector and mask must have same number of elements.");
9346
9347 if (N1.isUndef() || N2.isUndef())
9348 return N3;
9349
9350 break;
9351 }
9352 case ISD::PARTIAL_REDUCE_UMLA:
9353 case ISD::PARTIAL_REDUCE_SMLA:
9354 case ISD::PARTIAL_REDUCE_SUMLA:
9355 case ISD::PARTIAL_REDUCE_FMLA: {
9356 [[maybe_unused]] EVT AccVT = N1.getValueType();
9357 [[maybe_unused]] EVT Input1VT = N2.getValueType();
9358 [[maybe_unused]] EVT Input2VT = N3.getValueType();
9359 assert(Input1VT.isVector() && Input1VT == Input2VT &&
9360 "Expected the second and third operands of the PARTIAL_REDUCE_MLA "
9361 "node to have the same type!");
9362 assert(VT.isVector() && VT == AccVT &&
9363 "Expected the first operand of the PARTIAL_REDUCE_MLA node to have "
9364 "the same type as its result!");
9365 assert(Input1VT.getVectorElementCount().hasKnownScalarFactor(
9366 AccVT.getVectorElementCount()) &&
9367 "Expected the element count of the second and third operands of the "
9368 "PARTIAL_REDUCE_MLA node to be a positive integer multiple of the "
9369 "element count of the first operand and the result!");
9370 assert(N2.getScalarValueSizeInBits() <= N1.getScalarValueSizeInBits() &&
9371 "Expected the second and third operands of the PARTIAL_REDUCE_MLA "
9372 "node to have an element type which is the same as or smaller than "
9373 "the element type of the first operand and result!");
9374 break;
9375 }
9376 }
9377
9378 // Perform trivial constant folding for arithmetic operators.
9379 switch (Opcode) {
9380 case ISD::PARTIAL_REDUCE_SMLA:
9381 case ISD::PARTIAL_REDUCE_UMLA:
9382 case ISD::PARTIAL_REDUCE_SUMLA:
9383 case ISD::FMA:
9384 case ISD::FMAD:
9385 case ISD::SETCC:
9386 case ISD::FSHL:
9387 case ISD::FSHR:
9388 if (SDValue SV =
9389 FoldConstantArithmetic(Opcode, DL, VT, Ops: {N1, N2, N3}, Flags))
9390 return SV;
9391 break;
9392 }
9393
9394 // Memoize node if it doesn't produce a glue result.
9395 SDNode *N;
9396 SDVTList VTs = getVTList(VT);
9397 SDValue Ops[] = {N1, N2, N3};
9398 if (VT != MVT::Glue) {
9399 SDNodeKey ID(Opcode, VTs, Ops);
9400 FoldingSetInsertToken InsertToken;
9401 if (SDNode *E = lookupNode(Key: ID, DL, InsertToken)) {
9402 E->intersectFlagsWith(Flags);
9403 return SDValue(E, 0);
9404 }
9405
9406 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
9407 N->setFlags(Flags);
9408 createOperands(Node: N, Vals: Ops);
9409 CSEMap.insert(N, Token: InsertToken);
9410 } else {
9411 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
9412 createOperands(Node: N, Vals: Ops);
9413 }
9414
9415 InsertNode(N);
9416 SDValue V = SDValue(N, 0);
9417 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
9418 return V;
9419}
9420
9421SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
9422 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
9423 const SDNodeFlags Flags) {
9424 SDValue Ops[] = { N1, N2, N3, N4 };
9425 return getNode(Opcode, DL, VT, Ops, Flags);
9426}
9427
9428SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
9429 SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
9430 SDNodeFlags Flags;
9431 if (Inserter)
9432 Flags = Inserter->getFlags();
9433 return getNode(Opcode, DL, VT, N1, N2, N3, N4, Flags);
9434}
9435
9436SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
9437 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
9438 SDValue N5, const SDNodeFlags Flags) {
9439 SDValue Ops[] = { N1, N2, N3, N4, N5 };
9440 return getNode(Opcode, DL, VT, Ops, Flags);
9441}
9442
9443SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
9444 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
9445 SDValue N5) {
9446 SDNodeFlags Flags;
9447 if (Inserter)
9448 Flags = Inserter->getFlags();
9449 return getNode(Opcode, DL, VT, N1, N2, N3, N4, N5, Flags);
9450}
9451
9452/// getStackArgumentTokenFactor - Compute a TokenFactor to force all
9453/// the incoming stack arguments to be loaded from the stack.
9454SDValue SelectionDAG::getStackArgumentTokenFactor(SDValue Chain) {
9455 SmallVector<SDValue, 8> ArgChains;
9456
9457 // Include the original chain at the beginning of the list. When this is
9458 // used by target LowerCall hooks, this helps legalize find the
9459 // CALLSEQ_BEGIN node.
9460 ArgChains.push_back(Elt: Chain);
9461
9462 // Add a chain value for each stack argument.
9463 for (SDNode *U : getEntryNode().getNode()->users())
9464 if (LoadSDNode *L = dyn_cast<LoadSDNode>(Val: U))
9465 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val: L->getBasePtr()))
9466 if (FI->getIndex() < 0)
9467 ArgChains.push_back(Elt: SDValue(L, 1));
9468
9469 // Build a tokenfactor for all the chains.
9470 return getNode(Opcode: ISD::TokenFactor, DL: SDLoc(Chain), VT: MVT::Other, Ops: ArgChains);
9471}
9472
9473/// getMemsetValue - Vectorized representation of the memset value
9474/// operand.
9475static SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG,
9476 const SDLoc &dl) {
9477 assert(!Value.isUndef());
9478
9479 unsigned NumBits = VT.getScalarSizeInBits();
9480 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: Value)) {
9481 assert(C->getAPIntValue().getBitWidth() == 8);
9482 APInt Val = APInt::getSplat(NewLen: NumBits, V: C->getAPIntValue());
9483 if (VT.isInteger()) {
9484 bool IsOpaque = VT.getSizeInBits() > 64 ||
9485 !DAG.getTargetLoweringInfo().isLegalStoreImmediate(Value: C->getSExtValue());
9486 return DAG.getConstant(Val, DL: dl, VT, isT: false, isO: IsOpaque);
9487 }
9488 return DAG.getConstantFP(V: APFloat(VT.getFltSemantics(), Val), DL: dl, VT);
9489 }
9490
9491 assert(Value.getValueType() == MVT::i8 && "memset with non-byte fill value?");
9492 EVT IntVT = VT.getScalarType();
9493 if (!IntVT.isInteger())
9494 IntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: IntVT.getSizeInBits());
9495
9496 Value = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: IntVT, N1: Value);
9497 if (NumBits > 8) {
9498 // Use a multiplication with 0x010101... to extend the input to the
9499 // required length.
9500 APInt Magic = APInt::getSplat(NewLen: NumBits, V: APInt(8, 0x01));
9501 Value = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: IntVT, N1: Value,
9502 N2: DAG.getConstant(Val: Magic, DL: dl, VT: IntVT));
9503 }
9504
9505 if (VT != Value.getValueType() && !VT.isInteger())
9506 Value = DAG.getBitcast(VT: VT.getScalarType(), V: Value);
9507 if (VT != Value.getValueType())
9508 Value = DAG.getSplatBuildVector(VT, DL: dl, Op: Value);
9509
9510 return Value;
9511}
9512
9513/// getMemsetStringVal - Similar to getMemsetValue. Except this is only
9514/// used when a memcpy is turned into a memset when the source is a constant
9515/// string ptr.
9516static SDValue getMemsetStringVal(EVT VT, const SDLoc &dl, SelectionDAG &DAG,
9517 const TargetLowering &TLI,
9518 const ConstantDataArraySlice &Slice) {
9519 // Handle vector with all elements zero.
9520 if (Slice.Array == nullptr) {
9521 if (VT.isInteger())
9522 return DAG.getConstant(Val: 0, DL: dl, VT);
9523 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT,
9524 N1: DAG.getConstant(Val: 0, DL: dl, VT: VT.changeTypeToInteger()));
9525 }
9526
9527 assert(!VT.isVector() && "Can't handle vector type here!");
9528 unsigned NumVTBits = VT.getSizeInBits();
9529 unsigned NumVTBytes = NumVTBits / 8;
9530 unsigned NumBytes = std::min(a: NumVTBytes, b: unsigned(Slice.Length));
9531
9532 APInt Val(NumVTBits, 0);
9533 if (DAG.getDataLayout().isLittleEndian()) {
9534 for (unsigned i = 0; i != NumBytes; ++i)
9535 Val |= (uint64_t)(unsigned char)Slice[i] << i*8;
9536 } else {
9537 for (unsigned i = 0; i != NumBytes; ++i)
9538 Val |= (uint64_t)(unsigned char)Slice[i] << (NumVTBytes-i-1)*8;
9539 }
9540
9541 // If the "cost" of materializing the integer immediate is less than the cost
9542 // of a load, then it is cost effective to turn the load into the immediate.
9543 Type *Ty = VT.getTypeForEVT(Context&: *DAG.getContext());
9544 if (TLI.shouldConvertConstantLoadToIntImm(Imm: Val, Ty))
9545 return DAG.getConstant(Val, DL: dl, VT);
9546 return SDValue();
9547}
9548
9549SDValue SelectionDAG::getMemBasePlusOffset(SDValue Base, TypeSize Offset,
9550 const SDLoc &DL,
9551 const SDNodeFlags Flags) {
9552 SDValue Index = getTypeSize(DL, VT: Base.getValueType(), TS: Offset);
9553 return getMemBasePlusOffset(Base, Offset: Index, DL, Flags);
9554}
9555
9556SDValue SelectionDAG::getMemBasePlusOffset(SDValue Ptr, SDValue Offset,
9557 const SDLoc &DL,
9558 const SDNodeFlags Flags) {
9559 assert(Offset.getValueType().isInteger());
9560 EVT BasePtrVT = Ptr.getValueType();
9561 if (TLI->shouldPreservePtrArith(F: this->getMachineFunction().getFunction(),
9562 PtrVT: BasePtrVT))
9563 return getNode(Opcode: ISD::PTRADD, DL, VT: BasePtrVT, N1: Ptr, N2: Offset, Flags);
9564 // InBounds only applies to PTRADD, don't set it if we generate ADD.
9565 SDNodeFlags AddFlags = Flags;
9566 AddFlags.setInBounds(false);
9567 return getNode(Opcode: ISD::ADD, DL, VT: BasePtrVT, N1: Ptr, N2: Offset, Flags: AddFlags);
9568}
9569
9570/// Returns true if memcpy source is constant data.
9571static bool isMemSrcFromConstant(SDValue Src, ConstantDataArraySlice &Slice) {
9572 uint64_t SrcDelta = 0;
9573 GlobalAddressSDNode *G = nullptr;
9574 if (Src.getOpcode() == ISD::GlobalAddress)
9575 G = cast<GlobalAddressSDNode>(Val&: Src);
9576 else if (Src->isAnyAdd() &&
9577 Src.getOperand(i: 0).getOpcode() == ISD::GlobalAddress &&
9578 Src.getOperand(i: 1).getOpcode() == ISD::Constant) {
9579 G = cast<GlobalAddressSDNode>(Val: Src.getOperand(i: 0));
9580 SrcDelta = Src.getConstantOperandVal(i: 1);
9581 }
9582 if (!G)
9583 return false;
9584
9585 return getConstantDataArrayInfo(V: G->getGlobal(), Slice, ElementSize: 8,
9586 Offset: SrcDelta + G->getOffset());
9587}
9588
9589static bool shouldLowerMemFuncForSize(const MachineFunction &MF,
9590 SelectionDAG &DAG) {
9591 // On Darwin, -Os means optimize for size without hurting performance, so
9592 // only really optimize for size when -Oz (MinSize) is used.
9593 if (MF.getTarget().getTargetTriple().isOSDarwin())
9594 return MF.getFunction().hasMinSize();
9595 return DAG.shouldOptForSize();
9596}
9597
9598static void chainLoadsAndStoresForMemcpy(SelectionDAG &DAG, const SDLoc &dl,
9599 SmallVector<SDValue, 32> &OutChains, unsigned From,
9600 unsigned To, SmallVector<SDValue, 16> &OutLoadChains,
9601 SmallVector<SDValue, 16> &OutStoreChains) {
9602 assert(OutLoadChains.size() && "Missing loads in memcpy inlining");
9603 assert(OutStoreChains.size() && "Missing stores in memcpy inlining");
9604 SmallVector<SDValue, 16> GluedLoadChains;
9605 for (unsigned i = From; i < To; ++i) {
9606 OutChains.push_back(Elt: OutLoadChains[i]);
9607 GluedLoadChains.push_back(Elt: OutLoadChains[i]);
9608 }
9609
9610 // Chain for all loads.
9611 SDValue LoadToken = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other,
9612 Ops: GluedLoadChains);
9613
9614 for (unsigned i = From; i < To; ++i) {
9615 StoreSDNode *ST = dyn_cast<StoreSDNode>(Val&: OutStoreChains[i]);
9616 SDValue NewStore = DAG.getTruncStore(Chain: LoadToken, dl, Val: ST->getValue(),
9617 Ptr: ST->getBasePtr(), SVT: ST->getMemoryVT(),
9618 MMO: ST->getMemOperand());
9619 OutChains.push_back(Elt: NewStore);
9620 }
9621}
9622
9623static SDValue
9624getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain,
9625 SDValue Dst, SDValue Src, uint64_t Size, Align DstAlign,
9626 Align SrcAlign, bool isVol, bool AlwaysInline,
9627 MachinePointerInfo DstPtrInfo,
9628 MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo,
9629 BatchAAResults *BatchAA, const MDNode *DstMemCacheHint,
9630 const MDNode *SrcMemCacheHint) {
9631 // Turn a memcpy of undef to nop.
9632 // FIXME: We need to honor volatile even is Src is undef.
9633 if (Src.isUndef())
9634 return Chain;
9635
9636 // Expand memcpy to a series of load and store ops if the size operand falls
9637 // below a certain threshold.
9638 // TODO: In the AlwaysInline case, if the size is big then generate a loop
9639 // rather than maybe a humongous number of loads and stores.
9640 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9641 const DataLayout &DL = DAG.getDataLayout();
9642 LLVMContext &C = *DAG.getContext();
9643 std::vector<EVT> MemOps;
9644 bool DstAlignCanChange = false;
9645 MachineFunction &MF = DAG.getMachineFunction();
9646 MachineFrameInfo &MFI = MF.getFrameInfo();
9647 bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
9648 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Dst);
9649 if (FI && !MFI.isFixedObjectIndex(ObjectIdx: FI->getIndex()))
9650 DstAlignCanChange = true;
9651 SrcAlign = std::max(a: SrcAlign, b: DAG.InferPtrAlign(Ptr: Src).valueOrOne());
9652 ConstantDataArraySlice Slice;
9653 // If marked as volatile, perform a copy even when marked as constant.
9654 bool CopyFromConstant = !isVol && isMemSrcFromConstant(Src, Slice);
9655 bool isZeroConstant = CopyFromConstant && Slice.Array == nullptr;
9656 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(OptSize);
9657 const MemOp Op = isZeroConstant
9658 ? MemOp::Set(Size, DstAlignCanChange, DstAlign,
9659 /*IsZeroMemset*/ true, IsVolatile: isVol)
9660 : MemOp::Copy(Size, DstAlignCanChange, DstAlign,
9661 SrcAlign, IsVolatile: isVol, MemcpyStrSrc: CopyFromConstant);
9662 if (!TLI.findOptimalMemOpLowering(
9663 Context&: C, MemOps, Limit, Op, DstAS: DstPtrInfo.getAddrSpace(),
9664 SrcAS: SrcPtrInfo.getAddrSpace(), FuncAttributes: MF.getFunction().getAttributes(), LargestVT: nullptr))
9665 return SDValue();
9666
9667 if (DstAlignCanChange) {
9668 Type *Ty = MemOps[0].getTypeForEVT(Context&: C);
9669 Align NewDstAlign = DL.getABITypeAlign(Ty);
9670
9671 // Don't promote to an alignment that would require dynamic stack
9672 // realignment which may conflict with optimizations such as tail call
9673 // optimization.
9674 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
9675 if (!TRI->hasStackRealignment(MF))
9676 if (MaybeAlign StackAlign = DL.getStackAlignment())
9677 NewDstAlign = std::min(a: NewDstAlign, b: *StackAlign);
9678
9679 if (NewDstAlign > DstAlign) {
9680 // Give the stack frame object a larger alignment if needed.
9681 if (MFI.getObjectAlign(ObjectIdx: FI->getIndex()) < NewDstAlign)
9682 MFI.setObjectAlignment(ObjectIdx: FI->getIndex(), Alignment: NewDstAlign);
9683 DstAlign = NewDstAlign;
9684 }
9685 }
9686
9687 // Prepare AAInfo for loads/stores after lowering this memcpy.
9688 AAMDNodes NewAAInfo = AAInfo;
9689 NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
9690
9691 const Value *SrcVal = dyn_cast_if_present<const Value *>(Val&: SrcPtrInfo.V);
9692 bool isConstant =
9693 BatchAA && SrcVal &&
9694 BatchAA->pointsToConstantMemory(Loc: MemoryLocation(SrcVal, Size, AAInfo));
9695
9696 MachineMemOperand::Flags MMOFlags =
9697 isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone;
9698 SmallVector<SDValue, 16> OutLoadChains;
9699 SmallVector<SDValue, 16> OutStoreChains;
9700 SmallVector<SDValue, 32> OutChains;
9701 unsigned NumMemOps = MemOps.size();
9702 uint64_t SrcOff = 0, DstOff = 0;
9703 for (unsigned i = 0; i != NumMemOps; ++i) {
9704 EVT VT = MemOps[i];
9705 unsigned VTSize = VT.getSizeInBits() / 8;
9706 SDValue Value, Store;
9707
9708 if (VTSize > Size) {
9709 // Issuing an unaligned load / store pair that overlaps with the previous
9710 // pair. Adjust the offset accordingly.
9711 assert(i == NumMemOps-1 && i != 0);
9712 SrcOff -= VTSize - Size;
9713 DstOff -= VTSize - Size;
9714 }
9715
9716 if (CopyFromConstant &&
9717 (isZeroConstant || (VT.isInteger() && !VT.isVector()))) {
9718 // It's unlikely a store of a vector immediate can be done in a single
9719 // instruction. It would require a load from a constantpool first.
9720 // We only handle zero vectors here.
9721 // FIXME: Handle other cases where store of vector immediate is done in
9722 // a single instruction.
9723 ConstantDataArraySlice SubSlice;
9724 if (SrcOff < Slice.Length) {
9725 SubSlice = Slice;
9726 SubSlice.move(Delta: SrcOff);
9727 } else {
9728 // This is an out-of-bounds access and hence UB. Pretend we read zero.
9729 SubSlice.Array = nullptr;
9730 SubSlice.Offset = 0;
9731 SubSlice.Length = VTSize;
9732 }
9733 Value = getMemsetStringVal(VT, dl, DAG, TLI, Slice: SubSlice);
9734 if (Value.getNode()) {
9735 Store = DAG.getStore(
9736 Chain, dl, Val: Value,
9737 Ptr: DAG.getObjectPtrOffset(SL: dl, Ptr: Dst, Offset: TypeSize::getFixed(ExactSize: DstOff)),
9738 PtrInfo: DstPtrInfo.getWithOffset(O: DstOff), Alignment: DstAlign, MMOFlags,
9739 Metadata: MMOMetadata(NewAAInfo, /*Ranges=*/nullptr, DstMemCacheHint));
9740 OutChains.push_back(Elt: Store);
9741 }
9742 }
9743
9744 if (!Store.getNode()) {
9745 // The type might not be legal for the target. This should only happen
9746 // if the type is smaller than a legal type, as on PPC, so the right
9747 // thing to do is generate a LoadExt/StoreTrunc pair. These simplify
9748 // to Load/Store if NVT==VT.
9749 // FIXME does the case above also need this?
9750 EVT NVT = TLI.getTypeToTransformTo(Context&: C, VT);
9751 assert(NVT.bitsGE(VT));
9752
9753 bool isDereferenceable =
9754 SrcPtrInfo.getWithOffset(O: SrcOff).isDereferenceable(Size: VTSize, C, DL);
9755 MachineMemOperand::Flags SrcMMOFlags = MMOFlags;
9756 if (isDereferenceable)
9757 SrcMMOFlags |= MachineMemOperand::MODereferenceable;
9758 if (isConstant)
9759 SrcMMOFlags |= MachineMemOperand::MOInvariant;
9760
9761 Value = DAG.getExtLoad(
9762 ExtType: ISD::EXTLOAD, dl, VT: NVT, Chain,
9763 Ptr: DAG.getObjectPtrOffset(SL: dl, Ptr: Src, Offset: TypeSize::getFixed(ExactSize: SrcOff)),
9764 PtrInfo: SrcPtrInfo.getWithOffset(O: SrcOff), MemVT: VT,
9765 Alignment: commonAlignment(A: SrcAlign, Offset: SrcOff), MMOFlags: SrcMMOFlags,
9766 Metadata: MMOMetadata(NewAAInfo, /*Ranges=*/nullptr, SrcMemCacheHint));
9767 OutLoadChains.push_back(Elt: Value.getValue(R: 1));
9768
9769 Store = DAG.getTruncStore(
9770 Chain, dl, Val: Value,
9771 Ptr: DAG.getObjectPtrOffset(SL: dl, Ptr: Dst, Offset: TypeSize::getFixed(ExactSize: DstOff)),
9772 PtrInfo: DstPtrInfo.getWithOffset(O: DstOff), SVT: VT, Alignment: DstAlign, MMOFlags,
9773 Metadata: MMOMetadata(NewAAInfo, /*Ranges=*/nullptr, DstMemCacheHint));
9774 OutStoreChains.push_back(Elt: Store);
9775 }
9776 SrcOff += VTSize;
9777 DstOff += VTSize;
9778 Size -= VTSize;
9779 }
9780
9781 unsigned GluedLdStLimit = MaxLdStGlue == 0 ?
9782 TLI.getMaxGluedStoresPerMemcpy() : MaxLdStGlue;
9783 unsigned NumLdStInMemcpy = OutStoreChains.size();
9784
9785 if (NumLdStInMemcpy) {
9786 // It may be that memcpy might be converted to memset if it's memcpy
9787 // of constants. In such a case, we won't have loads and stores, but
9788 // just stores. In the absence of loads, there is nothing to gang up.
9789 if ((GluedLdStLimit <= 1) || !EnableMemCpyDAGOpt) {
9790 // If target does not care, just leave as it.
9791 for (unsigned i = 0; i < NumLdStInMemcpy; ++i) {
9792 OutChains.push_back(Elt: OutLoadChains[i]);
9793 OutChains.push_back(Elt: OutStoreChains[i]);
9794 }
9795 } else {
9796 // Ld/St less than/equal limit set by target.
9797 if (NumLdStInMemcpy <= GluedLdStLimit) {
9798 chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, From: 0,
9799 To: NumLdStInMemcpy, OutLoadChains,
9800 OutStoreChains);
9801 } else {
9802 unsigned NumberLdChain = NumLdStInMemcpy / GluedLdStLimit;
9803 unsigned RemainingLdStInMemcpy = NumLdStInMemcpy % GluedLdStLimit;
9804 unsigned GlueIter = 0;
9805
9806 // Residual ld/st.
9807 if (RemainingLdStInMemcpy) {
9808 chainLoadsAndStoresForMemcpy(
9809 DAG, dl, OutChains, From: NumLdStInMemcpy - RemainingLdStInMemcpy,
9810 To: NumLdStInMemcpy, OutLoadChains, OutStoreChains);
9811 }
9812
9813 for (unsigned cnt = 0; cnt < NumberLdChain; ++cnt) {
9814 unsigned IndexFrom = NumLdStInMemcpy - RemainingLdStInMemcpy -
9815 GlueIter - GluedLdStLimit;
9816 unsigned IndexTo = NumLdStInMemcpy - RemainingLdStInMemcpy - GlueIter;
9817 chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, From: IndexFrom, To: IndexTo,
9818 OutLoadChains, OutStoreChains);
9819 GlueIter += GluedLdStLimit;
9820 }
9821 }
9822 }
9823 }
9824 return DAG.getTokenFactor(DL: dl, Vals&: OutChains);
9825}
9826
9827static SDValue getMemmoveLoadsAndStores(
9828 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
9829 uint64_t Size, Align DstAlign, Align SrcAlign, bool isVol,
9830 bool AlwaysInline, MachinePointerInfo DstPtrInfo,
9831 MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo) {
9832 // Turn a memmove of undef to nop.
9833 // FIXME: We need to honor volatile even is Src is undef.
9834 if (Src.isUndef())
9835 return Chain;
9836
9837 // Expand memmove to a series of load and store ops if the size operand falls
9838 // below a certain threshold.
9839 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9840 const DataLayout &DL = DAG.getDataLayout();
9841 LLVMContext &C = *DAG.getContext();
9842 std::vector<EVT> MemOps;
9843 bool DstAlignCanChange = false;
9844 MachineFunction &MF = DAG.getMachineFunction();
9845 MachineFrameInfo &MFI = MF.getFrameInfo();
9846 bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
9847 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Dst);
9848 if (FI && !MFI.isFixedObjectIndex(ObjectIdx: FI->getIndex()))
9849 DstAlignCanChange = true;
9850 SrcAlign = std::max(a: SrcAlign, b: DAG.InferPtrAlign(Ptr: Src).valueOrOne());
9851 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize);
9852 if (!TLI.findOptimalMemOpLowering(
9853 Context&: C, MemOps, Limit,
9854 Op: MemOp::Move(Size, DstAlignCanChange, DstAlign, SrcAlign, IsVolatile: isVol),
9855 DstAS: DstPtrInfo.getAddrSpace(), SrcAS: SrcPtrInfo.getAddrSpace(),
9856 FuncAttributes: MF.getFunction().getAttributes(), LargestVT: nullptr))
9857 return SDValue();
9858
9859 if (DstAlignCanChange) {
9860 Type *Ty = MemOps[0].getTypeForEVT(Context&: C);
9861 Align NewDstAlign = DL.getABITypeAlign(Ty);
9862
9863 // Don't promote to an alignment that would require dynamic stack
9864 // realignment which may conflict with optimizations such as tail call
9865 // optimization.
9866 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
9867 if (!TRI->hasStackRealignment(MF))
9868 if (MaybeAlign StackAlign = DL.getStackAlignment())
9869 NewDstAlign = std::min(a: NewDstAlign, b: *StackAlign);
9870
9871 if (NewDstAlign > DstAlign) {
9872 // Give the stack frame object a larger alignment if needed.
9873 if (MFI.getObjectAlign(ObjectIdx: FI->getIndex()) < NewDstAlign)
9874 MFI.setObjectAlignment(ObjectIdx: FI->getIndex(), Alignment: NewDstAlign);
9875 DstAlign = NewDstAlign;
9876 }
9877 }
9878
9879 // Prepare AAInfo for loads/stores after lowering this memmove.
9880 AAMDNodes NewAAInfo = AAInfo;
9881 NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
9882
9883 MachineMemOperand::Flags MMOFlags =
9884 isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone;
9885 uint64_t SrcOff = 0;
9886 SmallVector<SDValue, 8> LoadValues;
9887 SmallVector<SDValue, 8> LoadChains;
9888 SmallVector<SDValue, 8> OutChains;
9889 unsigned NumMemOps = MemOps.size();
9890 for (unsigned i = 0; i < NumMemOps; i++) {
9891 EVT VT = MemOps[i];
9892 unsigned VTSize = VT.getSizeInBits() / 8;
9893 SDValue Value;
9894 bool IsOverlapping = false;
9895
9896 if (i == NumMemOps - 1 && i != 0 && VTSize > Size - SrcOff) {
9897 // Issuing an unaligned load / store pair that overlaps with the previous
9898 // pair. Adjust the offset accordingly.
9899 SrcOff = Size - VTSize;
9900 IsOverlapping = true;
9901 }
9902
9903 // Calculate the actual alignment at the current offset. The alignment at
9904 // SrcOff may be lower than the base alignment, especially when using
9905 // overlapping loads.
9906 Align SrcAlignAtOffset = commonAlignment(A: SrcAlign, Offset: SrcOff);
9907 if (IsOverlapping) {
9908 // Verify that the target allows misaligned memory accesses at the
9909 // adjusted offset when using overlapping loads.
9910 unsigned Fast;
9911 if (!TLI.allowsMisalignedMemoryAccesses(VT, AddrSpace: SrcPtrInfo.getAddrSpace(),
9912 Alignment: SrcAlignAtOffset, Flags: MMOFlags,
9913 &Fast) ||
9914 !Fast) {
9915 // This should have been caught by findOptimalMemOpLowering, but verify
9916 // here for safety.
9917 return SDValue();
9918 }
9919 }
9920
9921 bool isDereferenceable =
9922 SrcPtrInfo.getWithOffset(O: SrcOff).isDereferenceable(Size: VTSize, C, DL);
9923 MachineMemOperand::Flags SrcMMOFlags = MMOFlags;
9924 if (isDereferenceable)
9925 SrcMMOFlags |= MachineMemOperand::MODereferenceable;
9926 Value =
9927 DAG.getLoad(VT, dl, Chain,
9928 Ptr: DAG.getObjectPtrOffset(SL: dl, Ptr: Src, Offset: TypeSize::getFixed(ExactSize: SrcOff)),
9929 PtrInfo: SrcPtrInfo.getWithOffset(O: SrcOff), Alignment: SrcAlignAtOffset,
9930 MMOFlags: SrcMMOFlags, Metadata: NewAAInfo);
9931 LoadValues.push_back(Elt: Value);
9932 LoadChains.push_back(Elt: Value.getValue(R: 1));
9933 SrcOff += VTSize;
9934 }
9935 Chain = DAG.getTokenFactor(DL: dl, Vals&: LoadChains);
9936 OutChains.clear();
9937 uint64_t DstOff = 0;
9938 for (unsigned i = 0; i < NumMemOps; i++) {
9939 EVT VT = MemOps[i];
9940 unsigned VTSize = VT.getSizeInBits() / 8;
9941 SDValue Store;
9942 bool IsOverlapping = false;
9943
9944 if (i == NumMemOps - 1 && i != 0 && VTSize > Size - DstOff) {
9945 // Issuing an unaligned load / store pair that overlaps with the previous
9946 // pair. Adjust the offset accordingly.
9947 DstOff = Size - VTSize;
9948 IsOverlapping = true;
9949 }
9950
9951 // Calculate the actual alignment at the current offset. The alignment at
9952 // DstOff may be lower than the base alignment, especially when using
9953 // overlapping stores.
9954 Align DstAlignAtOffset = commonAlignment(A: DstAlign, Offset: DstOff);
9955 if (IsOverlapping) {
9956 // Verify that the target allows misaligned memory accesses at the
9957 // adjusted offset when using overlapping stores.
9958 unsigned Fast;
9959 if (!TLI.allowsMisalignedMemoryAccesses(VT, AddrSpace: DstPtrInfo.getAddrSpace(),
9960 Alignment: DstAlignAtOffset, Flags: MMOFlags,
9961 &Fast) ||
9962 !Fast) {
9963 // This should have been caught by findOptimalMemOpLowering, but verify
9964 // here for safety.
9965 return SDValue();
9966 }
9967 }
9968 Store = DAG.getStore(
9969 Chain, dl, Val: LoadValues[i],
9970 Ptr: DAG.getObjectPtrOffset(SL: dl, Ptr: Dst, Offset: TypeSize::getFixed(ExactSize: DstOff)),
9971 PtrInfo: DstPtrInfo.getWithOffset(O: DstOff), Alignment: DstAlignAtOffset, MMOFlags,
9972 Metadata: NewAAInfo);
9973 OutChains.push_back(Elt: Store);
9974 DstOff += VTSize;
9975 }
9976
9977 return DAG.getTokenFactor(DL: dl, Vals&: OutChains);
9978}
9979
9980/// Lower the call to 'memset' intrinsic function into a series of store
9981/// operations.
9982///
9983/// \param DAG Selection DAG where lowered code is placed.
9984/// \param dl Link to corresponding IR location.
9985/// \param Chain Control flow dependency.
9986/// \param Dst Pointer to destination memory location.
9987/// \param Src Value of byte to write into the memory.
9988/// \param Size Number of bytes to write.
9989/// \param Alignment Alignment of the destination in bytes.
9990/// \param isVol True if destination is volatile.
9991/// \param AlwaysInline Makes sure no function call is generated.
9992/// \param DstPtrInfo IR information on the memory pointer.
9993/// \returns New head in the control flow, if lowering was successful, empty
9994/// SDValue otherwise.
9995///
9996/// The function tries to replace 'llvm.memset' intrinsic with several store
9997/// operations and value calculation code. This is usually profitable for small
9998/// memory size or when the semantic requires inlining.
9999static SDValue getMemsetStores(SelectionDAG &DAG, const SDLoc &dl,
10000 SDValue Chain, SDValue Dst, SDValue Src,
10001 uint64_t Size, Align Alignment, bool isVol,
10002 bool AlwaysInline, MachinePointerInfo DstPtrInfo,
10003 const AAMDNodes &AAInfo) {
10004 // Turn a memset of undef to nop.
10005 // FIXME: We need to honor volatile even is Src is undef.
10006 if (Src.isUndef())
10007 return Chain;
10008
10009 // Expand memset to a series of load/store ops if the size operand
10010 // falls below a certain threshold.
10011 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
10012 std::vector<EVT> MemOps;
10013 bool DstAlignCanChange = false;
10014 LLVMContext &C = *DAG.getContext();
10015 MachineFunction &MF = DAG.getMachineFunction();
10016 MachineFrameInfo &MFI = MF.getFrameInfo();
10017 bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
10018 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Dst);
10019 if (FI && !MFI.isFixedObjectIndex(ObjectIdx: FI->getIndex()))
10020 DstAlignCanChange = true;
10021 bool IsZeroVal = isNullConstant(V: Src);
10022 unsigned Limit = AlwaysInline ? ~0 : TLI.getMaxStoresPerMemset(OptSize);
10023
10024 EVT LargestVT;
10025 if (!TLI.findOptimalMemOpLowering(
10026 Context&: C, MemOps, Limit,
10027 Op: MemOp::Set(Size, DstAlignCanChange, DstAlign: Alignment, IsZeroMemset: IsZeroVal, IsVolatile: isVol),
10028 DstAS: DstPtrInfo.getAddrSpace(), SrcAS: ~0u, FuncAttributes: MF.getFunction().getAttributes(),
10029 LargestVT: &LargestVT))
10030 return SDValue();
10031
10032 if (DstAlignCanChange) {
10033 Type *Ty = MemOps[0].getTypeForEVT(Context&: *DAG.getContext());
10034 const DataLayout &DL = DAG.getDataLayout();
10035 Align NewAlign = DL.getABITypeAlign(Ty);
10036
10037 // Don't promote to an alignment that would require dynamic stack
10038 // realignment which may conflict with optimizations such as tail call
10039 // optimization.
10040 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
10041 if (!TRI->hasStackRealignment(MF))
10042 if (MaybeAlign StackAlign = DL.getStackAlignment())
10043 NewAlign = std::min(a: NewAlign, b: *StackAlign);
10044
10045 if (NewAlign > Alignment) {
10046 // Give the stack frame object a larger alignment if needed.
10047 if (MFI.getObjectAlign(ObjectIdx: FI->getIndex()) < NewAlign)
10048 MFI.setObjectAlignment(ObjectIdx: FI->getIndex(), Alignment: NewAlign);
10049 Alignment = NewAlign;
10050 }
10051 }
10052
10053 SmallVector<SDValue, 8> OutChains;
10054 uint64_t DstOff = 0;
10055 unsigned NumMemOps = MemOps.size();
10056
10057 // Find the largest store and generate the bit pattern for it.
10058 // If target didn't set LargestVT, compute it from MemOps.
10059 if (!LargestVT.isSimple()) {
10060 LargestVT = MemOps[0];
10061 for (unsigned i = 1; i < NumMemOps; i++)
10062 if (MemOps[i].bitsGT(VT: LargestVT))
10063 LargestVT = MemOps[i];
10064 }
10065 SDValue MemSetValue = getMemsetValue(Value: Src, VT: LargestVT, DAG, dl);
10066
10067 // Prepare AAInfo for loads/stores after lowering this memset.
10068 AAMDNodes NewAAInfo = AAInfo;
10069 NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
10070
10071 for (unsigned i = 0; i < NumMemOps; i++) {
10072 EVT VT = MemOps[i];
10073 unsigned VTSize = VT.getSizeInBits() / 8;
10074 // The target should specify store types that exactly cover the memset size
10075 // (with the last store potentially being oversized for overlapping stores).
10076 assert(Size > 0 && "Target specified more stores than needed in "
10077 "findOptimalMemOpLowering");
10078 if (VTSize > Size) {
10079 // Issuing an unaligned load / store pair that overlaps with the previous
10080 // pair. Adjust the offset accordingly.
10081 assert(i == NumMemOps-1 && i != 0);
10082 DstOff -= VTSize - Size;
10083 }
10084
10085 // If this store is smaller than the largest store see whether we can get
10086 // the smaller value for free with a truncate or extract vector element and
10087 // then store.
10088 SDValue Value = MemSetValue;
10089 if (VT.bitsLT(VT: LargestVT)) {
10090 unsigned Index;
10091 unsigned NElts = LargestVT.getSizeInBits() / VT.getSizeInBits();
10092 EVT SVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: VT.getScalarType(), NumElements: NElts);
10093 if (!LargestVT.isVector() && !VT.isVector() &&
10094 TLI.isTruncateFree(FromVT: LargestVT, ToVT: VT))
10095 Value = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, N1: MemSetValue);
10096 else if (LargestVT.isVector() && !VT.isVector() &&
10097 TLI.shallExtractConstSplatVectorElementToStore(
10098 VectorTy: LargestVT.getTypeForEVT(Context&: *DAG.getContext()),
10099 ElemSizeInBits: VT.getSizeInBits(), Index) &&
10100 TLI.isTypeLegal(VT: SVT) &&
10101 LargestVT.getSizeInBits() == SVT.getSizeInBits()) {
10102 // Target which can combine store(extractelement VectorTy, Idx) can get
10103 // the smaller value for free.
10104 SDValue TailValue = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: SVT, N1: MemSetValue);
10105 Value = DAG.getExtractVectorElt(DL: dl, VT, Vec: TailValue, Idx: Index);
10106 } else
10107 Value = getMemsetValue(Value: Src, VT, DAG, dl);
10108 }
10109 assert(Value.getValueType() == VT && "Value with wrong type.");
10110 SDValue Store = DAG.getStore(
10111 Chain, dl, Val: Value,
10112 Ptr: DAG.getObjectPtrOffset(SL: dl, Ptr: Dst, Offset: TypeSize::getFixed(ExactSize: DstOff)),
10113 PtrInfo: DstPtrInfo.getWithOffset(O: DstOff), Alignment,
10114 MMOFlags: isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone,
10115 Metadata: NewAAInfo);
10116 OutChains.push_back(Elt: Store);
10117 DstOff += VT.getSizeInBits() / 8;
10118 // For oversized overlapping stores, only subtract the remaining bytes.
10119 // For normal stores, subtract the full store size.
10120 if (VTSize > Size) {
10121 Size = 0;
10122 } else {
10123 Size -= VTSize;
10124 }
10125 }
10126
10127 // After processing all stores, Size should be exactly 0. Any remaining bytes
10128 // indicate a bug in the target's findOptimalMemOpLowering implementation.
10129 assert(Size == 0 && "Target's findOptimalMemOpLowering did not specify "
10130 "stores that exactly cover the memset size");
10131
10132 return DAG.getTokenFactor(DL: dl, Vals&: OutChains);
10133}
10134
10135static void checkAddrSpaceIsValidForLibcall(const TargetLowering *TLI,
10136 const DataLayout &DL, unsigned AS) {
10137 // Lowering memcpy / memset / memmove intrinsics to calls is only valid if all
10138 // pointer operands can be losslessly bitcasted to pointers of address space 0
10139 if (AS != 0 && !TLI->getTargetMachine().isNoopAddrSpaceCast(DL, SrcAS: AS, DestAS: 0)) {
10140 report_fatal_error(reason: "cannot lower memory intrinsic in address space " +
10141 Twine(AS));
10142 }
10143}
10144
10145/// The length of a memory intrinsic (e.g. number of bytes to copy) is unsigned
10146/// and may have any integer type. We zero-extend or truncate it to the pointer
10147/// type of the narrower address space being accessed.
10148static SDValue getMemIntrinsicSize(SelectionDAG &DAG, const SDLoc &dl,
10149 SDValue Size, unsigned DstAS,
10150 unsigned SrcAS) {
10151 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
10152 MVT DstVT = TLI.getPointerTy(DL: DAG.getDataLayout(), AS: DstAS);
10153 MVT SrcVT = TLI.getPointerTy(DL: DAG.getDataLayout(), AS: SrcAS);
10154 return DAG.getZExtOrTrunc(Op: Size, DL: dl, VT: DstVT.bitsLT(VT: SrcVT) ? DstVT : SrcVT);
10155}
10156
10157static bool isInTailCallPositionWrapper(const CallInst *CI,
10158 const SelectionDAG *SelDAG,
10159 bool AllowReturnsFirstArg) {
10160 if (!CI || !CI->isTailCall())
10161 return false;
10162 // TODO: Fix "returns-first-arg" determination so it doesn't depend on which
10163 // helper symbol we lower to.
10164 return isInTailCallPosition(Call: *CI, TM: SelDAG->getTarget(),
10165 ReturnsFirstArg: AllowReturnsFirstArg &&
10166 funcReturnsFirstArgOfCall(CI: *CI));
10167}
10168
10169static std::pair<SDValue, SDValue>
10170getRuntimeCallSDValueHelper(SDValue Chain, const SDLoc &dl,
10171 TargetLowering::ArgListTy &&Args,
10172 const CallInst *CI, RTLIB::Libcall Call,
10173 SelectionDAG *DAG, const TargetLowering *TLI) {
10174 RTLIB::LibcallImpl LCImpl = DAG->getLibcalls().getLibcallImpl(Call);
10175
10176 if (LCImpl == RTLIB::Unsupported)
10177 return {};
10178
10179 TargetLowering::CallLoweringInfo CLI(*DAG);
10180 bool IsTailCall =
10181 isInTailCallPositionWrapper(CI, SelDAG: DAG, /*AllowReturnsFirstArg=*/true) &&
10182 // Lowering doesn't support tail calling inside a function with
10183 // a swifterror argument yet.
10184 !DAG->hasSwiftErrorArg();
10185 SDValue Callee =
10186 DAG->getExternalSymbol(Libcall: LCImpl, VT: TLI->getPointerTy(DL: DAG->getDataLayout()));
10187
10188 CLI.setDebugLoc(dl)
10189 .setChain(Chain)
10190 .setLibCallee(CC: DAG->getLibcalls().getLibcallImplCallingConv(Call: LCImpl),
10191 ResultType: CI->getType(), Target: Callee, ArgsList: std::move(Args))
10192 .setTailCall(IsTailCall);
10193
10194 return TLI->LowerCallTo(CLI);
10195}
10196
10197std::pair<SDValue, SDValue> SelectionDAG::getStrcmp(SDValue Chain,
10198 const SDLoc &dl, SDValue S1,
10199 SDValue S2,
10200 const CallInst *CI) {
10201 PointerType *PT = PointerType::getUnqual(C&: *getContext());
10202 TargetLowering::ArgListTy Args = {{S1, PT}, {S2, PT}};
10203 return getRuntimeCallSDValueHelper(Chain, dl, Args: std::move(Args), CI,
10204 Call: RTLIB::STRCMP, DAG: this, TLI);
10205}
10206
10207std::pair<SDValue, SDValue> SelectionDAG::getStrstr(SDValue Chain,
10208 const SDLoc &dl, SDValue S1,
10209 SDValue S2,
10210 const CallInst *CI) {
10211 PointerType *PT = PointerType::getUnqual(C&: *getContext());
10212 TargetLowering::ArgListTy Args = {{S1, PT}, {S2, PT}};
10213 return getRuntimeCallSDValueHelper(Chain, dl, Args: std::move(Args), CI,
10214 Call: RTLIB::STRSTR, DAG: this, TLI);
10215}
10216
10217std::pair<SDValue, SDValue> SelectionDAG::getMemccpy(SDValue Chain,
10218 const SDLoc &dl,
10219 SDValue Dst, SDValue Src,
10220 SDValue C, SDValue Size,
10221 const CallInst *CI) {
10222 PointerType *PT = PointerType::getUnqual(C&: *getContext());
10223
10224 TargetLowering::ArgListTy Args = {
10225 {Dst, PT},
10226 {Src, PT},
10227 {C, Type::getInt32Ty(C&: *getContext())},
10228 {Size, getDataLayout().getIntPtrType(C&: *getContext())}};
10229 return getRuntimeCallSDValueHelper(Chain, dl, Args: std::move(Args), CI,
10230 Call: RTLIB::MEMCCPY, DAG: this, TLI);
10231}
10232
10233std::pair<SDValue, SDValue>
10234SelectionDAG::getMemcmp(SDValue Chain, const SDLoc &dl, SDValue Mem0,
10235 SDValue Mem1, SDValue Size, const CallInst *CI) {
10236 PointerType *PT = PointerType::getUnqual(C&: *getContext());
10237 TargetLowering::ArgListTy Args = {
10238 {Mem0, PT},
10239 {Mem1, PT},
10240 {Size, getDataLayout().getIntPtrType(C&: *getContext())}};
10241 return getRuntimeCallSDValueHelper(Chain, dl, Args: std::move(Args), CI,
10242 Call: RTLIB::MEMCMP, DAG: this, TLI);
10243}
10244
10245std::pair<SDValue, SDValue> SelectionDAG::getStrcpy(SDValue Chain,
10246 const SDLoc &dl,
10247 SDValue Dst, SDValue Src,
10248 const CallInst *CI) {
10249 PointerType *PT = PointerType::getUnqual(C&: *getContext());
10250 TargetLowering::ArgListTy Args = {{Dst, PT}, {Src, PT}};
10251 return getRuntimeCallSDValueHelper(Chain, dl, Args: std::move(Args), CI,
10252 Call: RTLIB::STRCPY, DAG: this, TLI);
10253}
10254
10255std::pair<SDValue, SDValue> SelectionDAG::getStrlen(SDValue Chain,
10256 const SDLoc &dl,
10257 SDValue Src,
10258 const CallInst *CI) {
10259 // Emit a library call.
10260 TargetLowering::ArgListTy Args = {
10261 {Src, PointerType::getUnqual(C&: *getContext())}};
10262 return getRuntimeCallSDValueHelper(Chain, dl, Args: std::move(Args), CI,
10263 Call: RTLIB::STRLEN, DAG: this, TLI);
10264}
10265
10266bool SelectionDAG::hasSwiftErrorArg() const {
10267 return TLI->supportSwiftError() &&
10268 MF->getFunction().getAttributes().hasAttrSomewhere(
10269 Kind: Attribute::SwiftError);
10270}
10271
10272SDValue SelectionDAG::getMemcpy(
10273 SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size,
10274 Align DstAlign, Align SrcAlign, bool isVol, bool AlwaysInline,
10275 const CallInst *CI, std::optional<bool> OverrideTailCall,
10276 MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo,
10277 const AAMDNodes &AAInfo, BatchAAResults *BatchAA) {
10278 Size = getMemIntrinsicSize(DAG&: *this, dl, Size, DstAS: DstPtrInfo.getAddrSpace(),
10279 SrcAS: SrcPtrInfo.getAddrSpace());
10280 // Check to see if we should lower the memcpy to loads and stores first.
10281 // For cases within the target-specified limits, this is the best choice.
10282 const MDNode *DstMemCacheHint =
10283 CI ? getMemCacheHintMetadata(I: *CI, /*OperandNo=*/0) : nullptr;
10284 const MDNode *SrcMemCacheHint =
10285 CI ? getMemCacheHintMetadata(I: *CI, /*OperandNo=*/1) : nullptr;
10286
10287 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Val&: Size);
10288 if (ConstantSize) {
10289 // Memcpy with size zero? Just return the original chain.
10290 if (ConstantSize->isZero())
10291 return Chain;
10292
10293 SDValue Result = getMemcpyLoadsAndStores(
10294 DAG&: *this, dl, Chain, Dst, Src, Size: ConstantSize->getZExtValue(), DstAlign,
10295 SrcAlign, isVol, AlwaysInline: false, DstPtrInfo, SrcPtrInfo, AAInfo, BatchAA,
10296 DstMemCacheHint, SrcMemCacheHint);
10297 if (Result.getNode())
10298 return Result;
10299 }
10300
10301 // Then check to see if we should lower the memcpy with target-specific
10302 // code. If the target chooses to do this, this is the next best.
10303 if (TSI) {
10304 SDValue Result = TSI->EmitTargetCodeForMemcpy(
10305 DAG&: *this, dl, Chain, Op1: Dst, Op2: Src, Op3: Size, DstAlign, SrcAlign, isVolatile: isVol,
10306 AlwaysInline, DstPtrInfo, SrcPtrInfo);
10307 if (Result.getNode())
10308 return Result;
10309 }
10310
10311 // If we really need inline code and the target declined to provide it,
10312 // use a (potentially long) sequence of loads and stores.
10313 if (AlwaysInline) {
10314 assert(ConstantSize && "AlwaysInline requires a constant size!");
10315 return getMemcpyLoadsAndStores(
10316 DAG&: *this, dl, Chain, Dst, Src, Size: ConstantSize->getZExtValue(), DstAlign,
10317 SrcAlign, isVol, AlwaysInline: true, DstPtrInfo, SrcPtrInfo, AAInfo, BatchAA,
10318 DstMemCacheHint, SrcMemCacheHint);
10319 }
10320
10321 checkAddrSpaceIsValidForLibcall(TLI, DL: getDataLayout(),
10322 AS: DstPtrInfo.getAddrSpace());
10323 checkAddrSpaceIsValidForLibcall(TLI, DL: getDataLayout(),
10324 AS: SrcPtrInfo.getAddrSpace());
10325
10326 // FIXME: If the memcpy is volatile (isVol), lowering it to a plain libc
10327 // memcpy is not guaranteed to be safe. libc memcpys aren't required to
10328 // respect volatile, so they may do things like read or write memory
10329 // beyond the given memory regions. But fixing this isn't easy, and most
10330 // people don't care.
10331
10332 // Emit a library call.
10333 TargetLowering::ArgListTy Args;
10334 Type *PtrTy = PointerType::getUnqual(C&: *getContext());
10335 Args.emplace_back(args&: Dst, args&: PtrTy);
10336 Args.emplace_back(args&: Src, args&: PtrTy);
10337 Args.emplace_back(args&: Size, args: getDataLayout().getIntPtrType(C&: *getContext()));
10338 // FIXME: pass in SDLoc
10339 TargetLowering::CallLoweringInfo CLI(*this);
10340 bool IsTailCall = false;
10341 RTLIB::LibcallImpl MemCpyImpl = TLI->getMemcpyImpl();
10342
10343 if (OverrideTailCall.has_value()) {
10344 IsTailCall = *OverrideTailCall;
10345 } else {
10346 bool LowersToMemcpy = MemCpyImpl == RTLIB::impl_memcpy;
10347 IsTailCall = isInTailCallPositionWrapper(CI, SelDAG: this, AllowReturnsFirstArg: LowersToMemcpy);
10348 }
10349 // Lowering doesn't support tail calling inside a function with a
10350 // swifterror argument yet.
10351 IsTailCall &= !hasSwiftErrorArg();
10352
10353 CLI.setDebugLoc(dl)
10354 .setChain(Chain)
10355 .setLibCallee(
10356 CC: Libcalls->getLibcallImplCallingConv(Call: MemCpyImpl),
10357 ResultType: Dst.getValueType().getTypeForEVT(Context&: *getContext()),
10358 Target: getExternalSymbol(Libcall: MemCpyImpl, VT: TLI->getPointerTy(DL: getDataLayout())),
10359 ArgsList: std::move(Args))
10360 .setDiscardResult()
10361 .setTailCall(IsTailCall);
10362
10363 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
10364 return CallResult.second;
10365}
10366
10367static TargetLoweringBase::ArgListEntry
10368getAtomicMemOpSizeArg(SelectionDAG *DAG, const SDLoc &dl, SDValue Size) {
10369 const Module &M = *DAG->getMachineFunction().getFunction().getParent();
10370 Type *SizeTTy = DAG->getLibInfo().getSizeTType(M);
10371 return TargetLoweringBase::ArgListEntry(
10372 DAG->getZExtOrTrunc(Op: Size, DL: dl, VT: MVT::getVT(Ty: SizeTTy)), SizeTTy);
10373}
10374
10375SDValue SelectionDAG::getAtomicMemcpy(SDValue Chain, const SDLoc &dl,
10376 SDValue Dst, SDValue Src, SDValue Size,
10377 Type *SizeTy, unsigned ElemSz,
10378 bool isTailCall,
10379 MachinePointerInfo DstPtrInfo,
10380 MachinePointerInfo SrcPtrInfo) {
10381 // Lowering doesn't support tail calling inside a function with a
10382 // swifterror argument yet.
10383 isTailCall &= !hasSwiftErrorArg();
10384
10385 // Emit a library call.
10386 TargetLowering::ArgListTy Args;
10387 Type *ArgTy = getDataLayout().getIntPtrType(C&: *getContext());
10388 Args.emplace_back(args&: Dst, args&: ArgTy);
10389 Args.emplace_back(args&: Src, args&: ArgTy);
10390 Args.emplace_back(args: getAtomicMemOpSizeArg(DAG: this, dl, Size));
10391
10392 RTLIB::Libcall LibraryCall =
10393 RTLIB::getMEMCPY_ELEMENT_UNORDERED_ATOMIC(ElementSize: ElemSz);
10394 RTLIB::LibcallImpl LibcallImpl = Libcalls->getLibcallImpl(Call: LibraryCall);
10395 if (LibcallImpl == RTLIB::Unsupported)
10396 report_fatal_error(reason: "Unsupported element size");
10397
10398 TargetLowering::CallLoweringInfo CLI(*this);
10399 CLI.setDebugLoc(dl)
10400 .setChain(Chain)
10401 .setLibCallee(
10402 CC: Libcalls->getLibcallImplCallingConv(Call: LibcallImpl),
10403 ResultType: Type::getVoidTy(C&: *getContext()),
10404 Target: getExternalSymbol(Libcall: LibcallImpl, VT: TLI->getPointerTy(DL: getDataLayout())),
10405 ArgsList: std::move(Args))
10406 .setDiscardResult()
10407 .setTailCall(isTailCall);
10408
10409 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
10410 return CallResult.second;
10411}
10412
10413SDValue SelectionDAG::getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst,
10414 SDValue Src, SDValue Size, Align DstAlign,
10415 Align SrcAlign, bool isVol, const CallInst *CI,
10416 std::optional<bool> OverrideTailCall,
10417 MachinePointerInfo DstPtrInfo,
10418 MachinePointerInfo SrcPtrInfo,
10419 const AAMDNodes &AAInfo,
10420 BatchAAResults *BatchAA) {
10421 Size = getMemIntrinsicSize(DAG&: *this, dl, Size, DstAS: DstPtrInfo.getAddrSpace(),
10422 SrcAS: SrcPtrInfo.getAddrSpace());
10423 // Check to see if we should lower the memmove to loads and stores first.
10424 // For cases within the target-specified limits, this is the best choice.
10425 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Val&: Size);
10426 if (ConstantSize) {
10427 // Memmove with size zero? Just return the original chain.
10428 if (ConstantSize->isZero())
10429 return Chain;
10430
10431 SDValue Result = getMemmoveLoadsAndStores(
10432 DAG&: *this, dl, Chain, Dst, Src, Size: ConstantSize->getZExtValue(), DstAlign,
10433 SrcAlign, isVol, AlwaysInline: false, DstPtrInfo, SrcPtrInfo, AAInfo);
10434 if (Result.getNode())
10435 return Result;
10436 }
10437
10438 // Then check to see if we should lower the memmove with target-specific
10439 // code. If the target chooses to do this, this is the next best.
10440 if (TSI) {
10441 SDValue Result = TSI->EmitTargetCodeForMemmove(
10442 DAG&: *this, dl, Chain, Op1: Dst, Op2: Src, Op3: Size, DstAlign, SrcAlign, isVolatile: isVol, DstPtrInfo,
10443 SrcPtrInfo);
10444 if (Result.getNode())
10445 return Result;
10446 }
10447
10448 checkAddrSpaceIsValidForLibcall(TLI, DL: getDataLayout(),
10449 AS: DstPtrInfo.getAddrSpace());
10450 checkAddrSpaceIsValidForLibcall(TLI, DL: getDataLayout(),
10451 AS: SrcPtrInfo.getAddrSpace());
10452
10453 // FIXME: If the memmove is volatile, lowering it to plain libc memmove may
10454 // not be safe. See memcpy above for more details.
10455
10456 // Emit a library call.
10457 TargetLowering::ArgListTy Args;
10458 Type *PtrTy = PointerType::getUnqual(C&: *getContext());
10459 Args.emplace_back(args&: Dst, args&: PtrTy);
10460 Args.emplace_back(args&: Src, args&: PtrTy);
10461 Args.emplace_back(args&: Size, args: getDataLayout().getIntPtrType(C&: *getContext()));
10462 // FIXME: pass in SDLoc
10463 TargetLowering::CallLoweringInfo CLI(*this);
10464
10465 RTLIB::LibcallImpl MemmoveImpl = Libcalls->getLibcallImpl(Call: RTLIB::MEMMOVE);
10466
10467 bool IsTailCall = false;
10468 if (OverrideTailCall.has_value()) {
10469 IsTailCall = *OverrideTailCall;
10470 } else {
10471 bool LowersToMemmove = MemmoveImpl == RTLIB::impl_memmove;
10472 IsTailCall = isInTailCallPositionWrapper(CI, SelDAG: this, AllowReturnsFirstArg: LowersToMemmove);
10473 }
10474 // Lowering doesn't support tail calling inside a function with a
10475 // swifterror argument yet.
10476 IsTailCall &= !hasSwiftErrorArg();
10477
10478 CLI.setDebugLoc(dl)
10479 .setChain(Chain)
10480 .setLibCallee(
10481 CC: Libcalls->getLibcallImplCallingConv(Call: MemmoveImpl),
10482 ResultType: Dst.getValueType().getTypeForEVT(Context&: *getContext()),
10483 Target: getExternalSymbol(Libcall: MemmoveImpl, VT: TLI->getPointerTy(DL: getDataLayout())),
10484 ArgsList: std::move(Args))
10485 .setDiscardResult()
10486 .setTailCall(IsTailCall);
10487
10488 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
10489 return CallResult.second;
10490}
10491
10492SDValue SelectionDAG::getAtomicMemmove(SDValue Chain, const SDLoc &dl,
10493 SDValue Dst, SDValue Src, SDValue Size,
10494 Type *SizeTy, unsigned ElemSz,
10495 bool isTailCall,
10496 MachinePointerInfo DstPtrInfo,
10497 MachinePointerInfo SrcPtrInfo) {
10498 // Lowering doesn't support tail calling inside a function with a
10499 // swifterror argument yet.
10500 isTailCall &= !hasSwiftErrorArg();
10501
10502 // Emit a library call.
10503 TargetLowering::ArgListTy Args;
10504 Type *IntPtrTy = getDataLayout().getIntPtrType(C&: *getContext());
10505 Args.emplace_back(args&: Dst, args&: IntPtrTy);
10506 Args.emplace_back(args&: Src, args&: IntPtrTy);
10507 Args.emplace_back(args: getAtomicMemOpSizeArg(DAG: this, dl, Size));
10508
10509 RTLIB::Libcall LibraryCall =
10510 RTLIB::getMEMMOVE_ELEMENT_UNORDERED_ATOMIC(ElementSize: ElemSz);
10511 RTLIB::LibcallImpl LibcallImpl = Libcalls->getLibcallImpl(Call: LibraryCall);
10512 if (LibcallImpl == RTLIB::Unsupported)
10513 report_fatal_error(reason: "Unsupported element size");
10514
10515 TargetLowering::CallLoweringInfo CLI(*this);
10516 CLI.setDebugLoc(dl)
10517 .setChain(Chain)
10518 .setLibCallee(
10519 CC: Libcalls->getLibcallImplCallingConv(Call: LibcallImpl),
10520 ResultType: Type::getVoidTy(C&: *getContext()),
10521 Target: getExternalSymbol(Libcall: LibcallImpl, VT: TLI->getPointerTy(DL: getDataLayout())),
10522 ArgsList: std::move(Args))
10523 .setDiscardResult()
10524 .setTailCall(isTailCall);
10525
10526 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
10527 return CallResult.second;
10528}
10529
10530SDValue SelectionDAG::getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst,
10531 SDValue Src, SDValue Size, Align Alignment,
10532 bool isVol, bool AlwaysInline,
10533 const CallInst *CI,
10534 MachinePointerInfo DstPtrInfo,
10535 const AAMDNodes &AAInfo) {
10536 Size = getMemIntrinsicSize(DAG&: *this, dl, Size, DstAS: DstPtrInfo.getAddrSpace(),
10537 SrcAS: DstPtrInfo.getAddrSpace());
10538 // Check to see if we should lower the memset to stores first.
10539 // For cases within the target-specified limits, this is the best choice.
10540 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Val&: Size);
10541 if (ConstantSize) {
10542 // Memset with size zero? Just return the original chain.
10543 if (ConstantSize->isZero())
10544 return Chain;
10545
10546 SDValue Result = getMemsetStores(DAG&: *this, dl, Chain, Dst, Src,
10547 Size: ConstantSize->getZExtValue(), Alignment,
10548 isVol, AlwaysInline: false, DstPtrInfo, AAInfo);
10549
10550 if (Result.getNode())
10551 return Result;
10552 }
10553
10554 // Then check to see if we should lower the memset with target-specific
10555 // code. If the target chooses to do this, this is the next best.
10556 if (TSI) {
10557 SDValue Result = TSI->EmitTargetCodeForMemset(
10558 DAG&: *this, dl, Chain, Op1: Dst, Op2: Src, Op3: Size, Alignment, isVolatile: isVol, AlwaysInline, DstPtrInfo);
10559 if (Result.getNode())
10560 return Result;
10561 }
10562
10563 // If we really need inline code and the target declined to provide it,
10564 // use a (potentially long) sequence of loads and stores.
10565 if (AlwaysInline) {
10566 assert(ConstantSize && "AlwaysInline requires a constant size!");
10567 SDValue Result = getMemsetStores(DAG&: *this, dl, Chain, Dst, Src,
10568 Size: ConstantSize->getZExtValue(), Alignment,
10569 isVol, AlwaysInline: true, DstPtrInfo, AAInfo);
10570 assert(Result &&
10571 "getMemsetStores must return a valid sequence when AlwaysInline");
10572 return Result;
10573 }
10574
10575 checkAddrSpaceIsValidForLibcall(TLI, DL: getDataLayout(),
10576 AS: DstPtrInfo.getAddrSpace());
10577
10578 // Emit a library call.
10579 auto &Ctx = *getContext();
10580 const auto& DL = getDataLayout();
10581
10582 TargetLowering::CallLoweringInfo CLI(*this);
10583 // FIXME: pass in SDLoc
10584 CLI.setDebugLoc(dl).setChain(Chain);
10585
10586 RTLIB::LibcallImpl BzeroImpl = Libcalls->getLibcallImpl(Call: RTLIB::BZERO);
10587 bool UseBZero = BzeroImpl != RTLIB::Unsupported && isNullConstant(V: Src);
10588
10589 // If zeroing out and bzero is present, use it.
10590 if (UseBZero) {
10591 TargetLowering::ArgListTy Args;
10592 Args.emplace_back(args&: Dst, args: PointerType::getUnqual(C&: Ctx));
10593 Args.emplace_back(args&: Size, args: DL.getIntPtrType(C&: Ctx));
10594 CLI.setLibCallee(
10595 CC: Libcalls->getLibcallImplCallingConv(Call: BzeroImpl), ResultType: Type::getVoidTy(C&: Ctx),
10596 Target: getExternalSymbol(Libcall: BzeroImpl, VT: TLI->getPointerTy(DL)), ArgsList: std::move(Args));
10597 } else {
10598 RTLIB::LibcallImpl MemsetImpl = Libcalls->getLibcallImpl(Call: RTLIB::MEMSET);
10599
10600 MVT IntVT = MVT::getIntegerVT(BitWidth: LibInfo->getIntSize());
10601 TargetLowering::ArgListTy Args;
10602 Args.emplace_back(args&: Dst, args: PointerType::getUnqual(C&: Ctx));
10603 Args.emplace_back(args: getAnyExtOrTrunc(Op: Src, DL: dl, VT: IntVT),
10604 args: Type::getIntNTy(C&: Ctx, N: LibInfo->getIntSize()),
10605 args: LibInfo->getExtAttrForI32Param());
10606 Args.emplace_back(args&: Size, args: DL.getIntPtrType(C&: Ctx));
10607
10608 CLI.setLibCallee(CC: Libcalls->getLibcallImplCallingConv(Call: MemsetImpl),
10609 ResultType: Dst.getValueType().getTypeForEVT(Context&: Ctx),
10610 Target: getExternalSymbol(Libcall: MemsetImpl, VT: TLI->getPointerTy(DL)),
10611 ArgsList: std::move(Args));
10612 }
10613
10614 RTLIB::LibcallImpl MemsetImpl = Libcalls->getLibcallImpl(Call: RTLIB::MEMSET);
10615 bool LowersToMemset = MemsetImpl == RTLIB::impl_memset;
10616
10617 // If we're going to use bzero, make sure not to tail call unless the
10618 // subsequent return doesn't need a value, as bzero doesn't return the first
10619 // arg unlike memset.
10620 bool ReturnsFirstArg = CI && funcReturnsFirstArgOfCall(CI: *CI) && !UseBZero;
10621 bool IsTailCall = CI && CI->isTailCall() &&
10622 isInTailCallPosition(Call: *CI, TM: getTarget(),
10623 ReturnsFirstArg: ReturnsFirstArg && LowersToMemset) &&
10624 // Lowering doesn't support tail calling inside a function
10625 // with a swifterror argument yet.
10626 !hasSwiftErrorArg();
10627 CLI.setDiscardResult().setTailCall(IsTailCall);
10628
10629 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
10630 return CallResult.second;
10631}
10632
10633SDValue SelectionDAG::getAtomicMemset(SDValue Chain, const SDLoc &dl,
10634 SDValue Dst, SDValue Value, SDValue Size,
10635 Type *SizeTy, unsigned ElemSz,
10636 bool isTailCall,
10637 MachinePointerInfo DstPtrInfo) {
10638 // Lowering doesn't support tail calling inside a function with a
10639 // swifterror argument yet.
10640 isTailCall &= !hasSwiftErrorArg();
10641
10642 // Emit a library call.
10643 MVT IntVT = MVT::getIntegerVT(BitWidth: LibInfo->getIntSize());
10644 TargetLowering::ArgListTy Args;
10645 Args.emplace_back(args&: Dst, args: getDataLayout().getIntPtrType(C&: *getContext()));
10646 Args.emplace_back(args: getAnyExtOrTrunc(Op: Value, DL: dl, VT: IntVT),
10647 args: Type::getIntNTy(C&: *getContext(), N: LibInfo->getIntSize()),
10648 args: LibInfo->getExtAttrForI32Param());
10649 Args.emplace_back(args: getAtomicMemOpSizeArg(DAG: this, dl, Size));
10650
10651 RTLIB::Libcall LibraryCall =
10652 RTLIB::getMEMSET_ELEMENT_UNORDERED_ATOMIC(ElementSize: ElemSz);
10653 RTLIB::LibcallImpl LibcallImpl = Libcalls->getLibcallImpl(Call: LibraryCall);
10654 if (LibcallImpl == RTLIB::Unsupported)
10655 report_fatal_error(reason: "Unsupported element size");
10656
10657 TargetLowering::CallLoweringInfo CLI(*this);
10658 CLI.setDebugLoc(dl)
10659 .setChain(Chain)
10660 .setLibCallee(
10661 CC: Libcalls->getLibcallImplCallingConv(Call: LibcallImpl),
10662 ResultType: Type::getVoidTy(C&: *getContext()),
10663 Target: getExternalSymbol(Libcall: LibcallImpl, VT: TLI->getPointerTy(DL: getDataLayout())),
10664 ArgsList: std::move(Args))
10665 .setDiscardResult()
10666 .setTailCall(isTailCall);
10667
10668 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
10669 return CallResult.second;
10670}
10671
10672SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
10673 SDVTList VTList, ArrayRef<SDValue> Ops,
10674 MachineMemOperand *MMO,
10675 ISD::LoadExtType ExtType) {
10676 SDNodeKey ID(Opcode, VTList, Ops);
10677 ID.AddInteger(I: MemVT.getRawBits());
10678 ID.AddInteger(I: getSyntheticNodeSubclassData<AtomicSDNode>(
10679 IROrder: dl.getIROrder(), Args&: Opcode, Args&: VTList, Args&: MemVT, Args&: MMO, Args&: ExtType));
10680 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10681 ID.AddInteger(I: MMO->getFlags());
10682 FoldingSetInsertToken InsertToken;
10683 if (auto *E = cast_or_null<AtomicSDNode>(Val: lookupNode(Key: ID, DL: dl, InsertToken))) {
10684 E->refineAlignment(NewMMO: MMO);
10685 E->refineMMOMetadata(NewMMO: MMO);
10686 return SDValue(E, 0);
10687 }
10688
10689 auto *N = newSDNode<AtomicSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: Opcode,
10690 Args&: VTList, Args&: MemVT, Args&: MMO, Args&: ExtType);
10691 createOperands(Node: N, Vals: Ops);
10692
10693 CSEMap.insert(N, Token: InsertToken);
10694 InsertNode(N);
10695 SDValue V(N, 0);
10696 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10697 return V;
10698}
10699
10700SDValue SelectionDAG::getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl,
10701 EVT MemVT, SDVTList VTs, SDValue Chain,
10702 SDValue Ptr, SDValue Cmp, SDValue Swp,
10703 MachineMemOperand *MMO) {
10704 assert(Opcode == ISD::ATOMIC_CMP_SWAP ||
10705 Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
10706 assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
10707
10708 SDValue Ops[] = {Chain, Ptr, Cmp, Swp};
10709 return getAtomic(Opcode, dl, MemVT, VTList: VTs, Ops, MMO);
10710}
10711
10712SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
10713 SDValue Chain, SDValue Ptr, SDValue Val,
10714 MachineMemOperand *MMO) {
10715 assert((Opcode == ISD::ATOMIC_LOAD_ADD || Opcode == ISD::ATOMIC_LOAD_SUB ||
10716 Opcode == ISD::ATOMIC_LOAD_AND || Opcode == ISD::ATOMIC_LOAD_CLR ||
10717 Opcode == ISD::ATOMIC_LOAD_OR || Opcode == ISD::ATOMIC_LOAD_XOR ||
10718 Opcode == ISD::ATOMIC_LOAD_NAND || Opcode == ISD::ATOMIC_LOAD_MIN ||
10719 Opcode == ISD::ATOMIC_LOAD_MAX || Opcode == ISD::ATOMIC_LOAD_UMIN ||
10720 Opcode == ISD::ATOMIC_LOAD_UMAX || Opcode == ISD::ATOMIC_LOAD_FADD ||
10721 Opcode == ISD::ATOMIC_LOAD_FSUB || Opcode == ISD::ATOMIC_LOAD_FMAX ||
10722 Opcode == ISD::ATOMIC_LOAD_FMIN ||
10723 Opcode == ISD::ATOMIC_LOAD_FMINIMUM ||
10724 Opcode == ISD::ATOMIC_LOAD_FMAXIMUM ||
10725 Opcode == ISD::ATOMIC_LOAD_FMINIMUMNUM ||
10726 Opcode == ISD::ATOMIC_LOAD_FMAXIMUMNUM ||
10727 Opcode == ISD::ATOMIC_LOAD_UINC_WRAP ||
10728 Opcode == ISD::ATOMIC_LOAD_UDEC_WRAP ||
10729 Opcode == ISD::ATOMIC_LOAD_USUB_COND ||
10730 Opcode == ISD::ATOMIC_LOAD_USUB_SAT || Opcode == ISD::ATOMIC_SWAP ||
10731 Opcode == ISD::ATOMIC_STORE) &&
10732 "Invalid Atomic Op");
10733
10734 EVT VT = Val.getValueType();
10735
10736 SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(VT: MVT::Other) :
10737 getVTList(VT1: VT, VT2: MVT::Other);
10738 SDValue Ops[] = {Chain, Ptr, Val};
10739 return getAtomic(Opcode, dl, MemVT, VTList: VTs, Ops, MMO);
10740}
10741
10742SDValue SelectionDAG::getAtomicLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
10743 EVT MemVT, EVT VT, SDValue Chain,
10744 SDValue Ptr, MachineMemOperand *MMO) {
10745 SDVTList VTs = getVTList(VT1: VT, VT2: MVT::Other);
10746 SDValue Ops[] = {Chain, Ptr};
10747 return getAtomic(Opcode: ISD::ATOMIC_LOAD, dl, MemVT, VTList: VTs, Ops, MMO, ExtType);
10748}
10749
10750/// getMergeValues - Create a MERGE_VALUES node from the given operands.
10751SDValue SelectionDAG::getMergeValues(ArrayRef<SDValue> Ops, const SDLoc &dl) {
10752 if (Ops.size() == 1)
10753 return Ops[0];
10754
10755 SmallVector<EVT, 4> VTs;
10756 VTs.reserve(N: Ops.size());
10757 for (const SDValue &Op : Ops)
10758 VTs.push_back(Elt: Op.getValueType());
10759 return getNode(Opcode: ISD::MERGE_VALUES, DL: dl, VTList: getVTList(VTs), Ops);
10760}
10761
10762SDValue SelectionDAG::getErrorMergeValues(ArrayRef<EVT> ResultTypes,
10763 SDValue Chain, const SDLoc &dl) {
10764 SmallVector<SDValue, 4> RetValues;
10765 RetValues.reserve(N: ResultTypes.size());
10766 for (EVT VT : ResultTypes)
10767 RetValues.push_back(Elt: VT == MVT::Other ? Chain : getPOISON(VT));
10768 return getMergeValues(Ops: RetValues, dl);
10769}
10770
10771SDValue SelectionDAG::getMemIntrinsicNode(
10772 unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef<SDValue> Ops,
10773 EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment,
10774 MachineMemOperand::Flags Flags, LocationSize Size,
10775 const AAMDNodes &AAInfo) {
10776 if (Size.hasValue() && !Size.getValue())
10777 Size = LocationSize::precise(Value: MemVT.getStoreSize());
10778
10779 MachineFunction &MF = getMachineFunction();
10780 MachineMemOperand *MMO =
10781 MF.getMachineMemOperand(PtrInfo, F: Flags, Size, BaseAlignment: Alignment, Metadata: AAInfo);
10782
10783 return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, MMO);
10784}
10785
10786SDValue SelectionDAG::getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl,
10787 SDVTList VTList,
10788 ArrayRef<SDValue> Ops, EVT MemVT,
10789 MachineMemOperand *MMO) {
10790 return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, MMOs: ArrayRef(MMO));
10791}
10792
10793SDValue SelectionDAG::getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl,
10794 SDVTList VTList,
10795 ArrayRef<SDValue> Ops, EVT MemVT,
10796 ArrayRef<MachineMemOperand *> MMOs) {
10797 assert(!MMOs.empty() && "Must have at least one MMO");
10798 assert(
10799 (Opcode == ISD::INTRINSIC_VOID || Opcode == ISD::INTRINSIC_W_CHAIN ||
10800 Opcode == ISD::PREFETCH ||
10801 (Opcode <= (unsigned)std::numeric_limits<int>::max() &&
10802 Opcode >= ISD::BUILTIN_OP_END && TSI->isTargetMemoryOpcode(Opcode))) &&
10803 "Opcode is not a memory-accessing opcode!");
10804
10805 PointerUnion<MachineMemOperand *, MachineMemOperand **> MemRefs;
10806 if (MMOs.size() == 1) {
10807 MemRefs = MMOs[0];
10808 } else {
10809 // Allocate: [size_t count][MMO*][MMO*]...
10810 size_t AllocSize =
10811 sizeof(size_t) + MMOs.size() * sizeof(MachineMemOperand *);
10812 void *Buffer = Allocator.Allocate(Size: AllocSize, Alignment: alignof(size_t));
10813 size_t *CountPtr = static_cast<size_t *>(Buffer);
10814 *CountPtr = MMOs.size();
10815 MachineMemOperand **Array =
10816 reinterpret_cast<MachineMemOperand **>(CountPtr + 1);
10817 llvm::copy(Range&: MMOs, Out: Array);
10818 MemRefs = Array;
10819 }
10820
10821 // Memoize the node unless it returns a glue result.
10822 MemIntrinsicSDNode *N;
10823 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
10824 SDNodeKey ID(Opcode, VTList, Ops);
10825 ID.AddInteger(I: getSyntheticNodeSubclassData<MemIntrinsicSDNode>(
10826 Opc: Opcode, Order: dl.getIROrder(), VTs: VTList, MemoryVT: MemVT, MemRefs));
10827 ID.AddInteger(I: MemVT.getRawBits());
10828 for (const MachineMemOperand *MMO : MMOs) {
10829 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10830 ID.AddInteger(I: MMO->getFlags());
10831 }
10832 FoldingSetInsertToken InsertToken;
10833 if (SDNode *E = lookupNode(Key: ID, DL: dl, InsertToken)) {
10834 cast<MemIntrinsicSDNode>(Val: E)->refineAlignment(NewMMOs: MMOs);
10835 return SDValue(E, 0);
10836 }
10837
10838 N = newSDNode<MemIntrinsicSDNode>(Args&: Opcode, Args: dl.getIROrder(), Args: dl.getDebugLoc(),
10839 Args&: VTList, Args&: MemVT, Args&: MemRefs);
10840 createOperands(Node: N, Vals: Ops);
10841 CSEMap.insert(N, Token: InsertToken);
10842 } else {
10843 N = newSDNode<MemIntrinsicSDNode>(Args&: Opcode, Args: dl.getIROrder(), Args: dl.getDebugLoc(),
10844 Args&: VTList, Args&: MemVT, Args&: MemRefs);
10845 createOperands(Node: N, Vals: Ops);
10846 }
10847 InsertNode(N);
10848 SDValue V(N, 0);
10849 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10850 return V;
10851}
10852
10853SDValue SelectionDAG::getLifetimeNode(bool IsStart, const SDLoc &dl,
10854 SDValue Chain, int FrameIndex) {
10855 const unsigned Opcode = IsStart ? ISD::LIFETIME_START : ISD::LIFETIME_END;
10856 const auto VTs = getVTList(VT: MVT::Other);
10857 SDValue Ops[2] = {
10858 Chain,
10859 getFrameIndex(FI: FrameIndex,
10860 VT: getTargetLoweringInfo().getFrameIndexTy(DL: getDataLayout()),
10861 isTarget: true)};
10862
10863 SDNodeKey ID(Opcode, VTs, Ops);
10864 FoldingSetInsertToken InsertToken;
10865 if (SDNode *E = lookupNode(Key: ID, DL: dl, InsertToken))
10866 return SDValue(E, 0);
10867
10868 LifetimeSDNode *N =
10869 newSDNode<LifetimeSDNode>(Args: Opcode, Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args: VTs);
10870 createOperands(Node: N, Vals: Ops);
10871 CSEMap.insert(N, Token: InsertToken);
10872 InsertNode(N);
10873 SDValue V(N, 0);
10874 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10875 return V;
10876}
10877
10878SDValue SelectionDAG::getPseudoProbeNode(const SDLoc &Dl, SDValue Chain,
10879 uint64_t Guid, uint64_t Index,
10880 uint32_t Attr) {
10881 const unsigned Opcode = ISD::PSEUDO_PROBE;
10882 const auto VTs = getVTList(VT: MVT::Other);
10883 SDValue Ops[] = {Chain};
10884 SDNodeKey ID(Opcode, VTs, Ops);
10885 ID.AddInteger(I: Guid);
10886 ID.AddInteger(I: Index);
10887 ID.AddInteger(I: Attr);
10888 FoldingSetInsertToken InsertToken;
10889 if (SDNode *E = lookupNode(Key: ID, DL: Dl, InsertToken))
10890 return SDValue(E, 0);
10891
10892 auto *N = newSDNode<PseudoProbeSDNode>(
10893 Args: Opcode, Args: Dl.getIROrder(), Args: Dl.getDebugLoc(), Args: VTs, Args&: Guid, Args&: Index, Args&: Attr);
10894 createOperands(Node: N, Vals: Ops);
10895 CSEMap.insert(N, Token: InsertToken);
10896 InsertNode(N);
10897 SDValue V(N, 0);
10898 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10899 return V;
10900}
10901
10902/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
10903/// MachinePointerInfo record from it. This is particularly useful because the
10904/// code generator has many cases where it doesn't bother passing in a
10905/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
10906static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info,
10907 SelectionDAG &DAG, SDValue Ptr,
10908 int64_t Offset = 0) {
10909 // If this is FI+Offset, we can model it.
10910 if (const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Ptr))
10911 return MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(),
10912 FI: FI->getIndex(), Offset);
10913
10914 // If this is (FI+Offset1)+Offset2, we can model it.
10915 if (Ptr.getOpcode() != ISD::ADD ||
10916 !isa<ConstantSDNode>(Val: Ptr.getOperand(i: 1)) ||
10917 !isa<FrameIndexSDNode>(Val: Ptr.getOperand(i: 0)))
10918 return Info;
10919
10920 int FI = cast<FrameIndexSDNode>(Val: Ptr.getOperand(i: 0))->getIndex();
10921 return MachinePointerInfo::getFixedStack(
10922 MF&: DAG.getMachineFunction(), FI,
10923 Offset: Offset + cast<ConstantSDNode>(Val: Ptr.getOperand(i: 1))->getSExtValue());
10924}
10925
10926/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
10927/// MachinePointerInfo record from it. This is particularly useful because the
10928/// code generator has many cases where it doesn't bother passing in a
10929/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
10930static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info,
10931 SelectionDAG &DAG, SDValue Ptr,
10932 SDValue OffsetOp) {
10933 // If the 'Offset' value isn't a constant, we can't handle this.
10934 if (ConstantSDNode *OffsetNode = dyn_cast<ConstantSDNode>(Val&: OffsetOp))
10935 return InferPointerInfo(Info, DAG, Ptr, Offset: OffsetNode->getSExtValue());
10936 if (OffsetOp.isUndef())
10937 return InferPointerInfo(Info, DAG, Ptr);
10938 return Info;
10939}
10940
10941SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
10942 EVT VT, const SDLoc &dl, SDValue Chain,
10943 SDValue Ptr, SDValue Offset,
10944 MachinePointerInfo PtrInfo, EVT MemVT,
10945 Align Alignment,
10946 MachineMemOperand::Flags MMOFlags,
10947 const MMOMetadata &Metadata) {
10948 assert(Chain.getValueType() == MVT::Other &&
10949 "Invalid chain type");
10950
10951 MMOFlags |= MachineMemOperand::MOLoad;
10952 assert((MMOFlags & MachineMemOperand::MOStore) == 0);
10953 // If we don't have a PtrInfo, infer the trivial frame index case to simplify
10954 // clients.
10955 if (PtrInfo.V.isNull())
10956 PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr, OffsetOp: Offset);
10957
10958 TypeSize Size = MemVT.getStoreSize();
10959 MachineFunction &MF = getMachineFunction();
10960 MachineMemOperand *MMO =
10961 MF.getMachineMemOperand(PtrInfo, F: MMOFlags, Size, BaseAlignment: Alignment, Metadata);
10962 return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO);
10963}
10964
10965SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
10966 EVT VT, const SDLoc &dl, SDValue Chain,
10967 SDValue Ptr, SDValue Offset, EVT MemVT,
10968 MachineMemOperand *MMO) {
10969 if (VT == MemVT) {
10970 ExtType = ISD::NON_EXTLOAD;
10971 } else if (ExtType == ISD::NON_EXTLOAD) {
10972 assert(VT == MemVT && "Non-extending load from different memory type!");
10973 } else {
10974 // Extending load.
10975 assert(MemVT.getScalarType().bitsLT(VT.getScalarType()) &&
10976 "Should only be an extending load, not truncating!");
10977 assert(VT.isInteger() == MemVT.isInteger() &&
10978 "Cannot convert from FP to Int or Int -> FP!");
10979 assert(VT.isVector() == MemVT.isVector() &&
10980 "Cannot use an ext load to convert to or from a vector!");
10981 assert((!VT.isVector() ||
10982 VT.getVectorElementCount() == MemVT.getVectorElementCount()) &&
10983 "Cannot use an ext load to change the number of vector elements!");
10984 }
10985
10986 assert((!MMO->getRanges() ||
10987 (mdconst::extract<ConstantInt>(MMO->getRanges()->getOperand(0))
10988 ->getBitWidth() == MemVT.getScalarSizeInBits() &&
10989 MemVT.isInteger())) &&
10990 "Range metadata and load type must match!");
10991
10992 bool Indexed = AM != ISD::UNINDEXED;
10993 assert((Indexed || Offset.getOpcode() == ISD::POISON) &&
10994 "Unindexed load with an offset!");
10995
10996 SDVTList VTs = Indexed ?
10997 getVTList(VT1: VT, VT2: Ptr.getValueType(), VT3: MVT::Other) : getVTList(VT1: VT, VT2: MVT::Other);
10998 SDValue Ops[] = { Chain, Ptr, Offset };
10999 SDNodeKey ID(ISD::LOAD, VTs, Ops);
11000 ID.AddInteger(I: MemVT.getRawBits());
11001 ID.AddInteger(I: getSyntheticNodeSubclassData<LoadSDNode>(
11002 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: ExtType, Args&: MemVT, Args&: MMO));
11003 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11004 ID.AddInteger(I: MMO->getFlags());
11005 FoldingSetInsertToken InsertToken;
11006 if (auto *E = cast_or_null<LoadSDNode>(Val: lookupNode(Key: ID, DL: dl, InsertToken))) {
11007 E->refineAlignment(NewMMO: MMO);
11008 E->refineMMOMetadata(NewMMO: MMO);
11009 return SDValue(E, 0);
11010 }
11011 auto *N = newSDNode<LoadSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM,
11012 Args&: ExtType, Args&: MemVT, Args&: MMO);
11013 createOperands(Node: N, Vals: Ops);
11014
11015 CSEMap.insert(N, Token: InsertToken);
11016 InsertNode(N);
11017 SDValue V(N, 0);
11018 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11019 return V;
11020}
11021
11022SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain,
11023 SDValue Ptr, MachinePointerInfo PtrInfo,
11024 MaybeAlign Alignment,
11025 MachineMemOperand::Flags MMOFlags,
11026 const MMOMetadata &Metadata) {
11027 SDValue Undef = getPOISON(VT: Ptr.getValueType());
11028 return getLoad(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Offset: Undef,
11029 PtrInfo, MemVT: VT, Alignment, MMOFlags, Metadata);
11030}
11031
11032SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain,
11033 SDValue Ptr, MachineMemOperand *MMO) {
11034 SDValue Undef = getPOISON(VT: Ptr.getValueType());
11035 return getLoad(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Offset: Undef,
11036 MemVT: VT, MMO);
11037}
11038
11039SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
11040 EVT VT, SDValue Chain, SDValue Ptr,
11041 MachinePointerInfo PtrInfo, EVT MemVT,
11042 MaybeAlign Alignment,
11043 MachineMemOperand::Flags MMOFlags,
11044 const MMOMetadata &Metadata) {
11045 SDValue Undef = getPOISON(VT: Ptr.getValueType());
11046 return getLoad(AM: ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Offset: Undef, PtrInfo,
11047 MemVT, Alignment, MMOFlags, Metadata);
11048}
11049
11050SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
11051 EVT VT, SDValue Chain, SDValue Ptr, EVT MemVT,
11052 MachineMemOperand *MMO) {
11053 SDValue Undef = getPOISON(VT: Ptr.getValueType());
11054 return getLoad(AM: ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Offset: Undef,
11055 MemVT, MMO);
11056}
11057
11058SDValue SelectionDAG::getIndexedLoad(SDValue OrigLoad, const SDLoc &dl,
11059 SDValue Base, SDValue Offset,
11060 ISD::MemIndexedMode AM) {
11061 LoadSDNode *LD = cast<LoadSDNode>(Val&: OrigLoad);
11062 assert(LD->getOffset().getOpcode() == ISD::POISON &&
11063 "Load is already a indexed load!");
11064 // Don't propagate the invariant or dereferenceable flags.
11065 auto MMOFlags =
11066 LD->getMemOperand()->getFlags() &
11067 ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
11068 return getLoad(
11069 AM, ExtType: LD->getExtensionType(), VT: OrigLoad.getValueType(), dl, Chain: LD->getChain(),
11070 Ptr: Base, Offset, PtrInfo: LD->getPointerInfo(), MemVT: LD->getMemoryVT(), Alignment: LD->getAlign(),
11071 MMOFlags,
11072 Metadata: MMOMetadata(LD->getAAInfo(), LD->getRanges(), LD->getMemCacheHint()));
11073}
11074
11075SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
11076 SDValue Ptr, MachinePointerInfo PtrInfo,
11077 Align Alignment,
11078 MachineMemOperand::Flags MMOFlags,
11079 const MMOMetadata &Metadata) {
11080 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
11081
11082 MMOFlags |= MachineMemOperand::MOStore;
11083 assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
11084 assert(!Metadata.Ranges && "range metadata is invalid for stores");
11085
11086 if (PtrInfo.V.isNull())
11087 PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr);
11088
11089 MachineFunction &MF = getMachineFunction();
11090 TypeSize Size = Val.getValueType().getStoreSize();
11091 MachineMemOperand *MMO =
11092 MF.getMachineMemOperand(PtrInfo, F: MMOFlags, Size, BaseAlignment: Alignment, Metadata);
11093 return getStore(Chain, dl, Val, Ptr, MMO);
11094}
11095
11096SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
11097 SDValue Ptr, MachineMemOperand *MMO) {
11098 SDValue Undef = getPOISON(VT: Ptr.getValueType());
11099 return getStore(Chain, dl, Val, Ptr, Offset: Undef, SVT: Val.getValueType(), MMO,
11100 AM: ISD::UNINDEXED);
11101}
11102
11103SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
11104 SDValue Ptr, SDValue Offset, EVT SVT,
11105 MachineMemOperand *MMO, ISD::MemIndexedMode AM,
11106 bool IsTruncating) {
11107 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
11108 EVT VT = Val.getValueType();
11109 if (VT == SVT) {
11110 IsTruncating = false;
11111 } else if (!IsTruncating) {
11112 assert(VT == SVT && "No-truncating store from different memory type!");
11113 } else {
11114 assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&
11115 "Should only be a truncating store, not extending!");
11116 assert(VT.isInteger() == SVT.isInteger() && "Can't do FP-INT conversion!");
11117 assert(VT.isVector() == SVT.isVector() &&
11118 "Cannot use trunc store to convert to or from a vector!");
11119 assert((!VT.isVector() ||
11120 VT.getVectorElementCount() == SVT.getVectorElementCount()) &&
11121 "Cannot use trunc store to change the number of vector elements!");
11122 }
11123
11124 bool Indexed = AM != ISD::UNINDEXED;
11125 assert((Indexed || Offset.getOpcode() == ISD::POISON) &&
11126 "Unindexed store with an offset!");
11127 SDVTList VTs = Indexed ? getVTList(VT1: Ptr.getValueType(), VT2: MVT::Other)
11128 : getVTList(VT: MVT::Other);
11129 SDValue Ops[] = {Chain, Val, Ptr, Offset};
11130 SDNodeKey ID(ISD::STORE, VTs, Ops);
11131 ID.AddInteger(I: SVT.getRawBits());
11132 ID.AddInteger(I: getSyntheticNodeSubclassData<StoreSDNode>(
11133 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: IsTruncating, Args&: SVT, Args&: MMO));
11134 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11135 ID.AddInteger(I: MMO->getFlags());
11136 FoldingSetInsertToken InsertToken;
11137 if (SDNode *E = lookupNode(Key: ID, DL: dl, InsertToken)) {
11138 cast<StoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11139 cast<StoreSDNode>(Val: E)->refineMMOMetadata(NewMMO: MMO);
11140 return SDValue(E, 0);
11141 }
11142 auto *N = newSDNode<StoreSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM,
11143 Args&: IsTruncating, Args&: SVT, Args&: MMO);
11144 createOperands(Node: N, Vals: Ops);
11145
11146 CSEMap.insert(N, Token: InsertToken);
11147 InsertNode(N);
11148 SDValue V(N, 0);
11149 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11150 return V;
11151}
11152
11153SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
11154 SDValue Ptr, SDValue Offset,
11155 MachinePointerInfo PtrInfo, EVT SVT,
11156 Align Alignment,
11157 MachineMemOperand::Flags MMOFlags,
11158 const MMOMetadata &Metadata) {
11159 assert(Chain.getValueType() == MVT::Other &&
11160 "Invalid chain type");
11161
11162 MMOFlags |= MachineMemOperand::MOStore;
11163 assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
11164 assert(!Metadata.Ranges && "range metadata is invalid for stores");
11165
11166 if (PtrInfo.V.isNull())
11167 PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr);
11168
11169 MachineFunction &MF = getMachineFunction();
11170 MachineMemOperand *MMO = MF.getMachineMemOperand(
11171 PtrInfo, F: MMOFlags, Size: SVT.getStoreSize(), BaseAlignment: Alignment, Metadata);
11172 return getTruncStore(Chain, dl, Val, Ptr, Offset, SVT, MMO);
11173}
11174
11175SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
11176 SDValue Ptr, MachinePointerInfo PtrInfo,
11177 EVT SVT, Align Alignment,
11178 MachineMemOperand::Flags MMOFlags,
11179 const MMOMetadata &Metadata) {
11180 return getTruncStore(Chain, dl, Val, Ptr, Offset: getPOISON(VT: Ptr.getValueType()),
11181 PtrInfo, SVT, Alignment, MMOFlags, Metadata);
11182}
11183
11184SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
11185 SDValue Ptr, SDValue Offset, EVT SVT,
11186 MachineMemOperand *MMO) {
11187 return getStore(Chain, dl, Val, Ptr, Offset, SVT, MMO, AM: ISD::UNINDEXED, IsTruncating: true);
11188}
11189
11190SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
11191 SDValue Ptr, EVT SVT,
11192 MachineMemOperand *MMO) {
11193 return getStore(Chain, dl, Val, Ptr, Offset: getPOISON(VT: Ptr.getValueType()), SVT, MMO,
11194 AM: ISD::UNINDEXED, IsTruncating: true);
11195}
11196
11197SDValue SelectionDAG::getIndexedStore(SDValue OrigStore, const SDLoc &dl,
11198 SDValue Base, SDValue Offset,
11199 ISD::MemIndexedMode AM) {
11200 StoreSDNode *ST = cast<StoreSDNode>(Val&: OrigStore);
11201 assert(ST->getOffset().getOpcode() == ISD::POISON &&
11202 "Store is already a indexed store!");
11203 return getStore(Chain: ST->getChain(), dl, Val: ST->getValue(), Ptr: Base, Offset,
11204 SVT: ST->getMemoryVT(), MMO: ST->getMemOperand(), AM,
11205 IsTruncating: ST->isTruncatingStore());
11206}
11207
11208SDValue SelectionDAG::getLoadVP(
11209 ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &dl,
11210 SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL,
11211 MachinePointerInfo PtrInfo, EVT MemVT, Align Alignment,
11212 MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo,
11213 const MDNode *Ranges, bool IsExpanding) {
11214 MMOFlags |= MachineMemOperand::MOLoad;
11215 assert((MMOFlags & MachineMemOperand::MOStore) == 0);
11216 // If we don't have a PtrInfo, infer the trivial frame index case to simplify
11217 // clients.
11218 if (PtrInfo.V.isNull())
11219 PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr, OffsetOp: Offset);
11220
11221 TypeSize Size = MemVT.getStoreSize();
11222 MachineFunction &MF = getMachineFunction();
11223 MachineMemOperand *MMO = MF.getMachineMemOperand(
11224 PtrInfo, F: MMOFlags, Size, BaseAlignment: Alignment, Metadata: MMOMetadata(AAInfo, Ranges));
11225 return getLoadVP(AM, ExtType, VT, dl, Chain, Ptr, Offset, Mask, EVL, MemVT,
11226 MMO, IsExpanding);
11227}
11228
11229SDValue SelectionDAG::getLoadVP(ISD::MemIndexedMode AM,
11230 ISD::LoadExtType ExtType, EVT VT,
11231 const SDLoc &dl, SDValue Chain, SDValue Ptr,
11232 SDValue Offset, SDValue Mask, SDValue EVL,
11233 EVT MemVT, MachineMemOperand *MMO,
11234 bool IsExpanding) {
11235 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
11236 assert(Mask.getValueType().getVectorElementCount() ==
11237 VT.getVectorElementCount() &&
11238 "Vector width mismatch between mask and data");
11239
11240 bool Indexed = AM != ISD::UNINDEXED;
11241 assert((Indexed || Offset.getOpcode() == ISD::POISON) &&
11242 "Unindexed load with an offset!");
11243
11244 SDVTList VTs = Indexed ? getVTList(VT1: VT, VT2: Ptr.getValueType(), VT3: MVT::Other)
11245 : getVTList(VT1: VT, VT2: MVT::Other);
11246 SDValue Ops[] = {Chain, Ptr, Offset, Mask, EVL};
11247 SDNodeKey ID(ISD::VP_LOAD, VTs, Ops);
11248 ID.AddInteger(I: MemVT.getRawBits());
11249 ID.AddInteger(I: getSyntheticNodeSubclassData<VPLoadSDNode>(
11250 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: ExtType, Args&: IsExpanding, Args&: MemVT, Args&: MMO));
11251 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11252 ID.AddInteger(I: MMO->getFlags());
11253 FoldingSetInsertToken InsertToken;
11254 if (auto *E = cast_or_null<VPLoadSDNode>(Val: lookupNode(Key: ID, DL: dl, InsertToken))) {
11255 E->refineAlignment(NewMMO: MMO);
11256 E->refineMMOMetadata(NewMMO: MMO);
11257 return SDValue(E, 0);
11258 }
11259 auto *N = newSDNode<VPLoadSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM,
11260 Args&: ExtType, Args&: IsExpanding, Args&: MemVT, Args&: MMO);
11261 createOperands(Node: N, Vals: Ops);
11262
11263 CSEMap.insert(N, Token: InsertToken);
11264 InsertNode(N);
11265 SDValue V(N, 0);
11266 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11267 return V;
11268}
11269
11270SDValue SelectionDAG::getLoadVP(EVT VT, const SDLoc &dl, SDValue Chain,
11271 SDValue Ptr, SDValue Mask, SDValue EVL,
11272 MachinePointerInfo PtrInfo,
11273 MaybeAlign Alignment,
11274 MachineMemOperand::Flags MMOFlags,
11275 const AAMDNodes &AAInfo, const MDNode *Ranges,
11276 bool IsExpanding) {
11277 SDValue Undef = getPOISON(VT: Ptr.getValueType());
11278 return getLoadVP(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Offset: Undef,
11279 Mask, EVL, PtrInfo, MemVT: VT, Alignment, MMOFlags, AAInfo, Ranges,
11280 IsExpanding);
11281}
11282
11283SDValue SelectionDAG::getLoadVP(EVT VT, const SDLoc &dl, SDValue Chain,
11284 SDValue Ptr, SDValue Mask, SDValue EVL,
11285 MachineMemOperand *MMO, bool IsExpanding) {
11286 SDValue Undef = getPOISON(VT: Ptr.getValueType());
11287 return getLoadVP(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Offset: Undef,
11288 Mask, EVL, MemVT: VT, MMO, IsExpanding);
11289}
11290
11291SDValue SelectionDAG::getExtLoadVP(ISD::LoadExtType ExtType, const SDLoc &dl,
11292 EVT VT, SDValue Chain, SDValue Ptr,
11293 SDValue Mask, SDValue EVL,
11294 MachinePointerInfo PtrInfo, EVT MemVT,
11295 MaybeAlign Alignment,
11296 MachineMemOperand::Flags MMOFlags,
11297 const AAMDNodes &AAInfo, bool IsExpanding) {
11298 SDValue Undef = getPOISON(VT: Ptr.getValueType());
11299 return getLoadVP(AM: ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Offset: Undef, Mask,
11300 EVL, PtrInfo, MemVT, Alignment, MMOFlags, AAInfo, Ranges: nullptr,
11301 IsExpanding);
11302}
11303
11304SDValue SelectionDAG::getExtLoadVP(ISD::LoadExtType ExtType, const SDLoc &dl,
11305 EVT VT, SDValue Chain, SDValue Ptr,
11306 SDValue Mask, SDValue EVL, EVT MemVT,
11307 MachineMemOperand *MMO, bool IsExpanding) {
11308 SDValue Undef = getPOISON(VT: Ptr.getValueType());
11309 return getLoadVP(AM: ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Offset: Undef, Mask,
11310 EVL, MemVT, MMO, IsExpanding);
11311}
11312
11313SDValue SelectionDAG::getStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val,
11314 SDValue Ptr, SDValue Offset, SDValue Mask,
11315 SDValue EVL, EVT MemVT, MachineMemOperand *MMO,
11316 ISD::MemIndexedMode AM, bool IsTruncating,
11317 bool IsCompressing) {
11318 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
11319 assert(Mask.getValueType().getVectorElementCount() ==
11320 Val.getValueType().getVectorElementCount() &&
11321 "Vector width mismatch between mask and data");
11322
11323 bool Indexed = AM != ISD::UNINDEXED;
11324 assert((Indexed || Offset.getOpcode() == ISD::POISON) &&
11325 "Unindexed vp_store with an offset!");
11326 SDVTList VTs = Indexed ? getVTList(VT1: Ptr.getValueType(), VT2: MVT::Other)
11327 : getVTList(VT: MVT::Other);
11328 SDValue Ops[] = {Chain, Val, Ptr, Offset, Mask, EVL};
11329 SDNodeKey ID(ISD::VP_STORE, VTs, Ops);
11330 ID.AddInteger(I: MemVT.getRawBits());
11331 ID.AddInteger(I: getSyntheticNodeSubclassData<VPStoreSDNode>(
11332 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO));
11333 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11334 ID.AddInteger(I: MMO->getFlags());
11335 FoldingSetInsertToken InsertToken;
11336 if (SDNode *E = lookupNode(Key: ID, DL: dl, InsertToken)) {
11337 cast<VPStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11338 return SDValue(E, 0);
11339 }
11340 auto *N = newSDNode<VPStoreSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM,
11341 Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO);
11342 createOperands(Node: N, Vals: Ops);
11343
11344 CSEMap.insert(N, Token: InsertToken);
11345 InsertNode(N);
11346 SDValue V(N, 0);
11347 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11348 return V;
11349}
11350
11351SDValue SelectionDAG::getTruncStoreVP(SDValue Chain, const SDLoc &dl,
11352 SDValue Val, SDValue Ptr, SDValue Mask,
11353 SDValue EVL, MachinePointerInfo PtrInfo,
11354 EVT SVT, Align Alignment,
11355 MachineMemOperand::Flags MMOFlags,
11356 const AAMDNodes &AAInfo,
11357 bool IsCompressing) {
11358 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
11359
11360 MMOFlags |= MachineMemOperand::MOStore;
11361 assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
11362
11363 if (PtrInfo.V.isNull())
11364 PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr);
11365
11366 MachineFunction &MF = getMachineFunction();
11367 MachineMemOperand *MMO = MF.getMachineMemOperand(
11368 PtrInfo, F: MMOFlags, Size: SVT.getStoreSize(), BaseAlignment: Alignment, Metadata: AAInfo);
11369 return getTruncStoreVP(Chain, dl, Val, Ptr, Mask, EVL, SVT, MMO,
11370 IsCompressing);
11371}
11372
11373SDValue SelectionDAG::getTruncStoreVP(SDValue Chain, const SDLoc &dl,
11374 SDValue Val, SDValue Ptr, SDValue Mask,
11375 SDValue EVL, EVT SVT,
11376 MachineMemOperand *MMO,
11377 bool IsCompressing) {
11378 EVT VT = Val.getValueType();
11379
11380 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
11381 if (VT == SVT)
11382 return getStoreVP(Chain, dl, Val, Ptr, Offset: getPOISON(VT: Ptr.getValueType()), Mask,
11383 EVL, MemVT: VT, MMO, AM: ISD::UNINDEXED,
11384 /*IsTruncating*/ false, IsCompressing);
11385
11386 assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&
11387 "Should only be a truncating store, not extending!");
11388 assert(VT.isInteger() == SVT.isInteger() && "Can't do FP-INT conversion!");
11389 assert(VT.isVector() == SVT.isVector() &&
11390 "Cannot use trunc store to convert to or from a vector!");
11391 assert((!VT.isVector() ||
11392 VT.getVectorElementCount() == SVT.getVectorElementCount()) &&
11393 "Cannot use trunc store to change the number of vector elements!");
11394
11395 SDVTList VTs = getVTList(VT: MVT::Other);
11396 SDValue Undef = getPOISON(VT: Ptr.getValueType());
11397 SDValue Ops[] = {Chain, Val, Ptr, Undef, Mask, EVL};
11398 SDNodeKey ID(ISD::VP_STORE, VTs, Ops);
11399 ID.AddInteger(I: SVT.getRawBits());
11400 ID.AddInteger(I: getSyntheticNodeSubclassData<VPStoreSDNode>(
11401 IROrder: dl.getIROrder(), Args&: VTs, Args: ISD::UNINDEXED, Args: true, Args&: IsCompressing, Args&: SVT, Args&: MMO));
11402 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11403 ID.AddInteger(I: MMO->getFlags());
11404 FoldingSetInsertToken InsertToken;
11405 if (SDNode *E = lookupNode(Key: ID, DL: dl, InsertToken)) {
11406 cast<VPStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11407 return SDValue(E, 0);
11408 }
11409 auto *N =
11410 newSDNode<VPStoreSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs,
11411 Args: ISD::UNINDEXED, Args: true, Args&: IsCompressing, Args&: SVT, Args&: MMO);
11412 createOperands(Node: N, Vals: Ops);
11413
11414 CSEMap.insert(N, Token: InsertToken);
11415 InsertNode(N);
11416 SDValue V(N, 0);
11417 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11418 return V;
11419}
11420
11421SDValue SelectionDAG::getStridedLoadVP(
11422 ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &DL,
11423 SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask,
11424 SDValue EVL, EVT MemVT, MachineMemOperand *MMO, bool IsExpanding) {
11425 bool Indexed = AM != ISD::UNINDEXED;
11426 assert((Indexed || Offset.getOpcode() == ISD::POISON) &&
11427 "Unindexed load with an offset!");
11428
11429 SDValue Ops[] = {Chain, Ptr, Offset, Stride, Mask, EVL};
11430 SDVTList VTs = Indexed ? getVTList(VT1: VT, VT2: Ptr.getValueType(), VT3: MVT::Other)
11431 : getVTList(VT1: VT, VT2: MVT::Other);
11432 SDNodeKey ID(ISD::EXPERIMENTAL_VP_STRIDED_LOAD, VTs, Ops);
11433 ID.AddInteger(I: MemVT.getRawBits());
11434 ID.AddInteger(I: getSyntheticNodeSubclassData<VPStridedLoadSDNode>(
11435 IROrder: DL.getIROrder(), Args&: VTs, Args&: AM, Args&: ExtType, Args&: IsExpanding, Args&: MemVT, Args&: MMO));
11436 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11437
11438 FoldingSetInsertToken InsertToken;
11439 if (SDNode *E = lookupNode(Key: ID, DL, InsertToken)) {
11440 cast<VPStridedLoadSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11441 return SDValue(E, 0);
11442 }
11443
11444 auto *N =
11445 newSDNode<VPStridedLoadSDNode>(Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs, Args&: AM,
11446 Args&: ExtType, Args&: IsExpanding, Args&: MemVT, Args&: MMO);
11447 createOperands(Node: N, Vals: Ops);
11448 CSEMap.insert(N, Token: InsertToken);
11449 InsertNode(N);
11450 SDValue V(N, 0);
11451 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11452 return V;
11453}
11454
11455SDValue SelectionDAG::getStridedLoadVP(EVT VT, const SDLoc &DL, SDValue Chain,
11456 SDValue Ptr, SDValue Stride,
11457 SDValue Mask, SDValue EVL,
11458 MachineMemOperand *MMO,
11459 bool IsExpanding) {
11460 SDValue Undef = getPOISON(VT: Ptr.getValueType());
11461 return getStridedLoadVP(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, DL, Chain, Ptr,
11462 Offset: Undef, Stride, Mask, EVL, MemVT: VT, MMO, IsExpanding);
11463}
11464
11465SDValue SelectionDAG::getExtStridedLoadVP(
11466 ISD::LoadExtType ExtType, const SDLoc &DL, EVT VT, SDValue Chain,
11467 SDValue Ptr, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT,
11468 MachineMemOperand *MMO, bool IsExpanding) {
11469 SDValue Undef = getPOISON(VT: Ptr.getValueType());
11470 return getStridedLoadVP(AM: ISD::UNINDEXED, ExtType, VT, DL, Chain, Ptr, Offset: Undef,
11471 Stride, Mask, EVL, MemVT, MMO, IsExpanding);
11472}
11473
11474SDValue SelectionDAG::getStridedStoreVP(SDValue Chain, const SDLoc &DL,
11475 SDValue Val, SDValue Ptr,
11476 SDValue Offset, SDValue Stride,
11477 SDValue Mask, SDValue EVL, EVT MemVT,
11478 MachineMemOperand *MMO,
11479 ISD::MemIndexedMode AM,
11480 bool IsTruncating, bool IsCompressing) {
11481 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
11482 bool Indexed = AM != ISD::UNINDEXED;
11483 assert((Indexed || Offset.getOpcode() == ISD::POISON) &&
11484 "Unindexed vp_store with an offset!");
11485 SDVTList VTs = Indexed ? getVTList(VT1: Ptr.getValueType(), VT2: MVT::Other)
11486 : getVTList(VT: MVT::Other);
11487 SDValue Ops[] = {Chain, Val, Ptr, Offset, Stride, Mask, EVL};
11488 SDNodeKey ID(ISD::EXPERIMENTAL_VP_STRIDED_STORE, VTs, Ops);
11489 ID.AddInteger(I: MemVT.getRawBits());
11490 ID.AddInteger(I: getSyntheticNodeSubclassData<VPStridedStoreSDNode>(
11491 IROrder: DL.getIROrder(), Args&: VTs, Args&: AM, Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO));
11492 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11493 FoldingSetInsertToken InsertToken;
11494 if (SDNode *E = lookupNode(Key: ID, DL, InsertToken)) {
11495 cast<VPStridedStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11496 return SDValue(E, 0);
11497 }
11498 auto *N = newSDNode<VPStridedStoreSDNode>(Args: DL.getIROrder(), Args: DL.getDebugLoc(),
11499 Args&: VTs, Args&: AM, Args&: IsTruncating,
11500 Args&: IsCompressing, Args&: MemVT, Args&: MMO);
11501 createOperands(Node: N, Vals: Ops);
11502
11503 CSEMap.insert(N, Token: InsertToken);
11504 InsertNode(N);
11505 SDValue V(N, 0);
11506 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11507 return V;
11508}
11509
11510SDValue SelectionDAG::getGatherVP(SDVTList VTs, EVT VT, const SDLoc &dl,
11511 ArrayRef<SDValue> Ops, MachineMemOperand *MMO,
11512 ISD::MemIndexType IndexType) {
11513 assert(Ops.size() == 6 && "Incompatible number of operands");
11514
11515 SDNodeKey ID(ISD::VP_GATHER, VTs, Ops);
11516 ID.AddInteger(I: VT.getRawBits());
11517 ID.AddInteger(I: getSyntheticNodeSubclassData<VPGatherSDNode>(
11518 IROrder: dl.getIROrder(), Args&: VTs, Args&: VT, Args&: MMO, Args&: IndexType));
11519 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11520 ID.AddInteger(I: MMO->getFlags());
11521 FoldingSetInsertToken InsertToken;
11522 if (SDNode *E = lookupNode(Key: ID, DL: dl, InsertToken)) {
11523 cast<VPGatherSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11524 return SDValue(E, 0);
11525 }
11526
11527 auto *N = newSDNode<VPGatherSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs,
11528 Args&: VT, Args&: MMO, Args&: IndexType);
11529 createOperands(Node: N, Vals: Ops);
11530
11531 assert(N->getMask().getValueType().getVectorElementCount() ==
11532 N->getValueType(0).getVectorElementCount() &&
11533 "Vector width mismatch between mask and data");
11534 assert(N->getIndex().getValueType().getVectorElementCount().isScalable() ==
11535 N->getValueType(0).getVectorElementCount().isScalable() &&
11536 "Scalable flags of index and data do not match");
11537 assert(ElementCount::isKnownGE(
11538 N->getIndex().getValueType().getVectorElementCount(),
11539 N->getValueType(0).getVectorElementCount()) &&
11540 "Vector width mismatch between index and data");
11541 assert(isa<ConstantSDNode>(N->getScale()) &&
11542 N->getScale()->getAsAPIntVal().isPowerOf2() &&
11543 "Scale should be a constant power of 2");
11544
11545 CSEMap.insert(N, Token: InsertToken);
11546 InsertNode(N);
11547 SDValue V(N, 0);
11548 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11549 return V;
11550}
11551
11552SDValue SelectionDAG::getScatterVP(SDVTList VTs, EVT VT, const SDLoc &dl,
11553 ArrayRef<SDValue> Ops,
11554 MachineMemOperand *MMO,
11555 ISD::MemIndexType IndexType) {
11556 assert(Ops.size() == 7 && "Incompatible number of operands");
11557
11558 SDNodeKey ID(ISD::VP_SCATTER, VTs, Ops);
11559 ID.AddInteger(I: VT.getRawBits());
11560 ID.AddInteger(I: getSyntheticNodeSubclassData<VPScatterSDNode>(
11561 IROrder: dl.getIROrder(), Args&: VTs, Args&: VT, Args&: MMO, Args&: IndexType));
11562 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11563 ID.AddInteger(I: MMO->getFlags());
11564 FoldingSetInsertToken InsertToken;
11565 if (SDNode *E = lookupNode(Key: ID, DL: dl, InsertToken)) {
11566 cast<VPScatterSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11567 return SDValue(E, 0);
11568 }
11569 auto *N = newSDNode<VPScatterSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs,
11570 Args&: VT, Args&: MMO, Args&: IndexType);
11571 createOperands(Node: N, Vals: Ops);
11572
11573 assert(N->getMask().getValueType().getVectorElementCount() ==
11574 N->getValue().getValueType().getVectorElementCount() &&
11575 "Vector width mismatch between mask and data");
11576 assert(
11577 N->getIndex().getValueType().getVectorElementCount().isScalable() ==
11578 N->getValue().getValueType().getVectorElementCount().isScalable() &&
11579 "Scalable flags of index and data do not match");
11580 assert(ElementCount::isKnownGE(
11581 N->getIndex().getValueType().getVectorElementCount(),
11582 N->getValue().getValueType().getVectorElementCount()) &&
11583 "Vector width mismatch between index and data");
11584 assert(isa<ConstantSDNode>(N->getScale()) &&
11585 N->getScale()->getAsAPIntVal().isPowerOf2() &&
11586 "Scale should be a constant power of 2");
11587
11588 CSEMap.insert(N, Token: InsertToken);
11589 InsertNode(N);
11590 SDValue V(N, 0);
11591 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11592 return V;
11593}
11594
11595SDValue SelectionDAG::getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain,
11596 SDValue Base, SDValue Offset, SDValue Mask,
11597 SDValue PassThru, EVT MemVT,
11598 MachineMemOperand *MMO,
11599 ISD::MemIndexedMode AM,
11600 ISD::LoadExtType ExtTy, bool isExpanding) {
11601 bool Indexed = AM != ISD::UNINDEXED;
11602 assert((Indexed || Offset.getOpcode() == ISD::POISON) &&
11603 "Unindexed masked load with an offset!");
11604 SDVTList VTs = Indexed ? getVTList(VT1: VT, VT2: Base.getValueType(), VT3: MVT::Other)
11605 : getVTList(VT1: VT, VT2: MVT::Other);
11606 SDValue Ops[] = {Chain, Base, Offset, Mask, PassThru};
11607 SDNodeKey ID(ISD::MLOAD, VTs, Ops);
11608 ID.AddInteger(I: MemVT.getRawBits());
11609 ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedLoadSDNode>(
11610 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: ExtTy, Args&: isExpanding, Args&: MemVT, Args&: MMO));
11611 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11612 ID.AddInteger(I: MMO->getFlags());
11613 FoldingSetInsertToken InsertToken;
11614 if (SDNode *E = lookupNode(Key: ID, DL: dl, InsertToken)) {
11615 cast<MaskedLoadSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11616 return SDValue(E, 0);
11617 }
11618 auto *N = newSDNode<MaskedLoadSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs,
11619 Args&: AM, Args&: ExtTy, Args&: isExpanding, Args&: MemVT, Args&: MMO);
11620 createOperands(Node: N, Vals: Ops);
11621
11622 CSEMap.insert(N, Token: InsertToken);
11623 InsertNode(N);
11624 SDValue V(N, 0);
11625 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11626 return V;
11627}
11628
11629SDValue SelectionDAG::getIndexedMaskedLoad(SDValue OrigLoad, const SDLoc &dl,
11630 SDValue Base, SDValue Offset,
11631 ISD::MemIndexedMode AM) {
11632 MaskedLoadSDNode *LD = cast<MaskedLoadSDNode>(Val&: OrigLoad);
11633 assert(LD->getOffset().getOpcode() == ISD::POISON &&
11634 "Masked load is already a indexed load!");
11635 return getMaskedLoad(VT: OrigLoad.getValueType(), dl, Chain: LD->getChain(), Base,
11636 Offset, Mask: LD->getMask(), PassThru: LD->getPassThru(),
11637 MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand(), AM,
11638 ExtTy: LD->getExtensionType(), isExpanding: LD->isExpandingLoad());
11639}
11640
11641SDValue SelectionDAG::getMaskedStore(SDValue Chain, const SDLoc &dl,
11642 SDValue Val, SDValue Base, SDValue Offset,
11643 SDValue Mask, EVT MemVT,
11644 MachineMemOperand *MMO,
11645 ISD::MemIndexedMode AM, bool IsTruncating,
11646 bool IsCompressing) {
11647 assert(Chain.getValueType() == MVT::Other &&
11648 "Invalid chain type");
11649 bool Indexed = AM != ISD::UNINDEXED;
11650 assert((Indexed || Offset.getOpcode() == ISD::POISON) &&
11651 "Unindexed masked store with an offset!");
11652 SDVTList VTs = Indexed ? getVTList(VT1: Base.getValueType(), VT2: MVT::Other)
11653 : getVTList(VT: MVT::Other);
11654 SDValue Ops[] = {Chain, Val, Base, Offset, Mask};
11655 SDNodeKey ID(ISD::MSTORE, VTs, Ops);
11656 ID.AddInteger(I: MemVT.getRawBits());
11657 ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedStoreSDNode>(
11658 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO));
11659 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11660 ID.AddInteger(I: MMO->getFlags());
11661 FoldingSetInsertToken InsertToken;
11662 if (SDNode *E = lookupNode(Key: ID, DL: dl, InsertToken)) {
11663 cast<MaskedStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11664 return SDValue(E, 0);
11665 }
11666 auto *N =
11667 newSDNode<MaskedStoreSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM,
11668 Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO);
11669 createOperands(Node: N, Vals: Ops);
11670
11671 CSEMap.insert(N, Token: InsertToken);
11672 InsertNode(N);
11673 SDValue V(N, 0);
11674 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11675 return V;
11676}
11677
11678SDValue SelectionDAG::getIndexedMaskedStore(SDValue OrigStore, const SDLoc &dl,
11679 SDValue Base, SDValue Offset,
11680 ISD::MemIndexedMode AM) {
11681 MaskedStoreSDNode *ST = cast<MaskedStoreSDNode>(Val&: OrigStore);
11682 assert(ST->getOffset().getOpcode() == ISD::POISON &&
11683 "Masked store is already a indexed store!");
11684 return getMaskedStore(Chain: ST->getChain(), dl, Val: ST->getValue(), Base, Offset,
11685 Mask: ST->getMask(), MemVT: ST->getMemoryVT(), MMO: ST->getMemOperand(),
11686 AM, IsTruncating: ST->isTruncatingStore(), IsCompressing: ST->isCompressingStore());
11687}
11688
11689SDValue SelectionDAG::getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl,
11690 ArrayRef<SDValue> Ops,
11691 MachineMemOperand *MMO,
11692 ISD::MemIndexType IndexType,
11693 ISD::LoadExtType ExtTy) {
11694 assert(Ops.size() == 6 && "Incompatible number of operands");
11695
11696 SDNodeKey ID(ISD::MGATHER, VTs, Ops);
11697 ID.AddInteger(I: MemVT.getRawBits());
11698 ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedGatherSDNode>(
11699 IROrder: dl.getIROrder(), Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType, Args&: ExtTy));
11700 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11701 ID.AddInteger(I: MMO->getFlags());
11702 FoldingSetInsertToken InsertToken;
11703 if (SDNode *E = lookupNode(Key: ID, DL: dl, InsertToken)) {
11704 cast<MaskedGatherSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11705 return SDValue(E, 0);
11706 }
11707
11708 auto *N = newSDNode<MaskedGatherSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(),
11709 Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType, Args&: ExtTy);
11710 createOperands(Node: N, Vals: Ops);
11711
11712 assert(N->getPassThru().getValueType() == N->getValueType(0) &&
11713 "Incompatible type of the PassThru value in MaskedGatherSDNode");
11714 assert(N->getMask().getValueType().getVectorElementCount() ==
11715 N->getValueType(0).getVectorElementCount() &&
11716 "Vector width mismatch between mask and data");
11717 assert(N->getIndex().getValueType().getVectorElementCount().isScalable() ==
11718 N->getValueType(0).getVectorElementCount().isScalable() &&
11719 "Scalable flags of index and data do not match");
11720 assert(ElementCount::isKnownGE(
11721 N->getIndex().getValueType().getVectorElementCount(),
11722 N->getValueType(0).getVectorElementCount()) &&
11723 "Vector width mismatch between index and data");
11724 assert(isa<ConstantSDNode>(N->getScale()) &&
11725 N->getScale()->getAsAPIntVal().isPowerOf2() &&
11726 "Scale should be a constant power of 2");
11727
11728 CSEMap.insert(N, Token: InsertToken);
11729 InsertNode(N);
11730 SDValue V(N, 0);
11731 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11732 return V;
11733}
11734
11735SDValue SelectionDAG::getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl,
11736 ArrayRef<SDValue> Ops,
11737 MachineMemOperand *MMO,
11738 ISD::MemIndexType IndexType,
11739 bool IsTrunc) {
11740 assert(Ops.size() == 6 && "Incompatible number of operands");
11741
11742 SDNodeKey ID(ISD::MSCATTER, VTs, Ops);
11743 ID.AddInteger(I: MemVT.getRawBits());
11744 ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedScatterSDNode>(
11745 IROrder: dl.getIROrder(), Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType, Args&: IsTrunc));
11746 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11747 ID.AddInteger(I: MMO->getFlags());
11748 FoldingSetInsertToken InsertToken;
11749 if (SDNode *E = lookupNode(Key: ID, DL: dl, InsertToken)) {
11750 cast<MaskedScatterSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11751 return SDValue(E, 0);
11752 }
11753
11754 auto *N = newSDNode<MaskedScatterSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(),
11755 Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType, Args&: IsTrunc);
11756 createOperands(Node: N, Vals: Ops);
11757
11758 assert(N->getMask().getValueType().getVectorElementCount() ==
11759 N->getValue().getValueType().getVectorElementCount() &&
11760 "Vector width mismatch between mask and data");
11761 assert(
11762 N->getIndex().getValueType().getVectorElementCount().isScalable() ==
11763 N->getValue().getValueType().getVectorElementCount().isScalable() &&
11764 "Scalable flags of index and data do not match");
11765 assert(ElementCount::isKnownGE(
11766 N->getIndex().getValueType().getVectorElementCount(),
11767 N->getValue().getValueType().getVectorElementCount()) &&
11768 "Vector width mismatch between index and data");
11769 assert(isa<ConstantSDNode>(N->getScale()) &&
11770 N->getScale()->getAsAPIntVal().isPowerOf2() &&
11771 "Scale should be a constant power of 2");
11772
11773 CSEMap.insert(N, Token: InsertToken);
11774 InsertNode(N);
11775 SDValue V(N, 0);
11776 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11777 return V;
11778}
11779
11780SDValue SelectionDAG::getMaskedHistogram(SDVTList VTs, EVT MemVT,
11781 const SDLoc &dl, ArrayRef<SDValue> Ops,
11782 MachineMemOperand *MMO,
11783 ISD::MemIndexType IndexType) {
11784 assert(Ops.size() == 7 && "Incompatible number of operands");
11785
11786 SDNodeKey ID(ISD::EXPERIMENTAL_VECTOR_HISTOGRAM, VTs, Ops);
11787 ID.AddInteger(I: MemVT.getRawBits());
11788 ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedHistogramSDNode>(
11789 IROrder: dl.getIROrder(), Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType));
11790 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11791 ID.AddInteger(I: MMO->getFlags());
11792 FoldingSetInsertToken InsertToken;
11793 if (SDNode *E = lookupNode(Key: ID, DL: dl, InsertToken)) {
11794 cast<MaskedGatherSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11795 return SDValue(E, 0);
11796 }
11797
11798 auto *N = newSDNode<MaskedHistogramSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(),
11799 Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType);
11800 createOperands(Node: N, Vals: Ops);
11801
11802 assert(N->getMask().getValueType().getVectorElementCount() ==
11803 N->getIndex().getValueType().getVectorElementCount() &&
11804 "Vector width mismatch between mask and data");
11805 assert(isa<ConstantSDNode>(N->getScale()) &&
11806 N->getScale()->getAsAPIntVal().isPowerOf2() &&
11807 "Scale should be a constant power of 2");
11808 assert(N->getInc().getValueType().isInteger() && "Non integer update value");
11809
11810 CSEMap.insert(N, Token: InsertToken);
11811 InsertNode(N);
11812 SDValue V(N, 0);
11813 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11814 return V;
11815}
11816
11817SDValue SelectionDAG::getLoadFFVP(EVT VT, const SDLoc &DL, SDValue Chain,
11818 SDValue Ptr, SDValue Mask, SDValue EVL,
11819 MachineMemOperand *MMO) {
11820 SDVTList VTs = getVTList(VT1: VT, VT2: EVL.getValueType(), VT3: MVT::Other);
11821 SDValue Ops[] = {Chain, Ptr, Mask, EVL};
11822 SDNodeKey ID(ISD::VP_LOAD_FF, VTs, Ops);
11823 ID.AddInteger(I: VT.getRawBits());
11824 ID.AddInteger(I: getSyntheticNodeSubclassData<VPLoadFFSDNode>(IROrder: DL.getIROrder(),
11825 Args&: VTs, Args&: VT, Args&: MMO));
11826 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11827 ID.AddInteger(I: MMO->getFlags());
11828 FoldingSetInsertToken InsertToken;
11829 if (SDNode *E = lookupNode(Key: ID, DL, InsertToken)) {
11830 cast<VPLoadFFSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11831 return SDValue(E, 0);
11832 }
11833 auto *N = newSDNode<VPLoadFFSDNode>(Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs,
11834 Args&: VT, Args&: MMO);
11835 createOperands(Node: N, Vals: Ops);
11836
11837 CSEMap.insert(N, Token: InsertToken);
11838 InsertNode(N);
11839 SDValue V(N, 0);
11840 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11841 return V;
11842}
11843
11844SDValue SelectionDAG::getGetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr,
11845 EVT MemVT, MachineMemOperand *MMO) {
11846 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
11847 SDVTList VTs = getVTList(VT: MVT::Other);
11848 SDValue Ops[] = {Chain, Ptr};
11849 SDNodeKey ID(ISD::GET_FPENV_MEM, VTs, Ops);
11850 ID.AddInteger(I: MemVT.getRawBits());
11851 ID.AddInteger(I: getSyntheticNodeSubclassData<FPStateAccessSDNode>(
11852 Opc: ISD::GET_FPENV_MEM, Order: dl.getIROrder(), VTs, MemoryVT: MemVT, MMO));
11853 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11854 ID.AddInteger(I: MMO->getFlags());
11855 FoldingSetInsertToken InsertToken;
11856 if (SDNode *E = lookupNode(Key: ID, DL: dl, InsertToken))
11857 return SDValue(E, 0);
11858
11859 auto *N = newSDNode<FPStateAccessSDNode>(Args: ISD::GET_FPENV_MEM, Args: dl.getIROrder(),
11860 Args: dl.getDebugLoc(), Args&: VTs, Args&: MemVT, Args&: MMO);
11861 createOperands(Node: N, Vals: Ops);
11862
11863 CSEMap.insert(N, Token: InsertToken);
11864 InsertNode(N);
11865 SDValue V(N, 0);
11866 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11867 return V;
11868}
11869
11870SDValue SelectionDAG::getSetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr,
11871 EVT MemVT, MachineMemOperand *MMO) {
11872 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
11873 SDVTList VTs = getVTList(VT: MVT::Other);
11874 SDValue Ops[] = {Chain, Ptr};
11875 SDNodeKey ID(ISD::SET_FPENV_MEM, VTs, Ops);
11876 ID.AddInteger(I: MemVT.getRawBits());
11877 ID.AddInteger(I: getSyntheticNodeSubclassData<FPStateAccessSDNode>(
11878 Opc: ISD::SET_FPENV_MEM, Order: dl.getIROrder(), VTs, MemoryVT: MemVT, MMO));
11879 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11880 ID.AddInteger(I: MMO->getFlags());
11881 FoldingSetInsertToken InsertToken;
11882 if (SDNode *E = lookupNode(Key: ID, DL: dl, InsertToken))
11883 return SDValue(E, 0);
11884
11885 auto *N = newSDNode<FPStateAccessSDNode>(Args: ISD::SET_FPENV_MEM, Args: dl.getIROrder(),
11886 Args: dl.getDebugLoc(), Args&: VTs, Args&: MemVT, Args&: MMO);
11887 createOperands(Node: N, Vals: Ops);
11888
11889 CSEMap.insert(N, Token: InsertToken);
11890 InsertNode(N);
11891 SDValue V(N, 0);
11892 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11893 return V;
11894}
11895
11896SDValue SelectionDAG::simplifySelect(SDValue Cond, SDValue T, SDValue F) {
11897 // select undef, T, F --> T (if T is a constant), otherwise F
11898 // select, ?, undef, F --> F
11899 // select, ?, T, undef --> T
11900 if (Cond.isUndef())
11901 return isConstantValueOfAnyType(N: T) ? T : F;
11902 if (T.isUndef())
11903 return isGuaranteedNotToBePoison(Op: F) ? F : getFreeze(V: F);
11904 if (F.isUndef())
11905 return isGuaranteedNotToBePoison(Op: T) ? T : getFreeze(V: T);
11906
11907 // select true, T, F --> T
11908 // select false, T, F --> F
11909 if (auto C = isBoolConstant(N: Cond))
11910 return *C ? T : F;
11911
11912 // select ?, T, T --> T
11913 if (T == F)
11914 return T;
11915
11916 return SDValue();
11917}
11918
11919SDValue SelectionDAG::simplifyShift(SDValue X, SDValue Y) {
11920 // shift undef, Y --> 0 (can always assume that the undef value is 0)
11921 if (X.isUndef())
11922 return getConstant(Val: 0, DL: SDLoc(X.getNode()), VT: X.getValueType());
11923 // shift X, undef --> undef (because it may shift by the bitwidth)
11924 if (Y.isUndef())
11925 return getUNDEF(VT: X.getValueType());
11926
11927 // shift 0, Y --> 0
11928 // shift X, 0 --> X
11929 if (isNullOrNullSplat(V: X) || isNullOrNullSplat(V: Y))
11930 return X;
11931
11932 // shift X, C >= bitwidth(X) --> undef
11933 // All vector elements must be too big (or undef) to avoid partial undefs.
11934 auto isShiftTooBig = [X](ConstantSDNode *Val) {
11935 return !Val || Val->getAPIntValue().uge(RHS: X.getScalarValueSizeInBits());
11936 };
11937 if (ISD::matchUnaryPredicate(Op: Y, Match: isShiftTooBig, AllowUndefs: true))
11938 return getUNDEF(VT: X.getValueType());
11939
11940 // shift i1/vXi1 X, Y --> X (any non-zero shift amount is undefined).
11941 if (X.getValueType().getScalarType() == MVT::i1)
11942 return X;
11943
11944 return SDValue();
11945}
11946
11947SDValue SelectionDAG::simplifyFPBinop(unsigned Opcode, SDValue X, SDValue Y,
11948 SDNodeFlags Flags) {
11949 // If this operation has 'nnan' or 'ninf' and at least 1 disallowed operand
11950 // (an undef operand can be chosen to be Nan/Inf), then the result of this
11951 // operation is poison. That result can be relaxed to undef.
11952 ConstantFPSDNode *XC = isConstOrConstSplatFP(N: X, /* AllowUndefs */ true);
11953 ConstantFPSDNode *YC = isConstOrConstSplatFP(N: Y, /* AllowUndefs */ true);
11954 bool HasNan = (XC && XC->getValueAPF().isNaN()) ||
11955 (YC && YC->getValueAPF().isNaN());
11956 bool HasInf = (XC && XC->getValueAPF().isInfinity()) ||
11957 (YC && YC->getValueAPF().isInfinity());
11958
11959 if (Flags.hasNoNaNs() && (HasNan || X.isUndef() || Y.isUndef()))
11960 return getUNDEF(VT: X.getValueType());
11961
11962 if (Flags.hasNoInfs() && (HasInf || X.isUndef() || Y.isUndef()))
11963 return getUNDEF(VT: X.getValueType());
11964
11965 if (!YC)
11966 return SDValue();
11967
11968 // X + -0.0 --> X
11969 if (Opcode == ISD::FADD)
11970 if (YC->getValueAPF().isNegZero())
11971 return X;
11972
11973 // X - +0.0 --> X
11974 if (Opcode == ISD::FSUB)
11975 if (YC->getValueAPF().isPosZero())
11976 return X;
11977
11978 // X * 1.0 --> X
11979 // X / 1.0 --> X
11980 if (Opcode == ISD::FMUL || Opcode == ISD::FDIV)
11981 if (YC->getValueAPF().isOne())
11982 return X;
11983
11984 // X * 0.0 --> 0.0
11985 if (Opcode == ISD::FMUL && Flags.hasNoNaNs() && Flags.hasNoSignedZeros())
11986 if (YC->getValueAPF().isZero())
11987 return getConstantFP(Val: 0.0, DL: SDLoc(Y), VT: Y.getValueType());
11988
11989 return SDValue();
11990}
11991
11992SDValue SelectionDAG::getVAArg(EVT VT, const SDLoc &dl, SDValue Chain,
11993 SDValue Ptr, SDValue SV, unsigned Align) {
11994 SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Val: Align, DL: dl, VT: MVT::i32) };
11995 return getNode(Opcode: ISD::VAARG, DL: dl, VTList: getVTList(VT1: VT, VT2: MVT::Other), Ops);
11996}
11997
11998SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
11999 ArrayRef<SDUse> Ops) {
12000 switch (Ops.size()) {
12001 case 0: return getNode(Opcode, DL, VT);
12002 case 1: return getNode(Opcode, DL, VT, N1: Ops[0].get());
12003 case 2: return getNode(Opcode, DL, VT, N1: Ops[0], N2: Ops[1]);
12004 case 3: return getNode(Opcode, DL, VT, N1: Ops[0], N2: Ops[1], N3: Ops[2]);
12005 default: break;
12006 }
12007
12008 // Copy from an SDUse array into an SDValue array for use with
12009 // the regular getNode logic.
12010 SmallVector<SDValue, 8> NewOps(Ops);
12011 return getNode(Opcode, DL, VT, Ops: NewOps);
12012}
12013
12014SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
12015 ArrayRef<SDValue> Ops) {
12016 SDNodeFlags Flags;
12017 if (Inserter)
12018 Flags = Inserter->getFlags();
12019 return getNode(Opcode, DL, VT, Ops, Flags);
12020}
12021
12022SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
12023 ArrayRef<SDValue> Ops, const SDNodeFlags Flags) {
12024 unsigned NumOps = Ops.size();
12025 switch (NumOps) {
12026 case 0: return getNode(Opcode, DL, VT);
12027 case 1: return getNode(Opcode, DL, VT, N1: Ops[0], Flags);
12028 case 2: return getNode(Opcode, DL, VT, N1: Ops[0], N2: Ops[1], Flags);
12029 case 3: return getNode(Opcode, DL, VT, N1: Ops[0], N2: Ops[1], N3: Ops[2], Flags);
12030 default: break;
12031 }
12032
12033#ifndef NDEBUG
12034 for (const auto &Op : Ops)
12035 assert(Op.getOpcode() != ISD::DELETED_NODE &&
12036 "Operand is DELETED_NODE!");
12037#endif
12038
12039 switch (Opcode) {
12040 default: break;
12041 case ISD::BUILD_VECTOR:
12042 // Attempt to simplify BUILD_VECTOR.
12043 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, DAG&: *this))
12044 return V;
12045 break;
12046 case ISD::CONCAT_VECTORS:
12047 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, DAG&: *this))
12048 return V;
12049 break;
12050 case ISD::SELECT_CC:
12051 assert(NumOps == 5 && "SELECT_CC takes 5 operands!");
12052 assert(Ops[0].getValueType() == Ops[1].getValueType() &&
12053 "LHS and RHS of condition must have same type!");
12054 assert(Ops[2].getValueType() == Ops[3].getValueType() &&
12055 "True and False arms of SelectCC must have same type!");
12056 assert(Ops[2].getValueType() == VT &&
12057 "select_cc node must be of same type as true and false value!");
12058 assert((!Ops[0].getValueType().isVector() ||
12059 Ops[0].getValueType().getVectorElementCount() ==
12060 VT.getVectorElementCount()) &&
12061 "Expected select_cc with vector result to have the same sized "
12062 "comparison type!");
12063 break;
12064 case ISD::BR_CC:
12065 assert(NumOps == 5 && "BR_CC takes 5 operands!");
12066 assert(Ops[2].getValueType() == Ops[3].getValueType() &&
12067 "LHS/RHS of comparison should match types!");
12068 break;
12069 case ISD::VP_REDUCE_MUL:
12070 // If it is VP_REDUCE_MUL mask operation then turn it to VP_REDUCE_AND
12071 if (VT == MVT::i1)
12072 Opcode = ISD::VP_REDUCE_AND;
12073 break;
12074 case ISD::VP_REDUCE_ADD:
12075 // If it is VP_REDUCE_ADD mask operation then turn it to VP_REDUCE_XOR
12076 if (VT == MVT::i1)
12077 Opcode = ISD::VP_REDUCE_XOR;
12078 break;
12079 case ISD::VP_REDUCE_SMAX:
12080 case ISD::VP_REDUCE_UMIN:
12081 // If it is VP_REDUCE_SMAX/VP_REDUCE_UMIN mask operation then turn it to
12082 // VP_REDUCE_AND.
12083 if (VT == MVT::i1)
12084 Opcode = ISD::VP_REDUCE_AND;
12085 break;
12086 case ISD::VP_REDUCE_SMIN:
12087 case ISD::VP_REDUCE_UMAX:
12088 // If it is VP_REDUCE_SMIN/VP_REDUCE_UMAX mask operation then turn it to
12089 // VP_REDUCE_OR.
12090 if (VT == MVT::i1)
12091 Opcode = ISD::VP_REDUCE_OR;
12092 break;
12093 }
12094
12095 // Memoize nodes.
12096 SDNode *N;
12097 SDVTList VTs = getVTList(VT);
12098
12099 if (VT != MVT::Glue) {
12100 SDNodeKey ID(Opcode, VTs, Ops);
12101 FoldingSetInsertToken InsertToken;
12102
12103 if (SDNode *E = lookupNode(Key: ID, DL, InsertToken)) {
12104 E->intersectFlagsWith(Flags);
12105 return SDValue(E, 0);
12106 }
12107
12108 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
12109 createOperands(Node: N, Vals: Ops);
12110
12111 CSEMap.insert(N, Token: InsertToken);
12112 } else {
12113 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
12114 createOperands(Node: N, Vals: Ops);
12115 }
12116
12117 N->setFlags(Flags);
12118 InsertNode(N);
12119 SDValue V(N, 0);
12120 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
12121 return V;
12122}
12123
12124SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
12125 ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops) {
12126 SDNodeFlags Flags;
12127 if (Inserter)
12128 Flags = Inserter->getFlags();
12129 return getNode(Opcode, DL, VTList: getVTList(VTs: ResultTys), Ops, Flags);
12130}
12131
12132SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
12133 ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops,
12134 const SDNodeFlags Flags) {
12135 return getNode(Opcode, DL, VTList: getVTList(VTs: ResultTys), Ops, Flags);
12136}
12137
12138SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
12139 ArrayRef<SDValue> Ops) {
12140 SDNodeFlags Flags;
12141 if (Inserter)
12142 Flags = Inserter->getFlags();
12143 return getNode(Opcode, DL, VTList, Ops, Flags);
12144}
12145
12146SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
12147 ArrayRef<SDValue> Ops, const SDNodeFlags Flags) {
12148 if (VTList.NumVTs == 1)
12149 return getNode(Opcode, DL, VT: VTList.VTs[0], Ops, Flags);
12150
12151#ifndef NDEBUG
12152 for (const auto &Op : Ops)
12153 assert(Op.getOpcode() != ISD::DELETED_NODE &&
12154 "Operand is DELETED_NODE!");
12155#endif
12156
12157 if (SDValue V = FoldConstantArithmetic(Opcode, DL, VTList, Ops, Flags))
12158 return V;
12159
12160 switch (Opcode) {
12161 case ISD::SADDO:
12162 case ISD::UADDO:
12163 case ISD::SSUBO:
12164 case ISD::USUBO: {
12165 assert(VTList.NumVTs == 2 && Ops.size() == 2 &&
12166 "Invalid add/sub overflow op!");
12167 assert(VTList.VTs[0].isInteger() && VTList.VTs[1].isInteger() &&
12168 Ops[0].getValueType() == Ops[1].getValueType() &&
12169 Ops[0].getValueType() == VTList.VTs[0] &&
12170 "Binary operator types must match!");
12171 SDValue N1 = Ops[0], N2 = Ops[1];
12172 canonicalizeCommutativeBinop(Opcode, N1, N2);
12173
12174 // (X +- 0) -> X with zero-overflow.
12175 ConstantSDNode *N2CV = isConstOrConstSplat(N: N2, /*AllowUndefs*/ false,
12176 /*AllowTruncation*/ true);
12177 if (N2CV && N2CV->isZero()) {
12178 SDValue ZeroOverFlow = getConstant(Val: 0, DL, VT: VTList.VTs[1]);
12179 return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList, Ops: {N1, ZeroOverFlow}, Flags);
12180 }
12181
12182 if (VTList.VTs[0].getScalarType() == MVT::i1 &&
12183 VTList.VTs[1].getScalarType() == MVT::i1) {
12184 SDValue F1 = getFreeze(V: N1);
12185 SDValue F2 = getFreeze(V: N2);
12186 // {vXi1,vXi1} (u/s)addo(vXi1 x, vXi1y) -> {xor(x,y),and(x,y)}
12187 if (Opcode == ISD::UADDO || Opcode == ISD::SADDO)
12188 return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList,
12189 Ops: {getNode(Opcode: ISD::XOR, DL, VT: VTList.VTs[0], N1: F1, N2: F2),
12190 getNode(Opcode: ISD::AND, DL, VT: VTList.VTs[1], N1: F1, N2: F2)},
12191 Flags);
12192 // {vXi1,vXi1} (u/s)subo(vXi1 x, vXi1y) -> {xor(x,y),and(~x,y)}
12193 if (Opcode == ISD::USUBO || Opcode == ISD::SSUBO) {
12194 SDValue NotF1 = getNOT(DL, Val: F1, VT: VTList.VTs[0]);
12195 return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList,
12196 Ops: {getNode(Opcode: ISD::XOR, DL, VT: VTList.VTs[0], N1: F1, N2: F2),
12197 getNode(Opcode: ISD::AND, DL, VT: VTList.VTs[1], N1: NotF1, N2: F2)},
12198 Flags);
12199 }
12200 }
12201 break;
12202 }
12203 case ISD::SADDO_CARRY:
12204 case ISD::UADDO_CARRY:
12205 case ISD::SSUBO_CARRY:
12206 case ISD::USUBO_CARRY:
12207 assert(VTList.NumVTs == 2 && Ops.size() == 3 &&
12208 "Invalid add/sub overflow op!");
12209 assert(VTList.VTs[0].isInteger() && VTList.VTs[1].isInteger() &&
12210 Ops[0].getValueType() == Ops[1].getValueType() &&
12211 Ops[0].getValueType() == VTList.VTs[0] &&
12212 Ops[2].getValueType() == VTList.VTs[1] &&
12213 "Binary operator types must match!");
12214 break;
12215 case ISD::SMUL_LOHI:
12216 case ISD::UMUL_LOHI: {
12217 assert(VTList.NumVTs == 2 && Ops.size() == 2 && "Invalid mul lo/hi op!");
12218 assert(VTList.VTs[0].isInteger() && VTList.VTs[0] == VTList.VTs[1] &&
12219 VTList.VTs[0] == Ops[0].getValueType() &&
12220 VTList.VTs[0] == Ops[1].getValueType() &&
12221 "Binary operator types must match!");
12222 // Constant fold.
12223 ConstantSDNode *LHS = dyn_cast<ConstantSDNode>(Val: Ops[0]);
12224 ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Val: Ops[1]);
12225 if (LHS && RHS) {
12226 unsigned Width = VTList.VTs[0].getScalarSizeInBits();
12227 unsigned OutWidth = Width * 2;
12228 APInt Val = LHS->getAPIntValue();
12229 APInt Mul = RHS->getAPIntValue();
12230 if (Opcode == ISD::SMUL_LOHI) {
12231 Val = Val.sext(width: OutWidth);
12232 Mul = Mul.sext(width: OutWidth);
12233 } else {
12234 Val = Val.zext(width: OutWidth);
12235 Mul = Mul.zext(width: OutWidth);
12236 }
12237 Val *= Mul;
12238
12239 SDValue Hi =
12240 getConstant(Val: Val.extractBits(numBits: Width, bitPosition: Width), DL, VT: VTList.VTs[0]);
12241 SDValue Lo = getConstant(Val: Val.trunc(width: Width), DL, VT: VTList.VTs[0]);
12242 return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList, Ops: {Lo, Hi}, Flags);
12243 }
12244 break;
12245 }
12246 case ISD::FFREXP: {
12247 assert(VTList.NumVTs == 2 && Ops.size() == 1 && "Invalid ffrexp op!");
12248 assert(VTList.VTs[0].isFloatingPoint() && VTList.VTs[1].isInteger() &&
12249 VTList.VTs[0] == Ops[0].getValueType() && "frexp type mismatch");
12250
12251 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val: Ops[0])) {
12252 int FrexpExp;
12253 APFloat FrexpMant =
12254 frexp(X: C->getValueAPF(), Exp&: FrexpExp, RM: APFloat::rmNearestTiesToEven);
12255 SDValue Result0 = getConstantFP(V: FrexpMant, DL, VT: VTList.VTs[0]);
12256 SDValue Result1 = getSignedConstant(Val: FrexpMant.isFinite() ? FrexpExp : 0,
12257 DL, VT: VTList.VTs[1]);
12258 return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList, Ops: {Result0, Result1}, Flags);
12259 }
12260
12261 break;
12262 }
12263 case ISD::STRICT_FP_EXTEND:
12264 assert(VTList.NumVTs == 2 && Ops.size() == 2 &&
12265 "Invalid STRICT_FP_EXTEND!");
12266 assert(VTList.VTs[0].isFloatingPoint() &&
12267 Ops[1].getValueType().isFloatingPoint() && "Invalid FP cast!");
12268 assert(VTList.VTs[0].isVector() == Ops[1].getValueType().isVector() &&
12269 "STRICT_FP_EXTEND result type should be vector iff the operand "
12270 "type is vector!");
12271 assert((!VTList.VTs[0].isVector() ||
12272 VTList.VTs[0].getVectorElementCount() ==
12273 Ops[1].getValueType().getVectorElementCount()) &&
12274 "Vector element count mismatch!");
12275 assert(Ops[1].getValueType().bitsLT(VTList.VTs[0]) &&
12276 "Invalid fpext node, dst <= src!");
12277 break;
12278 case ISD::STRICT_FP_ROUND:
12279 assert(VTList.NumVTs == 2 && Ops.size() == 3 && "Invalid STRICT_FP_ROUND!");
12280 assert(VTList.VTs[0].isVector() == Ops[1].getValueType().isVector() &&
12281 "STRICT_FP_ROUND result type should be vector iff the operand "
12282 "type is vector!");
12283 assert((!VTList.VTs[0].isVector() ||
12284 VTList.VTs[0].getVectorElementCount() ==
12285 Ops[1].getValueType().getVectorElementCount()) &&
12286 "Vector element count mismatch!");
12287 assert(VTList.VTs[0].isFloatingPoint() &&
12288 Ops[1].getValueType().isFloatingPoint() &&
12289 VTList.VTs[0].bitsLT(Ops[1].getValueType()) &&
12290 Ops[2].getOpcode() == ISD::TargetConstant &&
12291 (Ops[2]->getAsZExtVal() == 0 || Ops[2]->getAsZExtVal() == 1) &&
12292 "Invalid STRICT_FP_ROUND!");
12293 break;
12294 }
12295
12296 // Memoize the node unless it returns a glue result.
12297 SDNode *N;
12298 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
12299 SDNodeKey ID(Opcode, VTList, Ops);
12300 FoldingSetInsertToken InsertToken;
12301 if (SDNode *E = lookupNode(Key: ID, DL, InsertToken)) {
12302 E->intersectFlagsWith(Flags);
12303 return SDValue(E, 0);
12304 }
12305
12306 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTList);
12307 createOperands(Node: N, Vals: Ops);
12308 CSEMap.insert(N, Token: InsertToken);
12309 } else {
12310 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTList);
12311 createOperands(Node: N, Vals: Ops);
12312 }
12313
12314 N->setFlags(Flags);
12315 InsertNode(N);
12316 SDValue V(N, 0);
12317 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
12318 return V;
12319}
12320
12321SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
12322 SDVTList VTList) {
12323 return getNode(Opcode, DL, VTList, Ops: ArrayRef<SDValue>());
12324}
12325
12326SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
12327 SDValue N1) {
12328 SDValue Ops[] = { N1 };
12329 return getNode(Opcode, DL, VTList, Ops);
12330}
12331
12332SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
12333 SDValue N1, SDValue N2) {
12334 SDValue Ops[] = { N1, N2 };
12335 return getNode(Opcode, DL, VTList, Ops);
12336}
12337
12338SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
12339 SDValue N1, SDValue N2, SDValue N3) {
12340 SDValue Ops[] = { N1, N2, N3 };
12341 return getNode(Opcode, DL, VTList, Ops);
12342}
12343
12344SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
12345 SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
12346 SDValue Ops[] = { N1, N2, N3, N4 };
12347 return getNode(Opcode, DL, VTList, Ops);
12348}
12349
12350SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
12351 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
12352 SDValue N5) {
12353 SDValue Ops[] = { N1, N2, N3, N4, N5 };
12354 return getNode(Opcode, DL, VTList, Ops);
12355}
12356
12357SDVTList SelectionDAG::getVTList(EVT VT) {
12358 if (!VT.isExtended())
12359 return makeVTList(VTs: SDNode::getValueTypeList(VT: VT.getSimpleVT()), NumVTs: 1);
12360
12361 EVT VTs[] = {VT};
12362 return getVTList(VTs);
12363}
12364
12365SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2) {
12366 EVT VTs[] = {VT1, VT2};
12367 return getVTList(VTs);
12368}
12369
12370SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3) {
12371 EVT VTs[] = {VT1, VT2, VT3};
12372 return getVTList(VTs);
12373}
12374
12375SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4) {
12376 EVT VTs[] = {VT1, VT2, VT3, VT4};
12377 return getVTList(VTs);
12378}
12379
12380SDVTList SelectionDAG::getVTList(ArrayRef<EVT> VTs) {
12381 auto It = VTLists.find(V: VTs);
12382 if (It == VTLists.end()) {
12383 EVT *Array = Allocator.Allocate<EVT>(Num: VTs.size());
12384 llvm::copy(Range&: VTs, Out: Array);
12385 It = VTLists.insert(V: ArrayRef(Array, VTs.size())).first;
12386 }
12387 return makeVTList(VTs: It->data(), NumVTs: It->size());
12388}
12389
12390/// UpdateNodeOperands - *Mutate* the specified node in-place to have the
12391/// specified operands. If the resultant node already exists in the DAG,
12392/// this does not modify the specified node, instead it returns the node that
12393/// already exists. If the resultant node does not exist in the DAG, the
12394/// input node is returned. As a degenerate case, if you specify the same
12395/// input operands as the node already has, the input node is returned.
12396SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op) {
12397 assert(N->getNumOperands() == 1 && "Update with wrong number of operands");
12398
12399 // Check to see if there is no change.
12400 if (Op == N->getOperand(Num: 0)) return N;
12401
12402 // See if the modified node already exists.
12403 FoldingSetInsertToken InsertToken;
12404 if (SDNode *Existing = FindModifiedNodeSlot(N, Op, InsertToken))
12405 return Existing;
12406
12407 // Nope it doesn't. Remove the node from its current place in the maps.
12408 if (InsertToken)
12409 if (!RemoveNodeFromCSEMaps(N))
12410 InsertToken = {};
12411
12412 // Now we update the operands.
12413 N->OperandList[0].set(Op);
12414
12415 updateDivergence(N);
12416 // If this gets put into a CSE map, add it.
12417 if (InsertToken)
12418 CSEMap.insert(N, Token: InsertToken);
12419 return N;
12420}
12421
12422SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2) {
12423 assert(N->getNumOperands() == 2 && "Update with wrong number of operands");
12424
12425 // Check to see if there is no change.
12426 if (Op1 == N->getOperand(Num: 0) && Op2 == N->getOperand(Num: 1))
12427 return N; // No operands changed, just return the input node.
12428
12429 // See if the modified node already exists.
12430 FoldingSetInsertToken InsertToken;
12431 if (SDNode *Existing = FindModifiedNodeSlot(N, Op1, Op2, InsertToken))
12432 return Existing;
12433
12434 // Nope it doesn't. Remove the node from its current place in the maps.
12435 if (InsertToken)
12436 if (!RemoveNodeFromCSEMaps(N))
12437 InsertToken = {};
12438
12439 // Now we update the operands.
12440 if (N->OperandList[0] != Op1)
12441 N->OperandList[0].set(Op1);
12442 if (N->OperandList[1] != Op2)
12443 N->OperandList[1].set(Op2);
12444
12445 updateDivergence(N);
12446 // If this gets put into a CSE map, add it.
12447 if (InsertToken)
12448 CSEMap.insert(N, Token: InsertToken);
12449 return N;
12450}
12451
12452SDNode *SelectionDAG::
12453UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, SDValue Op3) {
12454 SDValue Ops[] = { Op1, Op2, Op3 };
12455 return UpdateNodeOperands(N, Ops);
12456}
12457
12458SDNode *SelectionDAG::
12459UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
12460 SDValue Op3, SDValue Op4) {
12461 SDValue Ops[] = { Op1, Op2, Op3, Op4 };
12462 return UpdateNodeOperands(N, Ops);
12463}
12464
12465SDNode *SelectionDAG::
12466UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
12467 SDValue Op3, SDValue Op4, SDValue Op5) {
12468 SDValue Ops[] = { Op1, Op2, Op3, Op4, Op5 };
12469 return UpdateNodeOperands(N, Ops);
12470}
12471
12472SDNode *SelectionDAG::
12473UpdateNodeOperands(SDNode *N, ArrayRef<SDValue> Ops) {
12474 unsigned NumOps = Ops.size();
12475 assert(N->getNumOperands() == NumOps &&
12476 "Update with wrong number of operands");
12477
12478 // If no operands changed just return the input node.
12479 if (std::equal(first1: Ops.begin(), last1: Ops.end(), first2: N->op_begin()))
12480 return N;
12481
12482 // See if the modified node already exists.
12483 FoldingSetInsertToken InsertToken;
12484 if (SDNode *Existing = FindModifiedNodeSlot(N, Ops, InsertToken))
12485 return Existing;
12486
12487 // Nope it doesn't. Remove the node from its current place in the maps.
12488 if (InsertToken)
12489 if (!RemoveNodeFromCSEMaps(N))
12490 InsertToken = {};
12491
12492 // Now we update the operands.
12493 for (unsigned i = 0; i != NumOps; ++i)
12494 if (N->OperandList[i] != Ops[i])
12495 N->OperandList[i].set(Ops[i]);
12496
12497 updateDivergence(N);
12498 // If this gets put into a CSE map, add it.
12499 if (InsertToken)
12500 CSEMap.insert(N, Token: InsertToken);
12501 return N;
12502}
12503
12504/// DropOperands - Release the operands and set this node to have
12505/// zero operands.
12506void SDNode::DropOperands() {
12507 // Unlike the code in MorphNodeTo that does this, we don't need to
12508 // watch for dead nodes here.
12509 for (op_iterator I = op_begin(), E = op_end(); I != E; ) {
12510 SDUse &Use = *I++;
12511 Use.set(SDValue());
12512 }
12513}
12514
12515void SelectionDAG::setNodeMemRefs(MachineSDNode *N,
12516 ArrayRef<MachineMemOperand *> NewMemRefs) {
12517 if (NewMemRefs.empty()) {
12518 N->clearMemRefs();
12519 return;
12520 }
12521
12522 // Check if we can avoid allocating by storing a single reference directly.
12523 if (NewMemRefs.size() == 1) {
12524 N->MemRefs = NewMemRefs[0];
12525 N->NumMemRefs = 1;
12526 return;
12527 }
12528
12529 MachineMemOperand **MemRefsBuffer =
12530 Allocator.template Allocate<MachineMemOperand *>(Num: NewMemRefs.size());
12531 llvm::copy(Range&: NewMemRefs, Out: MemRefsBuffer);
12532 N->MemRefs = MemRefsBuffer;
12533 N->NumMemRefs = static_cast<int>(NewMemRefs.size());
12534}
12535
12536/// SelectNodeTo - These are wrappers around MorphNodeTo that accept a
12537/// machine opcode.
12538///
12539SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12540 EVT VT) {
12541 SDVTList VTs = getVTList(VT);
12542 return SelectNodeTo(N, MachineOpc, VTs, Ops: {});
12543}
12544
12545SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12546 EVT VT, SDValue Op1) {
12547 SDVTList VTs = getVTList(VT);
12548 SDValue Ops[] = { Op1 };
12549 return SelectNodeTo(N, MachineOpc, VTs, Ops);
12550}
12551
12552SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12553 EVT VT, SDValue Op1,
12554 SDValue Op2) {
12555 SDVTList VTs = getVTList(VT);
12556 SDValue Ops[] = { Op1, Op2 };
12557 return SelectNodeTo(N, MachineOpc, VTs, Ops);
12558}
12559
12560SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12561 EVT VT, SDValue Op1,
12562 SDValue Op2, SDValue Op3) {
12563 SDVTList VTs = getVTList(VT);
12564 SDValue Ops[] = { Op1, Op2, Op3 };
12565 return SelectNodeTo(N, MachineOpc, VTs, Ops);
12566}
12567
12568SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12569 EVT VT, ArrayRef<SDValue> Ops) {
12570 SDVTList VTs = getVTList(VT);
12571 return SelectNodeTo(N, MachineOpc, VTs, Ops);
12572}
12573
12574SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12575 EVT VT1, EVT VT2, ArrayRef<SDValue> Ops) {
12576 SDVTList VTs = getVTList(VT1, VT2);
12577 return SelectNodeTo(N, MachineOpc, VTs, Ops);
12578}
12579
12580SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12581 EVT VT1, EVT VT2) {
12582 SDVTList VTs = getVTList(VT1, VT2);
12583 return SelectNodeTo(N, MachineOpc, VTs, Ops: {});
12584}
12585
12586SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12587 EVT VT1, EVT VT2, EVT VT3,
12588 ArrayRef<SDValue> Ops) {
12589 SDVTList VTs = getVTList(VT1, VT2, VT3);
12590 return SelectNodeTo(N, MachineOpc, VTs, Ops);
12591}
12592
12593SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12594 EVT VT1, EVT VT2,
12595 SDValue Op1, SDValue Op2) {
12596 SDVTList VTs = getVTList(VT1, VT2);
12597 SDValue Ops[] = { Op1, Op2 };
12598 return SelectNodeTo(N, MachineOpc, VTs, Ops);
12599}
12600
12601SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12602 SDVTList VTs,ArrayRef<SDValue> Ops) {
12603 SDNode *New = MorphNodeTo(N, Opc: ~MachineOpc, VTs, Ops);
12604 // Reset the NodeID to -1.
12605 New->setNodeId(-1);
12606 if (New != N) {
12607 ReplaceAllUsesWith(From: N, To: New);
12608 RemoveDeadNode(N);
12609 }
12610 return New;
12611}
12612
12613/// UpdateSDLocOnMergeSDNode - If the opt level is -O0 then it throws away
12614/// the line number information on the merged node since it is not possible to
12615/// preserve the information that operation is associated with multiple lines.
12616/// This will make the debugger working better at -O0, were there is a higher
12617/// probability having other instructions associated with that line.
12618///
12619/// For IROrder, we keep the smaller of the two
12620SDNode *SelectionDAG::UpdateSDLocOnMergeSDNode(SDNode *N, const SDLoc &OLoc) {
12621 DebugLoc NLoc = N->getDebugLoc();
12622 if (NLoc && OptLevel == CodeGenOptLevel::None && OLoc.getDebugLoc() != NLoc) {
12623 N->setDebugLoc(DebugLoc());
12624 }
12625 unsigned Order = std::min(a: N->getIROrder(), b: OLoc.getIROrder());
12626 N->setIROrder(Order);
12627 return N;
12628}
12629
12630/// MorphNodeTo - This *mutates* the specified node to have the specified
12631/// return type, opcode, and operands.
12632///
12633/// Note that MorphNodeTo returns the resultant node. If there is already a
12634/// node of the specified opcode and operands, it returns that node instead of
12635/// the current one. Note that the SDLoc need not be the same.
12636///
12637/// Using MorphNodeTo is faster than creating a new node and swapping it in
12638/// with ReplaceAllUsesWith both because it often avoids allocating a new
12639/// node, and because it doesn't require CSE recalculation for any of
12640/// the node's users.
12641///
12642/// However, note that MorphNodeTo recursively deletes dead nodes from the DAG.
12643/// As a consequence it isn't appropriate to use from within the DAG combiner or
12644/// the legalizer which maintain worklists that would need to be updated when
12645/// deleting things.
12646SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
12647 SDVTList VTs, ArrayRef<SDValue> Ops) {
12648 // If an identical node already exists, use it.
12649 FoldingSetInsertToken InsertToken;
12650 if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) {
12651 SDNodeKey ID(Opc, VTs, Ops);
12652 AddNodeIDCustom(ID&: ID.Tail, N, Opc);
12653 if (SDNode *ON = lookupNode(Key: ID, DL: SDLoc(N), InsertToken))
12654 return UpdateSDLocOnMergeSDNode(N: ON, OLoc: SDLoc(N));
12655 }
12656
12657 if (!RemoveNodeFromCSEMaps(N))
12658 InsertToken = {};
12659
12660 // Start the morphing.
12661 N->NodeType = Opc;
12662 N->ValueList = VTs.VTs;
12663 N->NumValues = VTs.NumVTs;
12664
12665 // Clear the operands list, updating used nodes to remove this from their
12666 // use list. Keep track of any operands that become dead as a result.
12667 SmallPtrSet<SDNode*, 16> DeadNodeSet;
12668 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
12669 SDUse &Use = *I++;
12670 SDNode *Used = Use.getNode();
12671 Use.set(SDValue());
12672 if (Used->use_empty())
12673 DeadNodeSet.insert(Ptr: Used);
12674 }
12675
12676 // For MachineNode, initialize the memory references information.
12677 if (MachineSDNode *MN = dyn_cast<MachineSDNode>(Val: N))
12678 MN->clearMemRefs();
12679
12680 // Swap for an appropriately sized array from the recycler.
12681 removeOperands(Node: N);
12682 createOperands(Node: N, Vals: Ops);
12683
12684 // Delete any nodes that are still dead after adding the uses for the
12685 // new operands.
12686 if (!DeadNodeSet.empty()) {
12687 SmallVector<SDNode *, 16> DeadNodes;
12688 for (SDNode *N : DeadNodeSet)
12689 if (N->use_empty())
12690 DeadNodes.push_back(Elt: N);
12691 RemoveDeadNodes(DeadNodes);
12692 }
12693
12694 if (InsertToken)
12695 CSEMap.insert(N, Token: InsertToken); // Memoize the new node.
12696 return N;
12697}
12698
12699SDNode* SelectionDAG::mutateStrictFPToFP(SDNode *Node) {
12700 unsigned OrigOpc = Node->getOpcode();
12701 unsigned NewOpc;
12702 switch (OrigOpc) {
12703 default:
12704 llvm_unreachable("mutateStrictFPToFP called with unexpected opcode!");
12705#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
12706 case ISD::STRICT_##DAGN: NewOpc = ISD::DAGN; break;
12707#define CMP_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
12708 case ISD::STRICT_##DAGN: NewOpc = ISD::SETCC; break;
12709#include "llvm/IR/ConstrainedOps.def"
12710 }
12711
12712 assert(Node->getNumValues() == 2 && "Unexpected number of results!");
12713
12714 // We're taking this node out of the chain, so we need to re-link things.
12715 SDValue InputChain = Node->getOperand(Num: 0);
12716 SDValue OutputChain = SDValue(Node, 1);
12717 ReplaceAllUsesOfValueWith(From: OutputChain, To: InputChain);
12718
12719 SmallVector<SDValue, 3> Ops;
12720 for (unsigned i = 1, e = Node->getNumOperands(); i != e; ++i)
12721 Ops.push_back(Elt: Node->getOperand(Num: i));
12722
12723 SDVTList VTs = getVTList(VT: Node->getValueType(ResNo: 0));
12724 SDNode *Res = MorphNodeTo(N: Node, Opc: NewOpc, VTs, Ops);
12725
12726 // MorphNodeTo can operate in two ways: if an existing node with the
12727 // specified operands exists, it can just return it. Otherwise, it
12728 // updates the node in place to have the requested operands.
12729 if (Res == Node) {
12730 // If we updated the node in place, reset the node ID. To the isel,
12731 // this should be just like a newly allocated machine node.
12732 Res->setNodeId(-1);
12733 } else {
12734 ReplaceAllUsesWith(From: Node, To: Res);
12735 RemoveDeadNode(N: Node);
12736 }
12737
12738 return Res;
12739}
12740
12741/// getMachineNode - These are used for target selectors to create a new node
12742/// with specified return type(s), MachineInstr opcode, and operands.
12743///
12744/// Note that getMachineNode returns the resultant node. If there is already a
12745/// node of the specified opcode and operands, it returns that node instead of
12746/// the current one.
12747MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12748 EVT VT) {
12749 SDVTList VTs = getVTList(VT);
12750 return getMachineNode(Opcode, dl, VTs, Ops: {});
12751}
12752
12753MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12754 EVT VT, SDValue Op1) {
12755 SDVTList VTs = getVTList(VT);
12756 SDValue Ops[] = { Op1 };
12757 return getMachineNode(Opcode, dl, VTs, Ops);
12758}
12759
12760MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12761 EVT VT, SDValue Op1, SDValue Op2) {
12762 SDVTList VTs = getVTList(VT);
12763 SDValue Ops[] = { Op1, Op2 };
12764 return getMachineNode(Opcode, dl, VTs, Ops);
12765}
12766
12767MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12768 EVT VT, SDValue Op1, SDValue Op2,
12769 SDValue Op3) {
12770 SDVTList VTs = getVTList(VT);
12771 SDValue Ops[] = { Op1, Op2, Op3 };
12772 return getMachineNode(Opcode, dl, VTs, Ops);
12773}
12774
12775MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12776 EVT VT, ArrayRef<SDValue> Ops) {
12777 SDVTList VTs = getVTList(VT);
12778 return getMachineNode(Opcode, dl, VTs, Ops);
12779}
12780
12781MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12782 EVT VT1, EVT VT2, SDValue Op1,
12783 SDValue Op2) {
12784 SDVTList VTs = getVTList(VT1, VT2);
12785 SDValue Ops[] = { Op1, Op2 };
12786 return getMachineNode(Opcode, dl, VTs, Ops);
12787}
12788
12789MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12790 EVT VT1, EVT VT2, SDValue Op1,
12791 SDValue Op2, SDValue Op3) {
12792 SDVTList VTs = getVTList(VT1, VT2);
12793 SDValue Ops[] = { Op1, Op2, Op3 };
12794 return getMachineNode(Opcode, dl, VTs, Ops);
12795}
12796
12797MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12798 EVT VT1, EVT VT2,
12799 ArrayRef<SDValue> Ops) {
12800 SDVTList VTs = getVTList(VT1, VT2);
12801 return getMachineNode(Opcode, dl, VTs, Ops);
12802}
12803
12804MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12805 EVT VT1, EVT VT2, EVT VT3,
12806 SDValue Op1, SDValue Op2) {
12807 SDVTList VTs = getVTList(VT1, VT2, VT3);
12808 SDValue Ops[] = { Op1, Op2 };
12809 return getMachineNode(Opcode, dl, VTs, Ops);
12810}
12811
12812MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12813 EVT VT1, EVT VT2, EVT VT3,
12814 SDValue Op1, SDValue Op2,
12815 SDValue Op3) {
12816 SDVTList VTs = getVTList(VT1, VT2, VT3);
12817 SDValue Ops[] = { Op1, Op2, Op3 };
12818 return getMachineNode(Opcode, dl, VTs, Ops);
12819}
12820
12821MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12822 EVT VT1, EVT VT2, EVT VT3,
12823 ArrayRef<SDValue> Ops) {
12824 SDVTList VTs = getVTList(VT1, VT2, VT3);
12825 return getMachineNode(Opcode, dl, VTs, Ops);
12826}
12827
12828MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12829 ArrayRef<EVT> ResultTys,
12830 ArrayRef<SDValue> Ops) {
12831 SDVTList VTs = getVTList(VTs: ResultTys);
12832 return getMachineNode(Opcode, dl, VTs, Ops);
12833}
12834
12835MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &DL,
12836 SDVTList VTs,
12837 ArrayRef<SDValue> Ops) {
12838 bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue;
12839 MachineSDNode *N;
12840 FoldingSetInsertToken InsertToken;
12841
12842 if (DoCSE) {
12843 SDNodeKey ID(~Opcode, VTs, Ops);
12844 if (SDNode *E = lookupNode(Key: ID, DL, InsertToken)) {
12845 return cast<MachineSDNode>(Val: UpdateSDLocOnMergeSDNode(N: E, OLoc: DL));
12846 }
12847 }
12848
12849 // Allocate a new MachineSDNode.
12850 N = newSDNode<MachineSDNode>(Args: ~Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
12851 createOperands(Node: N, Vals: Ops);
12852
12853 if (DoCSE)
12854 CSEMap.insert(N, Token: InsertToken);
12855
12856 InsertNode(N);
12857 NewSDValueDbgMsg(V: SDValue(N, 0), Msg: "Creating new machine node: ", G: this);
12858 return N;
12859}
12860
12861/// getTargetExtractSubreg - A convenience function for creating
12862/// TargetOpcode::EXTRACT_SUBREG nodes.
12863SDValue SelectionDAG::getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT,
12864 SDValue Operand) {
12865 SDValue SRIdxVal = getTargetConstant(Val: SRIdx, DL, VT: MVT::i32);
12866 SDNode *Subreg = getMachineNode(Opcode: TargetOpcode::EXTRACT_SUBREG, dl: DL,
12867 VT, Op1: Operand, Op2: SRIdxVal);
12868 return SDValue(Subreg, 0);
12869}
12870
12871/// getTargetInsertSubreg - A convenience function for creating
12872/// TargetOpcode::INSERT_SUBREG nodes.
12873SDValue SelectionDAG::getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT,
12874 SDValue Operand, SDValue Subreg) {
12875 SDValue SRIdxVal = getTargetConstant(Val: SRIdx, DL, VT: MVT::i32);
12876 SDNode *Result = getMachineNode(Opcode: TargetOpcode::INSERT_SUBREG, dl: DL,
12877 VT, Op1: Operand, Op2: Subreg, Op3: SRIdxVal);
12878 return SDValue(Result, 0);
12879}
12880
12881/// getNodeIfExists - Get the specified node if it's already available, or
12882/// else return NULL.
12883SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,
12884 ArrayRef<SDValue> Ops,
12885 bool AllowCommute) {
12886 SDNodeFlags Flags;
12887 if (Inserter)
12888 Flags = Inserter->getFlags();
12889 return getNodeIfExists(Opcode, VTList, Ops, Flags, AllowCommute);
12890}
12891
12892SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,
12893 ArrayRef<SDValue> Ops,
12894 const SDNodeFlags Flags,
12895 bool AllowCommute) {
12896 if (VTList.VTs[VTList.NumVTs - 1] == MVT::Glue)
12897 return nullptr;
12898
12899 auto Lookup = [&](ArrayRef<SDValue> LookupOps) -> SDNode * {
12900 SDNodeKey ID(Opcode, VTList, LookupOps);
12901 FoldingSetInsertToken InsertToken;
12902 if (SDNode *E = lookupNode(Key: ID, InsertToken)) {
12903 E->intersectFlagsWith(Flags);
12904 return E;
12905 }
12906 return nullptr;
12907 };
12908
12909 if (SDNode *Existing = Lookup(Ops))
12910 return Existing;
12911
12912 if (AllowCommute && TLI->isCommutativeBinOp(Opcode))
12913 return Lookup({Ops[1], Ops[0]});
12914
12915 return nullptr;
12916}
12917
12918/// doesNodeExist - Check if a node exists without modifying its flags.
12919bool SelectionDAG::doesNodeExist(unsigned Opcode, SDVTList VTList,
12920 ArrayRef<SDValue> Ops) {
12921 if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) {
12922 SDNodeKey ID(Opcode, VTList, Ops);
12923 FoldingSetInsertToken InsertToken;
12924 if (lookupNode(Key: ID, DL: SDLoc(), InsertToken))
12925 return true;
12926 }
12927 return false;
12928}
12929
12930/// getDbgValue - Creates a SDDbgValue node.
12931///
12932/// SDNode
12933SDDbgValue *SelectionDAG::getDbgValue(DIVariable *Var, DIExpression *Expr,
12934 SDNode *N, unsigned R, bool IsIndirect,
12935 const DebugLoc &DL, unsigned O) {
12936 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
12937 "Expected inlined-at fields to agree");
12938 return new (DbgInfo->getAlloc())
12939 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromNode(Node: N, ResNo: R),
12940 {}, IsIndirect, DL, O,
12941 /*IsVariadic=*/false);
12942}
12943
12944/// Constant
12945SDDbgValue *SelectionDAG::getConstantDbgValue(DIVariable *Var,
12946 DIExpression *Expr,
12947 const Value *C,
12948 const DebugLoc &DL, unsigned O) {
12949 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
12950 "Expected inlined-at fields to agree");
12951 return new (DbgInfo->getAlloc())
12952 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromConst(Const: C), {},
12953 /*IsIndirect=*/false, DL, O,
12954 /*IsVariadic=*/false);
12955}
12956
12957/// FrameIndex
12958SDDbgValue *SelectionDAG::getFrameIndexDbgValue(DIVariable *Var,
12959 DIExpression *Expr, unsigned FI,
12960 bool IsIndirect,
12961 const DebugLoc &DL,
12962 unsigned O) {
12963 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
12964 "Expected inlined-at fields to agree");
12965 return getFrameIndexDbgValue(Var, Expr, FI, Dependencies: {}, IsIndirect, DL, O);
12966}
12967
12968/// FrameIndex with dependencies
12969SDDbgValue *SelectionDAG::getFrameIndexDbgValue(DIVariable *Var,
12970 DIExpression *Expr, unsigned FI,
12971 ArrayRef<SDNode *> Dependencies,
12972 bool IsIndirect,
12973 const DebugLoc &DL,
12974 unsigned O) {
12975 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
12976 "Expected inlined-at fields to agree");
12977 return new (DbgInfo->getAlloc())
12978 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromFrameIdx(FrameIdx: FI),
12979 Dependencies, IsIndirect, DL, O,
12980 /*IsVariadic=*/false);
12981}
12982
12983/// VReg
12984SDDbgValue *SelectionDAG::getVRegDbgValue(DIVariable *Var, DIExpression *Expr,
12985 Register VReg, bool IsIndirect,
12986 const DebugLoc &DL, unsigned O) {
12987 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
12988 "Expected inlined-at fields to agree");
12989 return new (DbgInfo->getAlloc())
12990 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromVReg(VReg),
12991 {}, IsIndirect, DL, O,
12992 /*IsVariadic=*/false);
12993}
12994
12995SDDbgValue *SelectionDAG::getDbgValueList(DIVariable *Var, DIExpression *Expr,
12996 ArrayRef<SDDbgOperand> Locs,
12997 ArrayRef<SDNode *> Dependencies,
12998 bool IsIndirect, const DebugLoc &DL,
12999 unsigned O, bool IsVariadic) {
13000 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
13001 "Expected inlined-at fields to agree");
13002 return new (DbgInfo->getAlloc())
13003 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, Locs, Dependencies, IsIndirect,
13004 DL, O, IsVariadic);
13005}
13006
13007void SelectionDAG::transferDbgValues(SDValue From, SDValue To,
13008 unsigned OffsetInBits, unsigned SizeInBits,
13009 bool InvalidateDbg) {
13010 SDNode *FromNode = From.getNode();
13011 SDNode *ToNode = To.getNode();
13012 assert(FromNode && ToNode && "Can't modify dbg values");
13013
13014 // PR35338
13015 // TODO: assert(From != To && "Redundant dbg value transfer");
13016 // TODO: assert(FromNode != ToNode && "Intranode dbg value transfer");
13017 if (From == To || FromNode == ToNode)
13018 return;
13019
13020 if (!FromNode->getHasDebugValue())
13021 return;
13022
13023 SDDbgOperand FromLocOp =
13024 SDDbgOperand::fromNode(Node: From.getNode(), ResNo: From.getResNo());
13025 SDDbgOperand ToLocOp = SDDbgOperand::fromNode(Node: To.getNode(), ResNo: To.getResNo());
13026
13027 SmallVector<SDDbgValue *, 2> ClonedDVs;
13028 for (SDDbgValue *Dbg : GetDbgValues(SD: FromNode)) {
13029 if (Dbg->isInvalidated())
13030 continue;
13031
13032 // TODO: assert(!Dbg->isInvalidated() && "Transfer of invalid dbg value");
13033
13034 // Create a new location ops vector that is equal to the old vector, but
13035 // with each instance of FromLocOp replaced with ToLocOp.
13036 bool Changed = false;
13037 auto NewLocOps = Dbg->copyLocationOps();
13038 std::replace_if(
13039 first: NewLocOps.begin(), last: NewLocOps.end(),
13040 pred: [&Changed, FromLocOp](const SDDbgOperand &Op) {
13041 bool Match = Op == FromLocOp;
13042 Changed |= Match;
13043 return Match;
13044 },
13045 new_value: ToLocOp);
13046 // Ignore this SDDbgValue if we didn't find a matching location.
13047 if (!Changed)
13048 continue;
13049
13050 DIVariable *Var = Dbg->getVariable();
13051 auto *Expr = Dbg->getExpression();
13052 // If a fragment is requested, update the expression.
13053 if (SizeInBits) {
13054 // When splitting a larger (e.g., sign-extended) value whose
13055 // lower bits are described with an SDDbgValue, do not attempt
13056 // to transfer the SDDbgValue to the upper bits.
13057 if (auto FI = Expr->getFragmentInfo())
13058 if (OffsetInBits + SizeInBits > FI->SizeInBits)
13059 continue;
13060 auto Fragment = DIExpression::createFragmentExpression(Expr, OffsetInBits,
13061 SizeInBits);
13062 if (!Fragment)
13063 continue;
13064 Expr = *Fragment;
13065 }
13066
13067 auto AdditionalDependencies = Dbg->getAdditionalDependencies();
13068 // Clone the SDDbgValue and move it to To.
13069 SDDbgValue *Clone = getDbgValueList(
13070 Var, Expr, Locs: NewLocOps, Dependencies: AdditionalDependencies, IsIndirect: Dbg->isIndirect(),
13071 DL: Dbg->getDebugLoc(), O: std::max(a: ToNode->getIROrder(), b: Dbg->getOrder()),
13072 IsVariadic: Dbg->isVariadic());
13073 ClonedDVs.push_back(Elt: Clone);
13074
13075 if (InvalidateDbg) {
13076 // Invalidate value and indicate the SDDbgValue should not be emitted.
13077 Dbg->setIsInvalidated();
13078 Dbg->setIsEmitted();
13079 }
13080 }
13081
13082 for (SDDbgValue *Dbg : ClonedDVs) {
13083 assert(is_contained(Dbg->getSDNodes(), ToNode) &&
13084 "Transferred DbgValues should depend on the new SDNode");
13085 AddDbgValue(DB: Dbg, isParameter: false);
13086 }
13087}
13088
13089void SelectionDAG::salvageDebugInfo(SDNode &N) {
13090 if (!N.getHasDebugValue())
13091 return;
13092
13093 auto GetLocationOperand = [](SDNode *Node, unsigned ResNo) {
13094 if (auto *FISDN = dyn_cast<FrameIndexSDNode>(Val: Node))
13095 return SDDbgOperand::fromFrameIdx(FrameIdx: FISDN->getIndex());
13096 return SDDbgOperand::fromNode(Node, ResNo);
13097 };
13098
13099 SmallVector<SDDbgValue *, 2> ClonedDVs;
13100 for (auto *DV : GetDbgValues(SD: &N)) {
13101 if (DV->isInvalidated())
13102 continue;
13103 switch (N.getOpcode()) {
13104 default:
13105 break;
13106 case ISD::ADD: {
13107 SDValue N0 = N.getOperand(Num: 0);
13108 SDValue N1 = N.getOperand(Num: 1);
13109 if (!isa<ConstantSDNode>(Val: N0)) {
13110 bool RHSConstant = isa<ConstantSDNode>(Val: N1);
13111 uint64_t Offset;
13112 if (RHSConstant)
13113 Offset = N.getConstantOperandVal(Num: 1);
13114 // We are not allowed to turn indirect debug values variadic, so
13115 // don't salvage those.
13116 if (!RHSConstant && DV->isIndirect())
13117 continue;
13118
13119 // Rewrite an ADD constant node into a DIExpression. Since we are
13120 // performing arithmetic to compute the variable's *value* in the
13121 // DIExpression, we need to mark the expression with a
13122 // DW_OP_stack_value.
13123 auto *DIExpr = DV->getExpression();
13124 auto NewLocOps = DV->copyLocationOps();
13125 bool Changed = false;
13126 size_t OrigLocOpsSize = NewLocOps.size();
13127 for (size_t i = 0; i < OrigLocOpsSize; ++i) {
13128 // We're not given a ResNo to compare against because the whole
13129 // node is going away. We know that any ISD::ADD only has one
13130 // result, so we can assume any node match is using the result.
13131 if (NewLocOps[i].getKind() != SDDbgOperand::SDNODE ||
13132 NewLocOps[i].getSDNode() != &N)
13133 continue;
13134 NewLocOps[i] = GetLocationOperand(N0.getNode(), N0.getResNo());
13135 if (RHSConstant) {
13136 SmallVector<uint64_t, 3> ExprOps;
13137 DIExpression::appendOffset(Ops&: ExprOps, Offset);
13138 DIExpr = DIExpression::appendOpsToArg(Expr: DIExpr, Ops: ExprOps, ArgNo: i, StackValue: true);
13139 } else {
13140 // Convert to a variadic expression (if not already).
13141 // convertToVariadicExpression() returns a const pointer, so we use
13142 // a temporary const variable here.
13143 const auto *TmpDIExpr =
13144 DIExpression::convertToVariadicExpression(Expr: DIExpr);
13145 SmallVector<uint64_t, 3> ExprOps;
13146 ExprOps.push_back(Elt: dwarf::DW_OP_LLVM_arg);
13147 ExprOps.push_back(Elt: NewLocOps.size());
13148 ExprOps.push_back(Elt: dwarf::DW_OP_plus);
13149 SDDbgOperand RHS =
13150 SDDbgOperand::fromNode(Node: N1.getNode(), ResNo: N1.getResNo());
13151 NewLocOps.push_back(Elt: RHS);
13152 DIExpr = DIExpression::appendOpsToArg(Expr: TmpDIExpr, Ops: ExprOps, ArgNo: i, StackValue: true);
13153 }
13154 Changed = true;
13155 }
13156 (void)Changed;
13157 assert(Changed && "Salvage target doesn't use N");
13158
13159 bool IsVariadic =
13160 DV->isVariadic() || OrigLocOpsSize != NewLocOps.size();
13161
13162 auto AdditionalDependencies = DV->getAdditionalDependencies();
13163 SDDbgValue *Clone = getDbgValueList(
13164 Var: DV->getVariable(), Expr: DIExpr, Locs: NewLocOps, Dependencies: AdditionalDependencies,
13165 IsIndirect: DV->isIndirect(), DL: DV->getDebugLoc(), O: DV->getOrder(), IsVariadic);
13166 ClonedDVs.push_back(Elt: Clone);
13167 DV->setIsInvalidated();
13168 DV->setIsEmitted();
13169 LLVM_DEBUG(dbgs() << "SALVAGE: Rewriting";
13170 N0.getNode()->dumprFull(this);
13171 dbgs() << " into " << *DIExpr << '\n');
13172 }
13173 break;
13174 }
13175 case ISD::TRUNCATE: {
13176 SDValue N0 = N.getOperand(Num: 0);
13177 TypeSize FromSize = N0.getValueSizeInBits();
13178 TypeSize ToSize = N.getValueSizeInBits(ResNo: 0);
13179
13180 DIExpression *DbgExpression = DV->getExpression();
13181 auto ExtOps = DIExpression::getExtOps(FromSize, ToSize, Signed: false);
13182 auto NewLocOps = DV->copyLocationOps();
13183 bool Changed = false;
13184 for (size_t i = 0; i < NewLocOps.size(); ++i) {
13185 if (NewLocOps[i].getKind() != SDDbgOperand::SDNODE ||
13186 NewLocOps[i].getSDNode() != &N)
13187 continue;
13188
13189 NewLocOps[i] = GetLocationOperand(N0.getNode(), N0.getResNo());
13190 DbgExpression = DIExpression::appendOpsToArg(Expr: DbgExpression, Ops: ExtOps, ArgNo: i);
13191 Changed = true;
13192 }
13193 assert(Changed && "Salvage target doesn't use N");
13194 (void)Changed;
13195
13196 SDDbgValue *Clone =
13197 getDbgValueList(Var: DV->getVariable(), Expr: DbgExpression, Locs: NewLocOps,
13198 Dependencies: DV->getAdditionalDependencies(), IsIndirect: DV->isIndirect(),
13199 DL: DV->getDebugLoc(), O: DV->getOrder(), IsVariadic: DV->isVariadic());
13200
13201 ClonedDVs.push_back(Elt: Clone);
13202 DV->setIsInvalidated();
13203 DV->setIsEmitted();
13204 LLVM_DEBUG(dbgs() << "SALVAGE: Rewriting"; N0.getNode()->dumprFull(this);
13205 dbgs() << " into " << *DbgExpression << '\n');
13206 break;
13207 }
13208 }
13209 }
13210
13211 for (SDDbgValue *Dbg : ClonedDVs) {
13212 assert((!Dbg->getSDNodes().empty() ||
13213 llvm::any_of(Dbg->getLocationOps(),
13214 [&](const SDDbgOperand &Op) {
13215 return Op.getKind() == SDDbgOperand::FRAMEIX;
13216 })) &&
13217 "Salvaged DbgValue should depend on a new SDNode");
13218 AddDbgValue(DB: Dbg, isParameter: false);
13219 }
13220}
13221
13222/// Creates a SDDbgLabel node.
13223SDDbgLabel *SelectionDAG::getDbgLabel(DILabel *Label,
13224 const DebugLoc &DL, unsigned O) {
13225 assert(cast<DILabel>(Label)->isValidLocationForIntrinsic(DL) &&
13226 "Expected inlined-at fields to agree");
13227 return new (DbgInfo->getAlloc()) SDDbgLabel(Label, DL, O);
13228}
13229
13230namespace {
13231
13232/// RAUWUpdateListener - Helper for ReplaceAllUsesWith - When the node
13233/// pointed to by a use iterator is deleted, increment the use iterator
13234/// so that it doesn't dangle.
13235///
13236class RAUWUpdateListener : public SelectionDAG::DAGUpdateListener {
13237 SDNode::use_iterator &UI;
13238 SDNode::use_iterator &UE;
13239
13240 void NodeDeleted(SDNode *N, SDNode *E) override {
13241 // Increment the iterator as needed.
13242 while (UI != UE && N == UI->getUser())
13243 ++UI;
13244 }
13245
13246public:
13247 RAUWUpdateListener(SelectionDAG &d,
13248 SDNode::use_iterator &ui,
13249 SDNode::use_iterator &ue)
13250 : SelectionDAG::DAGUpdateListener(d), UI(ui), UE(ue) {}
13251};
13252
13253} // end anonymous namespace
13254
13255/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
13256/// This can cause recursive merging of nodes in the DAG.
13257///
13258/// This version assumes From has a single result value.
13259///
13260void SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To) {
13261 SDNode *From = FromN.getNode();
13262 assert(From->getNumValues() == 1 && FromN.getResNo() == 0 &&
13263 "Cannot replace with this method!");
13264 assert(From != To.getNode() && "Cannot replace uses of with self");
13265
13266 // Preserve Debug Values
13267 transferDbgValues(From: FromN, To);
13268 // Preserve extra info.
13269 copyExtraInfo(From, To: To.getNode());
13270
13271 // Iterate over all the existing uses of From. New uses will be added
13272 // to the beginning of the use list, which we avoid visiting.
13273 // This specifically avoids visiting uses of From that arise while the
13274 // replacement is happening, because any such uses would be the result
13275 // of CSE: If an existing node looks like From after one of its operands
13276 // is replaced by To, we don't want to replace of all its users with To
13277 // too. See PR3018 for more info.
13278 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
13279 RAUWUpdateListener Listener(*this, UI, UE);
13280 while (UI != UE) {
13281 SDNode *User = UI->getUser();
13282
13283 // This node is about to morph, remove its old self from the CSE maps.
13284 RemoveNodeFromCSEMaps(N: User);
13285
13286 // A user can appear in a use list multiple times, and when this
13287 // happens the uses are usually next to each other in the list.
13288 // To help reduce the number of CSE recomputations, process all
13289 // the uses of this user that we can find this way.
13290 do {
13291 SDUse &Use = *UI;
13292 ++UI;
13293 Use.set(To);
13294 if (To->isDivergent() != From->isDivergent())
13295 updateDivergence(N: User);
13296 } while (UI != UE && UI->getUser() == User);
13297 // Now that we have modified User, add it back to the CSE maps. If it
13298 // already exists there, recursively merge the results together.
13299 AddModifiedNodeToCSEMaps(N: User);
13300 }
13301
13302 // If we just RAUW'd the root, take note.
13303 if (FromN == getRoot())
13304 setRoot(To);
13305}
13306
13307/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
13308/// This can cause recursive merging of nodes in the DAG.
13309///
13310/// This version assumes that for each value of From, there is a
13311/// corresponding value in To in the same position with the same type.
13312///
13313void SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To) {
13314#ifndef NDEBUG
13315 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
13316 assert((!From->hasAnyUseOfValue(i) ||
13317 From->getValueType(i) == To->getValueType(i)) &&
13318 "Cannot use this version of ReplaceAllUsesWith!");
13319#endif
13320
13321 // Handle the trivial case.
13322 if (From == To)
13323 return;
13324
13325 // Preserve Debug Info. Only do this if there's a use.
13326 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
13327 if (From->hasAnyUseOfValue(Value: i)) {
13328 assert((i < To->getNumValues()) && "Invalid To location");
13329 transferDbgValues(From: SDValue(From, i), To: SDValue(To, i));
13330 }
13331 // Preserve extra info.
13332 copyExtraInfo(From, To);
13333
13334 // Iterate over just the existing users of From. See the comments in
13335 // the ReplaceAllUsesWith above.
13336 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
13337 RAUWUpdateListener Listener(*this, UI, UE);
13338 while (UI != UE) {
13339 SDNode *User = UI->getUser();
13340
13341 // This node is about to morph, remove its old self from the CSE maps.
13342 RemoveNodeFromCSEMaps(N: User);
13343
13344 // A user can appear in a use list multiple times, and when this
13345 // happens the uses are usually next to each other in the list.
13346 // To help reduce the number of CSE recomputations, process all
13347 // the uses of this user that we can find this way.
13348 do {
13349 SDUse &Use = *UI;
13350 ++UI;
13351 Use.setNode(To);
13352 if (To->isDivergent() != From->isDivergent())
13353 updateDivergence(N: User);
13354 } while (UI != UE && UI->getUser() == User);
13355
13356 // Now that we have modified User, add it back to the CSE maps. If it
13357 // already exists there, recursively merge the results together.
13358 AddModifiedNodeToCSEMaps(N: User);
13359 }
13360
13361 // If we just RAUW'd the root, take note.
13362 if (From == getRoot().getNode())
13363 setRoot(SDValue(To, getRoot().getResNo()));
13364}
13365
13366/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
13367/// This can cause recursive merging of nodes in the DAG.
13368///
13369/// This version can replace From with any result values. To must match the
13370/// number and types of values returned by From.
13371void SelectionDAG::ReplaceAllUsesWith(SDNode *From, const SDValue *To) {
13372 if (From->getNumValues() == 1) // Handle the simple case efficiently.
13373 return ReplaceAllUsesWith(FromN: SDValue(From, 0), To: To[0]);
13374
13375 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i) {
13376 // Preserve Debug Info.
13377 transferDbgValues(From: SDValue(From, i), To: To[i]);
13378 // Preserve extra info.
13379 copyExtraInfo(From, To: To[i].getNode());
13380 }
13381
13382 // Iterate over just the existing users of From. See the comments in
13383 // the ReplaceAllUsesWith above.
13384 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
13385 RAUWUpdateListener Listener(*this, UI, UE);
13386 while (UI != UE) {
13387 SDNode *User = UI->getUser();
13388
13389 // This node is about to morph, remove its old self from the CSE maps.
13390 RemoveNodeFromCSEMaps(N: User);
13391
13392 // A user can appear in a use list multiple times, and when this happens the
13393 // uses are usually next to each other in the list. To help reduce the
13394 // number of CSE and divergence recomputations, process all the uses of this
13395 // user that we can find this way.
13396 bool To_IsDivergent = false;
13397 do {
13398 SDUse &Use = *UI;
13399 const SDValue &ToOp = To[Use.getResNo()];
13400 ++UI;
13401 Use.set(ToOp);
13402 if (ToOp.getValueType() != MVT::Other)
13403 To_IsDivergent |= ToOp->isDivergent();
13404 } while (UI != UE && UI->getUser() == User);
13405
13406 if (To_IsDivergent != From->isDivergent())
13407 updateDivergence(N: User);
13408
13409 // Now that we have modified User, add it back to the CSE maps. If it
13410 // already exists there, recursively merge the results together.
13411 AddModifiedNodeToCSEMaps(N: User);
13412 }
13413
13414 // If we just RAUW'd the root, take note.
13415 if (From == getRoot().getNode())
13416 setRoot(SDValue(To[getRoot().getResNo()]));
13417}
13418
13419/// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving
13420/// uses of other values produced by From.getNode() alone. The Deleted
13421/// vector is handled the same way as for ReplaceAllUsesWith.
13422void SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To){
13423 // Handle the really simple, really trivial case efficiently.
13424 if (From == To) return;
13425
13426 // Handle the simple, trivial, case efficiently.
13427 if (From.getNode()->getNumValues() == 1) {
13428 ReplaceAllUsesWith(FromN: From, To);
13429 return;
13430 }
13431
13432 // Preserve Debug Info.
13433 transferDbgValues(From, To);
13434 copyExtraInfo(From: From.getNode(), To: To.getNode());
13435
13436 // Iterate over just the existing users of From. See the comments in
13437 // the ReplaceAllUsesWith above.
13438 SDNode::use_iterator UI = From.getNode()->use_begin(),
13439 UE = From.getNode()->use_end();
13440 RAUWUpdateListener Listener(*this, UI, UE);
13441 while (UI != UE) {
13442 SDNode *User = UI->getUser();
13443 bool UserRemovedFromCSEMaps = false;
13444
13445 // A user can appear in a use list multiple times, and when this
13446 // happens the uses are usually next to each other in the list.
13447 // To help reduce the number of CSE recomputations, process all
13448 // the uses of this user that we can find this way.
13449 do {
13450 SDUse &Use = *UI;
13451
13452 // Skip uses of different values from the same node.
13453 if (Use.getResNo() != From.getResNo()) {
13454 ++UI;
13455 continue;
13456 }
13457
13458 // If this node hasn't been modified yet, it's still in the CSE maps,
13459 // so remove its old self from the CSE maps.
13460 if (!UserRemovedFromCSEMaps) {
13461 RemoveNodeFromCSEMaps(N: User);
13462 UserRemovedFromCSEMaps = true;
13463 }
13464
13465 ++UI;
13466 Use.set(To);
13467 if (To->isDivergent() != From->isDivergent())
13468 updateDivergence(N: User);
13469 } while (UI != UE && UI->getUser() == User);
13470 // We are iterating over all uses of the From node, so if a use
13471 // doesn't use the specific value, no changes are made.
13472 if (!UserRemovedFromCSEMaps)
13473 continue;
13474
13475 // Now that we have modified User, add it back to the CSE maps. If it
13476 // already exists there, recursively merge the results together.
13477 AddModifiedNodeToCSEMaps(N: User);
13478 }
13479
13480 // If we just RAUW'd the root, take note.
13481 if (From == getRoot())
13482 setRoot(To);
13483}
13484
13485namespace {
13486
13487/// UseMemo - This class is used by SelectionDAG::ReplaceAllUsesOfValuesWith
13488/// to record information about a use.
13489struct UseMemo {
13490 SDNode *User;
13491 unsigned Index;
13492 SDUse *Use;
13493};
13494
13495/// operator< - Sort Memos by User.
13496bool operator<(const UseMemo &L, const UseMemo &R) {
13497 return (intptr_t)L.User < (intptr_t)R.User;
13498}
13499
13500/// RAUOVWUpdateListener - Helper for ReplaceAllUsesOfValuesWith - When the node
13501/// pointed to by a UseMemo is deleted, set the User to nullptr to indicate that
13502/// the node already has been taken care of recursively.
13503class RAUOVWUpdateListener : public SelectionDAG::DAGUpdateListener {
13504 SmallVectorImpl<UseMemo> &Uses;
13505
13506 void NodeDeleted(SDNode *N, SDNode *E) override {
13507 for (UseMemo &Memo : Uses)
13508 if (Memo.User == N)
13509 Memo.User = nullptr;
13510 }
13511
13512public:
13513 RAUOVWUpdateListener(SelectionDAG &d, SmallVectorImpl<UseMemo> &uses)
13514 : SelectionDAG::DAGUpdateListener(d), Uses(uses) {}
13515};
13516
13517} // end anonymous namespace
13518
13519/// Return true if a glue output should propagate divergence information.
13520static bool gluePropagatesDivergence(const SDNode *Node) {
13521 switch (Node->getOpcode()) {
13522 case ISD::CopyFromReg:
13523 case ISD::CopyToReg:
13524 return false;
13525 default:
13526 return true;
13527 }
13528
13529 llvm_unreachable("covered opcode switch");
13530}
13531
13532bool SelectionDAG::calculateDivergence(SDNode *N) {
13533 if (TLI->isSDNodeAlwaysUniform(N)) {
13534 assert(!TLI->isSDNodeSourceOfDivergence(N, FLI, UA) &&
13535 "Conflicting divergence information!");
13536 return false;
13537 }
13538 if (TLI->isSDNodeSourceOfDivergence(N, FLI, UA))
13539 return true;
13540 for (const auto &Op : N->ops()) {
13541 EVT VT = Op.getValueType();
13542
13543 // Skip Chain. It does not carry divergence.
13544 if (VT != MVT::Other && Op.getNode()->isDivergent() &&
13545 (VT != MVT::Glue || gluePropagatesDivergence(Node: Op.getNode())))
13546 return true;
13547 }
13548 return false;
13549}
13550
13551void SelectionDAG::updateDivergence(SDNode *N) {
13552 SmallVector<SDNode *, 16> Worklist(1, N);
13553 do {
13554 N = Worklist.pop_back_val();
13555 bool IsDivergent = calculateDivergence(N);
13556 if (N->SDNodeBits.IsDivergent != IsDivergent) {
13557 N->SDNodeBits.IsDivergent = IsDivergent;
13558 llvm::append_range(C&: Worklist, R: N->users());
13559 }
13560 } while (!Worklist.empty());
13561}
13562
13563void SelectionDAG::CreateTopologicalOrder(std::vector<SDNode *> &Order) {
13564 DenseMap<SDNode *, unsigned> Degree;
13565 Order.reserve(n: AllNodes.size());
13566 for (auto &N : allnodes()) {
13567 unsigned NOps = N.getNumOperands();
13568 Degree[&N] = NOps;
13569 if (0 == NOps)
13570 Order.push_back(x: &N);
13571 }
13572 for (size_t I = 0; I != Order.size(); ++I) {
13573 SDNode *N = Order[I];
13574 for (auto *U : N->users()) {
13575 unsigned &UnsortedOps = Degree[U];
13576 if (0 == --UnsortedOps)
13577 Order.push_back(x: U);
13578 }
13579 }
13580}
13581
13582#if !defined(NDEBUG) && LLVM_ENABLE_ABI_BREAKING_CHECKS
13583void SelectionDAG::VerifyDAGDivergence() {
13584 std::vector<SDNode *> TopoOrder;
13585 CreateTopologicalOrder(TopoOrder);
13586 for (auto *N : TopoOrder) {
13587 assert(calculateDivergence(N) == N->isDivergent() &&
13588 "Divergence bit inconsistency detected");
13589 }
13590}
13591#endif
13592
13593/// ReplaceAllUsesOfValuesWith - Replace any uses of From with To, leaving
13594/// uses of other values produced by From.getNode() alone. The same value
13595/// may appear in both the From and To list. The Deleted vector is
13596/// handled the same way as for ReplaceAllUsesWith.
13597void SelectionDAG::ReplaceAllUsesOfValuesWith(const SDValue *From,
13598 const SDValue *To,
13599 unsigned Num){
13600 // Handle the simple, trivial case efficiently.
13601 if (Num == 1)
13602 return ReplaceAllUsesOfValueWith(From: *From, To: *To);
13603
13604 transferDbgValues(From: *From, To: *To);
13605 copyExtraInfo(From: From->getNode(), To: To->getNode());
13606
13607 // Read up all the uses and make records of them. This helps
13608 // processing new uses that are introduced during the
13609 // replacement process.
13610 SmallVector<UseMemo, 4> Uses;
13611 for (unsigned i = 0; i != Num; ++i) {
13612 unsigned FromResNo = From[i].getResNo();
13613 SDNode *FromNode = From[i].getNode();
13614 for (SDUse &Use : FromNode->uses()) {
13615 if (Use.getResNo() == FromResNo) {
13616 UseMemo Memo = {.User: Use.getUser(), .Index: i, .Use: &Use};
13617 Uses.push_back(Elt: Memo);
13618 }
13619 }
13620 }
13621
13622 // Sort the uses, so that all the uses from a given User are together.
13623 llvm::sort(C&: Uses);
13624 RAUOVWUpdateListener Listener(*this, Uses);
13625
13626 for (unsigned UseIndex = 0, UseIndexEnd = Uses.size();
13627 UseIndex != UseIndexEnd; ) {
13628 // We know that this user uses some value of From. If it is the right
13629 // value, update it.
13630 SDNode *User = Uses[UseIndex].User;
13631 // If the node has been deleted by recursive CSE updates when updating
13632 // another node, then just skip this entry.
13633 if (User == nullptr) {
13634 ++UseIndex;
13635 continue;
13636 }
13637
13638 // This node is about to morph, remove its old self from the CSE maps.
13639 RemoveNodeFromCSEMaps(N: User);
13640
13641 // The Uses array is sorted, so all the uses for a given User
13642 // are next to each other in the list.
13643 // To help reduce the number of CSE recomputations, process all
13644 // the uses of this user that we can find this way.
13645 do {
13646 unsigned i = Uses[UseIndex].Index;
13647 SDUse &Use = *Uses[UseIndex].Use;
13648 ++UseIndex;
13649
13650 Use.set(To[i]);
13651 } while (UseIndex != UseIndexEnd && Uses[UseIndex].User == User);
13652
13653 // Now that we have modified User, add it back to the CSE maps. If it
13654 // already exists there, recursively merge the results together.
13655 AddModifiedNodeToCSEMaps(N: User);
13656 }
13657}
13658
13659/// AssignTopologicalOrder - Assign a unique node id for each node in the DAG
13660/// based on their topological order. It returns the maximum id and a vector
13661/// of the SDNodes* in assigned order by reference.
13662unsigned SelectionDAG::AssignTopologicalOrder() {
13663 unsigned DAGSize = 0;
13664
13665 // SortedPos tracks the progress of the algorithm. Nodes before it are
13666 // sorted, nodes after it are unsorted. When the algorithm completes
13667 // it is at the end of the list.
13668 allnodes_iterator SortedPos = allnodes_begin();
13669
13670 // Visit all the nodes. Move nodes with no operands to the front of
13671 // the list immediately. Annotate nodes that do have operands with their
13672 // operand count. Before we do this, the Node Id fields of the nodes
13673 // may contain arbitrary values. After, the Node Id fields for nodes
13674 // before SortedPos will contain the topological sort index, and the
13675 // Node Id fields for nodes At SortedPos and after will contain the
13676 // count of outstanding operands.
13677 for (SDNode &N : llvm::make_early_inc_range(Range: allnodes())) {
13678 checkForCycles(N: &N, DAG: this);
13679 unsigned Degree = N.getNumOperands();
13680 if (Degree == 0) {
13681 // A node with no uses, add it to the result array immediately.
13682 N.setNodeId(DAGSize++);
13683 allnodes_iterator Q(&N);
13684 if (Q != SortedPos)
13685 SortedPos = AllNodes.insert(where: SortedPos, New: AllNodes.remove(IT&: Q));
13686 assert(SortedPos != AllNodes.end() && "Overran node list");
13687 ++SortedPos;
13688 } else {
13689 // Temporarily use the Node Id as scratch space for the degree count.
13690 N.setNodeId(Degree);
13691 }
13692 }
13693
13694 // Visit all the nodes. As we iterate, move nodes into sorted order,
13695 // such that by the time the end is reached all nodes will be sorted.
13696 for (SDNode &Node : allnodes()) {
13697 SDNode *N = &Node;
13698 checkForCycles(N, DAG: this);
13699 // N is in sorted position, so all its uses have one less operand
13700 // that needs to be sorted.
13701 for (SDNode *P : N->users()) {
13702 unsigned Degree = P->getNodeId();
13703 assert(Degree != 0 && "Invalid node degree");
13704 --Degree;
13705 if (Degree == 0) {
13706 // All of P's operands are sorted, so P may sorted now.
13707 P->setNodeId(DAGSize++);
13708 if (P->getIterator() != SortedPos)
13709 SortedPos = AllNodes.insert(where: SortedPos, New: AllNodes.remove(IT: P));
13710 assert(SortedPos != AllNodes.end() && "Overran node list");
13711 ++SortedPos;
13712 } else {
13713 // Update P's outstanding operand count.
13714 P->setNodeId(Degree);
13715 }
13716 }
13717 if (Node.getIterator() == SortedPos) {
13718#ifndef NDEBUG
13719 allnodes_iterator I(N);
13720 SDNode *S = &*++I;
13721 dbgs() << "Overran sorted position:\n";
13722 S->dumprFull(this); dbgs() << "\n";
13723 dbgs() << "Checking if this is due to cycles\n";
13724 checkForCycles(this, true);
13725#endif
13726 llvm_unreachable(nullptr);
13727 }
13728 }
13729
13730 assert(SortedPos == AllNodes.end() &&
13731 "Topological sort incomplete!");
13732 assert(AllNodes.front().getOpcode() == ISD::EntryToken &&
13733 "First node in topological sort is not the entry token!");
13734 assert(AllNodes.front().getNodeId() == 0 &&
13735 "First node in topological sort has non-zero id!");
13736 assert(AllNodes.front().getNumOperands() == 0 &&
13737 "First node in topological sort has operands!");
13738 assert(AllNodes.back().getNodeId() == (int)DAGSize-1 &&
13739 "Last node in topologic sort has unexpected id!");
13740 assert(AllNodes.back().use_empty() &&
13741 "Last node in topologic sort has users!");
13742 assert(DAGSize == allnodes_size() && "Node count mismatch!");
13743 return DAGSize;
13744}
13745
13746void SelectionDAG::getTopologicallyOrderedNodes(
13747 SmallVectorImpl<const SDNode *> &SortedNodes) const {
13748 SortedNodes.clear();
13749 // Node -> remaining number of outstanding operands.
13750 DenseMap<const SDNode *, unsigned> RemainingOperands;
13751
13752 // Put nodes without any operands into SortedNodes first.
13753 for (const SDNode &N : allnodes()) {
13754 checkForCycles(N: &N, DAG: this);
13755 unsigned NumOperands = N.getNumOperands();
13756 if (NumOperands == 0)
13757 SortedNodes.push_back(Elt: &N);
13758 else
13759 // Record their total number of outstanding operands.
13760 RemainingOperands[&N] = NumOperands;
13761 }
13762
13763 // A node is pushed into SortedNodes when all of its operands (predecessors in
13764 // the graph) are also in SortedNodes.
13765 for (unsigned i = 0U; i < SortedNodes.size(); ++i) {
13766 const SDNode *N = SortedNodes[i];
13767 for (const SDNode *U : N->users()) {
13768 // HandleSDNode is never part of a DAG and therefore has no entry in
13769 // RemainingOperands.
13770 if (U->getOpcode() == ISD::HANDLENODE)
13771 continue;
13772 unsigned &NumRemOperands = RemainingOperands[U];
13773 assert(NumRemOperands && "Invalid number of remaining operands");
13774 --NumRemOperands;
13775 if (!NumRemOperands)
13776 SortedNodes.push_back(Elt: U);
13777 }
13778 }
13779
13780 assert(SortedNodes.size() == AllNodes.size() && "Node count mismatch");
13781 assert(SortedNodes.front()->getOpcode() == ISD::EntryToken &&
13782 "First node in topological sort is not the entry token");
13783 assert(SortedNodes.front()->getNumOperands() == 0 &&
13784 "First node in topological sort has operands");
13785}
13786
13787/// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the
13788/// value is produced by SD.
13789void SelectionDAG::AddDbgValue(SDDbgValue *DB, bool isParameter) {
13790 for (SDNode *SD : DB->getSDNodes()) {
13791 if (!SD)
13792 continue;
13793 assert(DbgInfo->getSDDbgValues(SD).empty() || SD->getHasDebugValue());
13794 SD->setHasDebugValue(true);
13795 }
13796 DbgInfo->add(V: DB, isParameter);
13797}
13798
13799void SelectionDAG::AddDbgLabel(SDDbgLabel *DB) { DbgInfo->add(L: DB); }
13800
13801SDValue SelectionDAG::makeEquivalentMemoryOrdering(SDValue OldChain,
13802 SDValue NewMemOpChain) {
13803 assert(isa<MemSDNode>(NewMemOpChain) && "Expected a memop node");
13804 assert(NewMemOpChain.getValueType() == MVT::Other && "Expected a token VT");
13805 // The new memory operation must have the same position as the old load in
13806 // terms of memory dependency. Create a TokenFactor for the old load and new
13807 // memory operation and update uses of the old load's output chain to use that
13808 // TokenFactor.
13809 if (OldChain == NewMemOpChain || OldChain.use_empty())
13810 return NewMemOpChain;
13811
13812 SDValue TokenFactor = getNode(Opcode: ISD::TokenFactor, DL: SDLoc(OldChain), VT: MVT::Other,
13813 N1: OldChain, N2: NewMemOpChain);
13814 ReplaceAllUsesOfValueWith(From: OldChain, To: TokenFactor);
13815 UpdateNodeOperands(N: TokenFactor.getNode(), Op1: OldChain, Op2: NewMemOpChain);
13816 return TokenFactor;
13817}
13818
13819SDValue SelectionDAG::makeEquivalentMemoryOrdering(LoadSDNode *OldLoad,
13820 SDValue NewMemOp) {
13821 assert(isa<MemSDNode>(NewMemOp.getNode()) && "Expected a memop node");
13822 SDValue OldChain = SDValue(OldLoad, 1);
13823 SDValue NewMemOpChain = NewMemOp.getValue(R: 1);
13824 return makeEquivalentMemoryOrdering(OldChain, NewMemOpChain);
13825}
13826
13827SDValue SelectionDAG::getSymbolFunctionGlobalAddress(SDValue Op,
13828 Function **OutFunction) {
13829 assert(isa<ExternalSymbolSDNode>(Op) && "Node should be an ExternalSymbol");
13830
13831 auto *Symbol = cast<ExternalSymbolSDNode>(Val&: Op)->getSymbol();
13832 auto *Module = MF->getFunction().getParent();
13833 auto *Function = Module->getFunction(Name: Symbol);
13834
13835 if (OutFunction != nullptr)
13836 *OutFunction = Function;
13837
13838 if (Function != nullptr) {
13839 auto PtrTy = TLI->getPointerTy(DL: getDataLayout(), AS: Function->getAddressSpace());
13840 return getGlobalAddress(GV: Function, DL: SDLoc(Op), VT: PtrTy);
13841 }
13842
13843 std::string ErrorStr;
13844 raw_string_ostream ErrorFormatter(ErrorStr);
13845 ErrorFormatter << "Undefined external symbol ";
13846 ErrorFormatter << '"' << Symbol << '"';
13847 report_fatal_error(reason: Twine(ErrorStr));
13848}
13849
13850//===----------------------------------------------------------------------===//
13851// SDNode Class
13852//===----------------------------------------------------------------------===//
13853
13854bool llvm::isNullConstant(SDValue V) {
13855 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val&: V);
13856 return Const != nullptr && Const->isZero();
13857}
13858
13859bool llvm::isNullConstantOrUndef(SDValue V) {
13860 return V.isUndef() || isNullConstant(V);
13861}
13862
13863bool llvm::isNullFPConstant(SDValue V) {
13864 ConstantFPSDNode *Const = dyn_cast<ConstantFPSDNode>(Val&: V);
13865 return Const != nullptr && Const->isZero() && !Const->isNegative();
13866}
13867
13868bool llvm::isAllOnesConstant(SDValue V) {
13869 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val&: V);
13870 return Const != nullptr && Const->isAllOnes();
13871}
13872
13873bool llvm::isOneConstant(SDValue V) {
13874 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val&: V);
13875 return Const != nullptr && Const->isOne();
13876}
13877
13878bool llvm::isMinSignedConstant(SDValue V) {
13879 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val&: V);
13880 return Const != nullptr && Const->isMinSignedValue();
13881}
13882
13883bool SelectionDAG::isIdentityElement(unsigned Opcode, SDNodeFlags Flags,
13884 SDValue V, unsigned OperandNo,
13885 unsigned Depth) const {
13886 APInt DemandedElts = getDemandAllEltsMask(V);
13887 return isIdentityElement(Opc: Opcode, Flags, V, DemandedElts, OperandNo, Depth);
13888}
13889
13890bool SelectionDAG::isIdentityElement(unsigned Opcode, SDNodeFlags Flags,
13891 SDValue V, const APInt &DemandedElts,
13892 unsigned OperandNo, unsigned Depth) const {
13893 // NOTE: The cases should match with IR's ConstantExpr::getBinOpIdentity().
13894 // TODO: Target-specific opcodes could be added.
13895 if (V.getValueType().isInteger()) {
13896 KnownBits Known = computeKnownBits(Op: V, DemandedElts, Depth);
13897 if (Known.isConstant()) {
13898 const APInt &Const = Known.getConstant();
13899 switch (Opcode) {
13900 case ISD::ADD:
13901 case ISD::OR:
13902 case ISD::XOR:
13903 case ISD::UMAX:
13904 return Const.isZero();
13905 case ISD::MUL:
13906 return Const.isOne();
13907 case ISD::AND:
13908 case ISD::UMIN:
13909 return Const.isAllOnes();
13910 case ISD::SMAX:
13911 return Const.isMinSignedValue();
13912 case ISD::SMIN:
13913 return Const.isMaxSignedValue();
13914 case ISD::SUB:
13915 case ISD::SHL:
13916 case ISD::SRA:
13917 case ISD::SRL:
13918 return OperandNo == 1 && Const.isZero();
13919 case ISD::UDIV:
13920 case ISD::SDIV:
13921 return OperandNo == 1 && Const.isOne();
13922 }
13923 }
13924 } else if (auto *ConstFP = isConstOrConstSplatFP(N: V, DemandedElts)) {
13925 switch (Opcode) {
13926 case ISD::FADD:
13927 return ConstFP->isZero() &&
13928 (Flags.hasNoSignedZeros() || ConstFP->isNegative());
13929 case ISD::FSUB:
13930 return OperandNo == 1 && ConstFP->isZero() &&
13931 (Flags.hasNoSignedZeros() || !ConstFP->isNegative());
13932 case ISD::FMUL:
13933 return ConstFP->isOne();
13934 case ISD::FDIV:
13935 return OperandNo == 1 && ConstFP->isOne();
13936 case ISD::FMINNUM:
13937 case ISD::FMAXNUM:
13938 case ISD::FMINIMUMNUM:
13939 case ISD::FMAXIMUMNUM: {
13940 // Neutral element for fminnum/fminimumnum is NaN, Inf or FLT_MAX,
13941 // depending on fast-math flags (FMF).
13942 EVT VT = V.getValueType();
13943 const fltSemantics &Semantics = VT.getFltSemantics();
13944 APFloat NeutralAF = !Flags.hasNoNaNs() ? APFloat::getQNaN(Sem: Semantics)
13945 : !Flags.hasNoInfs() ? APFloat::getInf(Sem: Semantics)
13946 : APFloat::getLargest(Sem: Semantics);
13947 if (Opcode == ISD::FMAXNUM || Opcode == ISD::FMAXIMUMNUM)
13948 NeutralAF.changeSign();
13949
13950 return ConstFP->isExactlyValue(V: NeutralAF);
13951 }
13952 case ISD::FMINIMUM:
13953 case ISD::FMAXIMUM: {
13954 // Neutral element for fminimum is Inf or FLT_MAX, depending on FMF.
13955 const APFloat &VAPF = ConstFP->getValueAPF();
13956 bool NeutralNegative = (Opcode == ISD::FMAXIMUM);
13957 if (Flags.hasNoInfs())
13958 return VAPF.isLargest() && VAPF.isNegative() == NeutralNegative;
13959 return VAPF.isInfinity() && VAPF.isNegative() == NeutralNegative;
13960 }
13961 }
13962 }
13963 return false;
13964}
13965
13966SDValue llvm::peekThroughBitcasts(SDValue V) {
13967 while (V.getOpcode() == ISD::BITCAST)
13968 V = V.getOperand(i: 0);
13969 return V;
13970}
13971
13972SDValue llvm::peekThroughOneUseBitcasts(SDValue V) {
13973 while (V.getOpcode() == ISD::BITCAST && V.getOperand(i: 0).hasOneUse())
13974 V = V.getOperand(i: 0);
13975 return V;
13976}
13977
13978SDValue llvm::peekThroughExtractSubvectors(SDValue V) {
13979 while (V.getOpcode() == ISD::EXTRACT_SUBVECTOR)
13980 V = V.getOperand(i: 0);
13981 return V;
13982}
13983
13984SDValue llvm::peekThroughInsertVectorElt(SDValue V, const APInt &DemandedElts) {
13985 while (V.getOpcode() == ISD::INSERT_VECTOR_ELT) {
13986 SDValue InVec = V.getOperand(i: 0);
13987 SDValue EltNo = V.getOperand(i: 2);
13988 EVT VT = InVec.getValueType();
13989 auto *IndexC = dyn_cast<ConstantSDNode>(Val&: EltNo);
13990 if (IndexC && VT.isFixedLengthVector() &&
13991 IndexC->getAPIntValue().ult(RHS: VT.getVectorNumElements()) &&
13992 !DemandedElts[IndexC->getZExtValue()]) {
13993 V = InVec;
13994 continue;
13995 }
13996 break;
13997 }
13998 return V;
13999}
14000
14001SDValue llvm::peekThroughTruncates(SDValue V) {
14002 while (V.getOpcode() == ISD::TRUNCATE)
14003 V = V.getOperand(i: 0);
14004 return V;
14005}
14006
14007bool llvm::isBitwiseNot(SDValue V, bool AllowUndefs) {
14008 if (V.getOpcode() != ISD::XOR)
14009 return false;
14010 V = peekThroughBitcasts(V: V.getOperand(i: 1));
14011 unsigned NumBits = V.getScalarValueSizeInBits();
14012 ConstantSDNode *C =
14013 isConstOrConstSplat(N: V, AllowUndefs, /*AllowTruncation*/ true);
14014 return C && (C->getAPIntValue().countr_one() >= NumBits);
14015}
14016
14017ConstantSDNode *llvm::isConstOrConstSplat(SDValue N, bool AllowUndefs,
14018 bool AllowTruncation) {
14019 APInt DemandedElts = getDemandAllEltsMask(V: N);
14020 return isConstOrConstSplat(N, DemandedElts, AllowUndefs, AllowTruncation);
14021}
14022
14023ConstantSDNode *llvm::isConstOrConstSplat(SDValue N, const APInt &DemandedElts,
14024 bool AllowUndefs,
14025 bool AllowTruncation) {
14026 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val&: N))
14027 return CN;
14028
14029 // SplatVectors can truncate their operands. Ignore that case here unless
14030 // AllowTruncation is set.
14031 if (N->getOpcode() == ISD::SPLAT_VECTOR) {
14032 EVT VecEltVT = N->getValueType(ResNo: 0).getVectorElementType();
14033 if (auto *CN = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 0))) {
14034 EVT CVT = CN->getValueType(ResNo: 0);
14035 assert(CVT.bitsGE(VecEltVT) && "Illegal splat_vector element extension");
14036 if (AllowTruncation || CVT == VecEltVT)
14037 return CN;
14038 }
14039 }
14040
14041 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Val&: N)) {
14042 BitVector UndefElements;
14043 ConstantSDNode *CN = BV->getConstantSplatNode(DemandedElts, UndefElements: &UndefElements);
14044
14045 // BuildVectors can truncate their operands. Ignore that case here unless
14046 // AllowTruncation is set.
14047 // TODO: Look into whether we should allow UndefElements in non-DemandedElts
14048 if (CN && (UndefElements.none() || AllowUndefs)) {
14049 EVT CVT = CN->getValueType(ResNo: 0);
14050 EVT NSVT = N.getValueType().getScalarType();
14051 assert(CVT.bitsGE(NSVT) && "Illegal build vector element extension");
14052 if (AllowTruncation || (CVT == NSVT))
14053 return CN;
14054 }
14055 }
14056
14057 if (N.getOpcode() == ISD::MERGE_VALUES)
14058 return isConstOrConstSplat(N: N.getOperand(i: N.getResNo()), DemandedElts,
14059 AllowUndefs, AllowTruncation);
14060
14061 return nullptr;
14062}
14063
14064ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N, bool AllowUndefs) {
14065 APInt DemandedElts = getDemandAllEltsMask(V: N);
14066 return isConstOrConstSplatFP(N, DemandedElts, AllowUndefs);
14067}
14068
14069ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N,
14070 const APInt &DemandedElts,
14071 bool AllowUndefs) {
14072 if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(Val&: N))
14073 return CN;
14074
14075 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Val&: N)) {
14076 BitVector UndefElements;
14077 ConstantFPSDNode *CN =
14078 BV->getConstantFPSplatNode(DemandedElts, UndefElements: &UndefElements);
14079 // TODO: Look into whether we should allow UndefElements in non-DemandedElts
14080 if (CN && (UndefElements.none() || AllowUndefs))
14081 return CN;
14082 }
14083
14084 if (N.getOpcode() == ISD::SPLAT_VECTOR)
14085 if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(Val: N.getOperand(i: 0)))
14086 return CN;
14087
14088 return nullptr;
14089}
14090
14091bool llvm::isNullOrNullSplat(SDValue N, bool AllowUndefs) {
14092 // TODO: may want to use peekThroughBitcast() here.
14093 ConstantSDNode *C =
14094 isConstOrConstSplat(N, AllowUndefs, /*AllowTruncation=*/true);
14095 return C && C->isZero();
14096}
14097
14098bool llvm::isOneOrOneSplat(SDValue N, bool AllowUndefs) {
14099 ConstantSDNode *C =
14100 isConstOrConstSplat(N, AllowUndefs, /*AllowTruncation*/ true);
14101 return C && C->isOne();
14102}
14103
14104bool llvm::isOneOrOneSplatFP(SDValue N, bool AllowUndefs) {
14105 ConstantFPSDNode *C = isConstOrConstSplatFP(N, AllowUndefs);
14106 return C && C->isOne();
14107}
14108
14109bool llvm::isAllOnesOrAllOnesSplat(SDValue N, bool AllowUndefs) {
14110 N = peekThroughBitcasts(V: N);
14111 unsigned BitWidth = N.getScalarValueSizeInBits();
14112 ConstantSDNode *C =
14113 isConstOrConstSplat(N, AllowUndefs, /*AllowTruncation=*/true);
14114 return C && C->getAPIntValue().countTrailingOnes() >= BitWidth;
14115}
14116
14117bool llvm::isZeroOrZeroSplat(SDValue N, bool AllowUndefs) {
14118 N = peekThroughBitcasts(V: N);
14119 ConstantSDNode *C = isConstOrConstSplat(N, AllowUndefs, AllowTruncation: true);
14120 return C && C->isZero();
14121}
14122
14123bool llvm::isZeroOrZeroSplatFP(SDValue N, bool AllowUndefs) {
14124 ConstantFPSDNode *C = isConstOrConstSplatFP(N, AllowUndefs);
14125 return C && C->isZero();
14126}
14127
14128HandleSDNode::~HandleSDNode() {
14129 DropOperands();
14130}
14131
14132MemSDNode::MemSDNode(
14133 unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTs, EVT memvt,
14134 PointerUnion<MachineMemOperand *, MachineMemOperand **> memrefs)
14135 : SDNode(Opc, Order, dl, VTs), MemoryVT(memvt), MemRefs(memrefs) {
14136 bool IsVolatile = false;
14137 bool IsNonTemporal = false;
14138 bool IsDereferenceable = true;
14139 bool IsInvariant = true;
14140 for (const MachineMemOperand *MMO : memoperands()) {
14141 IsVolatile |= MMO->isVolatile();
14142 IsNonTemporal |= MMO->isNonTemporal();
14143 IsDereferenceable &= MMO->isDereferenceable();
14144 IsInvariant &= MMO->isInvariant();
14145 }
14146 MemSDNodeBits.IsVolatile = IsVolatile;
14147 MemSDNodeBits.IsNonTemporal = IsNonTemporal;
14148 MemSDNodeBits.IsDereferenceable = IsDereferenceable;
14149 MemSDNodeBits.IsInvariant = IsInvariant;
14150
14151 // For the single-MMO case, we check here that the size of the memory operand
14152 // fits within the size of the MMO. This is because the MMO might indicate
14153 // only a possible address range instead of specifying the affected memory
14154 // addresses precisely.
14155 assert((getNumMemOperands() != 1 || !getMemOperand()->getType().isValid() ||
14156 TypeSize::isKnownLE(memvt.getStoreSize(),
14157 getMemOperand()->getSize().getValue())) &&
14158 "Size mismatch!");
14159}
14160
14161namespace {
14162
14163 struct EVTArray {
14164 std::vector<EVT> VTs;
14165
14166 EVTArray() {
14167 VTs.reserve(n: MVT::VALUETYPE_SIZE);
14168 for (unsigned i = 0; i < MVT::VALUETYPE_SIZE; ++i)
14169 VTs.push_back(x: MVT((MVT::SimpleValueType)i));
14170 }
14171 };
14172
14173} // end anonymous namespace
14174
14175/// getValueTypeList - Return a pointer to the specified value type.
14176///
14177const EVT *SDNode::getValueTypeList(MVT VT) {
14178 static EVTArray SimpleVTArray;
14179
14180 assert(VT < MVT::VALUETYPE_SIZE && "Value type out of range!");
14181 return &SimpleVTArray.VTs[VT.SimpleTy];
14182}
14183
14184/// hasAnyUseOfValue - Return true if there are any use of the indicated
14185/// value. This method ignores uses of other values defined by this operation.
14186bool SDNode::hasAnyUseOfValue(unsigned Value) const {
14187 assert(Value < getNumValues() && "Bad value!");
14188
14189 for (SDUse &U : uses())
14190 if (U.getResNo() == Value)
14191 return true;
14192
14193 return false;
14194}
14195
14196/// isOnlyUserOf - Return true if this node is the only use of N.
14197bool SDNode::isOnlyUserOf(const SDNode *N) const {
14198 bool Seen = false;
14199 for (const SDNode *User : N->users()) {
14200 if (User == this)
14201 Seen = true;
14202 else
14203 return false;
14204 }
14205
14206 return Seen;
14207}
14208
14209/// Return true if the only users of N are contained in Nodes.
14210bool SDNode::areOnlyUsersOf(ArrayRef<const SDNode *> Nodes, const SDNode *N) {
14211 bool Seen = false;
14212 for (const SDNode *User : N->users()) {
14213 if (llvm::is_contained(Range&: Nodes, Element: User))
14214 Seen = true;
14215 else
14216 return false;
14217 }
14218
14219 return Seen;
14220}
14221
14222/// Return true if the referenced return value is an operand of N.
14223bool SDValue::isOperandOf(const SDNode *N) const {
14224 return is_contained(Range: N->op_values(), Element: *this);
14225}
14226
14227bool SDNode::isOperandOf(const SDNode *N) const {
14228 return any_of(Range: N->op_values(),
14229 P: [this](SDValue Op) { return this == Op.getNode(); });
14230}
14231
14232/// reachesChainWithoutSideEffects - Return true if this operand (which must
14233/// be a chain) reaches the specified operand without crossing any
14234/// side-effecting instructions on any chain path. In practice, this looks
14235/// through token factors and non-volatile loads. In order to remain efficient,
14236/// this only looks a couple of nodes in, it does not do an exhaustive search.
14237///
14238/// Note that we only need to examine chains when we're searching for
14239/// side-effects; SelectionDAG requires that all side-effects are represented
14240/// by chains, even if another operand would force a specific ordering. This
14241/// constraint is necessary to allow transformations like splitting loads.
14242bool SDValue::reachesChainWithoutSideEffects(SDValue Dest,
14243 unsigned Depth) const {
14244 if (*this == Dest) return true;
14245
14246 // Don't search too deeply, we just want to be able to see through
14247 // TokenFactor's etc.
14248 if (Depth == 0) return false;
14249
14250 // If this is a token factor, all inputs to the TF happen in parallel.
14251 if (getOpcode() == ISD::TokenFactor) {
14252 // First, try a shallow search.
14253 if (is_contained(Range: (*this)->ops(), Element: Dest)) {
14254 // We found the chain we want as an operand of this TokenFactor.
14255 // Essentially, we reach the chain without side-effects if we could
14256 // serialize the TokenFactor into a simple chain of operations with
14257 // Dest as the last operation. This is automatically true if the
14258 // chain has one use: there are no other ordering constraints.
14259 // If the chain has more than one use, we give up: some other
14260 // use of Dest might force a side-effect between Dest and the current
14261 // node.
14262 if (Dest.hasOneUse())
14263 return true;
14264 }
14265 // Next, try a deep search: check whether every operand of the TokenFactor
14266 // reaches Dest.
14267 return llvm::all_of(Range: (*this)->ops(), P: [=](SDValue Op) {
14268 return Op.reachesChainWithoutSideEffects(Dest, Depth: Depth - 1);
14269 });
14270 }
14271
14272 // Loads don't have side effects, look through them.
14273 if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(Val: *this)) {
14274 if (Ld->isUnordered())
14275 return Ld->getChain().reachesChainWithoutSideEffects(Dest, Depth: Depth-1);
14276 }
14277 return false;
14278}
14279
14280bool SDNode::hasPredecessor(const SDNode *N) const {
14281 SmallPtrSet<const SDNode *, 32> Visited;
14282 SmallVector<const SDNode *, 16> Worklist;
14283 Worklist.push_back(Elt: this);
14284 return hasPredecessorHelper(N, Visited, Worklist);
14285}
14286
14287void SDNode::intersectFlagsWith(const SDNodeFlags Flags) {
14288 this->Flags &= Flags;
14289}
14290
14291SDValue
14292SelectionDAG::matchBinOpReduction(SDNode *Extract, ISD::NodeType &BinOp,
14293 ArrayRef<ISD::NodeType> CandidateBinOps,
14294 bool AllowPartials) {
14295 // The pattern must end in an extract from index 0.
14296 if (Extract->getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
14297 !isNullConstant(V: Extract->getOperand(Num: 1)))
14298 return SDValue();
14299
14300 // Match against one of the candidate binary ops.
14301 SDValue Op = Extract->getOperand(Num: 0);
14302 if (llvm::none_of(Range&: CandidateBinOps, P: [Op](ISD::NodeType BinOp) {
14303 return Op.getOpcode() == unsigned(BinOp);
14304 }))
14305 return SDValue();
14306
14307 // Floating-point reductions may require relaxed constraints on the final step
14308 // of the reduction because they may reorder intermediate operations.
14309 unsigned CandidateBinOp = Op.getOpcode();
14310 if (Op.getValueType().isFloatingPoint()) {
14311 SDNodeFlags Flags = Op->getFlags();
14312 switch (CandidateBinOp) {
14313 case ISD::FADD:
14314 if (!Flags.hasNoSignedZeros() || !Flags.hasAllowReassociation())
14315 return SDValue();
14316 break;
14317 default:
14318 llvm_unreachable("Unhandled FP opcode for binop reduction");
14319 }
14320 }
14321
14322 // Matching failed - attempt to see if we did enough stages that a partial
14323 // reduction from a subvector is possible.
14324 auto PartialReduction = [&](SDValue Op, unsigned NumSubElts) {
14325 if (!AllowPartials || !Op)
14326 return SDValue();
14327 EVT OpVT = Op.getValueType();
14328 EVT OpSVT = OpVT.getScalarType();
14329 EVT SubVT = EVT::getVectorVT(Context&: *getContext(), VT: OpSVT, NumElements: NumSubElts);
14330 if (TLI->getExtractSubvectorCost(ResVT: SubVT, SrcVT: OpVT, Index: 0) >
14331 TargetLowering::ExtractSubvectorCost::Cheap)
14332 return SDValue();
14333 BinOp = (ISD::NodeType)CandidateBinOp;
14334 return getExtractSubvector(DL: SDLoc(Op), VT: SubVT, Vec: Op, Idx: 0);
14335 };
14336
14337 // At each stage, we're looking for something that looks like:
14338 // %s = shufflevector <8 x i32> %op, <8 x i32> undef,
14339 // <8 x i32> <i32 2, i32 3, i32 undef, i32 undef,
14340 // i32 undef, i32 undef, i32 undef, i32 undef>
14341 // %a = binop <8 x i32> %op, %s
14342 // Where the mask changes according to the stage. E.g. for a 3-stage pyramid,
14343 // we expect something like:
14344 // <4,5,6,7,u,u,u,u>
14345 // <2,3,u,u,u,u,u,u>
14346 // <1,u,u,u,u,u,u,u>
14347 // While a partial reduction match would be:
14348 // <2,3,u,u,u,u,u,u>
14349 // <1,u,u,u,u,u,u,u>
14350 unsigned Stages = Log2_32(Value: Op.getValueType().getVectorNumElements());
14351 SDValue PrevOp;
14352 for (unsigned i = 0; i < Stages; ++i) {
14353 unsigned MaskEnd = (1 << i);
14354
14355 if (Op.getOpcode() != CandidateBinOp)
14356 return PartialReduction(PrevOp, MaskEnd);
14357
14358 SDValue Op0 = Op.getOperand(i: 0);
14359 SDValue Op1 = Op.getOperand(i: 1);
14360
14361 ShuffleVectorSDNode *Shuffle = dyn_cast<ShuffleVectorSDNode>(Val&: Op0);
14362 if (Shuffle) {
14363 Op = Op1;
14364 } else {
14365 Shuffle = dyn_cast<ShuffleVectorSDNode>(Val&: Op1);
14366 Op = Op0;
14367 }
14368
14369 // The first operand of the shuffle should be the same as the other operand
14370 // of the binop.
14371 if (!Shuffle || Shuffle->getOperand(Num: 0) != Op)
14372 return PartialReduction(PrevOp, MaskEnd);
14373
14374 // Verify the shuffle has the expected (at this stage of the pyramid) mask.
14375 for (int Index = 0; Index < (int)MaskEnd; ++Index)
14376 if (Shuffle->getMaskElt(Idx: Index) != (int)(MaskEnd + Index))
14377 return PartialReduction(PrevOp, MaskEnd);
14378
14379 PrevOp = Op;
14380 }
14381
14382 // Handle subvector reductions, which tend to appear after the shuffle
14383 // reduction stages.
14384 while (Op.getOpcode() == CandidateBinOp) {
14385 unsigned NumElts = Op.getValueType().getVectorNumElements();
14386 SDValue Op0 = Op.getOperand(i: 0);
14387 SDValue Op1 = Op.getOperand(i: 1);
14388 if (Op0.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
14389 Op1.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
14390 Op0.getOperand(i: 0) != Op1.getOperand(i: 0))
14391 break;
14392 SDValue Src = Op0.getOperand(i: 0);
14393 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
14394 if (NumSrcElts != (2 * NumElts))
14395 break;
14396 if (!(Op0.getConstantOperandAPInt(i: 1) == 0 &&
14397 Op1.getConstantOperandAPInt(i: 1) == NumElts) &&
14398 !(Op1.getConstantOperandAPInt(i: 1) == 0 &&
14399 Op0.getConstantOperandAPInt(i: 1) == NumElts))
14400 break;
14401 Op = Src;
14402 }
14403
14404 BinOp = (ISD::NodeType)CandidateBinOp;
14405 return Op;
14406}
14407
14408SDValue SelectionDAG::UnrollVectorOp(SDNode *N, unsigned ResNE) {
14409 EVT VT = N->getValueType(ResNo: 0);
14410 EVT EltVT = VT.getVectorElementType();
14411 unsigned NE = getMaxRuntimeNumElements(VT);
14412
14413 if (VT.isScalableVector() && (NE == 0 || ResNE != 0))
14414 reportFatalUsageError(reason: "Cannot unroll scalable vector!");
14415
14416 assert(NE && "Nothing to unroll!");
14417 SDLoc dl(N);
14418
14419 // If ResNE is 0, fully unroll the vector op.
14420 if (ResNE == 0)
14421 ResNE = NE;
14422 else if (NE > ResNE)
14423 NE = ResNE;
14424
14425 if (N->getNumValues() == 2) {
14426 SmallVector<SDValue, 8> Scalars0, Scalars1;
14427 SmallVector<SDValue, 4> Operands(N->getNumOperands());
14428 EVT VT1 = N->getValueType(ResNo: 1);
14429 EVT EltVT1 = VT1.getVectorElementType();
14430
14431 unsigned i;
14432 for (i = 0; i != NE; ++i) {
14433 for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) {
14434 SDValue Operand = N->getOperand(Num: j);
14435 EVT OperandVT = Operand.getValueType();
14436
14437 // A vector operand; extract a single element.
14438 EVT OperandEltVT = OperandVT.getVectorElementType();
14439 Operands[j] = getExtractVectorElt(DL: dl, VT: OperandEltVT, Vec: Operand, Idx: i);
14440 }
14441
14442 SDValue EltOp = getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {EltVT, EltVT1}, Ops: Operands);
14443 Scalars0.push_back(Elt: EltOp);
14444 Scalars1.push_back(Elt: EltOp.getValue(R: 1));
14445 }
14446
14447 for (; i < ResNE; ++i) {
14448 Scalars0.push_back(Elt: getUNDEF(VT: EltVT));
14449 Scalars1.push_back(Elt: getUNDEF(VT: EltVT1));
14450 }
14451
14452 ElementCount ResEC = VT.isScalableVector() ? VT.getVectorElementCount()
14453 : ElementCount::getFixed(MinVal: ResNE);
14454 EVT VecVT = EVT::getVectorVT(Context&: *getContext(), VT: EltVT, EC: ResEC);
14455 EVT VecVT1 = EVT::getVectorVT(Context&: *getContext(), VT: EltVT1, EC: ResEC);
14456 SDValue Vec0 = buildVectorFromUnrolledParts(VT: VecVT, DL: dl, Scalars: Scalars0);
14457 SDValue Vec1 = buildVectorFromUnrolledParts(VT: VecVT1, DL: dl, Scalars: Scalars1);
14458 return getMergeValues(Ops: {Vec0, Vec1}, dl);
14459 }
14460
14461 assert(N->getNumValues() == 1 &&
14462 "Can't unroll a vector with multiple results!");
14463
14464 SmallVector<SDValue, 8> Scalars;
14465 SmallVector<SDValue, 4> Operands(N->getNumOperands());
14466
14467 unsigned i;
14468 for (i= 0; i != NE; ++i) {
14469 for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) {
14470 SDValue Operand = N->getOperand(Num: j);
14471 EVT OperandVT = Operand.getValueType();
14472 if (OperandVT.isVector()) {
14473 // A vector operand; extract a single element.
14474 EVT OperandEltVT = OperandVT.getVectorElementType();
14475 Operands[j] = getExtractVectorElt(DL: dl, VT: OperandEltVT, Vec: Operand, Idx: i);
14476 } else {
14477 // A scalar operand; just use it as is.
14478 Operands[j] = Operand;
14479 }
14480 }
14481
14482 switch (N->getOpcode()) {
14483 default: {
14484 Scalars.push_back(Elt: getNode(Opcode: N->getOpcode(), DL: dl, VT: EltVT, Ops: Operands,
14485 Flags: N->getFlags()));
14486 break;
14487 }
14488 case ISD::VSELECT:
14489 Scalars.push_back(
14490 Elt: getNode(Opcode: ISD::SELECT, DL: dl, VT: EltVT, Ops: Operands, Flags: N->getFlags()));
14491 break;
14492 case ISD::SHL:
14493 case ISD::SRA:
14494 case ISD::SRL:
14495 case ISD::ROTL:
14496 case ISD::ROTR:
14497 Scalars.push_back(Elt: getNode(Opcode: N->getOpcode(), DL: dl, VT: EltVT, N1: Operands[0],
14498 N2: getShiftAmountOperand(LHSTy: Operands[0].getValueType(),
14499 Op: Operands[1])));
14500 break;
14501 case ISD::SIGN_EXTEND_INREG: {
14502 EVT ExtVT = cast<VTSDNode>(Val&: Operands[1])->getVT().getVectorElementType();
14503 Scalars.push_back(Elt: getNode(Opcode: N->getOpcode(), DL: dl, VT: EltVT,
14504 N1: Operands[0],
14505 N2: getValueType(VT: ExtVT)));
14506 break;
14507 }
14508 case ISD::ADDRSPACECAST: {
14509 const auto *ASC = cast<AddrSpaceCastSDNode>(Val: N);
14510 Scalars.push_back(
14511 Elt: getAddrSpaceCast(dl, VT: EltVT, Ptr: Operands[0], SrcAS: ASC->getSrcAddressSpace(),
14512 DestAS: ASC->getDestAddressSpace(), Flags: ASC->getFlags()));
14513 break;
14514 }
14515 }
14516 }
14517
14518 for (; i < ResNE; ++i)
14519 Scalars.push_back(Elt: getUNDEF(VT: EltVT));
14520
14521 EVT VecVT = VT.isScalableVector()
14522 ? VT
14523 : EVT::getVectorVT(Context&: *getContext(), VT: EltVT, NumElements: ResNE);
14524 return buildVectorFromUnrolledParts(VT: VecVT, DL: dl, Scalars);
14525}
14526
14527std::pair<SDValue, SDValue> SelectionDAG::UnrollVectorOverflowOp(
14528 SDNode *N, unsigned ResNE) {
14529 unsigned Opcode = N->getOpcode();
14530 assert((Opcode == ISD::UADDO || Opcode == ISD::SADDO ||
14531 Opcode == ISD::USUBO || Opcode == ISD::SSUBO ||
14532 Opcode == ISD::UMULO || Opcode == ISD::SMULO) &&
14533 "Expected an overflow opcode");
14534
14535 EVT ResVT = N->getValueType(ResNo: 0);
14536 EVT OvVT = N->getValueType(ResNo: 1);
14537 EVT ResEltVT = ResVT.getVectorElementType();
14538 EVT OvEltVT = OvVT.getVectorElementType();
14539 SDLoc dl(N);
14540
14541 // If ResNE is 0, fully unroll the vector op.
14542 unsigned NE = ResVT.getVectorNumElements();
14543 if (ResNE == 0)
14544 ResNE = NE;
14545 else if (NE > ResNE)
14546 NE = ResNE;
14547
14548 SmallVector<SDValue, 8> LHSScalars;
14549 SmallVector<SDValue, 8> RHSScalars;
14550 ExtractVectorElements(Op: N->getOperand(Num: 0), Args&: LHSScalars, Start: 0, Count: NE);
14551 ExtractVectorElements(Op: N->getOperand(Num: 1), Args&: RHSScalars, Start: 0, Count: NE);
14552
14553 EVT SVT = TLI->getSetCCResultType(DL: getDataLayout(), Context&: *getContext(), VT: ResEltVT);
14554 SDVTList VTs = getVTList(VT1: ResEltVT, VT2: SVT);
14555 SmallVector<SDValue, 8> ResScalars;
14556 SmallVector<SDValue, 8> OvScalars;
14557 for (unsigned i = 0; i < NE; ++i) {
14558 SDValue Res = getNode(Opcode, DL: dl, VTList: VTs, N1: LHSScalars[i], N2: RHSScalars[i]);
14559 SDValue Ov =
14560 getSelect(DL: dl, VT: OvEltVT, Cond: Res.getValue(R: 1),
14561 LHS: getBoolConstant(V: true, DL: dl, VT: OvEltVT, OpVT: ResVT),
14562 RHS: getConstant(Val: 0, DL: dl, VT: OvEltVT));
14563
14564 ResScalars.push_back(Elt: Res);
14565 OvScalars.push_back(Elt: Ov);
14566 }
14567
14568 ResScalars.append(NumInputs: ResNE - NE, Elt: getUNDEF(VT: ResEltVT));
14569 OvScalars.append(NumInputs: ResNE - NE, Elt: getUNDEF(VT: OvEltVT));
14570
14571 EVT NewResVT = EVT::getVectorVT(Context&: *getContext(), VT: ResEltVT, NumElements: ResNE);
14572 EVT NewOvVT = EVT::getVectorVT(Context&: *getContext(), VT: OvEltVT, NumElements: ResNE);
14573 return std::make_pair(x: getBuildVector(VT: NewResVT, DL: dl, Ops: ResScalars),
14574 y: getBuildVector(VT: NewOvVT, DL: dl, Ops: OvScalars));
14575}
14576
14577static bool areNonVolatileConsecutiveLoadsOrStores(LSBaseSDNode *LS,
14578 LSBaseSDNode *Base,
14579 unsigned Bytes, int Dist,
14580 const SelectionDAG &DAG) {
14581 if (LS->isVolatile() || Base->isVolatile())
14582 return false;
14583 // TODO: probably too restrictive for atomics, revisit
14584 if (!LS->isSimple())
14585 return false;
14586 if (LS->isIndexed() || Base->isIndexed())
14587 return false;
14588 if (LS->getChain() != Base->getChain())
14589 return false;
14590 EVT VT = LS->getMemoryVT();
14591 if (VT.getSizeInBits() / 8 != Bytes)
14592 return false;
14593
14594 auto BaseLocDecomp = BaseIndexOffset::match(N: Base, DAG);
14595 auto LocDecomp = BaseIndexOffset::match(N: LS, DAG);
14596
14597 int64_t Offset = 0;
14598 if (BaseLocDecomp.equalBaseIndex(Other: LocDecomp, DAG, Off&: Offset))
14599 return (Dist * (int64_t)Bytes == Offset);
14600 return false;
14601}
14602
14603bool SelectionDAG::areNonVolatileConsecutiveLoads(LoadSDNode *LD,
14604 LoadSDNode *Base,
14605 unsigned Bytes,
14606 int Dist) const {
14607 return areNonVolatileConsecutiveLoadsOrStores(LS: LD, Base, Bytes, Dist, DAG: *this);
14608}
14609
14610bool SelectionDAG::areNonVolatileConsecutiveStores(StoreSDNode *ST,
14611 StoreSDNode *Base,
14612 unsigned Bytes,
14613 int Dist) const {
14614 return areNonVolatileConsecutiveLoadsOrStores(LS: ST, Base, Bytes, Dist, DAG: *this);
14615}
14616
14617/// InferPtrAlignment - Infer alignment of a load / store address. Return
14618/// std::nullopt if it cannot be inferred.
14619MaybeAlign SelectionDAG::InferPtrAlign(SDValue Ptr) const {
14620 // If this is a GlobalAddress + cst, return the alignment.
14621 const GlobalValue *GV = nullptr;
14622 int64_t GVOffset = 0;
14623 if (TLI->isGAPlusOffset(N: Ptr.getNode(), GA&: GV, Offset&: GVOffset)) {
14624 unsigned PtrWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType());
14625 KnownBits Known(PtrWidth);
14626 llvm::computeKnownBits(V: GV, Known, DL: getDataLayout());
14627 unsigned AlignBits = Known.countMinTrailingZeros();
14628 if (AlignBits)
14629 return commonAlignment(A: Align::fromLog2(Log2Value: std::min(a: 31U, b: AlignBits)),
14630 Offset: GVOffset);
14631 }
14632
14633 // If this is a direct reference to a stack slot, use information about the
14634 // stack slot's alignment.
14635 int FrameIdx = INT_MIN;
14636 int64_t FrameOffset = 0;
14637 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Ptr)) {
14638 FrameIdx = FI->getIndex();
14639 } else if (isBaseWithConstantOffset(Op: Ptr) &&
14640 isa<FrameIndexSDNode>(Val: Ptr.getOperand(i: 0))) {
14641 // Handle FI+Cst
14642 FrameIdx = cast<FrameIndexSDNode>(Val: Ptr.getOperand(i: 0))->getIndex();
14643 FrameOffset = Ptr.getConstantOperandVal(i: 1);
14644 }
14645
14646 if (FrameIdx != INT_MIN) {
14647 const MachineFrameInfo &MFI = getMachineFunction().getFrameInfo();
14648 return commonAlignment(A: MFI.getObjectAlign(ObjectIdx: FrameIdx), Offset: FrameOffset);
14649 }
14650
14651 return std::nullopt;
14652}
14653
14654/// Split the scalar node with EXTRACT_ELEMENT using the provided
14655/// VTs and return the low/high part.
14656std::pair<SDValue, SDValue> SelectionDAG::SplitScalar(const SDValue &N,
14657 const SDLoc &DL,
14658 const EVT &LoVT,
14659 const EVT &HiVT) {
14660 assert(!LoVT.isVector() && !HiVT.isVector() && !N.getValueType().isVector() &&
14661 "Split node must be a scalar type");
14662 SDValue Lo =
14663 getNode(Opcode: ISD::EXTRACT_ELEMENT, DL, VT: LoVT, N1: N, N2: getIntPtrConstant(Val: 0, DL));
14664 SDValue Hi =
14665 getNode(Opcode: ISD::EXTRACT_ELEMENT, DL, VT: HiVT, N1: N, N2: getIntPtrConstant(Val: 1, DL));
14666 return std::make_pair(x&: Lo, y&: Hi);
14667}
14668
14669/// GetSplitDestVTs - Compute the VTs needed for the low/hi parts of a type
14670/// which is split (or expanded) into two not necessarily identical pieces.
14671std::pair<EVT, EVT> SelectionDAG::GetSplitDestVTs(const EVT &VT) const {
14672 // Currently all types are split in half.
14673 EVT LoVT, HiVT;
14674 if (!VT.isVector())
14675 LoVT = HiVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT);
14676 else
14677 LoVT = HiVT = VT.getHalfNumVectorElementsVT(Context&: *getContext());
14678
14679 return std::make_pair(x&: LoVT, y&: HiVT);
14680}
14681
14682/// GetDependentSplitDestVTs - Compute the VTs needed for the low/hi parts of a
14683/// type, dependent on an enveloping VT that has been split into two identical
14684/// pieces. Sets the HiIsEmpty flag when hi type has zero storage size.
14685std::pair<EVT, EVT>
14686SelectionDAG::GetDependentSplitDestVTs(const EVT &VT, const EVT &EnvVT,
14687 bool *HiIsEmpty) const {
14688 EVT EltTp = VT.getVectorElementType();
14689 // Examples:
14690 // custom VL=8 with enveloping VL=8/8 yields 8/0 (hi empty)
14691 // custom VL=9 with enveloping VL=8/8 yields 8/1
14692 // custom VL=10 with enveloping VL=8/8 yields 8/2
14693 // etc.
14694 ElementCount VTNumElts = VT.getVectorElementCount();
14695 ElementCount EnvNumElts = EnvVT.getVectorElementCount();
14696 assert(VTNumElts.isScalable() == EnvNumElts.isScalable() &&
14697 "Mixing fixed width and scalable vectors when enveloping a type");
14698 EVT LoVT, HiVT;
14699 if (VTNumElts.getKnownMinValue() > EnvNumElts.getKnownMinValue()) {
14700 LoVT = EVT::getVectorVT(Context&: *getContext(), VT: EltTp, EC: EnvNumElts);
14701 HiVT = EVT::getVectorVT(Context&: *getContext(), VT: EltTp, EC: VTNumElts - EnvNumElts);
14702 *HiIsEmpty = false;
14703 } else {
14704 // Flag that hi type has zero storage size, but return split envelop type
14705 // (this would be easier if vector types with zero elements were allowed).
14706 LoVT = EVT::getVectorVT(Context&: *getContext(), VT: EltTp, EC: VTNumElts);
14707 HiVT = EVT::getVectorVT(Context&: *getContext(), VT: EltTp, EC: EnvNumElts);
14708 *HiIsEmpty = true;
14709 }
14710 return std::make_pair(x&: LoVT, y&: HiVT);
14711}
14712
14713/// SplitVector - Split the vector with EXTRACT_SUBVECTOR and return the
14714/// low/high part.
14715std::pair<SDValue, SDValue>
14716SelectionDAG::SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT,
14717 const EVT &HiVT) {
14718 assert(LoVT.isScalableVector() == HiVT.isScalableVector() &&
14719 LoVT.isScalableVector() == N.getValueType().isScalableVector() &&
14720 "Splitting vector with an invalid mixture of fixed and scalable "
14721 "vector types");
14722 assert(LoVT.getVectorMinNumElements() + HiVT.getVectorMinNumElements() <=
14723 N.getValueType().getVectorMinNumElements() &&
14724 "More vector elements requested than available!");
14725 SDValue Lo, Hi;
14726 Lo = getExtractSubvector(DL, VT: LoVT, Vec: N, Idx: 0);
14727 // For scalable vectors it is safe to use LoVT.getVectorMinNumElements()
14728 // (rather than having to use ElementCount), because EXTRACT_SUBVECTOR scales
14729 // IDX with the runtime scaling factor of the result vector type. For
14730 // fixed-width result vectors, that runtime scaling factor is 1.
14731 Hi = getExtractSubvector(DL, VT: HiVT, Vec: N, Idx: LoVT.getVectorMinNumElements());
14732 return std::make_pair(x&: Lo, y&: Hi);
14733}
14734
14735std::pair<SDValue, SDValue> SelectionDAG::SplitEVL(SDValue N, EVT VecVT,
14736 const SDLoc &DL) {
14737 // Split the vector length parameter.
14738 // %evl -> umin(%evl, %halfnumelts) and usubsat(%evl - %halfnumelts).
14739 EVT VT = N.getValueType();
14740 assert(VecVT.getVectorElementCount().isKnownEven() &&
14741 "Expecting the mask to be an evenly-sized vector");
14742 SDValue HalfNumElts = getElementCount(
14743 DL, VT, EC: VecVT.getVectorElementCount().divideCoefficientBy(RHS: 2));
14744 SDValue Lo = getNode(Opcode: ISD::UMIN, DL, VT, N1: N, N2: HalfNumElts);
14745 SDValue Hi = getNode(Opcode: ISD::USUBSAT, DL, VT, N1: N, N2: HalfNumElts);
14746 return std::make_pair(x&: Lo, y&: Hi);
14747}
14748
14749/// Widen the vector up to the next power of two using INSERT_SUBVECTOR.
14750SDValue SelectionDAG::WidenVector(const SDValue &N, const SDLoc &DL) {
14751 EVT VT = N.getValueType();
14752 EVT WideVT = EVT::getVectorVT(Context&: *getContext(), VT: VT.getVectorElementType(),
14753 NumElements: NextPowerOf2(A: VT.getVectorNumElements()));
14754 return getInsertSubvector(DL, Vec: getPOISON(VT: WideVT), SubVec: N, Idx: 0);
14755}
14756
14757void SelectionDAG::ExtractVectorElements(SDValue Op,
14758 SmallVectorImpl<SDValue> &Args,
14759 unsigned Start, unsigned Count,
14760 EVT EltVT) {
14761 EVT VT = Op.getValueType();
14762 if (Count == 0)
14763 Count = VT.getVectorNumElements();
14764 if (EltVT == EVT())
14765 EltVT = VT.getVectorElementType();
14766 SDLoc SL(Op);
14767 for (unsigned i = Start, e = Start + Count; i != e; ++i) {
14768 Args.push_back(Elt: getExtractVectorElt(DL: SL, VT: EltVT, Vec: Op, Idx: i));
14769 }
14770}
14771
14772// getAddressSpace - Return the address space this GlobalAddress belongs to.
14773unsigned GlobalAddressSDNode::getAddressSpace() const {
14774 return getGlobal()->getType()->getAddressSpace();
14775}
14776
14777Type *ConstantPoolSDNode::getType() const {
14778 if (isMachineConstantPoolEntry())
14779 return Val.MachineCPVal->getType();
14780 return Val.ConstVal->getType();
14781}
14782
14783bool BuildVectorSDNode::isConstantSplat(APInt &SplatValue, APInt &SplatUndef,
14784 unsigned &SplatBitSize,
14785 bool &HasAnyUndefs,
14786 unsigned MinSplatBits,
14787 bool IsBigEndian) const {
14788 EVT VT = getValueType(ResNo: 0);
14789 assert(VT.isVector() && "Expected a vector type");
14790 unsigned VecWidth = VT.getSizeInBits();
14791 if (MinSplatBits > VecWidth)
14792 return false;
14793
14794 // FIXME: The widths are based on this node's type, but build vectors can
14795 // truncate their operands.
14796 SplatValue = APInt(VecWidth, 0);
14797 SplatUndef = APInt(VecWidth, 0);
14798
14799 // Get the bits. Bits with undefined values (when the corresponding element
14800 // of the vector is an ISD::UNDEF value) are set in SplatUndef and cleared
14801 // in SplatValue. If any of the values are not constant, give up and return
14802 // false.
14803 unsigned int NumOps = getNumOperands();
14804 assert(NumOps > 0 && "isConstantSplat has 0-size build vector");
14805 unsigned EltWidth = VT.getScalarSizeInBits();
14806
14807 for (unsigned j = 0; j < NumOps; ++j) {
14808 unsigned i = IsBigEndian ? NumOps - 1 - j : j;
14809 SDValue OpVal = getOperand(Num: i);
14810 unsigned BitPos = j * EltWidth;
14811
14812 if (OpVal.isUndef())
14813 SplatUndef.setBits(loBit: BitPos, hiBit: BitPos + EltWidth);
14814 else if (auto *CN = dyn_cast<ConstantSDNode>(Val&: OpVal))
14815 SplatValue.insertBits(SubBits: CN->getAPIntValue().zextOrTrunc(width: EltWidth), bitPosition: BitPos);
14816 else if (auto *CN = dyn_cast<ConstantFPSDNode>(Val&: OpVal))
14817 SplatValue.insertBits(SubBits: CN->getValueAPF().bitcastToAPInt(), bitPosition: BitPos);
14818 else
14819 return false;
14820 }
14821
14822 // The build_vector is all constants or undefs. Find the smallest element
14823 // size that splats the vector.
14824 HasAnyUndefs = (SplatUndef != 0);
14825
14826 // FIXME: This does not work for vectors with elements less than 8 bits.
14827 while (VecWidth > 8) {
14828 // If we can't split in half, stop here.
14829 if (VecWidth & 1)
14830 break;
14831
14832 unsigned HalfSize = VecWidth / 2;
14833 APInt HighValue = SplatValue.extractBits(numBits: HalfSize, bitPosition: HalfSize);
14834 APInt LowValue = SplatValue.extractBits(numBits: HalfSize, bitPosition: 0);
14835 APInt HighUndef = SplatUndef.extractBits(numBits: HalfSize, bitPosition: HalfSize);
14836 APInt LowUndef = SplatUndef.extractBits(numBits: HalfSize, bitPosition: 0);
14837
14838 // If the two halves do not match (ignoring undef bits), stop here.
14839 if ((HighValue & ~LowUndef) != (LowValue & ~HighUndef) ||
14840 MinSplatBits > HalfSize)
14841 break;
14842
14843 SplatValue = HighValue | LowValue;
14844 SplatUndef = HighUndef & LowUndef;
14845
14846 VecWidth = HalfSize;
14847 }
14848
14849 // FIXME: The loop above only tries to split in halves. But if the input
14850 // vector for example is <3 x i16> it wouldn't be able to detect a
14851 // SplatBitSize of 16. No idea if that is a design flaw currently limiting
14852 // optimizations. I guess that back in the days when this helper was created
14853 // vectors normally was power-of-2 sized.
14854
14855 SplatBitSize = VecWidth;
14856 return true;
14857}
14858
14859SDValue BuildVectorSDNode::getSplatValue(const APInt &DemandedElts,
14860 BitVector *UndefElements) const {
14861 unsigned NumOps = getNumOperands();
14862 if (UndefElements) {
14863 UndefElements->clear();
14864 UndefElements->resize(N: NumOps);
14865 }
14866 assert(NumOps == DemandedElts.getBitWidth() && "Unexpected vector size");
14867 if (!DemandedElts)
14868 return SDValue();
14869 SDValue Splatted;
14870 for (unsigned i = 0; i != NumOps; ++i) {
14871 if (!DemandedElts[i])
14872 continue;
14873 SDValue Op = getOperand(Num: i);
14874 if (Op.isUndef()) {
14875 if (UndefElements)
14876 (*UndefElements)[i] = true;
14877 } else if (!Splatted) {
14878 Splatted = Op;
14879 } else if (Splatted != Op) {
14880 return SDValue();
14881 }
14882 }
14883
14884 if (!Splatted) {
14885 unsigned FirstDemandedIdx = DemandedElts.countr_zero();
14886 assert(getOperand(FirstDemandedIdx).isUndef() &&
14887 "Can only have a splat without a constant for all undefs.");
14888 return getOperand(Num: FirstDemandedIdx);
14889 }
14890
14891 return Splatted;
14892}
14893
14894SDValue BuildVectorSDNode::getSplatValue(BitVector *UndefElements) const {
14895 APInt DemandedElts = APInt::getAllOnes(numBits: getNumOperands());
14896 return getSplatValue(DemandedElts, UndefElements);
14897}
14898
14899bool BuildVectorSDNode::getRepeatedSequence(const APInt &DemandedElts,
14900 SmallVectorImpl<SDValue> &Sequence,
14901 BitVector *UndefElements) const {
14902 unsigned NumOps = getNumOperands();
14903 Sequence.clear();
14904 if (UndefElements) {
14905 UndefElements->clear();
14906 UndefElements->resize(N: NumOps);
14907 }
14908 assert(NumOps == DemandedElts.getBitWidth() && "Unexpected vector size");
14909 if (!DemandedElts || NumOps < 2 || !isPowerOf2_32(Value: NumOps))
14910 return false;
14911
14912 // Set the undefs even if we don't find a sequence (like getSplatValue).
14913 if (UndefElements)
14914 for (unsigned I = 0; I != NumOps; ++I)
14915 if (DemandedElts[I] && getOperand(Num: I).isUndef())
14916 (*UndefElements)[I] = true;
14917
14918 // Iteratively widen the sequence length looking for repetitions.
14919 for (unsigned SeqLen = 1; SeqLen < NumOps; SeqLen *= 2) {
14920 Sequence.append(NumInputs: SeqLen, Elt: SDValue());
14921 for (unsigned I = 0; I != NumOps; ++I) {
14922 if (!DemandedElts[I])
14923 continue;
14924 SDValue &SeqOp = Sequence[I % SeqLen];
14925 SDValue Op = getOperand(Num: I);
14926 if (Op.isUndef()) {
14927 if (!SeqOp)
14928 SeqOp = Op;
14929 continue;
14930 }
14931 if (SeqOp && !SeqOp.isUndef() && SeqOp != Op) {
14932 Sequence.clear();
14933 break;
14934 }
14935 SeqOp = Op;
14936 }
14937 if (!Sequence.empty())
14938 return true;
14939 }
14940
14941 assert(Sequence.empty() && "Failed to empty non-repeating sequence pattern");
14942 return false;
14943}
14944
14945bool BuildVectorSDNode::getRepeatedSequence(SmallVectorImpl<SDValue> &Sequence,
14946 BitVector *UndefElements) const {
14947 APInt DemandedElts = APInt::getAllOnes(numBits: getNumOperands());
14948 return getRepeatedSequence(DemandedElts, Sequence, UndefElements);
14949}
14950
14951ConstantSDNode *
14952BuildVectorSDNode::getConstantSplatNode(const APInt &DemandedElts,
14953 BitVector *UndefElements) const {
14954 return dyn_cast_or_null<ConstantSDNode>(
14955 Val: getSplatValue(DemandedElts, UndefElements));
14956}
14957
14958ConstantSDNode *
14959BuildVectorSDNode::getConstantSplatNode(BitVector *UndefElements) const {
14960 return dyn_cast_or_null<ConstantSDNode>(Val: getSplatValue(UndefElements));
14961}
14962
14963ConstantFPSDNode *
14964BuildVectorSDNode::getConstantFPSplatNode(const APInt &DemandedElts,
14965 BitVector *UndefElements) const {
14966 return dyn_cast_or_null<ConstantFPSDNode>(
14967 Val: getSplatValue(DemandedElts, UndefElements));
14968}
14969
14970ConstantFPSDNode *
14971BuildVectorSDNode::getConstantFPSplatNode(BitVector *UndefElements) const {
14972 return dyn_cast_or_null<ConstantFPSDNode>(Val: getSplatValue(UndefElements));
14973}
14974
14975int32_t
14976BuildVectorSDNode::getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements,
14977 uint32_t BitWidth) const {
14978 if (ConstantFPSDNode *CN =
14979 dyn_cast_or_null<ConstantFPSDNode>(Val: getSplatValue(UndefElements))) {
14980 bool IsExact;
14981 APSInt IntVal(BitWidth);
14982 const APFloat &APF = CN->getValueAPF();
14983 if (APF.convertToInteger(Result&: IntVal, RM: APFloat::rmTowardZero, IsExact: &IsExact) !=
14984 APFloat::opOK ||
14985 !IsExact)
14986 return -1;
14987
14988 return IntVal.exactLogBase2();
14989 }
14990 return -1;
14991}
14992
14993bool BuildVectorSDNode::getConstantRawBits(
14994 bool IsLittleEndian, unsigned DstEltSizeInBits,
14995 SmallVectorImpl<APInt> &RawBitElements, BitVector &UndefElements) const {
14996 // Early-out if this contains anything but Undef/Constant/ConstantFP.
14997 if (!isConstant())
14998 return false;
14999
15000 unsigned NumSrcOps = getNumOperands();
15001 unsigned SrcEltSizeInBits = getValueType(ResNo: 0).getScalarSizeInBits();
15002 assert(((NumSrcOps * SrcEltSizeInBits) % DstEltSizeInBits) == 0 &&
15003 "Invalid bitcast scale");
15004
15005 // Extract raw src bits.
15006 SmallVector<APInt> SrcBitElements(NumSrcOps,
15007 APInt::getZero(numBits: SrcEltSizeInBits));
15008 BitVector SrcUndeElements(NumSrcOps, false);
15009
15010 for (unsigned I = 0; I != NumSrcOps; ++I) {
15011 SDValue Op = getOperand(Num: I);
15012 if (Op.isUndef()) {
15013 SrcUndeElements.set(I);
15014 continue;
15015 }
15016 auto *CInt = dyn_cast<ConstantSDNode>(Val&: Op);
15017 auto *CFP = dyn_cast<ConstantFPSDNode>(Val&: Op);
15018 assert((CInt || CFP) && "Unknown constant");
15019 SrcBitElements[I] = CInt ? CInt->getAPIntValue().trunc(width: SrcEltSizeInBits)
15020 : CFP->getValueAPF().bitcastToAPInt();
15021 }
15022
15023 // Recast to dst width.
15024 recastRawBits(IsLittleEndian, DstEltSizeInBits, DstBitElements&: RawBitElements,
15025 SrcBitElements, DstUndefElements&: UndefElements, SrcUndefElements: SrcUndeElements);
15026 return true;
15027}
15028
15029void BuildVectorSDNode::recastRawBits(bool IsLittleEndian,
15030 unsigned DstEltSizeInBits,
15031 SmallVectorImpl<APInt> &DstBitElements,
15032 ArrayRef<APInt> SrcBitElements,
15033 BitVector &DstUndefElements,
15034 const BitVector &SrcUndefElements) {
15035 unsigned NumSrcOps = SrcBitElements.size();
15036 unsigned SrcEltSizeInBits = SrcBitElements[0].getBitWidth();
15037 assert(((NumSrcOps * SrcEltSizeInBits) % DstEltSizeInBits) == 0 &&
15038 "Invalid bitcast scale");
15039 assert(NumSrcOps == SrcUndefElements.size() &&
15040 "Vector size mismatch");
15041
15042 unsigned NumDstOps = (NumSrcOps * SrcEltSizeInBits) / DstEltSizeInBits;
15043 DstUndefElements.clear();
15044 DstUndefElements.resize(N: NumDstOps, t: false);
15045 DstBitElements.assign(NumElts: NumDstOps, Elt: APInt::getZero(numBits: DstEltSizeInBits));
15046
15047 // Concatenate src elements constant bits together into dst element.
15048 if (SrcEltSizeInBits <= DstEltSizeInBits) {
15049 unsigned Scale = DstEltSizeInBits / SrcEltSizeInBits;
15050 for (unsigned I = 0; I != NumDstOps; ++I) {
15051 DstUndefElements.set(I);
15052 APInt &DstBits = DstBitElements[I];
15053 for (unsigned J = 0; J != Scale; ++J) {
15054 unsigned Idx = (I * Scale) + (IsLittleEndian ? J : (Scale - J - 1));
15055 if (SrcUndefElements[Idx])
15056 continue;
15057 DstUndefElements.reset(Idx: I);
15058 const APInt &SrcBits = SrcBitElements[Idx];
15059 assert(SrcBits.getBitWidth() == SrcEltSizeInBits &&
15060 "Illegal constant bitwidths");
15061 DstBits.insertBits(SubBits: SrcBits, bitPosition: J * SrcEltSizeInBits);
15062 }
15063 }
15064 return;
15065 }
15066
15067 // Split src element constant bits into dst elements.
15068 unsigned Scale = SrcEltSizeInBits / DstEltSizeInBits;
15069 for (unsigned I = 0; I != NumSrcOps; ++I) {
15070 if (SrcUndefElements[I]) {
15071 DstUndefElements.set(I: I * Scale, E: (I + 1) * Scale);
15072 continue;
15073 }
15074 const APInt &SrcBits = SrcBitElements[I];
15075 for (unsigned J = 0; J != Scale; ++J) {
15076 unsigned Idx = (I * Scale) + (IsLittleEndian ? J : (Scale - J - 1));
15077 APInt &DstBits = DstBitElements[Idx];
15078 DstBits = SrcBits.extractBits(numBits: DstEltSizeInBits, bitPosition: J * DstEltSizeInBits);
15079 }
15080 }
15081}
15082
15083bool BuildVectorSDNode::isConstant() const {
15084 for (const SDValue &Op : op_values()) {
15085 unsigned Opc = Op.getOpcode();
15086 if (!Op.isUndef() && Opc != ISD::Constant && Opc != ISD::ConstantFP)
15087 return false;
15088 }
15089 return true;
15090}
15091
15092std::optional<std::pair<APInt, APInt>>
15093BuildVectorSDNode::isArithmeticSequence() const {
15094 unsigned NumOps = getNumOperands();
15095 if (NumOps < 2)
15096 return std::nullopt;
15097
15098 unsigned EltSize = getValueType(ResNo: 0).getScalarSizeInBits();
15099 APInt Start, Stride;
15100 int FirstIdx = -1, SecondIdx = -1;
15101
15102 // Find the first two non-undef constant elements to determine Start and
15103 // Stride, then verify all remaining elements match the sequence.
15104 for (unsigned I = 0; I < NumOps; ++I) {
15105 SDValue Op = getOperand(Num: I);
15106 if (Op->isUndef())
15107 continue;
15108 if (!isa<ConstantSDNode>(Val: Op))
15109 return std::nullopt;
15110
15111 APInt Val = getConstantOperandAPInt(Num: I).trunc(width: EltSize);
15112 if (FirstIdx < 0) {
15113 FirstIdx = I;
15114 Start = Val;
15115 } else if (SecondIdx < 0) {
15116 SecondIdx = I;
15117 // Compute stride using modular arithmetic. Simple division would handle
15118 // common strides (1, 2, -1, etc.), but modular inverse maximizes matches.
15119 // Example: <0, poison, poison, 0xFF> has stride 0x55 since 3*0x55 = 0xFF
15120 // Note that modular arithmetic is agnostic to signed/unsigned.
15121 unsigned IdxDiff = I - FirstIdx;
15122 APInt ValDiff = Val - Start;
15123
15124 // Step 1: Factor out common powers of 2 from IdxDiff and ValDiff.
15125 unsigned CommonPow2Bits = llvm::countr_zero(Val: IdxDiff);
15126 if (ValDiff.countr_zero() < CommonPow2Bits)
15127 return std::nullopt; // ValDiff not divisible by 2^CommonPow2Bits
15128 IdxDiff >>= CommonPow2Bits;
15129 ValDiff.lshrInPlace(ShiftAmt: CommonPow2Bits);
15130
15131 // Step 2: IdxDiff is now odd, so its inverse mod 2^EltSize exists.
15132 // TODO: There are 2^CommonPow2Bits valid strides; currently we only try
15133 // one, but we could try all candidates to handle more cases.
15134 Stride = ValDiff * APInt(EltSize, IdxDiff).multiplicativeInverse();
15135 if (Stride.isZero())
15136 return std::nullopt;
15137
15138 // Step 3: Adjust Start based on the first defined element's index.
15139 Start -= Stride * FirstIdx;
15140 } else {
15141 // Verify this element matches the sequence.
15142 if (Val != Start + Stride * I)
15143 return std::nullopt;
15144 }
15145 }
15146
15147 // Need at least two defined elements.
15148 if (SecondIdx < 0)
15149 return std::nullopt;
15150
15151 return std::make_pair(x&: Start, y&: Stride);
15152}
15153
15154bool ShuffleVectorSDNode::isSplatMask(ArrayRef<int> Mask) {
15155 // Find the first non-undef value in the shuffle mask.
15156 unsigned i, e;
15157 for (i = 0, e = Mask.size(); i != e && Mask[i] < 0; ++i)
15158 /* search */;
15159
15160 // If all elements are undefined, this shuffle can be considered a splat
15161 // (although it should eventually get simplified away completely).
15162 if (i == e)
15163 return true;
15164
15165 // Make sure all remaining elements are either undef or the same as the first
15166 // non-undef value.
15167 for (int Idx = Mask[i]; i != e; ++i)
15168 if (Mask[i] >= 0 && Mask[i] != Idx)
15169 return false;
15170 return true;
15171}
15172
15173// Returns true if it is a constant integer BuildVector or constant integer,
15174// possibly hidden by a bitcast.
15175bool SelectionDAG::isConstantIntBuildVectorOrConstantInt(
15176 SDValue N, bool AllowOpaques) const {
15177 N = peekThroughBitcasts(V: N);
15178
15179 if (auto *C = dyn_cast<ConstantSDNode>(Val&: N))
15180 return AllowOpaques || !C->isOpaque();
15181
15182 if (ISD::isBuildVectorOfConstantSDNodes(N: N.getNode()))
15183 return true;
15184
15185 // Treat a GlobalAddress supporting constant offset folding as a
15186 // constant integer.
15187 if (auto *GA = dyn_cast<GlobalAddressSDNode>(Val&: N))
15188 if (GA->getOpcode() == ISD::GlobalAddress &&
15189 TLI->isOffsetFoldingLegal(GA))
15190 return true;
15191
15192 if ((N.getOpcode() == ISD::SPLAT_VECTOR) &&
15193 isa<ConstantSDNode>(Val: N.getOperand(i: 0)))
15194 return true;
15195 return false;
15196}
15197
15198// Returns true if it is a constant float BuildVector or constant float.
15199bool SelectionDAG::isConstantFPBuildVectorOrConstantFP(SDValue N) const {
15200 if (isa<ConstantFPSDNode>(Val: N))
15201 return true;
15202
15203 if (ISD::isBuildVectorOfConstantFPSDNodes(N: N.getNode()))
15204 return true;
15205
15206 if ((N.getOpcode() == ISD::SPLAT_VECTOR) &&
15207 isa<ConstantFPSDNode>(Val: N.getOperand(i: 0)))
15208 return true;
15209
15210 return false;
15211}
15212
15213std::optional<bool> SelectionDAG::isBoolConstant(SDValue N) const {
15214 ConstantSDNode *Const =
15215 isConstOrConstSplat(N, AllowUndefs: false, /*AllowTruncation=*/true);
15216 if (!Const)
15217 return std::nullopt;
15218
15219 EVT VT = N.getValueType();
15220 const APInt CVal = Const->getAPIntValue().trunc(width: VT.getScalarSizeInBits());
15221 switch (TLI->getBooleanContents(Type: VT)) {
15222 case TargetLowering::ZeroOrOneBooleanContent:
15223 if (CVal.isOne())
15224 return true;
15225 if (CVal.isZero())
15226 return false;
15227 return std::nullopt;
15228 case TargetLowering::ZeroOrNegativeOneBooleanContent:
15229 if (CVal.isAllOnes())
15230 return true;
15231 if (CVal.isZero())
15232 return false;
15233 return std::nullopt;
15234 case TargetLowering::UndefinedBooleanContent:
15235 return CVal[0];
15236 }
15237 llvm_unreachable("Unknown BooleanContent enum");
15238}
15239
15240void SelectionDAG::createOperands(SDNode *Node, ArrayRef<SDValue> Vals) {
15241 assert(!Node->OperandList && "Node already has operands");
15242 assert(SDNode::getMaxNumOperands() >= Vals.size() &&
15243 "too many operands to fit into SDNode");
15244 SDUse *Ops = OperandRecycler.allocate(
15245 Cap: ArrayRecycler<SDUse>::Capacity::get(N: Vals.size()), Allocator&: OperandAllocator);
15246
15247 bool IsDivergent = false;
15248 for (unsigned I = 0; I != Vals.size(); ++I) {
15249 Ops[I].setUser(Node);
15250 Ops[I].setInitial(Vals[I]);
15251 EVT VT = Ops[I].getValueType();
15252
15253 // Skip Chain. It does not carry divergence.
15254 if (VT != MVT::Other &&
15255 (VT != MVT::Glue || gluePropagatesDivergence(Node: Ops[I].getNode())) &&
15256 Ops[I].getNode()->isDivergent()) {
15257 IsDivergent = true;
15258 }
15259 }
15260 Node->NumOperands = Vals.size();
15261 Node->OperandList = Ops;
15262 if (!TLI->isSDNodeAlwaysUniform(N: Node)) {
15263 IsDivergent |= TLI->isSDNodeSourceOfDivergence(N: Node, FLI, UA);
15264 Node->SDNodeBits.IsDivergent = IsDivergent;
15265 }
15266 checkForCycles(N: Node);
15267}
15268
15269SDValue SelectionDAG::getTokenFactor(const SDLoc &DL,
15270 SmallVectorImpl<SDValue> &Vals) {
15271 size_t Limit = SDNode::getMaxNumOperands();
15272 while (Vals.size() > Limit) {
15273 unsigned SliceIdx = Vals.size() - Limit;
15274 auto ExtractedTFs = ArrayRef<SDValue>(Vals).slice(N: SliceIdx, M: Limit);
15275 SDValue NewTF = getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: ExtractedTFs);
15276 Vals.erase(CS: Vals.begin() + SliceIdx, CE: Vals.end());
15277 Vals.emplace_back(Args&: NewTF);
15278 }
15279 return getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: Vals);
15280}
15281
15282SDValue SelectionDAG::getIdentityElement(unsigned Opcode, const SDLoc &DL,
15283 EVT VT, SDNodeFlags Flags) {
15284 switch (Opcode) {
15285 default:
15286 return SDValue();
15287 case ISD::ADD:
15288 case ISD::OR:
15289 case ISD::XOR:
15290 case ISD::UMAX:
15291 case ISD::MUL:
15292 case ISD::AND:
15293 case ISD::UMIN:
15294 case ISD::SMAX:
15295 case ISD::SMIN:
15296 return getConstant(Val: getIntegerIdentity(Opcode, BitWidth: VT.getScalarSizeInBits()), DL,
15297 VT);
15298 case ISD::FADD:
15299 // If flags allow, prefer positive zero since it's generally cheaper
15300 // to materialize on most targets.
15301 return getConstantFP(Val: Flags.hasNoSignedZeros() ? 0.0 : -0.0, DL, VT);
15302 case ISD::FMUL:
15303 return getConstantFP(Val: 1.0, DL, VT);
15304 case ISD::FMINNUM:
15305 case ISD::FMAXNUM:
15306 case ISD::FMINIMUMNUM:
15307 case ISD::FMAXIMUMNUM: {
15308 // Neutral element for fminnum/fminimumnum is NaN, Inf or FLT_MAX,
15309 // depending on fast-math flags (FMF).
15310 const fltSemantics &Semantics = VT.getFltSemantics();
15311 APFloat NeutralAF = !Flags.hasNoNaNs() ? APFloat::getQNaN(Sem: Semantics) :
15312 !Flags.hasNoInfs() ? APFloat::getInf(Sem: Semantics) :
15313 APFloat::getLargest(Sem: Semantics);
15314 if (Opcode == ISD::FMAXNUM || Opcode == ISD::FMAXIMUMNUM)
15315 NeutralAF.changeSign();
15316
15317 return getConstantFP(V: NeutralAF, DL, VT);
15318 }
15319 case ISD::FMINIMUM:
15320 case ISD::FMAXIMUM: {
15321 // Neutral element for fminimum is Inf or FLT_MAX, depending on FMF.
15322 const fltSemantics &Semantics = VT.getFltSemantics();
15323 APFloat NeutralAF = !Flags.hasNoInfs() ? APFloat::getInf(Sem: Semantics)
15324 : APFloat::getLargest(Sem: Semantics);
15325 if (Opcode == ISD::FMAXIMUM)
15326 NeutralAF.changeSign();
15327
15328 return getConstantFP(V: NeutralAF, DL, VT);
15329 }
15330
15331 }
15332}
15333
15334SDValue SelectionDAG::getPartialReduceMLS(unsigned Opc, const SDLoc &DL,
15335 SDValue Acc, SDValue LHS,
15336 SDValue RHS) {
15337 EVT AccVT = Acc.getValueType();
15338 if (AccVT.isFloatingPoint()) {
15339 assert(Opc == ISD::PARTIAL_REDUCE_FMLA && "Unexpected opcode");
15340 SDValue NegRHS = getNode(Opcode: ISD::FNEG, DL, VT: RHS.getValueType(), N1: RHS);
15341 return getNode(Opcode: Opc, DL, VT: AccVT, N1: Acc, N2: LHS, N3: NegRHS);
15342 }
15343 assert((Opc == ISD::PARTIAL_REDUCE_UMLA || Opc == ISD::PARTIAL_REDUCE_SMLA ||
15344 Opc == ISD::PARTIAL_REDUCE_SUMLA) &&
15345 "Unexpected opcode");
15346 SDValue NegAcc = getNegative(Val: Acc, DL, VT: AccVT);
15347 SDValue MLA = getNode(Opcode: Opc, DL, VT: AccVT, N1: NegAcc, N2: LHS, N3: RHS);
15348 return getNegative(Val: MLA, DL, VT: AccVT);
15349}
15350
15351/// Helper used to make a call to a library function that has one argument of
15352/// pointer type.
15353///
15354/// Such functions include 'fegetmode', 'fesetenv' and some others, which are
15355/// used to get or set floating-point state. They have one argument of pointer
15356/// type, which points to the memory region containing bits of the
15357/// floating-point state. The value returned by such function is ignored in the
15358/// created call.
15359///
15360/// \param LibFunc Reference to library function (value of RTLIB::Libcall).
15361/// \param Ptr Pointer used to save/load state.
15362/// \param InChain Ingoing token chain.
15363/// \param Node Node being legalized
15364/// \returns Outgoing chain token.
15365SDValue SelectionDAG::makeStateFunctionCall(unsigned LibFunc, SDValue Ptr,
15366 SDValue InChain, SDNode *Node) {
15367 assert(InChain.getValueType() == MVT::Other && "Expected token chain");
15368 RTLIB::LibcallImpl LibcallImpl =
15369 Libcalls->getLibcallImpl(Call: static_cast<RTLIB::Libcall>(LibFunc));
15370 if (LibcallImpl == RTLIB::Unsupported) {
15371 getContext()->emitError(ErrorStr: Twine("no libcall available for ") +
15372 Node->getOperationName(G: this));
15373 return InChain;
15374 }
15375
15376 TargetLowering::ArgListTy Args;
15377 Args.emplace_back(args&: Ptr, args: Ptr.getValueType().getTypeForEVT(Context&: *getContext()));
15378 SDValue Callee =
15379 getExternalSymbol(Libcall: LibcallImpl, VT: TLI->getPointerTy(DL: getDataLayout()));
15380 TargetLowering::CallLoweringInfo CLI(*this);
15381 CLI.setDebugLoc(SDLoc(Node))
15382 .setChain(InChain)
15383 .setLibCallee(CC: Libcalls->getLibcallImplCallingConv(Call: LibcallImpl),
15384 ResultType: Type::getVoidTy(C&: *getContext()), Target: Callee, ArgsList: std::move(Args));
15385 return TLI->LowerCallTo(CLI).second;
15386}
15387
15388void SelectionDAG::copyExtraInfo(SDNode *From, SDNode *To) {
15389 assert(From && To && "Invalid SDNode; empty source SDValue?");
15390 auto I = SDEI.find(Val: From);
15391 if (I == SDEI.end())
15392 return;
15393
15394 // Use of operator[] on the DenseMap may cause an insertion, which invalidates
15395 // the iterator, hence the need to make a copy to prevent a use-after-free.
15396 NodeExtraInfo NEI = I->second;
15397 if (LLVM_LIKELY(!NEI.PCSections)) {
15398 // No deep copy required for the types of extra info set.
15399 //
15400 // FIXME: Investigate if other types of extra info also need deep copy. This
15401 // depends on the types of nodes they can be attached to: if some extra info
15402 // is only ever attached to nodes where a replacement To node is always the
15403 // node where later use and propagation of the extra info has the intended
15404 // semantics, no deep copy is required.
15405 SDEI[To] = std::move(NEI);
15406 return;
15407 }
15408
15409 const SDNode *EntrySDN = getEntryNode().getNode();
15410
15411 // We need to copy NodeExtraInfo to all _new_ nodes that are being introduced
15412 // through the replacement of From with To. Otherwise, replacements of a node
15413 // (From) with more complex nodes (To and its operands) may result in lost
15414 // extra info where the root node (To) is insignificant in further propagating
15415 // and using extra info when further lowering to MIR.
15416 //
15417 // In the first step pre-populate the visited set with the nodes reachable
15418 // from the old From node. This avoids copying NodeExtraInfo to parts of the
15419 // DAG that is not new and should be left untouched.
15420 SmallVector<const SDNode *> Leafs{From}; // Leafs reachable with VisitFrom.
15421 DenseSet<const SDNode *> FromReach; // The set of nodes reachable from From.
15422 auto VisitFrom = [&](auto &&Self, const SDNode *N, int MaxDepth) {
15423 if (MaxDepth == 0) {
15424 // Remember this node in case we need to increase MaxDepth and continue
15425 // populating FromReach from this node.
15426 Leafs.emplace_back(Args&: N);
15427 return;
15428 }
15429 if (!FromReach.insert(V: N).second)
15430 return;
15431 for (const SDValue &Op : N->op_values())
15432 Self(Self, Op.getNode(), MaxDepth - 1);
15433 };
15434
15435 // Copy extra info to To and all its transitive operands (that are new).
15436 SmallPtrSet<const SDNode *, 8> Visited;
15437 auto DeepCopyTo = [&](auto &&Self, const SDNode *N) {
15438 if (FromReach.contains(V: N))
15439 return true;
15440 if (!Visited.insert(Ptr: N).second)
15441 return true;
15442 if (EntrySDN == N)
15443 return false;
15444 for (const SDValue &Op : N->op_values()) {
15445 if (N == To && Op.getNode() == EntrySDN) {
15446 // Special case: New node's operand is the entry node; just need to
15447 // copy extra info to new node.
15448 break;
15449 }
15450 if (!Self(Self, Op.getNode()))
15451 return false;
15452 }
15453 // Copy only if entry node was not reached.
15454 SDEI[N] = std::move(NEI);
15455 return true;
15456 };
15457
15458 // We first try with a lower MaxDepth, assuming that the path to common
15459 // operands between From and To is relatively short. This significantly
15460 // improves performance in the common case. The initial MaxDepth is big
15461 // enough to avoid retry in the common case; the last MaxDepth is large
15462 // enough to avoid having to use the fallback below (and protects from
15463 // potential stack exhaustion from recursion).
15464 for (int PrevDepth = 0, MaxDepth = 16; MaxDepth <= 1024;
15465 PrevDepth = MaxDepth, MaxDepth *= 2, Visited.clear()) {
15466 // StartFrom is the previous (or initial) set of leafs reachable at the
15467 // previous maximum depth.
15468 SmallVector<const SDNode *> StartFrom;
15469 std::swap(LHS&: StartFrom, RHS&: Leafs);
15470 for (const SDNode *N : StartFrom)
15471 VisitFrom(VisitFrom, N, MaxDepth - PrevDepth);
15472 if (LLVM_LIKELY(DeepCopyTo(DeepCopyTo, To)))
15473 return;
15474 // This should happen very rarely (reached the entry node).
15475 LLVM_DEBUG(dbgs() << __func__ << ": MaxDepth=" << MaxDepth << " too low\n");
15476 assert(!Leafs.empty());
15477 }
15478
15479 // This should not happen - but if it did, that means the subgraph reachable
15480 // from From has depth greater or equal to maximum MaxDepth, and VisitFrom()
15481 // could not visit all reachable common operands. Consequently, we were able
15482 // to reach the entry node.
15483 errs() << "warning: incomplete propagation of SelectionDAG::NodeExtraInfo\n";
15484 assert(false && "From subgraph too complex - increase max. MaxDepth?");
15485 // Best-effort fallback if assertions disabled.
15486 SDEI[To] = std::move(NEI);
15487}
15488
15489unsigned SelectionDAG::getMaxRuntimeNumElements(EVT VT) const {
15490 assert(VT.isVector() && "Can only unroll vector types!");
15491 if (VT.isFixedLengthVector())
15492 return VT.getVectorNumElements();
15493
15494 const MachineFunction &MF = getMachineFunction();
15495 const Function &F = MF.getFunction();
15496
15497 APInt MaxVScale = getVScaleRange(F: &F, BitWidth: sizeof(unsigned) * 8).getUnsignedMax();
15498 if (MaxVScale.ugt(RHS: VScaleUnrollLimit))
15499 return 0;
15500
15501 bool Overflow;
15502 APInt MinNElts(sizeof(unsigned) * 8, VT.getVectorMinNumElements());
15503 APInt MaxNElts = MinNElts.umul_ov(RHS: MaxVScale, Overflow);
15504 if (Overflow)
15505 return 0;
15506
15507 return MaxNElts.getZExtValue();
15508}
15509
15510SDValue SelectionDAG::buildVectorFromUnrolledParts(EVT VT, const SDLoc &DL,
15511 ArrayRef<SDValue> Scalars) {
15512 assert(Scalars.size() == getMaxRuntimeNumElements(VT) &&
15513 "Element count mismatch!");
15514 if (VT.isFixedLengthVector())
15515 return getBuildVector(VT, DL, Ops: Scalars);
15516
15517 SDValue Vec = getPOISON(VT);
15518 // Iterate in reverse so result remains poison until we encounter a lane that
15519 // exists, after which all lower-numbered lanes must also exist.
15520 for (unsigned IdxVal : reverse(C: seq(Size: Scalars.size())))
15521 Vec = getInsertVectorElt(DL, Vec, Elt: Scalars[IdxVal], Idx: IdxVal);
15522
15523 return Vec;
15524}
15525
15526#ifndef NDEBUG
15527static void checkForCyclesHelper(const SDNode *N,
15528 SmallPtrSetImpl<const SDNode*> &Visited,
15529 SmallPtrSetImpl<const SDNode*> &Checked,
15530 const llvm::SelectionDAG *DAG) {
15531 // If this node has already been checked, don't check it again.
15532 if (Checked.count(N))
15533 return;
15534
15535 // If a node has already been visited on this depth-first walk, reject it as
15536 // a cycle.
15537 if (!Visited.insert(N).second) {
15538 errs() << "Detected cycle in SelectionDAG\n";
15539 dbgs() << "Offending node:\n";
15540 N->dumprFull(DAG); dbgs() << "\n";
15541 abort();
15542 }
15543
15544 for (const SDValue &Op : N->op_values())
15545 checkForCyclesHelper(Op.getNode(), Visited, Checked, DAG);
15546
15547 Checked.insert(N);
15548 Visited.erase(N);
15549}
15550#endif
15551
15552void llvm::checkForCycles(const llvm::SDNode *N,
15553 const llvm::SelectionDAG *DAG,
15554 bool force) {
15555#ifndef NDEBUG
15556 bool check = force;
15557#ifdef EXPENSIVE_CHECKS
15558 check = true;
15559#endif // EXPENSIVE_CHECKS
15560 if (check) {
15561 assert(N && "Checking nonexistent SDNode");
15562 SmallPtrSet<const SDNode*, 32> visited;
15563 SmallPtrSet<const SDNode*, 32> checked;
15564 checkForCyclesHelper(N, visited, checked, DAG);
15565 }
15566#endif // !NDEBUG
15567}
15568
15569void llvm::checkForCycles(const llvm::SelectionDAG *DAG, bool force) {
15570 checkForCycles(N: DAG->getRoot().getNode(), DAG, force);
15571}
15572