1//===- SelectionDAG.cpp - Implement the SelectionDAG data structures ------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements the SelectionDAG class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/CodeGen/SelectionDAG.h"
14#include "SDNodeDbgValue.h"
15#include "llvm/ADT/APFloat.h"
16#include "llvm/ADT/APInt.h"
17#include "llvm/ADT/APSInt.h"
18#include "llvm/ADT/ArrayRef.h"
19#include "llvm/ADT/BitVector.h"
20#include "llvm/ADT/DenseSet.h"
21#include "llvm/ADT/FoldingSet.h"
22#include "llvm/ADT/STLExtras.h"
23#include "llvm/ADT/SmallPtrSet.h"
24#include "llvm/ADT/SmallVector.h"
25#include "llvm/ADT/Twine.h"
26#include "llvm/Analysis/AliasAnalysis.h"
27#include "llvm/Analysis/MemoryLocation.h"
28#include "llvm/Analysis/TargetLibraryInfo.h"
29#include "llvm/Analysis/ValueTracking.h"
30#include "llvm/Analysis/VectorUtils.h"
31#include "llvm/BinaryFormat/Dwarf.h"
32#include "llvm/CodeGen/Analysis.h"
33#include "llvm/CodeGen/CodeGenCommonISel.h"
34#include "llvm/CodeGen/FunctionLoweringInfo.h"
35#include "llvm/CodeGen/ISDOpcodes.h"
36#include "llvm/CodeGen/MachineBasicBlock.h"
37#include "llvm/CodeGen/MachineConstantPool.h"
38#include "llvm/CodeGen/MachineFrameInfo.h"
39#include "llvm/CodeGen/MachineFunction.h"
40#include "llvm/CodeGen/MachineMemOperand.h"
41#include "llvm/CodeGen/RuntimeLibcallUtil.h"
42#include "llvm/CodeGen/SDPatternMatch.h"
43#include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
44#include "llvm/CodeGen/SelectionDAGNodes.h"
45#include "llvm/CodeGen/SelectionDAGTargetInfo.h"
46#include "llvm/CodeGen/TargetFrameLowering.h"
47#include "llvm/CodeGen/TargetLowering.h"
48#include "llvm/CodeGen/TargetRegisterInfo.h"
49#include "llvm/CodeGen/TargetSubtargetInfo.h"
50#include "llvm/CodeGen/ValueTypes.h"
51#include "llvm/CodeGenTypes/MachineValueType.h"
52#include "llvm/IR/Constant.h"
53#include "llvm/IR/Constants.h"
54#include "llvm/IR/DataLayout.h"
55#include "llvm/IR/DebugInfoMetadata.h"
56#include "llvm/IR/DebugLoc.h"
57#include "llvm/IR/DerivedTypes.h"
58#include "llvm/IR/Function.h"
59#include "llvm/IR/GlobalValue.h"
60#include "llvm/IR/Metadata.h"
61#include "llvm/IR/Type.h"
62#include "llvm/Support/Casting.h"
63#include "llvm/Support/CodeGen.h"
64#include "llvm/Support/Compiler.h"
65#include "llvm/Support/Debug.h"
66#include "llvm/Support/ErrorHandling.h"
67#include "llvm/Support/KnownBits.h"
68#include "llvm/Support/KnownFPClass.h"
69#include "llvm/Support/MathExtras.h"
70#include "llvm/Support/raw_ostream.h"
71#include "llvm/Target/TargetMachine.h"
72#include "llvm/Target/TargetOptions.h"
73#include "llvm/TargetParser/Triple.h"
74#include "llvm/Transforms/Utils/SizeOpts.h"
75#include <algorithm>
76#include <cassert>
77#include <cstdint>
78#include <cstdlib>
79#include <limits>
80#include <optional>
81#include <string>
82#include <utility>
83#include <vector>
84
85using namespace llvm;
86using namespace llvm::SDPatternMatch;
87
88/// makeVTList - Return an instance of the SDVTList struct initialized with the
89/// specified members.
90static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) {
91 SDVTList Res = {.VTs: VTs, .NumVTs: NumVTs};
92 return Res;
93}
94
95// Default null implementations of the callbacks.
96void SelectionDAG::DAGUpdateListener::NodeDeleted(SDNode*, SDNode*) {}
97void SelectionDAG::DAGUpdateListener::NodeUpdated(SDNode*) {}
98void SelectionDAG::DAGUpdateListener::NodeInserted(SDNode *) {}
99
100void SelectionDAG::DAGNodeDeletedListener::anchor() {}
101void SelectionDAG::DAGNodeInsertedListener::anchor() {}
102
103#define DEBUG_TYPE "selectiondag"
104
105static cl::opt<bool> EnableMemCpyDAGOpt("enable-memcpy-dag-opt",
106 cl::Hidden, cl::init(Val: true),
107 cl::desc("Gang up loads and stores generated by inlining of memcpy"));
108
109static cl::opt<int> MaxLdStGlue("ldstmemcpy-glue-max",
110 cl::desc("Number limit for gluing ld/st of memcpy."),
111 cl::Hidden, cl::init(Val: 0));
112
113static cl::opt<unsigned>
114 MaxSteps("has-predecessor-max-steps", cl::Hidden, cl::init(Val: 8192),
115 cl::desc("DAG combiner limit number of steps when searching DAG "
116 "for predecessor nodes"));
117
118static void NewSDValueDbgMsg(SDValue V, StringRef Msg, SelectionDAG *G) {
119 LLVM_DEBUG(dbgs() << Msg; V.getNode()->dump(G););
120}
121
122unsigned SelectionDAG::getHasPredecessorMaxSteps() { return MaxSteps; }
123
124//===----------------------------------------------------------------------===//
125// ConstantFPSDNode Class
126//===----------------------------------------------------------------------===//
127
128/// isExactlyValue - We don't rely on operator== working on double values, as
129/// it returns true for things that are clearly not equal, like -0.0 and 0.0.
130/// As such, this method can be used to do an exact bit-for-bit comparison of
131/// two floating point values.
132bool ConstantFPSDNode::isExactlyValue(const APFloat& V) const {
133 return getValueAPF().bitwiseIsEqual(RHS: V);
134}
135
136bool ConstantFPSDNode::isValueValidForType(EVT VT,
137 const APFloat& Val) {
138 assert(VT.isFloatingPoint() && "Can only convert between FP types");
139
140 // convert modifies in place, so make a copy.
141 APFloat Val2 = APFloat(Val);
142 bool losesInfo;
143 (void)Val2.convert(ToSemantics: VT.getFltSemantics(), RM: APFloat::rmNearestTiesToEven,
144 losesInfo: &losesInfo);
145 return !losesInfo;
146}
147
148//===----------------------------------------------------------------------===//
149// ISD Namespace
150//===----------------------------------------------------------------------===//
151
152bool ISD::isConstantSplatVector(const SDNode *N, APInt &SplatVal) {
153 if (N->getOpcode() == ISD::SPLAT_VECTOR) {
154 if (auto OptAPInt = N->getOperand(Num: 0)->bitcastToAPInt()) {
155 unsigned EltSize =
156 N->getValueType(ResNo: 0).getVectorElementType().getSizeInBits();
157 SplatVal = OptAPInt->trunc(width: EltSize);
158 return true;
159 }
160 }
161
162 auto *BV = dyn_cast<BuildVectorSDNode>(Val: N);
163 if (!BV)
164 return false;
165
166 APInt SplatUndef;
167 unsigned SplatBitSize;
168 bool HasUndefs;
169 unsigned EltSize = N->getValueType(ResNo: 0).getVectorElementType().getSizeInBits();
170 // Endianness does not matter here. We are checking for a splat given the
171 // element size of the vector, and if we find such a splat for little endian
172 // layout, then that should be valid also for big endian (as the full vector
173 // size is known to be a multiple of the element size).
174 const bool IsBigEndian = false;
175 return BV->isConstantSplat(SplatValue&: SplatVal, SplatUndef, SplatBitSize, HasAnyUndefs&: HasUndefs,
176 MinSplatBits: EltSize, isBigEndian: IsBigEndian) &&
177 EltSize == SplatBitSize;
178}
179
180// FIXME: AllOnes and AllZeros duplicate a lot of code. Could these be
181// specializations of the more general isConstantSplatVector()?
182
183bool ISD::isConstantSplatVectorAllOnes(const SDNode *N, bool BuildVectorOnly) {
184 // Look through a bit convert.
185 while (N->getOpcode() == ISD::BITCAST)
186 N = N->getOperand(Num: 0).getNode();
187
188 if (!BuildVectorOnly && N->getOpcode() == ISD::SPLAT_VECTOR) {
189 APInt SplatVal;
190 return isConstantSplatVector(N, SplatVal) && SplatVal.isAllOnes();
191 }
192
193 if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
194
195 unsigned i = 0, e = N->getNumOperands();
196
197 // Skip over all of the undef values.
198 while (i != e && N->getOperand(Num: i).isUndef())
199 ++i;
200
201 // Do not accept an all-undef vector.
202 if (i == e) return false;
203
204 // Do not accept build_vectors that aren't all constants or which have non-~0
205 // elements. We have to be a bit careful here, as the type of the constant
206 // may not be the same as the type of the vector elements due to type
207 // legalization (the elements are promoted to a legal type for the target and
208 // a vector of a type may be legal when the base element type is not).
209 // We only want to check enough bits to cover the vector elements, because
210 // we care if the resultant vector is all ones, not whether the individual
211 // constants are.
212 SDValue NotZero = N->getOperand(Num: i);
213 if (auto OptAPInt = NotZero->bitcastToAPInt()) {
214 unsigned EltSize = N->getValueType(ResNo: 0).getScalarSizeInBits();
215 if (OptAPInt->countr_one() < EltSize)
216 return false;
217 } else
218 return false;
219
220 // Okay, we have at least one ~0 value, check to see if the rest match or are
221 // undefs. Even with the above element type twiddling, this should be OK, as
222 // the same type legalization should have applied to all the elements.
223 for (++i; i != e; ++i)
224 if (N->getOperand(Num: i) != NotZero && !N->getOperand(Num: i).isUndef())
225 return false;
226 return true;
227}
228
229bool ISD::isConstantSplatVectorAllZeros(const SDNode *N, bool BuildVectorOnly) {
230 // Look through a bit convert.
231 while (N->getOpcode() == ISD::BITCAST)
232 N = N->getOperand(Num: 0).getNode();
233
234 if (!BuildVectorOnly && N->getOpcode() == ISD::SPLAT_VECTOR) {
235 APInt SplatVal;
236 return isConstantSplatVector(N, SplatVal) && SplatVal.isZero();
237 }
238
239 if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
240
241 bool IsAllUndef = true;
242 for (const SDValue &Op : N->op_values()) {
243 if (Op.isUndef())
244 continue;
245 IsAllUndef = false;
246 // Do not accept build_vectors that aren't all constants or which have non-0
247 // elements. We have to be a bit careful here, as the type of the constant
248 // may not be the same as the type of the vector elements due to type
249 // legalization (the elements are promoted to a legal type for the target
250 // and a vector of a type may be legal when the base element type is not).
251 // We only want to check enough bits to cover the vector elements, because
252 // we care if the resultant vector is all zeros, not whether the individual
253 // constants are.
254 if (auto OptAPInt = Op->bitcastToAPInt()) {
255 unsigned EltSize = N->getValueType(ResNo: 0).getScalarSizeInBits();
256 if (OptAPInt->countr_zero() < EltSize)
257 return false;
258 } else
259 return false;
260 }
261
262 // Do not accept an all-undef vector.
263 if (IsAllUndef)
264 return false;
265 return true;
266}
267
268bool ISD::isBuildVectorAllOnes(const SDNode *N) {
269 return isConstantSplatVectorAllOnes(N, /*BuildVectorOnly*/ true);
270}
271
272bool ISD::isBuildVectorAllZeros(const SDNode *N) {
273 return isConstantSplatVectorAllZeros(N, /*BuildVectorOnly*/ true);
274}
275
276bool ISD::isBuildVectorOfConstantSDNodes(const SDNode *N) {
277 if (N->getOpcode() != ISD::BUILD_VECTOR)
278 return false;
279
280 for (const SDValue &Op : N->op_values()) {
281 if (Op.isUndef())
282 continue;
283 if (!isa<ConstantSDNode>(Val: Op))
284 return false;
285 }
286 return true;
287}
288
289bool ISD::isBuildVectorOfConstantFPSDNodes(const SDNode *N) {
290 if (N->getOpcode() != ISD::BUILD_VECTOR)
291 return false;
292
293 for (const SDValue &Op : N->op_values()) {
294 if (Op.isUndef())
295 continue;
296 if (!isa<ConstantFPSDNode>(Val: Op))
297 return false;
298 }
299 return true;
300}
301
302bool ISD::isVectorShrinkable(const SDNode *N, unsigned NewEltSize,
303 bool Signed) {
304 assert(N->getValueType(0).isVector() && "Expected a vector!");
305
306 unsigned EltSize = N->getValueType(ResNo: 0).getScalarSizeInBits();
307 if (EltSize <= NewEltSize)
308 return false;
309
310 if (N->getOpcode() == ISD::ZERO_EXTEND) {
311 return (N->getOperand(Num: 0).getValueType().getScalarSizeInBits() <=
312 NewEltSize) &&
313 !Signed;
314 }
315 if (N->getOpcode() == ISD::SIGN_EXTEND) {
316 return (N->getOperand(Num: 0).getValueType().getScalarSizeInBits() <=
317 NewEltSize) &&
318 Signed;
319 }
320 if (N->getOpcode() != ISD::BUILD_VECTOR)
321 return false;
322
323 for (const SDValue &Op : N->op_values()) {
324 if (Op.isUndef())
325 continue;
326 if (!isa<ConstantSDNode>(Val: Op))
327 return false;
328
329 APInt C = Op->getAsAPIntVal().trunc(width: EltSize);
330 if (Signed && C.trunc(width: NewEltSize).sext(width: EltSize) != C)
331 return false;
332 if (!Signed && C.trunc(width: NewEltSize).zext(width: EltSize) != C)
333 return false;
334 }
335
336 return true;
337}
338
339bool ISD::allOperandsUndef(const SDNode *N) {
340 // Return false if the node has no operands.
341 // This is "logically inconsistent" with the definition of "all" but
342 // is probably the desired behavior.
343 if (N->getNumOperands() == 0)
344 return false;
345 return all_of(Range: N->op_values(), P: [](SDValue Op) { return Op.isUndef(); });
346}
347
348bool ISD::isFreezeUndef(const SDNode *N) {
349 return N->getOpcode() == ISD::FREEZE && N->getOperand(Num: 0).isUndef();
350}
351
352template <typename ConstNodeType>
353bool ISD::matchUnaryPredicateImpl(SDValue Op, const APInt &DemandedElts,
354 std::function<bool(ConstNodeType *)> Match,
355 bool AllowUndefs, bool AllowTruncation) {
356 // FIXME: Add support for scalar UNDEF cases?
357 if (auto *C = dyn_cast<ConstNodeType>(Op))
358 return Match(C);
359
360 // FIXME: Add support for vector UNDEF cases?
361 if (ISD::BUILD_VECTOR != Op.getOpcode() &&
362 ISD::SPLAT_VECTOR != Op.getOpcode())
363 return false;
364
365 if (ISD::SPLAT_VECTOR == Op.getOpcode() && !DemandedElts)
366 return true;
367
368 EVT SVT = Op.getValueType().getScalarType();
369 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
370 if (ISD::SPLAT_VECTOR != Op.getOpcode() && !DemandedElts[i])
371 continue;
372
373 if (AllowUndefs && Op.getOperand(i).isUndef()) {
374 if (!Match(nullptr))
375 return false;
376 continue;
377 }
378
379 auto *Cst = dyn_cast<ConstNodeType>(Op.getOperand(i));
380 if (!Cst || (!AllowTruncation && Cst->getValueType(0) != SVT) ||
381 !Match(Cst))
382 return false;
383 }
384 return true;
385}
386// Build used template types.
387template bool ISD::matchUnaryPredicateImpl<ConstantSDNode>(
388 SDValue, const APInt &, std::function<bool(ConstantSDNode *)>, bool, bool);
389template bool ISD::matchUnaryPredicateImpl<ConstantFPSDNode>(
390 SDValue, const APInt &, std::function<bool(ConstantFPSDNode *)>, bool,
391 bool);
392
393bool ISD::matchBinaryPredicate(
394 SDValue LHS, SDValue RHS, const APInt &DemandedElts,
395 std::function<bool(ConstantSDNode *, ConstantSDNode *)> Match,
396 bool AllowUndefs, bool AllowTypeMismatch) {
397 if (!AllowTypeMismatch && LHS.getValueType() != RHS.getValueType())
398 return false;
399
400 // TODO: Add support for scalar UNDEF cases?
401 if (auto *LHSCst = dyn_cast<ConstantSDNode>(Val&: LHS))
402 if (auto *RHSCst = dyn_cast<ConstantSDNode>(Val&: RHS))
403 return Match(LHSCst, RHSCst);
404
405 // TODO: Add support for vector UNDEF cases?
406 if (LHS.getOpcode() != RHS.getOpcode() ||
407 (LHS.getOpcode() != ISD::BUILD_VECTOR &&
408 LHS.getOpcode() != ISD::SPLAT_VECTOR))
409 return false;
410
411 if (ISD::SPLAT_VECTOR == LHS.getOpcode() && !DemandedElts)
412 return true;
413
414 EVT SVT = LHS.getValueType().getScalarType();
415 for (unsigned i = 0, e = LHS.getNumOperands(); i != e; ++i) {
416 if (ISD::SPLAT_VECTOR != LHS.getOpcode() && !DemandedElts[i])
417 continue;
418 SDValue LHSOp = LHS.getOperand(i);
419 SDValue RHSOp = RHS.getOperand(i);
420 bool LHSUndef = AllowUndefs && LHSOp.isUndef();
421 bool RHSUndef = AllowUndefs && RHSOp.isUndef();
422 auto *LHSCst = dyn_cast<ConstantSDNode>(Val&: LHSOp);
423 auto *RHSCst = dyn_cast<ConstantSDNode>(Val&: RHSOp);
424 if ((!LHSCst && !LHSUndef) || (!RHSCst && !RHSUndef))
425 return false;
426 if (!AllowTypeMismatch && (LHSOp.getValueType() != SVT ||
427 LHSOp.getValueType() != RHSOp.getValueType()))
428 return false;
429 if (!Match(LHSCst, RHSCst))
430 return false;
431 }
432 return true;
433}
434
435ISD::NodeType ISD::getInverseMinMaxOpcode(unsigned MinMaxOpc) {
436 switch (MinMaxOpc) {
437 default:
438 llvm_unreachable("unrecognized opcode");
439 case ISD::UMIN:
440 return ISD::UMAX;
441 case ISD::UMAX:
442 return ISD::UMIN;
443 case ISD::SMIN:
444 return ISD::SMAX;
445 case ISD::SMAX:
446 return ISD::SMIN;
447 }
448}
449
450ISD::NodeType ISD::getOppositeSignednessMinMaxOpcode(unsigned MinMaxOpc) {
451 switch (MinMaxOpc) {
452 default:
453 llvm_unreachable("unrecognized min/max opcode");
454 case ISD::SMIN:
455 return ISD::UMIN;
456 case ISD::SMAX:
457 return ISD::UMAX;
458 case ISD::UMIN:
459 return ISD::SMIN;
460 case ISD::UMAX:
461 return ISD::SMAX;
462 }
463}
464
465ISD::NodeType ISD::getVecReduceBaseOpcode(unsigned VecReduceOpcode) {
466 switch (VecReduceOpcode) {
467 default:
468 llvm_unreachable("Expected VECREDUCE opcode");
469 case ISD::VECREDUCE_FADD:
470 case ISD::VECREDUCE_SEQ_FADD:
471 case ISD::VP_REDUCE_FADD:
472 case ISD::VP_REDUCE_SEQ_FADD:
473 return ISD::FADD;
474 case ISD::VECREDUCE_FMUL:
475 case ISD::VECREDUCE_SEQ_FMUL:
476 case ISD::VP_REDUCE_FMUL:
477 case ISD::VP_REDUCE_SEQ_FMUL:
478 return ISD::FMUL;
479 case ISD::VECREDUCE_ADD:
480 case ISD::VP_REDUCE_ADD:
481 return ISD::ADD;
482 case ISD::VECREDUCE_MUL:
483 case ISD::VP_REDUCE_MUL:
484 return ISD::MUL;
485 case ISD::VECREDUCE_AND:
486 case ISD::VP_REDUCE_AND:
487 return ISD::AND;
488 case ISD::VECREDUCE_OR:
489 case ISD::VP_REDUCE_OR:
490 return ISD::OR;
491 case ISD::VECREDUCE_XOR:
492 case ISD::VP_REDUCE_XOR:
493 return ISD::XOR;
494 case ISD::VECREDUCE_SMAX:
495 case ISD::VP_REDUCE_SMAX:
496 return ISD::SMAX;
497 case ISD::VECREDUCE_SMIN:
498 case ISD::VP_REDUCE_SMIN:
499 return ISD::SMIN;
500 case ISD::VECREDUCE_UMAX:
501 case ISD::VP_REDUCE_UMAX:
502 return ISD::UMAX;
503 case ISD::VECREDUCE_UMIN:
504 case ISD::VP_REDUCE_UMIN:
505 return ISD::UMIN;
506 case ISD::VECREDUCE_FMAX:
507 case ISD::VP_REDUCE_FMAX:
508 return ISD::FMAXNUM;
509 case ISD::VECREDUCE_FMIN:
510 case ISD::VP_REDUCE_FMIN:
511 return ISD::FMINNUM;
512 case ISD::VECREDUCE_FMAXIMUM:
513 case ISD::VP_REDUCE_FMAXIMUM:
514 return ISD::FMAXIMUM;
515 case ISD::VECREDUCE_FMINIMUM:
516 case ISD::VP_REDUCE_FMINIMUM:
517 return ISD::FMINIMUM;
518 case ISD::VECREDUCE_FMAXIMUMNUM:
519 return ISD::FMAXIMUMNUM;
520 case ISD::VECREDUCE_FMINIMUMNUM:
521 return ISD::FMINIMUMNUM;
522 }
523}
524
525ISD::NodeType ISD::getUnmaskedBinOpOpcode(unsigned MaskedOpc) {
526 switch (MaskedOpc) {
527 case ISD::MASKED_UDIV:
528 return ISD::UDIV;
529 case ISD::MASKED_SDIV:
530 return ISD::SDIV;
531 case ISD::MASKED_UREM:
532 return ISD::UREM;
533 case ISD::MASKED_SREM:
534 return ISD::SREM;
535 default:
536 llvm_unreachable("Expected masked binop opcode");
537 }
538}
539
540bool ISD::isVPOpcode(unsigned Opcode) {
541 switch (Opcode) {
542 default:
543 return false;
544#define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) \
545 case ISD::VPSD: \
546 return true;
547#include "llvm/IR/VPIntrinsics.def"
548 }
549}
550
551bool ISD::isVPBinaryOp(unsigned Opcode) {
552 switch (Opcode) {
553 default:
554 break;
555#define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) case ISD::VPSD:
556#define VP_PROPERTY_BINARYOP return true;
557#define END_REGISTER_VP_SDNODE(VPSD) break;
558#include "llvm/IR/VPIntrinsics.def"
559 }
560 return false;
561}
562
563bool ISD::isVPReduction(unsigned Opcode) {
564 switch (Opcode) {
565 default:
566 return false;
567 case ISD::VP_REDUCE_ADD:
568 case ISD::VP_REDUCE_MUL:
569 case ISD::VP_REDUCE_AND:
570 case ISD::VP_REDUCE_OR:
571 case ISD::VP_REDUCE_XOR:
572 case ISD::VP_REDUCE_SMAX:
573 case ISD::VP_REDUCE_SMIN:
574 case ISD::VP_REDUCE_UMAX:
575 case ISD::VP_REDUCE_UMIN:
576 case ISD::VP_REDUCE_FMAX:
577 case ISD::VP_REDUCE_FMIN:
578 case ISD::VP_REDUCE_FMAXIMUM:
579 case ISD::VP_REDUCE_FMINIMUM:
580 case ISD::VP_REDUCE_FADD:
581 case ISD::VP_REDUCE_FMUL:
582 case ISD::VP_REDUCE_SEQ_FADD:
583 case ISD::VP_REDUCE_SEQ_FMUL:
584 return true;
585 }
586}
587
588/// The operand position of the vector mask.
589std::optional<unsigned> ISD::getVPMaskIdx(unsigned Opcode) {
590 switch (Opcode) {
591 default:
592 return std::nullopt;
593#define BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, TDNAME, MASKPOS, ...) \
594 case ISD::VPSD: \
595 return MASKPOS;
596#include "llvm/IR/VPIntrinsics.def"
597 }
598}
599
600/// The operand position of the explicit vector length parameter.
601std::optional<unsigned> ISD::getVPExplicitVectorLengthIdx(unsigned Opcode) {
602 switch (Opcode) {
603 default:
604 return std::nullopt;
605#define BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, TDNAME, MASKPOS, EVLPOS) \
606 case ISD::VPSD: \
607 return EVLPOS;
608#include "llvm/IR/VPIntrinsics.def"
609 }
610}
611
612std::optional<unsigned> ISD::getBaseOpcodeForVP(unsigned VPOpcode,
613 bool hasFPExcept) {
614 // FIXME: Return strict opcodes in case of fp exceptions.
615 switch (VPOpcode) {
616 default:
617 return std::nullopt;
618#define BEGIN_REGISTER_VP_SDNODE(VPOPC, ...) case ISD::VPOPC:
619#define VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) return ISD::SDOPC;
620#define END_REGISTER_VP_SDNODE(VPOPC) break;
621#include "llvm/IR/VPIntrinsics.def"
622 }
623 return std::nullopt;
624}
625
626std::optional<unsigned> ISD::getVPForBaseOpcode(unsigned Opcode) {
627 switch (Opcode) {
628 default:
629 return std::nullopt;
630#define BEGIN_REGISTER_VP_SDNODE(VPOPC, ...) break;
631#define VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) case ISD::SDOPC:
632#define END_REGISTER_VP_SDNODE(VPOPC) return ISD::VPOPC;
633#include "llvm/IR/VPIntrinsics.def"
634 }
635}
636
637ISD::NodeType ISD::getExtForLoadExtType(bool IsFP, ISD::LoadExtType ExtType) {
638 switch (ExtType) {
639 case ISD::EXTLOAD:
640 return IsFP ? ISD::FP_EXTEND : ISD::ANY_EXTEND;
641 case ISD::SEXTLOAD:
642 return ISD::SIGN_EXTEND;
643 case ISD::ZEXTLOAD:
644 return ISD::ZERO_EXTEND;
645 default:
646 break;
647 }
648
649 llvm_unreachable("Invalid LoadExtType");
650}
651
652ISD::CondCode ISD::getSetCCSwappedOperands(ISD::CondCode Operation) {
653 // To perform this operation, we just need to swap the L and G bits of the
654 // operation.
655 unsigned OldL = (Operation >> 2) & 1;
656 unsigned OldG = (Operation >> 1) & 1;
657 return ISD::CondCode((Operation & ~6) | // Keep the N, U, E bits
658 (OldL << 1) | // New G bit
659 (OldG << 2)); // New L bit.
660}
661
662static ISD::CondCode getSetCCInverseImpl(ISD::CondCode Op, bool isIntegerLike) {
663 unsigned Operation = Op;
664 if (isIntegerLike)
665 Operation ^= 7; // Flip L, G, E bits, but not U.
666 else
667 Operation ^= 15; // Flip all of the condition bits.
668
669 if (Operation > ISD::SETTRUE2)
670 Operation &= ~8; // Don't let N and U bits get set.
671
672 return ISD::CondCode(Operation);
673}
674
675ISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, EVT Type) {
676 return getSetCCInverseImpl(Op, isIntegerLike: Type.isInteger());
677}
678
679ISD::CondCode ISD::GlobalISel::getSetCCInverse(ISD::CondCode Op,
680 bool isIntegerLike) {
681 return getSetCCInverseImpl(Op, isIntegerLike);
682}
683
684/// For an integer comparison, return 1 if the comparison is a signed operation
685/// and 2 if the result is an unsigned comparison. Return zero if the operation
686/// does not depend on the sign of the input (setne and seteq).
687static int isSignedOp(ISD::CondCode Opcode) {
688 switch (Opcode) {
689 default: llvm_unreachable("Illegal integer setcc operation!");
690 case ISD::SETEQ:
691 case ISD::SETNE: return 0;
692 case ISD::SETLT:
693 case ISD::SETLE:
694 case ISD::SETGT:
695 case ISD::SETGE: return 1;
696 case ISD::SETULT:
697 case ISD::SETULE:
698 case ISD::SETUGT:
699 case ISD::SETUGE: return 2;
700 }
701}
702
703ISD::CondCode ISD::getSetCCOrOperation(ISD::CondCode Op1, ISD::CondCode Op2,
704 EVT Type) {
705 bool IsInteger = Type.isInteger();
706 if (IsInteger && (isSignedOp(Opcode: Op1) | isSignedOp(Opcode: Op2)) == 3)
707 // Cannot fold a signed integer setcc with an unsigned integer setcc.
708 return ISD::SETCC_INVALID;
709
710 unsigned Op = Op1 | Op2; // Combine all of the condition bits.
711
712 // If the N and U bits get set, then the resultant comparison DOES suddenly
713 // care about orderedness, and it is true when ordered.
714 if (Op > ISD::SETTRUE2)
715 Op &= ~16; // Clear the U bit if the N bit is set.
716
717 // Canonicalize illegal integer setcc's.
718 if (IsInteger && Op == ISD::SETUNE) // e.g. SETUGT | SETULT
719 Op = ISD::SETNE;
720
721 return ISD::CondCode(Op);
722}
723
724ISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2,
725 EVT Type) {
726 bool IsInteger = Type.isInteger();
727 if (IsInteger && (isSignedOp(Opcode: Op1) | isSignedOp(Opcode: Op2)) == 3)
728 // Cannot fold a signed setcc with an unsigned setcc.
729 return ISD::SETCC_INVALID;
730
731 // Combine all of the condition bits.
732 ISD::CondCode Result = ISD::CondCode(Op1 & Op2);
733
734 // Canonicalize illegal integer setcc's.
735 if (IsInteger) {
736 switch (Result) {
737 default: break;
738 case ISD::SETUO : Result = ISD::SETFALSE; break; // SETUGT & SETULT
739 case ISD::SETOEQ: // SETEQ & SETU[LG]E
740 case ISD::SETUEQ: Result = ISD::SETEQ ; break; // SETUGE & SETULE
741 case ISD::SETOLT: Result = ISD::SETULT ; break; // SETULT & SETNE
742 case ISD::SETOGT: Result = ISD::SETUGT ; break; // SETUGT & SETNE
743 }
744 }
745
746 return Result;
747}
748
749//===----------------------------------------------------------------------===//
750// SDNode Profile Support
751//===----------------------------------------------------------------------===//
752
753/// AddNodeIDOpcode - Add the node opcode to the NodeID data.
754static void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC) {
755 ID.AddInteger(I: OpC);
756}
757
758/// AddNodeIDValueTypes - Value type lists are intern'd so we can represent them
759/// solely with their pointer.
760static void AddNodeIDValueTypes(FoldingSetNodeID &ID, SDVTList VTList) {
761 ID.AddPointer(Ptr: VTList.VTs);
762}
763
764/// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
765static void AddNodeIDOperands(FoldingSetNodeID &ID,
766 ArrayRef<SDValue> Ops) {
767 for (const auto &Op : Ops) {
768 ID.AddPointer(Ptr: Op.getNode());
769 ID.AddInteger(I: Op.getResNo());
770 }
771}
772
773/// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
774static void AddNodeIDOperands(FoldingSetNodeID &ID,
775 ArrayRef<SDUse> Ops) {
776 for (const auto &Op : Ops) {
777 ID.AddPointer(Ptr: Op.getNode());
778 ID.AddInteger(I: Op.getResNo());
779 }
780}
781
782static void AddNodeIDNode(FoldingSetNodeID &ID, unsigned OpC,
783 SDVTList VTList, ArrayRef<SDValue> OpList) {
784 AddNodeIDOpcode(ID, OpC);
785 AddNodeIDValueTypes(ID, VTList);
786 AddNodeIDOperands(ID, Ops: OpList);
787}
788
789/// If this is an SDNode with special info, add this info to the NodeID data.
790static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) {
791 switch (N->getOpcode()) {
792 case ISD::TargetExternalSymbol:
793 case ISD::ExternalSymbol:
794 case ISD::MCSymbol:
795 llvm_unreachable("Should only be used on nodes with operands");
796 default: break; // Normal nodes don't need extra info.
797 case ISD::TargetConstant:
798 case ISD::Constant: {
799 const ConstantSDNode *C = cast<ConstantSDNode>(Val: N);
800 ID.AddPointer(Ptr: C->getConstantIntValue());
801 ID.AddBoolean(B: C->isOpaque());
802 break;
803 }
804 case ISD::TargetConstantFP:
805 case ISD::ConstantFP:
806 ID.AddPointer(Ptr: cast<ConstantFPSDNode>(Val: N)->getConstantFPValue());
807 break;
808 case ISD::TargetGlobalAddress:
809 case ISD::GlobalAddress:
810 case ISD::TargetGlobalTLSAddress:
811 case ISD::GlobalTLSAddress: {
812 const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Val: N);
813 ID.AddPointer(Ptr: GA->getGlobal());
814 ID.AddInteger(I: GA->getOffset());
815 ID.AddInteger(I: GA->getTargetFlags());
816 break;
817 }
818 case ISD::BasicBlock:
819 ID.AddPointer(Ptr: cast<BasicBlockSDNode>(Val: N)->getBasicBlock());
820 break;
821 case ISD::Register:
822 ID.AddInteger(I: cast<RegisterSDNode>(Val: N)->getReg().id());
823 break;
824 case ISD::RegisterMask:
825 ID.AddPointer(Ptr: cast<RegisterMaskSDNode>(Val: N)->getRegMask());
826 break;
827 case ISD::SRCVALUE:
828 ID.AddPointer(Ptr: cast<SrcValueSDNode>(Val: N)->getValue());
829 break;
830 case ISD::FrameIndex:
831 case ISD::TargetFrameIndex:
832 ID.AddInteger(I: cast<FrameIndexSDNode>(Val: N)->getIndex());
833 break;
834 case ISD::PSEUDO_PROBE:
835 ID.AddInteger(I: cast<PseudoProbeSDNode>(Val: N)->getGuid());
836 ID.AddInteger(I: cast<PseudoProbeSDNode>(Val: N)->getIndex());
837 ID.AddInteger(I: cast<PseudoProbeSDNode>(Val: N)->getAttributes());
838 break;
839 case ISD::JumpTable:
840 case ISD::TargetJumpTable:
841 ID.AddInteger(I: cast<JumpTableSDNode>(Val: N)->getIndex());
842 ID.AddInteger(I: cast<JumpTableSDNode>(Val: N)->getTargetFlags());
843 break;
844 case ISD::ConstantPool:
845 case ISD::TargetConstantPool: {
846 const ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Val: N);
847 ID.AddInteger(I: CP->getAlign().value());
848 ID.AddInteger(I: CP->getOffset());
849 if (CP->isMachineConstantPoolEntry())
850 CP->getMachineCPVal()->addSelectionDAGCSEId(ID);
851 else
852 ID.AddPointer(Ptr: CP->getConstVal());
853 ID.AddInteger(I: CP->getTargetFlags());
854 break;
855 }
856 case ISD::TargetIndex: {
857 const TargetIndexSDNode *TI = cast<TargetIndexSDNode>(Val: N);
858 ID.AddInteger(I: TI->getIndex());
859 ID.AddInteger(I: TI->getOffset());
860 ID.AddInteger(I: TI->getTargetFlags());
861 break;
862 }
863 case ISD::LOAD: {
864 const LoadSDNode *LD = cast<LoadSDNode>(Val: N);
865 ID.AddInteger(I: LD->getMemoryVT().getRawBits());
866 ID.AddInteger(I: LD->getRawSubclassData());
867 ID.AddInteger(I: LD->getPointerInfo().getAddrSpace());
868 ID.AddInteger(I: LD->getMemOperand()->getFlags());
869 break;
870 }
871 case ISD::STORE: {
872 const StoreSDNode *ST = cast<StoreSDNode>(Val: N);
873 ID.AddInteger(I: ST->getMemoryVT().getRawBits());
874 ID.AddInteger(I: ST->getRawSubclassData());
875 ID.AddInteger(I: ST->getPointerInfo().getAddrSpace());
876 ID.AddInteger(I: ST->getMemOperand()->getFlags());
877 break;
878 }
879 case ISD::VP_LOAD: {
880 const VPLoadSDNode *ELD = cast<VPLoadSDNode>(Val: N);
881 ID.AddInteger(I: ELD->getMemoryVT().getRawBits());
882 ID.AddInteger(I: ELD->getRawSubclassData());
883 ID.AddInteger(I: ELD->getPointerInfo().getAddrSpace());
884 ID.AddInteger(I: ELD->getMemOperand()->getFlags());
885 break;
886 }
887 case ISD::VP_LOAD_FF: {
888 const auto *LD = cast<VPLoadFFSDNode>(Val: N);
889 ID.AddInteger(I: LD->getMemoryVT().getRawBits());
890 ID.AddInteger(I: LD->getRawSubclassData());
891 ID.AddInteger(I: LD->getPointerInfo().getAddrSpace());
892 ID.AddInteger(I: LD->getMemOperand()->getFlags());
893 break;
894 }
895 case ISD::VP_STORE: {
896 const VPStoreSDNode *EST = cast<VPStoreSDNode>(Val: N);
897 ID.AddInteger(I: EST->getMemoryVT().getRawBits());
898 ID.AddInteger(I: EST->getRawSubclassData());
899 ID.AddInteger(I: EST->getPointerInfo().getAddrSpace());
900 ID.AddInteger(I: EST->getMemOperand()->getFlags());
901 break;
902 }
903 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD: {
904 const VPStridedLoadSDNode *SLD = cast<VPStridedLoadSDNode>(Val: N);
905 ID.AddInteger(I: SLD->getMemoryVT().getRawBits());
906 ID.AddInteger(I: SLD->getRawSubclassData());
907 ID.AddInteger(I: SLD->getPointerInfo().getAddrSpace());
908 break;
909 }
910 case ISD::EXPERIMENTAL_VP_STRIDED_STORE: {
911 const VPStridedStoreSDNode *SST = cast<VPStridedStoreSDNode>(Val: N);
912 ID.AddInteger(I: SST->getMemoryVT().getRawBits());
913 ID.AddInteger(I: SST->getRawSubclassData());
914 ID.AddInteger(I: SST->getPointerInfo().getAddrSpace());
915 break;
916 }
917 case ISD::VP_GATHER: {
918 const VPGatherSDNode *EG = cast<VPGatherSDNode>(Val: N);
919 ID.AddInteger(I: EG->getMemoryVT().getRawBits());
920 ID.AddInteger(I: EG->getRawSubclassData());
921 ID.AddInteger(I: EG->getPointerInfo().getAddrSpace());
922 ID.AddInteger(I: EG->getMemOperand()->getFlags());
923 break;
924 }
925 case ISD::VP_SCATTER: {
926 const VPScatterSDNode *ES = cast<VPScatterSDNode>(Val: N);
927 ID.AddInteger(I: ES->getMemoryVT().getRawBits());
928 ID.AddInteger(I: ES->getRawSubclassData());
929 ID.AddInteger(I: ES->getPointerInfo().getAddrSpace());
930 ID.AddInteger(I: ES->getMemOperand()->getFlags());
931 break;
932 }
933 case ISD::MLOAD: {
934 const MaskedLoadSDNode *MLD = cast<MaskedLoadSDNode>(Val: N);
935 ID.AddInteger(I: MLD->getMemoryVT().getRawBits());
936 ID.AddInteger(I: MLD->getRawSubclassData());
937 ID.AddInteger(I: MLD->getPointerInfo().getAddrSpace());
938 ID.AddInteger(I: MLD->getMemOperand()->getFlags());
939 break;
940 }
941 case ISD::MSTORE: {
942 const MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(Val: N);
943 ID.AddInteger(I: MST->getMemoryVT().getRawBits());
944 ID.AddInteger(I: MST->getRawSubclassData());
945 ID.AddInteger(I: MST->getPointerInfo().getAddrSpace());
946 ID.AddInteger(I: MST->getMemOperand()->getFlags());
947 break;
948 }
949 case ISD::MGATHER: {
950 const MaskedGatherSDNode *MG = cast<MaskedGatherSDNode>(Val: N);
951 ID.AddInteger(I: MG->getMemoryVT().getRawBits());
952 ID.AddInteger(I: MG->getRawSubclassData());
953 ID.AddInteger(I: MG->getPointerInfo().getAddrSpace());
954 ID.AddInteger(I: MG->getMemOperand()->getFlags());
955 break;
956 }
957 case ISD::MSCATTER: {
958 const MaskedScatterSDNode *MS = cast<MaskedScatterSDNode>(Val: N);
959 ID.AddInteger(I: MS->getMemoryVT().getRawBits());
960 ID.AddInteger(I: MS->getRawSubclassData());
961 ID.AddInteger(I: MS->getPointerInfo().getAddrSpace());
962 ID.AddInteger(I: MS->getMemOperand()->getFlags());
963 break;
964 }
965 case ISD::ATOMIC_CMP_SWAP:
966 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
967 case ISD::ATOMIC_SWAP:
968 case ISD::ATOMIC_LOAD_ADD:
969 case ISD::ATOMIC_LOAD_SUB:
970 case ISD::ATOMIC_LOAD_AND:
971 case ISD::ATOMIC_LOAD_CLR:
972 case ISD::ATOMIC_LOAD_OR:
973 case ISD::ATOMIC_LOAD_XOR:
974 case ISD::ATOMIC_LOAD_NAND:
975 case ISD::ATOMIC_LOAD_MIN:
976 case ISD::ATOMIC_LOAD_MAX:
977 case ISD::ATOMIC_LOAD_UMIN:
978 case ISD::ATOMIC_LOAD_UMAX:
979 case ISD::ATOMIC_LOAD:
980 case ISD::ATOMIC_STORE: {
981 const AtomicSDNode *AT = cast<AtomicSDNode>(Val: N);
982 ID.AddInteger(I: AT->getMemoryVT().getRawBits());
983 ID.AddInteger(I: AT->getRawSubclassData());
984 ID.AddInteger(I: AT->getPointerInfo().getAddrSpace());
985 ID.AddInteger(I: AT->getMemOperand()->getFlags());
986 break;
987 }
988 case ISD::VECTOR_SHUFFLE: {
989 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Val: N)->getMask();
990 for (int M : Mask)
991 ID.AddInteger(I: M);
992 break;
993 }
994 case ISD::ADDRSPACECAST: {
995 const AddrSpaceCastSDNode *ASC = cast<AddrSpaceCastSDNode>(Val: N);
996 ID.AddInteger(I: ASC->getSrcAddressSpace());
997 ID.AddInteger(I: ASC->getDestAddressSpace());
998 break;
999 }
1000 case ISD::TargetBlockAddress:
1001 case ISD::BlockAddress: {
1002 const BlockAddressSDNode *BA = cast<BlockAddressSDNode>(Val: N);
1003 ID.AddPointer(Ptr: BA->getBlockAddress());
1004 ID.AddInteger(I: BA->getOffset());
1005 ID.AddInteger(I: BA->getTargetFlags());
1006 break;
1007 }
1008 case ISD::AssertAlign:
1009 ID.AddInteger(I: cast<AssertAlignSDNode>(Val: N)->getAlign().value());
1010 break;
1011 case ISD::PREFETCH:
1012 case ISD::INTRINSIC_VOID:
1013 case ISD::INTRINSIC_W_CHAIN:
1014 // Handled by MemIntrinsicSDNode check after the switch.
1015 break;
1016 case ISD::MDNODE_SDNODE:
1017 ID.AddPointer(Ptr: cast<MDNodeSDNode>(Val: N)->getMD());
1018 break;
1019 } // end switch (N->getOpcode())
1020
1021 // MemIntrinsic nodes could also have subclass data, address spaces, and flags
1022 // to check.
1023 if (auto *MN = dyn_cast<MemIntrinsicSDNode>(Val: N)) {
1024 ID.AddInteger(I: MN->getRawSubclassData());
1025 ID.AddInteger(I: MN->getMemoryVT().getRawBits());
1026 for (const MachineMemOperand *MMO : MN->memoperands()) {
1027 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
1028 ID.AddInteger(I: MMO->getFlags());
1029 }
1030 }
1031}
1032
1033/// AddNodeIDNode - Generic routine for adding a nodes info to the NodeID
1034/// data.
1035static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) {
1036 AddNodeIDOpcode(ID, OpC: N->getOpcode());
1037 // Add the return value info.
1038 AddNodeIDValueTypes(ID, VTList: N->getVTList());
1039 // Add the operand info.
1040 AddNodeIDOperands(ID, Ops: N->ops());
1041
1042 // Handle SDNode leafs with special info.
1043 AddNodeIDCustom(ID, N);
1044}
1045
1046//===----------------------------------------------------------------------===//
1047// SelectionDAG Class
1048//===----------------------------------------------------------------------===//
1049
1050/// doNotCSE - Return true if CSE should not be performed for this node.
1051static bool doNotCSE(SDNode *N) {
1052 if (N->getValueType(ResNo: 0) == MVT::Glue)
1053 return true; // Never CSE anything that produces a glue result.
1054
1055 switch (N->getOpcode()) {
1056 default: break;
1057 case ISD::HANDLENODE:
1058 case ISD::EH_LABEL:
1059 return true; // Never CSE these nodes.
1060 }
1061
1062 // Check that remaining values produced are not flags.
1063 for (unsigned i = 1, e = N->getNumValues(); i != e; ++i)
1064 if (N->getValueType(ResNo: i) == MVT::Glue)
1065 return true; // Never CSE anything that produces a glue result.
1066
1067 return false;
1068}
1069
1070/// Construct a DemandedElts mask which demands all elements of \p V.
1071/// If \p V is not a fixed-length vector, then this will return a single bit.
1072static APInt getDemandAllEltsMask(SDValue V) {
1073 EVT VT = V.getValueType();
1074 // Since the number of lanes in a scalable vector is unknown at compile time,
1075 // we track one bit which is implicitly broadcast to all lanes. This means
1076 // that all lanes in a scalable vector are considered demanded.
1077 return VT.isFixedLengthVector() ? APInt::getAllOnes(numBits: VT.getVectorNumElements())
1078 : APInt(1, 1);
1079}
1080
1081/// RemoveDeadNodes - This method deletes all unreachable nodes in the
1082/// SelectionDAG.
1083void SelectionDAG::RemoveDeadNodes() {
1084 // Create a dummy node (which is not added to allnodes), that adds a reference
1085 // to the root node, preventing it from being deleted.
1086 HandleSDNode Dummy(getRoot());
1087
1088 SmallVector<SDNode*, 128> DeadNodes;
1089
1090 // Add all obviously-dead nodes to the DeadNodes worklist.
1091 for (SDNode &Node : allnodes())
1092 if (Node.use_empty())
1093 DeadNodes.push_back(Elt: &Node);
1094
1095 RemoveDeadNodes(DeadNodes);
1096
1097 // If the root changed (e.g. it was a dead load, update the root).
1098 setRoot(Dummy.getValue());
1099}
1100
1101/// RemoveDeadNodes - This method deletes the unreachable nodes in the
1102/// given list, and any nodes that become unreachable as a result.
1103void SelectionDAG::RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes) {
1104
1105 // Process the worklist, deleting the nodes and adding their uses to the
1106 // worklist.
1107 while (!DeadNodes.empty()) {
1108 SDNode *N = DeadNodes.pop_back_val();
1109 // Skip to next node if we've already managed to delete the node. This could
1110 // happen if replacing a node causes a node previously added to the node to
1111 // be deleted.
1112 if (N->getOpcode() == ISD::DELETED_NODE)
1113 continue;
1114
1115 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1116 DUL->NodeDeleted(N, nullptr);
1117
1118 // Take the node out of the appropriate CSE map.
1119 RemoveNodeFromCSEMaps(N);
1120
1121 // Next, brutally remove the operand list. This is safe to do, as there are
1122 // no cycles in the graph.
1123 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
1124 SDUse &Use = *I++;
1125 SDNode *Operand = Use.getNode();
1126 Use.set(SDValue());
1127
1128 // Now that we removed this operand, see if there are no uses of it left.
1129 if (Operand->use_empty())
1130 DeadNodes.push_back(Elt: Operand);
1131 }
1132
1133 DeallocateNode(N);
1134 }
1135}
1136
1137void SelectionDAG::RemoveDeadNode(SDNode *N){
1138 SmallVector<SDNode*, 16> DeadNodes(1, N);
1139
1140 // Create a dummy node that adds a reference to the root node, preventing
1141 // it from being deleted. (This matters if the root is an operand of the
1142 // dead node.)
1143 HandleSDNode Dummy(getRoot());
1144
1145 RemoveDeadNodes(DeadNodes);
1146}
1147
1148void SelectionDAG::DeleteNode(SDNode *N) {
1149 // First take this out of the appropriate CSE map.
1150 RemoveNodeFromCSEMaps(N);
1151
1152 // Finally, remove uses due to operands of this node, remove from the
1153 // AllNodes list, and delete the node.
1154 DeleteNodeNotInCSEMaps(N);
1155}
1156
1157void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) {
1158 assert(N->getIterator() != AllNodes.begin() &&
1159 "Cannot delete the entry node!");
1160 assert(N->use_empty() && "Cannot delete a node that is not dead!");
1161
1162 // Drop all of the operands and decrement used node's use counts.
1163 N->DropOperands();
1164
1165 DeallocateNode(N);
1166}
1167
1168void SDDbgInfo::add(SDDbgValue *V, bool isParameter) {
1169 assert(!(V->isVariadic() && isParameter));
1170 if (isParameter)
1171 ByvalParmDbgValues.push_back(Elt: V);
1172 else
1173 DbgValues.push_back(Elt: V);
1174 for (const SDNode *Node : V->getSDNodes())
1175 if (Node)
1176 DbgValMap[Node].push_back(Elt: V);
1177}
1178
1179void SDDbgInfo::erase(const SDNode *Node) {
1180 DbgValMapType::iterator I = DbgValMap.find(Val: Node);
1181 if (I == DbgValMap.end())
1182 return;
1183 for (auto &Val: I->second)
1184 Val->setIsInvalidated();
1185 DbgValMap.erase(I);
1186}
1187
1188void SelectionDAG::DeallocateNode(SDNode *N) {
1189 // If we have operands, deallocate them.
1190 removeOperands(Node: N);
1191
1192 NodeAllocator.Deallocate(E: AllNodes.remove(IT: N));
1193
1194 // Set the opcode to DELETED_NODE to help catch bugs when node
1195 // memory is reallocated.
1196 // FIXME: There are places in SDag that have grown a dependency on the opcode
1197 // value in the released node.
1198 __asan_unpoison_memory_region(&N->NodeType, sizeof(N->NodeType));
1199 N->NodeType = ISD::DELETED_NODE;
1200
1201 // If any of the SDDbgValue nodes refer to this SDNode, invalidate
1202 // them and forget about that node.
1203 DbgInfo->erase(Node: N);
1204
1205 // Invalidate extra info.
1206 SDEI.erase(Val: N);
1207}
1208
1209#ifndef NDEBUG
1210/// VerifySDNode - Check the given SDNode. Aborts if it is invalid.
1211void SelectionDAG::verifyNode(SDNode *N) const {
1212 switch (N->getOpcode()) {
1213 default:
1214 if (N->isTargetOpcode())
1215 getSelectionDAGInfo().verifyTargetNode(*this, N);
1216 break;
1217 case ISD::BUILD_PAIR: {
1218 EVT VT = N->getValueType(0);
1219 assert(N->getNumValues() == 1 && "Too many results!");
1220 assert(!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) &&
1221 "Wrong return type!");
1222 assert(N->getNumOperands() == 2 && "Wrong number of operands!");
1223 assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() &&
1224 "Mismatched operand types!");
1225 assert(N->getOperand(0).getValueType().isInteger() == VT.isInteger() &&
1226 "Wrong operand type!");
1227 assert(VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() &&
1228 "Wrong return type size");
1229 break;
1230 }
1231 case ISD::BUILD_VECTOR: {
1232 assert(N->getNumValues() == 1 && "Too many results!");
1233 assert(N->getValueType(0).isVector() && "Wrong return type!");
1234 assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() &&
1235 "Wrong number of operands!");
1236 EVT EltVT = N->getValueType(0).getVectorElementType();
1237 for (const SDUse &Op : N->ops()) {
1238 assert((Op.getValueType() == EltVT ||
1239 (EltVT.isInteger() && Op.getValueType().isInteger() &&
1240 EltVT.bitsLE(Op.getValueType()))) &&
1241 "Wrong operand type!");
1242 assert(Op.getValueType() == N->getOperand(0).getValueType() &&
1243 "Operands must all have the same type");
1244 }
1245 break;
1246 }
1247 case ISD::SADDO:
1248 case ISD::UADDO:
1249 case ISD::SSUBO:
1250 case ISD::USUBO:
1251 assert(N->getNumValues() == 2 && "Wrong number of results!");
1252 assert(N->getVTList().NumVTs == 2 && N->getNumOperands() == 2 &&
1253 "Invalid add/sub overflow op!");
1254 assert(N->getVTList().VTs[0].isInteger() &&
1255 N->getVTList().VTs[1].isInteger() &&
1256 N->getOperand(0).getValueType() == N->getOperand(1).getValueType() &&
1257 N->getOperand(0).getValueType() == N->getVTList().VTs[0] &&
1258 "Binary operator types must match!");
1259 break;
1260 }
1261}
1262#endif // NDEBUG
1263
1264/// Insert a newly allocated node into the DAG.
1265///
1266/// Handles insertion into the all nodes list and CSE map, as well as
1267/// verification and other common operations when a new node is allocated.
1268void SelectionDAG::InsertNode(SDNode *N) {
1269 AllNodes.push_back(val: N);
1270#ifndef NDEBUG
1271 N->PersistentId = NextPersistentId++;
1272 verifyNode(N);
1273#endif
1274 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1275 DUL->NodeInserted(N);
1276}
1277
1278/// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that
1279/// correspond to it. This is useful when we're about to delete or repurpose
1280/// the node. We don't want future request for structurally identical nodes
1281/// to return N anymore.
1282bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) {
1283 bool Erased = false;
1284 switch (N->getOpcode()) {
1285 case ISD::HANDLENODE: return false; // noop.
1286 case ISD::CONDCODE:
1287 assert(CondCodeNodes[cast<CondCodeSDNode>(N)->get()] &&
1288 "Cond code doesn't exist!");
1289 Erased = CondCodeNodes[cast<CondCodeSDNode>(Val: N)->get()] != nullptr;
1290 CondCodeNodes[cast<CondCodeSDNode>(Val: N)->get()] = nullptr;
1291 break;
1292 case ISD::ExternalSymbol:
1293 Erased = ExternalSymbols.erase(Key: cast<ExternalSymbolSDNode>(Val: N)->getSymbol());
1294 break;
1295 case ISD::TargetExternalSymbol: {
1296 ExternalSymbolSDNode *ESN = cast<ExternalSymbolSDNode>(Val: N);
1297 Erased = TargetExternalSymbols.erase(x: std::pair<std::string, unsigned>(
1298 ESN->getSymbol(), ESN->getTargetFlags()));
1299 break;
1300 }
1301 case ISD::MCSymbol: {
1302 auto *MCSN = cast<MCSymbolSDNode>(Val: N);
1303 Erased = MCSymbols.erase(Val: MCSN->getMCSymbol());
1304 break;
1305 }
1306 case ISD::VALUETYPE: {
1307 EVT VT = cast<VTSDNode>(Val: N)->getVT();
1308 if (VT.isExtended()) {
1309 Erased = ExtendedValueTypeNodes.erase(x: VT);
1310 } else {
1311 Erased = ValueTypeNodes[VT.getSimpleVT().SimpleTy] != nullptr;
1312 ValueTypeNodes[VT.getSimpleVT().SimpleTy] = nullptr;
1313 }
1314 break;
1315 }
1316 default:
1317 // Remove it from the CSE Map.
1318 assert(N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!");
1319 assert(N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!");
1320 Erased = CSEMap.erase(N);
1321 break;
1322 }
1323#ifndef NDEBUG
1324 // Verify that the node was actually in one of the CSE maps, unless it has a
1325 // glue result (which cannot be CSE'd) or is one of the special cases that are
1326 // not subject to CSE.
1327 if (!Erased && N->getValueType(N->getNumValues()-1) != MVT::Glue &&
1328 !N->isMachineOpcode() && !doNotCSE(N)) {
1329 N->dump(this);
1330 dbgs() << "\n";
1331 llvm_unreachable("Node is not in map!");
1332 }
1333#endif
1334 return Erased;
1335}
1336
1337/// AddModifiedNodeToCSEMaps - The specified node has been removed from the CSE
1338/// maps and modified in place. Add it back to the CSE maps, unless an identical
1339/// node already exists, in which case transfer all its users to the existing
1340/// node. This transfer can potentially trigger recursive merging.
1341void
1342SelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N) {
1343 // For node types that aren't CSE'd, just act as if no identical node
1344 // already exists.
1345 if (!doNotCSE(N)) {
1346 SDNode *Existing = CSEMap.getOrInsert(N);
1347 if (Existing != N) {
1348 // If there was already an existing matching node, use ReplaceAllUsesWith
1349 // to replace the dead one with the existing one. This can cause
1350 // recursive merging of other unrelated nodes down the line.
1351 Existing->intersectFlagsWith(Flags: N->getFlags());
1352 if (auto *MemNode = dyn_cast<MemSDNode>(Val: Existing)) {
1353 ArrayRef<MachineMemOperand *> NewMMOs =
1354 cast<MemSDNode>(Val: N)->memoperands();
1355 // Range and cache hint metadata are not part of the DAG CSE key because
1356 // we prefer to CSE even when metadata does not match. Merge potentially
1357 // differing metadata conservatively.
1358 MemNode->refineMMOMetadata(NewMMOs);
1359 }
1360 ReplaceAllUsesWith(From: N, To: Existing);
1361
1362 // N is now dead. Inform the listeners and delete it.
1363 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1364 DUL->NodeDeleted(N, Existing);
1365 DeleteNodeNotInCSEMaps(N);
1366 return;
1367 }
1368 }
1369
1370 // If the node doesn't already exist, we updated it. Inform listeners.
1371 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1372 DUL->NodeUpdated(N);
1373}
1374
1375/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1376/// were replaced with those specified. If this node is never memoized,
1377/// return null, otherwise return a pointer to the slot it would take. If a
1378/// node already exists with these operands, the slot will be non-null.
1379SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op,
1380 FoldingSetInsertToken &InsertToken) {
1381 if (doNotCSE(N))
1382 return nullptr;
1383
1384 SDValue Ops[] = { Op };
1385 FoldingSetNodeID ID;
1386 AddNodeIDNode(ID, OpC: N->getOpcode(), VTList: N->getVTList(), OpList: Ops);
1387 AddNodeIDCustom(ID, N);
1388 SDNode *Node = lookupNode(ID, DL: SDLoc(N), InsertToken);
1389 if (Node)
1390 Node->intersectFlagsWith(Flags: N->getFlags());
1391 return Node;
1392}
1393
1394/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1395/// were replaced with those specified. If this node is never memoized,
1396/// return null, otherwise return a pointer to the slot it would take. If a
1397/// node already exists with these operands, the slot will be non-null.
1398SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op1, SDValue Op2,
1399 FoldingSetInsertToken &InsertToken) {
1400 if (doNotCSE(N))
1401 return nullptr;
1402
1403 SDValue Ops[] = { Op1, Op2 };
1404 FoldingSetNodeID ID;
1405 AddNodeIDNode(ID, OpC: N->getOpcode(), VTList: N->getVTList(), OpList: Ops);
1406 AddNodeIDCustom(ID, N);
1407 SDNode *Node = lookupNode(ID, DL: SDLoc(N), InsertToken);
1408 if (Node)
1409 Node->intersectFlagsWith(Flags: N->getFlags());
1410 return Node;
1411}
1412
1413/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1414/// were replaced with those specified. If this node is never memoized,
1415/// return null, otherwise return a pointer to the slot it would take. If a
1416/// node already exists with these operands, the slot will be non-null.
1417SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, ArrayRef<SDValue> Ops,
1418 FoldingSetInsertToken &InsertToken) {
1419 if (doNotCSE(N))
1420 return nullptr;
1421
1422 FoldingSetNodeID ID;
1423 AddNodeIDNode(ID, OpC: N->getOpcode(), VTList: N->getVTList(), OpList: Ops);
1424 AddNodeIDCustom(ID, N);
1425 SDNode *Node = lookupNode(ID, DL: SDLoc(N), InsertToken);
1426 if (Node)
1427 Node->intersectFlagsWith(Flags: N->getFlags());
1428 return Node;
1429}
1430
1431Align SelectionDAG::getEVTAlign(EVT VT) const {
1432 Type *Ty = VT == MVT::iPTR ? PointerType::get(C&: *getContext(), AddressSpace: 0)
1433 : VT.getTypeForEVT(Context&: *getContext());
1434
1435 return getDataLayout().getABITypeAlign(Ty);
1436}
1437
1438// EntryNode could meaningfully have debug info if we can find it...
1439SelectionDAG::SelectionDAG(const TargetMachine &tm, CodeGenOptLevel OL)
1440 : TM(tm), OptLevel(OL), EntryNode(ISD::EntryToken, 0, DebugLoc(),
1441 getVTList(VT1: MVT::Other, VT2: MVT::Glue)),
1442 Root(getEntryNode()) {
1443 InsertNode(N: &EntryNode);
1444 DbgInfo = new SDDbgInfo();
1445}
1446
1447void SelectionDAG::init(MachineFunction &NewMF,
1448 OptimizationRemarkEmitter &NewORE, Pass *PassPtr,
1449 const TargetLibraryInfo *LibraryInfo,
1450 const LibcallLoweringInfo *LibcallsInfo,
1451 UniformityInfo *NewUA, ProfileSummaryInfo *PSIin,
1452 BlockFrequencyInfo *BFIin, MachineModuleInfo &MMIin,
1453 FunctionVarLocs const *VarLocs) {
1454 MF = &NewMF;
1455 SDAGISelPass = PassPtr;
1456 ORE = &NewORE;
1457 TLI = getSubtarget().getTargetLowering();
1458 TSI = getSubtarget().getSelectionDAGInfo();
1459 LibInfo = LibraryInfo;
1460 Libcalls = LibcallsInfo;
1461 Context = &MF->getFunction().getContext();
1462 UA = NewUA;
1463 PSI = PSIin;
1464 BFI = BFIin;
1465 MMI = &MMIin;
1466 FnVarLocs = VarLocs;
1467}
1468
1469SelectionDAG::~SelectionDAG() {
1470 assert(!UpdateListeners && "Dangling registered DAGUpdateListeners");
1471 allnodes_clear();
1472 OperandRecycler.clear(OperandAllocator);
1473 delete DbgInfo;
1474}
1475
1476bool SelectionDAG::shouldOptForSize() const {
1477 return llvm::shouldOptimizeForSize(BB: FLI->MBB->getBasicBlock(), PSI, BFI);
1478}
1479
1480void SelectionDAG::allnodes_clear() {
1481 assert(&*AllNodes.begin() == &EntryNode);
1482 AllNodes.remove(IT: AllNodes.begin());
1483 while (!AllNodes.empty())
1484 DeallocateNode(N: &AllNodes.front());
1485#ifndef NDEBUG
1486 NextPersistentId = 0;
1487#endif
1488}
1489
1490SDNode *SelectionDAG::lookupNode(const FoldingSetNodeID &ID,
1491 FoldingSetInsertToken &InsertToken) {
1492 SDNode *N = CSEMap.lookup(ID, Token&: InsertToken);
1493 if (N) {
1494 switch (N->getOpcode()) {
1495 default: break;
1496 case ISD::Constant:
1497 case ISD::ConstantFP:
1498 llvm_unreachable("Querying for Constant and ConstantFP nodes requires "
1499 "debug location. Use another overload.");
1500 }
1501 }
1502 return N;
1503}
1504
1505SDNode *SelectionDAG::lookupNode(const FoldingSetNodeID &ID, const SDLoc &DL,
1506 FoldingSetInsertToken &InsertToken) {
1507 SDNode *N = CSEMap.lookup(ID, Token&: InsertToken);
1508 if (N) {
1509 switch (N->getOpcode()) {
1510 case ISD::Constant:
1511 case ISD::ConstantFP:
1512 // Erase debug location from the node if the node is used at several
1513 // different places. Do not propagate one location to all uses as it
1514 // will cause a worse single stepping debugging experience.
1515 if (N->getDebugLoc() != DL.getDebugLoc())
1516 N->setDebugLoc(DebugLoc());
1517 break;
1518 default:
1519 // When the node's point of use is located earlier in the instruction
1520 // sequence than its prior point of use, update its debug info to the
1521 // earlier location.
1522 if (DL.getIROrder() && DL.getIROrder() < N->getIROrder())
1523 N->setDebugLoc(DL.getDebugLoc());
1524 break;
1525 }
1526 }
1527 return N;
1528}
1529
1530void SelectionDAG::clear() {
1531 allnodes_clear();
1532 OperandRecycler.clear(OperandAllocator);
1533 OperandAllocator.Reset();
1534 CSEMap.clear();
1535
1536 ExtendedValueTypeNodes.clear();
1537 ExternalSymbols.clear();
1538 TargetExternalSymbols.clear();
1539 MCSymbols.clear();
1540 SDEI.clear();
1541 llvm::fill(Range&: CondCodeNodes, Value: nullptr);
1542 llvm::fill(Range&: ValueTypeNodes, Value: nullptr);
1543
1544 EntryNode.UseList = nullptr;
1545 InsertNode(N: &EntryNode);
1546 Root = getEntryNode();
1547 DbgInfo->clear();
1548}
1549
1550SDValue SelectionDAG::getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT) {
1551 return VT.bitsGT(VT: Op.getValueType())
1552 ? getNode(Opcode: ISD::FP_EXTEND, DL, VT, Operand: Op)
1553 : getNode(Opcode: ISD::FP_ROUND, DL, VT, N1: Op,
1554 N2: getIntPtrConstant(Val: 0, DL, /*isTarget=*/true));
1555}
1556
1557std::pair<SDValue, SDValue>
1558SelectionDAG::getStrictFPExtendOrRound(SDValue Op, SDValue Chain,
1559 const SDLoc &DL, EVT VT) {
1560 assert(!VT.bitsEq(Op.getValueType()) &&
1561 "Strict no-op FP extend/round not allowed.");
1562 SDValue Res =
1563 VT.bitsGT(VT: Op.getValueType())
1564 ? getNode(Opcode: ISD::STRICT_FP_EXTEND, DL, ResultTys: {VT, MVT::Other}, Ops: {Chain, Op})
1565 : getNode(Opcode: ISD::STRICT_FP_ROUND, DL, ResultTys: {VT, MVT::Other},
1566 Ops: {Chain, Op, getIntPtrConstant(Val: 0, DL, /*isTarget=*/true)});
1567
1568 return std::pair<SDValue, SDValue>(Res, SDValue(Res.getNode(), 1));
1569}
1570
1571SDValue SelectionDAG::getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1572 return VT.bitsGT(VT: Op.getValueType()) ?
1573 getNode(Opcode: ISD::ANY_EXTEND, DL, VT, Operand: Op) :
1574 getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Op);
1575}
1576
1577SDValue SelectionDAG::getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1578 return VT.bitsGT(VT: Op.getValueType()) ?
1579 getNode(Opcode: ISD::SIGN_EXTEND, DL, VT, Operand: Op) :
1580 getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Op);
1581}
1582
1583SDValue SelectionDAG::getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1584 return VT.bitsGT(VT: Op.getValueType()) ?
1585 getNode(Opcode: ISD::ZERO_EXTEND, DL, VT, Operand: Op) :
1586 getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Op);
1587}
1588
1589SDValue SelectionDAG::getBitcastedAnyExtOrTrunc(SDValue Op, const SDLoc &DL,
1590 EVT VT) {
1591 assert(!VT.isVector());
1592 auto Type = Op.getValueType();
1593 SDValue DestOp;
1594 if (Type == VT)
1595 return Op;
1596 auto Size = Op.getValueSizeInBits();
1597 DestOp = getBitcast(VT: EVT::getIntegerVT(Context&: *Context, BitWidth: Size), V: Op);
1598 if (DestOp.getValueType() == VT)
1599 return DestOp;
1600
1601 return getAnyExtOrTrunc(Op: DestOp, DL, VT);
1602}
1603
1604SDValue SelectionDAG::getBitcastedSExtOrTrunc(SDValue Op, const SDLoc &DL,
1605 EVT VT) {
1606 assert(!VT.isVector());
1607 auto Type = Op.getValueType();
1608 SDValue DestOp;
1609 if (Type == VT)
1610 return Op;
1611 auto Size = Op.getValueSizeInBits();
1612 DestOp = getBitcast(VT: MVT::getIntegerVT(BitWidth: Size), V: Op);
1613 if (DestOp.getValueType() == VT)
1614 return DestOp;
1615
1616 return getSExtOrTrunc(Op: DestOp, DL, VT);
1617}
1618
1619SDValue SelectionDAG::getBitcastedZExtOrTrunc(SDValue Op, const SDLoc &DL,
1620 EVT VT) {
1621 assert(!VT.isVector());
1622 auto Type = Op.getValueType();
1623 SDValue DestOp;
1624 if (Type == VT)
1625 return Op;
1626 auto Size = Op.getValueSizeInBits();
1627 DestOp = getBitcast(VT: MVT::getIntegerVT(BitWidth: Size), V: Op);
1628 if (DestOp.getValueType() == VT)
1629 return DestOp;
1630
1631 return getZExtOrTrunc(Op: DestOp, DL, VT);
1632}
1633
1634SDValue SelectionDAG::getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT,
1635 EVT OpVT) {
1636 if (VT.bitsLE(VT: Op.getValueType()))
1637 return getNode(Opcode: ISD::TRUNCATE, DL: SL, VT, Operand: Op);
1638
1639 TargetLowering::BooleanContent BType = TLI->getBooleanContents(Type: OpVT);
1640 return getNode(Opcode: TLI->getExtendForContent(Content: BType), DL: SL, VT, Operand: Op);
1641}
1642
1643SDValue SelectionDAG::getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) {
1644 EVT OpVT = Op.getValueType();
1645 assert(VT.isInteger() && OpVT.isInteger() &&
1646 "Cannot getZeroExtendInReg FP types");
1647 assert(VT.isVector() == OpVT.isVector() &&
1648 "getZeroExtendInReg type should be vector iff the operand "
1649 "type is vector!");
1650 assert((!VT.isVector() ||
1651 VT.getVectorElementCount() == OpVT.getVectorElementCount()) &&
1652 "Vector element counts must match in getZeroExtendInReg");
1653 assert(VT.getScalarType().bitsLE(OpVT.getScalarType()) && "Not extending!");
1654 if (OpVT == VT)
1655 return Op;
1656 // TODO: Use computeKnownBits instead of AssertZext.
1657 if (Op.getOpcode() == ISD::AssertZext && cast<VTSDNode>(Val: Op.getOperand(i: 1))
1658 ->getVT()
1659 .getScalarType()
1660 .bitsLE(VT: VT.getScalarType()))
1661 return Op;
1662 APInt Imm = APInt::getLowBitsSet(numBits: OpVT.getScalarSizeInBits(),
1663 loBitsSet: VT.getScalarSizeInBits());
1664 return getNode(Opcode: ISD::AND, DL, VT: OpVT, N1: Op, N2: getConstant(Val: Imm, DL, VT: OpVT));
1665}
1666
1667SDValue SelectionDAG::getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1668 // Only unsigned pointer semantics are supported right now. In the future this
1669 // might delegate to TLI to check pointer signedness.
1670 return getZExtOrTrunc(Op, DL, VT);
1671}
1672
1673SDValue SelectionDAG::getPtrExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) {
1674 // Only unsigned pointer semantics are supported right now. In the future this
1675 // might delegate to TLI to check pointer signedness.
1676 return getZeroExtendInReg(Op, DL, VT);
1677}
1678
1679SDValue SelectionDAG::getNegative(SDValue Val, const SDLoc &DL, EVT VT) {
1680 return getNode(Opcode: ISD::SUB, DL, VT, N1: getConstant(Val: 0, DL, VT), N2: Val);
1681}
1682
1683/// getNOT - Create a bitwise NOT operation as (XOR Val, -1).
1684SDValue SelectionDAG::getNOT(const SDLoc &DL, SDValue Val, EVT VT) {
1685 return getNode(Opcode: ISD::XOR, DL, VT, N1: Val, N2: getAllOnesConstant(DL, VT));
1686}
1687
1688SDValue SelectionDAG::getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT) {
1689 SDValue TrueValue = getBoolConstant(V: true, DL, VT, OpVT: VT);
1690 return getNode(Opcode: ISD::XOR, DL, VT, N1: Val, N2: TrueValue);
1691}
1692
1693SDValue SelectionDAG::getBoolConstant(bool V, const SDLoc &DL, EVT VT,
1694 EVT OpVT) {
1695 if (!V)
1696 return getConstant(Val: 0, DL, VT);
1697
1698 switch (TLI->getBooleanContents(Type: OpVT)) {
1699 case TargetLowering::ZeroOrOneBooleanContent:
1700 case TargetLowering::UndefinedBooleanContent:
1701 return getConstant(Val: 1, DL, VT);
1702 case TargetLowering::ZeroOrNegativeOneBooleanContent:
1703 return getAllOnesConstant(DL, VT);
1704 }
1705 llvm_unreachable("Unexpected boolean content enum!");
1706}
1707
1708SDValue SelectionDAG::getConstant(uint64_t Val, const SDLoc &DL, EVT VT,
1709 bool isT, bool isO) {
1710 return getConstant(Val: APInt(VT.getScalarSizeInBits(), Val, /*isSigned=*/false),
1711 DL, VT, isTarget: isT, isOpaque: isO);
1712}
1713
1714SDValue SelectionDAG::getConstant(const APInt &Val, const SDLoc &DL, EVT VT,
1715 bool isT, bool isO) {
1716 return getConstant(Val: *ConstantInt::get(Context&: *Context, V: Val), DL, VT, isTarget: isT, isOpaque: isO);
1717}
1718
1719SDValue SelectionDAG::getConstant(const ConstantInt &Val, const SDLoc &DL,
1720 EVT VT, bool isT, bool isO) {
1721 assert(VT.isInteger() && "Cannot create FP integer constant!");
1722
1723 EVT EltVT = VT.getScalarType();
1724 const ConstantInt *Elt = &Val;
1725
1726 // Vector splats are explicit within the DAG, with ConstantSDNode holding the
1727 // to-be-splatted scalar ConstantInt.
1728 if (isa<VectorType>(Val: Elt->getType()))
1729 Elt = ConstantInt::get(Context&: *getContext(), V: Elt->getValue());
1730
1731 // In some cases the vector type is legal but the element type is illegal and
1732 // needs to be promoted, for example v8i8 on ARM. In this case, promote the
1733 // inserted value (the type does not need to match the vector element type).
1734 // Any extra bits introduced will be truncated away.
1735 if (VT.isVector() && TLI->getTypeAction(Context&: *getContext(), VT: EltVT) ==
1736 TargetLowering::TypePromoteInteger) {
1737 EltVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT: EltVT);
1738 APInt NewVal;
1739 if (TLI->isSExtCheaperThanZExt(FromTy: VT.getScalarType(), ToTy: EltVT))
1740 NewVal = Elt->getValue().sextOrTrunc(width: EltVT.getSizeInBits());
1741 else
1742 NewVal = Elt->getValue().zextOrTrunc(width: EltVT.getSizeInBits());
1743 Elt = ConstantInt::get(Context&: *getContext(), V: NewVal);
1744 }
1745 // In other cases the element type is illegal and needs to be expanded, for
1746 // example v2i64 on MIPS32. In this case, find the nearest legal type, split
1747 // the value into n parts and use a vector type with n-times the elements.
1748 // Then bitcast to the type requested.
1749 // Legalizing constants too early makes the DAGCombiner's job harder so we
1750 // only legalize if the DAG tells us we must produce legal types.
1751 else if (NewNodesMustHaveLegalTypes && VT.isVector() &&
1752 TLI->getTypeAction(Context&: *getContext(), VT: EltVT) ==
1753 TargetLowering::TypeExpandInteger) {
1754 const APInt &NewVal = Elt->getValue();
1755 EVT ViaEltVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT: EltVT);
1756 unsigned ViaEltSizeInBits = ViaEltVT.getSizeInBits();
1757
1758 // For scalable vectors, try to use a SPLAT_VECTOR_PARTS node.
1759 if (VT.isScalableVector() ||
1760 TLI->isOperationLegal(Op: ISD::SPLAT_VECTOR, VT)) {
1761 assert(EltVT.getSizeInBits() % ViaEltSizeInBits == 0 &&
1762 "Can only handle an even split!");
1763 unsigned Parts = EltVT.getSizeInBits() / ViaEltSizeInBits;
1764
1765 SmallVector<SDValue, 2> ScalarParts;
1766 for (unsigned i = 0; i != Parts; ++i)
1767 ScalarParts.push_back(Elt: getConstant(
1768 Val: NewVal.extractBits(numBits: ViaEltSizeInBits, bitPosition: i * ViaEltSizeInBits), DL,
1769 VT: ViaEltVT, isT, isO));
1770
1771 return getNode(Opcode: ISD::SPLAT_VECTOR_PARTS, DL, VT, Ops: ScalarParts);
1772 }
1773
1774 unsigned ViaVecNumElts = VT.getSizeInBits() / ViaEltSizeInBits;
1775 EVT ViaVecVT = EVT::getVectorVT(Context&: *getContext(), VT: ViaEltVT, NumElements: ViaVecNumElts);
1776
1777 // Check the temporary vector is the correct size. If this fails then
1778 // getTypeToTransformTo() probably returned a type whose size (in bits)
1779 // isn't a power-of-2 factor of the requested type size.
1780 assert(ViaVecVT.getSizeInBits() == VT.getSizeInBits());
1781
1782 SmallVector<SDValue, 2> EltParts;
1783 for (unsigned i = 0; i < ViaVecNumElts / VT.getVectorNumElements(); ++i)
1784 EltParts.push_back(Elt: getConstant(
1785 Val: NewVal.extractBits(numBits: ViaEltSizeInBits, bitPosition: i * ViaEltSizeInBits), DL,
1786 VT: ViaEltVT, isT, isO));
1787
1788 // EltParts is currently in little endian order. If we actually want
1789 // big-endian order then reverse it now.
1790 if (getDataLayout().isBigEndian())
1791 std::reverse(first: EltParts.begin(), last: EltParts.end());
1792
1793 // The elements must be reversed when the element order is different
1794 // to the endianness of the elements (because the BITCAST is itself a
1795 // vector shuffle in this situation). However, we do not need any code to
1796 // perform this reversal because getConstant() is producing a vector
1797 // splat.
1798 // This situation occurs in MIPS MSA.
1799
1800 SmallVector<SDValue, 8> Ops;
1801 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i)
1802 llvm::append_range(C&: Ops, R&: EltParts);
1803
1804 SDValue V =
1805 getNode(Opcode: ISD::BITCAST, DL, VT, Operand: getBuildVector(VT: ViaVecVT, DL, Ops));
1806 return V;
1807 }
1808
1809 assert(Elt->getBitWidth() == EltVT.getSizeInBits() &&
1810 "APInt size does not match type size!");
1811 unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant;
1812 SDVTList VTs = getVTList(VT: EltVT);
1813 FoldingSetNodeID ID;
1814 AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: {});
1815 ID.AddPointer(Ptr: Elt);
1816 ID.AddBoolean(B: isO);
1817 FoldingSetInsertToken InsertToken;
1818 SDNode *N = nullptr;
1819 if ((N = lookupNode(ID, DL, InsertToken)))
1820 if (!VT.isVector())
1821 return SDValue(N, 0);
1822
1823 if (!N) {
1824 N = newSDNode<ConstantSDNode>(Args&: isT, Args&: isO, Args&: Elt, Args&: VTs);
1825 if (!isT)
1826 N->setDebugLoc(DL.getDebugLoc());
1827 CSEMap.insert(N, Token: InsertToken);
1828 InsertNode(N);
1829 NewSDValueDbgMsg(V: SDValue(N, 0), Msg: "Creating constant: ", G: this);
1830 }
1831
1832 SDValue Result(N, 0);
1833 if (VT.isVector())
1834 Result = getSplat(VT, DL, Op: Result);
1835 return Result;
1836}
1837
1838SDValue SelectionDAG::getSignedConstant(int64_t Val, const SDLoc &DL, EVT VT,
1839 bool isT, bool isO) {
1840 unsigned Size = VT.getScalarSizeInBits();
1841 return getConstant(Val: APInt(Size, Val, /*isSigned=*/true), DL, VT, isT, isO);
1842}
1843
1844SDValue SelectionDAG::getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget,
1845 bool IsOpaque) {
1846 return getConstant(Val: APInt::getAllOnes(numBits: VT.getScalarSizeInBits()), DL, VT,
1847 isT: IsTarget, isO: IsOpaque);
1848}
1849
1850SDValue SelectionDAG::getIntPtrConstant(uint64_t Val, const SDLoc &DL,
1851 bool isTarget) {
1852 return getConstant(Val, DL, VT: TLI->getPointerTy(DL: getDataLayout()), isT: isTarget);
1853}
1854
1855SDValue SelectionDAG::getShiftAmountConstant(uint64_t Val, EVT VT,
1856 const SDLoc &DL) {
1857 assert(VT.isInteger() && "Shift amount is not an integer type!");
1858 EVT ShiftVT = TLI->getShiftAmountTy(LHSTy: VT, DL: getDataLayout());
1859 return getConstant(Val, DL, VT: ShiftVT);
1860}
1861
1862SDValue SelectionDAG::getShiftAmountConstant(const APInt &Val, EVT VT,
1863 const SDLoc &DL) {
1864 assert(Val.ult(VT.getScalarSizeInBits()) && "Out of range shift");
1865 return getShiftAmountConstant(Val: Val.getZExtValue(), VT, DL);
1866}
1867
1868SDValue SelectionDAG::getVectorIdxConstant(uint64_t Val, const SDLoc &DL,
1869 bool isTarget) {
1870 return getConstant(Val, DL, VT: TLI->getVectorIdxTy(DL: getDataLayout()), isT: isTarget);
1871}
1872
1873SDValue SelectionDAG::getConstantFP(const APFloat &V, const SDLoc &DL, EVT VT,
1874 bool isTarget) {
1875 return getConstantFP(V: *ConstantFP::get(Context&: *getContext(), V), DL, VT, isTarget);
1876}
1877
1878SDValue SelectionDAG::getConstantFP(const ConstantFP &V, const SDLoc &DL,
1879 EVT VT, bool isTarget) {
1880 assert(VT.isFloatingPoint() && "Cannot create integer FP constant!");
1881
1882 EVT EltVT = VT.getScalarType();
1883 const ConstantFP *Elt = &V;
1884
1885 // Vector splats are explicit within the DAG, with ConstantFPSDNode holding
1886 // the to-be-splatted scalar ConstantFP.
1887 if (isa<VectorType>(Val: Elt->getType()))
1888 Elt = ConstantFP::get(Context&: *getContext(), V: Elt->getValue());
1889
1890 // Do the map lookup using the actual bit pattern for the floating point
1891 // value, so that we don't have problems with 0.0 comparing equal to -0.0, and
1892 // we don't have issues with SNANs.
1893 unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP;
1894 SDVTList VTs = getVTList(VT: EltVT);
1895 FoldingSetNodeID ID;
1896 AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: {});
1897 ID.AddPointer(Ptr: Elt);
1898 FoldingSetInsertToken InsertToken;
1899 SDNode *N = nullptr;
1900 if ((N = lookupNode(ID, DL, InsertToken)))
1901 if (!VT.isVector())
1902 return SDValue(N, 0);
1903
1904 if (!N) {
1905 N = newSDNode<ConstantFPSDNode>(Args&: isTarget, Args&: Elt, Args&: VTs);
1906 CSEMap.insert(N, Token: InsertToken);
1907 InsertNode(N);
1908 }
1909
1910 SDValue Result(N, 0);
1911 if (VT.isVector())
1912 Result = getSplat(VT, DL, Op: Result);
1913 NewSDValueDbgMsg(V: Result, Msg: "Creating fp constant: ", G: this);
1914 return Result;
1915}
1916
1917SDValue SelectionDAG::getConstantFP(double Val, const SDLoc &DL, EVT VT,
1918 bool isTarget) {
1919 EVT EltVT = VT.getScalarType();
1920 if (EltVT == MVT::f32)
1921 return getConstantFP(V: APFloat((float)Val), DL, VT, isTarget);
1922 if (EltVT == MVT::f64)
1923 return getConstantFP(V: APFloat(Val), DL, VT, isTarget);
1924 if (EltVT == MVT::f80 || EltVT == MVT::f128 || EltVT == MVT::ppcf128 ||
1925 EltVT == MVT::f16 || EltVT == MVT::bf16) {
1926 bool Ignored;
1927 APFloat APF = APFloat(Val);
1928 APF.convert(ToSemantics: EltVT.getFltSemantics(), RM: APFloat::rmNearestTiesToEven,
1929 losesInfo: &Ignored);
1930 return getConstantFP(V: APF, DL, VT, isTarget);
1931 }
1932 llvm_unreachable("Unsupported type in getConstantFP");
1933}
1934
1935SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, const SDLoc &DL,
1936 EVT VT, int64_t Offset, bool isTargetGA,
1937 unsigned TargetFlags) {
1938 assert((TargetFlags == 0 || isTargetGA) &&
1939 "Cannot set target flags on target-independent globals");
1940
1941 // Truncate (with sign-extension) the offset value to the pointer size.
1942 unsigned BitWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType());
1943 if (BitWidth < 64)
1944 Offset = SignExtend64(X: Offset, B: BitWidth);
1945
1946 unsigned Opc;
1947 if (GV->isThreadLocal())
1948 Opc = isTargetGA ? ISD::TargetGlobalTLSAddress : ISD::GlobalTLSAddress;
1949 else
1950 Opc = isTargetGA ? ISD::TargetGlobalAddress : ISD::GlobalAddress;
1951
1952 SDVTList VTs = getVTList(VT);
1953 FoldingSetNodeID ID;
1954 AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: {});
1955 ID.AddPointer(Ptr: GV);
1956 ID.AddInteger(I: Offset);
1957 ID.AddInteger(I: TargetFlags);
1958 FoldingSetInsertToken InsertToken;
1959 if (SDNode *E = lookupNode(ID, DL, InsertToken))
1960 return SDValue(E, 0);
1961
1962 auto *N = newSDNode<GlobalAddressSDNode>(
1963 Args&: Opc, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: GV, Args&: VTs, Args&: Offset, Args&: TargetFlags);
1964 CSEMap.insert(N, Token: InsertToken);
1965 InsertNode(N);
1966 return SDValue(N, 0);
1967}
1968
1969SDValue SelectionDAG::getDeactivationSymbol(const GlobalValue *GV) {
1970 SDVTList VTs = getVTList(VT: MVT::Untyped);
1971 FoldingSetNodeID ID;
1972 AddNodeIDNode(ID, OpC: ISD::DEACTIVATION_SYMBOL, VTList: VTs, OpList: {});
1973 ID.AddPointer(Ptr: GV);
1974 FoldingSetInsertToken InsertToken;
1975 if (SDNode *E = lookupNode(ID, DL: SDLoc(), InsertToken))
1976 return SDValue(E, 0);
1977
1978 auto *N = newSDNode<DeactivationSymbolSDNode>(Args&: GV, Args&: VTs);
1979 CSEMap.insert(N, Token: InsertToken);
1980 InsertNode(N);
1981 return SDValue(N, 0);
1982}
1983
1984SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) {
1985 unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex;
1986 SDVTList VTs = getVTList(VT);
1987 FoldingSetNodeID ID;
1988 AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: {});
1989 ID.AddInteger(I: FI);
1990 FoldingSetInsertToken InsertToken;
1991 if (SDNode *E = lookupNode(ID, InsertToken))
1992 return SDValue(E, 0);
1993
1994 auto *N = newSDNode<FrameIndexSDNode>(Args&: FI, Args&: VTs, Args&: isTarget);
1995 CSEMap.insert(N, Token: InsertToken);
1996 InsertNode(N);
1997 return SDValue(N, 0);
1998}
1999
2000SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget,
2001 unsigned TargetFlags) {
2002 assert((TargetFlags == 0 || isTarget) &&
2003 "Cannot set target flags on target-independent jump tables");
2004 unsigned Opc = isTarget ? ISD::TargetJumpTable : ISD::JumpTable;
2005 SDVTList VTs = getVTList(VT);
2006 FoldingSetNodeID ID;
2007 AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: {});
2008 ID.AddInteger(I: JTI);
2009 ID.AddInteger(I: TargetFlags);
2010 FoldingSetInsertToken InsertToken;
2011 if (SDNode *E = lookupNode(ID, InsertToken))
2012 return SDValue(E, 0);
2013
2014 auto *N = newSDNode<JumpTableSDNode>(Args&: JTI, Args&: VTs, Args&: isTarget, Args&: TargetFlags);
2015 CSEMap.insert(N, Token: InsertToken);
2016 InsertNode(N);
2017 return SDValue(N, 0);
2018}
2019
2020SDValue SelectionDAG::getJumpTableDebugInfo(int JTI, SDValue Chain,
2021 const SDLoc &DL) {
2022 EVT PTy = getTargetLoweringInfo().getPointerTy(DL: getDataLayout());
2023 return getNode(Opcode: ISD::JUMP_TABLE_DEBUG_INFO, DL, VT: MVT::Other, N1: Chain,
2024 N2: getTargetConstant(Val: static_cast<uint64_t>(JTI), DL, VT: PTy, isOpaque: true));
2025}
2026
2027SDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT,
2028 MaybeAlign Alignment, int Offset,
2029 bool isTarget, unsigned TargetFlags) {
2030 assert((TargetFlags == 0 || isTarget) &&
2031 "Cannot set target flags on target-independent globals");
2032 if (!Alignment)
2033 Alignment = shouldOptForSize()
2034 ? getDataLayout().getABITypeAlign(Ty: C->getType())
2035 : getDataLayout().getPrefTypeAlign(Ty: C->getType());
2036 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
2037 SDVTList VTs = getVTList(VT);
2038 FoldingSetNodeID ID;
2039 AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: {});
2040 ID.AddInteger(I: Alignment->value());
2041 ID.AddInteger(I: Offset);
2042 ID.AddPointer(Ptr: C);
2043 ID.AddInteger(I: TargetFlags);
2044 FoldingSetInsertToken InsertToken;
2045 if (SDNode *E = lookupNode(ID, InsertToken))
2046 return SDValue(E, 0);
2047
2048 auto *N = newSDNode<ConstantPoolSDNode>(Args&: isTarget, Args&: C, Args&: VTs, Args&: Offset, Args&: *Alignment,
2049 Args&: TargetFlags);
2050 CSEMap.insert(N, Token: InsertToken);
2051 InsertNode(N);
2052 SDValue V = SDValue(N, 0);
2053 NewSDValueDbgMsg(V, Msg: "Creating new constant pool: ", G: this);
2054 return V;
2055}
2056
2057SDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT,
2058 MaybeAlign Alignment, int Offset,
2059 bool isTarget, unsigned TargetFlags) {
2060 assert((TargetFlags == 0 || isTarget) &&
2061 "Cannot set target flags on target-independent globals");
2062 if (!Alignment)
2063 Alignment = getDataLayout().getPrefTypeAlign(Ty: C->getType());
2064 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
2065 SDVTList VTs = getVTList(VT);
2066 FoldingSetNodeID ID;
2067 AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: {});
2068 ID.AddInteger(I: Alignment->value());
2069 ID.AddInteger(I: Offset);
2070 C->addSelectionDAGCSEId(ID);
2071 ID.AddInteger(I: TargetFlags);
2072 FoldingSetInsertToken InsertToken;
2073 if (SDNode *E = lookupNode(ID, InsertToken))
2074 return SDValue(E, 0);
2075
2076 auto *N = newSDNode<ConstantPoolSDNode>(Args&: isTarget, Args&: C, Args&: VTs, Args&: Offset, Args&: *Alignment,
2077 Args&: TargetFlags);
2078 CSEMap.insert(N, Token: InsertToken);
2079 InsertNode(N);
2080 return SDValue(N, 0);
2081}
2082
2083SDValue SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) {
2084 FoldingSetNodeID ID;
2085 AddNodeIDNode(ID, OpC: ISD::BasicBlock, VTList: getVTList(VT: MVT::Other), OpList: {});
2086 ID.AddPointer(Ptr: MBB);
2087 FoldingSetInsertToken InsertToken;
2088 if (SDNode *E = lookupNode(ID, InsertToken))
2089 return SDValue(E, 0);
2090
2091 auto *N = newSDNode<BasicBlockSDNode>(Args&: MBB);
2092 CSEMap.insert(N, Token: InsertToken);
2093 InsertNode(N);
2094 return SDValue(N, 0);
2095}
2096
2097SDValue SelectionDAG::getValueType(EVT VT) {
2098 if (VT.isSimple() && (unsigned)VT.getSimpleVT().SimpleTy >=
2099 ValueTypeNodes.size())
2100 ValueTypeNodes.resize(new_size: VT.getSimpleVT().SimpleTy+1);
2101
2102 SDNode *&N = VT.isExtended() ?
2103 ExtendedValueTypeNodes[VT] : ValueTypeNodes[VT.getSimpleVT().SimpleTy];
2104
2105 if (N) return SDValue(N, 0);
2106 N = newSDNode<VTSDNode>(Args&: VT);
2107 InsertNode(N);
2108 return SDValue(N, 0);
2109}
2110
2111SDValue SelectionDAG::getExternalSymbol(const char *Sym, EVT VT) {
2112 SDNode *&N = ExternalSymbols[Sym];
2113 if (N) return SDValue(N, 0);
2114 N = newSDNode<ExternalSymbolSDNode>(Args: false, Args&: Sym, Args: 0, Args: getVTList(VT));
2115 InsertNode(N);
2116 return SDValue(N, 0);
2117}
2118
2119SDValue SelectionDAG::getExternalSymbol(RTLIB::LibcallImpl Libcall, EVT VT) {
2120 StringRef SymName = RTLIB::RuntimeLibcallsInfo::getLibcallImplName(CallImpl: Libcall);
2121 return getExternalSymbol(Sym: SymName.data(), VT);
2122}
2123
2124SDValue SelectionDAG::getMCSymbol(MCSymbol *Sym, EVT VT) {
2125 SDNode *&N = MCSymbols[Sym];
2126 if (N)
2127 return SDValue(N, 0);
2128 N = newSDNode<MCSymbolSDNode>(Args&: Sym, Args: getVTList(VT));
2129 InsertNode(N);
2130 return SDValue(N, 0);
2131}
2132
2133SDValue SelectionDAG::getTargetExternalSymbol(const char *Sym, EVT VT,
2134 unsigned TargetFlags) {
2135 SDNode *&N =
2136 TargetExternalSymbols[std::pair<std::string, unsigned>(Sym, TargetFlags)];
2137 if (N) return SDValue(N, 0);
2138 N = newSDNode<ExternalSymbolSDNode>(Args: true, Args&: Sym, Args&: TargetFlags, Args: getVTList(VT));
2139 InsertNode(N);
2140 return SDValue(N, 0);
2141}
2142
2143SDValue SelectionDAG::getTargetExternalSymbol(RTLIB::LibcallImpl Libcall,
2144 EVT VT, unsigned TargetFlags) {
2145 StringRef SymName = RTLIB::RuntimeLibcallsInfo::getLibcallImplName(CallImpl: Libcall);
2146 return getTargetExternalSymbol(Sym: SymName.data(), VT, TargetFlags);
2147}
2148
2149SDValue SelectionDAG::getCondCode(ISD::CondCode Cond) {
2150 if ((unsigned)Cond >= CondCodeNodes.size())
2151 CondCodeNodes.resize(new_size: Cond+1);
2152
2153 if (!CondCodeNodes[Cond]) {
2154 auto *N = newSDNode<CondCodeSDNode>(Args&: Cond);
2155 CondCodeNodes[Cond] = N;
2156 InsertNode(N);
2157 }
2158
2159 return SDValue(CondCodeNodes[Cond], 0);
2160}
2161
2162SDValue SelectionDAG::getVScale(const SDLoc &DL, EVT VT, APInt MulImm) {
2163 assert(MulImm.getBitWidth() == VT.getSizeInBits() &&
2164 "APInt size does not match type size!");
2165
2166 if (MulImm == 0)
2167 return getConstant(Val: 0, DL, VT);
2168
2169 const MachineFunction &MF = getMachineFunction();
2170 const Function &F = MF.getFunction();
2171 ConstantRange CR = getVScaleRange(F: &F, BitWidth: 64);
2172 if (const APInt *C = CR.getSingleElement())
2173 return getConstant(Val: MulImm * C->getZExtValue(), DL, VT);
2174
2175 return getNode(Opcode: ISD::VSCALE, DL, VT, Operand: getConstant(Val: MulImm, DL, VT));
2176}
2177
2178/// \returns a value of type \p VT that represents the runtime value of \p
2179/// Quantity, i.e. scaled by vscale if it's scalable, or a fixed constant
2180/// otherwise. Quantity should be a FixedOrScalableQuantity, i.e. ElementCount
2181/// or TypeSize.
2182template <typename Ty>
2183static SDValue getFixedOrScalableQuantity(SelectionDAG &DAG, const SDLoc &DL,
2184 EVT VT, Ty Quantity) {
2185 if (Quantity.isScalable())
2186 return DAG.getVScale(
2187 DL, VT, MulImm: APInt(VT.getSizeInBits(), Quantity.getKnownMinValue()));
2188
2189 return DAG.getConstant(Quantity.getKnownMinValue(), DL, VT);
2190}
2191
2192SDValue SelectionDAG::getElementCount(const SDLoc &DL, EVT VT,
2193 ElementCount EC) {
2194 return getFixedOrScalableQuantity(DAG&: *this, DL, VT, Quantity: EC);
2195}
2196
2197SDValue SelectionDAG::getTypeSize(const SDLoc &DL, EVT VT, TypeSize TS) {
2198 return getFixedOrScalableQuantity(DAG&: *this, DL, VT, Quantity: TS);
2199}
2200
2201SDValue SelectionDAG::getMaskFromElementCount(const SDLoc &DL, EVT DataVT,
2202 ElementCount EC) {
2203 EVT IdxVT = TLI->getVectorIdxTy(DL: getDataLayout());
2204 EVT MaskVT = TLI->getSetCCResultType(DL: getDataLayout(), Context&: *getContext(), VT: DataVT);
2205 return getNode(Opcode: ISD::GET_ACTIVE_LANE_MASK, DL, VT: MaskVT,
2206 N1: getConstant(Val: 0, DL, VT: IdxVT), N2: getElementCount(DL, VT: IdxVT, EC));
2207}
2208
2209SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT) {
2210 APInt One(ResVT.getScalarSizeInBits(), 1);
2211 return getStepVector(DL, ResVT, StepVal: One);
2212}
2213
2214SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT,
2215 const APInt &StepVal) {
2216 assert(ResVT.getScalarSizeInBits() == StepVal.getBitWidth());
2217 if (ResVT.isScalableVector())
2218 return getNode(
2219 Opcode: ISD::STEP_VECTOR, DL, VT: ResVT,
2220 Operand: getTargetConstant(Val: StepVal, DL, VT: ResVT.getVectorElementType()));
2221
2222 SmallVector<SDValue, 16> OpsStepConstants;
2223 for (uint64_t i = 0; i < ResVT.getVectorNumElements(); i++)
2224 OpsStepConstants.push_back(
2225 Elt: getConstant(Val: StepVal * i, DL, VT: ResVT.getVectorElementType()));
2226 return getBuildVector(VT: ResVT, DL, Ops: OpsStepConstants);
2227}
2228
2229/// Swaps the values of N1 and N2. Swaps all indices in the shuffle mask M that
2230/// point at N1 to point at N2 and indices that point at N2 to point at N1.
2231static void commuteShuffle(SDValue &N1, SDValue &N2, MutableArrayRef<int> M) {
2232 std::swap(a&: N1, b&: N2);
2233 ShuffleVectorSDNode::commuteMask(Mask: M);
2234}
2235
2236SDValue SelectionDAG::getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1,
2237 SDValue N2, ArrayRef<int> Mask) {
2238 assert(VT.getVectorNumElements() == Mask.size() &&
2239 "Must have the same number of vector elements as mask elements!");
2240 assert(VT == N1.getValueType() && VT == N2.getValueType() &&
2241 "Invalid VECTOR_SHUFFLE");
2242
2243 // Canonicalize shuffle undef, undef -> undef
2244 if (N1.isUndef() && N2.isUndef()) {
2245 if (N1.getOpcode() == ISD::POISON && N2.getOpcode() == ISD::POISON)
2246 return getPOISON(VT);
2247 return getUNDEF(VT);
2248 }
2249
2250 // Validate that all indices in Mask are within the range of the elements
2251 // input to the shuffle.
2252 int NElts = Mask.size();
2253 assert(llvm::all_of(Mask,
2254 [&](int M) { return M < (NElts * 2) && M >= -1; }) &&
2255 "Index out of range");
2256
2257 // Copy the mask so we can do any needed cleanup.
2258 SmallVector<int, 8> MaskVec(Mask);
2259
2260 // Canonicalize shuffle v, v -> v, poison
2261 if (N1 == N2) {
2262 N2 = getPOISON(VT);
2263 for (int i = 0; i != NElts; ++i)
2264 if (MaskVec[i] >= NElts) MaskVec[i] -= NElts;
2265 }
2266
2267 // Canonicalize shuffle undef, v -> v, undef. Commute the shuffle mask.
2268 if (N1.isUndef())
2269 commuteShuffle(N1, N2, M: MaskVec);
2270
2271 if (TLI->hasVectorBlend()) {
2272 // If shuffling a splat, try to blend the splat instead. We do this here so
2273 // that even when this arises during lowering we don't have to re-handle it.
2274 auto BlendSplat = [&](BuildVectorSDNode *BV, int Offset) {
2275 BitVector UndefElements;
2276 SDValue Splat = BV->getSplatValue(UndefElements: &UndefElements);
2277 if (!Splat)
2278 return;
2279
2280 for (int i = 0; i < NElts; ++i) {
2281 if (MaskVec[i] < Offset || MaskVec[i] >= (Offset + NElts))
2282 continue;
2283
2284 // If this input comes from undef, mark it as such.
2285 if (UndefElements[MaskVec[i] - Offset]) {
2286 MaskVec[i] = -1;
2287 continue;
2288 }
2289
2290 // If we can blend a non-undef lane, use that instead.
2291 if (!UndefElements[i])
2292 MaskVec[i] = i + Offset;
2293 }
2294 };
2295 if (auto *N1BV = dyn_cast<BuildVectorSDNode>(Val&: N1))
2296 BlendSplat(N1BV, 0);
2297 if (auto *N2BV = dyn_cast<BuildVectorSDNode>(Val&: N2))
2298 BlendSplat(N2BV, NElts);
2299 }
2300
2301 // Canonicalize all index into lhs, -> shuffle lhs, poison
2302 // Canonicalize all index into rhs, -> shuffle rhs, poison
2303 bool AllLHS = true, AllRHS = true;
2304 bool N2Undef = N2.isUndef();
2305 for (int i = 0; i != NElts; ++i) {
2306 if (MaskVec[i] >= NElts) {
2307 if (N2Undef)
2308 MaskVec[i] = -1;
2309 else
2310 AllLHS = false;
2311 } else if (MaskVec[i] >= 0) {
2312 AllRHS = false;
2313 }
2314 }
2315 if (AllLHS && AllRHS)
2316 return getPOISON(VT);
2317 if (AllLHS && !N2Undef)
2318 N2 = getPOISON(VT);
2319 if (AllRHS) {
2320 N1 = getPOISON(VT);
2321 commuteShuffle(N1, N2, M: MaskVec);
2322 }
2323 // Reset our undef status after accounting for the mask.
2324 N2Undef = N2.isUndef();
2325 // Re-check whether both sides ended up undef.
2326 if (N1.isUndef() && N2Undef) {
2327 if (N1.getOpcode() == ISD::POISON && N2.getOpcode() == ISD::POISON)
2328 return getPOISON(VT);
2329 return getUNDEF(VT);
2330 }
2331
2332 // If Identity shuffle return that node.
2333 bool Identity = true, AllSame = true;
2334 for (int i = 0; i != NElts; ++i) {
2335 if (MaskVec[i] >= 0 && MaskVec[i] != i) Identity = false;
2336 if (MaskVec[i] != MaskVec[0]) AllSame = false;
2337 }
2338 if (Identity && NElts)
2339 return N1;
2340
2341 // Shuffling a constant splat doesn't change the result.
2342 if (N2Undef) {
2343 SDValue V = N1;
2344
2345 // Look through any bitcasts. We check that these don't change the number
2346 // (and size) of elements and just changes their types.
2347 while (V.getOpcode() == ISD::BITCAST)
2348 V = V->getOperand(Num: 0);
2349
2350 // A splat should always show up as a build vector node.
2351 if (auto *BV = dyn_cast<BuildVectorSDNode>(Val&: V)) {
2352 BitVector UndefElements;
2353 SDValue Splat = BV->getSplatValue(UndefElements: &UndefElements);
2354 // If this is a splat of an undef, shuffling it is also undef.
2355 if (Splat && Splat.isUndef())
2356 return Splat.getOpcode() == ISD::POISON ? getPOISON(VT) : getUNDEF(VT);
2357
2358 bool SameNumElts =
2359 V.getValueType().getVectorNumElements() == VT.getVectorNumElements();
2360
2361 // We only have a splat which can skip shuffles if there is a splatted
2362 // value and no undef lanes rearranged by the shuffle.
2363 if (Splat && UndefElements.none()) {
2364 // Splat of <x, x, ..., x>, return <x, x, ..., x>, provided that the
2365 // number of elements match or the value splatted is a zero constant.
2366 if (SameNumElts || isNullConstant(V: Splat))
2367 return N1;
2368 }
2369
2370 // If the shuffle itself creates a splat, build the vector directly.
2371 if (AllSame && SameNumElts) {
2372 EVT BuildVT = BV->getValueType(ResNo: 0);
2373 const SDValue &Splatted = BV->getOperand(Num: MaskVec[0]);
2374 SDValue NewBV = getSplatBuildVector(VT: BuildVT, DL: dl, Op: Splatted);
2375
2376 // We may have jumped through bitcasts, so the type of the
2377 // BUILD_VECTOR may not match the type of the shuffle.
2378 if (BuildVT != VT)
2379 NewBV = getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: NewBV);
2380 return NewBV;
2381 }
2382 }
2383 }
2384
2385 SDVTList VTs = getVTList(VT);
2386 FoldingSetNodeID ID;
2387 SDValue Ops[2] = { N1, N2 };
2388 AddNodeIDNode(ID, OpC: ISD::VECTOR_SHUFFLE, VTList: VTs, OpList: Ops);
2389 for (int i = 0; i != NElts; ++i)
2390 ID.AddInteger(I: MaskVec[i]);
2391
2392 FoldingSetInsertToken InsertToken;
2393 if (SDNode *E = lookupNode(ID, DL: dl, InsertToken))
2394 return SDValue(E, 0);
2395
2396 // Allocate the mask array for the node out of the BumpPtrAllocator, since
2397 // SDNode doesn't have access to it. This memory will be "leaked" when
2398 // the node is deallocated, but recovered when the NodeAllocator is released.
2399 int *MaskAlloc = OperandAllocator.Allocate<int>(Num: NElts);
2400 llvm::copy(Range&: MaskVec, Out: MaskAlloc);
2401
2402 auto *N = newSDNode<ShuffleVectorSDNode>(Args&: VTs, Args: dl.getIROrder(),
2403 Args: dl.getDebugLoc(), Args&: MaskAlloc);
2404 createOperands(Node: N, Vals: Ops);
2405
2406 CSEMap.insert(N, Token: InsertToken);
2407 InsertNode(N);
2408 SDValue V = SDValue(N, 0);
2409 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
2410 return V;
2411}
2412
2413SDValue SelectionDAG::getCommutedVectorShuffle(const ShuffleVectorSDNode &SV) {
2414 EVT VT = SV.getValueType(ResNo: 0);
2415 SmallVector<int, 8> MaskVec(SV.getMask());
2416 ShuffleVectorSDNode::commuteMask(Mask: MaskVec);
2417
2418 SDValue Op0 = SV.getOperand(Num: 0);
2419 SDValue Op1 = SV.getOperand(Num: 1);
2420 return getVectorShuffle(VT, dl: SDLoc(&SV), N1: Op1, N2: Op0, Mask: MaskVec);
2421}
2422
2423SDValue SelectionDAG::getRegister(Register Reg, EVT VT) {
2424 SDVTList VTs = getVTList(VT);
2425 FoldingSetNodeID ID;
2426 AddNodeIDNode(ID, OpC: ISD::Register, VTList: VTs, OpList: {});
2427 ID.AddInteger(I: Reg.id());
2428 FoldingSetInsertToken InsertToken;
2429 if (SDNode *E = lookupNode(ID, InsertToken))
2430 return SDValue(E, 0);
2431
2432 auto *N = newSDNode<RegisterSDNode>(Args&: Reg, Args&: VTs);
2433 N->SDNodeBits.IsDivergent = TLI->isSDNodeSourceOfDivergence(N, FLI, UA);
2434 CSEMap.insert(N, Token: InsertToken);
2435 InsertNode(N);
2436 return SDValue(N, 0);
2437}
2438
2439SDValue SelectionDAG::getRegisterMask(const uint32_t *RegMask) {
2440 FoldingSetNodeID ID;
2441 AddNodeIDNode(ID, OpC: ISD::RegisterMask, VTList: getVTList(VT: MVT::Untyped), OpList: {});
2442 ID.AddPointer(Ptr: RegMask);
2443 FoldingSetInsertToken InsertToken;
2444 if (SDNode *E = lookupNode(ID, InsertToken))
2445 return SDValue(E, 0);
2446
2447 auto *N = newSDNode<RegisterMaskSDNode>(Args&: RegMask);
2448 CSEMap.insert(N, Token: InsertToken);
2449 InsertNode(N);
2450 return SDValue(N, 0);
2451}
2452
2453SDValue SelectionDAG::getEHLabel(const SDLoc &dl, SDValue Root,
2454 MCSymbol *Label) {
2455 return getLabelNode(Opcode: ISD::EH_LABEL, dl, Root, Label);
2456}
2457
2458SDValue SelectionDAG::getLabelNode(unsigned Opcode, const SDLoc &dl,
2459 SDValue Root, MCSymbol *Label) {
2460 FoldingSetNodeID ID;
2461 SDValue Ops[] = { Root };
2462 AddNodeIDNode(ID, OpC: Opcode, VTList: getVTList(VT: MVT::Other), OpList: Ops);
2463 ID.AddPointer(Ptr: Label);
2464 FoldingSetInsertToken InsertToken;
2465 if (SDNode *E = lookupNode(ID, InsertToken))
2466 return SDValue(E, 0);
2467
2468 auto *N =
2469 newSDNode<LabelSDNode>(Args&: Opcode, Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: Label);
2470 createOperands(Node: N, Vals: Ops);
2471
2472 CSEMap.insert(N, Token: InsertToken);
2473 InsertNode(N);
2474 return SDValue(N, 0);
2475}
2476
2477SDValue SelectionDAG::getBlockAddress(const BlockAddress *BA, EVT VT,
2478 int64_t Offset, bool isTarget,
2479 unsigned TargetFlags) {
2480 unsigned Opc = isTarget ? ISD::TargetBlockAddress : ISD::BlockAddress;
2481 SDVTList VTs = getVTList(VT);
2482
2483 FoldingSetNodeID ID;
2484 AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: {});
2485 ID.AddPointer(Ptr: BA);
2486 ID.AddInteger(I: Offset);
2487 ID.AddInteger(I: TargetFlags);
2488 FoldingSetInsertToken InsertToken;
2489 if (SDNode *E = lookupNode(ID, InsertToken))
2490 return SDValue(E, 0);
2491
2492 auto *N = newSDNode<BlockAddressSDNode>(Args&: Opc, Args&: VTs, Args&: BA, Args&: Offset, Args&: TargetFlags);
2493 CSEMap.insert(N, Token: InsertToken);
2494 InsertNode(N);
2495 return SDValue(N, 0);
2496}
2497
2498SDValue SelectionDAG::getSrcValue(const Value *V) {
2499 FoldingSetNodeID ID;
2500 AddNodeIDNode(ID, OpC: ISD::SRCVALUE, VTList: getVTList(VT: MVT::Other), OpList: {});
2501 ID.AddPointer(Ptr: V);
2502
2503 FoldingSetInsertToken InsertToken;
2504 if (SDNode *E = lookupNode(ID, InsertToken))
2505 return SDValue(E, 0);
2506
2507 auto *N = newSDNode<SrcValueSDNode>(Args&: V);
2508 CSEMap.insert(N, Token: InsertToken);
2509 InsertNode(N);
2510 return SDValue(N, 0);
2511}
2512
2513SDValue SelectionDAG::getMDNode(const MDNode *MD) {
2514 FoldingSetNodeID ID;
2515 AddNodeIDNode(ID, OpC: ISD::MDNODE_SDNODE, VTList: getVTList(VT: MVT::Other), OpList: {});
2516 ID.AddPointer(Ptr: MD);
2517
2518 FoldingSetInsertToken InsertToken;
2519 if (SDNode *E = lookupNode(ID, InsertToken))
2520 return SDValue(E, 0);
2521
2522 auto *N = newSDNode<MDNodeSDNode>(Args&: MD);
2523 CSEMap.insert(N, Token: InsertToken);
2524 InsertNode(N);
2525 return SDValue(N, 0);
2526}
2527
2528SDValue SelectionDAG::getBitcast(EVT VT, SDValue V) {
2529 if (VT == V.getValueType())
2530 return V;
2531
2532 return getNode(Opcode: ISD::BITCAST, DL: SDLoc(V), VT, Operand: V);
2533}
2534
2535SDValue SelectionDAG::getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr,
2536 unsigned SrcAS, unsigned DestAS) {
2537 SDVTList VTs = getVTList(VT);
2538 SDValue Ops[] = {Ptr};
2539 FoldingSetNodeID ID;
2540 AddNodeIDNode(ID, OpC: ISD::ADDRSPACECAST, VTList: VTs, OpList: Ops);
2541 ID.AddInteger(I: SrcAS);
2542 ID.AddInteger(I: DestAS);
2543
2544 FoldingSetInsertToken InsertToken;
2545 if (SDNode *E = lookupNode(ID, DL: dl, InsertToken))
2546 return SDValue(E, 0);
2547
2548 auto *N = newSDNode<AddrSpaceCastSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(),
2549 Args&: VTs, Args&: SrcAS, Args&: DestAS);
2550 createOperands(Node: N, Vals: Ops);
2551
2552 CSEMap.insert(N, Token: InsertToken);
2553 InsertNode(N);
2554 return SDValue(N, 0);
2555}
2556
2557SDValue SelectionDAG::getFreeze(SDValue V) {
2558 return getNode(Opcode: ISD::FREEZE, DL: SDLoc(V), VT: V.getValueType(), Operand: V);
2559}
2560
2561SDValue SelectionDAG::getFreeze(SDValue V, const APInt &DemandedElts,
2562 UndefPoisonKind Kind) {
2563 if (isGuaranteedNotToBeUndefOrPoison(Op: V, DemandedElts, Kind))
2564 return V;
2565 return getFreeze(V);
2566}
2567
2568/// getShiftAmountOperand - Return the specified value casted to
2569/// the target's desired shift amount type.
2570SDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) {
2571 EVT OpTy = Op.getValueType();
2572 EVT ShTy = TLI->getShiftAmountTy(LHSTy, DL: getDataLayout());
2573 if (OpTy == ShTy || OpTy.isVector()) return Op;
2574
2575 return getZExtOrTrunc(Op, DL: SDLoc(Op), VT: ShTy);
2576}
2577
2578SDValue SelectionDAG::expandVAArg(SDNode *Node) {
2579 SDLoc dl(Node);
2580 const TargetLowering &TLI = getTargetLoweringInfo();
2581 const Value *V = cast<SrcValueSDNode>(Val: Node->getOperand(Num: 2))->getValue();
2582 EVT VT = Node->getValueType(ResNo: 0);
2583 SDValue Tmp1 = Node->getOperand(Num: 0);
2584 SDValue Tmp2 = Node->getOperand(Num: 1);
2585 const MaybeAlign MA(Node->getConstantOperandVal(Num: 3));
2586
2587 SDValue VAListLoad = getLoad(VT: TLI.getPointerTy(DL: getDataLayout()), dl, Chain: Tmp1,
2588 Ptr: Tmp2, PtrInfo: MachinePointerInfo(V));
2589 SDValue VAList = VAListLoad;
2590
2591 if (MA && *MA > TLI.getMinStackArgumentAlignment()) {
2592 VAList = getNode(Opcode: ISD::ADD, DL: dl, VT: VAList.getValueType(), N1: VAList,
2593 N2: getConstant(Val: MA->value() - 1, DL: dl, VT: VAList.getValueType()));
2594
2595 VAList = getNode(
2596 Opcode: ISD::AND, DL: dl, VT: VAList.getValueType(), N1: VAList,
2597 N2: getSignedConstant(Val: -(int64_t)MA->value(), DL: dl, VT: VAList.getValueType()));
2598 }
2599
2600 // Increment the pointer, VAList, to the next vaarg
2601 Tmp1 = getNode(Opcode: ISD::ADD, DL: dl, VT: VAList.getValueType(), N1: VAList,
2602 N2: getConstant(Val: getDataLayout().getTypeAllocSize(
2603 Ty: VT.getTypeForEVT(Context&: *getContext())),
2604 DL: dl, VT: VAList.getValueType()));
2605 // Store the incremented VAList to the legalized pointer
2606 Tmp1 =
2607 getStore(Chain: VAListLoad.getValue(R: 1), dl, Val: Tmp1, Ptr: Tmp2, PtrInfo: MachinePointerInfo(V));
2608 // Load the actual argument out of the pointer VAList
2609 return getLoad(VT, dl, Chain: Tmp1, Ptr: VAList, PtrInfo: MachinePointerInfo());
2610}
2611
2612SDValue SelectionDAG::expandVACopy(SDNode *Node) {
2613 SDLoc dl(Node);
2614 const TargetLowering &TLI = getTargetLoweringInfo();
2615 // This defaults to loading a pointer from the input and storing it to the
2616 // output, returning the chain.
2617 const Value *VD = cast<SrcValueSDNode>(Val: Node->getOperand(Num: 3))->getValue();
2618 const Value *VS = cast<SrcValueSDNode>(Val: Node->getOperand(Num: 4))->getValue();
2619 SDValue Tmp1 =
2620 getLoad(VT: TLI.getPointerTy(DL: getDataLayout()), dl, Chain: Node->getOperand(Num: 0),
2621 Ptr: Node->getOperand(Num: 2), PtrInfo: MachinePointerInfo(VS));
2622 return getStore(Chain: Tmp1.getValue(R: 1), dl, Val: Tmp1, Ptr: Node->getOperand(Num: 1),
2623 PtrInfo: MachinePointerInfo(VD));
2624}
2625
2626Align SelectionDAG::getReducedAlign(EVT VT, bool UseABI) {
2627 const DataLayout &DL = getDataLayout();
2628 Type *Ty = VT.getTypeForEVT(Context&: *getContext());
2629 Align RedAlign = UseABI ? DL.getABITypeAlign(Ty) : DL.getPrefTypeAlign(Ty);
2630
2631 if (TLI->isTypeLegal(VT) || !VT.isVector())
2632 return RedAlign;
2633
2634 const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering();
2635 const Align StackAlign = TFI->getStackAlign();
2636
2637 // See if we can choose a smaller ABI alignment in cases where it's an
2638 // illegal vector type that will get broken down.
2639 if (RedAlign > StackAlign) {
2640 EVT IntermediateVT;
2641 MVT RegisterVT;
2642 unsigned NumIntermediates;
2643 TLI->getVectorTypeBreakdown(Context&: *getContext(), VT, IntermediateVT,
2644 NumIntermediates, RegisterVT);
2645 Ty = IntermediateVT.getTypeForEVT(Context&: *getContext());
2646 Align RedAlign2 = UseABI ? DL.getABITypeAlign(Ty) : DL.getPrefTypeAlign(Ty);
2647 if (RedAlign2 < RedAlign)
2648 RedAlign = RedAlign2;
2649
2650 if (!getMachineFunction().getFrameInfo().isStackRealignable())
2651 // If the stack is not realignable, the alignment should be limited to the
2652 // StackAlignment
2653 RedAlign = std::min(a: RedAlign, b: StackAlign);
2654 }
2655
2656 return RedAlign;
2657}
2658
2659SDValue SelectionDAG::CreateStackTemporary(TypeSize Bytes, Align Alignment) {
2660 MachineFrameInfo &MFI = MF->getFrameInfo();
2661 const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering();
2662 int StackID = 0;
2663 if (Bytes.isScalable())
2664 StackID = TFI->getStackIDForScalableVectors();
2665 // The stack id gives an indication of whether the object is scalable or
2666 // not, so it's safe to pass in the minimum size here.
2667 int FrameIdx = MFI.CreateStackObject(Size: Bytes.getKnownMinValue(), Alignment,
2668 isSpillSlot: false, Alloca: nullptr, ID: StackID);
2669 return getFrameIndex(FI: FrameIdx, VT: TLI->getFrameIndexTy(DL: getDataLayout()));
2670}
2671
2672SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) {
2673 Type *Ty = VT.getTypeForEVT(Context&: *getContext());
2674 Align StackAlign =
2675 std::max(a: getDataLayout().getPrefTypeAlign(Ty), b: Align(minAlign));
2676 return CreateStackTemporary(Bytes: VT.getStoreSize(), Alignment: StackAlign);
2677}
2678
2679SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) {
2680 TypeSize VT1Size = VT1.getStoreSize();
2681 TypeSize VT2Size = VT2.getStoreSize();
2682 assert(VT1Size.isScalable() == VT2Size.isScalable() &&
2683 "Don't know how to choose the maximum size when creating a stack "
2684 "temporary");
2685 TypeSize Bytes = VT1Size.getKnownMinValue() > VT2Size.getKnownMinValue()
2686 ? VT1Size
2687 : VT2Size;
2688
2689 Type *Ty1 = VT1.getTypeForEVT(Context&: *getContext());
2690 Type *Ty2 = VT2.getTypeForEVT(Context&: *getContext());
2691 const DataLayout &DL = getDataLayout();
2692 Align Align = std::max(a: DL.getPrefTypeAlign(Ty: Ty1), b: DL.getPrefTypeAlign(Ty: Ty2));
2693 return CreateStackTemporary(Bytes, Alignment: Align);
2694}
2695
2696SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1, SDValue N2,
2697 ISD::CondCode Cond, const SDLoc &dl,
2698 SDNodeFlags Flags) {
2699 EVT OpVT = N1.getValueType();
2700
2701 auto GetUndefBooleanConstant = [&]() {
2702 if (VT.getScalarType() == MVT::i1 ||
2703 TLI->getBooleanContents(Type: OpVT) ==
2704 TargetLowering::UndefinedBooleanContent)
2705 return getUNDEF(VT);
2706 // ZeroOrOne / ZeroOrNegative require specific values for the high bits,
2707 // so we cannot use getUNDEF(). Return zero instead.
2708 return getConstant(Val: 0, DL: dl, VT);
2709 };
2710
2711 // These setcc operations always fold.
2712 switch (Cond) {
2713 default: break;
2714 case ISD::SETFALSE:
2715 case ISD::SETFALSE2: return getBoolConstant(V: false, DL: dl, VT, OpVT);
2716 case ISD::SETTRUE:
2717 case ISD::SETTRUE2: return getBoolConstant(V: true, DL: dl, VT, OpVT);
2718
2719 case ISD::SETOEQ:
2720 case ISD::SETOGT:
2721 case ISD::SETOGE:
2722 case ISD::SETOLT:
2723 case ISD::SETOLE:
2724 case ISD::SETONE:
2725 case ISD::SETO:
2726 case ISD::SETUO:
2727 case ISD::SETUEQ:
2728 case ISD::SETUNE:
2729 assert(!OpVT.isInteger() && "Illegal setcc for integer!");
2730 break;
2731 }
2732
2733 if (OpVT.isInteger()) {
2734 // For EQ and NE, we can always pick a value for the undef to make the
2735 // predicate pass or fail, so we can return undef.
2736 // Matches behavior in llvm::ConstantFoldCompareInstruction.
2737 // icmp eq/ne X, undef -> undef.
2738 if ((N1.isUndef() || N2.isUndef()) &&
2739 (Cond == ISD::SETEQ || Cond == ISD::SETNE))
2740 return GetUndefBooleanConstant();
2741
2742 // If both operands are undef, we can return undef for int comparison.
2743 // icmp undef, undef -> undef.
2744 if (N1.isUndef() && N2.isUndef())
2745 return GetUndefBooleanConstant();
2746
2747 // icmp X, X -> true/false
2748 // icmp X, undef -> true/false because undef could be X.
2749 if (N1.isUndef() || N2.isUndef() || N1 == N2)
2750 return getBoolConstant(V: ISD::isTrueWhenEqual(Cond), DL: dl, VT, OpVT);
2751 }
2752
2753 if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(Val&: N2)) {
2754 const APInt &C2 = N2C->getAPIntValue();
2755 if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(Val&: N1)) {
2756 const APInt &C1 = N1C->getAPIntValue();
2757
2758 return getBoolConstant(V: ICmpInst::compare(LHS: C1, RHS: C2, Pred: getICmpCondCode(Pred: Cond)),
2759 DL: dl, VT, OpVT);
2760 }
2761 }
2762
2763 auto *N1CFP = dyn_cast<ConstantFPSDNode>(Val&: N1);
2764 auto *N2CFP = dyn_cast<ConstantFPSDNode>(Val&: N2);
2765
2766 if (N1CFP && N2CFP) {
2767 APFloat::cmpResult R = N1CFP->getValueAPF().compare(RHS: N2CFP->getValueAPF());
2768 switch (Cond) {
2769 default: break;
2770 case ISD::SETEQ: if (R==APFloat::cmpUnordered)
2771 return GetUndefBooleanConstant();
2772 [[fallthrough]];
2773 case ISD::SETOEQ: return getBoolConstant(V: R==APFloat::cmpEqual, DL: dl, VT,
2774 OpVT);
2775 case ISD::SETNE: if (R==APFloat::cmpUnordered)
2776 return GetUndefBooleanConstant();
2777 [[fallthrough]];
2778 case ISD::SETONE: return getBoolConstant(V: R==APFloat::cmpGreaterThan ||
2779 R==APFloat::cmpLessThan, DL: dl, VT,
2780 OpVT);
2781 case ISD::SETLT: if (R==APFloat::cmpUnordered)
2782 return GetUndefBooleanConstant();
2783 [[fallthrough]];
2784 case ISD::SETOLT: return getBoolConstant(V: R==APFloat::cmpLessThan, DL: dl, VT,
2785 OpVT);
2786 case ISD::SETGT: if (R==APFloat::cmpUnordered)
2787 return GetUndefBooleanConstant();
2788 [[fallthrough]];
2789 case ISD::SETOGT: return getBoolConstant(V: R==APFloat::cmpGreaterThan, DL: dl,
2790 VT, OpVT);
2791 case ISD::SETLE: if (R==APFloat::cmpUnordered)
2792 return GetUndefBooleanConstant();
2793 [[fallthrough]];
2794 case ISD::SETOLE: return getBoolConstant(V: R==APFloat::cmpLessThan ||
2795 R==APFloat::cmpEqual, DL: dl, VT,
2796 OpVT);
2797 case ISD::SETGE: if (R==APFloat::cmpUnordered)
2798 return GetUndefBooleanConstant();
2799 [[fallthrough]];
2800 case ISD::SETOGE: return getBoolConstant(V: R==APFloat::cmpGreaterThan ||
2801 R==APFloat::cmpEqual, DL: dl, VT, OpVT);
2802 case ISD::SETO: return getBoolConstant(V: R!=APFloat::cmpUnordered, DL: dl, VT,
2803 OpVT);
2804 case ISD::SETUO: return getBoolConstant(V: R==APFloat::cmpUnordered, DL: dl, VT,
2805 OpVT);
2806 case ISD::SETUEQ: return getBoolConstant(V: R==APFloat::cmpUnordered ||
2807 R==APFloat::cmpEqual, DL: dl, VT,
2808 OpVT);
2809 case ISD::SETUNE: return getBoolConstant(V: R!=APFloat::cmpEqual, DL: dl, VT,
2810 OpVT);
2811 case ISD::SETULT: return getBoolConstant(V: R==APFloat::cmpUnordered ||
2812 R==APFloat::cmpLessThan, DL: dl, VT,
2813 OpVT);
2814 case ISD::SETUGT: return getBoolConstant(V: R==APFloat::cmpGreaterThan ||
2815 R==APFloat::cmpUnordered, DL: dl, VT,
2816 OpVT);
2817 case ISD::SETULE: return getBoolConstant(V: R!=APFloat::cmpGreaterThan, DL: dl,
2818 VT, OpVT);
2819 case ISD::SETUGE: return getBoolConstant(V: R!=APFloat::cmpLessThan, DL: dl, VT,
2820 OpVT);
2821 }
2822 } else if (N1CFP && OpVT.isSimple() && !N2.isUndef()) {
2823 // Ensure that the constant occurs on the RHS.
2824 ISD::CondCode SwappedCond = ISD::getSetCCSwappedOperands(Operation: Cond);
2825 if (!TLI->isCondCodeLegal(CC: SwappedCond, VT: OpVT.getSimpleVT()))
2826 return SDValue();
2827 return getSetCC(DL: dl, VT, LHS: N2, RHS: N1, Cond: SwappedCond, /*Chain=*/{},
2828 /*IsSignaling=*/false, Flags);
2829 } else if ((N2CFP && N2CFP->getValueAPF().isNaN()) ||
2830 (OpVT.isFloatingPoint() && (N1.isUndef() || N2.isUndef()))) {
2831 // If an operand is known to be a nan (or undef that could be a nan), we can
2832 // fold it.
2833 // Choosing NaN for the undef will always make unordered comparison succeed
2834 // and ordered comparison fails.
2835 // Matches behavior in llvm::ConstantFoldCompareInstruction.
2836 switch (ISD::getUnorderedFlavor(Cond)) {
2837 default:
2838 llvm_unreachable("Unknown flavor!");
2839 case 0: // Known false.
2840 return getBoolConstant(V: false, DL: dl, VT, OpVT);
2841 case 1: // Known true.
2842 return getBoolConstant(V: true, DL: dl, VT, OpVT);
2843 case 2: // Undefined.
2844 return GetUndefBooleanConstant();
2845 }
2846 }
2847
2848 // Could not fold it.
2849 return SDValue();
2850}
2851
2852/// SignBitIsZero - Return true if the sign bit of Op is known to be zero. We
2853/// use this predicate to simplify operations downstream.
2854bool SelectionDAG::SignBitIsZero(SDValue Op, unsigned Depth) const {
2855 unsigned BitWidth = Op.getScalarValueSizeInBits();
2856 return MaskedValueIsZero(Op, Mask: APInt::getSignMask(BitWidth), Depth);
2857}
2858
2859// TODO: Should have argument to specify if sign bit of nan is ignorable.
2860bool SelectionDAG::SignBitIsZeroFP(SDValue Op, unsigned Depth) const {
2861 if (Depth >= MaxRecursionDepth)
2862 return false; // Limit search depth.
2863
2864 unsigned Opc = Op.getOpcode();
2865 switch (Opc) {
2866 case ISD::FABS:
2867 return true;
2868 case ISD::AssertNoFPClass: {
2869 FPClassTest NoFPClass =
2870 static_cast<FPClassTest>(Op.getConstantOperandVal(i: 1));
2871
2872 const FPClassTest TestMask = fcNan | fcNegative;
2873 return (NoFPClass & TestMask) == TestMask;
2874 }
2875 case ISD::ARITH_FENCE:
2876 return SignBitIsZeroFP(Op: Op.getOperand(i: 0), Depth: Depth + 1);
2877 case ISD::FEXP:
2878 case ISD::FEXP2:
2879 case ISD::FEXP10:
2880 return Op->getFlags().hasNoNaNs();
2881 case ISD::FMINNUM:
2882 case ISD::FMINNUM_IEEE:
2883 case ISD::FMINIMUM:
2884 case ISD::FMINIMUMNUM:
2885 return SignBitIsZeroFP(Op: Op.getOperand(i: 1), Depth: Depth + 1) &&
2886 SignBitIsZeroFP(Op: Op.getOperand(i: 0), Depth: Depth + 1);
2887 case ISD::FMAXNUM:
2888 case ISD::FMAXNUM_IEEE:
2889 case ISD::FMAXIMUM:
2890 case ISD::FMAXIMUMNUM:
2891 // TODO: If we can ignore the sign bit of nans, only one side being known 0
2892 // is sufficient.
2893 return SignBitIsZeroFP(Op: Op.getOperand(i: 1), Depth: Depth + 1) &&
2894 SignBitIsZeroFP(Op: Op.getOperand(i: 0), Depth: Depth + 1);
2895 default:
2896 return false;
2897 }
2898
2899 llvm_unreachable("covered opcode switch");
2900}
2901
2902/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero. We use
2903/// this predicate to simplify operations downstream. Mask is known to be zero
2904/// for bits that V cannot have.
2905bool SelectionDAG::MaskedValueIsZero(SDValue V, const APInt &Mask,
2906 unsigned Depth) const {
2907 return Mask.isSubsetOf(RHS: computeKnownBits(Op: V, Depth).Zero);
2908}
2909
2910/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero in
2911/// DemandedElts. We use this predicate to simplify operations downstream.
2912/// Mask is known to be zero for bits that V cannot have.
2913bool SelectionDAG::MaskedValueIsZero(SDValue V, const APInt &Mask,
2914 const APInt &DemandedElts,
2915 unsigned Depth) const {
2916 return Mask.isSubsetOf(RHS: computeKnownBits(Op: V, DemandedElts, Depth).Zero);
2917}
2918
2919/// MaskedVectorIsZero - Return true if 'Op' is known to be zero in
2920/// DemandedElts. We use this predicate to simplify operations downstream.
2921bool SelectionDAG::MaskedVectorIsZero(SDValue V, const APInt &DemandedElts,
2922 unsigned Depth /* = 0 */) const {
2923 return computeKnownBits(Op: V, DemandedElts, Depth).isZero();
2924}
2925
2926/// MaskedValueIsAllOnes - Return true if '(Op & Mask) == Mask'.
2927bool SelectionDAG::MaskedValueIsAllOnes(SDValue V, const APInt &Mask,
2928 unsigned Depth) const {
2929 return Mask.isSubsetOf(RHS: computeKnownBits(Op: V, Depth).One);
2930}
2931
2932APInt SelectionDAG::computeVectorKnownZeroElements(SDValue Op,
2933 const APInt &DemandedElts,
2934 unsigned Depth) const {
2935 EVT VT = Op.getValueType();
2936 assert(VT.isVector() && !VT.isScalableVector() && "Only for fixed vectors!");
2937
2938 unsigned NumElts = VT.getVectorNumElements();
2939 assert(DemandedElts.getBitWidth() == NumElts && "Unexpected demanded mask.");
2940
2941 APInt KnownZeroElements = APInt::getZero(numBits: NumElts);
2942 for (unsigned EltIdx = 0; EltIdx != NumElts; ++EltIdx) {
2943 if (!DemandedElts[EltIdx])
2944 continue; // Don't query elements that are not demanded.
2945 APInt Mask = APInt::getOneBitSet(numBits: NumElts, BitNo: EltIdx);
2946 if (MaskedVectorIsZero(V: Op, DemandedElts: Mask, Depth))
2947 KnownZeroElements.setBit(EltIdx);
2948 }
2949 return KnownZeroElements;
2950}
2951
2952/// isSplatValue - Return true if the vector V has the same value
2953/// across all DemandedElts. For scalable vectors, we don't know the
2954/// number of lanes at compile time. Instead, we use a 1 bit APInt
2955/// to represent a conservative value for all lanes; that is, that
2956/// one bit value is implicitly splatted across all lanes.
2957bool SelectionDAG::isSplatValue(SDValue V, const APInt &DemandedElts,
2958 APInt &UndefElts, unsigned Depth) const {
2959 unsigned Opcode = V.getOpcode();
2960 EVT VT = V.getValueType();
2961 assert(VT.isVector() && "Vector type expected");
2962 assert((!VT.isScalableVector() || DemandedElts.getBitWidth() == 1) &&
2963 "scalable demanded bits are ignored");
2964
2965 if (!DemandedElts)
2966 return false; // No demanded elts, better to assume we don't know anything.
2967
2968 if (Depth >= MaxRecursionDepth)
2969 return false; // Limit search depth.
2970
2971 // Deal with some common cases here that work for both fixed and scalable
2972 // vector types.
2973 switch (Opcode) {
2974 case ISD::SPLAT_VECTOR:
2975 UndefElts = V.getOperand(i: 0).isUndef()
2976 ? APInt::getAllOnes(numBits: DemandedElts.getBitWidth())
2977 : APInt(DemandedElts.getBitWidth(), 0);
2978 return true;
2979 case ISD::ADD:
2980 case ISD::SUB:
2981 case ISD::AND:
2982 case ISD::XOR:
2983 case ISD::OR: {
2984 APInt UndefLHS, UndefRHS;
2985 SDValue LHS = V.getOperand(i: 0);
2986 SDValue RHS = V.getOperand(i: 1);
2987 // Only recognize splats with the same demanded undef elements for both
2988 // operands, otherwise we might fail to handle binop-specific undef
2989 // handling.
2990 // e.g. (and undef, 0) -> 0 etc.
2991 if (isSplatValue(V: LHS, DemandedElts, UndefElts&: UndefLHS, Depth: Depth + 1) &&
2992 isSplatValue(V: RHS, DemandedElts, UndefElts&: UndefRHS, Depth: Depth + 1) &&
2993 (DemandedElts & UndefLHS) == (DemandedElts & UndefRHS)) {
2994 UndefElts = UndefLHS | UndefRHS;
2995 return true;
2996 }
2997 return false;
2998 }
2999 case ISD::ABS:
3000 case ISD::ABS_MIN_POISON:
3001 case ISD::TRUNCATE:
3002 case ISD::SIGN_EXTEND:
3003 case ISD::ZERO_EXTEND:
3004 return isSplatValue(V: V.getOperand(i: 0), DemandedElts, UndefElts, Depth: Depth + 1);
3005 default:
3006 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
3007 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)
3008 return TLI->isSplatValueForTargetNode(Op: V, DemandedElts, UndefElts, DAG: *this,
3009 Depth);
3010 break;
3011 }
3012
3013 // We don't support other cases than those above for scalable vectors at
3014 // the moment.
3015 if (VT.isScalableVector())
3016 return false;
3017
3018 unsigned NumElts = VT.getVectorNumElements();
3019 assert(NumElts == DemandedElts.getBitWidth() && "Vector size mismatch");
3020 UndefElts = APInt::getZero(numBits: NumElts);
3021
3022 switch (Opcode) {
3023 case ISD::BUILD_VECTOR: {
3024 SDValue Scl;
3025 for (unsigned i = 0; i != NumElts; ++i) {
3026 SDValue Op = V.getOperand(i);
3027 if (Op.isUndef()) {
3028 UndefElts.setBit(i);
3029 continue;
3030 }
3031 if (!DemandedElts[i])
3032 continue;
3033 if (Scl && Scl != Op)
3034 return false;
3035 Scl = Op;
3036 }
3037 return true;
3038 }
3039 case ISD::VECTOR_SHUFFLE: {
3040 // Check if this is a shuffle node doing a splat or a shuffle of a splat.
3041 APInt DemandedLHS = APInt::getZero(numBits: NumElts);
3042 APInt DemandedRHS = APInt::getZero(numBits: NumElts);
3043 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Val&: V)->getMask();
3044 for (int i = 0; i != (int)NumElts; ++i) {
3045 int M = Mask[i];
3046 if (M < 0) {
3047 UndefElts.setBit(i);
3048 continue;
3049 }
3050 if (!DemandedElts[i])
3051 continue;
3052 if (M < (int)NumElts)
3053 DemandedLHS.setBit(M);
3054 else
3055 DemandedRHS.setBit(M - NumElts);
3056 }
3057
3058 // If we aren't demanding either op, assume there's no splat.
3059 // If we are demanding both ops, assume there's no splat.
3060 if ((DemandedLHS.isZero() && DemandedRHS.isZero()) ||
3061 (!DemandedLHS.isZero() && !DemandedRHS.isZero()))
3062 return false;
3063
3064 // See if the demanded elts of the source op is a splat or we only demand
3065 // one element, which should always be a splat.
3066 // TODO: Handle source ops splats with undefs.
3067 auto CheckSplatSrc = [&](SDValue Src, const APInt &SrcElts) {
3068 APInt SrcUndefs;
3069 return (SrcElts.popcount() == 1) ||
3070 (isSplatValue(V: Src, DemandedElts: SrcElts, UndefElts&: SrcUndefs, Depth: Depth + 1) &&
3071 (SrcElts & SrcUndefs).isZero());
3072 };
3073 if (!DemandedLHS.isZero())
3074 return CheckSplatSrc(V.getOperand(i: 0), DemandedLHS);
3075 return CheckSplatSrc(V.getOperand(i: 1), DemandedRHS);
3076 }
3077 case ISD::EXTRACT_SUBVECTOR: {
3078 // Offset the demanded elts by the subvector index.
3079 SDValue Src = V.getOperand(i: 0);
3080 // We don't support scalable vectors at the moment.
3081 if (Src.getValueType().isScalableVector())
3082 return false;
3083 uint64_t Idx = V.getConstantOperandVal(i: 1);
3084 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3085 APInt UndefSrcElts;
3086 APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
3087 if (isSplatValue(V: Src, DemandedElts: DemandedSrcElts, UndefElts&: UndefSrcElts, Depth: Depth + 1)) {
3088 UndefElts = UndefSrcElts.extractBits(numBits: NumElts, bitPosition: Idx);
3089 return true;
3090 }
3091 break;
3092 }
3093 case ISD::ANY_EXTEND_VECTOR_INREG:
3094 case ISD::SIGN_EXTEND_VECTOR_INREG:
3095 case ISD::ZERO_EXTEND_VECTOR_INREG: {
3096 // Widen the demanded elts by the src element count.
3097 SDValue Src = V.getOperand(i: 0);
3098 // We don't support scalable vectors at the moment.
3099 if (Src.getValueType().isScalableVector())
3100 return false;
3101 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3102 APInt UndefSrcElts;
3103 APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts);
3104 if (isSplatValue(V: Src, DemandedElts: DemandedSrcElts, UndefElts&: UndefSrcElts, Depth: Depth + 1)) {
3105 UndefElts = UndefSrcElts.trunc(width: NumElts);
3106 return true;
3107 }
3108 break;
3109 }
3110 case ISD::BITCAST: {
3111 SDValue Src = V.getOperand(i: 0);
3112 EVT SrcVT = Src.getValueType();
3113 unsigned SrcBitWidth = SrcVT.getScalarSizeInBits();
3114 unsigned BitWidth = VT.getScalarSizeInBits();
3115
3116 // Ignore bitcasts from unsupported types.
3117 // TODO: Add fp support?
3118 if (!SrcVT.isVector() || !SrcVT.isInteger() || !VT.isInteger())
3119 break;
3120
3121 // Bitcast 'small element' vector to 'large element' vector.
3122 if ((BitWidth % SrcBitWidth) == 0) {
3123 // See if each sub element is a splat.
3124 unsigned Scale = BitWidth / SrcBitWidth;
3125 unsigned NumSrcElts = SrcVT.getVectorNumElements();
3126 APInt ScaledDemandedElts =
3127 APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumSrcElts);
3128 for (unsigned I = 0; I != Scale; ++I) {
3129 APInt SubUndefElts;
3130 APInt SubDemandedElt = APInt::getOneBitSet(numBits: Scale, BitNo: I);
3131 APInt SubDemandedElts = APInt::getSplat(NewLen: NumSrcElts, V: SubDemandedElt);
3132 SubDemandedElts &= ScaledDemandedElts;
3133 if (!isSplatValue(V: Src, DemandedElts: SubDemandedElts, UndefElts&: SubUndefElts, Depth: Depth + 1))
3134 return false;
3135 // TODO: Add support for merging sub undef elements.
3136 if (!SubUndefElts.isZero())
3137 return false;
3138 }
3139 return true;
3140 }
3141 break;
3142 }
3143 }
3144
3145 return false;
3146}
3147
3148/// Helper wrapper to main isSplatValue function.
3149bool SelectionDAG::isSplatValue(SDValue V, bool AllowUndefs) const {
3150 EVT VT = V.getValueType();
3151 assert(VT.isVector() && "Vector type expected");
3152
3153 APInt UndefElts;
3154 // Since the number of lanes in a scalable vector is unknown at compile time,
3155 // we track one bit which is implicitly broadcast to all lanes. This means
3156 // that all lanes in a scalable vector are considered demanded.
3157 APInt DemandedElts
3158 = APInt::getAllOnes(numBits: VT.isScalableVector() ? 1 : VT.getVectorNumElements());
3159 return isSplatValue(V, DemandedElts, UndefElts) &&
3160 (AllowUndefs || !UndefElts);
3161}
3162
3163SDValue SelectionDAG::getSplatSourceVector(SDValue V, int &SplatIdx) {
3164 V = peekThroughExtractSubvectors(V);
3165
3166 EVT VT = V.getValueType();
3167 unsigned Opcode = V.getOpcode();
3168 switch (Opcode) {
3169 default: {
3170 APInt UndefElts;
3171 // Since the number of lanes in a scalable vector is unknown at compile time,
3172 // we track one bit which is implicitly broadcast to all lanes. This means
3173 // that all lanes in a scalable vector are considered demanded.
3174 APInt DemandedElts
3175 = APInt::getAllOnes(numBits: VT.isScalableVector() ? 1 : VT.getVectorNumElements());
3176
3177 if (isSplatValue(V, DemandedElts, UndefElts)) {
3178 if (VT.isScalableVector()) {
3179 // DemandedElts and UndefElts are ignored for scalable vectors, since
3180 // the only supported cases are SPLAT_VECTOR nodes.
3181 SplatIdx = 0;
3182 } else {
3183 // Handle case where all demanded elements are UNDEF.
3184 if (DemandedElts.isSubsetOf(RHS: UndefElts)) {
3185 SplatIdx = 0;
3186 return getUNDEF(VT);
3187 }
3188 SplatIdx = (UndefElts & DemandedElts).countr_one();
3189 }
3190 return V;
3191 }
3192 break;
3193 }
3194 case ISD::SPLAT_VECTOR:
3195 SplatIdx = 0;
3196 return V;
3197 case ISD::VECTOR_SHUFFLE: {
3198 assert(!VT.isScalableVector());
3199 // Check if this is a shuffle node doing a splat.
3200 // TODO - remove this and rely purely on SelectionDAG::isSplatValue,
3201 // getTargetVShiftNode currently struggles without the splat source.
3202 auto *SVN = cast<ShuffleVectorSDNode>(Val&: V);
3203 if (!SVN->isSplat())
3204 break;
3205 int Idx = SVN->getSplatIndex();
3206 int NumElts = V.getValueType().getVectorNumElements();
3207 SplatIdx = Idx % NumElts;
3208 return V.getOperand(i: Idx / NumElts);
3209 }
3210 }
3211
3212 return SDValue();
3213}
3214
3215SDValue SelectionDAG::getSplatValue(SDValue V, bool LegalTypes) {
3216 int SplatIdx;
3217 if (SDValue SrcVector = getSplatSourceVector(V, SplatIdx)) {
3218 EVT SVT = SrcVector.getValueType().getScalarType();
3219 EVT LegalSVT = SVT;
3220 if (LegalTypes && !TLI->isTypeLegal(VT: SVT)) {
3221 if (!SVT.isInteger())
3222 return SDValue();
3223 LegalSVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT: LegalSVT);
3224 if (LegalSVT.bitsLT(VT: SVT))
3225 return SDValue();
3226 }
3227 return getExtractVectorElt(DL: SDLoc(V), VT: LegalSVT, Vec: SrcVector, Idx: SplatIdx);
3228 }
3229 return SDValue();
3230}
3231
3232std::optional<ConstantRange>
3233SelectionDAG::getValidShiftAmountRange(SDValue V, const APInt &DemandedElts,
3234 unsigned Depth) const {
3235 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
3236 V.getOpcode() == ISD::SRA) &&
3237 "Unknown shift node");
3238 // Shifting more than the bitwidth is not valid.
3239 unsigned BitWidth = V.getScalarValueSizeInBits();
3240
3241 if (auto *Cst = dyn_cast<ConstantSDNode>(Val: V.getOperand(i: 1))) {
3242 const APInt &ShAmt = Cst->getAPIntValue();
3243 if (ShAmt.uge(RHS: BitWidth))
3244 return std::nullopt;
3245 return ConstantRange(ShAmt);
3246 }
3247
3248 if (auto *BV = dyn_cast<BuildVectorSDNode>(Val: V.getOperand(i: 1))) {
3249 const APInt *MinAmt = nullptr, *MaxAmt = nullptr;
3250 for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) {
3251 if (!DemandedElts[i])
3252 continue;
3253 auto *SA = dyn_cast<ConstantSDNode>(Val: BV->getOperand(Num: i));
3254 if (!SA) {
3255 MinAmt = MaxAmt = nullptr;
3256 break;
3257 }
3258 const APInt &ShAmt = SA->getAPIntValue();
3259 if (ShAmt.uge(RHS: BitWidth))
3260 return std::nullopt;
3261 if (!MinAmt || MinAmt->ugt(RHS: ShAmt))
3262 MinAmt = &ShAmt;
3263 if (!MaxAmt || MaxAmt->ult(RHS: ShAmt))
3264 MaxAmt = &ShAmt;
3265 }
3266 assert(((!MinAmt && !MaxAmt) || (MinAmt && MaxAmt)) &&
3267 "Failed to find matching min/max shift amounts");
3268 if (MinAmt && MaxAmt)
3269 return ConstantRange(*MinAmt, *MaxAmt + 1);
3270 }
3271
3272 // Use computeKnownBits to find a hidden constant/knownbits (usually type
3273 // legalized). e.g. Hidden behind multiple bitcasts/build_vector/casts etc.
3274 KnownBits KnownAmt = computeKnownBits(Op: V.getOperand(i: 1), DemandedElts, Depth);
3275 if (KnownAmt.getMaxValue().ult(RHS: BitWidth))
3276 return ConstantRange::fromKnownBits(Known: KnownAmt, /*IsSigned=*/false);
3277
3278 return std::nullopt;
3279}
3280
3281std::optional<unsigned>
3282SelectionDAG::getValidShiftAmount(SDValue V, const APInt &DemandedElts,
3283 unsigned Depth) const {
3284 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
3285 V.getOpcode() == ISD::SRA) &&
3286 "Unknown shift node");
3287 if (std::optional<ConstantRange> AmtRange =
3288 getValidShiftAmountRange(V, DemandedElts, Depth))
3289 if (const APInt *ShAmt = AmtRange->getSingleElement())
3290 return ShAmt->getZExtValue();
3291 return std::nullopt;
3292}
3293
3294std::optional<unsigned>
3295SelectionDAG::getValidShiftAmount(SDValue V, unsigned Depth) const {
3296 APInt DemandedElts = getDemandAllEltsMask(V);
3297 return getValidShiftAmount(V, DemandedElts, Depth);
3298}
3299
3300std::optional<unsigned>
3301SelectionDAG::getValidMinimumShiftAmount(SDValue V, const APInt &DemandedElts,
3302 unsigned Depth) const {
3303 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
3304 V.getOpcode() == ISD::SRA) &&
3305 "Unknown shift node");
3306 if (std::optional<ConstantRange> AmtRange =
3307 getValidShiftAmountRange(V, DemandedElts, Depth))
3308 return AmtRange->getUnsignedMin().getZExtValue();
3309 return std::nullopt;
3310}
3311
3312std::optional<unsigned>
3313SelectionDAG::getValidMinimumShiftAmount(SDValue V, unsigned Depth) const {
3314 APInt DemandedElts = getDemandAllEltsMask(V);
3315 return getValidMinimumShiftAmount(V, DemandedElts, Depth);
3316}
3317
3318std::optional<unsigned>
3319SelectionDAG::getValidMaximumShiftAmount(SDValue V, const APInt &DemandedElts,
3320 unsigned Depth) const {
3321 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
3322 V.getOpcode() == ISD::SRA) &&
3323 "Unknown shift node");
3324 if (std::optional<ConstantRange> AmtRange =
3325 getValidShiftAmountRange(V, DemandedElts, Depth))
3326 return AmtRange->getUnsignedMax().getZExtValue();
3327 return std::nullopt;
3328}
3329
3330std::optional<unsigned>
3331SelectionDAG::getValidMaximumShiftAmount(SDValue V, unsigned Depth) const {
3332 APInt DemandedElts = getDemandAllEltsMask(V);
3333 return getValidMaximumShiftAmount(V, DemandedElts, Depth);
3334}
3335
3336/// Determine which bits of Op are known to be either zero or one and return
3337/// them in Known. For vectors, the known bits are those that are shared by
3338/// every vector element.
3339KnownBits SelectionDAG::computeKnownBits(SDValue Op, unsigned Depth) const {
3340 APInt DemandedElts = getDemandAllEltsMask(V: Op);
3341 return computeKnownBits(Op, DemandedElts, Depth);
3342}
3343
3344/// Determine which bits of Op are known to be either zero or one and return
3345/// them in Known. The DemandedElts argument allows us to only collect the known
3346/// bits that are shared by the requested vector elements.
3347KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts,
3348 unsigned Depth) const {
3349 unsigned BitWidth = Op.getScalarValueSizeInBits();
3350
3351 KnownBits Known(BitWidth); // Don't know anything.
3352
3353 if (auto OptAPInt = Op->bitcastToAPInt()) {
3354 // We know all of the bits for a constant!
3355 return KnownBits::makeConstant(C: *std::move(OptAPInt));
3356 }
3357
3358 if (Depth >= MaxRecursionDepth)
3359 return Known; // Limit search depth.
3360
3361 KnownBits Known2;
3362 unsigned NumElts = DemandedElts.getBitWidth();
3363 assert((!Op.getValueType().isScalableVector() || NumElts == 1) &&
3364 "DemandedElts for scalable vectors must be 1 to represent all lanes");
3365 assert((!Op.getValueType().isFixedLengthVector() ||
3366 NumElts == Op.getValueType().getVectorNumElements()) &&
3367 "Unexpected vector size");
3368
3369 if (!DemandedElts)
3370 return Known; // No demanded elts, better to assume we don't know anything.
3371
3372 unsigned Opcode = Op.getOpcode();
3373 switch (Opcode) {
3374 case ISD::FREEZE: {
3375 if (isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 0), DemandedElts,
3376 Kind: UndefPoisonKind::UndefOrPoison))
3377 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3378 break;
3379 }
3380 case ISD::MERGE_VALUES:
3381 return computeKnownBits(Op: Op.getOperand(i: Op.getResNo()), DemandedElts,
3382 Depth: Depth + 1);
3383 case ISD::SPLAT_VECTOR: {
3384 SDValue SrcOp = Op.getOperand(i: 0);
3385 assert(SrcOp.getValueSizeInBits() >= BitWidth &&
3386 "Expected SPLAT_VECTOR implicit truncation");
3387 // Implicitly truncate the bits to match the official semantics of
3388 // SPLAT_VECTOR.
3389 Known = computeKnownBits(Op: SrcOp, Depth: Depth + 1).trunc(BitWidth);
3390 break;
3391 }
3392 case ISD::SPLAT_VECTOR_PARTS: {
3393 unsigned ScalarSize = Op.getOperand(i: 0).getScalarValueSizeInBits();
3394 assert(ScalarSize * Op.getNumOperands() == BitWidth &&
3395 "Expected SPLAT_VECTOR_PARTS scalars to cover element width");
3396 for (auto [I, SrcOp] : enumerate(First: Op->ops())) {
3397 Known.insertBits(SubBits: computeKnownBits(Op: SrcOp, Depth: Depth + 1), BitPosition: ScalarSize * I);
3398 }
3399 break;
3400 }
3401 case ISD::STEP_VECTOR: {
3402 const APInt &Step = Op.getConstantOperandAPInt(i: 0);
3403
3404 if (Step.isPowerOf2())
3405 Known.Zero.setLowBits(Step.logBase2());
3406
3407 const Function &F = getMachineFunction().getFunction();
3408
3409 if (!isUIntN(N: BitWidth, x: Op.getValueType().getVectorMinNumElements()))
3410 break;
3411 const APInt MinNumElts =
3412 APInt(BitWidth, Op.getValueType().getVectorMinNumElements());
3413
3414 bool Overflow;
3415 const APInt MaxNumElts = getVScaleRange(F: &F, BitWidth)
3416 .getUnsignedMax()
3417 .umul_ov(RHS: MinNumElts, Overflow);
3418 if (Overflow)
3419 break;
3420
3421 const APInt MaxValue = (MaxNumElts - 1).umul_ov(RHS: Step, Overflow);
3422 if (Overflow)
3423 break;
3424
3425 Known.Zero.setHighBits(MaxValue.countl_zero());
3426 break;
3427 }
3428 case ISD::BUILD_VECTOR:
3429 assert(!Op.getValueType().isScalableVector());
3430 // Collect the known bits that are shared by every demanded vector element.
3431 Known.setAllConflict();
3432 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
3433 if (!DemandedElts[i])
3434 continue;
3435
3436 SDValue SrcOp = Op.getOperand(i);
3437 if (SrcOp.getOpcode() == ISD::POISON)
3438 continue;
3439
3440 Known2 = computeKnownBits(Op: SrcOp, Depth: Depth + 1);
3441
3442 // BUILD_VECTOR can implicitly truncate sources, we must handle this.
3443 if (SrcOp.getValueSizeInBits() != BitWidth) {
3444 assert(SrcOp.getValueSizeInBits() > BitWidth &&
3445 "Expected BUILD_VECTOR implicit truncation");
3446 Known2 = Known2.trunc(BitWidth);
3447 }
3448
3449 // Known bits are the values that are shared by every demanded element.
3450 Known = Known.intersectWith(RHS: Known2);
3451
3452 // If we don't know any bits, early out.
3453 if (Known.isUnknown())
3454 break;
3455 }
3456
3457 // If every demanded element was poison, we know nothing.
3458 if (Known.hasConflict())
3459 Known.resetAll();
3460 break;
3461 case ISD::VECTOR_COMPRESS: {
3462 SDValue Vec = Op.getOperand(i: 0);
3463 SDValue PassThru = Op.getOperand(i: 2);
3464 Known = computeKnownBits(Op: PassThru, DemandedElts, Depth: Depth + 1);
3465 // If we don't know any bits, early out.
3466 if (Known.isUnknown())
3467 break;
3468 Known2 = computeKnownBits(Op: Vec, Depth: Depth + 1);
3469 Known = Known.intersectWith(RHS: Known2);
3470 break;
3471 }
3472 case ISD::VECTOR_SHUFFLE: {
3473 assert(!Op.getValueType().isScalableVector());
3474 // Collect the known bits that are shared by every vector element referenced
3475 // by the shuffle.
3476 APInt DemandedLHS, DemandedRHS;
3477 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
3478 assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
3479 if (!getShuffleDemandedElts(SrcWidth: NumElts, Mask: SVN->getMask(), DemandedElts,
3480 DemandedLHS, DemandedRHS))
3481 break;
3482
3483 // Known bits are the values that are shared by every demanded element.
3484 Known.setAllConflict();
3485 if (!!DemandedLHS) {
3486 SDValue LHS = Op.getOperand(i: 0);
3487 Known2 = computeKnownBits(Op: LHS, DemandedElts: DemandedLHS, Depth: Depth + 1);
3488 Known = Known.intersectWith(RHS: Known2);
3489 }
3490 // If we don't know any bits, early out.
3491 if (Known.isUnknown())
3492 break;
3493 if (!!DemandedRHS) {
3494 SDValue RHS = Op.getOperand(i: 1);
3495 Known2 = computeKnownBits(Op: RHS, DemandedElts: DemandedRHS, Depth: Depth + 1);
3496 Known = Known.intersectWith(RHS: Known2);
3497 }
3498 break;
3499 }
3500 case ISD::VSCALE: {
3501 const Function &F = getMachineFunction().getFunction();
3502 const APInt &Multiplier = Op.getConstantOperandAPInt(i: 0);
3503 Known = getVScaleRange(F: &F, BitWidth).multiply(Other: Multiplier).toKnownBits();
3504 break;
3505 }
3506 case ISD::CONCAT_VECTORS: {
3507 if (Op.getValueType().isScalableVector())
3508 break;
3509 // Split DemandedElts and test each of the demanded subvectors.
3510 Known.setAllConflict();
3511 EVT SubVectorVT = Op.getOperand(i: 0).getValueType();
3512 unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements();
3513 unsigned NumSubVectors = Op.getNumOperands();
3514 for (unsigned i = 0; i != NumSubVectors; ++i) {
3515 APInt DemandedSub =
3516 DemandedElts.extractBits(numBits: NumSubVectorElts, bitPosition: i * NumSubVectorElts);
3517 if (!!DemandedSub) {
3518 SDValue Sub = Op.getOperand(i);
3519 Known2 = computeKnownBits(Op: Sub, DemandedElts: DemandedSub, Depth: Depth + 1);
3520 Known = Known.intersectWith(RHS: Known2);
3521 }
3522 // If we don't know any bits, early out.
3523 if (Known.isUnknown())
3524 break;
3525 }
3526 break;
3527 }
3528 case ISD::INSERT_SUBVECTOR: {
3529 if (Op.getValueType().isScalableVector())
3530 break;
3531 // Demand any elements from the subvector and the remainder from the src its
3532 // inserted into.
3533 SDValue Src = Op.getOperand(i: 0);
3534 SDValue Sub = Op.getOperand(i: 1);
3535 uint64_t Idx = Op.getConstantOperandVal(i: 2);
3536 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
3537 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
3538 APInt DemandedSrcElts = DemandedElts;
3539 DemandedSrcElts.clearBits(LoBit: Idx, HiBit: Idx + NumSubElts);
3540
3541 Known.setAllConflict();
3542 if (!!DemandedSubElts) {
3543 Known = computeKnownBits(Op: Sub, DemandedElts: DemandedSubElts, Depth: Depth + 1);
3544 if (Known.isUnknown())
3545 break; // early-out.
3546 }
3547 if (!!DemandedSrcElts) {
3548 Known2 = computeKnownBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
3549 Known = Known.intersectWith(RHS: Known2);
3550 }
3551 break;
3552 }
3553 case ISD::EXTRACT_SUBVECTOR: {
3554 // Offset the demanded elts by the subvector index.
3555 SDValue Src = Op.getOperand(i: 0);
3556
3557 APInt DemandedSrcElts;
3558 if (Src.getValueType().isScalableVector())
3559 DemandedSrcElts = APInt(1, 1); // <=> 'demand all elements'
3560 else {
3561 uint64_t Idx = Op.getConstantOperandVal(i: 1);
3562 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3563 DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
3564 }
3565 Known = computeKnownBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
3566 break;
3567 }
3568 case ISD::SCALAR_TO_VECTOR: {
3569 if (Op.getValueType().isScalableVector())
3570 break;
3571 // We know about scalar_to_vector as much as we know about it source,
3572 // which becomes the first element of otherwise unknown vector.
3573 if (DemandedElts != 1)
3574 break;
3575
3576 SDValue N0 = Op.getOperand(i: 0);
3577 Known = computeKnownBits(Op: N0, Depth: Depth + 1);
3578 if (N0.getValueSizeInBits() != BitWidth)
3579 Known = Known.trunc(BitWidth);
3580
3581 break;
3582 }
3583 case ISD::BITCAST: {
3584 if (Op.getValueType().isScalableVector())
3585 break;
3586
3587 SDValue N0 = Op.getOperand(i: 0);
3588 EVT SubVT = N0.getValueType();
3589 unsigned SubBitWidth = SubVT.getScalarSizeInBits();
3590
3591 // Ignore bitcasts from unsupported types.
3592 if (!(SubVT.isInteger() || SubVT.isFloatingPoint()))
3593 break;
3594
3595 // Fast handling of 'identity' bitcasts.
3596 if (BitWidth == SubBitWidth) {
3597 Known = computeKnownBits(Op: N0, DemandedElts, Depth: Depth + 1);
3598 break;
3599 }
3600
3601 bool IsLE = getDataLayout().isLittleEndian();
3602
3603 // Bitcast 'small element' vector to 'large element' scalar/vector.
3604 if ((BitWidth % SubBitWidth) == 0) {
3605 assert(N0.getValueType().isVector() && "Expected bitcast from vector");
3606
3607 // Collect known bits for the (larger) output by collecting the known
3608 // bits from each set of sub elements and shift these into place.
3609 // We need to separately call computeKnownBits for each set of
3610 // sub elements as the knownbits for each is likely to be different.
3611 unsigned SubScale = BitWidth / SubBitWidth;
3612 APInt SubDemandedElts(NumElts * SubScale, 0);
3613 for (unsigned i = 0; i != NumElts; ++i)
3614 if (DemandedElts[i])
3615 SubDemandedElts.setBit(i * SubScale);
3616
3617 for (unsigned i = 0; i != SubScale; ++i) {
3618 Known2 = computeKnownBits(Op: N0, DemandedElts: SubDemandedElts.shl(shiftAmt: i),
3619 Depth: Depth + 1);
3620 unsigned Shifts = IsLE ? i : SubScale - 1 - i;
3621 Known.insertBits(SubBits: Known2, BitPosition: SubBitWidth * Shifts);
3622 }
3623 }
3624
3625 // Bitcast 'large element' scalar/vector to 'small element' vector.
3626 if ((SubBitWidth % BitWidth) == 0) {
3627 assert(Op.getValueType().isVector() && "Expected bitcast to vector");
3628
3629 // Collect known bits for the (smaller) output by collecting the known
3630 // bits from the overlapping larger input elements and extracting the
3631 // sub sections we actually care about.
3632 unsigned SubScale = SubBitWidth / BitWidth;
3633 APInt SubDemandedElts =
3634 APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumElts / SubScale);
3635 Known2 = computeKnownBits(Op: N0, DemandedElts: SubDemandedElts, Depth: Depth + 1);
3636
3637 Known.setAllConflict();
3638 for (unsigned i = 0; i != NumElts; ++i)
3639 if (DemandedElts[i]) {
3640 unsigned Shifts = IsLE ? i : NumElts - 1 - i;
3641 unsigned Offset = (Shifts % SubScale) * BitWidth;
3642 Known = Known.intersectWith(RHS: Known2.extractBits(NumBits: BitWidth, BitPosition: Offset));
3643 // If we don't know any bits, early out.
3644 if (Known.isUnknown())
3645 break;
3646 }
3647 }
3648 break;
3649 }
3650 case ISD::AND:
3651 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3652 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3653
3654 Known &= Known2;
3655 break;
3656 case ISD::OR:
3657 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3658 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3659
3660 Known |= Known2;
3661 break;
3662 case ISD::XOR:
3663 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3664 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3665
3666 Known ^= Known2;
3667 break;
3668 case ISD::MUL: {
3669 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3670 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3671 bool SelfMultiply = Op.getOperand(i: 0) == Op.getOperand(i: 1);
3672 // TODO: SelfMultiply can be poison, but not undef.
3673 if (SelfMultiply)
3674 SelfMultiply &= isGuaranteedNotToBeUndefOrPoison(
3675 Op: Op.getOperand(i: 0), DemandedElts, Kind: UndefPoisonKind::UndefOrPoison,
3676 Depth: Depth + 1);
3677 Known = KnownBits::mul(LHS: Known, RHS: Known2, NoUndefSelfMultiply: SelfMultiply);
3678
3679 // If the multiplication is known not to overflow, the product of a number
3680 // with itself is non-negative. Only do this if we didn't already computed
3681 // the opposite value for the sign bit.
3682 if (Op->getFlags().hasNoSignedWrap() &&
3683 Op.getOperand(i: 0) == Op.getOperand(i: 1) &&
3684 !Known.isNegative())
3685 Known.makeNonNegative();
3686 break;
3687 }
3688 case ISD::MULHU: {
3689 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3690 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3691 Known = KnownBits::mulhu(LHS: Known, RHS: Known2);
3692 break;
3693 }
3694 case ISD::MULHS: {
3695 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3696 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3697 Known = KnownBits::mulhs(LHS: Known, RHS: Known2);
3698 break;
3699 }
3700 case ISD::ABDU: {
3701 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3702 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3703 Known = KnownBits::abdu(LHS: Known, RHS: Known2);
3704 break;
3705 }
3706 case ISD::ABDS: {
3707 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3708 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3709 Known = KnownBits::abds(LHS: Known, RHS: Known2);
3710 unsigned SignBits1 =
3711 ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3712 if (SignBits1 == 1)
3713 break;
3714 unsigned SignBits0 =
3715 ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3716 Known.Zero.setHighBits(std::min(a: SignBits0, b: SignBits1) - 1);
3717 break;
3718 }
3719 case ISD::UMUL_LOHI: {
3720 assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result");
3721 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3722 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3723 bool SelfMultiply = Op.getOperand(i: 0) == Op.getOperand(i: 1);
3724 if (Op.getResNo() == 0)
3725 Known = KnownBits::mul(LHS: Known, RHS: Known2, NoUndefSelfMultiply: SelfMultiply);
3726 else
3727 Known = KnownBits::mulhu(LHS: Known, RHS: Known2);
3728 break;
3729 }
3730 case ISD::SMUL_LOHI: {
3731 assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result");
3732 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3733 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3734 bool SelfMultiply = Op.getOperand(i: 0) == Op.getOperand(i: 1);
3735 if (Op.getResNo() == 0)
3736 Known = KnownBits::mul(LHS: Known, RHS: Known2, NoUndefSelfMultiply: SelfMultiply);
3737 else
3738 Known = KnownBits::mulhs(LHS: Known, RHS: Known2);
3739 break;
3740 }
3741 case ISD::AVGFLOORU: {
3742 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3743 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3744 Known = KnownBits::avgFloorU(LHS: Known, RHS: Known2);
3745 break;
3746 }
3747 case ISD::AVGCEILU: {
3748 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3749 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3750 Known = KnownBits::avgCeilU(LHS: Known, RHS: Known2);
3751 break;
3752 }
3753 case ISD::AVGFLOORS: {
3754 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3755 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3756 Known = KnownBits::avgFloorS(LHS: Known, RHS: Known2);
3757 break;
3758 }
3759 case ISD::AVGCEILS: {
3760 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3761 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3762 Known = KnownBits::avgCeilS(LHS: Known, RHS: Known2);
3763 break;
3764 }
3765 case ISD::SELECT:
3766 case ISD::VSELECT:
3767 Known = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth+1);
3768 // If we don't know any bits, early out.
3769 if (Known.isUnknown())
3770 break;
3771 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth+1);
3772
3773 // Only known if known in both the LHS and RHS.
3774 Known = Known.intersectWith(RHS: Known2);
3775 break;
3776 case ISD::SELECT_CC:
3777 Known = computeKnownBits(Op: Op.getOperand(i: 3), DemandedElts, Depth: Depth+1);
3778 // If we don't know any bits, early out.
3779 if (Known.isUnknown())
3780 break;
3781 Known2 = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth+1);
3782
3783 // Only known if known in both the LHS and RHS.
3784 Known = Known.intersectWith(RHS: Known2);
3785 break;
3786 case ISD::SMULO:
3787 case ISD::UMULO:
3788 if (Op.getResNo() != 1)
3789 break;
3790 // The boolean result conforms to getBooleanContents.
3791 // If we know the result of a setcc has the top bits zero, use this info.
3792 // We know that we have an integer-based boolean since these operations
3793 // are only available for integer.
3794 if (TLI->getBooleanContents(isVec: Op.getValueType().isVector(), isFloat: false) ==
3795 TargetLowering::ZeroOrOneBooleanContent &&
3796 BitWidth > 1)
3797 Known.Zero.setBitsFrom(1);
3798 break;
3799 case ISD::SETCC:
3800 case ISD::SETCCCARRY:
3801 case ISD::STRICT_FSETCC:
3802 case ISD::STRICT_FSETCCS: {
3803 unsigned OpNo = Op->isStrictFPOpcode() ? 1 : 0;
3804 // If we know the result of a setcc has the top bits zero, use this info.
3805 if (TLI->getBooleanContents(Type: Op.getOperand(i: OpNo).getValueType()) ==
3806 TargetLowering::ZeroOrOneBooleanContent &&
3807 BitWidth > 1)
3808 Known.Zero.setBitsFrom(1);
3809 break;
3810 }
3811 case ISD::SHL: {
3812 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3813 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3814
3815 bool NUW = Op->getFlags().hasNoUnsignedWrap();
3816 bool NSW = Op->getFlags().hasNoSignedWrap();
3817
3818 bool ShAmtNonZero = Known2.isNonZero();
3819
3820 Known = KnownBits::shl(LHS: Known, RHS: Known2, NUW, NSW, ShAmtNonZero);
3821
3822 // Minimum shift low bits are known zero.
3823 if (std::optional<unsigned> ShMinAmt =
3824 getValidMinimumShiftAmount(V: Op, DemandedElts, Depth: Depth + 1))
3825 Known.Zero.setLowBits(*ShMinAmt);
3826 break;
3827 }
3828 case ISD::SRL:
3829 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3830 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3831 Known = KnownBits::lshr(LHS: Known, RHS: Known2, /*ShAmtNonZero=*/false,
3832 Exact: Op->getFlags().hasExact());
3833
3834 // Minimum shift high bits are known zero.
3835 if (std::optional<unsigned> ShMinAmt =
3836 getValidMinimumShiftAmount(V: Op, DemandedElts, Depth: Depth + 1))
3837 Known.Zero.setHighBits(*ShMinAmt);
3838 break;
3839 case ISD::SRA:
3840 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3841 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3842 Known = KnownBits::ashr(LHS: Known, RHS: Known2, /*ShAmtNonZero=*/false,
3843 Exact: Op->getFlags().hasExact());
3844 break;
3845 case ISD::ROTL:
3846 case ISD::ROTR:
3847 if (ConstantSDNode *C =
3848 isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts)) {
3849 unsigned Amt = C->getAPIntValue().urem(RHS: BitWidth);
3850
3851 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3852
3853 // Canonicalize to ROTR.
3854 if (Opcode == ISD::ROTL && Amt != 0)
3855 Amt = BitWidth - Amt;
3856
3857 Known.Zero = Known.Zero.rotr(rotateAmt: Amt);
3858 Known.One = Known.One.rotr(rotateAmt: Amt);
3859 }
3860 break;
3861 case ISD::FSHL:
3862 case ISD::FSHR:
3863 if (ConstantSDNode *C = isConstOrConstSplat(N: Op.getOperand(i: 2), DemandedElts)) {
3864 unsigned Amt = C->getAPIntValue().urem(RHS: BitWidth);
3865
3866 // For fshl, 0-shift returns the 1st arg.
3867 // For fshr, 0-shift returns the 2nd arg.
3868 if (Amt == 0) {
3869 Known = computeKnownBits(Op: Op.getOperand(i: Opcode == ISD::FSHL ? 0 : 1),
3870 DemandedElts, Depth: Depth + 1);
3871 break;
3872 }
3873
3874 // fshl: (X << (Z % BW)) | (Y >> (BW - (Z % BW)))
3875 // fshr: (X << (BW - (Z % BW))) | (Y >> (Z % BW))
3876 const APInt ShAmt(BitWidth, Amt);
3877 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3878 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3879 Known = Opcode == ISD::FSHL ? KnownBits::fshl(LHS: Known, RHS: Known2, Amt: ShAmt)
3880 : KnownBits::fshr(LHS: Known, RHS: Known2, Amt: ShAmt);
3881 }
3882 break;
3883 case ISD::SHL_PARTS:
3884 case ISD::SRA_PARTS:
3885 case ISD::SRL_PARTS: {
3886 assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result");
3887
3888 // Collect lo/hi source values and concatenate.
3889 unsigned LoBits = Op.getOperand(i: 0).getScalarValueSizeInBits();
3890 unsigned HiBits = Op.getOperand(i: 1).getScalarValueSizeInBits();
3891 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3892 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3893 Known = Known2.concat(Lo: Known);
3894
3895 // Collect shift amount.
3896 Known2 = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth + 1);
3897
3898 if (Opcode == ISD::SHL_PARTS)
3899 Known = KnownBits::shl(LHS: Known, RHS: Known2);
3900 else if (Opcode == ISD::SRA_PARTS)
3901 Known = KnownBits::ashr(LHS: Known, RHS: Known2);
3902 else // if (Opcode == ISD::SRL_PARTS)
3903 Known = KnownBits::lshr(LHS: Known, RHS: Known2);
3904
3905 // TODO: Minimum shift low/high bits are known zero.
3906
3907 if (Op.getResNo() == 0)
3908 Known = Known.extractBits(NumBits: LoBits, BitPosition: 0);
3909 else
3910 Known = Known.extractBits(NumBits: HiBits, BitPosition: LoBits);
3911 break;
3912 }
3913 case ISD::SIGN_EXTEND_INREG: {
3914 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3915 EVT EVT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
3916 Known = Known.sextInReg(SrcBitWidth: EVT.getScalarSizeInBits());
3917 break;
3918 }
3919 case ISD::CTTZ:
3920 case ISD::CTTZ_ZERO_POISON: {
3921 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3922 // If we have a known 1, its position is our upper bound.
3923 unsigned PossibleTZ = Known2.countMaxTrailingZeros();
3924 unsigned LowBits = llvm::bit_width(Value: PossibleTZ);
3925 Known.Zero.setBitsFrom(LowBits);
3926 break;
3927 }
3928 case ISD::CTLZ:
3929 case ISD::CTLZ_ZERO_POISON: {
3930 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3931 // If we have a known 1, its position is our upper bound.
3932 unsigned PossibleLZ = Known2.countMaxLeadingZeros();
3933 unsigned LowBits = llvm::bit_width(Value: PossibleLZ);
3934 Known.Zero.setBitsFrom(LowBits);
3935 break;
3936 }
3937 case ISD::CTLS: {
3938 unsigned MinRedundantSignBits =
3939 ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1) - 1;
3940 ConstantRange Range(APInt(BitWidth, MinRedundantSignBits),
3941 APInt(BitWidth, BitWidth));
3942 Known = Range.toKnownBits();
3943 break;
3944 }
3945 case ISD::CTPOP: {
3946 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3947 // If we know some of the bits are zero, they can't be one.
3948 unsigned PossibleOnes = Known2.countMaxPopulation();
3949 Known.Zero.setBitsFrom(llvm::bit_width(Value: PossibleOnes));
3950 break;
3951 }
3952 case ISD::PARITY: {
3953 // Parity returns 0 everywhere but the LSB.
3954 Known.Zero.setBitsFrom(1);
3955 break;
3956 }
3957 case ISD::PDEP: {
3958 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3959 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3960 Known = KnownBits::pdep(Val: Known2, Mask: Known);
3961 break;
3962 }
3963 case ISD::PEXT: {
3964 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3965 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3966 Known = KnownBits::pext(Val: Known2, Mask: Known);
3967 break;
3968 }
3969 case ISD::CLMUL: {
3970 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3971 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3972 Known = KnownBits::clmul(LHS: Known, RHS: Known2);
3973 break;
3974 }
3975 case ISD::MGATHER:
3976 case ISD::MLOAD: {
3977 ISD::LoadExtType ETy =
3978 (Opcode == ISD::MGATHER)
3979 ? cast<MaskedGatherSDNode>(Val&: Op)->getExtensionType()
3980 : cast<MaskedLoadSDNode>(Val&: Op)->getExtensionType();
3981 if (ETy == ISD::ZEXTLOAD) {
3982 EVT MemVT = cast<MemSDNode>(Val&: Op)->getMemoryVT();
3983 KnownBits Known0(MemVT.getScalarSizeInBits());
3984 return Known0.zext(BitWidth);
3985 }
3986 break;
3987 }
3988 case ISD::LOAD: {
3989 LoadSDNode *LD = cast<LoadSDNode>(Val&: Op);
3990 const Constant *Cst = TLI->getTargetConstantFromLoad(LD);
3991 if (ISD::isNON_EXTLoad(N: LD) && Cst) {
3992 // Determine any common known bits from the loaded constant pool value.
3993 Type *CstTy = Cst->getType();
3994 if ((NumElts * BitWidth) == CstTy->getPrimitiveSizeInBits() &&
3995 !Op.getValueType().isScalableVector()) {
3996 // If its a vector splat, then we can (quickly) reuse the scalar path.
3997 // NOTE: We assume all elements match and none are UNDEF.
3998 if (CstTy->isVectorTy()) {
3999 if (const Constant *Splat = Cst->getSplatValue()) {
4000 Cst = Splat;
4001 CstTy = Cst->getType();
4002 }
4003 }
4004 // TODO - do we need to handle different bitwidths?
4005 if (CstTy->isVectorTy() && BitWidth == CstTy->getScalarSizeInBits()) {
4006 // Iterate across all vector elements finding common known bits.
4007 Known.setAllConflict();
4008 for (unsigned i = 0; i != NumElts; ++i) {
4009 if (!DemandedElts[i])
4010 continue;
4011 if (Constant *Elt = Cst->getAggregateElement(Elt: i)) {
4012 if (auto *CInt = dyn_cast<ConstantInt>(Val: Elt)) {
4013 const APInt &Value = CInt->getValue();
4014 Known.One &= Value;
4015 Known.Zero &= ~Value;
4016 continue;
4017 }
4018 if (auto *CFP = dyn_cast<ConstantFP>(Val: Elt)) {
4019 APInt Value = CFP->getValueAPF().bitcastToAPInt();
4020 Known.One &= Value;
4021 Known.Zero &= ~Value;
4022 continue;
4023 }
4024 }
4025 Known.One.clearAllBits();
4026 Known.Zero.clearAllBits();
4027 break;
4028 }
4029 } else if (BitWidth == CstTy->getPrimitiveSizeInBits()) {
4030 if (auto *CInt = dyn_cast<ConstantInt>(Val: Cst)) {
4031 Known = KnownBits::makeConstant(C: CInt->getValue());
4032 } else if (auto *CFP = dyn_cast<ConstantFP>(Val: Cst)) {
4033 Known =
4034 KnownBits::makeConstant(C: CFP->getValueAPF().bitcastToAPInt());
4035 }
4036 }
4037 }
4038 } else if (Op.getResNo() == 0) {
4039 unsigned ScalarMemorySize = LD->getMemoryVT().getScalarSizeInBits();
4040 KnownBits KnownScalarMemory(ScalarMemorySize);
4041 if (const MDNode *MD = LD->getRanges())
4042 computeKnownBitsFromRangeMetadata(Ranges: *MD, Known&: KnownScalarMemory);
4043
4044 // Extend the Known bits from memory to the size of the scalar result.
4045 if (ISD::isZEXTLoad(N: Op.getNode()))
4046 Known = KnownScalarMemory.zext(BitWidth);
4047 else if (ISD::isSEXTLoad(N: Op.getNode()))
4048 Known = KnownScalarMemory.sext(BitWidth);
4049 else if (ISD::isEXTLoad(N: Op.getNode()))
4050 Known = KnownScalarMemory.anyext(BitWidth);
4051 else
4052 Known = KnownScalarMemory;
4053 assert(Known.getBitWidth() == BitWidth);
4054 return Known;
4055 }
4056 break;
4057 }
4058 case ISD::ZERO_EXTEND_VECTOR_INREG: {
4059 if (Op.getValueType().isScalableVector())
4060 break;
4061 EVT InVT = Op.getOperand(i: 0).getValueType();
4062 APInt InDemandedElts = DemandedElts.zext(width: InVT.getVectorNumElements());
4063 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts: InDemandedElts, Depth: Depth + 1);
4064 Known = Known.zext(BitWidth);
4065 break;
4066 }
4067 case ISD::ZERO_EXTEND: {
4068 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4069 Known = Known.zext(BitWidth);
4070 break;
4071 }
4072 case ISD::SIGN_EXTEND_VECTOR_INREG: {
4073 if (Op.getValueType().isScalableVector())
4074 break;
4075 EVT InVT = Op.getOperand(i: 0).getValueType();
4076 APInt InDemandedElts = DemandedElts.zext(width: InVT.getVectorNumElements());
4077 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts: InDemandedElts, Depth: Depth + 1);
4078 // If the sign bit is known to be zero or one, then sext will extend
4079 // it to the top bits, else it will just zext.
4080 Known = Known.sext(BitWidth);
4081 break;
4082 }
4083 case ISD::SIGN_EXTEND: {
4084 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4085 // If the sign bit is known to be zero or one, then sext will extend
4086 // it to the top bits, else it will just zext.
4087 Known = Known.sext(BitWidth);
4088 break;
4089 }
4090 case ISD::ANY_EXTEND_VECTOR_INREG: {
4091 if (Op.getValueType().isScalableVector())
4092 break;
4093 EVT InVT = Op.getOperand(i: 0).getValueType();
4094 APInt InDemandedElts = DemandedElts.zext(width: InVT.getVectorNumElements());
4095 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts: InDemandedElts, Depth: Depth + 1);
4096 Known = Known.anyext(BitWidth);
4097 break;
4098 }
4099 case ISD::ANY_EXTEND: {
4100 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4101 Known = Known.anyext(BitWidth);
4102 break;
4103 }
4104 case ISD::TRUNCATE: {
4105 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4106 Known = Known.trunc(BitWidth);
4107 break;
4108 }
4109 case ISD::TRUNCATE_SSAT_S: {
4110 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4111 Known = Known.truncSSat(BitWidth);
4112 break;
4113 }
4114 case ISD::TRUNCATE_SSAT_U: {
4115 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4116 Known = Known.truncSSatU(BitWidth);
4117 break;
4118 }
4119 case ISD::TRUNCATE_USAT_U: {
4120 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4121 Known = Known.truncUSat(BitWidth);
4122 break;
4123 }
4124 case ISD::AssertZext: {
4125 EVT VT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
4126 APInt InMask = APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: VT.getSizeInBits());
4127 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4128 Known.Zero |= (~InMask);
4129 Known.One &= (~Known.Zero);
4130 break;
4131 }
4132 case ISD::AssertAlign: {
4133 unsigned LogOfAlign = Log2(A: cast<AssertAlignSDNode>(Val&: Op)->getAlign());
4134 assert(LogOfAlign != 0);
4135
4136 // TODO: Should use maximum with source
4137 // If a node is guaranteed to be aligned, set low zero bits accordingly as
4138 // well as clearing one bits.
4139 Known.Zero.setLowBits(LogOfAlign);
4140 Known.One.clearLowBits(loBits: LogOfAlign);
4141 break;
4142 }
4143 case ISD::AssertNoFPClass: {
4144 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4145
4146 FPClassTest NoFPClass =
4147 static_cast<FPClassTest>(Op.getConstantOperandVal(i: 1));
4148 const FPClassTest NegativeTestMask = fcNan | fcNegative;
4149 if ((NoFPClass & NegativeTestMask) == NegativeTestMask) {
4150 // Cannot be negative.
4151 Known.makeNonNegative();
4152 }
4153
4154 const FPClassTest PositiveTestMask = fcNan | fcPositive;
4155 if ((NoFPClass & PositiveTestMask) == PositiveTestMask) {
4156 // Cannot be positive.
4157 Known.makeNegative();
4158 }
4159
4160 break;
4161 }
4162 case ISD::FABS:
4163 // fabs clears the sign bit
4164 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4165 Known.makeNonNegative();
4166 break;
4167 case ISD::FGETSIGN:
4168 // All bits are zero except the low bit.
4169 Known.Zero.setBitsFrom(1);
4170 break;
4171 case ISD::ADD: {
4172 SDNodeFlags Flags = Op.getNode()->getFlags();
4173 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4174 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4175 bool SelfAdd = Op.getOperand(i: 0) == Op.getOperand(i: 1) &&
4176 isGuaranteedNotToBeUndefOrPoison(
4177 Op: Op.getOperand(i: 0), DemandedElts,
4178 Kind: UndefPoisonKind::UndefOrPoison, Depth: Depth + 1);
4179 Known = KnownBits::add(LHS: Known, RHS: Known2, NSW: Flags.hasNoSignedWrap(),
4180 NUW: Flags.hasNoUnsignedWrap(), SelfAdd);
4181 break;
4182 }
4183 case ISD::SUB: {
4184 SDNodeFlags Flags = Op.getNode()->getFlags();
4185 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4186 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4187 Known = KnownBits::sub(LHS: Known, RHS: Known2, NSW: Flags.hasNoSignedWrap(),
4188 NUW: Flags.hasNoUnsignedWrap());
4189 break;
4190 }
4191 case ISD::USUBO:
4192 case ISD::SSUBO:
4193 case ISD::USUBO_CARRY:
4194 case ISD::SSUBO_CARRY:
4195 if (Op.getResNo() == 1) {
4196 // If we know the result of a setcc has the top bits zero, use this info.
4197 if (TLI->getBooleanContents(Type: Op.getOperand(i: 0).getValueType()) ==
4198 TargetLowering::ZeroOrOneBooleanContent &&
4199 BitWidth > 1)
4200 Known.Zero.setBitsFrom(1);
4201 break;
4202 }
4203 [[fallthrough]];
4204 case ISD::SUBC: {
4205 assert(Op.getResNo() == 0 &&
4206 "We only compute knownbits for the difference here.");
4207
4208 // With USUBO_CARRY and SSUBO_CARRY a borrow bit may be added in.
4209 KnownBits Borrow(1);
4210 if (Opcode == ISD::USUBO_CARRY || Opcode == ISD::SSUBO_CARRY) {
4211 Borrow = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth + 1);
4212 // Borrow has bit width 1
4213 Borrow = Borrow.trunc(BitWidth: 1);
4214 } else {
4215 Borrow.setAllZero();
4216 }
4217
4218 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4219 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4220 Known = KnownBits::computeForSubBorrow(LHS: Known, RHS: Known2, Borrow);
4221 break;
4222 }
4223 case ISD::UADDO:
4224 case ISD::SADDO:
4225 case ISD::UADDO_CARRY:
4226 case ISD::SADDO_CARRY:
4227 if (Op.getResNo() == 1) {
4228 // If we know the result of a setcc has the top bits zero, use this info.
4229 if (TLI->getBooleanContents(Type: Op.getOperand(i: 0).getValueType()) ==
4230 TargetLowering::ZeroOrOneBooleanContent &&
4231 BitWidth > 1)
4232 Known.Zero.setBitsFrom(1);
4233 break;
4234 }
4235 [[fallthrough]];
4236 case ISD::ADDC:
4237 case ISD::ADDE: {
4238 assert(Op.getResNo() == 0 && "We only compute knownbits for the sum here.");
4239
4240 // With ADDE and UADDO_CARRY, a carry bit may be added in.
4241 KnownBits Carry(1);
4242 if (Opcode == ISD::ADDE)
4243 // Can't track carry from glue, set carry to unknown.
4244 Carry.resetAll();
4245 else if (Opcode == ISD::UADDO_CARRY || Opcode == ISD::SADDO_CARRY) {
4246 Carry = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth + 1);
4247 // Carry has bit width 1
4248 Carry = Carry.trunc(BitWidth: 1);
4249 } else {
4250 Carry.setAllZero();
4251 }
4252
4253 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4254 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4255 Known = KnownBits::computeForAddCarry(LHS: Known, RHS: Known2, Carry);
4256 break;
4257 }
4258 case ISD::UDIV: {
4259 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4260 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4261 Known = KnownBits::udiv(LHS: Known, RHS: Known2, Exact: Op->getFlags().hasExact());
4262 break;
4263 }
4264 case ISD::SDIV: {
4265 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4266 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4267 Known = KnownBits::sdiv(LHS: Known, RHS: Known2, Exact: Op->getFlags().hasExact());
4268 break;
4269 }
4270 case ISD::SREM: {
4271 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4272 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4273 Known = KnownBits::srem(LHS: Known, RHS: Known2);
4274 break;
4275 }
4276 case ISD::UREM: {
4277 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4278 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4279 Known = KnownBits::urem(LHS: Known, RHS: Known2);
4280 break;
4281 }
4282 case ISD::EXTRACT_ELEMENT: {
4283 Known = computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth+1);
4284 const unsigned Index = Op.getConstantOperandVal(i: 1);
4285 const unsigned EltBitWidth = Op.getValueSizeInBits();
4286
4287 // Remove low part of known bits mask
4288 Known.Zero = Known.Zero.getHiBits(numBits: Known.getBitWidth() - Index * EltBitWidth);
4289 Known.One = Known.One.getHiBits(numBits: Known.getBitWidth() - Index * EltBitWidth);
4290
4291 // Remove high part of known bit mask
4292 Known = Known.trunc(BitWidth: EltBitWidth);
4293 break;
4294 }
4295 case ISD::EXTRACT_VECTOR_ELT: {
4296 SDValue InVec = Op.getOperand(i: 0);
4297 SDValue EltNo = Op.getOperand(i: 1);
4298 EVT VecVT = InVec.getValueType();
4299 // computeKnownBits not yet implemented for scalable vectors.
4300 if (VecVT.isScalableVector())
4301 break;
4302 const unsigned EltBitWidth = VecVT.getScalarSizeInBits();
4303 const unsigned NumSrcElts = VecVT.getVectorNumElements();
4304
4305 // If BitWidth > EltBitWidth the value is anyext:ed. So we do not know
4306 // anything about the extended bits.
4307 if (BitWidth > EltBitWidth)
4308 Known = Known.trunc(BitWidth: EltBitWidth);
4309
4310 // If we know the element index, just demand that vector element, else for
4311 // an unknown element index, ignore DemandedElts and demand them all.
4312 APInt DemandedSrcElts = APInt::getAllOnes(numBits: NumSrcElts);
4313 auto *ConstEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
4314 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(RHS: NumSrcElts))
4315 DemandedSrcElts =
4316 APInt::getOneBitSet(numBits: NumSrcElts, BitNo: ConstEltNo->getZExtValue());
4317
4318 Known = computeKnownBits(Op: InVec, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
4319 if (BitWidth > EltBitWidth)
4320 Known = Known.anyext(BitWidth);
4321 break;
4322 }
4323 case ISD::INSERT_VECTOR_ELT: {
4324 if (Op.getValueType().isScalableVector())
4325 break;
4326
4327 // If we know the element index, split the demand between the
4328 // source vector and the inserted element, otherwise assume we need
4329 // the original demanded vector elements and the value.
4330 SDValue InVec = Op.getOperand(i: 0);
4331 SDValue InVal = Op.getOperand(i: 1);
4332 SDValue EltNo = Op.getOperand(i: 2);
4333 bool DemandedVal = true;
4334 APInt DemandedVecElts = DemandedElts;
4335 auto *CEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
4336 if (CEltNo && CEltNo->getAPIntValue().ult(RHS: NumElts)) {
4337 unsigned EltIdx = CEltNo->getZExtValue();
4338 DemandedVal = !!DemandedElts[EltIdx];
4339 DemandedVecElts.clearBit(BitPosition: EltIdx);
4340 }
4341 Known.setAllConflict();
4342 if (DemandedVal) {
4343 Known2 = computeKnownBits(Op: InVal, Depth: Depth + 1);
4344 Known = Known.intersectWith(RHS: Known2.zextOrTrunc(BitWidth));
4345 }
4346 if (!!DemandedVecElts) {
4347 Known2 = computeKnownBits(Op: InVec, DemandedElts: DemandedVecElts, Depth: Depth + 1);
4348 Known = Known.intersectWith(RHS: Known2);
4349 }
4350 break;
4351 }
4352 case ISD::BITREVERSE: {
4353 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4354 Known = Known2.reverseBits();
4355 break;
4356 }
4357 case ISD::BSWAP: {
4358 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4359 Known = Known2.byteSwap();
4360 break;
4361 }
4362 case ISD::ABS:
4363 case ISD::ABS_MIN_POISON: {
4364 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4365 Known = Known2.abs();
4366 Known.Zero.setHighBits(
4367 ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1) - 1);
4368 break;
4369 }
4370 case ISD::USUBSAT: {
4371 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4372 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4373 Known = KnownBits::usub_sat(LHS: Known, RHS: Known2);
4374 break;
4375 }
4376 case ISD::UMIN: {
4377 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4378 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4379 Known = KnownBits::umin(LHS: Known, RHS: Known2);
4380 break;
4381 }
4382 case ISD::UMAX: {
4383 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4384 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4385 Known = KnownBits::umax(LHS: Known, RHS: Known2);
4386 break;
4387 }
4388 case ISD::SMIN:
4389 case ISD::SMAX: {
4390 // If we have a clamp pattern, we know that the number of sign bits will be
4391 // the minimum of the clamp min/max range.
4392 bool IsMax = (Opcode == ISD::SMAX);
4393 ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr;
4394 if ((CstLow = isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts)))
4395 if (Op.getOperand(i: 0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX))
4396 CstHigh =
4397 isConstOrConstSplat(N: Op.getOperand(i: 0).getOperand(i: 1), DemandedElts);
4398 if (CstLow && CstHigh) {
4399 if (!IsMax)
4400 std::swap(a&: CstLow, b&: CstHigh);
4401
4402 const APInt &ValueLow = CstLow->getAPIntValue();
4403 const APInt &ValueHigh = CstHigh->getAPIntValue();
4404 if (ValueLow.sle(RHS: ValueHigh)) {
4405 unsigned LowSignBits = ValueLow.getNumSignBits();
4406 unsigned HighSignBits = ValueHigh.getNumSignBits();
4407 unsigned MinSignBits = std::min(a: LowSignBits, b: HighSignBits);
4408 if (ValueLow.isNegative() && ValueHigh.isNegative()) {
4409 Known.One.setHighBits(MinSignBits);
4410 break;
4411 }
4412 if (ValueLow.isNonNegative() && ValueHigh.isNonNegative()) {
4413 Known.Zero.setHighBits(MinSignBits);
4414 break;
4415 }
4416 }
4417 }
4418
4419 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4420 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4421 if (IsMax)
4422 Known = KnownBits::smax(LHS: Known, RHS: Known2);
4423 else
4424 Known = KnownBits::smin(LHS: Known, RHS: Known2);
4425
4426 // For SMAX, if CstLow is non-negative we know the result will be
4427 // non-negative and thus all sign bits are 0.
4428 // TODO: There's an equivalent of this for smin with negative constant for
4429 // known ones.
4430 if (IsMax && CstLow) {
4431 const APInt &ValueLow = CstLow->getAPIntValue();
4432 if (ValueLow.isNonNegative()) {
4433 unsigned SignBits = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
4434 Known.Zero.setHighBits(std::min(a: SignBits, b: ValueLow.getNumSignBits()));
4435 }
4436 }
4437
4438 break;
4439 }
4440 case ISD::UINT_TO_FP: {
4441 Known.makeNonNegative();
4442 break;
4443 }
4444 case ISD::SINT_TO_FP: {
4445 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4446 if (Known2.isNonNegative())
4447 Known.makeNonNegative();
4448 else if (Known2.isNegative())
4449 Known.makeNegative();
4450 break;
4451 }
4452 case ISD::FP_TO_UINT_SAT: {
4453 // FP_TO_UINT_SAT produces an unsigned value that fits in the saturating VT.
4454 EVT VT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
4455 Known.Zero |= APInt::getBitsSetFrom(numBits: BitWidth, loBit: VT.getScalarSizeInBits());
4456 break;
4457 }
4458 case ISD::ATOMIC_LOAD: {
4459 // If we are looking at the loaded value.
4460 if (Op.getResNo() == 0) {
4461 auto *AT = cast<AtomicSDNode>(Val&: Op);
4462 unsigned ScalarMemorySize = AT->getMemoryVT().getScalarSizeInBits();
4463 KnownBits KnownScalarMemory(ScalarMemorySize);
4464 if (const MDNode *MD = AT->getRanges())
4465 computeKnownBitsFromRangeMetadata(Ranges: *MD, Known&: KnownScalarMemory);
4466
4467 switch (AT->getExtensionType()) {
4468 case ISD::ZEXTLOAD:
4469 Known = KnownScalarMemory.zext(BitWidth);
4470 break;
4471 case ISD::SEXTLOAD:
4472 Known = KnownScalarMemory.sext(BitWidth);
4473 break;
4474 case ISD::EXTLOAD:
4475 switch (TLI->getExtendForAtomicOps()) {
4476 case ISD::ZERO_EXTEND:
4477 Known = KnownScalarMemory.zext(BitWidth);
4478 break;
4479 case ISD::SIGN_EXTEND:
4480 Known = KnownScalarMemory.sext(BitWidth);
4481 break;
4482 default:
4483 Known = KnownScalarMemory.anyext(BitWidth);
4484 break;
4485 }
4486 break;
4487 case ISD::NON_EXTLOAD:
4488 Known = KnownScalarMemory;
4489 break;
4490 }
4491 assert(Known.getBitWidth() == BitWidth);
4492 }
4493 break;
4494 }
4495 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
4496 if (Op.getResNo() == 1) {
4497 // The boolean result conforms to getBooleanContents.
4498 // If we know the result of a setcc has the top bits zero, use this info.
4499 // We know that we have an integer-based boolean since these operations
4500 // are only available for integer.
4501 if (TLI->getBooleanContents(isVec: Op.getValueType().isVector(), isFloat: false) ==
4502 TargetLowering::ZeroOrOneBooleanContent &&
4503 BitWidth > 1)
4504 Known.Zero.setBitsFrom(1);
4505 break;
4506 }
4507 [[fallthrough]];
4508 case ISD::ATOMIC_CMP_SWAP:
4509 case ISD::ATOMIC_SWAP:
4510 case ISD::ATOMIC_LOAD_ADD:
4511 case ISD::ATOMIC_LOAD_SUB:
4512 case ISD::ATOMIC_LOAD_AND:
4513 case ISD::ATOMIC_LOAD_CLR:
4514 case ISD::ATOMIC_LOAD_OR:
4515 case ISD::ATOMIC_LOAD_XOR:
4516 case ISD::ATOMIC_LOAD_NAND:
4517 case ISD::ATOMIC_LOAD_MIN:
4518 case ISD::ATOMIC_LOAD_MAX:
4519 case ISD::ATOMIC_LOAD_UMIN:
4520 case ISD::ATOMIC_LOAD_UMAX: {
4521 // If we are looking at the loaded value.
4522 if (Op.getResNo() == 0) {
4523 auto *AT = cast<AtomicSDNode>(Val&: Op);
4524 unsigned MemBits = AT->getMemoryVT().getScalarSizeInBits();
4525
4526 if (TLI->getExtendForAtomicOps() == ISD::ZERO_EXTEND)
4527 Known.Zero.setBitsFrom(MemBits);
4528 }
4529 break;
4530 }
4531 case ISD::FrameIndex:
4532 case ISD::TargetFrameIndex: {
4533 const MachineFunction &MF = getMachineFunction();
4534 int FrameIdx = cast<FrameIndexSDNode>(Val&: Op)->getIndex();
4535 TLI->computeKnownBitsForStackObjectPointer(
4536 Known, MF, Alignment: MF.getFrameInfo().getObjectAlign(ObjectIdx: FrameIdx));
4537 break;
4538 }
4539
4540 default:
4541 if (Opcode < ISD::BUILTIN_OP_END)
4542 break;
4543 [[fallthrough]];
4544 case ISD::INTRINSIC_WO_CHAIN:
4545 case ISD::INTRINSIC_W_CHAIN:
4546 case ISD::INTRINSIC_VOID:
4547 // Allow the target to implement this method for its nodes.
4548 TLI->computeKnownBitsForTargetNode(Op, Known, DemandedElts, DAG: *this, Depth);
4549 break;
4550 }
4551
4552 return Known;
4553}
4554
4555/// Convert ConstantRange OverflowResult into SelectionDAG::OverflowKind.
4556static SelectionDAG::OverflowKind mapOverflowResult(ConstantRange::OverflowResult OR) {
4557 switch (OR) {
4558 case ConstantRange::OverflowResult::MayOverflow:
4559 return SelectionDAG::OFK_Sometime;
4560 case ConstantRange::OverflowResult::AlwaysOverflowsLow:
4561 case ConstantRange::OverflowResult::AlwaysOverflowsHigh:
4562 return SelectionDAG::OFK_Always;
4563 case ConstantRange::OverflowResult::NeverOverflows:
4564 return SelectionDAG::OFK_Never;
4565 }
4566 llvm_unreachable("Unknown OverflowResult");
4567}
4568
4569SelectionDAG::OverflowKind
4570SelectionDAG::computeOverflowForSignedAdd(SDValue N0, SDValue N1) const {
4571 // X + 0 never overflow
4572 if (isNullConstant(V: N1))
4573 return OFK_Never;
4574
4575 // If both operands each have at least two sign bits, the addition
4576 // cannot overflow.
4577 if (ComputeNumSignBits(Op: N0) > 1 && ComputeNumSignBits(Op: N1) > 1)
4578 return OFK_Never;
4579
4580 // TODO: Add ConstantRange::signedAddMayOverflow handling.
4581 return OFK_Sometime;
4582}
4583
4584SelectionDAG::OverflowKind
4585SelectionDAG::computeOverflowForUnsignedAdd(SDValue N0, SDValue N1) const {
4586 // X + 0 never overflow
4587 if (isNullConstant(V: N1))
4588 return OFK_Never;
4589
4590 // mulhi + 1 never overflow
4591 KnownBits N1Known = computeKnownBits(Op: N1);
4592 if (N0.getOpcode() == ISD::UMUL_LOHI && N0.getResNo() == 1 &&
4593 N1Known.getMaxValue().ult(RHS: 2))
4594 return OFK_Never;
4595
4596 KnownBits N0Known = computeKnownBits(Op: N0);
4597 if (N1.getOpcode() == ISD::UMUL_LOHI && N1.getResNo() == 1 &&
4598 N0Known.getMaxValue().ult(RHS: 2))
4599 return OFK_Never;
4600
4601 // Fallback to ConstantRange::unsignedAddMayOverflow handling.
4602 ConstantRange N0Range = ConstantRange::fromKnownBits(Known: N0Known, IsSigned: false);
4603 ConstantRange N1Range = ConstantRange::fromKnownBits(Known: N1Known, IsSigned: false);
4604 return mapOverflowResult(OR: N0Range.unsignedAddMayOverflow(Other: N1Range));
4605}
4606
4607SelectionDAG::OverflowKind
4608SelectionDAG::computeOverflowForSignedSub(SDValue N0, SDValue N1) const {
4609 // X - 0 never overflow
4610 if (isNullConstant(V: N1))
4611 return OFK_Never;
4612
4613 // If both operands each have at least two sign bits, the subtraction
4614 // cannot overflow.
4615 if (ComputeNumSignBits(Op: N0) > 1 && ComputeNumSignBits(Op: N1) > 1)
4616 return OFK_Never;
4617
4618 KnownBits N0Known = computeKnownBits(Op: N0);
4619 KnownBits N1Known = computeKnownBits(Op: N1);
4620 ConstantRange N0Range = ConstantRange::fromKnownBits(Known: N0Known, IsSigned: true);
4621 ConstantRange N1Range = ConstantRange::fromKnownBits(Known: N1Known, IsSigned: true);
4622 return mapOverflowResult(OR: N0Range.signedSubMayOverflow(Other: N1Range));
4623}
4624
4625SelectionDAG::OverflowKind
4626SelectionDAG::computeOverflowForUnsignedSub(SDValue N0, SDValue N1) const {
4627 // X - 0 never overflow
4628 if (isNullConstant(V: N1))
4629 return OFK_Never;
4630
4631 ConstantRange N0Range =
4632 computeConstantRangeIncludingKnownBits(Op: N0, /*ForSigned=*/false);
4633 ConstantRange N1Range =
4634 computeConstantRangeIncludingKnownBits(Op: N1, /*ForSigned=*/false);
4635 return mapOverflowResult(OR: N0Range.unsignedSubMayOverflow(Other: N1Range));
4636}
4637
4638SelectionDAG::OverflowKind
4639SelectionDAG::computeOverflowForUnsignedMul(SDValue N0, SDValue N1) const {
4640 // X * 0 and X * 1 never overflow.
4641 if (isNullConstant(V: N1) || isOneConstant(V: N1))
4642 return OFK_Never;
4643
4644 ConstantRange N0Range = computeConstantRangeIncludingKnownBits(Op: N0, ForSigned: false);
4645 ConstantRange N1Range = computeConstantRangeIncludingKnownBits(Op: N1, ForSigned: false);
4646 return mapOverflowResult(OR: N0Range.unsignedMulMayOverflow(Other: N1Range));
4647}
4648
4649SelectionDAG::OverflowKind
4650SelectionDAG::computeOverflowForSignedMul(SDValue N0, SDValue N1) const {
4651 // X * 0 and X * 1 never overflow.
4652 if (isNullConstant(V: N1) || isOneConstant(V: N1))
4653 return OFK_Never;
4654
4655 // Get the size of the result.
4656 unsigned BitWidth = N0.getScalarValueSizeInBits();
4657
4658 // Sum of the sign bits.
4659 unsigned SignBits = ComputeNumSignBits(Op: N0) + ComputeNumSignBits(Op: N1);
4660
4661 // If we have enough sign bits, then there's no overflow.
4662 if (SignBits > BitWidth + 1)
4663 return OFK_Never;
4664
4665 if (SignBits == BitWidth + 1) {
4666 // The overflow occurs when the true multiplication of the
4667 // the operands is the minimum negative number.
4668 KnownBits N0Known = computeKnownBits(Op: N0);
4669 KnownBits N1Known = computeKnownBits(Op: N1);
4670 // If one of the operands is non-negative, then there's no
4671 // overflow.
4672 if (N0Known.isNonNegative() || N1Known.isNonNegative())
4673 return OFK_Never;
4674 }
4675
4676 return OFK_Sometime;
4677}
4678
4679ConstantRange SelectionDAG::computeConstantRange(SDValue Op, bool ForSigned,
4680 unsigned Depth) const {
4681 APInt DemandedElts = getDemandAllEltsMask(V: Op);
4682 return computeConstantRange(Op, DemandedElts, ForSigned, Depth);
4683}
4684
4685ConstantRange SelectionDAG::computeConstantRange(SDValue Op,
4686 const APInt &DemandedElts,
4687 bool ForSigned,
4688 unsigned Depth) const {
4689 EVT VT = Op.getValueType();
4690 unsigned BitWidth = VT.getScalarSizeInBits();
4691
4692 if (Depth >= MaxRecursionDepth)
4693 return ConstantRange::getFull(BitWidth);
4694
4695 if (ConstantSDNode *C = isConstOrConstSplat(N: Op, DemandedElts))
4696 return ConstantRange(C->getAPIntValue());
4697
4698 unsigned Opcode = Op.getOpcode();
4699 switch (Opcode) {
4700 case ISD::VSCALE: {
4701 const Function &F = getMachineFunction().getFunction();
4702 const APInt &Multiplier = Op.getConstantOperandAPInt(i: 0);
4703 return getVScaleRange(F: &F, BitWidth).multiply(Other: Multiplier);
4704 }
4705 default:
4706 break;
4707 }
4708
4709 return ConstantRange::getFull(BitWidth);
4710}
4711
4712ConstantRange
4713SelectionDAG::computeConstantRangeIncludingKnownBits(SDValue Op, bool ForSigned,
4714 unsigned Depth) const {
4715 APInt DemandedElts = getDemandAllEltsMask(V: Op);
4716 return computeConstantRangeIncludingKnownBits(Op, DemandedElts, ForSigned,
4717 Depth);
4718}
4719
4720ConstantRange SelectionDAG::computeConstantRangeIncludingKnownBits(
4721 SDValue Op, const APInt &DemandedElts, bool ForSigned,
4722 unsigned Depth) const {
4723 KnownBits Known = computeKnownBits(Op, DemandedElts, Depth);
4724 ConstantRange CR1 = ConstantRange::fromKnownBits(Known, IsSigned: ForSigned);
4725 ConstantRange CR2 = computeConstantRange(Op, DemandedElts, ForSigned, Depth);
4726 ConstantRange::PreferredRangeType RangeType =
4727 ForSigned ? ConstantRange::Signed : ConstantRange::Unsigned;
4728 return CR1.intersectWith(CR: CR2, Type: RangeType);
4729}
4730
4731bool SelectionDAG::isKnownToBeAPowerOfTwo(SDValue Val, bool OrZero,
4732 unsigned Depth) const {
4733 APInt DemandedElts = getDemandAllEltsMask(V: Val);
4734 return isKnownToBeAPowerOfTwo(Val, DemandedElts, OrZero, Depth);
4735}
4736
4737bool SelectionDAG::isKnownToBeAPowerOfTwo(SDValue Val,
4738 const APInt &DemandedElts,
4739 bool OrZero, unsigned Depth) const {
4740 if (Depth >= MaxRecursionDepth)
4741 return false; // Limit search depth.
4742
4743 EVT OpVT = Val.getValueType();
4744 unsigned BitWidth = OpVT.getScalarSizeInBits();
4745 [[maybe_unused]] unsigned NumElts = DemandedElts.getBitWidth();
4746 assert((!OpVT.isScalableVector() || NumElts == 1) &&
4747 "DemandedElts for scalable vectors must be 1 to represent all lanes");
4748 assert(
4749 (!OpVT.isFixedLengthVector() || NumElts == OpVT.getVectorNumElements()) &&
4750 "Unexpected vector size");
4751
4752 auto IsPowerOfTwoOrZero = [BitWidth, OrZero](const ConstantSDNode *C) {
4753 APInt V = C->getAPIntValue().zextOrTrunc(width: BitWidth);
4754 return (OrZero && V.isZero()) || V.isPowerOf2();
4755 };
4756
4757 // Is the constant a known power of 2 or zero?
4758 if (ISD::matchUnaryPredicate(Op: Val, DemandedElts, Match: IsPowerOfTwoOrZero,
4759 /*AllowUndefs=*/false, /*AllowTruncation=*/true))
4760 return true;
4761
4762 switch (Val.getOpcode()) {
4763 case ISD::EXTRACT_VECTOR_ELT: {
4764 SDValue InVec = Val.getOperand(i: 0);
4765 SDValue EltNo = Val.getOperand(i: 1);
4766 EVT VecVT = InVec.getValueType();
4767
4768 // Skip scalable vectors or implicit extensions.
4769 if (VecVT.isScalableVector() ||
4770 OpVT.getScalarSizeInBits() != VecVT.getScalarSizeInBits())
4771 break;
4772
4773 // If we know the element index, just demand that vector element, else for
4774 // an unknown element index, ignore DemandedElts and demand them all.
4775 const unsigned NumSrcElts = VecVT.getVectorNumElements();
4776 auto *ConstEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
4777 APInt DemandedSrcElts =
4778 ConstEltNo && ConstEltNo->getAPIntValue().ult(RHS: NumSrcElts)
4779 ? APInt::getOneBitSet(numBits: NumSrcElts, BitNo: ConstEltNo->getZExtValue())
4780 : APInt::getAllOnes(numBits: NumSrcElts);
4781 return isKnownToBeAPowerOfTwo(Val: InVec, DemandedElts: DemandedSrcElts, OrZero, Depth: Depth + 1);
4782 }
4783
4784 case ISD::AND: {
4785 // Looking for `x & -x` pattern:
4786 // If x == 0:
4787 // x & -x -> 0
4788 // If x != 0:
4789 // x & -x -> non-zero pow2
4790 // so if we find the pattern return whether we know `x` is non-zero.
4791 SDValue X, Z;
4792 if (sd_match(N: Val, P: m_And(L: m_Value(N&: X), R: m_Neg(V: m_Deferred(V&: X)))) ||
4793 (sd_match(N: Val, P: m_And(L: m_Value(N&: X), R: m_Sub(L: m_Value(N&: Z), R: m_Deferred(V&: X)))) &&
4794 MaskedVectorIsZero(V: Z, DemandedElts, Depth: Depth + 1)))
4795 return OrZero || isKnownNeverZero(Op: X, DemandedElts, Depth);
4796 break;
4797 }
4798
4799 case ISD::SHL: {
4800 // A left-shift of a constant one will have exactly one bit set because
4801 // shifting the bit off the end is undefined.
4802 auto *C = isConstOrConstSplat(N: Val.getOperand(i: 0), DemandedElts);
4803 if (C && C->getAPIntValue() == 1)
4804 return true;
4805 return (OrZero || isKnownNeverZero(Op: Val, DemandedElts, Depth)) &&
4806 isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), DemandedElts, OrZero,
4807 Depth: Depth + 1);
4808 }
4809
4810 case ISD::SRL: {
4811 // A logical right-shift of a constant sign-bit will have exactly
4812 // one bit set.
4813 auto *C = isConstOrConstSplat(N: Val.getOperand(i: 0), DemandedElts);
4814 if (C && C->getAPIntValue().isSignMask())
4815 return true;
4816 return (OrZero || isKnownNeverZero(Op: Val, DemandedElts, Depth)) &&
4817 isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), DemandedElts, OrZero,
4818 Depth: Depth + 1);
4819 }
4820
4821 case ISD::TRUNCATE:
4822 return (OrZero || isKnownNeverZero(Op: Val, DemandedElts, Depth)) &&
4823 isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), DemandedElts, OrZero,
4824 Depth: Depth + 1);
4825
4826 case ISD::ROTL:
4827 case ISD::ROTR:
4828 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), DemandedElts, OrZero,
4829 Depth: Depth + 1);
4830 case ISD::BSWAP:
4831 case ISD::BITREVERSE:
4832 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), DemandedElts, OrZero,
4833 Depth: Depth + 1);
4834
4835 case ISD::SMIN:
4836 case ISD::SMAX:
4837 case ISD::UMIN:
4838 case ISD::UMAX:
4839 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 1), DemandedElts, OrZero,
4840 Depth: Depth + 1) &&
4841 isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), DemandedElts, OrZero,
4842 Depth: Depth + 1);
4843
4844 case ISD::SELECT:
4845 case ISD::VSELECT:
4846 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 2), DemandedElts, OrZero,
4847 Depth: Depth + 1) &&
4848 isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 1), DemandedElts, OrZero,
4849 Depth: Depth + 1);
4850
4851 case ISD::ZERO_EXTEND:
4852 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), DemandedElts, OrZero,
4853 Depth: Depth + 1);
4854
4855 case ISD::VSCALE:
4856 // vscale(power-of-two) is a power-of-two
4857 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), /*OrZero=*/false,
4858 Depth: Depth + 1);
4859
4860 case ISD::VECTOR_SHUFFLE: {
4861 assert(!Val.getValueType().isScalableVector());
4862 // Demanded elements with undef shuffle mask elements are unknown
4863 // - we cannot guarantee they are a power of two, so return false.
4864 APInt DemandedLHS, DemandedRHS;
4865 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val);
4866 assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
4867 if (!getShuffleDemandedElts(SrcWidth: NumElts, Mask: SVN->getMask(), DemandedElts,
4868 DemandedLHS, DemandedRHS))
4869 return false;
4870
4871 // All demanded elements from LHS must be known power of two.
4872 if (!!DemandedLHS && !isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), DemandedElts: DemandedLHS,
4873 OrZero, Depth: Depth + 1))
4874 return false;
4875
4876 // All demanded elements from RHS must be known power of two.
4877 if (!!DemandedRHS && !isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 1), DemandedElts: DemandedRHS,
4878 OrZero, Depth: Depth + 1))
4879 return false;
4880
4881 return true;
4882 }
4883 }
4884
4885 // More could be done here, though the above checks are enough
4886 // to handle some common cases.
4887 return false;
4888}
4889
4890bool SelectionDAG::isKnownToBeAPowerOfTwoFP(SDValue Val, unsigned Depth) const {
4891 if (ConstantFPSDNode *C1 = isConstOrConstSplatFP(N: Val, AllowUndefs: true))
4892 return C1->getValueAPF().getExactLog2Abs() >= 0;
4893
4894 if (Val.getOpcode() == ISD::UINT_TO_FP || Val.getOpcode() == ISD::SINT_TO_FP)
4895 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), OrZero: Depth + 1);
4896
4897 return false;
4898}
4899
4900unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const {
4901 APInt DemandedElts = getDemandAllEltsMask(V: Op);
4902 return ComputeNumSignBits(Op, DemandedElts, Depth);
4903}
4904
4905unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, const APInt &DemandedElts,
4906 unsigned Depth) const {
4907 EVT VT = Op.getValueType();
4908 assert((VT.isInteger() || VT.isFloatingPoint()) && "Invalid VT!");
4909 unsigned VTBits = VT.getScalarSizeInBits();
4910 unsigned NumElts = DemandedElts.getBitWidth();
4911 unsigned Tmp, Tmp2;
4912 unsigned FirstAnswer = 1;
4913
4914 assert((!VT.isScalableVector() || NumElts == 1) &&
4915 "DemandedElts for scalable vectors must be 1 to represent all lanes");
4916
4917 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op)) {
4918 const APInt &Val = C->getAPIntValue();
4919 return Val.getNumSignBits();
4920 }
4921
4922 if (Depth >= MaxRecursionDepth)
4923 return 1; // Limit search depth.
4924
4925 if (!DemandedElts)
4926 return 1; // No demanded elts, better to assume we don't know anything.
4927
4928 unsigned Opcode = Op.getOpcode();
4929 switch (Opcode) {
4930 default: break;
4931 case ISD::AssertSext:
4932 Tmp = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT().getSizeInBits();
4933 return VTBits-Tmp+1;
4934 case ISD::AssertZext:
4935 Tmp = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT().getSizeInBits();
4936 return VTBits-Tmp;
4937 case ISD::FREEZE:
4938 if (isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 0), DemandedElts,
4939 Kind: UndefPoisonKind::UndefOrPoison))
4940 return ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4941 break;
4942 case ISD::MERGE_VALUES:
4943 return ComputeNumSignBits(Op: Op.getOperand(i: Op.getResNo()), DemandedElts,
4944 Depth: Depth + 1);
4945 case ISD::SPLAT_VECTOR: {
4946 // Check if the sign bits of source go down as far as the truncated value.
4947 unsigned NumSrcBits = Op.getOperand(i: 0).getValueSizeInBits();
4948 unsigned NumSrcSignBits = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
4949 if (NumSrcSignBits > (NumSrcBits - VTBits))
4950 return NumSrcSignBits - (NumSrcBits - VTBits);
4951 break;
4952 }
4953 case ISD::BUILD_VECTOR:
4954 assert(!VT.isScalableVector());
4955 Tmp = VTBits;
4956 for (unsigned i = 0, e = Op.getNumOperands(); (i < e) && (Tmp > 1); ++i) {
4957 if (!DemandedElts[i])
4958 continue;
4959
4960 SDValue SrcOp = Op.getOperand(i);
4961 // BUILD_VECTOR can implicitly truncate sources, we handle this specially
4962 // for constant nodes to ensure we only look at the sign bits.
4963 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: SrcOp)) {
4964 APInt T = C->getAPIntValue().trunc(width: VTBits);
4965 Tmp2 = T.getNumSignBits();
4966 } else {
4967 Tmp2 = ComputeNumSignBits(Op: SrcOp, Depth: Depth + 1);
4968
4969 if (SrcOp.getValueSizeInBits() != VTBits) {
4970 assert(SrcOp.getValueSizeInBits() > VTBits &&
4971 "Expected BUILD_VECTOR implicit truncation");
4972 unsigned ExtraBits = SrcOp.getValueSizeInBits() - VTBits;
4973 Tmp2 = (Tmp2 > ExtraBits ? Tmp2 - ExtraBits : 1);
4974 }
4975 }
4976 Tmp = std::min(a: Tmp, b: Tmp2);
4977 }
4978 return Tmp;
4979
4980 case ISD::VECTOR_COMPRESS: {
4981 SDValue Vec = Op.getOperand(i: 0);
4982 SDValue PassThru = Op.getOperand(i: 2);
4983 Tmp = ComputeNumSignBits(Op: PassThru, DemandedElts, Depth: Depth + 1);
4984 if (Tmp == 1)
4985 return 1;
4986 Tmp2 = ComputeNumSignBits(Op: Vec, Depth: Depth + 1);
4987 Tmp = std::min(a: Tmp, b: Tmp2);
4988 return Tmp;
4989 }
4990
4991 case ISD::VECTOR_SHUFFLE: {
4992 // Collect the minimum number of sign bits that are shared by every vector
4993 // element referenced by the shuffle.
4994 APInt DemandedLHS, DemandedRHS;
4995 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
4996 assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
4997 if (!getShuffleDemandedElts(SrcWidth: NumElts, Mask: SVN->getMask(), DemandedElts,
4998 DemandedLHS, DemandedRHS))
4999 return 1;
5000
5001 Tmp = std::numeric_limits<unsigned>::max();
5002 if (!!DemandedLHS)
5003 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts: DemandedLHS, Depth: Depth + 1);
5004 if (!!DemandedRHS) {
5005 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts: DemandedRHS, Depth: Depth + 1);
5006 Tmp = std::min(a: Tmp, b: Tmp2);
5007 }
5008 // If we don't know anything, early out and try computeKnownBits fall-back.
5009 if (Tmp == 1)
5010 break;
5011 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
5012 return Tmp;
5013 }
5014
5015 case ISD::BITCAST: {
5016 if (VT.isScalableVector())
5017 break;
5018 SDValue N0 = Op.getOperand(i: 0);
5019 EVT SrcVT = N0.getValueType();
5020 unsigned SrcBits = SrcVT.getScalarSizeInBits();
5021
5022 // Ignore bitcasts from unsupported types..
5023 if (!(SrcVT.isInteger() || SrcVT.isFloatingPoint()))
5024 break;
5025
5026 // Fast handling of 'identity' bitcasts.
5027 if (VTBits == SrcBits)
5028 return ComputeNumSignBits(Op: N0, DemandedElts, Depth: Depth + 1);
5029
5030 bool IsLE = getDataLayout().isLittleEndian();
5031
5032 // Bitcast 'large element' scalar/vector to 'small element' vector.
5033 if ((SrcBits % VTBits) == 0) {
5034 assert(VT.isVector() && "Expected bitcast to vector");
5035
5036 unsigned Scale = SrcBits / VTBits;
5037 APInt SrcDemandedElts =
5038 APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumElts / Scale);
5039
5040 // Fast case - sign splat can be simply split across the small elements.
5041 Tmp = ComputeNumSignBits(Op: N0, DemandedElts: SrcDemandedElts, Depth: Depth + 1);
5042 if (Tmp == SrcBits)
5043 return VTBits;
5044
5045 // Slow case - determine how far the sign extends into each sub-element.
5046 Tmp2 = VTBits;
5047 for (unsigned i = 0; i != NumElts; ++i)
5048 if (DemandedElts[i]) {
5049 unsigned SubOffset = i % Scale;
5050 SubOffset = (IsLE ? ((Scale - 1) - SubOffset) : SubOffset);
5051 SubOffset = SubOffset * VTBits;
5052 if (Tmp <= SubOffset)
5053 return 1;
5054 Tmp2 = std::min(a: Tmp2, b: Tmp - SubOffset);
5055 }
5056 return Tmp2;
5057 }
5058 break;
5059 }
5060
5061 case ISD::FP_TO_SINT_SAT:
5062 // FP_TO_SINT_SAT produces a signed value that fits in the saturating VT.
5063 Tmp = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT().getScalarSizeInBits();
5064 return VTBits - Tmp + 1;
5065 case ISD::SIGN_EXTEND:
5066 Tmp = VTBits - Op.getOperand(i: 0).getScalarValueSizeInBits();
5067 return ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth+1) + Tmp;
5068 case ISD::SIGN_EXTEND_INREG:
5069 // Max of the input and what this extends.
5070 Tmp = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT().getScalarSizeInBits();
5071 Tmp = VTBits-Tmp+1;
5072 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth+1);
5073 return std::max(a: Tmp, b: Tmp2);
5074 case ISD::SIGN_EXTEND_VECTOR_INREG: {
5075 if (VT.isScalableVector())
5076 break;
5077 SDValue Src = Op.getOperand(i: 0);
5078 EVT SrcVT = Src.getValueType();
5079 APInt DemandedSrcElts = DemandedElts.zext(width: SrcVT.getVectorNumElements());
5080 Tmp = VTBits - SrcVT.getScalarSizeInBits();
5081 return ComputeNumSignBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth+1) + Tmp;
5082 }
5083 case ISD::SRA:
5084 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5085 // SRA X, C -> adds C sign bits.
5086 if (std::optional<unsigned> ShAmt =
5087 getValidMinimumShiftAmount(V: Op, DemandedElts, Depth: Depth + 1))
5088 Tmp = std::min(a: Tmp + *ShAmt, b: VTBits);
5089 return Tmp;
5090 case ISD::SHL:
5091 if (std::optional<ConstantRange> ShAmtRange =
5092 getValidShiftAmountRange(V: Op, DemandedElts, Depth: Depth + 1)) {
5093 unsigned MaxShAmt = ShAmtRange->getUnsignedMax().getZExtValue();
5094 unsigned MinShAmt = ShAmtRange->getUnsignedMin().getZExtValue();
5095 // Try to look through ZERO/SIGN/ANY_EXTEND. If all extended bits are
5096 // shifted out, then we can compute the number of sign bits for the
5097 // operand being extended. A future improvement could be to pass along the
5098 // "shifted left by" information in the recursive calls to
5099 // ComputeKnownSignBits. Allowing us to handle this more generically.
5100 if (ISD::isExtOpcode(Opcode: Op.getOperand(i: 0).getOpcode())) {
5101 SDValue Ext = Op.getOperand(i: 0);
5102 EVT ExtVT = Ext.getValueType();
5103 SDValue Extendee = Ext.getOperand(i: 0);
5104 EVT ExtendeeVT = Extendee.getValueType();
5105 unsigned SizeDifference =
5106 ExtVT.getScalarSizeInBits() - ExtendeeVT.getScalarSizeInBits();
5107 if (SizeDifference <= MinShAmt) {
5108 Tmp = SizeDifference +
5109 ComputeNumSignBits(Op: Extendee, DemandedElts, Depth: Depth + 1);
5110 if (MaxShAmt < Tmp)
5111 return Tmp - MaxShAmt;
5112 }
5113 }
5114 // shl destroys sign bits, ensure it doesn't shift out all sign bits.
5115 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5116 if (MaxShAmt < Tmp)
5117 return Tmp - MaxShAmt;
5118 }
5119 break;
5120 case ISD::AND:
5121 case ISD::OR:
5122 case ISD::XOR: // NOT is handled here.
5123 // Logical binary ops preserve the number of sign bits at the worst.
5124 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth+1);
5125 if (Tmp != 1) {
5126 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth+1);
5127 FirstAnswer = std::min(a: Tmp, b: Tmp2);
5128 // We computed what we know about the sign bits as our first
5129 // answer. Now proceed to the generic code that uses
5130 // computeKnownBits, and pick whichever answer is better.
5131 }
5132 break;
5133
5134 case ISD::SELECT:
5135 case ISD::VSELECT:
5136 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth+1);
5137 if (Tmp == 1) return 1; // Early out.
5138 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth+1);
5139 return std::min(a: Tmp, b: Tmp2);
5140 case ISD::SELECT_CC:
5141 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth+1);
5142 if (Tmp == 1) return 1; // Early out.
5143 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 3), DemandedElts, Depth: Depth+1);
5144 return std::min(a: Tmp, b: Tmp2);
5145
5146 case ISD::SMIN:
5147 case ISD::SMAX: {
5148 // If we have a clamp pattern, we know that the number of sign bits will be
5149 // the minimum of the clamp min/max range.
5150 bool IsMax = (Opcode == ISD::SMAX);
5151 ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr;
5152 if ((CstLow = isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts)))
5153 if (Op.getOperand(i: 0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX))
5154 CstHigh =
5155 isConstOrConstSplat(N: Op.getOperand(i: 0).getOperand(i: 1), DemandedElts);
5156 if (CstLow && CstHigh) {
5157 if (!IsMax)
5158 std::swap(a&: CstLow, b&: CstHigh);
5159 if (CstLow->getAPIntValue().sle(RHS: CstHigh->getAPIntValue())) {
5160 Tmp = CstLow->getAPIntValue().getNumSignBits();
5161 Tmp2 = CstHigh->getAPIntValue().getNumSignBits();
5162 return std::min(a: Tmp, b: Tmp2);
5163 }
5164 }
5165
5166 // Fallback - just get the minimum number of sign bits of the operands.
5167 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5168 if (Tmp == 1)
5169 return 1; // Early out.
5170 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5171 return std::min(a: Tmp, b: Tmp2);
5172 }
5173 case ISD::UMIN:
5174 case ISD::UMAX:
5175 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5176 if (Tmp == 1)
5177 return 1; // Early out.
5178 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5179 return std::min(a: Tmp, b: Tmp2);
5180 case ISD::SSUBO_CARRY:
5181 case ISD::USUBO_CARRY:
5182 // sub_carry(x,x,c) -> 0/-1 (sext carry)
5183 if (Op.getResNo() == 0 && Op.getOperand(i: 0) == Op.getOperand(i: 1))
5184 return VTBits;
5185 [[fallthrough]];
5186 case ISD::SADDO:
5187 case ISD::UADDO:
5188 case ISD::SADDO_CARRY:
5189 case ISD::UADDO_CARRY:
5190 case ISD::SSUBO:
5191 case ISD::USUBO:
5192 case ISD::SMULO:
5193 case ISD::UMULO:
5194 if (Op.getResNo() != 1)
5195 break;
5196 // The boolean result conforms to getBooleanContents. Fall through.
5197 // If setcc returns 0/-1, all bits are sign bits.
5198 // We know that we have an integer-based boolean since these operations
5199 // are only available for integer.
5200 if (TLI->getBooleanContents(isVec: VT.isVector(), isFloat: false) ==
5201 TargetLowering::ZeroOrNegativeOneBooleanContent)
5202 return VTBits;
5203 break;
5204 case ISD::SETCC:
5205 case ISD::SETCCCARRY:
5206 case ISD::STRICT_FSETCC:
5207 case ISD::STRICT_FSETCCS: {
5208 unsigned OpNo = Op->isStrictFPOpcode() ? 1 : 0;
5209 // If setcc returns 0/-1, all bits are sign bits.
5210 if (TLI->getBooleanContents(Type: Op.getOperand(i: OpNo).getValueType()) ==
5211 TargetLowering::ZeroOrNegativeOneBooleanContent)
5212 return VTBits;
5213 break;
5214 }
5215 case ISD::GET_ACTIVE_LANE_MASK:
5216 // Semantically similar to icmp ult.
5217 if (TLI->getBooleanContents(isVec: VT.isVector(), /*isFloat=*/false) ==
5218 TargetLowering::ZeroOrNegativeOneBooleanContent)
5219 return VTBits;
5220 break;
5221 case ISD::ROTL:
5222 case ISD::ROTR: {
5223 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5224 ConstantSDNode *C = isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts);
5225 FirstAnswer = SignBitsOps::rot(
5226 SrcSignBits: Tmp, BitWidth: VTBits, RotAmt: C ? std::optional(C->getAPIntValue()) : std::nullopt,
5227 IsRotateRight: Opcode == ISD::ROTR);
5228 break;
5229 }
5230 case ISD::ADD:
5231 case ISD::ADDC:
5232 // TODO: Move Operand 1 check before Operand 0 check
5233 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5234 if (Tmp == 1) return 1; // Early out.
5235
5236 // Special case decrementing a value (ADD X, -1):
5237 if (ConstantSDNode *CRHS =
5238 isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts))
5239 if (CRHS->isAllOnes()) {
5240 KnownBits Known =
5241 computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5242
5243 // If the input is known to be 0 or 1, the output is 0/-1, which is all
5244 // sign bits set.
5245 if ((Known.Zero | 1).isAllOnes())
5246 return VTBits;
5247
5248 // If we are subtracting one from a positive number, there is no carry
5249 // out of the result.
5250 if (Known.isNonNegative())
5251 return Tmp;
5252 }
5253
5254 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5255 if (Tmp2 == 1) return 1; // Early out.
5256
5257 // Add can have at most one carry bit. Thus we know that the output
5258 // is, at worst, one more bit than the inputs.
5259 return std::min(a: Tmp, b: Tmp2) - 1;
5260 case ISD::SUB:
5261 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5262 if (Tmp2 == 1) return 1; // Early out.
5263
5264 // Handle NEG.
5265 if (ConstantSDNode *CLHS =
5266 isConstOrConstSplat(N: Op.getOperand(i: 0), DemandedElts))
5267 if (CLHS->isZero()) {
5268 KnownBits Known =
5269 computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5270 // If the input is known to be 0 or 1, the output is 0/-1, which is all
5271 // sign bits set.
5272 if ((Known.Zero | 1).isAllOnes())
5273 return VTBits;
5274
5275 // If the input is known to be positive (the sign bit is known clear),
5276 // the output of the NEG has the same number of sign bits as the input.
5277 if (Known.isNonNegative())
5278 return Tmp2;
5279
5280 // Otherwise, we treat this like a SUB.
5281 }
5282
5283 // Sub can have at most one carry bit. Thus we know that the output
5284 // is, at worst, one more bit than the inputs.
5285 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5286 if (Tmp == 1) return 1; // Early out.
5287 return std::min(a: Tmp, b: Tmp2) - 1;
5288 case ISD::MUL: {
5289 // The output of the Mul can be at most twice the valid bits in the inputs.
5290 unsigned SignBitsOp0 = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
5291 if (SignBitsOp0 == 1)
5292 break;
5293 unsigned SignBitsOp1 = ComputeNumSignBits(Op: Op.getOperand(i: 1), Depth: Depth + 1);
5294 if (SignBitsOp1 == 1)
5295 break;
5296 unsigned OutValidBits =
5297 (VTBits - SignBitsOp0 + 1) + (VTBits - SignBitsOp1 + 1);
5298 return OutValidBits > VTBits ? 1 : VTBits - OutValidBits + 1;
5299 }
5300 case ISD::AVGCEILS:
5301 case ISD::AVGFLOORS:
5302 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5303 if (Tmp == 1)
5304 return 1; // Early out.
5305 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5306 return std::min(a: Tmp, b: Tmp2);
5307 case ISD::SREM:
5308 // The sign bit is the LHS's sign bit, except when the result of the
5309 // remainder is zero. The magnitude of the result should be less than or
5310 // equal to the magnitude of the LHS. Therefore, the result should have
5311 // at least as many sign bits as the left hand side.
5312 return ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5313 case ISD::TRUNCATE: {
5314 // Check if the sign bits of source go down as far as the truncated value.
5315 unsigned NumSrcBits = Op.getOperand(i: 0).getScalarValueSizeInBits();
5316 unsigned NumSrcSignBits = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
5317 if (NumSrcSignBits > (NumSrcBits - VTBits))
5318 return NumSrcSignBits - (NumSrcBits - VTBits);
5319 break;
5320 }
5321 case ISD::EXTRACT_ELEMENT: {
5322 if (VT.isScalableVector())
5323 break;
5324 const int KnownSign = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth+1);
5325 const int BitWidth = Op.getValueSizeInBits();
5326 const int Items = Op.getOperand(i: 0).getValueSizeInBits() / BitWidth;
5327
5328 // Get reverse index (starting from 1), Op1 value indexes elements from
5329 // little end. Sign starts at big end.
5330 const int rIndex = Items - 1 - Op.getConstantOperandVal(i: 1);
5331
5332 // If the sign portion ends in our element the subtraction gives correct
5333 // result. Otherwise it gives either negative or > bitwidth result
5334 return std::clamp(val: KnownSign - rIndex * BitWidth, lo: 1, hi: BitWidth);
5335 }
5336 case ISD::INSERT_VECTOR_ELT: {
5337 if (VT.isScalableVector())
5338 break;
5339 // If we know the element index, split the demand between the
5340 // source vector and the inserted element, otherwise assume we need
5341 // the original demanded vector elements and the value.
5342 SDValue InVec = Op.getOperand(i: 0);
5343 SDValue InVal = Op.getOperand(i: 1);
5344 SDValue EltNo = Op.getOperand(i: 2);
5345 bool DemandedVal = true;
5346 APInt DemandedVecElts = DemandedElts;
5347 auto *CEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
5348 if (CEltNo && CEltNo->getAPIntValue().ult(RHS: NumElts)) {
5349 unsigned EltIdx = CEltNo->getZExtValue();
5350 DemandedVal = !!DemandedElts[EltIdx];
5351 DemandedVecElts.clearBit(BitPosition: EltIdx);
5352 }
5353 Tmp = std::numeric_limits<unsigned>::max();
5354 if (DemandedVal) {
5355 // TODO - handle implicit truncation of inserted elements.
5356 if (InVal.getScalarValueSizeInBits() != VTBits)
5357 break;
5358 Tmp2 = ComputeNumSignBits(Op: InVal, Depth: Depth + 1);
5359 Tmp = std::min(a: Tmp, b: Tmp2);
5360 }
5361 if (!!DemandedVecElts) {
5362 Tmp2 = ComputeNumSignBits(Op: InVec, DemandedElts: DemandedVecElts, Depth: Depth + 1);
5363 Tmp = std::min(a: Tmp, b: Tmp2);
5364 }
5365 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
5366 return Tmp;
5367 }
5368 case ISD::EXTRACT_VECTOR_ELT: {
5369 SDValue InVec = Op.getOperand(i: 0);
5370 SDValue EltNo = Op.getOperand(i: 1);
5371 EVT VecVT = InVec.getValueType();
5372 // ComputeNumSignBits not yet implemented for scalable vectors.
5373 if (VecVT.isScalableVector())
5374 break;
5375 const unsigned BitWidth = Op.getValueSizeInBits();
5376 const unsigned EltBitWidth = Op.getOperand(i: 0).getScalarValueSizeInBits();
5377 const unsigned NumSrcElts = VecVT.getVectorNumElements();
5378
5379 // If BitWidth > EltBitWidth the value is anyext:ed, and we do not know
5380 // anything about sign bits. But if the sizes match we can derive knowledge
5381 // about sign bits from the vector operand.
5382 if (BitWidth != EltBitWidth)
5383 break;
5384
5385 // If we know the element index, just demand that vector element, else for
5386 // an unknown element index, ignore DemandedElts and demand them all.
5387 APInt DemandedSrcElts = APInt::getAllOnes(numBits: NumSrcElts);
5388 auto *ConstEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
5389 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(RHS: NumSrcElts))
5390 DemandedSrcElts =
5391 APInt::getOneBitSet(numBits: NumSrcElts, BitNo: ConstEltNo->getZExtValue());
5392
5393 return ComputeNumSignBits(Op: InVec, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
5394 }
5395 case ISD::EXTRACT_SUBVECTOR: {
5396 // Offset the demanded elts by the subvector index.
5397 SDValue Src = Op.getOperand(i: 0);
5398
5399 APInt DemandedSrcElts;
5400 if (Src.getValueType().isScalableVector())
5401 DemandedSrcElts = APInt(1, 1);
5402 else {
5403 uint64_t Idx = Op.getConstantOperandVal(i: 1);
5404 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
5405 DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
5406 }
5407 return ComputeNumSignBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
5408 }
5409 case ISD::CONCAT_VECTORS: {
5410 if (VT.isScalableVector())
5411 break;
5412 // Determine the minimum number of sign bits across all demanded
5413 // elts of the input vectors. Early out if the result is already 1.
5414 Tmp = std::numeric_limits<unsigned>::max();
5415 EVT SubVectorVT = Op.getOperand(i: 0).getValueType();
5416 unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements();
5417 unsigned NumSubVectors = Op.getNumOperands();
5418 for (unsigned i = 0; (i < NumSubVectors) && (Tmp > 1); ++i) {
5419 APInt DemandedSub =
5420 DemandedElts.extractBits(numBits: NumSubVectorElts, bitPosition: i * NumSubVectorElts);
5421 if (!DemandedSub)
5422 continue;
5423 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i), DemandedElts: DemandedSub, Depth: Depth + 1);
5424 Tmp = std::min(a: Tmp, b: Tmp2);
5425 }
5426 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
5427 return Tmp;
5428 }
5429 case ISD::INSERT_SUBVECTOR: {
5430 if (VT.isScalableVector())
5431 break;
5432 // Demand any elements from the subvector and the remainder from the src its
5433 // inserted into.
5434 SDValue Src = Op.getOperand(i: 0);
5435 SDValue Sub = Op.getOperand(i: 1);
5436 uint64_t Idx = Op.getConstantOperandVal(i: 2);
5437 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
5438 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
5439 APInt DemandedSrcElts = DemandedElts;
5440 DemandedSrcElts.clearBits(LoBit: Idx, HiBit: Idx + NumSubElts);
5441
5442 Tmp = std::numeric_limits<unsigned>::max();
5443 if (!!DemandedSubElts) {
5444 Tmp = ComputeNumSignBits(Op: Sub, DemandedElts: DemandedSubElts, Depth: Depth + 1);
5445 if (Tmp == 1)
5446 return 1; // early-out
5447 }
5448 if (!!DemandedSrcElts) {
5449 Tmp2 = ComputeNumSignBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
5450 Tmp = std::min(a: Tmp, b: Tmp2);
5451 }
5452 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
5453 return Tmp;
5454 }
5455 case ISD::LOAD: {
5456 // If we are looking at the loaded value of the SDNode.
5457 if (Op.getResNo() != 0)
5458 break;
5459
5460 LoadSDNode *LD = cast<LoadSDNode>(Val&: Op);
5461 if (const MDNode *Ranges = LD->getRanges()) {
5462 if (DemandedElts != 1)
5463 break;
5464
5465 ConstantRange CR = getConstantRangeFromMetadata(RangeMD: *Ranges);
5466 if (VTBits > CR.getBitWidth()) {
5467 switch (LD->getExtensionType()) {
5468 case ISD::SEXTLOAD:
5469 CR = CR.signExtend(BitWidth: VTBits);
5470 break;
5471 case ISD::ZEXTLOAD:
5472 CR = CR.zeroExtend(BitWidth: VTBits);
5473 break;
5474 default:
5475 break;
5476 }
5477 }
5478
5479 if (VTBits != CR.getBitWidth())
5480 break;
5481 return std::min(a: CR.getSignedMin().getNumSignBits(),
5482 b: CR.getSignedMax().getNumSignBits());
5483 }
5484
5485 unsigned ExtType = LD->getExtensionType();
5486 switch (ExtType) {
5487 default:
5488 break;
5489 case ISD::SEXTLOAD: // e.g. i16->i32 = '17' bits known.
5490 Tmp = LD->getMemoryVT().getScalarSizeInBits();
5491 return VTBits - Tmp + 1;
5492 case ISD::ZEXTLOAD: // e.g. i16->i32 = '16' bits known.
5493 Tmp = LD->getMemoryVT().getScalarSizeInBits();
5494 return VTBits - Tmp;
5495 case ISD::NON_EXTLOAD:
5496 if (const Constant *Cst = TLI->getTargetConstantFromLoad(LD)) {
5497 // We only need to handle vectors - computeKnownBits should handle
5498 // scalar cases.
5499 Type *CstTy = Cst->getType();
5500 if (CstTy->isVectorTy() && !VT.isScalableVector() &&
5501 (NumElts * VTBits) == CstTy->getPrimitiveSizeInBits() &&
5502 VTBits == CstTy->getScalarSizeInBits()) {
5503 Tmp = VTBits;
5504 for (unsigned i = 0; i != NumElts; ++i) {
5505 if (!DemandedElts[i])
5506 continue;
5507 if (Constant *Elt = Cst->getAggregateElement(Elt: i)) {
5508 if (auto *CInt = dyn_cast<ConstantInt>(Val: Elt)) {
5509 const APInt &Value = CInt->getValue();
5510 Tmp = std::min(a: Tmp, b: Value.getNumSignBits());
5511 continue;
5512 }
5513 if (auto *CFP = dyn_cast<ConstantFP>(Val: Elt)) {
5514 APInt Value = CFP->getValueAPF().bitcastToAPInt();
5515 Tmp = std::min(a: Tmp, b: Value.getNumSignBits());
5516 continue;
5517 }
5518 }
5519 // Unknown type. Conservatively assume no bits match sign bit.
5520 return 1;
5521 }
5522 return Tmp;
5523 }
5524 }
5525 break;
5526 }
5527
5528 break;
5529 }
5530 case ISD::ATOMIC_CMP_SWAP:
5531 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
5532 case ISD::ATOMIC_SWAP:
5533 case ISD::ATOMIC_LOAD_ADD:
5534 case ISD::ATOMIC_LOAD_SUB:
5535 case ISD::ATOMIC_LOAD_AND:
5536 case ISD::ATOMIC_LOAD_CLR:
5537 case ISD::ATOMIC_LOAD_OR:
5538 case ISD::ATOMIC_LOAD_XOR:
5539 case ISD::ATOMIC_LOAD_NAND:
5540 case ISD::ATOMIC_LOAD_MIN:
5541 case ISD::ATOMIC_LOAD_MAX:
5542 case ISD::ATOMIC_LOAD_UMIN:
5543 case ISD::ATOMIC_LOAD_UMAX:
5544 case ISD::ATOMIC_LOAD: {
5545 auto *AT = cast<AtomicSDNode>(Val&: Op);
5546 // If we are looking at the loaded value.
5547 if (Op.getResNo() == 0) {
5548 Tmp = AT->getMemoryVT().getScalarSizeInBits();
5549 if (Tmp == VTBits)
5550 return 1; // early-out
5551
5552 // For atomic_load, prefer to use the extension type.
5553 if (Op->getOpcode() == ISD::ATOMIC_LOAD) {
5554 switch (AT->getExtensionType()) {
5555 default:
5556 break;
5557 case ISD::SEXTLOAD:
5558 return VTBits - Tmp + 1;
5559 case ISD::ZEXTLOAD:
5560 return VTBits - Tmp;
5561 }
5562 }
5563
5564 if (TLI->getExtendForAtomicOps() == ISD::SIGN_EXTEND)
5565 return VTBits - Tmp + 1;
5566 if (TLI->getExtendForAtomicOps() == ISD::ZERO_EXTEND)
5567 return VTBits - Tmp;
5568 }
5569 break;
5570 }
5571 }
5572
5573 // Allow the target to implement this method for its nodes.
5574 if (Opcode >= ISD::BUILTIN_OP_END ||
5575 Opcode == ISD::INTRINSIC_WO_CHAIN ||
5576 Opcode == ISD::INTRINSIC_W_CHAIN ||
5577 Opcode == ISD::INTRINSIC_VOID) {
5578 // TODO: This can probably be removed once target code is audited. This
5579 // is here purely to reduce patch size and review complexity.
5580 if (!VT.isScalableVector()) {
5581 unsigned NumBits =
5582 TLI->ComputeNumSignBitsForTargetNode(Op, DemandedElts, DAG: *this, Depth);
5583 if (NumBits > 1)
5584 FirstAnswer = std::max(a: FirstAnswer, b: NumBits);
5585 }
5586 }
5587
5588 // Finally, if we can prove that the top bits of the result are 0's or 1's,
5589 // use this information.
5590 KnownBits Known = computeKnownBits(Op, DemandedElts, Depth);
5591 return std::max(a: FirstAnswer, b: Known.countMinSignBits());
5592}
5593
5594unsigned SelectionDAG::ComputeMaxSignificantBits(SDValue Op,
5595 unsigned Depth) const {
5596 unsigned SignBits = ComputeNumSignBits(Op, Depth);
5597 return Op.getScalarValueSizeInBits() - SignBits + 1;
5598}
5599
5600unsigned SelectionDAG::ComputeMaxSignificantBits(SDValue Op,
5601 const APInt &DemandedElts,
5602 unsigned Depth) const {
5603 unsigned SignBits = ComputeNumSignBits(Op, DemandedElts, Depth);
5604 return Op.getScalarValueSizeInBits() - SignBits + 1;
5605}
5606
5607bool SelectionDAG::isGuaranteedNotToBeUndefOrPoison(SDValue Op,
5608 UndefPoisonKind Kind,
5609 unsigned Depth) const {
5610 // Early out for FREEZE.
5611 if (Op.getOpcode() == ISD::FREEZE)
5612 return true;
5613
5614 APInt DemandedElts = getDemandAllEltsMask(V: Op);
5615 return isGuaranteedNotToBeUndefOrPoison(Op, DemandedElts, Kind, Depth);
5616}
5617
5618bool SelectionDAG::isGuaranteedNotToBeUndefOrPoison(SDValue Op,
5619 const APInt &DemandedElts,
5620 UndefPoisonKind Kind,
5621 unsigned Depth) const {
5622 unsigned Opcode = Op.getOpcode();
5623
5624 // Early out for FREEZE.
5625 if (Opcode == ISD::FREEZE)
5626 return true;
5627
5628 if (Depth >= MaxRecursionDepth)
5629 return false; // Limit search depth.
5630
5631 if (isIntOrFPConstant(V: Op))
5632 return true;
5633
5634 switch (Opcode) {
5635 case ISD::CONDCODE:
5636 case ISD::VALUETYPE:
5637 case ISD::FrameIndex:
5638 case ISD::TargetFrameIndex:
5639 case ISD::CopyFromReg:
5640 return true;
5641
5642 case ISD::POISON:
5643 return !includesPoison(Kind);
5644
5645 case ISD::UNDEF:
5646 return !includesUndef(Kind);
5647
5648 case ISD::BITCAST: {
5649 SDValue Src = Op.getOperand(i: 0);
5650 EVT SrcVT = Src.getValueType();
5651 EVT DstVT = Op.getValueType();
5652
5653 if (!SrcVT.isVector() || !DstVT.isVector())
5654 return isGuaranteedNotToBeUndefOrPoison(Op: Src, Kind, Depth: Depth + 1);
5655
5656 unsigned SrcEltBits = SrcVT.getScalarSizeInBits();
5657 unsigned DstEltBits = DstVT.getScalarSizeInBits();
5658 ElementCount NumSrcElts = SrcVT.getVectorElementCount();
5659 [[maybe_unused]] ElementCount NumDstElts = DstVT.getVectorElementCount();
5660
5661 if (SrcEltBits == DstEltBits)
5662 return isGuaranteedNotToBeUndefOrPoison(Op: Src, DemandedElts, Kind,
5663 Depth: Depth + 1);
5664
5665 if (SrcEltBits < DstEltBits) {
5666 if (DstEltBits % SrcEltBits != 0)
5667 return isGuaranteedNotToBeUndefOrPoison(Op: Src, Kind, Depth: Depth + 1);
5668
5669 assert(NumSrcElts == NumDstElts * (DstEltBits / SrcEltBits) &&
5670 "Unexpected vector bitcast");
5671 APInt DemandedSrcElts =
5672 APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumSrcElts.getKnownMinValue());
5673 return isGuaranteedNotToBeUndefOrPoison(Op: Src, DemandedElts: DemandedSrcElts, Kind,
5674 Depth: Depth + 1);
5675 }
5676
5677 if (SrcEltBits % DstEltBits != 0)
5678 return isGuaranteedNotToBeUndefOrPoison(Op: Src, Kind, Depth: Depth + 1);
5679
5680 assert(NumDstElts == NumSrcElts * (SrcEltBits / DstEltBits) &&
5681 "Unexpected vector bitcast");
5682 APInt DemandedSrcElts =
5683 APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumSrcElts.getKnownMinValue());
5684 return isGuaranteedNotToBeUndefOrPoison(Op: Src, DemandedElts: DemandedSrcElts, Kind,
5685 Depth: Depth + 1);
5686 }
5687
5688 case ISD::BUILD_VECTOR:
5689 // NOTE: BUILD_VECTOR has implicit truncation of wider scalar elements -
5690 // this shouldn't affect the result.
5691 for (unsigned i = 0, e = Op.getNumOperands(); i < e; ++i) {
5692 if (!DemandedElts[i])
5693 continue;
5694 if (!isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i), Kind, Depth: Depth + 1))
5695 return false;
5696 }
5697 return true;
5698
5699 case ISD::CONCAT_VECTORS: {
5700 EVT VT = Op.getValueType();
5701 if (!VT.isFixedLengthVector())
5702 break;
5703
5704 EVT SubVT = Op.getOperand(i: 0).getValueType();
5705 unsigned NumSubElts = SubVT.getVectorNumElements();
5706 for (unsigned I = 0, E = Op.getNumOperands(); I != E; ++I) {
5707 APInt DemandedSubElts =
5708 DemandedElts.extractBits(numBits: NumSubElts, bitPosition: I * NumSubElts);
5709 if (!!DemandedSubElts &&
5710 !isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: I), DemandedElts: DemandedSubElts,
5711 Kind, Depth: Depth + 1))
5712 return false;
5713 }
5714 return true;
5715 }
5716
5717 case ISD::EXTRACT_SUBVECTOR: {
5718 SDValue Src = Op.getOperand(i: 0);
5719 if (Src.getValueType().isScalableVector())
5720 break;
5721 uint64_t Idx = Op.getConstantOperandVal(i: 1);
5722 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
5723 APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
5724 return isGuaranteedNotToBeUndefOrPoison(Op: Src, DemandedElts: DemandedSrcElts, Kind,
5725 Depth: Depth + 1);
5726 }
5727
5728 case ISD::INSERT_SUBVECTOR: {
5729 if (Op.getValueType().isScalableVector())
5730 break;
5731 SDValue Src = Op.getOperand(i: 0);
5732 SDValue Sub = Op.getOperand(i: 1);
5733 uint64_t Idx = Op.getConstantOperandVal(i: 2);
5734 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
5735 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
5736 APInt DemandedSrcElts = DemandedElts;
5737 DemandedSrcElts.clearBits(LoBit: Idx, HiBit: Idx + NumSubElts);
5738
5739 if (!!DemandedSubElts && !isGuaranteedNotToBeUndefOrPoison(
5740 Op: Sub, DemandedElts: DemandedSubElts, Kind, Depth: Depth + 1))
5741 return false;
5742 if (!!DemandedSrcElts && !isGuaranteedNotToBeUndefOrPoison(
5743 Op: Src, DemandedElts: DemandedSrcElts, Kind, Depth: Depth + 1))
5744 return false;
5745 return true;
5746 }
5747
5748 case ISD::EXTRACT_VECTOR_ELT: {
5749 SDValue Src = Op.getOperand(i: 0);
5750 auto *IndexC = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1));
5751 EVT SrcVT = Src.getValueType();
5752 if (SrcVT.isFixedLengthVector() && IndexC &&
5753 IndexC->getAPIntValue().ult(RHS: SrcVT.getVectorNumElements())) {
5754 APInt DemandedSrcElts = APInt::getOneBitSet(numBits: SrcVT.getVectorNumElements(),
5755 BitNo: IndexC->getZExtValue());
5756 return isGuaranteedNotToBeUndefOrPoison(Op: Src, DemandedElts: DemandedSrcElts, Kind,
5757 Depth: Depth + 1);
5758 }
5759 break;
5760 }
5761
5762 case ISD::INSERT_VECTOR_ELT: {
5763 SDValue InVec = Op.getOperand(i: 0);
5764 SDValue InVal = Op.getOperand(i: 1);
5765 SDValue EltNo = Op.getOperand(i: 2);
5766 EVT VT = InVec.getValueType();
5767 auto *IndexC = dyn_cast<ConstantSDNode>(Val&: EltNo);
5768 if (IndexC && VT.isFixedLengthVector() &&
5769 IndexC->getAPIntValue().ult(RHS: VT.getVectorNumElements())) {
5770 if (DemandedElts[IndexC->getZExtValue()] &&
5771 !isGuaranteedNotToBeUndefOrPoison(Op: InVal, Kind, Depth: Depth + 1))
5772 return false;
5773 APInt InVecDemandedElts = DemandedElts;
5774 InVecDemandedElts.clearBit(BitPosition: IndexC->getZExtValue());
5775 if (!!InVecDemandedElts &&
5776 !isGuaranteedNotToBeUndefOrPoison(
5777 Op: peekThroughInsertVectorElt(V: InVec, DemandedElts: InVecDemandedElts),
5778 DemandedElts: InVecDemandedElts, Kind, Depth: Depth + 1))
5779 return false;
5780 return true;
5781 }
5782 break;
5783 }
5784
5785 case ISD::SCALAR_TO_VECTOR:
5786 // Check upper (known undef) elements.
5787 if (DemandedElts.ugt(RHS: 1) && includesUndef(Kind))
5788 return false;
5789 // Check element zero.
5790 if (DemandedElts[0] &&
5791 !isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 0), Kind, Depth: Depth + 1))
5792 return false;
5793 return true;
5794
5795 case ISD::SPLAT_VECTOR:
5796 return isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 0), Kind, Depth: Depth + 1);
5797
5798 case ISD::SELECT: {
5799 return !canCreateUndefOrPoison(Op, DemandedElts, Kind,
5800 /*ConsiderFlags*/ true, Depth) &&
5801 isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 0), Kind,
5802 Depth: Depth + 1) &&
5803 isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 1), DemandedElts,
5804 Kind, Depth: Depth + 1) &&
5805 isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 2), DemandedElts,
5806 Kind, Depth: Depth + 1);
5807 }
5808
5809 case ISD::VECTOR_SHUFFLE: {
5810 APInt DemandedLHS, DemandedRHS;
5811 auto *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
5812 if (!getShuffleDemandedElts(SrcWidth: DemandedElts.getBitWidth(), Mask: SVN->getMask(),
5813 DemandedElts, DemandedLHS, DemandedRHS,
5814 /*AllowUndefElts=*/false))
5815 return false;
5816 if (!DemandedLHS.isZero() &&
5817 !isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 0), DemandedElts: DemandedLHS, Kind,
5818 Depth: Depth + 1))
5819 return false;
5820 if (!DemandedRHS.isZero() &&
5821 !isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 1), DemandedElts: DemandedRHS, Kind,
5822 Depth: Depth + 1))
5823 return false;
5824 return true;
5825 }
5826
5827 case ISD::SHL:
5828 case ISD::SRL:
5829 case ISD::SRA:
5830 // Shift amount operand is checked by canCreateUndefOrPoison. So it is
5831 // enough to check operand 0 if Op can't create undef/poison.
5832 return !canCreateUndefOrPoison(Op, DemandedElts, Kind,
5833 /*ConsiderFlags*/ true, Depth) &&
5834 isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 0), DemandedElts,
5835 Kind, Depth: Depth + 1);
5836
5837 case ISD::BSWAP:
5838 case ISD::CTPOP:
5839 case ISD::BITREVERSE:
5840 case ISD::AND:
5841 case ISD::OR:
5842 case ISD::XOR:
5843 case ISD::ADD:
5844 case ISD::SUB:
5845 case ISD::MUL:
5846 case ISD::SADDSAT:
5847 case ISD::UADDSAT:
5848 case ISD::SSUBSAT:
5849 case ISD::USUBSAT:
5850 case ISD::SSHLSAT:
5851 case ISD::USHLSAT:
5852 case ISD::SMIN:
5853 case ISD::SMAX:
5854 case ISD::UMIN:
5855 case ISD::UMAX:
5856 case ISD::ZERO_EXTEND:
5857 case ISD::SIGN_EXTEND:
5858 case ISD::ANY_EXTEND:
5859 case ISD::TRUNCATE:
5860 case ISD::VSELECT: {
5861 // If Op can't create undef/poison and none of its operands are undef/poison
5862 // then Op is never undef/poison. A difference from the more common check
5863 // below, outside the switch, is that we handle elementwise operations for
5864 // which the DemandedElts mask is valid for all operands here.
5865 return !canCreateUndefOrPoison(Op, DemandedElts, Kind,
5866 /*ConsiderFlags*/ true, Depth) &&
5867 all_of(Range: Op->ops(), P: [&](SDValue V) {
5868 return isGuaranteedNotToBeUndefOrPoison(Op: V, DemandedElts, Kind,
5869 Depth: Depth + 1);
5870 });
5871 }
5872
5873 // TODO: Search for noundef attributes from library functions.
5874
5875 // TODO: Pointers dereferenced by ISD::LOAD/STORE ops are noundef.
5876
5877 default:
5878 // Allow the target to implement this method for its nodes.
5879 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
5880 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)
5881 return TLI->isGuaranteedNotToBeUndefOrPoisonForTargetNode(
5882 Op, DemandedElts, DAG: *this, Kind, Depth);
5883 break;
5884 }
5885
5886 // If Op can't create undef/poison and none of its operands are undef/poison
5887 // then Op is never undef/poison.
5888 // NOTE: TargetNodes can handle this in themselves in
5889 // isGuaranteedNotToBeUndefOrPoisonForTargetNode or let
5890 // TargetLowering::isGuaranteedNotToBeUndefOrPoisonForTargetNode handle it.
5891 return !canCreateUndefOrPoison(Op, Kind, /*ConsiderFlags*/ true, Depth) &&
5892 all_of(Range: Op->ops(), P: [&](SDValue V) {
5893 return isGuaranteedNotToBeUndefOrPoison(Op: V, Kind, Depth: Depth + 1);
5894 });
5895}
5896
5897bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, UndefPoisonKind Kind,
5898 bool ConsiderFlags,
5899 unsigned Depth) const {
5900 APInt DemandedElts = getDemandAllEltsMask(V: Op);
5901 return canCreateUndefOrPoison(Op, DemandedElts, Kind, ConsiderFlags, Depth);
5902}
5903
5904bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, const APInt &DemandedElts,
5905 UndefPoisonKind Kind,
5906 bool ConsiderFlags,
5907 unsigned Depth) const {
5908 if (ConsiderFlags && includesPoison(Kind) && Op->hasPoisonGeneratingFlags())
5909 return true;
5910
5911 unsigned Opcode = Op.getOpcode();
5912 switch (Opcode) {
5913 case ISD::AssertSext:
5914 case ISD::AssertZext:
5915 case ISD::AssertAlign:
5916 case ISD::AssertNoFPClass:
5917 // Assertion nodes can create poison if the assertion fails.
5918 return includesPoison(Kind);
5919
5920 case ISD::FREEZE:
5921 case ISD::CONCAT_VECTORS:
5922 case ISD::INSERT_SUBVECTOR:
5923 case ISD::EXTRACT_SUBVECTOR:
5924 case ISD::SADDSAT:
5925 case ISD::UADDSAT:
5926 case ISD::SSUBSAT:
5927 case ISD::USUBSAT:
5928 case ISD::MULHU:
5929 case ISD::MULHS:
5930 case ISD::AVGFLOORS:
5931 case ISD::AVGFLOORU:
5932 case ISD::AVGCEILS:
5933 case ISD::AVGCEILU:
5934 case ISD::ABDU:
5935 case ISD::ABDS:
5936 case ISD::SMIN:
5937 case ISD::SMAX:
5938 case ISD::SCMP:
5939 case ISD::UMIN:
5940 case ISD::UMAX:
5941 case ISD::UCMP:
5942 case ISD::AND:
5943 case ISD::XOR:
5944 case ISD::ROTL:
5945 case ISD::ROTR:
5946 case ISD::FSHL:
5947 case ISD::FSHR:
5948 case ISD::BSWAP:
5949 case ISD::CTTZ:
5950 case ISD::CTLZ:
5951 case ISD::CTLS:
5952 case ISD::CTPOP:
5953 case ISD::BITREVERSE:
5954 case ISD::PARITY:
5955 case ISD::SIGN_EXTEND:
5956 case ISD::TRUNCATE:
5957 case ISD::SIGN_EXTEND_INREG:
5958 case ISD::SIGN_EXTEND_VECTOR_INREG:
5959 case ISD::ZERO_EXTEND_VECTOR_INREG:
5960 case ISD::BITCAST:
5961 case ISD::BUILD_VECTOR:
5962 case ISD::BUILD_PAIR:
5963 case ISD::SPLAT_VECTOR:
5964 case ISD::FABS:
5965 case ISD::FCEIL:
5966 case ISD::FFLOOR:
5967 case ISD::FTRUNC:
5968 case ISD::FRINT:
5969 case ISD::FNEARBYINT:
5970 case ISD::FROUND:
5971 case ISD::FROUNDEVEN:
5972 return false;
5973
5974 case ISD::ABS:
5975 // ISD::ABS defines abs(INT_MIN) -> INT_MIN and never generates poison.
5976 // Different to Intrinsic::abs.
5977 return false;
5978 case ISD::ABS_MIN_POISON:
5979 // ABS_MIN_POISON may produce poison if the input is INT_MIN.
5980 return ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1) <= 1;
5981
5982 case ISD::ADDC:
5983 case ISD::SUBC:
5984 case ISD::ADDE:
5985 case ISD::SUBE:
5986 case ISD::SADDO:
5987 case ISD::SSUBO:
5988 case ISD::SMULO:
5989 case ISD::SADDO_CARRY:
5990 case ISD::SSUBO_CARRY:
5991 case ISD::UADDO:
5992 case ISD::USUBO:
5993 case ISD::UMULO:
5994 case ISD::UADDO_CARRY:
5995 case ISD::USUBO_CARRY:
5996 // No poison on result or overflow flags.
5997 return false;
5998
5999 case ISD::SELECT_CC:
6000 case ISD::SETCC: {
6001 // Integer setcc cannot create undef or poison.
6002 if (Op.getOperand(i: 0).getValueType().isInteger())
6003 return false;
6004
6005 // FP compares are more complicated. They can create poison for nan/infinity
6006 // based on options and flags. The options and flags also cause special
6007 // nonan condition codes to be used. Those condition codes may be preserved
6008 // even if the nonan flag is dropped somewhere.
6009 unsigned CCOp = Opcode == ISD::SETCC ? 2 : 4;
6010 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: Op.getOperand(i: CCOp))->get();
6011 return (unsigned)CCCode & 0x10U;
6012 }
6013
6014 case ISD::OR:
6015 case ISD::ZERO_EXTEND:
6016 case ISD::SELECT:
6017 case ISD::VSELECT:
6018 case ISD::ADD:
6019 case ISD::SUB:
6020 case ISD::MUL:
6021 case ISD::FNEG:
6022 case ISD::FADD:
6023 case ISD::FSUB:
6024 case ISD::FMUL:
6025 case ISD::FDIV:
6026 case ISD::FREM:
6027 case ISD::FCOPYSIGN:
6028 case ISD::FMA:
6029 case ISD::FMAD:
6030 case ISD::FMULADD:
6031 case ISD::FP_EXTEND:
6032 case ISD::FMINNUM:
6033 case ISD::FMAXNUM:
6034 case ISD::FMINNUM_IEEE:
6035 case ISD::FMAXNUM_IEEE:
6036 case ISD::FMINIMUM:
6037 case ISD::FMAXIMUM:
6038 case ISD::FMINIMUMNUM:
6039 case ISD::FMAXIMUMNUM:
6040 case ISD::FP_TO_SINT_SAT:
6041 case ISD::FP_TO_UINT_SAT:
6042 case ISD::TRUNCATE_SSAT_S:
6043 case ISD::TRUNCATE_SSAT_U:
6044 case ISD::TRUNCATE_USAT_U:
6045 // No poison except from flags (which is handled above)
6046 return false;
6047
6048 case ISD::SHL:
6049 case ISD::SRL:
6050 case ISD::SRA:
6051 // If the max shift amount isn't in range, then the shift can
6052 // create poison.
6053 return includesPoison(Kind) &&
6054 !getValidMaximumShiftAmount(V: Op, DemandedElts, Depth: Depth + 1);
6055
6056 case ISD::CTTZ_ZERO_POISON:
6057 case ISD::CTLZ_ZERO_POISON:
6058 // If the amount is zero then the result will be poison.
6059 // TODO: Add isKnownNeverZero DemandedElts handling.
6060 return includesPoison(Kind) &&
6061 !isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6062
6063 case ISD::SCALAR_TO_VECTOR:
6064 // Check if we demand any upper (undef) elements.
6065 return includesUndef(Kind) && DemandedElts.ugt(RHS: 1);
6066
6067 case ISD::INSERT_VECTOR_ELT:
6068 case ISD::EXTRACT_VECTOR_ELT: {
6069 // Ensure that the element index is in bounds.
6070 if (includesPoison(Kind)) {
6071 EVT VecVT = Op.getOperand(i: 0).getValueType();
6072 SDValue Idx = Op.getOperand(i: Opcode == ISD::INSERT_VECTOR_ELT ? 2 : 1);
6073 KnownBits KnownIdx = computeKnownBits(Op: Idx, Depth: Depth + 1);
6074 return KnownIdx.getMaxValue().uge(RHS: VecVT.getVectorMinNumElements());
6075 }
6076 return false;
6077 }
6078
6079 case ISD::VECTOR_SHUFFLE: {
6080 // Check for any demanded shuffle element that is undef.
6081 auto *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
6082 for (auto [Idx, Elt] : enumerate(First: SVN->getMask()))
6083 if (Elt < 0 && DemandedElts[Idx])
6084 return true;
6085 return false;
6086 }
6087
6088 case ISD::VECTOR_COMPRESS:
6089 return false;
6090
6091 default:
6092 // Allow the target to implement this method for its nodes.
6093 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
6094 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)
6095 return TLI->canCreateUndefOrPoisonForTargetNode(
6096 Op, DemandedElts, DAG: *this, Kind, ConsiderFlags, Depth);
6097 break;
6098 }
6099
6100 // Be conservative and return true.
6101 return true;
6102}
6103
6104bool SelectionDAG::isADDLike(SDValue Op, bool NoWrap) const {
6105 unsigned Opcode = Op.getOpcode();
6106 if (Opcode == ISD::OR)
6107 return Op->getFlags().hasDisjoint() ||
6108 haveNoCommonBitsSet(A: Op.getOperand(i: 0), B: Op.getOperand(i: 1));
6109 if (Opcode == ISD::XOR)
6110 return !NoWrap && isMinSignedConstant(V: Op.getOperand(i: 1));
6111 return false;
6112}
6113
6114bool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const {
6115 return Op.getNumOperands() == 2 && isa<ConstantSDNode>(Val: Op.getOperand(i: 1)) &&
6116 (Op.isAnyAdd() || isADDLike(Op));
6117}
6118
6119KnownFPClass SelectionDAG::computeKnownFPClass(SDValue Op,
6120 FPClassTest InterestedClasses,
6121 unsigned Depth) const {
6122 APInt DemandedElts = getDemandAllEltsMask(V: Op);
6123 return computeKnownFPClass(Op, DemandedElts, InterestedClasses, Depth);
6124}
6125
6126KnownFPClass SelectionDAG::computeKnownFPClass(SDValue Op,
6127 const APInt &DemandedElts,
6128 FPClassTest InterestedClasses,
6129 unsigned Depth) const {
6130 KnownFPClass Known;
6131
6132 if (const auto *CFP = dyn_cast<ConstantFPSDNode>(Val&: Op))
6133 return KnownFPClass(CFP->getValueAPF());
6134
6135 if (Depth >= MaxRecursionDepth)
6136 return Known;
6137
6138 if (Op.getOpcode() == ISD::UNDEF)
6139 return Known;
6140
6141 EVT VT = Op.getValueType();
6142 assert(VT.isFloatingPoint() && "Computing KnownFPClass on non-FP op!");
6143 assert((!VT.isFixedLengthVector() ||
6144 DemandedElts.getBitWidth() == VT.getVectorNumElements()) &&
6145 "Unexpected vector size");
6146
6147 if (!DemandedElts)
6148 return Known;
6149
6150 unsigned Opcode = Op.getOpcode();
6151 switch (Opcode) {
6152 case ISD::POISON: {
6153 Known.KnownFPClasses = fcNone;
6154 Known.SignBit = false;
6155 break;
6156 }
6157 case ISD::FNEG: {
6158 Known = computeKnownFPClass(Op: Op.getOperand(i: 0), DemandedElts,
6159 InterestedClasses, Depth: Depth + 1);
6160 Known.fneg();
6161 break;
6162 }
6163 case ISD::BUILD_VECTOR: {
6164 assert(!VT.isScalableVector());
6165 bool First = true;
6166 for (unsigned I = 0, E = Op.getNumOperands(); I != E; ++I) {
6167 if (!DemandedElts[I])
6168 continue;
6169
6170 if (First) {
6171 Known =
6172 computeKnownFPClass(Op: Op.getOperand(i: I), InterestedClasses, Depth: Depth + 1);
6173 First = false;
6174 } else {
6175 Known |=
6176 computeKnownFPClass(Op: Op.getOperand(i: I), InterestedClasses, Depth: Depth + 1);
6177 }
6178
6179 if (Known.isUnknown())
6180 break;
6181 }
6182 break;
6183 }
6184 case ISD::EXTRACT_VECTOR_ELT: {
6185 SDValue Src = Op.getOperand(i: 0);
6186 auto *CIdx = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1));
6187 EVT SrcVT = Src.getValueType();
6188 if (SrcVT.isFixedLengthVector() && CIdx) {
6189 if (CIdx->getAPIntValue().ult(RHS: SrcVT.getVectorNumElements())) {
6190 APInt DemandedSrcElts = APInt::getOneBitSet(
6191 numBits: SrcVT.getVectorNumElements(), BitNo: CIdx->getZExtValue());
6192 Known = computeKnownFPClass(Op: Src, DemandedElts: DemandedSrcElts, InterestedClasses,
6193 Depth: Depth + 1);
6194 } else {
6195 // Out of bounds index is poison.
6196 Known.KnownFPClasses = fcNone;
6197 }
6198 } else {
6199 Known = computeKnownFPClass(Op: Src, InterestedClasses, Depth: Depth + 1);
6200 }
6201 break;
6202 }
6203 case ISD::SPLAT_VECTOR: {
6204 Known = computeKnownFPClass(Op: Op.getOperand(i: 0), InterestedClasses, Depth: Depth + 1);
6205 break;
6206 }
6207 case ISD::BITCAST: {
6208 // FIXME: It should not be necessary to check for an elementwise bitcast.
6209 // If a bitcast is not elementwise between vector / scalar types,
6210 // computeKnownBits already splices the known bits of the source elements
6211 // appropriately so as to line up with the bits of the result's demanded
6212 // elements.
6213 EVT SrcVT = Op.getOperand(i: 0).getValueType();
6214 if (VT.isScalableVector() || SrcVT.isScalableVector())
6215 break;
6216 unsigned VTNumElts = VT.isVector() ? VT.getVectorNumElements() : 1;
6217 unsigned SrcVTNumElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
6218 if (VTNumElts != SrcVTNumElts)
6219 break;
6220
6221 KnownBits Bits = computeKnownBits(Op, DemandedElts, Depth: Depth + 1);
6222 Known = KnownFPClass::bitcast(FltSemantics: VT.getFltSemantics(), Bits);
6223 break;
6224 }
6225 case ISD::FABS: {
6226 Known = computeKnownFPClass(Op: Op.getOperand(i: 0), DemandedElts,
6227 InterestedClasses, Depth: Depth + 1);
6228 Known.fabs();
6229 break;
6230 }
6231 case ISD::FCOPYSIGN: {
6232 Known = computeKnownFPClass(Op: Op.getOperand(i: 0), DemandedElts,
6233 InterestedClasses, Depth: Depth + 1);
6234 KnownFPClass KnownSign = computeKnownFPClass(Op: Op.getOperand(i: 1), DemandedElts,
6235 InterestedClasses, Depth: Depth + 1);
6236 Known.copysign(Sign: KnownSign);
6237 break;
6238 }
6239 case ISD::AssertNoFPClass: {
6240 Known = computeKnownFPClass(Op: Op.getOperand(i: 0), DemandedElts,
6241 InterestedClasses, Depth: Depth + 1);
6242 FPClassTest AssertedClasses =
6243 static_cast<FPClassTest>(Op->getConstantOperandVal(Num: 1));
6244 Known.KnownFPClasses &= ~AssertedClasses;
6245 break;
6246 }
6247 case ISD::EXTRACT_SUBVECTOR: {
6248 SDValue Src = Op.getOperand(i: 0);
6249 EVT SrcVT = Src.getValueType();
6250 if (SrcVT.isFixedLengthVector()) {
6251 unsigned Idx = Op.getConstantOperandVal(i: 1);
6252 unsigned NumSrcElts = SrcVT.getVectorNumElements();
6253
6254 APInt DemandedSrcElts = DemandedElts.zextOrTrunc(width: NumSrcElts).shl(shiftAmt: Idx);
6255 Known = computeKnownFPClass(Op: Src, DemandedElts: DemandedSrcElts, InterestedClasses,
6256 Depth: Depth + 1);
6257 } else {
6258 Known = computeKnownFPClass(Op: Src, InterestedClasses, Depth: Depth + 1);
6259 }
6260 break;
6261 }
6262 case ISD::INSERT_SUBVECTOR: {
6263 SDValue BaseVector = Op.getOperand(i: 0);
6264 SDValue SubVector = Op.getOperand(i: 1);
6265 EVT BaseVT = BaseVector.getValueType();
6266 if (BaseVT.isFixedLengthVector()) {
6267 unsigned Idx = Op.getConstantOperandVal(i: 2);
6268 unsigned NumBaseElts = BaseVT.getVectorNumElements();
6269 unsigned NumSubElts = SubVector.getValueType().getVectorNumElements();
6270
6271 APInt DemandedMask =
6272 APInt::getBitsSet(numBits: NumBaseElts, loBit: Idx, hiBit: Idx + NumSubElts);
6273 APInt DemandedSrcElts = DemandedElts & ~DemandedMask;
6274 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
6275
6276 if (!DemandedSrcElts.isZero())
6277 Known = computeKnownFPClass(Op: BaseVector, DemandedElts: DemandedSrcElts,
6278 InterestedClasses, Depth: Depth + 1);
6279 if (!DemandedSubElts.isZero()) {
6280 KnownFPClass SubKnown = computeKnownFPClass(
6281 Op: SubVector, DemandedElts: DemandedSubElts, InterestedClasses, Depth: Depth + 1);
6282 Known = DemandedSrcElts.isZero() ? SubKnown : (Known | SubKnown);
6283 }
6284 } else {
6285 Known = computeKnownFPClass(Op: SubVector, InterestedClasses, Depth: Depth + 1);
6286 if (!Known.isUnknown())
6287 Known |= computeKnownFPClass(Op: BaseVector, InterestedClasses, Depth: Depth + 1);
6288 }
6289 break;
6290 }
6291 case ISD::SELECT:
6292 case ISD::VSELECT: {
6293 // TODO: Add adjustKnownFPClassForSelectArm clamp recognition as in
6294 // IR-level ValueTracking.
6295 KnownFPClass KnownFalseClass = computeKnownFPClass(
6296 Op: Op.getOperand(i: 2), DemandedElts, InterestedClasses, Depth: Depth + 1);
6297 if (KnownFalseClass.isUnknown())
6298 break;
6299 KnownFPClass KnownTrueClass = computeKnownFPClass(
6300 Op: Op.getOperand(i: 1), DemandedElts, InterestedClasses, Depth: Depth + 1);
6301 Known = KnownTrueClass.intersectWith(RHS: KnownFalseClass);
6302 break;
6303 }
6304 default:
6305 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
6306 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID) {
6307 TLI->computeKnownFPClassForTargetNode(Op, Known, DemandedElts, DAG: *this,
6308 Depth);
6309 }
6310 break;
6311 }
6312
6313 return Known;
6314}
6315
6316bool SelectionDAG::isKnownNeverNaN(SDValue Op, bool SNaN,
6317 unsigned Depth) const {
6318 APInt DemandedElts = getDemandAllEltsMask(V: Op);
6319 return isKnownNeverNaN(Op, DemandedElts, SNaN, Depth);
6320}
6321
6322bool SelectionDAG::isKnownNeverNaN(SDValue Op, const APInt &DemandedElts,
6323 bool SNaN, unsigned Depth) const {
6324 assert(!DemandedElts.isZero() && "No demanded elements");
6325
6326 // If we're told that NaNs won't happen, assume they won't.
6327 if (Op->getFlags().hasNoNaNs())
6328 return true;
6329
6330 if (Depth >= MaxRecursionDepth)
6331 return false; // Limit search depth.
6332
6333 unsigned Opcode = Op.getOpcode();
6334 switch (Opcode) {
6335 case ISD::FADD:
6336 case ISD::FSUB:
6337 case ISD::FMUL:
6338 case ISD::FDIV:
6339 case ISD::FREM:
6340 case ISD::FSIN:
6341 case ISD::FCOS:
6342 case ISD::FTAN:
6343 case ISD::FASIN:
6344 case ISD::FACOS:
6345 case ISD::FATAN:
6346 case ISD::FATAN2:
6347 case ISD::FSINH:
6348 case ISD::FCOSH:
6349 case ISD::FTANH:
6350 case ISD::FMA:
6351 case ISD::FMULADD:
6352 case ISD::FMAD: {
6353 if (SNaN)
6354 return true;
6355 // TODO: Need isKnownNeverInfinity
6356 return false;
6357 }
6358 case ISD::FCANONICALIZE:
6359 case ISD::FEXP:
6360 case ISD::FEXP2:
6361 case ISD::FEXP10:
6362 case ISD::FTRUNC:
6363 case ISD::FFLOOR:
6364 case ISD::FCEIL:
6365 case ISD::FROUND:
6366 case ISD::FROUNDEVEN:
6367 case ISD::LROUND:
6368 case ISD::LLROUND:
6369 case ISD::FRINT:
6370 case ISD::LRINT:
6371 case ISD::LLRINT:
6372 case ISD::FNEARBYINT:
6373 case ISD::FLDEXP: {
6374 if (SNaN)
6375 return true;
6376 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1);
6377 }
6378 case ISD::FABS:
6379 case ISD::FNEG:
6380 case ISD::FCOPYSIGN: {
6381 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1);
6382 }
6383 case ISD::SELECT:
6384 return isKnownNeverNaN(Op: Op.getOperand(i: 1), DemandedElts, SNaN, Depth: Depth + 1) &&
6385 isKnownNeverNaN(Op: Op.getOperand(i: 2), DemandedElts, SNaN, Depth: Depth + 1);
6386 case ISD::FP_EXTEND:
6387 case ISD::FP_ROUND: {
6388 if (SNaN)
6389 return true;
6390 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1);
6391 }
6392 case ISD::SINT_TO_FP:
6393 case ISD::UINT_TO_FP:
6394 return true;
6395 case ISD::FSQRT: // Need is known positive
6396 case ISD::FLOG:
6397 case ISD::FLOG2:
6398 case ISD::FLOG10:
6399 case ISD::FPOWI:
6400 case ISD::FPOW: {
6401 if (SNaN)
6402 return true;
6403 // TODO: Refine on operand
6404 return false;
6405 }
6406 case ISD::FMINNUM:
6407 case ISD::FMAXNUM:
6408 case ISD::FMINIMUMNUM:
6409 case ISD::FMAXIMUMNUM: {
6410 // Only one needs to be known not-nan, since it will be returned if the
6411 // other ends up being one.
6412 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1) ||
6413 isKnownNeverNaN(Op: Op.getOperand(i: 1), DemandedElts, SNaN, Depth: Depth + 1);
6414 }
6415 case ISD::FMINNUM_IEEE:
6416 case ISD::FMAXNUM_IEEE: {
6417 if (SNaN)
6418 return true;
6419 // This can return a NaN if either operand is an sNaN, or if both operands
6420 // are NaN.
6421 return (isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN: false, Depth: Depth + 1) &&
6422 isKnownNeverSNaN(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1)) ||
6423 (isKnownNeverNaN(Op: Op.getOperand(i: 1), DemandedElts, SNaN: false, Depth: Depth + 1) &&
6424 isKnownNeverSNaN(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1));
6425 }
6426 case ISD::FMINIMUM:
6427 case ISD::FMAXIMUM: {
6428 // TODO: Does this quiet or return the origina NaN as-is?
6429 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1) &&
6430 isKnownNeverNaN(Op: Op.getOperand(i: 1), DemandedElts, SNaN, Depth: Depth + 1);
6431 }
6432 case ISD::EXTRACT_VECTOR_ELT: {
6433 SDValue Src = Op.getOperand(i: 0);
6434 auto *Idx = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1));
6435 EVT SrcVT = Src.getValueType();
6436 if (SrcVT.isFixedLengthVector() && Idx &&
6437 Idx->getAPIntValue().ult(RHS: SrcVT.getVectorNumElements())) {
6438 APInt DemandedSrcElts = APInt::getOneBitSet(numBits: SrcVT.getVectorNumElements(),
6439 BitNo: Idx->getZExtValue());
6440 return isKnownNeverNaN(Op: Src, DemandedElts: DemandedSrcElts, SNaN, Depth: Depth + 1);
6441 }
6442 return isKnownNeverNaN(Op: Src, SNaN, Depth: Depth + 1);
6443 }
6444 case ISD::EXTRACT_SUBVECTOR: {
6445 SDValue Src = Op.getOperand(i: 0);
6446 if (Src.getValueType().isFixedLengthVector()) {
6447 unsigned Idx = Op.getConstantOperandVal(i: 1);
6448 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
6449 APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
6450 return isKnownNeverNaN(Op: Src, DemandedElts: DemandedSrcElts, SNaN, Depth: Depth + 1);
6451 }
6452 return isKnownNeverNaN(Op: Src, SNaN, Depth: Depth + 1);
6453 }
6454 case ISD::INSERT_SUBVECTOR: {
6455 SDValue BaseVector = Op.getOperand(i: 0);
6456 SDValue SubVector = Op.getOperand(i: 1);
6457 EVT BaseVectorVT = BaseVector.getValueType();
6458 if (BaseVectorVT.isFixedLengthVector()) {
6459 unsigned Idx = Op.getConstantOperandVal(i: 2);
6460 unsigned NumBaseElts = BaseVectorVT.getVectorNumElements();
6461 unsigned NumSubElts = SubVector.getValueType().getVectorNumElements();
6462
6463 // Clear/Extract the bits at the position where the subvector will be
6464 // inserted.
6465 APInt DemandedMask =
6466 APInt::getBitsSet(numBits: NumBaseElts, loBit: Idx, hiBit: Idx + NumSubElts);
6467 APInt DemandedSrcElts = DemandedElts & ~DemandedMask;
6468 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
6469
6470 bool NeverNaN = true;
6471 if (!DemandedSrcElts.isZero())
6472 NeverNaN &=
6473 isKnownNeverNaN(Op: BaseVector, DemandedElts: DemandedSrcElts, SNaN, Depth: Depth + 1);
6474 if (NeverNaN && !DemandedSubElts.isZero())
6475 NeverNaN &=
6476 isKnownNeverNaN(Op: SubVector, DemandedElts: DemandedSubElts, SNaN, Depth: Depth + 1);
6477 return NeverNaN;
6478 }
6479 return isKnownNeverNaN(Op: BaseVector, SNaN, Depth: Depth + 1) &&
6480 isKnownNeverNaN(Op: SubVector, SNaN, Depth: Depth + 1);
6481 }
6482 case ISD::BUILD_VECTOR: {
6483 unsigned NumElts = Op.getNumOperands();
6484 for (unsigned I = 0; I != NumElts; ++I)
6485 if (DemandedElts[I] &&
6486 !isKnownNeverNaN(Op: Op.getOperand(i: I), SNaN, Depth: Depth + 1))
6487 return false;
6488 return true;
6489 }
6490 case ISD::SPLAT_VECTOR:
6491 return isKnownNeverNaN(Op: Op.getOperand(i: 0), SNaN, Depth: Depth + 1);
6492 case ISD::AssertNoFPClass: {
6493 FPClassTest NoFPClass =
6494 static_cast<FPClassTest>(Op.getConstantOperandVal(i: 1));
6495 if ((NoFPClass & fcNan) == fcNan)
6496 return true;
6497 if (SNaN && (NoFPClass & fcSNan) == fcSNan)
6498 return true;
6499 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1);
6500 }
6501 default:
6502 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
6503 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID) {
6504 return TLI->isKnownNeverNaNForTargetNode(Op, DemandedElts, DAG: *this, SNaN,
6505 Depth);
6506 }
6507 break;
6508 }
6509
6510 FPClassTest NanMask = SNaN ? fcSNan : fcNan;
6511 KnownFPClass Known = computeKnownFPClass(Op, DemandedElts, InterestedClasses: NanMask, Depth);
6512 return Known.isKnownNever(Mask: NanMask);
6513}
6514
6515bool SelectionDAG::isKnownNeverLogicalZero(SDValue Op, unsigned Depth) const {
6516 APInt DemandedElts = getDemandAllEltsMask(V: Op);
6517 return isKnownNeverLogicalZero(Op, DemandedElts, Depth);
6518}
6519
6520bool SelectionDAG::isKnownNeverLogicalZero(SDValue Op,
6521 const APInt &DemandedElts,
6522 unsigned Depth) const {
6523 assert(!DemandedElts.isZero() && "No demanded elements");
6524 EVT VT = Op.getValueType();
6525 KnownFPClass Known =
6526 computeKnownFPClass(Op, DemandedElts, InterestedClasses: fcZero | fcSubnormal, Depth);
6527 return Known.isKnownNeverLogicalZero(Mode: getDenormalMode(VT));
6528}
6529
6530bool SelectionDAG::isKnownNeverZero(SDValue Op, unsigned Depth) const {
6531 APInt DemandedElts = getDemandAllEltsMask(V: Op);
6532 return isKnownNeverZero(Op, DemandedElts, Depth);
6533}
6534
6535bool SelectionDAG::isKnownNeverZero(SDValue Op, const APInt &DemandedElts,
6536 unsigned Depth) const {
6537 if (Depth >= MaxRecursionDepth)
6538 return false; // Limit search depth.
6539
6540 EVT OpVT = Op.getValueType();
6541 unsigned BitWidth = OpVT.getScalarSizeInBits();
6542
6543 assert(!Op.getValueType().isFloatingPoint() &&
6544 "Floating point types unsupported - use isKnownNeverLogicalZero");
6545
6546 // If the value is a constant, we can obviously see if it is a zero or not.
6547 auto IsNeverZero = [BitWidth](const ConstantSDNode *C) {
6548 APInt V = C->getAPIntValue().zextOrTrunc(width: BitWidth);
6549 return !V.isZero();
6550 };
6551
6552 if (ISD::matchUnaryPredicate(Op, DemandedElts, Match: IsNeverZero,
6553 /*AllowUndefs=*/false, /*AllowTruncation=*/true))
6554 return true;
6555
6556 // TODO: Recognize more cases here. Most of the cases are also incomplete to
6557 // some degree.
6558 switch (Op.getOpcode()) {
6559 default:
6560 break;
6561
6562 case ISD::EXTRACT_VECTOR_ELT: {
6563 SDValue InVec = Op.getOperand(i: 0);
6564 SDValue EltNo = Op.getOperand(i: 1);
6565 EVT VecVT = InVec.getValueType();
6566
6567 // Skip scalable vectors or implicit extensions.
6568 if (VecVT.isScalableVector() ||
6569 OpVT.getScalarSizeInBits() != VecVT.getScalarSizeInBits())
6570 break;
6571
6572 // If we know the element index, just demand that vector element, else for
6573 // an unknown element index, ignore DemandedElts and demand them all.
6574 const unsigned NumSrcElts = VecVT.getVectorNumElements();
6575 APInt DemandedSrcElts = APInt::getAllOnes(numBits: NumSrcElts);
6576 auto *ConstEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
6577 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(RHS: NumSrcElts))
6578 DemandedSrcElts =
6579 APInt::getOneBitSet(numBits: NumSrcElts, BitNo: ConstEltNo->getZExtValue());
6580
6581 return isKnownNeverZero(Op: InVec, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
6582 }
6583
6584 case ISD::OR:
6585 return isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1) ||
6586 isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6587
6588 case ISD::VSELECT:
6589 case ISD::SELECT:
6590 return isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1) &&
6591 isKnownNeverZero(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth + 1);
6592
6593 case ISD::SHL: {
6594 if (Op->getFlags().hasNoSignedWrap() || Op->getFlags().hasNoUnsignedWrap())
6595 return isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6596 KnownBits ValKnown =
6597 computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6598 // 1 << X is never zero.
6599 if (ValKnown.One[0])
6600 return true;
6601 // If max shift cnt of known ones is non-zero, result is non-zero.
6602 APInt MaxCnt = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1)
6603 .getMaxValue();
6604 if (MaxCnt.ult(RHS: ValKnown.getBitWidth()) &&
6605 !ValKnown.One.shl(ShiftAmt: MaxCnt).isZero())
6606 return true;
6607 break;
6608 }
6609
6610 case ISD::VECTOR_SHUFFLE: {
6611 if (Op.getValueType().isScalableVector())
6612 return false;
6613
6614 unsigned NumElts = DemandedElts.getBitWidth();
6615
6616 // All demanded elements from LHS and RHS must be known non-zero.
6617 // Demanded elements with undef shuffle mask elements are unknown.
6618
6619 APInt DemandedLHS, DemandedRHS;
6620 auto *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
6621 assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
6622 if (!getShuffleDemandedElts(SrcWidth: NumElts, Mask: SVN->getMask(), DemandedElts,
6623 DemandedLHS, DemandedRHS))
6624 return false;
6625
6626 return (!DemandedLHS ||
6627 isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts: DemandedLHS, Depth: Depth + 1)) &&
6628 (!DemandedRHS ||
6629 isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts: DemandedRHS, Depth: Depth + 1));
6630 }
6631
6632 case ISD::UADDSAT:
6633 case ISD::UMAX:
6634 return isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1) ||
6635 isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6636
6637 case ISD::UMIN:
6638 return isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1) &&
6639 isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6640
6641 // For smin/smax: If either operand is known negative/positive
6642 // respectively we don't need the other to be known at all.
6643 case ISD::SMAX: {
6644 KnownBits Op1 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
6645 if (Op1.isStrictlyPositive())
6646 return true;
6647
6648 KnownBits Op0 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6649 if (Op0.isStrictlyPositive())
6650 return true;
6651
6652 if (Op1.isNonZero() && Op0.isNonZero())
6653 return true;
6654
6655 return isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1) &&
6656 isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6657 }
6658 case ISD::SMIN: {
6659 KnownBits Op1 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
6660 if (Op1.isNegative())
6661 return true;
6662
6663 KnownBits Op0 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6664 if (Op0.isNegative())
6665 return true;
6666
6667 if (Op1.isNonZero() && Op0.isNonZero())
6668 return true;
6669
6670 return isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1) &&
6671 isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6672 }
6673
6674 case ISD::ROTL:
6675 case ISD::ROTR:
6676 case ISD::BITREVERSE:
6677 case ISD::BSWAP:
6678 case ISD::CTPOP:
6679 case ISD::ABS:
6680 case ISD::ABS_MIN_POISON:
6681 return isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6682
6683 case ISD::SRA:
6684 case ISD::SRL: {
6685 if (Op->getFlags().hasExact())
6686 return isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6687 KnownBits ValKnown =
6688 computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6689 if (ValKnown.isNegative())
6690 return true;
6691 // If max shift cnt of known ones is non-zero, result is non-zero.
6692 APInt MaxCnt = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1)
6693 .getMaxValue();
6694 if (MaxCnt.ult(RHS: ValKnown.getBitWidth()) &&
6695 !ValKnown.One.lshr(ShiftAmt: MaxCnt).isZero())
6696 return true;
6697 break;
6698 }
6699 case ISD::UDIV:
6700 case ISD::SDIV:
6701 // div exact can only produce a zero if the dividend is zero.
6702 // TODO: For udiv this is also true if Op1 u<= Op0
6703 if (Op->getFlags().hasExact())
6704 return isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6705 break;
6706
6707 case ISD::ADD:
6708 if (Op->getFlags().hasNoUnsignedWrap())
6709 if (isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1) ||
6710 isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1))
6711 return true;
6712 // TODO: There are a lot more cases we can prove for add.
6713 break;
6714
6715 case ISD::SUB: {
6716 if (isNullConstant(V: Op.getOperand(i: 0)))
6717 return isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
6718
6719 std::optional<bool> ne = KnownBits::ne(
6720 LHS: computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1),
6721 RHS: computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1));
6722 return ne && *ne;
6723 }
6724
6725 case ISD::MUL:
6726 if (Op->getFlags().hasNoSignedWrap() || Op->getFlags().hasNoUnsignedWrap())
6727 if (isKnownNeverZero(Op: Op.getOperand(i: 1), Depth: Depth + 1) &&
6728 isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1))
6729 return true;
6730 break;
6731
6732 case ISD::ZERO_EXTEND:
6733 case ISD::SIGN_EXTEND:
6734 return isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6735 case ISD::VSCALE: {
6736 const Function &F = getMachineFunction().getFunction();
6737 const APInt &Multiplier = Op.getConstantOperandAPInt(i: 0);
6738 ConstantRange CR =
6739 getVScaleRange(F: &F, BitWidth: Op.getScalarValueSizeInBits()).multiply(Other: Multiplier);
6740 if (!CR.contains(Val: APInt(CR.getBitWidth(), 0)))
6741 return true;
6742 break;
6743 }
6744 }
6745
6746 return computeKnownBits(Op, DemandedElts, Depth).isNonZero();
6747}
6748
6749bool SelectionDAG::cannotBeOrderedNegativeFP(SDValue Op) const {
6750 if (ConstantFPSDNode *C1 = isConstOrConstSplatFP(N: Op, AllowUndefs: true))
6751 return !C1->isNegative();
6752
6753 switch (Op.getOpcode()) {
6754 case ISD::FABS:
6755 case ISD::FEXP:
6756 case ISD::FEXP2:
6757 case ISD::FEXP10:
6758 return true;
6759 default:
6760 return false;
6761 }
6762
6763 llvm_unreachable("covered opcode switch");
6764}
6765
6766bool SelectionDAG::canIgnoreSignBitOfZero(const SDUse &Use) const {
6767 assert(Use.getValueType().isFloatingPoint());
6768 const SDNode *User = Use.getUser();
6769 if (User->getFlags().hasNoSignedZeros())
6770 return true;
6771
6772 unsigned OperandNo = Use.getOperandNo();
6773 // Check if this use is insensitive to the sign of zero
6774 switch (User->getOpcode()) {
6775 case ISD::SETCC:
6776 // Comparisons: IEEE-754 specifies +0.0 == -0.0.
6777 case ISD::FABS:
6778 // fabs always produces +0.0.
6779 return true;
6780 case ISD::FCOPYSIGN:
6781 // copysign overwrites the sign bit of the first operand.
6782 return OperandNo == 0;
6783 case ISD::FADD:
6784 case ISD::FSUB: {
6785 // Arithmetic with non-zero constants fixes the uncertainty around the
6786 // sign bit.
6787 SDValue Other = User->getOperand(Num: 1 - OperandNo);
6788 return isKnownNeverLogicalZero(Op: Other);
6789 }
6790 case ISD::FP_TO_SINT:
6791 case ISD::FP_TO_UINT:
6792 // fp-to-int conversions normalize signed zeros.
6793 return true;
6794 default:
6795 return false;
6796 }
6797}
6798
6799bool SelectionDAG::canIgnoreSignBitOfZero(SDValue Op) const {
6800 if (Op->getFlags().hasNoSignedZeros())
6801 return true;
6802 // FIXME: Limit the amount of checked uses to not introduce a compile-time
6803 // regression. Ideally, this should be implemented as a demanded-bits
6804 // optimization that stems from the users.
6805 if (Op->use_size() > 2)
6806 return false;
6807 return all_of(Range: Op->uses(),
6808 P: [&](const SDUse &Use) { return canIgnoreSignBitOfZero(Use); });
6809}
6810
6811bool SelectionDAG::isEqualTo(SDValue A, SDValue B) const {
6812 // Check the obvious case.
6813 if (A == B) return true;
6814
6815 // For negative and positive zero.
6816 if (const ConstantFPSDNode *CA = dyn_cast<ConstantFPSDNode>(Val&: A))
6817 if (const ConstantFPSDNode *CB = dyn_cast<ConstantFPSDNode>(Val&: B))
6818 if (CA->isZero() && CB->isZero()) return true;
6819
6820 // Otherwise they may not be equal.
6821 return false;
6822}
6823
6824// Only bits set in Mask must be negated, other bits may be arbitrary.
6825SDValue llvm::getBitwiseNotOperand(SDValue V, SDValue Mask, bool AllowUndefs) {
6826 if (isBitwiseNot(V, AllowUndefs))
6827 return V.getOperand(i: 0);
6828
6829 // Handle any_extend (not (truncate X)) pattern, where Mask only sets
6830 // bits in the non-extended part.
6831 ConstantSDNode *MaskC = isConstOrConstSplat(N: Mask);
6832 if (!MaskC || V.getOpcode() != ISD::ANY_EXTEND)
6833 return SDValue();
6834 SDValue ExtArg = V.getOperand(i: 0);
6835 if (ExtArg.getScalarValueSizeInBits() >=
6836 MaskC->getAPIntValue().getActiveBits() &&
6837 isBitwiseNot(V: ExtArg, AllowUndefs) &&
6838 ExtArg.getOperand(i: 0).getOpcode() == ISD::TRUNCATE &&
6839 ExtArg.getOperand(i: 0).getOperand(i: 0).getValueType() == V.getValueType())
6840 return ExtArg.getOperand(i: 0).getOperand(i: 0);
6841 return SDValue();
6842}
6843
6844static bool haveNoCommonBitsSetCommutative(SDValue A, SDValue B) {
6845 // Match masked merge pattern (X & ~M) op (Y & M)
6846 // Including degenerate case (X & ~M) op M
6847 auto MatchNoCommonBitsPattern = [&](SDValue Not, SDValue Mask,
6848 SDValue Other) {
6849 if (SDValue NotOperand =
6850 getBitwiseNotOperand(V: Not, Mask, /* AllowUndefs */ true)) {
6851 if (NotOperand->getOpcode() == ISD::ZERO_EXTEND ||
6852 NotOperand->getOpcode() == ISD::TRUNCATE)
6853 NotOperand = NotOperand->getOperand(Num: 0);
6854
6855 if (Other == NotOperand)
6856 return true;
6857 if (Other->getOpcode() == ISD::AND)
6858 return NotOperand == Other->getOperand(Num: 0) ||
6859 NotOperand == Other->getOperand(Num: 1);
6860 }
6861 return false;
6862 };
6863
6864 if (A->getOpcode() == ISD::ZERO_EXTEND || A->getOpcode() == ISD::TRUNCATE)
6865 A = A->getOperand(Num: 0);
6866
6867 if (B->getOpcode() == ISD::ZERO_EXTEND || B->getOpcode() == ISD::TRUNCATE)
6868 B = B->getOperand(Num: 0);
6869
6870 if (A->getOpcode() == ISD::AND)
6871 return MatchNoCommonBitsPattern(A->getOperand(Num: 0), A->getOperand(Num: 1), B) ||
6872 MatchNoCommonBitsPattern(A->getOperand(Num: 1), A->getOperand(Num: 0), B);
6873 return false;
6874}
6875
6876// FIXME: unify with llvm::haveNoCommonBitsSet.
6877bool SelectionDAG::haveNoCommonBitsSet(SDValue A, SDValue B) const {
6878 assert(A.getValueType() == B.getValueType() &&
6879 "Values must have the same type");
6880 if (haveNoCommonBitsSetCommutative(A, B) ||
6881 haveNoCommonBitsSetCommutative(A: B, B: A))
6882 return true;
6883 return KnownBits::haveNoCommonBitsSet(LHS: computeKnownBits(Op: A),
6884 RHS: computeKnownBits(Op: B));
6885}
6886
6887static SDValue FoldSTEP_VECTOR(const SDLoc &DL, EVT VT, SDValue Step,
6888 SelectionDAG &DAG) {
6889 if (cast<ConstantSDNode>(Val&: Step)->isZero())
6890 return DAG.getConstant(Val: 0, DL, VT);
6891
6892 return SDValue();
6893}
6894
6895static SDValue FoldBUILD_VECTOR(const SDLoc &DL, EVT VT,
6896 ArrayRef<SDValue> Ops,
6897 SelectionDAG &DAG) {
6898 int NumOps = Ops.size();
6899 assert(NumOps != 0 && "Can't build an empty vector!");
6900 assert(!VT.isScalableVector() &&
6901 "BUILD_VECTOR cannot be used with scalable types");
6902 assert(VT.getVectorNumElements() == (unsigned)NumOps &&
6903 "Incorrect element count in BUILD_VECTOR!");
6904
6905 // BUILD_VECTOR of UNDEFs is UNDEF.
6906 bool AllPoison = true;
6907 if (llvm::all_of(Range&: Ops, P: [&AllPoison](SDValue Op) {
6908 AllPoison &= Op.getOpcode() == ISD::POISON;
6909 return Op.isUndef();
6910 }))
6911 return AllPoison ? DAG.getPOISON(VT) : DAG.getUNDEF(VT);
6912
6913 // BUILD_VECTOR of seq extract/insert from the same vector + type is Identity.
6914 SDValue IdentitySrc;
6915 bool IsIdentity = true;
6916 for (int i = 0; i != NumOps; ++i) {
6917 if (Ops[i].getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
6918 Ops[i].getOperand(i: 0).getValueType() != VT ||
6919 (IdentitySrc && Ops[i].getOperand(i: 0) != IdentitySrc) ||
6920 !isa<ConstantSDNode>(Val: Ops[i].getOperand(i: 1)) ||
6921 Ops[i].getConstantOperandAPInt(i: 1) != i) {
6922 IsIdentity = false;
6923 break;
6924 }
6925 IdentitySrc = Ops[i].getOperand(i: 0);
6926 }
6927 if (IsIdentity)
6928 return IdentitySrc;
6929
6930 return SDValue();
6931}
6932
6933/// Try to simplify vector concatenation to an input value, undef, or build
6934/// vector.
6935static SDValue foldCONCAT_VECTORS(const SDLoc &DL, EVT VT,
6936 ArrayRef<SDValue> Ops,
6937 SelectionDAG &DAG) {
6938 assert(!Ops.empty() && "Can't concatenate an empty list of vectors!");
6939 assert(llvm::all_of(Ops,
6940 [Ops](SDValue Op) {
6941 return Ops[0].getValueType() == Op.getValueType();
6942 }) &&
6943 "Concatenation of vectors with inconsistent value types!");
6944 assert((Ops[0].getValueType().getVectorElementCount() * Ops.size()) ==
6945 VT.getVectorElementCount() &&
6946 "Incorrect element count in vector concatenation!");
6947
6948 if (Ops.size() == 1)
6949 return Ops[0];
6950
6951 // Concat of UNDEFs is UNDEF.
6952 bool AllPoison = true;
6953 if (llvm::all_of(Range&: Ops, P: [&AllPoison](SDValue Op) {
6954 AllPoison &= Op.getOpcode() == ISD::POISON;
6955 return Op.isUndef();
6956 }))
6957 return AllPoison ? DAG.getPOISON(VT) : DAG.getUNDEF(VT);
6958
6959 // Scan the operands and look for extract operations from a single source
6960 // that correspond to insertion at the same location via this concatenation:
6961 // concat (extract X, 0*subvec_elts), (extract X, 1*subvec_elts), ...
6962 SDValue IdentitySrc;
6963 bool IsIdentity = true;
6964 for (unsigned i = 0, e = Ops.size(); i != e; ++i) {
6965 SDValue Op = Ops[i];
6966 unsigned IdentityIndex = i * Op.getValueType().getVectorMinNumElements();
6967 if (Op.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
6968 Op.getOperand(i: 0).getValueType() != VT ||
6969 (IdentitySrc && Op.getOperand(i: 0) != IdentitySrc) ||
6970 Op.getConstantOperandVal(i: 1) != IdentityIndex) {
6971 IsIdentity = false;
6972 break;
6973 }
6974 assert((!IdentitySrc || IdentitySrc == Op.getOperand(0)) &&
6975 "Unexpected identity source vector for concat of extracts");
6976 IdentitySrc = Op.getOperand(i: 0);
6977 }
6978 if (IsIdentity) {
6979 assert(IdentitySrc && "Failed to set source vector of extracts");
6980 return IdentitySrc;
6981 }
6982
6983 // The code below this point is only designed to work for fixed width
6984 // vectors, so we bail out for now.
6985 if (VT.isScalableVector())
6986 return SDValue();
6987
6988 // A CONCAT_VECTOR of scalar sources, such as UNDEF, BUILD_VECTOR and
6989 // single-element INSERT_VECTOR_ELT operands can be simplified to one big
6990 // BUILD_VECTOR.
6991 // FIXME: Add support for SCALAR_TO_VECTOR as well.
6992 EVT SVT = VT.getScalarType();
6993 SmallVector<SDValue, 16> Elts;
6994 for (SDValue Op : Ops) {
6995 EVT OpVT = Op.getValueType();
6996 if (Op.getOpcode() == ISD::POISON)
6997 Elts.append(NumInputs: OpVT.getVectorNumElements(), Elt: DAG.getPOISON(VT: SVT));
6998 else if (Op.getOpcode() == ISD::UNDEF)
6999 Elts.append(NumInputs: OpVT.getVectorNumElements(), Elt: DAG.getUNDEF(VT: SVT));
7000 else if (Op.getOpcode() == ISD::BUILD_VECTOR)
7001 Elts.append(in_start: Op->op_begin(), in_end: Op->op_end());
7002 else if (Op.getOpcode() == ISD::INSERT_VECTOR_ELT &&
7003 OpVT.getVectorNumElements() == 1 &&
7004 isNullConstant(V: Op.getOperand(i: 2)))
7005 Elts.push_back(Elt: Op.getOperand(i: 1));
7006 else
7007 return SDValue();
7008 }
7009
7010 // BUILD_VECTOR requires all inputs to be of the same type, find the
7011 // maximum type and extend them all.
7012 for (SDValue Op : Elts)
7013 SVT = (SVT.bitsLT(VT: Op.getValueType()) ? Op.getValueType() : SVT);
7014
7015 if (SVT.bitsGT(VT: VT.getScalarType())) {
7016 for (SDValue &Op : Elts) {
7017 if (Op.getOpcode() == ISD::POISON)
7018 Op = DAG.getPOISON(VT: SVT);
7019 else if (Op.getOpcode() == ISD::UNDEF)
7020 Op = DAG.getUNDEF(VT: SVT);
7021 else
7022 Op = DAG.getTargetLoweringInfo().isZExtFree(FromTy: Op.getValueType(), ToTy: SVT)
7023 ? DAG.getZExtOrTrunc(Op, DL, VT: SVT)
7024 : DAG.getSExtOrTrunc(Op, DL, VT: SVT);
7025 }
7026 }
7027
7028 SDValue V = DAG.getBuildVector(VT, DL, Ops: Elts);
7029 NewSDValueDbgMsg(V, Msg: "New node fold concat vectors: ", G: &DAG);
7030 return V;
7031}
7032
7033/// Gets or creates the specified node.
7034SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT) {
7035 SDVTList VTs = getVTList(VT);
7036 FoldingSetNodeID ID;
7037 AddNodeIDNode(ID, OpC: Opcode, VTList: VTs, OpList: {});
7038 FoldingSetInsertToken InsertToken;
7039 if (SDNode *E = lookupNode(ID, DL, InsertToken))
7040 return SDValue(E, 0);
7041
7042 auto *N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
7043 CSEMap.insert(N, Token: InsertToken);
7044
7045 InsertNode(N);
7046 SDValue V = SDValue(N, 0);
7047 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
7048 return V;
7049}
7050
7051SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7052 SDValue N1) {
7053 SDNodeFlags Flags;
7054 if (Inserter)
7055 Flags = Inserter->getFlags();
7056 return getNode(Opcode, DL, VT, Operand: N1, Flags);
7057}
7058
7059SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7060 SDValue N1, const SDNodeFlags Flags) {
7061 assert(N1.getOpcode() != ISD::DELETED_NODE && "Operand is DELETED_NODE!");
7062
7063 // Constant fold unary operations with a vector integer or float operand.
7064 switch (Opcode) {
7065 default:
7066 // FIXME: Entirely reasonable to perform folding of other unary
7067 // operations here as the need arises.
7068 break;
7069 case ISD::FNEG:
7070 case ISD::FABS:
7071 case ISD::FCEIL:
7072 case ISD::FTRUNC:
7073 case ISD::FFLOOR:
7074 case ISD::FP_EXTEND:
7075 case ISD::FP_TO_SINT:
7076 case ISD::FP_TO_UINT:
7077 case ISD::FP_TO_FP16:
7078 case ISD::FP_TO_BF16:
7079 case ISD::TRUNCATE:
7080 case ISD::ANY_EXTEND:
7081 case ISD::ZERO_EXTEND:
7082 case ISD::SIGN_EXTEND:
7083 case ISD::UINT_TO_FP:
7084 case ISD::SINT_TO_FP:
7085 case ISD::FP16_TO_FP:
7086 case ISD::BF16_TO_FP:
7087 case ISD::BITCAST:
7088 case ISD::ABS:
7089 case ISD::ABS_MIN_POISON:
7090 case ISD::BITREVERSE:
7091 case ISD::BSWAP:
7092 case ISD::CTLZ:
7093 case ISD::CTLZ_ZERO_POISON:
7094 case ISD::CTTZ:
7095 case ISD::CTTZ_ZERO_POISON:
7096 case ISD::CTPOP:
7097 case ISD::CTLS:
7098 case ISD::VECREDUCE_ADD:
7099 case ISD::VECREDUCE_SMAX:
7100 case ISD::VECREDUCE_SMIN:
7101 case ISD::VECREDUCE_UMAX:
7102 case ISD::VECREDUCE_UMIN:
7103 case ISD::VECREDUCE_MUL:
7104 case ISD::VECREDUCE_AND:
7105 case ISD::VECREDUCE_OR:
7106 case ISD::VECREDUCE_XOR:
7107 case ISD::STEP_VECTOR: {
7108 SDValue Ops = {N1};
7109 if (SDValue Fold = FoldConstantArithmetic(Opcode, DL, VT, Ops))
7110 return Fold;
7111 }
7112 }
7113
7114 unsigned OpOpcode = N1.getNode()->getOpcode();
7115 switch (Opcode) {
7116 case ISD::STEP_VECTOR:
7117 assert(VT.isScalableVector() &&
7118 "STEP_VECTOR can only be used with scalable types");
7119 assert(OpOpcode == ISD::TargetConstant &&
7120 VT.getVectorElementType() == N1.getValueType() &&
7121 "Unexpected step operand");
7122 break;
7123 case ISD::FREEZE:
7124 assert(VT == N1.getValueType() && "Unexpected VT!");
7125 if (isGuaranteedNotToBeUndefOrPoison(Op: N1, Kind: UndefPoisonKind::UndefOrPoison))
7126 return N1;
7127 break;
7128 case ISD::TokenFactor:
7129 case ISD::MERGE_VALUES:
7130 case ISD::CONCAT_VECTORS:
7131 return N1; // Factor, merge or concat of one node? No need.
7132 case ISD::BUILD_VECTOR: {
7133 // Attempt to simplify BUILD_VECTOR.
7134 SDValue Ops[] = {N1};
7135 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, DAG&: *this))
7136 return V;
7137 break;
7138 }
7139 case ISD::FP_ROUND: llvm_unreachable("Invalid method to make FP_ROUND node");
7140 case ISD::FP_EXTEND:
7141 assert(VT.isFloatingPoint() && N1.getValueType().isFloatingPoint() &&
7142 "Invalid FP cast!");
7143 if (N1.getValueType() == VT) return N1; // noop conversion.
7144 assert((!VT.isVector() || VT.getVectorElementCount() ==
7145 N1.getValueType().getVectorElementCount()) &&
7146 "Vector element count mismatch!");
7147 assert(N1.getValueType().bitsLT(VT) && "Invalid fpext node, dst < src!");
7148 if (N1.isUndef())
7149 return getUNDEF(VT);
7150 break;
7151 case ISD::FP_TO_SINT:
7152 case ISD::FP_TO_UINT:
7153 if (N1.isUndef())
7154 return getUNDEF(VT);
7155 break;
7156 case ISD::SINT_TO_FP:
7157 case ISD::UINT_TO_FP:
7158 // [us]itofp(undef) = 0, because the result value is bounded.
7159 if (N1.isUndef())
7160 return getConstantFP(Val: 0.0, DL, VT);
7161 break;
7162 case ISD::SIGN_EXTEND:
7163 assert(VT.isInteger() && N1.getValueType().isInteger() &&
7164 "Invalid SIGN_EXTEND!");
7165 assert(VT.isVector() == N1.getValueType().isVector() &&
7166 "SIGN_EXTEND result type type should be vector iff the operand "
7167 "type is vector!");
7168 if (N1.getValueType() == VT) return N1; // noop extension
7169 assert((!VT.isVector() || VT.getVectorElementCount() ==
7170 N1.getValueType().getVectorElementCount()) &&
7171 "Vector element count mismatch!");
7172 assert(N1.getValueType().bitsLT(VT) && "Invalid sext node, dst < src!");
7173 if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND) {
7174 SDNodeFlags Flags;
7175 if (OpOpcode == ISD::ZERO_EXTEND)
7176 Flags.setNonNeg(N1->getFlags().hasNonNeg());
7177 SDValue NewVal = getNode(Opcode: OpOpcode, DL, VT, N1: N1.getOperand(i: 0), Flags);
7178 transferDbgValues(From: N1, To: NewVal);
7179 return NewVal;
7180 }
7181
7182 if (OpOpcode == ISD::POISON)
7183 return getPOISON(VT);
7184
7185 if (N1.isUndef())
7186 // sext(undef) = 0, because the top bits will all be the same.
7187 return getConstant(Val: 0, DL, VT);
7188
7189 // Skip unnecessary sext_inreg pattern:
7190 // (sext (trunc x)) -> x iff the upper bits are all signbits.
7191 if (OpOpcode == ISD::TRUNCATE) {
7192 SDValue OpOp = N1.getOperand(i: 0);
7193 if (OpOp.getValueType() == VT) {
7194 unsigned NumSignExtBits =
7195 VT.getScalarSizeInBits() - N1.getScalarValueSizeInBits();
7196 if (ComputeNumSignBits(Op: OpOp) > NumSignExtBits) {
7197 transferDbgValues(From: N1, To: OpOp);
7198 return OpOp;
7199 }
7200 }
7201 }
7202 break;
7203 case ISD::ZERO_EXTEND:
7204 assert(VT.isInteger() && N1.getValueType().isInteger() &&
7205 "Invalid ZERO_EXTEND!");
7206 assert(VT.isVector() == N1.getValueType().isVector() &&
7207 "ZERO_EXTEND result type type should be vector iff the operand "
7208 "type is vector!");
7209 if (N1.getValueType() == VT) return N1; // noop extension
7210 assert((!VT.isVector() || VT.getVectorElementCount() ==
7211 N1.getValueType().getVectorElementCount()) &&
7212 "Vector element count mismatch!");
7213 assert(N1.getValueType().bitsLT(VT) && "Invalid zext node, dst < src!");
7214 if (OpOpcode == ISD::ZERO_EXTEND) { // (zext (zext x)) -> (zext x)
7215 SDNodeFlags Flags;
7216 Flags.setNonNeg(N1->getFlags().hasNonNeg());
7217 SDValue NewVal =
7218 getNode(Opcode: ISD::ZERO_EXTEND, DL, VT, N1: N1.getOperand(i: 0), Flags);
7219 transferDbgValues(From: N1, To: NewVal);
7220 return NewVal;
7221 }
7222
7223 if (OpOpcode == ISD::POISON)
7224 return getPOISON(VT);
7225
7226 if (N1.isUndef())
7227 // zext(undef) = 0, because the top bits will be zero.
7228 return getConstant(Val: 0, DL, VT);
7229
7230 // Skip unnecessary zext_inreg pattern:
7231 // (zext (trunc x)) -> x iff the upper bits are known zero.
7232 // TODO: Remove (zext (trunc (and x, c))) exception which some targets
7233 // use to recognise zext_inreg patterns.
7234 if (OpOpcode == ISD::TRUNCATE) {
7235 SDValue OpOp = N1.getOperand(i: 0);
7236 if (OpOp.getValueType() == VT) {
7237 if (OpOp.getOpcode() != ISD::AND) {
7238 APInt HiBits = APInt::getBitsSetFrom(numBits: VT.getScalarSizeInBits(),
7239 loBit: N1.getScalarValueSizeInBits());
7240 if (MaskedValueIsZero(V: OpOp, Mask: HiBits)) {
7241 transferDbgValues(From: N1, To: OpOp);
7242 return OpOp;
7243 }
7244 }
7245 }
7246 }
7247 break;
7248 case ISD::ANY_EXTEND:
7249 assert(VT.isInteger() && N1.getValueType().isInteger() &&
7250 "Invalid ANY_EXTEND!");
7251 assert(VT.isVector() == N1.getValueType().isVector() &&
7252 "ANY_EXTEND result type type should be vector iff the operand "
7253 "type is vector!");
7254 if (N1.getValueType() == VT) return N1; // noop extension
7255 assert((!VT.isVector() || VT.getVectorElementCount() ==
7256 N1.getValueType().getVectorElementCount()) &&
7257 "Vector element count mismatch!");
7258 assert(N1.getValueType().bitsLT(VT) && "Invalid anyext node, dst < src!");
7259
7260 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
7261 OpOpcode == ISD::ANY_EXTEND) {
7262 SDNodeFlags Flags;
7263 if (OpOpcode == ISD::ZERO_EXTEND)
7264 Flags.setNonNeg(N1->getFlags().hasNonNeg());
7265 // (ext (zext x)) -> (zext x) and (ext (sext x)) -> (sext x)
7266 return getNode(Opcode: OpOpcode, DL, VT, N1: N1.getOperand(i: 0), Flags);
7267 }
7268 if (N1.isUndef())
7269 return getUNDEF(VT);
7270
7271 // (ext (trunc x)) -> x
7272 if (OpOpcode == ISD::TRUNCATE) {
7273 SDValue OpOp = N1.getOperand(i: 0);
7274 if (OpOp.getValueType() == VT) {
7275 transferDbgValues(From: N1, To: OpOp);
7276 return OpOp;
7277 }
7278 }
7279 break;
7280 case ISD::TRUNCATE:
7281 assert(VT.isInteger() && N1.getValueType().isInteger() &&
7282 "Invalid TRUNCATE!");
7283 assert(VT.isVector() == N1.getValueType().isVector() &&
7284 "TRUNCATE result type type should be vector iff the operand "
7285 "type is vector!");
7286 if (N1.getValueType() == VT) return N1; // noop truncate
7287 assert((!VT.isVector() || VT.getVectorElementCount() ==
7288 N1.getValueType().getVectorElementCount()) &&
7289 "Vector element count mismatch!");
7290 assert(N1.getValueType().bitsGT(VT) && "Invalid truncate node, src < dst!");
7291 if (OpOpcode == ISD::TRUNCATE)
7292 return getNode(Opcode: ISD::TRUNCATE, DL, VT, N1: N1.getOperand(i: 0));
7293 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
7294 OpOpcode == ISD::ANY_EXTEND) {
7295 // If the source is smaller than the dest, we still need an extend.
7296 if (N1.getOperand(i: 0).getValueType().getScalarType().bitsLT(
7297 VT: VT.getScalarType())) {
7298 SDNodeFlags Flags;
7299 if (OpOpcode == ISD::ZERO_EXTEND)
7300 Flags.setNonNeg(N1->getFlags().hasNonNeg());
7301 return getNode(Opcode: OpOpcode, DL, VT, N1: N1.getOperand(i: 0), Flags);
7302 }
7303 if (N1.getOperand(i: 0).getValueType().bitsGT(VT))
7304 return getNode(Opcode: ISD::TRUNCATE, DL, VT, N1: N1.getOperand(i: 0));
7305 return N1.getOperand(i: 0);
7306 }
7307 if (N1.isUndef())
7308 return getUNDEF(VT);
7309 if (OpOpcode == ISD::VSCALE && !NewNodesMustHaveLegalTypes)
7310 return getVScale(DL, VT,
7311 MulImm: N1.getConstantOperandAPInt(i: 0).trunc(width: VT.getSizeInBits()));
7312 break;
7313 case ISD::ANY_EXTEND_VECTOR_INREG:
7314 case ISD::ZERO_EXTEND_VECTOR_INREG:
7315 case ISD::SIGN_EXTEND_VECTOR_INREG:
7316 assert(VT.isVector() && "This DAG node is restricted to vector types.");
7317 assert(N1.getValueType().bitsLE(VT) &&
7318 "The input must be the same size or smaller than the result.");
7319 assert(VT.getVectorMinNumElements() <
7320 N1.getValueType().getVectorMinNumElements() &&
7321 "The destination vector type must have fewer lanes than the input.");
7322 break;
7323 case ISD::ABS:
7324 assert(VT.isInteger() && VT == N1.getValueType() && "Invalid ABS!");
7325 if (N1.isUndef())
7326 return getConstant(Val: 0, DL, VT);
7327 break;
7328 case ISD::ABS_MIN_POISON:
7329 assert(VT.isInteger() && VT == N1.getValueType() &&
7330 "Invalid ABS_MIN_POISON!");
7331 if (N1.isUndef())
7332 return getConstant(Val: 0, DL, VT);
7333 break;
7334 case ISD::BSWAP:
7335 assert(VT.isInteger() && VT == N1.getValueType() && "Invalid BSWAP!");
7336 assert((VT.getScalarSizeInBits() % 16 == 0) &&
7337 "BSWAP types must be a multiple of 16 bits!");
7338 if (N1.isUndef())
7339 return getUNDEF(VT);
7340 // bswap(bswap(X)) -> X.
7341 if (OpOpcode == ISD::BSWAP)
7342 return N1.getOperand(i: 0);
7343 break;
7344 case ISD::BITREVERSE:
7345 assert(VT.isInteger() && VT == N1.getValueType() && "Invalid BITREVERSE!");
7346 if (N1.isUndef())
7347 return getUNDEF(VT);
7348 break;
7349 case ISD::BITCAST:
7350 assert(VT.getSizeInBits() == N1.getValueSizeInBits() &&
7351 "Cannot BITCAST between types of different sizes!");
7352 if (VT == N1.getValueType()) return N1; // noop conversion.
7353 if (OpOpcode == ISD::BITCAST) // bitconv(bitconv(x)) -> bitconv(x)
7354 return getNode(Opcode: ISD::BITCAST, DL, VT, N1: N1.getOperand(i: 0));
7355 if (N1.isUndef())
7356 return getUNDEF(VT);
7357 break;
7358 case ISD::SCALAR_TO_VECTOR:
7359 assert(VT.isVector() && !N1.getValueType().isVector() &&
7360 (VT.getVectorElementType() == N1.getValueType() ||
7361 (VT.getVectorElementType().isInteger() &&
7362 N1.getValueType().isInteger() &&
7363 VT.getVectorElementType().bitsLE(N1.getValueType()))) &&
7364 "Illegal SCALAR_TO_VECTOR node!");
7365 if (N1.isUndef())
7366 return getUNDEF(VT);
7367 // scalar_to_vector(extract_vector_elt V, 0) -> V, top bits are undefined.
7368 if (OpOpcode == ISD::EXTRACT_VECTOR_ELT &&
7369 isa<ConstantSDNode>(Val: N1.getOperand(i: 1)) &&
7370 N1.getConstantOperandVal(i: 1) == 0 &&
7371 N1.getOperand(i: 0).getValueType() == VT)
7372 return N1.getOperand(i: 0);
7373 break;
7374 case ISD::FNEG:
7375 // Negation of an unknown bag of bits is still completely undefined.
7376 if (N1.isUndef())
7377 return getUNDEF(VT);
7378
7379 if (OpOpcode == ISD::FNEG) // --X -> X
7380 return N1.getOperand(i: 0);
7381 break;
7382 case ISD::FABS:
7383 if (OpOpcode == ISD::FNEG) // abs(-X) -> abs(X)
7384 return getNode(Opcode: ISD::FABS, DL, VT, N1: N1.getOperand(i: 0));
7385 break;
7386 case ISD::VSCALE:
7387 assert(VT == N1.getValueType() && "Unexpected VT!");
7388 break;
7389 case ISD::CTPOP:
7390 if (N1.getValueType().getScalarType() == MVT::i1)
7391 return N1;
7392 break;
7393 case ISD::CTLZ:
7394 case ISD::CTTZ:
7395 if (N1.getValueType().getScalarType() == MVT::i1)
7396 return getNOT(DL, Val: N1, VT: N1.getValueType());
7397 break;
7398 case ISD::CTLS:
7399 if (N1.getValueType().getScalarType() == MVT::i1)
7400 return getConstant(Val: 0, DL, VT);
7401 break;
7402 case ISD::VECREDUCE_ADD:
7403 if (N1.getValueType().getScalarType() == MVT::i1)
7404 return getNode(Opcode: ISD::VECREDUCE_XOR, DL, VT, N1);
7405 break;
7406 case ISD::VECREDUCE_SMIN:
7407 case ISD::VECREDUCE_UMAX:
7408 if (N1.getValueType().getScalarType() == MVT::i1)
7409 return getNode(Opcode: ISD::VECREDUCE_OR, DL, VT, N1);
7410 break;
7411 case ISD::VECREDUCE_SMAX:
7412 case ISD::VECREDUCE_UMIN:
7413 if (N1.getValueType().getScalarType() == MVT::i1)
7414 return getNode(Opcode: ISD::VECREDUCE_AND, DL, VT, N1);
7415 break;
7416 case ISD::SPLAT_VECTOR:
7417 assert(VT.isVector() && "Wrong return type!");
7418 // FIXME: Hexagon uses i32 scalar for a floating point zero vector so allow
7419 // that for now.
7420 assert((VT.getVectorElementType() == N1.getValueType() ||
7421 (VT.isFloatingPoint() && N1.getValueType() == MVT::i32) ||
7422 (VT.getVectorElementType().isInteger() &&
7423 N1.getValueType().isInteger() &&
7424 VT.getVectorElementType().bitsLE(N1.getValueType()))) &&
7425 "Wrong operand type!");
7426 break;
7427 }
7428
7429 SDNode *N;
7430 SDVTList VTs = getVTList(VT);
7431 SDValue Ops[] = {N1};
7432 if (VT != MVT::Glue) { // Don't CSE glue producing nodes
7433 FoldingSetNodeID ID;
7434 AddNodeIDNode(ID, OpC: Opcode, VTList: VTs, OpList: Ops);
7435 FoldingSetInsertToken InsertToken;
7436 if (SDNode *E = lookupNode(ID, DL, InsertToken)) {
7437 E->intersectFlagsWith(Flags);
7438 return SDValue(E, 0);
7439 }
7440
7441 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
7442 N->setFlags(Flags);
7443 createOperands(Node: N, Vals: Ops);
7444 CSEMap.insert(N, Token: InsertToken);
7445 } else {
7446 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
7447 createOperands(Node: N, Vals: Ops);
7448 }
7449
7450 InsertNode(N);
7451 SDValue V = SDValue(N, 0);
7452 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
7453 return V;
7454}
7455
7456static APInt getIntegerIdentity(unsigned Opcode, unsigned BitWidth) {
7457 switch (Opcode) {
7458 default:
7459 llvm_unreachable("Unexpected integer identity opcode");
7460 case ISD::ADD:
7461 case ISD::OR:
7462 case ISD::XOR:
7463 case ISD::UMAX:
7464 return APInt::getZero(numBits: BitWidth);
7465 case ISD::MUL:
7466 return APInt(BitWidth, 1);
7467 case ISD::AND:
7468 case ISD::UMIN:
7469 return APInt::getAllOnes(numBits: BitWidth);
7470 case ISD::SMAX:
7471 return APInt::getSignedMinValue(numBits: BitWidth);
7472 case ISD::SMIN:
7473 return APInt::getSignedMaxValue(numBits: BitWidth);
7474 }
7475}
7476
7477static std::optional<APInt> FoldValue(unsigned Opcode, const APInt &C1,
7478 const APInt &C2) {
7479 switch (Opcode) {
7480 case ISD::ADD: return C1 + C2;
7481 case ISD::SUB: return C1 - C2;
7482 case ISD::MUL: return C1 * C2;
7483 case ISD::AND: return C1 & C2;
7484 case ISD::OR: return C1 | C2;
7485 case ISD::XOR: return C1 ^ C2;
7486 case ISD::SHL: return C1 << C2;
7487 case ISD::SRL: return C1.lshr(ShiftAmt: C2);
7488 case ISD::SRA: return C1.ashr(ShiftAmt: C2);
7489 case ISD::ROTL: return C1.rotl(rotateAmt: C2);
7490 case ISD::ROTR: return C1.rotr(rotateAmt: C2);
7491 case ISD::SMIN: return C1.sle(RHS: C2) ? C1 : C2;
7492 case ISD::SMAX: return C1.sge(RHS: C2) ? C1 : C2;
7493 case ISD::UMIN: return C1.ule(RHS: C2) ? C1 : C2;
7494 case ISD::UMAX: return C1.uge(RHS: C2) ? C1 : C2;
7495 case ISD::SADDSAT: return C1.sadd_sat(RHS: C2);
7496 case ISD::UADDSAT: return C1.uadd_sat(RHS: C2);
7497 case ISD::SSUBSAT: return C1.ssub_sat(RHS: C2);
7498 case ISD::USUBSAT: return C1.usub_sat(RHS: C2);
7499 case ISD::SSHLSAT: return C1.sshl_sat(RHS: C2);
7500 case ISD::USHLSAT: return C1.ushl_sat(RHS: C2);
7501 case ISD::UDIV:
7502 if (!C2.getBoolValue())
7503 break;
7504 return C1.udiv(RHS: C2);
7505 case ISD::UREM:
7506 if (!C2.getBoolValue())
7507 break;
7508 return C1.urem(RHS: C2);
7509 case ISD::SDIV:
7510 if (!C2.getBoolValue())
7511 break;
7512 return C1.sdiv(RHS: C2);
7513 case ISD::SREM:
7514 if (!C2.getBoolValue())
7515 break;
7516 return C1.srem(RHS: C2);
7517 case ISD::AVGFLOORS:
7518 return APIntOps::avgFloorS(C1, C2);
7519 case ISD::AVGFLOORU:
7520 return APIntOps::avgFloorU(C1, C2);
7521 case ISD::AVGCEILS:
7522 return APIntOps::avgCeilS(C1, C2);
7523 case ISD::AVGCEILU:
7524 return APIntOps::avgCeilU(C1, C2);
7525 case ISD::ABDS:
7526 return APIntOps::abds(A: C1, B: C2);
7527 case ISD::ABDU:
7528 return APIntOps::abdu(A: C1, B: C2);
7529 case ISD::MULHS:
7530 return APIntOps::mulhs(C1, C2);
7531 case ISD::MULHU:
7532 return APIntOps::mulhu(C1, C2);
7533 case ISD::CLMUL:
7534 return APIntOps::clmul(LHS: C1, RHS: C2);
7535 case ISD::CLMULR:
7536 return APIntOps::clmulr(LHS: C1, RHS: C2);
7537 case ISD::CLMULH:
7538 return APIntOps::clmulh(LHS: C1, RHS: C2);
7539 case ISD::PEXT:
7540 return APIntOps::pext(Val: C1, Mask: C2);
7541 case ISD::PDEP:
7542 return APIntOps::pdep(Val: C1, Mask: C2);
7543 }
7544 return std::nullopt;
7545}
7546// Handle constant folding with UNDEF.
7547// TODO: Handle more cases.
7548static std::optional<APInt> FoldValueWithUndef(unsigned Opcode, const APInt &C1,
7549 bool IsUndef1, const APInt &C2,
7550 bool IsUndef2) {
7551 if (!(IsUndef1 || IsUndef2))
7552 return FoldValue(Opcode, C1, C2);
7553
7554 // Fold and(x, undef) -> 0
7555 // Fold mul(x, undef) -> 0
7556 if (Opcode == ISD::AND || Opcode == ISD::MUL)
7557 return APInt::getZero(numBits: C1.getBitWidth());
7558
7559 return std::nullopt;
7560}
7561
7562SDValue SelectionDAG::FoldSymbolOffset(unsigned Opcode, EVT VT,
7563 const GlobalAddressSDNode *GA,
7564 const SDNode *N2) {
7565 if (GA->getOpcode() != ISD::GlobalAddress)
7566 return SDValue();
7567 if (!TLI->isOffsetFoldingLegal(GA))
7568 return SDValue();
7569 auto *C2 = dyn_cast<ConstantSDNode>(Val: N2);
7570 if (!C2)
7571 return SDValue();
7572 int64_t Offset = C2->getSExtValue();
7573 switch (Opcode) {
7574 case ISD::ADD:
7575 case ISD::PTRADD:
7576 break;
7577 case ISD::SUB: Offset = -uint64_t(Offset); break;
7578 default: return SDValue();
7579 }
7580 return getGlobalAddress(GV: GA->getGlobal(), DL: SDLoc(C2), VT,
7581 Offset: GA->getOffset() + uint64_t(Offset));
7582}
7583
7584bool SelectionDAG::isUndef(unsigned Opcode, ArrayRef<SDValue> Ops) {
7585 switch (Opcode) {
7586 case ISD::SDIV:
7587 case ISD::UDIV:
7588 case ISD::SREM:
7589 case ISD::UREM: {
7590 // If a divisor is zero/undef or any element of a divisor vector is
7591 // zero/undef, the whole op is undef.
7592 assert(Ops.size() == 2 && "Div/rem should have 2 operands");
7593 SDValue Divisor = Ops[1];
7594 if (Divisor.isUndef() || isNullConstant(V: Divisor))
7595 return true;
7596
7597 return ISD::isBuildVectorOfConstantSDNodes(N: Divisor.getNode()) &&
7598 llvm::any_of(Range: Divisor->op_values(),
7599 P: [](SDValue V) { return V.isUndef() ||
7600 isNullConstant(V); });
7601 // TODO: Handle signed overflow.
7602 }
7603 // TODO: Handle oversized shifts.
7604 default:
7605 return false;
7606 }
7607}
7608
7609SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL,
7610 EVT VT, ArrayRef<SDValue> Ops,
7611 SDNodeFlags Flags) {
7612 // If the opcode is a target-specific ISD node, there's nothing we can
7613 // do here and the operand rules may not line up with the below, so
7614 // bail early.
7615 // We can't create a scalar CONCAT_VECTORS so skip it. It will break
7616 // for concats involving SPLAT_VECTOR. Concats of BUILD_VECTORS are handled by
7617 // foldCONCAT_VECTORS in getNode before this is called.
7618 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::CONCAT_VECTORS)
7619 return SDValue();
7620
7621 unsigned NumOps = Ops.size();
7622 if (NumOps == 0)
7623 return SDValue();
7624
7625 if (isUndef(Opcode, Ops))
7626 return getUNDEF(VT);
7627
7628 // Handle unary special cases.
7629 if (NumOps == 1) {
7630 SDValue N1 = Ops[0];
7631
7632 // Constant fold unary operations with an integer constant operand. Even
7633 // opaque constant will be folded, because the folding of unary operations
7634 // doesn't create new constants with different values. Nevertheless, the
7635 // opaque flag is preserved during folding to prevent future folding with
7636 // other constants.
7637 if (auto *C = dyn_cast<ConstantSDNode>(Val&: N1)) {
7638 const APInt &Val = C->getAPIntValue();
7639 switch (Opcode) {
7640 case ISD::SIGN_EXTEND:
7641 return getConstant(Val: Val.sextOrTrunc(width: VT.getSizeInBits()), DL, VT,
7642 isT: C->isTargetOpcode(), isO: C->isOpaque());
7643 case ISD::TRUNCATE:
7644 if (C->isOpaque())
7645 break;
7646 [[fallthrough]];
7647 case ISD::ZERO_EXTEND:
7648 return getConstant(Val: Val.zextOrTrunc(width: VT.getSizeInBits()), DL, VT,
7649 isT: C->isTargetOpcode(), isO: C->isOpaque());
7650 case ISD::ANY_EXTEND:
7651 // Some targets like RISCV prefer to sign extend some types.
7652 if (TLI->isSExtCheaperThanZExt(FromTy: N1.getValueType(), ToTy: VT))
7653 return getConstant(Val: Val.sextOrTrunc(width: VT.getSizeInBits()), DL, VT,
7654 isT: C->isTargetOpcode(), isO: C->isOpaque());
7655 return getConstant(Val: Val.zextOrTrunc(width: VT.getSizeInBits()), DL, VT,
7656 isT: C->isTargetOpcode(), isO: C->isOpaque());
7657 case ISD::ABS:
7658 return getConstant(Val: Val.abs(), DL, VT, isT: C->isTargetOpcode(),
7659 isO: C->isOpaque());
7660 case ISD::ABS_MIN_POISON:
7661 if (Val.isMinSignedValue())
7662 return getPOISON(VT);
7663 return getConstant(Val: Val.abs(), DL, VT, isT: C->isTargetOpcode(),
7664 isO: C->isOpaque());
7665 case ISD::BITREVERSE:
7666 return getConstant(Val: Val.reverseBits(), DL, VT, isT: C->isTargetOpcode(),
7667 isO: C->isOpaque());
7668 case ISD::BSWAP:
7669 return getConstant(Val: Val.byteSwap(), DL, VT, isT: C->isTargetOpcode(),
7670 isO: C->isOpaque());
7671 case ISD::CTPOP:
7672 return getConstant(Val: Val.popcount(), DL, VT, isT: C->isTargetOpcode(),
7673 isO: C->isOpaque());
7674 case ISD::CTLZ:
7675 case ISD::CTLZ_ZERO_POISON:
7676 return getConstant(Val: Val.countl_zero(), DL, VT, isT: C->isTargetOpcode(),
7677 isO: C->isOpaque());
7678 case ISD::CTTZ:
7679 case ISD::CTTZ_ZERO_POISON:
7680 return getConstant(Val: Val.countr_zero(), DL, VT, isT: C->isTargetOpcode(),
7681 isO: C->isOpaque());
7682 case ISD::CTLS:
7683 // CTLS returns the number of extra sign bits so subtract one.
7684 return getConstant(Val: Val.getNumSignBits() - 1, DL, VT,
7685 isT: C->isTargetOpcode(), isO: C->isOpaque());
7686 case ISD::UINT_TO_FP:
7687 case ISD::SINT_TO_FP: {
7688 APFloat FPV(VT.getFltSemantics(), APInt::getZero(numBits: VT.getSizeInBits()));
7689 (void)FPV.convertFromAPInt(Input: Val, IsSigned: Opcode == ISD::SINT_TO_FP,
7690 RM: APFloat::rmNearestTiesToEven);
7691 return getConstantFP(V: FPV, DL, VT);
7692 }
7693 case ISD::FP16_TO_FP:
7694 case ISD::BF16_TO_FP: {
7695 bool Ignored;
7696 APFloat FPV(Opcode == ISD::FP16_TO_FP ? APFloat::IEEEhalf()
7697 : APFloat::BFloat(),
7698 (Val.getBitWidth() == 16) ? Val : Val.trunc(width: 16));
7699
7700 // This can return overflow, underflow, or inexact; we don't care.
7701 // FIXME need to be more flexible about rounding mode.
7702 (void)FPV.convert(ToSemantics: VT.getFltSemantics(), RM: APFloat::rmNearestTiesToEven,
7703 losesInfo: &Ignored);
7704 return getConstantFP(V: FPV, DL, VT);
7705 }
7706 case ISD::STEP_VECTOR:
7707 if (SDValue V = FoldSTEP_VECTOR(DL, VT, Step: N1, DAG&: *this))
7708 return V;
7709 break;
7710 case ISD::BITCAST:
7711 if (VT == MVT::f16 && C->getValueType(ResNo: 0) == MVT::i16)
7712 return getConstantFP(V: APFloat(APFloat::IEEEhalf(), Val), DL, VT);
7713 if (VT == MVT::f32 && C->getValueType(ResNo: 0) == MVT::i32)
7714 return getConstantFP(V: APFloat(APFloat::IEEEsingle(), Val), DL, VT);
7715 if (VT == MVT::f64 && C->getValueType(ResNo: 0) == MVT::i64)
7716 return getConstantFP(V: APFloat(APFloat::IEEEdouble(), Val), DL, VT);
7717 if (VT == MVT::f128 && C->getValueType(ResNo: 0) == MVT::i128)
7718 return getConstantFP(V: APFloat(APFloat::IEEEquad(), Val), DL, VT);
7719 break;
7720 }
7721 }
7722
7723 // Constant fold unary operations with a floating point constant operand.
7724 if (auto *C = dyn_cast<ConstantFPSDNode>(Val&: N1)) {
7725 APFloat V = C->getValueAPF(); // make copy
7726 switch (Opcode) {
7727 case ISD::FNEG:
7728 V.changeSign();
7729 return getConstantFP(V, DL, VT);
7730 case ISD::FABS:
7731 V.clearSign();
7732 return getConstantFP(V, DL, VT);
7733 case ISD::FCEIL: {
7734 APFloat::opStatus fs = V.roundToIntegral(RM: APFloat::rmTowardPositive);
7735 if (fs == APFloat::opOK || fs == APFloat::opInexact)
7736 return getConstantFP(V, DL, VT);
7737 return SDValue();
7738 }
7739 case ISD::FTRUNC: {
7740 APFloat::opStatus fs = V.roundToIntegral(RM: APFloat::rmTowardZero);
7741 if (fs == APFloat::opOK || fs == APFloat::opInexact)
7742 return getConstantFP(V, DL, VT);
7743 return SDValue();
7744 }
7745 case ISD::FFLOOR: {
7746 APFloat::opStatus fs = V.roundToIntegral(RM: APFloat::rmTowardNegative);
7747 if (fs == APFloat::opOK || fs == APFloat::opInexact)
7748 return getConstantFP(V, DL, VT);
7749 return SDValue();
7750 }
7751 case ISD::FP_EXTEND: {
7752 bool ignored;
7753 // This can return overflow, underflow, or inexact; we don't care.
7754 // FIXME need to be more flexible about rounding mode.
7755 (void)V.convert(ToSemantics: VT.getFltSemantics(), RM: APFloat::rmNearestTiesToEven,
7756 losesInfo: &ignored);
7757 return getConstantFP(V, DL, VT);
7758 }
7759 case ISD::FP_TO_SINT:
7760 case ISD::FP_TO_UINT: {
7761 bool ignored;
7762 APSInt IntVal(VT.getSizeInBits(), Opcode == ISD::FP_TO_UINT);
7763 // FIXME need to be more flexible about rounding mode.
7764 APFloat::opStatus s =
7765 V.convertToInteger(Result&: IntVal, RM: APFloat::rmTowardZero, IsExact: &ignored);
7766 if (s == APFloat::opInvalidOp) // inexact is OK, in fact usual
7767 break;
7768 return getConstant(Val: IntVal, DL, VT);
7769 }
7770 case ISD::FP_TO_FP16:
7771 case ISD::FP_TO_BF16: {
7772 bool Ignored;
7773 // This can return overflow, underflow, or inexact; we don't care.
7774 // FIXME need to be more flexible about rounding mode.
7775 (void)V.convert(ToSemantics: Opcode == ISD::FP_TO_FP16 ? APFloat::IEEEhalf()
7776 : APFloat::BFloat(),
7777 RM: APFloat::rmNearestTiesToEven, losesInfo: &Ignored);
7778 return getConstant(Val: V.bitcastToAPInt().getZExtValue(), DL, VT);
7779 }
7780 case ISD::BITCAST:
7781 if (VT == MVT::i16 && C->getValueType(ResNo: 0) == MVT::f16)
7782 return getConstant(Val: (uint16_t)V.bitcastToAPInt().getZExtValue(), DL,
7783 VT);
7784 if (VT == MVT::i16 && C->getValueType(ResNo: 0) == MVT::bf16)
7785 return getConstant(Val: (uint16_t)V.bitcastToAPInt().getZExtValue(), DL,
7786 VT);
7787 if (VT == MVT::i32 && C->getValueType(ResNo: 0) == MVT::f32)
7788 return getConstant(Val: (uint32_t)V.bitcastToAPInt().getZExtValue(), DL,
7789 VT);
7790 if (VT == MVT::i64 && C->getValueType(ResNo: 0) == MVT::f64)
7791 return getConstant(Val: V.bitcastToAPInt().getZExtValue(), DL, VT);
7792 break;
7793 }
7794 }
7795
7796 // Early-out if we failed to constant fold a bitcast.
7797 if (Opcode == ISD::BITCAST)
7798 return SDValue();
7799
7800 // Constant fold integer vector reductions with constant BUILD_VECTORs.
7801 if ((Opcode == ISD::VECREDUCE_ADD || Opcode == ISD::VECREDUCE_SMAX ||
7802 Opcode == ISD::VECREDUCE_SMIN || Opcode == ISD::VECREDUCE_UMAX ||
7803 Opcode == ISD::VECREDUCE_UMIN || Opcode == ISD::VECREDUCE_MUL ||
7804 Opcode == ISD::VECREDUCE_OR || Opcode == ISD::VECREDUCE_XOR ||
7805 Opcode == ISD::VECREDUCE_AND) &&
7806 ISD::isBuildVectorOfConstantSDNodes(N: N1.getNode())) {
7807 unsigned EltBits = N1.getValueType().getScalarSizeInBits();
7808 unsigned BaseOpcode = ISD::getVecReduceBaseOpcode(VecReduceOpcode: Opcode);
7809 APInt Acc = getIntegerIdentity(Opcode: BaseOpcode, BitWidth: EltBits);
7810 for (SDValue Elt : N1->op_values()) {
7811 if (Elt.getOpcode() == ISD::POISON)
7812 return getPOISON(VT);
7813 if (Elt.isUndef() || cast<ConstantSDNode>(Val&: Elt)->isOpaque())
7814 return SDValue();
7815 APInt Value = cast<ConstantSDNode>(Val&: Elt)->getAPIntValue().trunc(width: EltBits);
7816 std::optional<APInt> Folded = FoldValue(Opcode: BaseOpcode, C1: Acc, C2: Value);
7817 assert(Folded &&
7818 "Expected vector reduction base opcode to be foldable");
7819 Acc = *Folded;
7820 }
7821 EVT EltVT = N1.getValueType().getScalarType();
7822 return getAnyExtOrTrunc(Op: getConstant(Val: Acc, DL, VT: EltVT), DL, VT);
7823 }
7824 }
7825
7826 // Handle binops special cases.
7827 if (NumOps == 2) {
7828 if (SDValue CFP = foldConstantFPMath(Opcode, DL, VT, Ops))
7829 return CFP;
7830
7831 if (auto *C1 = dyn_cast<ConstantSDNode>(Val: Ops[0])) {
7832 if (auto *C2 = dyn_cast<ConstantSDNode>(Val: Ops[1])) {
7833 if (C1->isOpaque() || C2->isOpaque())
7834 return SDValue();
7835
7836 std::optional<APInt> FoldAttempt =
7837 FoldValue(Opcode, C1: C1->getAPIntValue(), C2: C2->getAPIntValue());
7838 if (!FoldAttempt)
7839 return SDValue();
7840
7841 SDValue Folded = getConstant(Val: *FoldAttempt, DL, VT);
7842 assert((!Folded || !VT.isVector()) &&
7843 "Can't fold vectors ops with scalar operands");
7844 return Folded;
7845 }
7846 }
7847
7848 // fold (add Sym, c) -> Sym+c
7849 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Val: Ops[0]))
7850 return FoldSymbolOffset(Opcode, VT, GA, N2: Ops[1].getNode());
7851 if (TLI->isCommutativeBinOp(Opcode))
7852 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Val: Ops[1]))
7853 return FoldSymbolOffset(Opcode, VT, GA, N2: Ops[0].getNode());
7854
7855 // fold (sext_in_reg c1) -> c2
7856 if (Opcode == ISD::SIGN_EXTEND_INREG) {
7857 EVT EVT = cast<VTSDNode>(Val: Ops[1])->getVT();
7858
7859 auto SignExtendInReg = [&](APInt Val, llvm::EVT ConstantVT) {
7860 unsigned FromBits = EVT.getScalarSizeInBits();
7861 Val <<= Val.getBitWidth() - FromBits;
7862 Val.ashrInPlace(ShiftAmt: Val.getBitWidth() - FromBits);
7863 return getConstant(Val, DL, VT: ConstantVT);
7864 };
7865
7866 if (auto *C1 = dyn_cast<ConstantSDNode>(Val: Ops[0])) {
7867 const APInt &Val = C1->getAPIntValue();
7868 return SignExtendInReg(Val, VT);
7869 }
7870
7871 if (ISD::isBuildVectorOfConstantSDNodes(N: Ops[0].getNode())) {
7872 SmallVector<SDValue, 8> ScalarOps;
7873 llvm::EVT OpVT = Ops[0].getOperand(i: 0).getValueType();
7874 for (int I = 0, E = VT.getVectorNumElements(); I != E; ++I) {
7875 SDValue Op = Ops[0].getOperand(i: I);
7876 if (Op.isUndef()) {
7877 ScalarOps.push_back(Elt: getUNDEF(VT: OpVT));
7878 continue;
7879 }
7880 const APInt &Val = cast<ConstantSDNode>(Val&: Op)->getAPIntValue();
7881 ScalarOps.push_back(Elt: SignExtendInReg(Val, OpVT));
7882 }
7883 return getBuildVector(VT, DL, Ops: ScalarOps);
7884 }
7885
7886 if (Ops[0].getOpcode() == ISD::SPLAT_VECTOR &&
7887 isa<ConstantSDNode>(Val: Ops[0].getOperand(i: 0)))
7888 return getNode(Opcode: ISD::SPLAT_VECTOR, DL, VT,
7889 N1: SignExtendInReg(Ops[0].getConstantOperandAPInt(i: 0),
7890 Ops[0].getOperand(i: 0).getValueType()));
7891 }
7892 }
7893
7894 // Handle fshl/fshr special cases.
7895 if (Opcode == ISD::FSHL || Opcode == ISD::FSHR) {
7896 auto *C1 = dyn_cast<ConstantSDNode>(Val: Ops[0]);
7897 auto *C2 = dyn_cast<ConstantSDNode>(Val: Ops[1]);
7898 auto *C3 = dyn_cast<ConstantSDNode>(Val: Ops[2]);
7899
7900 if (C1 && C2 && C3) {
7901 if (C1->isOpaque() || C2->isOpaque() || C3->isOpaque())
7902 return SDValue();
7903 const APInt &V1 = C1->getAPIntValue(), &V2 = C2->getAPIntValue(),
7904 &V3 = C3->getAPIntValue();
7905
7906 APInt FoldedVal = Opcode == ISD::FSHL ? APIntOps::fshl(Hi: V1, Lo: V2, Shift: V3)
7907 : APIntOps::fshr(Hi: V1, Lo: V2, Shift: V3);
7908 return getConstant(Val: FoldedVal, DL, VT);
7909 }
7910 }
7911
7912 // Handle fma/fmad special cases.
7913 if (Opcode == ISD::FMA || Opcode == ISD::FMAD || Opcode == ISD::FMULADD) {
7914 assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
7915 assert(Ops[0].getValueType() == VT && Ops[1].getValueType() == VT &&
7916 Ops[2].getValueType() == VT && "FMA types must match!");
7917 ConstantFPSDNode *C1 = dyn_cast<ConstantFPSDNode>(Val: Ops[0]);
7918 ConstantFPSDNode *C2 = dyn_cast<ConstantFPSDNode>(Val: Ops[1]);
7919 ConstantFPSDNode *C3 = dyn_cast<ConstantFPSDNode>(Val: Ops[2]);
7920 if (C1 && C2 && C3) {
7921 APFloat V1 = C1->getValueAPF();
7922 const APFloat &V2 = C2->getValueAPF();
7923 const APFloat &V3 = C3->getValueAPF();
7924 if (Opcode == ISD::FMAD || Opcode == ISD::FMULADD) {
7925 V1.multiply(RHS: V2, RM: APFloat::rmNearestTiesToEven);
7926 V1.add(RHS: V3, RM: APFloat::rmNearestTiesToEven);
7927 } else
7928 V1.fusedMultiplyAdd(Multiplicand: V2, Addend: V3, RM: APFloat::rmNearestTiesToEven);
7929 return getConstantFP(V: V1, DL, VT);
7930 }
7931 }
7932
7933 // This is for vector folding only from here on.
7934 if (!VT.isVector())
7935 return SDValue();
7936
7937 // Constant fold integer partial reductions with constant BUILD_VECTOR
7938 // operands. The reduction order is deliberately unspecified. Use the same
7939 // subvector layout as TargetLowering::expandPartialReduceMLA(), where input
7940 // lane I contributes to accumulator lane I % NumAccElts.
7941 if (Opcode == ISD::PARTIAL_REDUCE_SMLA ||
7942 Opcode == ISD::PARTIAL_REDUCE_UMLA ||
7943 Opcode == ISD::PARTIAL_REDUCE_SUMLA) {
7944 // These nodes have no scalar form, so unsupported cases must not fall
7945 // through to generic per-lane vector folding.
7946 if (!llvm::all_of(Range&: Ops, P: [](SDValue Op) {
7947 return ISD::isBuildVectorOfConstantSDNodes(N: Op.getNode());
7948 }))
7949 return SDValue();
7950
7951 unsigned AccEltBits = VT.getScalarSizeInBits();
7952 unsigned InputEltBits = Ops[1].getScalarValueSizeInBits();
7953 unsigned NumAccElts = VT.getVectorNumElements();
7954 unsigned NumInputElts = Ops[1].getValueType().getVectorNumElements();
7955 SmallVector<APInt, 8> Results(NumAccElts, APInt::getZero(numBits: AccEltBits));
7956 BitVector PoisonElts(NumAccElts);
7957
7958 for (unsigned I = 0; I != NumAccElts; ++I) {
7959 SDValue Elt = Ops[0].getOperand(i: I);
7960 if (Elt.getOpcode() == ISD::POISON) {
7961 PoisonElts.set(I);
7962 continue;
7963 }
7964 auto *C = dyn_cast<ConstantSDNode>(Val&: Elt);
7965 if (!C || C->isOpaque())
7966 return SDValue();
7967 Results[I] = C->getAPIntValue().trunc(width: AccEltBits);
7968 }
7969
7970 bool IsLHSSigned = Opcode != ISD::PARTIAL_REDUCE_UMLA;
7971 bool IsRHSSigned = Opcode == ISD::PARTIAL_REDUCE_SMLA;
7972 for (unsigned I = 0; I != NumInputElts; ++I) {
7973 const unsigned AccIdx = I % NumAccElts;
7974 SDValue LHSElt = Ops[1].getOperand(i: I);
7975 SDValue RHSElt = Ops[2].getOperand(i: I);
7976 if (LHSElt.getOpcode() == ISD::POISON ||
7977 RHSElt.getOpcode() == ISD::POISON) {
7978 PoisonElts.set(AccIdx);
7979 continue;
7980 }
7981
7982 auto *LHS = dyn_cast<ConstantSDNode>(Val&: LHSElt);
7983 auto *RHS = dyn_cast<ConstantSDNode>(Val&: RHSElt);
7984 if (!LHS || !RHS || LHS->isOpaque() || RHS->isOpaque())
7985 return SDValue();
7986
7987 APInt LHSVal = LHS->getAPIntValue().trunc(width: InputEltBits);
7988 APInt RHSVal = RHS->getAPIntValue().trunc(width: InputEltBits);
7989 LHSVal = IsLHSSigned ? LHSVal.sext(width: AccEltBits) : LHSVal.zext(width: AccEltBits);
7990 RHSVal = IsRHSSigned ? RHSVal.sext(width: AccEltBits) : RHSVal.zext(width: AccEltBits);
7991 Results[AccIdx] += LHSVal * RHSVal;
7992 }
7993
7994 // After type legalization the vector element type may not be a legal
7995 // scalar type (e.g. i16 on AArch64). Create the folded constants in the
7996 // promoted legal scalar type instead, matching the generic per-lane path
7997 // below. Bail out if legalization would narrow the type, since the lane
7998 // value would not fit.
7999 EVT AccEltVT = VT.getVectorElementType();
8000 EVT LegalSVT = AccEltVT;
8001 if (NewNodesMustHaveLegalTypes && LegalSVT.isInteger()) {
8002 LegalSVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT: LegalSVT);
8003 if (LegalSVT.bitsLT(VT: AccEltVT))
8004 return SDValue();
8005 }
8006
8007 SmallVector<SDValue, 8> ResultOps;
8008 for (unsigned I = 0; I != NumAccElts; ++I)
8009 ResultOps.push_back(
8010 Elt: PoisonElts[I] ? getPOISON(VT: LegalSVT)
8011 : getConstant(Val: Results[I].sext(width: LegalSVT.getSizeInBits()),
8012 DL, VT: LegalSVT));
8013 return getBuildVector(VT, DL, Ops: ResultOps);
8014 }
8015
8016 ElementCount NumElts = VT.getVectorElementCount();
8017
8018 // See if we can fold through any bitcasted integer ops.
8019 if (NumOps == 2 && VT.isFixedLengthVector() && VT.isInteger() &&
8020 Ops[0].getValueType() == VT && Ops[1].getValueType() == VT &&
8021 (Ops[0].getOpcode() == ISD::BITCAST ||
8022 Ops[1].getOpcode() == ISD::BITCAST)) {
8023 SDValue N1 = peekThroughBitcasts(V: Ops[0]);
8024 SDValue N2 = peekThroughBitcasts(V: Ops[1]);
8025 auto *BV1 = dyn_cast<BuildVectorSDNode>(Val&: N1);
8026 auto *BV2 = dyn_cast<BuildVectorSDNode>(Val&: N2);
8027 if (BV1 && BV2 && N1.getValueType().isInteger() &&
8028 N2.getValueType().isInteger()) {
8029 bool IsLE = getDataLayout().isLittleEndian();
8030 unsigned EltBits = VT.getScalarSizeInBits();
8031 SmallVector<APInt> RawBits1, RawBits2;
8032 BitVector UndefElts1, UndefElts2;
8033 if (BV1->getConstantRawBits(IsLittleEndian: IsLE, DstEltSizeInBits: EltBits, RawBitElements&: RawBits1, UndefElements&: UndefElts1) &&
8034 BV2->getConstantRawBits(IsLittleEndian: IsLE, DstEltSizeInBits: EltBits, RawBitElements&: RawBits2, UndefElements&: UndefElts2)) {
8035 SmallVector<APInt> RawBits;
8036 for (unsigned I = 0, E = NumElts.getFixedValue(); I != E; ++I) {
8037 std::optional<APInt> Fold = FoldValueWithUndef(
8038 Opcode, C1: RawBits1[I], IsUndef1: UndefElts1[I], C2: RawBits2[I], IsUndef2: UndefElts2[I]);
8039 if (!Fold)
8040 break;
8041 RawBits.push_back(Elt: *Fold);
8042 }
8043 if (RawBits.size() == NumElts.getFixedValue()) {
8044 // We have constant folded, but we might need to cast this again back
8045 // to the original (possibly legalized) type.
8046 EVT BVVT, BVEltVT;
8047 if (N1.getValueType() == VT) {
8048 BVVT = N1.getValueType();
8049 BVEltVT = BV1->getOperand(Num: 0).getValueType();
8050 } else {
8051 BVVT = N2.getValueType();
8052 BVEltVT = BV2->getOperand(Num: 0).getValueType();
8053 }
8054 unsigned BVEltBits = BVEltVT.getSizeInBits();
8055 SmallVector<APInt> DstBits;
8056 BitVector DstUndefs;
8057 BuildVectorSDNode::recastRawBits(IsLittleEndian: IsLE, DstEltSizeInBits: BVVT.getScalarSizeInBits(),
8058 DstBitElements&: DstBits, SrcBitElements: RawBits, DstUndefElements&: DstUndefs,
8059 SrcUndefElements: BitVector(RawBits.size(), false));
8060 SmallVector<SDValue> Ops(DstBits.size(), getUNDEF(VT: BVEltVT));
8061 for (unsigned I = 0, E = DstBits.size(); I != E; ++I) {
8062 if (DstUndefs[I])
8063 continue;
8064 Ops[I] = getConstant(Val: DstBits[I].sext(width: BVEltBits), DL, VT: BVEltVT);
8065 }
8066 return getBitcast(VT, V: getBuildVector(VT: BVVT, DL, Ops));
8067 }
8068 }
8069 }
8070 // Logic ops can be folded from raw integer bits - mainly for AVX512 masks.
8071 if (ISD::isBitwiseLogicOp(Opcode) && isa<ConstantSDNode>(Val: N1) &&
8072 isa<ConstantSDNode>(Val: N2)) {
8073 if (SDValue Res = FoldConstantArithmetic(Opcode, DL, VT: N1.getValueType(),
8074 Ops: {N1, N2}, Flags))
8075 return getBitcast(VT, V: Res);
8076 }
8077 }
8078
8079 // Fold (mul step_vector(C0), C1) to (step_vector(C0 * C1)).
8080 // (shl step_vector(C0), C1) -> (step_vector(C0 << C1))
8081 if ((Opcode == ISD::MUL || Opcode == ISD::SHL) &&
8082 Ops[0].getOpcode() == ISD::STEP_VECTOR) {
8083 APInt RHSVal;
8084 if (ISD::isConstantSplatVector(N: Ops[1].getNode(), SplatVal&: RHSVal)) {
8085 APInt NewStep = Opcode == ISD::MUL
8086 ? Ops[0].getConstantOperandAPInt(i: 0) * RHSVal
8087 : Ops[0].getConstantOperandAPInt(i: 0) << RHSVal;
8088 return getStepVector(DL, ResVT: VT, StepVal: NewStep);
8089 }
8090 }
8091
8092 auto IsScalarOrSameVectorSize = [NumElts](const SDValue &Op) {
8093 return !Op.getValueType().isVector() ||
8094 Op.getValueType().getVectorElementCount() == NumElts;
8095 };
8096
8097 auto IsBuildVectorSplatVectorOrUndef = [](const SDValue &Op) {
8098 return Op.isUndef() || Op.getOpcode() == ISD::CONDCODE ||
8099 Op.getOpcode() == ISD::BUILD_VECTOR ||
8100 Op.getOpcode() == ISD::SPLAT_VECTOR;
8101 };
8102
8103 // All operands must be vector types with the same number of elements as
8104 // the result type and must be either UNDEF or a build/splat vector
8105 // or UNDEF scalars.
8106 if (!llvm::all_of(Range&: Ops, P: IsBuildVectorSplatVectorOrUndef) ||
8107 !llvm::all_of(Range&: Ops, P: IsScalarOrSameVectorSize))
8108 return SDValue();
8109
8110 // If we are comparing vectors, then the result needs to be a i1 boolean that
8111 // is then extended back to the legal result type depending on how booleans
8112 // are represented.
8113 EVT SVT = (Opcode == ISD::SETCC ? MVT::i1 : VT.getScalarType());
8114 ISD::NodeType ExtendCode =
8115 (Opcode == ISD::SETCC && SVT != VT.getScalarType())
8116 ? TargetLowering::getExtendForContent(Content: TLI->getBooleanContents(Type: VT))
8117 : ISD::SIGN_EXTEND;
8118
8119 // Find legal integer scalar type for constant promotion and
8120 // ensure that its scalar size is at least as large as source.
8121 EVT LegalSVT = VT.getScalarType();
8122 if (NewNodesMustHaveLegalTypes && LegalSVT.isInteger()) {
8123 LegalSVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT: LegalSVT);
8124 if (LegalSVT.bitsLT(VT: VT.getScalarType()))
8125 return SDValue();
8126 }
8127
8128 // For scalable vector types we know we're dealing with SPLAT_VECTORs. We
8129 // only have one operand to check. For fixed-length vector types we may have
8130 // a combination of BUILD_VECTOR and SPLAT_VECTOR.
8131 unsigned NumVectorElts = NumElts.isScalable() ? 1 : NumElts.getFixedValue();
8132
8133 // Constant fold each scalar lane separately.
8134 SmallVector<SDValue, 4> ScalarResults;
8135 for (unsigned I = 0; I != NumVectorElts; I++) {
8136 SmallVector<SDValue, 4> ScalarOps;
8137 for (SDValue Op : Ops) {
8138 EVT InSVT = Op.getValueType().getScalarType();
8139 if (Op.getOpcode() != ISD::BUILD_VECTOR &&
8140 Op.getOpcode() != ISD::SPLAT_VECTOR) {
8141 if (Op.isUndef())
8142 ScalarOps.push_back(Elt: getUNDEF(VT: InSVT));
8143 else
8144 ScalarOps.push_back(Elt: Op);
8145 continue;
8146 }
8147
8148 SDValue ScalarOp =
8149 Op.getOperand(i: Op.getOpcode() == ISD::SPLAT_VECTOR ? 0 : I);
8150 EVT ScalarVT = ScalarOp.getValueType();
8151
8152 // Build vector (integer) scalar operands may need implicit
8153 // truncation - do this before constant folding.
8154 if (ScalarVT.isInteger() && ScalarVT.bitsGT(VT: InSVT)) {
8155 // Don't create illegally-typed nodes unless they're constants or undef
8156 // - if we fail to constant fold we can't guarantee the (dead) nodes
8157 // we're creating will be cleaned up before being visited for
8158 // legalization.
8159 if (NewNodesMustHaveLegalTypes && !ScalarOp.isUndef() &&
8160 !isa<ConstantSDNode>(Val: ScalarOp) &&
8161 TLI->getTypeAction(Context&: *getContext(), VT: InSVT) !=
8162 TargetLowering::TypeLegal)
8163 return SDValue();
8164 ScalarOp = getNode(Opcode: ISD::TRUNCATE, DL, VT: InSVT, N1: ScalarOp);
8165 }
8166
8167 ScalarOps.push_back(Elt: ScalarOp);
8168 }
8169
8170 // Constant fold the scalar operands.
8171 SDValue ScalarResult = getNode(Opcode, DL, VT: SVT, Ops: ScalarOps, Flags);
8172
8173 // Scalar folding only succeeded if the result is a constant or UNDEF.
8174 if (!ScalarResult.isUndef() && ScalarResult.getOpcode() != ISD::Constant &&
8175 ScalarResult.getOpcode() != ISD::ConstantFP)
8176 return SDValue();
8177
8178 // Legalize the (integer) scalar constant if necessary. We only do
8179 // this once we know the folding succeeded, since otherwise we would
8180 // get a node with illegal type which has a user.
8181 if (LegalSVT != SVT)
8182 ScalarResult = getNode(Opcode: ExtendCode, DL, VT: LegalSVT, N1: ScalarResult);
8183
8184 ScalarResults.push_back(Elt: ScalarResult);
8185 }
8186
8187 SDValue V = NumElts.isScalable() ? getSplatVector(VT, DL, Op: ScalarResults[0])
8188 : getBuildVector(VT, DL, Ops: ScalarResults);
8189 NewSDValueDbgMsg(V, Msg: "New node fold constant vector: ", G: this);
8190 return V;
8191}
8192
8193SDValue SelectionDAG::foldConstantFPMath(unsigned Opcode, const SDLoc &DL,
8194 EVT VT, ArrayRef<SDValue> Ops) {
8195 // TODO: Add support for unary/ternary fp opcodes.
8196 if (Ops.size() != 2)
8197 return SDValue();
8198
8199 // TODO: We don't do any constant folding for strict FP opcodes here, but we
8200 // should. That will require dealing with a potentially non-default
8201 // rounding mode, checking the "opStatus" return value from the APFloat
8202 // math calculations, and possibly other variations.
8203 SDValue N1 = Ops[0];
8204 SDValue N2 = Ops[1];
8205 ConstantFPSDNode *N1CFP = isConstOrConstSplatFP(N: N1, /*AllowUndefs*/ false);
8206 ConstantFPSDNode *N2CFP = isConstOrConstSplatFP(N: N2, /*AllowUndefs*/ false);
8207 if (N1CFP && N2CFP) {
8208 APFloat C1 = N1CFP->getValueAPF(); // make copy
8209 const APFloat &C2 = N2CFP->getValueAPF();
8210 switch (Opcode) {
8211 case ISD::FADD:
8212 C1.add(RHS: C2, RM: APFloat::rmNearestTiesToEven);
8213 return getConstantFP(V: C1, DL, VT);
8214 case ISD::FSUB:
8215 C1.subtract(RHS: C2, RM: APFloat::rmNearestTiesToEven);
8216 return getConstantFP(V: C1, DL, VT);
8217 case ISD::FMUL:
8218 C1.multiply(RHS: C2, RM: APFloat::rmNearestTiesToEven);
8219 return getConstantFP(V: C1, DL, VT);
8220 case ISD::FDIV:
8221 C1.divide(RHS: C2, RM: APFloat::rmNearestTiesToEven);
8222 return getConstantFP(V: C1, DL, VT);
8223 case ISD::FREM:
8224 C1.mod(RHS: C2);
8225 return getConstantFP(V: C1, DL, VT);
8226 case ISD::FCOPYSIGN:
8227 C1.copySign(RHS: C2);
8228 return getConstantFP(V: C1, DL, VT);
8229 case ISD::FMINNUM:
8230 return getConstantFP(V: minnum(A: C1, B: C2), DL, VT);
8231 case ISD::FMAXNUM:
8232 return getConstantFP(V: maxnum(A: C1, B: C2), DL, VT);
8233 case ISD::FMINIMUM:
8234 return getConstantFP(V: minimum(A: C1, B: C2), DL, VT);
8235 case ISD::FMAXIMUM:
8236 return getConstantFP(V: maximum(A: C1, B: C2), DL, VT);
8237 case ISD::FMINIMUMNUM:
8238 return getConstantFP(V: minimumnum(A: C1, B: C2), DL, VT);
8239 case ISD::FMAXIMUMNUM:
8240 return getConstantFP(V: maximumnum(A: C1, B: C2), DL, VT);
8241 default: break;
8242 }
8243 }
8244 if (N1CFP && Opcode == ISD::FP_ROUND) {
8245 APFloat C1 = N1CFP->getValueAPF(); // make copy
8246 bool Unused;
8247 // This can return overflow, underflow, or inexact; we don't care.
8248 // FIXME need to be more flexible about rounding mode.
8249 (void)C1.convert(ToSemantics: VT.getFltSemantics(), RM: APFloat::rmNearestTiesToEven,
8250 losesInfo: &Unused);
8251 return getConstantFP(V: C1, DL, VT);
8252 }
8253
8254 switch (Opcode) {
8255 case ISD::FSUB:
8256 // -0.0 - undef --> undef (consistent with "fneg undef")
8257 if (ConstantFPSDNode *N1C = isConstOrConstSplatFP(N: N1, /*AllowUndefs*/ true))
8258 if (N1C && N1C->getValueAPF().isNegZero() && N2.isUndef())
8259 return getUNDEF(VT);
8260 [[fallthrough]];
8261
8262 case ISD::FADD:
8263 case ISD::FMUL:
8264 case ISD::FDIV:
8265 case ISD::FREM:
8266 // If both operands are undef, the result is undef. If 1 operand is undef,
8267 // the result is NaN. This should match the behavior of the IR optimizer.
8268 if (N1.isUndef() && N2.isUndef())
8269 return getUNDEF(VT);
8270 if (N1.isUndef() || N2.isUndef())
8271 return getConstantFP(V: APFloat::getNaN(Sem: VT.getFltSemantics()), DL, VT);
8272 }
8273 return SDValue();
8274}
8275
8276SDValue SelectionDAG::FoldConstantBuildVector(BuildVectorSDNode *BV,
8277 const SDLoc &DL, EVT DstEltVT) {
8278 EVT SrcEltVT = BV->getValueType(ResNo: 0).getVectorElementType();
8279
8280 // If this is already the right type, we're done.
8281 if (SrcEltVT == DstEltVT)
8282 return SDValue(BV, 0);
8283
8284 unsigned SrcBitSize = SrcEltVT.getSizeInBits();
8285 unsigned DstBitSize = DstEltVT.getSizeInBits();
8286
8287 // If this is a conversion of N elements of one type to N elements of another
8288 // type, convert each element. This handles FP<->INT cases.
8289 if (SrcBitSize == DstBitSize) {
8290 SmallVector<SDValue, 8> Ops;
8291 for (SDValue Op : BV->op_values()) {
8292 // If the vector element type is not legal, the BUILD_VECTOR operands
8293 // are promoted and implicitly truncated. Make that explicit here.
8294 if (Op.getValueType() != SrcEltVT)
8295 Op = getNode(Opcode: ISD::TRUNCATE, DL, VT: SrcEltVT, N1: Op);
8296 Ops.push_back(Elt: getBitcast(VT: DstEltVT, V: Op));
8297 }
8298 EVT VT = EVT::getVectorVT(Context&: *getContext(), VT: DstEltVT,
8299 NumElements: BV->getValueType(ResNo: 0).getVectorNumElements());
8300 return getBuildVector(VT, DL, Ops);
8301 }
8302
8303 // Otherwise, we're growing or shrinking the elements. To avoid having to
8304 // handle annoying details of growing/shrinking FP values, we convert them to
8305 // int first.
8306 if (SrcEltVT.isFloatingPoint()) {
8307 // Convert the input float vector to a int vector where the elements are the
8308 // same sizes.
8309 EVT IntEltVT = EVT::getIntegerVT(Context&: *getContext(), BitWidth: SrcEltVT.getSizeInBits());
8310 if (SDValue Tmp = FoldConstantBuildVector(BV, DL, DstEltVT: IntEltVT))
8311 return FoldConstantBuildVector(BV: cast<BuildVectorSDNode>(Val&: Tmp), DL,
8312 DstEltVT);
8313 return SDValue();
8314 }
8315
8316 // Now we know the input is an integer vector. If the output is a FP type,
8317 // convert to integer first, then to FP of the right size.
8318 if (DstEltVT.isFloatingPoint()) {
8319 EVT IntEltVT = EVT::getIntegerVT(Context&: *getContext(), BitWidth: DstEltVT.getSizeInBits());
8320 if (SDValue Tmp = FoldConstantBuildVector(BV, DL, DstEltVT: IntEltVT))
8321 return FoldConstantBuildVector(BV: cast<BuildVectorSDNode>(Val&: Tmp), DL,
8322 DstEltVT);
8323 return SDValue();
8324 }
8325
8326 // Okay, we know the src/dst types are both integers of differing types.
8327 assert(SrcEltVT.isInteger() && DstEltVT.isInteger());
8328
8329 // Extract the constant raw bit data.
8330 BitVector UndefElements;
8331 SmallVector<APInt> RawBits;
8332 bool IsLE = getDataLayout().isLittleEndian();
8333 if (!BV->getConstantRawBits(IsLittleEndian: IsLE, DstEltSizeInBits: DstBitSize, RawBitElements&: RawBits, UndefElements))
8334 return SDValue();
8335
8336 SmallVector<SDValue, 8> Ops;
8337 for (unsigned I = 0, E = RawBits.size(); I != E; ++I) {
8338 if (UndefElements[I])
8339 Ops.push_back(Elt: getUNDEF(VT: DstEltVT));
8340 else
8341 Ops.push_back(Elt: getConstant(Val: RawBits[I], DL, VT: DstEltVT));
8342 }
8343
8344 EVT VT = EVT::getVectorVT(Context&: *getContext(), VT: DstEltVT, NumElements: Ops.size());
8345 return getBuildVector(VT, DL, Ops);
8346}
8347
8348SDValue SelectionDAG::getAssertAlign(const SDLoc &DL, SDValue Val, Align A) {
8349 assert(Val.getValueType().isInteger() && "Invalid AssertAlign!");
8350
8351 // There's no need to assert on a byte-aligned pointer. All pointers are at
8352 // least byte aligned.
8353 if (A == Align(1))
8354 return Val;
8355
8356 SDVTList VTs = getVTList(VT: Val.getValueType());
8357 FoldingSetNodeID ID;
8358 AddNodeIDNode(ID, OpC: ISD::AssertAlign, VTList: VTs, OpList: {Val});
8359 ID.AddInteger(I: A.value());
8360
8361 FoldingSetInsertToken InsertToken;
8362 if (SDNode *E = lookupNode(ID, DL, InsertToken))
8363 return SDValue(E, 0);
8364
8365 auto *N =
8366 newSDNode<AssertAlignSDNode>(Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs, Args&: A);
8367 createOperands(Node: N, Vals: {Val});
8368
8369 CSEMap.insert(N, Token: InsertToken);
8370 InsertNode(N);
8371
8372 SDValue V(N, 0);
8373 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
8374 return V;
8375}
8376
8377SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
8378 SDValue N1, SDValue N2) {
8379 SDNodeFlags Flags;
8380 if (Inserter)
8381 Flags = Inserter->getFlags();
8382 return getNode(Opcode, DL, VT, N1, N2, Flags);
8383}
8384
8385void SelectionDAG::canonicalizeCommutativeBinop(unsigned Opcode, SDValue &N1,
8386 SDValue &N2) const {
8387 if (!TLI->isCommutativeBinOp(Opcode))
8388 return;
8389
8390 // Canonicalize:
8391 // binop(const, nonconst) -> binop(nonconst, const)
8392 bool N1C = isConstantIntBuildVectorOrConstantInt(N: N1);
8393 bool N2C = isConstantIntBuildVectorOrConstantInt(N: N2);
8394 bool N1CFP = isConstantFPBuildVectorOrConstantFP(N: N1);
8395 bool N2CFP = isConstantFPBuildVectorOrConstantFP(N: N2);
8396 if ((N1C && !N2C) || (N1CFP && !N2CFP))
8397 std::swap(a&: N1, b&: N2);
8398
8399 // Canonicalize:
8400 // binop(splat(x), step_vector) -> binop(step_vector, splat(x))
8401 else if (N1.getOpcode() == ISD::SPLAT_VECTOR &&
8402 N2.getOpcode() == ISD::STEP_VECTOR)
8403 std::swap(a&: N1, b&: N2);
8404}
8405
8406SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
8407 SDValue N1, SDValue N2, const SDNodeFlags Flags) {
8408 assert(N1.getOpcode() != ISD::DELETED_NODE &&
8409 N2.getOpcode() != ISD::DELETED_NODE &&
8410 "Operand is DELETED_NODE!");
8411
8412 canonicalizeCommutativeBinop(Opcode, N1, N2);
8413
8414 auto *N1C = dyn_cast<ConstantSDNode>(Val&: N1);
8415 auto *N2C = dyn_cast<ConstantSDNode>(Val&: N2);
8416
8417 // Don't allow undefs in vector splats - we might be returning N2 when folding
8418 // to zero etc.
8419 ConstantSDNode *N2CV =
8420 isConstOrConstSplat(N: N2, /*AllowUndefs*/ false, /*AllowTruncation*/ true);
8421
8422 switch (Opcode) {
8423 default: break;
8424 case ISD::TokenFactor:
8425 assert(VT == MVT::Other && N1.getValueType() == MVT::Other &&
8426 N2.getValueType() == MVT::Other && "Invalid token factor!");
8427 // Fold trivial token factors.
8428 if (N1.getOpcode() == ISD::EntryToken) return N2;
8429 if (N2.getOpcode() == ISD::EntryToken) return N1;
8430 if (N1 == N2) return N1;
8431 break;
8432 case ISD::BUILD_VECTOR: {
8433 // Attempt to simplify BUILD_VECTOR.
8434 SDValue Ops[] = {N1, N2};
8435 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, DAG&: *this))
8436 return V;
8437 break;
8438 }
8439 case ISD::CONCAT_VECTORS: {
8440 SDValue Ops[] = {N1, N2};
8441 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, DAG&: *this))
8442 return V;
8443 break;
8444 }
8445 case ISD::AND:
8446 assert(VT.isInteger() && "This operator does not apply to FP types!");
8447 assert(N1.getValueType() == N2.getValueType() &&
8448 N1.getValueType() == VT && "Binary operator types must match!");
8449 // (X & 0) -> 0. This commonly occurs when legalizing i64 values, so it's
8450 // worth handling here.
8451 if (N2CV && N2CV->isZero())
8452 return N2;
8453 if (N2CV && N2CV->isAllOnes()) // X & -1 -> X
8454 return N1;
8455 break;
8456 case ISD::OR:
8457 case ISD::XOR:
8458 case ISD::ADD:
8459 case ISD::PTRADD:
8460 case ISD::SUB:
8461 assert(VT.isInteger() && "This operator does not apply to FP types!");
8462 assert(N1.getValueType() == N2.getValueType() &&
8463 N1.getValueType() == VT && "Binary operator types must match!");
8464 // The equal operand types requirement is unnecessarily strong for PTRADD.
8465 // However, the SelectionDAGBuilder does not generate PTRADDs with different
8466 // operand types, and we'd need to re-implement GEP's non-standard wrapping
8467 // logic everywhere where PTRADDs may be folded or combined to properly
8468 // support them. If/when we introduce pointer types to the SDAG, we will
8469 // need to relax this constraint.
8470
8471 // (X ^|+- 0) -> X. This commonly occurs when legalizing i64 values, so
8472 // it's worth handling here.
8473 if (N2CV && N2CV->isZero())
8474 return N1;
8475 if ((Opcode == ISD::ADD || Opcode == ISD::SUB) &&
8476 VT.getScalarType() == MVT::i1)
8477 return getNode(Opcode: ISD::XOR, DL, VT, N1, N2);
8478 // Fold (add (vscale * C0), (vscale * C1)) to (vscale * (C0 + C1)).
8479 if (Opcode == ISD::ADD && N1.getOpcode() == ISD::VSCALE &&
8480 N2.getOpcode() == ISD::VSCALE) {
8481 const APInt &C1 = N1->getConstantOperandAPInt(Num: 0);
8482 const APInt &C2 = N2->getConstantOperandAPInt(Num: 0);
8483 return getVScale(DL, VT, MulImm: C1 + C2);
8484 }
8485 break;
8486 case ISD::MUL:
8487 assert(VT.isInteger() && "This operator does not apply to FP types!");
8488 assert(N1.getValueType() == N2.getValueType() &&
8489 N1.getValueType() == VT && "Binary operator types must match!");
8490 if (VT.getScalarType() == MVT::i1)
8491 return getNode(Opcode: ISD::AND, DL, VT, N1, N2);
8492 if (N2CV && N2CV->isZero())
8493 return N2;
8494 if (N2C && (N1.getOpcode() == ISD::VSCALE) && Flags.hasNoSignedWrap()) {
8495 const APInt &MulImm = N1->getConstantOperandAPInt(Num: 0);
8496 const APInt &N2CImm = N2C->getAPIntValue();
8497 return getVScale(DL, VT, MulImm: MulImm * N2CImm);
8498 }
8499 break;
8500 case ISD::UDIV:
8501 case ISD::UREM:
8502 case ISD::MULHU:
8503 case ISD::MULHS:
8504 case ISD::SDIV:
8505 case ISD::SREM:
8506 case ISD::SADDSAT:
8507 case ISD::SSUBSAT:
8508 case ISD::UADDSAT:
8509 case ISD::USUBSAT:
8510 assert(VT.isInteger() && "This operator does not apply to FP types!");
8511 assert(N1.getValueType() == N2.getValueType() &&
8512 N1.getValueType() == VT && "Binary operator types must match!");
8513 if (VT.getScalarType() == MVT::i1) {
8514 // fold (add_sat x, y) -> (or x, y) for bool types.
8515 if (Opcode == ISD::SADDSAT || Opcode == ISD::UADDSAT)
8516 return getNode(Opcode: ISD::OR, DL, VT, N1, N2);
8517 // fold (sub_sat x, y) -> (and x, ~y) for bool types.
8518 if (Opcode == ISD::SSUBSAT || Opcode == ISD::USUBSAT)
8519 return getNode(Opcode: ISD::AND, DL, VT, N1, N2: getNOT(DL, Val: N2, VT));
8520 }
8521 break;
8522 case ISD::SCMP:
8523 case ISD::UCMP:
8524 assert(N1.getValueType() == N2.getValueType() &&
8525 "Types of operands of UCMP/SCMP must match");
8526 assert(N1.getValueType().isVector() == VT.isVector() &&
8527 "Operands and return type of must both be scalars or vectors");
8528 if (VT.isVector())
8529 assert(VT.getVectorElementCount() ==
8530 N1.getValueType().getVectorElementCount() &&
8531 "Result and operands must have the same number of elements");
8532 break;
8533 case ISD::AVGFLOORS:
8534 case ISD::AVGFLOORU:
8535 case ISD::AVGCEILS:
8536 case ISD::AVGCEILU:
8537 assert(VT.isInteger() && "This operator does not apply to FP types!");
8538 assert(N1.getValueType() == N2.getValueType() &&
8539 N1.getValueType() == VT && "Binary operator types must match!");
8540 break;
8541 case ISD::ABDS:
8542 case ISD::ABDU:
8543 assert(VT.isInteger() && "This operator does not apply to FP types!");
8544 assert(N1.getValueType() == N2.getValueType() &&
8545 N1.getValueType() == VT && "Binary operator types must match!");
8546 if (VT.getScalarType() == MVT::i1)
8547 return getNode(Opcode: ISD::XOR, DL, VT, N1, N2);
8548 break;
8549 case ISD::SMIN:
8550 case ISD::UMAX:
8551 assert(VT.isInteger() && "This operator does not apply to FP types!");
8552 assert(N1.getValueType() == N2.getValueType() &&
8553 N1.getValueType() == VT && "Binary operator types must match!");
8554 if (VT.getScalarType() == MVT::i1)
8555 return getNode(Opcode: ISD::OR, DL, VT, N1, N2);
8556 break;
8557 case ISD::SMAX:
8558 case ISD::UMIN:
8559 assert(VT.isInteger() && "This operator does not apply to FP types!");
8560 assert(N1.getValueType() == N2.getValueType() &&
8561 N1.getValueType() == VT && "Binary operator types must match!");
8562 if (VT.getScalarType() == MVT::i1)
8563 return getNode(Opcode: ISD::AND, DL, VT, N1, N2);
8564 break;
8565 case ISD::FADD:
8566 case ISD::FSUB:
8567 case ISD::FMUL:
8568 case ISD::FDIV:
8569 case ISD::FREM:
8570 assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
8571 assert(N1.getValueType() == N2.getValueType() &&
8572 N1.getValueType() == VT && "Binary operator types must match!");
8573 if (SDValue V = simplifyFPBinop(Opcode, X: N1, Y: N2, Flags))
8574 return V;
8575 break;
8576 case ISD::FCOPYSIGN: // N1 and result must match. N1/N2 need not match.
8577 assert(N1.getValueType() == VT &&
8578 N1.getValueType().isFloatingPoint() &&
8579 N2.getValueType().isFloatingPoint() &&
8580 "Invalid FCOPYSIGN!");
8581 break;
8582 case ISD::SHL:
8583 if (N2C && (N1.getOpcode() == ISD::VSCALE) && Flags.hasNoSignedWrap()) {
8584 const APInt &MulImm = N1->getConstantOperandAPInt(Num: 0);
8585 const APInt &ShiftImm = N2C->getAPIntValue();
8586 return getVScale(DL, VT, MulImm: MulImm << ShiftImm);
8587 }
8588 [[fallthrough]];
8589 case ISD::SRA:
8590 case ISD::SRL:
8591 if (SDValue V = simplifyShift(X: N1, Y: N2))
8592 return V;
8593 [[fallthrough]];
8594 case ISD::ROTL:
8595 case ISD::ROTR:
8596 case ISD::SSHLSAT:
8597 case ISD::USHLSAT:
8598 assert(VT == N1.getValueType() &&
8599 "Shift operators return type must be the same as their first arg");
8600 assert(VT.isInteger() && N2.getValueType().isInteger() &&
8601 "Shifts only work on integers");
8602 assert((!VT.isVector() || VT == N2.getValueType()) &&
8603 "Vector shift amounts must be in the same as their first arg");
8604 // Verify that the shift amount VT is big enough to hold valid shift
8605 // amounts. This catches things like trying to shift an i1024 value by an
8606 // i8, which is easy to fall into in generic code that uses
8607 // TLI.getShiftAmount().
8608 assert(N2.getValueType().getScalarSizeInBits() >=
8609 Log2_32_Ceil(VT.getScalarSizeInBits()) &&
8610 "Invalid use of small shift amount with oversized value!");
8611
8612 // Always fold shifts of i1 values so the code generator doesn't need to
8613 // handle them. Since we know the size of the shift has to be less than the
8614 // size of the value, the shift/rotate count is guaranteed to be zero.
8615 if (VT == MVT::i1)
8616 return N1;
8617 if (N2CV && N2CV->isZero())
8618 return N1;
8619 break;
8620 case ISD::FP_ROUND:
8621 assert(VT.isFloatingPoint() && N1.getValueType().isFloatingPoint() &&
8622 VT.bitsLE(N1.getValueType()) && N2C &&
8623 (N2C->getZExtValue() == 0 || N2C->getZExtValue() == 1) &&
8624 N2.getOpcode() == ISD::TargetConstant && "Invalid FP_ROUND!");
8625 if (N1.getValueType() == VT) return N1; // noop conversion.
8626 break;
8627 case ISD::IS_FPCLASS: {
8628 assert(N1.getValueType().isFloatingPoint() &&
8629 "IS_FPCLASS is used for a non-floating type");
8630 assert(isa<ConstantSDNode>(N2) && "FPClassTest is not Constant");
8631 // is.fpclass(poison, mask) -> poison
8632 if (N1.getOpcode() == ISD::POISON)
8633 return getPOISON(VT);
8634 FPClassTest Mask = static_cast<FPClassTest>(N2->getAsZExtVal());
8635 // If all tests are made, it doesn't matter what the value is.
8636 if ((Mask & fcAllFlags) == fcAllFlags)
8637 return getBoolConstant(V: true, DL, VT, OpVT: N1.getValueType());
8638 if ((Mask & fcAllFlags) == 0)
8639 return getBoolConstant(V: false, DL, VT, OpVT: N1.getValueType());
8640 break;
8641 }
8642 case ISD::AssertNoFPClass: {
8643 assert(N1.getValueType().isFloatingPoint() &&
8644 "AssertNoFPClass is used for a non-floating type");
8645 assert(isa<ConstantSDNode>(N2) && "NoFPClass is not Constant");
8646 FPClassTest NoFPClass = static_cast<FPClassTest>(N2->getAsZExtVal());
8647 assert(llvm::to_underlying(NoFPClass) <=
8648 BitmaskEnumDetail::Mask<FPClassTest>() &&
8649 "FPClassTest value too large");
8650 (void)NoFPClass;
8651 break;
8652 }
8653 case ISD::AssertSext:
8654 case ISD::AssertZext: {
8655 EVT EVT = cast<VTSDNode>(Val&: N2)->getVT();
8656 assert(VT == N1.getValueType() && "Not an inreg extend!");
8657 assert(VT.isInteger() && EVT.isInteger() &&
8658 "Cannot *_EXTEND_INREG FP types");
8659 assert(!EVT.isVector() &&
8660 "AssertSExt/AssertZExt type should be the vector element type "
8661 "rather than the vector type!");
8662 assert(EVT.bitsLE(VT.getScalarType()) && "Not extending!");
8663 if (VT.getScalarType() == EVT) return N1; // noop assertion.
8664 break;
8665 }
8666 case ISD::SIGN_EXTEND_INREG: {
8667 EVT EVT = cast<VTSDNode>(Val&: N2)->getVT();
8668 assert(VT == N1.getValueType() && "Not an inreg extend!");
8669 assert(VT.isInteger() && EVT.isInteger() &&
8670 "Cannot *_EXTEND_INREG FP types");
8671 assert(EVT.isVector() == VT.isVector() &&
8672 "SIGN_EXTEND_INREG type should be vector iff the operand "
8673 "type is vector!");
8674 assert((!EVT.isVector() ||
8675 EVT.getVectorElementCount() == VT.getVectorElementCount()) &&
8676 "Vector element counts must match in SIGN_EXTEND_INREG");
8677 assert(EVT.getScalarType().bitsLE(VT.getScalarType()) && "Not extending!");
8678 if (EVT == VT) return N1; // Not actually extending
8679 break;
8680 }
8681 case ISD::FP_TO_SINT_SAT:
8682 case ISD::FP_TO_UINT_SAT: {
8683 assert(VT.isInteger() && cast<VTSDNode>(N2)->getVT().isInteger() &&
8684 N1.getValueType().isFloatingPoint() && "Invalid FP_TO_*INT_SAT");
8685 assert(N1.getValueType().isVector() == VT.isVector() &&
8686 "FP_TO_*INT_SAT type should be vector iff the operand type is "
8687 "vector!");
8688 assert((!VT.isVector() || VT.getVectorElementCount() ==
8689 N1.getValueType().getVectorElementCount()) &&
8690 "Vector element counts must match in FP_TO_*INT_SAT");
8691 assert(!cast<VTSDNode>(N2)->getVT().isVector() &&
8692 "Type to saturate to must be a scalar.");
8693 assert(cast<VTSDNode>(N2)->getVT().bitsLE(VT.getScalarType()) &&
8694 "Not extending!");
8695 break;
8696 }
8697 case ISD::EXTRACT_VECTOR_ELT:
8698 assert(VT.getSizeInBits() >= N1.getValueType().getScalarSizeInBits() &&
8699 "The result of EXTRACT_VECTOR_ELT must be at least as wide as the \
8700 element type of the vector.");
8701
8702 // Extract from an undefined value or using an undefined index is undefined.
8703 if (N1.isUndef() || N2.isUndef())
8704 return getUNDEF(VT);
8705
8706 // EXTRACT_VECTOR_ELT of out-of-bounds element is POISON for fixed length
8707 // vectors. For scalable vectors we will provide appropriate support for
8708 // dealing with arbitrary indices.
8709 if (N2C && N1.getValueType().isFixedLengthVector() &&
8710 N2C->getAPIntValue().uge(RHS: N1.getValueType().getVectorNumElements()))
8711 return getPOISON(VT);
8712
8713 // EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is
8714 // expanding copies of large vectors from registers. This only works for
8715 // fixed length vectors, since we need to know the exact number of
8716 // elements.
8717 if (N2C && N1.getOpcode() == ISD::CONCAT_VECTORS &&
8718 N1.getOperand(i: 0).getValueType().isFixedLengthVector()) {
8719 unsigned Factor = N1.getOperand(i: 0).getValueType().getVectorNumElements();
8720 return getExtractVectorElt(DL, VT,
8721 Vec: N1.getOperand(i: N2C->getZExtValue() / Factor),
8722 Idx: N2C->getZExtValue() % Factor);
8723 }
8724
8725 // EXTRACT_VECTOR_ELT of BUILD_VECTOR or SPLAT_VECTOR is often formed while
8726 // lowering is expanding large vector constants.
8727 if (N2C && (N1.getOpcode() == ISD::BUILD_VECTOR ||
8728 N1.getOpcode() == ISD::SPLAT_VECTOR)) {
8729 assert((N1.getOpcode() != ISD::BUILD_VECTOR ||
8730 N1.getValueType().isFixedLengthVector()) &&
8731 "BUILD_VECTOR used for scalable vectors");
8732 unsigned Index =
8733 N1.getOpcode() == ISD::BUILD_VECTOR ? N2C->getZExtValue() : 0;
8734 SDValue Elt = N1.getOperand(i: Index);
8735
8736 if (VT != Elt.getValueType())
8737 // If the vector element type is not legal, the BUILD_VECTOR operands
8738 // are promoted and implicitly truncated, and the result implicitly
8739 // extended. Make that explicit here.
8740 Elt = getAnyExtOrTrunc(Op: Elt, DL, VT);
8741
8742 return Elt;
8743 }
8744
8745 // EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector
8746 // operations are lowered to scalars.
8747 if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT) {
8748 // If the indices are the same, return the inserted element else
8749 // if the indices are known different, extract the element from
8750 // the original vector.
8751 SDValue N1Op2 = N1.getOperand(i: 2);
8752 ConstantSDNode *N1Op2C = dyn_cast<ConstantSDNode>(Val&: N1Op2);
8753
8754 if (N1Op2C && N2C) {
8755 if (N1Op2C->getZExtValue() == N2C->getZExtValue()) {
8756 if (VT == N1.getOperand(i: 1).getValueType())
8757 return N1.getOperand(i: 1);
8758 if (VT.isFloatingPoint()) {
8759 assert(VT.getSizeInBits() > N1.getOperand(1).getValueType().getSizeInBits());
8760 return getFPExtendOrRound(Op: N1.getOperand(i: 1), DL, VT);
8761 }
8762 return getSExtOrTrunc(Op: N1.getOperand(i: 1), DL, VT);
8763 }
8764 return getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT, N1: N1.getOperand(i: 0), N2);
8765 }
8766 }
8767
8768 // EXTRACT_VECTOR_ELT of v1iX EXTRACT_SUBVECTOR could be formed
8769 // when vector types are scalarized and v1iX is legal.
8770 // vextract (v1iX extract_subvector(vNiX, Idx)) -> vextract(vNiX,Idx).
8771 // Here we are completely ignoring the extract element index (N2),
8772 // which is fine for fixed width vectors, since any index other than 0
8773 // is undefined anyway. However, this cannot be ignored for scalable
8774 // vectors - in theory we could support this, but we don't want to do this
8775 // without a profitability check.
8776 if (N1.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
8777 N1.getValueType().isFixedLengthVector() &&
8778 N1.getValueType().getVectorNumElements() == 1) {
8779 return getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT, N1: N1.getOperand(i: 0),
8780 N2: N1.getOperand(i: 1));
8781 }
8782 break;
8783 case ISD::EXTRACT_ELEMENT:
8784 assert(N2C && (unsigned)N2C->getZExtValue() < 2 && "Bad EXTRACT_ELEMENT!");
8785 assert(!N1.getValueType().isVector() && !VT.isVector() &&
8786 (N1.getValueType().isInteger() == VT.isInteger()) &&
8787 N1.getValueType() != VT &&
8788 "Wrong types for EXTRACT_ELEMENT!");
8789
8790 // EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding
8791 // 64-bit integers into 32-bit parts. Instead of building the extract of
8792 // the BUILD_PAIR, only to have legalize rip it apart, just do it now.
8793 if (N1.getOpcode() == ISD::BUILD_PAIR)
8794 return N1.getOperand(i: N2C->getZExtValue());
8795
8796 // EXTRACT_ELEMENT of a constant int is also very common.
8797 if (N1C) {
8798 unsigned ElementSize = VT.getSizeInBits();
8799 unsigned Shift = ElementSize * N2C->getZExtValue();
8800 const APInt &Val = N1C->getAPIntValue();
8801 return getConstant(Val: Val.extractBits(numBits: ElementSize, bitPosition: Shift), DL, VT);
8802 }
8803 break;
8804 case ISD::EXTRACT_SUBVECTOR: {
8805 EVT N1VT = N1.getValueType();
8806 assert(VT.isVector() && N1VT.isVector() &&
8807 "Extract subvector VTs must be vectors!");
8808 assert(VT.getVectorElementType() == N1VT.getVectorElementType() &&
8809 "Extract subvector VTs must have the same element type!");
8810 assert((VT.isFixedLengthVector() || N1VT.isScalableVector()) &&
8811 "Cannot extract a scalable vector from a fixed length vector!");
8812 assert((VT.isScalableVector() != N1VT.isScalableVector() ||
8813 VT.getVectorMinNumElements() <= N1VT.getVectorMinNumElements()) &&
8814 "Extract subvector must be from larger vector to smaller vector!");
8815 assert(N2C && "Extract subvector index must be a constant");
8816 assert((VT.isScalableVector() != N1VT.isScalableVector() ||
8817 (VT.getVectorMinNumElements() + N2C->getZExtValue()) <=
8818 N1VT.getVectorMinNumElements()) &&
8819 "Extract subvector overflow!");
8820 assert(N2C->getAPIntValue().getBitWidth() ==
8821 TLI->getVectorIdxWidth(getDataLayout()) &&
8822 "Constant index for EXTRACT_SUBVECTOR has an invalid size");
8823 assert(N2C->getZExtValue() % VT.getVectorMinNumElements() == 0 &&
8824 "Extract index is not a multiple of the output vector length");
8825
8826 // Trivial extraction.
8827 if (VT == N1VT)
8828 return N1;
8829
8830 // EXTRACT_SUBVECTOR of an UNDEF is an UNDEF.
8831 if (N1.isUndef())
8832 return getUNDEF(VT);
8833
8834 // EXTRACT_SUBVECTOR of CONCAT_VECTOR can be simplified if the pieces of
8835 // the concat have the same type as the extract.
8836 if (N1.getOpcode() == ISD::CONCAT_VECTORS &&
8837 VT == N1.getOperand(i: 0).getValueType()) {
8838 unsigned Factor = VT.getVectorMinNumElements();
8839 return N1.getOperand(i: N2C->getZExtValue() / Factor);
8840 }
8841
8842 // EXTRACT_SUBVECTOR of INSERT_SUBVECTOR is often created
8843 // during shuffle legalization.
8844 if (N1.getOpcode() == ISD::INSERT_SUBVECTOR && N2 == N1.getOperand(i: 2) &&
8845 VT == N1.getOperand(i: 1).getValueType())
8846 return N1.getOperand(i: 1);
8847 break;
8848 }
8849 }
8850
8851 if (N1.getOpcode() == ISD::POISON || N2.getOpcode() == ISD::POISON) {
8852 switch (Opcode) {
8853 case ISD::XOR:
8854 case ISD::ADD:
8855 case ISD::PTRADD:
8856 case ISD::SUB:
8857 case ISD::SIGN_EXTEND_INREG:
8858 case ISD::UDIV:
8859 case ISD::SDIV:
8860 case ISD::UREM:
8861 case ISD::SREM:
8862 case ISD::MUL:
8863 case ISD::AND:
8864 case ISD::SSUBSAT:
8865 case ISD::USUBSAT:
8866 case ISD::UMIN:
8867 case ISD::OR:
8868 case ISD::SADDSAT:
8869 case ISD::UADDSAT:
8870 case ISD::UMAX:
8871 case ISD::SMAX:
8872 case ISD::SMIN:
8873 // fold op(arg1, poison) -> poison, fold op(poison, arg2) -> poison.
8874 return N2.getOpcode() == ISD::POISON ? N2 : N1;
8875 }
8876 }
8877
8878 // Canonicalize an UNDEF to the RHS, even over a constant.
8879 if (N1.getOpcode() == ISD::UNDEF && N2.getOpcode() != ISD::UNDEF) {
8880 if (TLI->isCommutativeBinOp(Opcode)) {
8881 std::swap(a&: N1, b&: N2);
8882 } else {
8883 switch (Opcode) {
8884 case ISD::PTRADD:
8885 case ISD::SUB:
8886 // fold op(undef, non_undef_arg2) -> undef.
8887 return N1;
8888 case ISD::SIGN_EXTEND_INREG:
8889 case ISD::UDIV:
8890 case ISD::SDIV:
8891 case ISD::UREM:
8892 case ISD::SREM:
8893 case ISD::SSUBSAT:
8894 case ISD::USUBSAT:
8895 // fold op(undef, non_undef_arg2) -> 0.
8896 return getConstant(Val: 0, DL, VT);
8897 }
8898 }
8899 }
8900
8901 // Fold a bunch of operators when the RHS is undef.
8902 if (N2.getOpcode() == ISD::UNDEF) {
8903 switch (Opcode) {
8904 case ISD::XOR:
8905 if (N1.getOpcode() == ISD::UNDEF)
8906 // Handle undef ^ undef -> 0 special case. This is a common
8907 // idiom (misuse).
8908 return getConstant(Val: 0, DL, VT);
8909 [[fallthrough]];
8910 case ISD::ADD:
8911 case ISD::PTRADD:
8912 case ISD::SUB:
8913 // fold op(arg1, undef) -> undef.
8914 return N2;
8915 case ISD::UDIV:
8916 case ISD::SDIV:
8917 case ISD::UREM:
8918 case ISD::SREM:
8919 // fold op(arg1, undef) -> poison.
8920 return getPOISON(VT);
8921 case ISD::MUL:
8922 case ISD::AND:
8923 case ISD::SSUBSAT:
8924 case ISD::USUBSAT:
8925 case ISD::UMIN:
8926 // fold op(undef, undef) -> undef, fold op(arg1, undef) -> 0.
8927 return N1.getOpcode() == ISD::UNDEF ? N2 : getConstant(Val: 0, DL, VT);
8928 case ISD::OR:
8929 case ISD::SADDSAT:
8930 case ISD::UADDSAT:
8931 case ISD::UMAX:
8932 // fold op(undef, undef) -> undef, fold op(arg1, undef) -> -1.
8933 return N1.getOpcode() == ISD::UNDEF ? N2 : getAllOnesConstant(DL, VT);
8934 case ISD::SMAX:
8935 // fold op(undef, undef) -> undef, fold op(arg1, undef) -> MAX_INT.
8936 return N1.getOpcode() == ISD::UNDEF
8937 ? N2
8938 : getConstant(
8939 Val: APInt::getSignedMaxValue(numBits: VT.getScalarSizeInBits()), DL,
8940 VT);
8941 case ISD::SMIN:
8942 // fold op(undef, undef) -> undef, fold op(arg1, undef) -> MIN_INT.
8943 return N1.getOpcode() == ISD::UNDEF
8944 ? N2
8945 : getConstant(
8946 Val: APInt::getSignedMinValue(numBits: VT.getScalarSizeInBits()), DL,
8947 VT);
8948 }
8949 }
8950
8951 // Perform trivial constant folding.
8952 if (SDValue SV = FoldConstantArithmetic(Opcode, DL, VT, Ops: {N1, N2}, Flags))
8953 return SV;
8954
8955 // Memoize this node if possible.
8956 SDNode *N;
8957 SDVTList VTs = getVTList(VT);
8958 SDValue Ops[] = {N1, N2};
8959 if (VT != MVT::Glue) {
8960 FoldingSetNodeID ID;
8961 AddNodeIDNode(ID, OpC: Opcode, VTList: VTs, OpList: Ops);
8962 FoldingSetInsertToken InsertToken;
8963 if (SDNode *E = lookupNode(ID, DL, InsertToken)) {
8964 E->intersectFlagsWith(Flags);
8965 return SDValue(E, 0);
8966 }
8967
8968 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
8969 N->setFlags(Flags);
8970 createOperands(Node: N, Vals: Ops);
8971 CSEMap.insert(N, Token: InsertToken);
8972 } else {
8973 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
8974 createOperands(Node: N, Vals: Ops);
8975 }
8976
8977 InsertNode(N);
8978 SDValue V = SDValue(N, 0);
8979 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
8980 return V;
8981}
8982
8983SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
8984 SDValue N1, SDValue N2, SDValue N3) {
8985 SDNodeFlags Flags;
8986 if (Inserter)
8987 Flags = Inserter->getFlags();
8988 return getNode(Opcode, DL, VT, N1, N2, N3, Flags);
8989}
8990
8991SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
8992 SDValue N1, SDValue N2, SDValue N3,
8993 const SDNodeFlags Flags) {
8994 assert(N1.getOpcode() != ISD::DELETED_NODE &&
8995 N2.getOpcode() != ISD::DELETED_NODE &&
8996 N3.getOpcode() != ISD::DELETED_NODE &&
8997 "Operand is DELETED_NODE!");
8998 // Perform various simplifications.
8999 switch (Opcode) {
9000 case ISD::BUILD_VECTOR: {
9001 // Attempt to simplify BUILD_VECTOR.
9002 SDValue Ops[] = {N1, N2, N3};
9003 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, DAG&: *this))
9004 return V;
9005 break;
9006 }
9007 case ISD::CONCAT_VECTORS: {
9008 SDValue Ops[] = {N1, N2, N3};
9009 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, DAG&: *this))
9010 return V;
9011 break;
9012 }
9013 case ISD::SETCC: {
9014 assert(VT.isInteger() && "SETCC result type must be an integer!");
9015 assert(N1.getValueType() == N2.getValueType() &&
9016 "SETCC operands must have the same type!");
9017 assert(VT.isVector() == N1.getValueType().isVector() &&
9018 "SETCC type should be vector iff the operand type is vector!");
9019 assert((!VT.isVector() || VT.getVectorElementCount() ==
9020 N1.getValueType().getVectorElementCount()) &&
9021 "SETCC vector element counts must match!");
9022 // Use FoldSetCC to simplify SETCC's.
9023 if (SDValue V =
9024 FoldSetCC(VT, N1, N2, Cond: cast<CondCodeSDNode>(Val&: N3)->get(), dl: DL, Flags))
9025 return V;
9026 break;
9027 }
9028 case ISD::SELECT:
9029 case ISD::VSELECT:
9030 if (SDValue V = simplifySelect(Cond: N1, TVal: N2, FVal: N3))
9031 return V;
9032 break;
9033 case ISD::VECTOR_SHUFFLE:
9034 llvm_unreachable("should use getVectorShuffle constructor!");
9035 case ISD::VECTOR_SPLICE_LEFT:
9036 if (isNullConstant(V: N3))
9037 return N1;
9038 break;
9039 case ISD::VECTOR_SPLICE_RIGHT:
9040 if (isNullConstant(V: N3))
9041 return N2;
9042 break;
9043 case ISD::INSERT_VECTOR_ELT: {
9044 assert(VT.isVector() && VT == N1.getValueType() &&
9045 "INSERT_VECTOR_ELT vector type mismatch");
9046 assert(VT.isFloatingPoint() == N2.getValueType().isFloatingPoint() &&
9047 "INSERT_VECTOR_ELT scalar fp/int mismatch");
9048 assert((!VT.isFloatingPoint() ||
9049 VT.getVectorElementType() == N2.getValueType()) &&
9050 "INSERT_VECTOR_ELT fp scalar type mismatch");
9051 assert((!VT.isInteger() ||
9052 VT.getScalarSizeInBits() <= N2.getScalarValueSizeInBits()) &&
9053 "INSERT_VECTOR_ELT int scalar size mismatch");
9054
9055 auto *N3C = dyn_cast<ConstantSDNode>(Val&: N3);
9056 // INSERT_VECTOR_ELT into out-of-bounds element is an UNDEF, except
9057 // for scalable vectors where we will generate appropriate code to
9058 // deal with out-of-bounds cases correctly.
9059 if (N3C && VT.isFixedLengthVector() &&
9060 N3C->getZExtValue() >= VT.getVectorNumElements())
9061 return getUNDEF(VT);
9062
9063 // Undefined index can be assumed out-of-bounds, so that's UNDEF too.
9064 if (N3.isUndef())
9065 return getUNDEF(VT);
9066
9067 // If inserting poison, just use the input vector.
9068 if (N2.getOpcode() == ISD::POISON)
9069 return N1;
9070
9071 // Inserting undef into undef/poison is still undef.
9072 if (N2.getOpcode() == ISD::UNDEF && N1.isUndef())
9073 return getUNDEF(VT);
9074
9075 // If the inserted element is an UNDEF, just use the input vector.
9076 // But not if skipping the insert could make the result more poisonous.
9077 if (N2.isUndef()) {
9078 if (N3C && VT.isFixedLengthVector()) {
9079 APInt EltMask =
9080 APInt::getOneBitSet(numBits: VT.getVectorNumElements(), BitNo: N3C->getZExtValue());
9081 if (isGuaranteedNotToBePoison(Op: N1, DemandedElts: EltMask))
9082 return N1;
9083 } else if (isGuaranteedNotToBePoison(Op: N1))
9084 return N1;
9085 }
9086 break;
9087 }
9088 case ISD::INSERT_SUBVECTOR: {
9089 // If inserting poison, just use the input vector,
9090 if (N2.getOpcode() == ISD::POISON)
9091 return N1;
9092
9093 // Inserting undef into undef/poison is still undef.
9094 if (N2.getOpcode() == ISD::UNDEF && N1.isUndef())
9095 return getUNDEF(VT);
9096
9097 EVT N2VT = N2.getValueType();
9098 assert(VT == N1.getValueType() &&
9099 "Dest and insert subvector source types must match!");
9100 assert(VT.isVector() && N2VT.isVector() &&
9101 "Insert subvector VTs must be vectors!");
9102 assert(VT.getVectorElementType() == N2VT.getVectorElementType() &&
9103 "Insert subvector VTs must have the same element type!");
9104 assert((VT.isScalableVector() || N2VT.isFixedLengthVector()) &&
9105 "Cannot insert a scalable vector into a fixed length vector!");
9106 assert((VT.isScalableVector() != N2VT.isScalableVector() ||
9107 VT.getVectorMinNumElements() >= N2VT.getVectorMinNumElements()) &&
9108 "Insert subvector must be from smaller vector to larger vector!");
9109 assert(isa<ConstantSDNode>(N3) &&
9110 "Insert subvector index must be constant");
9111 assert((VT.isScalableVector() != N2VT.isScalableVector() ||
9112 (N2VT.getVectorMinNumElements() + N3->getAsZExtVal()) <=
9113 VT.getVectorMinNumElements()) &&
9114 "Insert subvector overflow!");
9115 assert(N3->getAsAPIntVal().getBitWidth() ==
9116 TLI->getVectorIdxWidth(getDataLayout()) &&
9117 "Constant index for INSERT_SUBVECTOR has an invalid size");
9118
9119 // Trivial insertion.
9120 if (VT == N2VT)
9121 return N2;
9122
9123 // If this is an insert of an extracted vector into an undef/poison vector,
9124 // we can just use the input to the extract. But not if skipping the
9125 // extract+insert could make the result more poisonous.
9126 if (N1.isUndef() && N2.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
9127 N2.getOperand(i: 1) == N3 && N2.getOperand(i: 0).getValueType() == VT) {
9128 if (N1.getOpcode() == ISD::POISON)
9129 return N2.getOperand(i: 0);
9130 if (VT.isFixedLengthVector() && N2VT.isFixedLengthVector()) {
9131 unsigned LoBit = N3->getAsZExtVal();
9132 unsigned HiBit = LoBit + N2VT.getVectorNumElements();
9133 APInt EltMask =
9134 APInt::getBitsSet(numBits: VT.getVectorNumElements(), loBit: LoBit, hiBit: HiBit);
9135 if (isGuaranteedNotToBePoison(Op: N2.getOperand(i: 0), DemandedElts: ~EltMask))
9136 return N2.getOperand(i: 0);
9137 } else if (isGuaranteedNotToBePoison(Op: N2.getOperand(i: 0)))
9138 return N2.getOperand(i: 0);
9139 }
9140
9141 // If the inserted subvector is UNDEF, just use the input vector.
9142 // But not if skipping the insert could make the result more poisonous.
9143 if (N2.isUndef()) {
9144 if (VT.isFixedLengthVector()) {
9145 unsigned LoBit = N3->getAsZExtVal();
9146 unsigned HiBit = LoBit + N2VT.getVectorNumElements();
9147 APInt EltMask =
9148 APInt::getBitsSet(numBits: VT.getVectorNumElements(), loBit: LoBit, hiBit: HiBit);
9149 if (isGuaranteedNotToBePoison(Op: N1, DemandedElts: EltMask))
9150 return N1;
9151 } else if (isGuaranteedNotToBePoison(Op: N1))
9152 return N1;
9153 }
9154 break;
9155 }
9156 case ISD::BITCAST:
9157 // Fold bit_convert nodes from a type to themselves.
9158 if (N1.getValueType() == VT)
9159 return N1;
9160 break;
9161 case ISD::VECTOR_COMPRESS: {
9162 [[maybe_unused]] EVT VecVT = N1.getValueType();
9163 [[maybe_unused]] EVT MaskVT = N2.getValueType();
9164 [[maybe_unused]] EVT PassthruVT = N3.getValueType();
9165 assert(VT == VecVT && "Vector and result type don't match.");
9166 assert(VecVT.isVector() && MaskVT.isVector() && PassthruVT.isVector() &&
9167 "All inputs must be vectors.");
9168 assert(VecVT == PassthruVT && "Vector and passthru types don't match.");
9169 assert(VecVT.getVectorElementCount() == MaskVT.getVectorElementCount() &&
9170 "Vector and mask must have same number of elements.");
9171
9172 if (N1.isUndef() || N2.isUndef())
9173 return N3;
9174
9175 break;
9176 }
9177 case ISD::PARTIAL_REDUCE_UMLA:
9178 case ISD::PARTIAL_REDUCE_SMLA:
9179 case ISD::PARTIAL_REDUCE_SUMLA:
9180 case ISD::PARTIAL_REDUCE_FMLA: {
9181 [[maybe_unused]] EVT AccVT = N1.getValueType();
9182 [[maybe_unused]] EVT Input1VT = N2.getValueType();
9183 [[maybe_unused]] EVT Input2VT = N3.getValueType();
9184 assert(Input1VT.isVector() && Input1VT == Input2VT &&
9185 "Expected the second and third operands of the PARTIAL_REDUCE_MLA "
9186 "node to have the same type!");
9187 assert(VT.isVector() && VT == AccVT &&
9188 "Expected the first operand of the PARTIAL_REDUCE_MLA node to have "
9189 "the same type as its result!");
9190 assert(Input1VT.getVectorElementCount().hasKnownScalarFactor(
9191 AccVT.getVectorElementCount()) &&
9192 "Expected the element count of the second and third operands of the "
9193 "PARTIAL_REDUCE_MLA node to be a positive integer multiple of the "
9194 "element count of the first operand and the result!");
9195 assert(N2.getScalarValueSizeInBits() <= N1.getScalarValueSizeInBits() &&
9196 "Expected the second and third operands of the PARTIAL_REDUCE_MLA "
9197 "node to have an element type which is the same as or smaller than "
9198 "the element type of the first operand and result!");
9199 break;
9200 }
9201 }
9202
9203 // Perform trivial constant folding for arithmetic operators.
9204 switch (Opcode) {
9205 case ISD::PARTIAL_REDUCE_SMLA:
9206 case ISD::PARTIAL_REDUCE_UMLA:
9207 case ISD::PARTIAL_REDUCE_SUMLA:
9208 case ISD::FMA:
9209 case ISD::FMAD:
9210 case ISD::SETCC:
9211 case ISD::FSHL:
9212 case ISD::FSHR:
9213 if (SDValue SV =
9214 FoldConstantArithmetic(Opcode, DL, VT, Ops: {N1, N2, N3}, Flags))
9215 return SV;
9216 break;
9217 }
9218
9219 // Memoize node if it doesn't produce a glue result.
9220 SDNode *N;
9221 SDVTList VTs = getVTList(VT);
9222 SDValue Ops[] = {N1, N2, N3};
9223 if (VT != MVT::Glue) {
9224 FoldingSetNodeID ID;
9225 AddNodeIDNode(ID, OpC: Opcode, VTList: VTs, OpList: Ops);
9226 FoldingSetInsertToken InsertToken;
9227 if (SDNode *E = lookupNode(ID, DL, InsertToken)) {
9228 E->intersectFlagsWith(Flags);
9229 return SDValue(E, 0);
9230 }
9231
9232 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
9233 N->setFlags(Flags);
9234 createOperands(Node: N, Vals: Ops);
9235 CSEMap.insert(N, Token: InsertToken);
9236 } else {
9237 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
9238 createOperands(Node: N, Vals: Ops);
9239 }
9240
9241 InsertNode(N);
9242 SDValue V = SDValue(N, 0);
9243 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
9244 return V;
9245}
9246
9247SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
9248 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
9249 const SDNodeFlags Flags) {
9250 SDValue Ops[] = { N1, N2, N3, N4 };
9251 return getNode(Opcode, DL, VT, Ops, Flags);
9252}
9253
9254SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
9255 SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
9256 SDNodeFlags Flags;
9257 if (Inserter)
9258 Flags = Inserter->getFlags();
9259 return getNode(Opcode, DL, VT, N1, N2, N3, N4, Flags);
9260}
9261
9262SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
9263 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
9264 SDValue N5, const SDNodeFlags Flags) {
9265 SDValue Ops[] = { N1, N2, N3, N4, N5 };
9266 return getNode(Opcode, DL, VT, Ops, Flags);
9267}
9268
9269SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
9270 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
9271 SDValue N5) {
9272 SDNodeFlags Flags;
9273 if (Inserter)
9274 Flags = Inserter->getFlags();
9275 return getNode(Opcode, DL, VT, N1, N2, N3, N4, N5, Flags);
9276}
9277
9278/// getStackArgumentTokenFactor - Compute a TokenFactor to force all
9279/// the incoming stack arguments to be loaded from the stack.
9280SDValue SelectionDAG::getStackArgumentTokenFactor(SDValue Chain) {
9281 SmallVector<SDValue, 8> ArgChains;
9282
9283 // Include the original chain at the beginning of the list. When this is
9284 // used by target LowerCall hooks, this helps legalize find the
9285 // CALLSEQ_BEGIN node.
9286 ArgChains.push_back(Elt: Chain);
9287
9288 // Add a chain value for each stack argument.
9289 for (SDNode *U : getEntryNode().getNode()->users())
9290 if (LoadSDNode *L = dyn_cast<LoadSDNode>(Val: U))
9291 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val: L->getBasePtr()))
9292 if (FI->getIndex() < 0)
9293 ArgChains.push_back(Elt: SDValue(L, 1));
9294
9295 // Build a tokenfactor for all the chains.
9296 return getNode(Opcode: ISD::TokenFactor, DL: SDLoc(Chain), VT: MVT::Other, Ops: ArgChains);
9297}
9298
9299/// getMemsetValue - Vectorized representation of the memset value
9300/// operand.
9301static SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG,
9302 const SDLoc &dl) {
9303 assert(!Value.isUndef());
9304
9305 unsigned NumBits = VT.getScalarSizeInBits();
9306 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: Value)) {
9307 assert(C->getAPIntValue().getBitWidth() == 8);
9308 APInt Val = APInt::getSplat(NewLen: NumBits, V: C->getAPIntValue());
9309 if (VT.isInteger()) {
9310 bool IsOpaque = VT.getSizeInBits() > 64 ||
9311 !DAG.getTargetLoweringInfo().isLegalStoreImmediate(Value: C->getSExtValue());
9312 return DAG.getConstant(Val, DL: dl, VT, isT: false, isO: IsOpaque);
9313 }
9314 return DAG.getConstantFP(V: APFloat(VT.getFltSemantics(), Val), DL: dl, VT);
9315 }
9316
9317 assert(Value.getValueType() == MVT::i8 && "memset with non-byte fill value?");
9318 EVT IntVT = VT.getScalarType();
9319 if (!IntVT.isInteger())
9320 IntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: IntVT.getSizeInBits());
9321
9322 Value = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: IntVT, N1: Value);
9323 if (NumBits > 8) {
9324 // Use a multiplication with 0x010101... to extend the input to the
9325 // required length.
9326 APInt Magic = APInt::getSplat(NewLen: NumBits, V: APInt(8, 0x01));
9327 Value = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: IntVT, N1: Value,
9328 N2: DAG.getConstant(Val: Magic, DL: dl, VT: IntVT));
9329 }
9330
9331 if (VT != Value.getValueType() && !VT.isInteger())
9332 Value = DAG.getBitcast(VT: VT.getScalarType(), V: Value);
9333 if (VT != Value.getValueType())
9334 Value = DAG.getSplatBuildVector(VT, DL: dl, Op: Value);
9335
9336 return Value;
9337}
9338
9339/// getMemsetStringVal - Similar to getMemsetValue. Except this is only
9340/// used when a memcpy is turned into a memset when the source is a constant
9341/// string ptr.
9342static SDValue getMemsetStringVal(EVT VT, const SDLoc &dl, SelectionDAG &DAG,
9343 const TargetLowering &TLI,
9344 const ConstantDataArraySlice &Slice) {
9345 // Handle vector with all elements zero.
9346 if (Slice.Array == nullptr) {
9347 if (VT.isInteger())
9348 return DAG.getConstant(Val: 0, DL: dl, VT);
9349 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT,
9350 N1: DAG.getConstant(Val: 0, DL: dl, VT: VT.changeTypeToInteger()));
9351 }
9352
9353 assert(!VT.isVector() && "Can't handle vector type here!");
9354 unsigned NumVTBits = VT.getSizeInBits();
9355 unsigned NumVTBytes = NumVTBits / 8;
9356 unsigned NumBytes = std::min(a: NumVTBytes, b: unsigned(Slice.Length));
9357
9358 APInt Val(NumVTBits, 0);
9359 if (DAG.getDataLayout().isLittleEndian()) {
9360 for (unsigned i = 0; i != NumBytes; ++i)
9361 Val |= (uint64_t)(unsigned char)Slice[i] << i*8;
9362 } else {
9363 for (unsigned i = 0; i != NumBytes; ++i)
9364 Val |= (uint64_t)(unsigned char)Slice[i] << (NumVTBytes-i-1)*8;
9365 }
9366
9367 // If the "cost" of materializing the integer immediate is less than the cost
9368 // of a load, then it is cost effective to turn the load into the immediate.
9369 Type *Ty = VT.getTypeForEVT(Context&: *DAG.getContext());
9370 if (TLI.shouldConvertConstantLoadToIntImm(Imm: Val, Ty))
9371 return DAG.getConstant(Val, DL: dl, VT);
9372 return SDValue();
9373}
9374
9375SDValue SelectionDAG::getMemBasePlusOffset(SDValue Base, TypeSize Offset,
9376 const SDLoc &DL,
9377 const SDNodeFlags Flags) {
9378 SDValue Index = getTypeSize(DL, VT: Base.getValueType(), TS: Offset);
9379 return getMemBasePlusOffset(Base, Offset: Index, DL, Flags);
9380}
9381
9382SDValue SelectionDAG::getMemBasePlusOffset(SDValue Ptr, SDValue Offset,
9383 const SDLoc &DL,
9384 const SDNodeFlags Flags) {
9385 assert(Offset.getValueType().isInteger());
9386 EVT BasePtrVT = Ptr.getValueType();
9387 if (TLI->shouldPreservePtrArith(F: this->getMachineFunction().getFunction(),
9388 PtrVT: BasePtrVT))
9389 return getNode(Opcode: ISD::PTRADD, DL, VT: BasePtrVT, N1: Ptr, N2: Offset, Flags);
9390 // InBounds only applies to PTRADD, don't set it if we generate ADD.
9391 SDNodeFlags AddFlags = Flags;
9392 AddFlags.setInBounds(false);
9393 return getNode(Opcode: ISD::ADD, DL, VT: BasePtrVT, N1: Ptr, N2: Offset, Flags: AddFlags);
9394}
9395
9396/// Returns true if memcpy source is constant data.
9397static bool isMemSrcFromConstant(SDValue Src, ConstantDataArraySlice &Slice) {
9398 uint64_t SrcDelta = 0;
9399 GlobalAddressSDNode *G = nullptr;
9400 if (Src.getOpcode() == ISD::GlobalAddress)
9401 G = cast<GlobalAddressSDNode>(Val&: Src);
9402 else if (Src->isAnyAdd() &&
9403 Src.getOperand(i: 0).getOpcode() == ISD::GlobalAddress &&
9404 Src.getOperand(i: 1).getOpcode() == ISD::Constant) {
9405 G = cast<GlobalAddressSDNode>(Val: Src.getOperand(i: 0));
9406 SrcDelta = Src.getConstantOperandVal(i: 1);
9407 }
9408 if (!G)
9409 return false;
9410
9411 return getConstantDataArrayInfo(V: G->getGlobal(), Slice, ElementSize: 8,
9412 Offset: SrcDelta + G->getOffset());
9413}
9414
9415static bool shouldLowerMemFuncForSize(const MachineFunction &MF,
9416 SelectionDAG &DAG) {
9417 // On Darwin, -Os means optimize for size without hurting performance, so
9418 // only really optimize for size when -Oz (MinSize) is used.
9419 if (MF.getTarget().getTargetTriple().isOSDarwin())
9420 return MF.getFunction().hasMinSize();
9421 return DAG.shouldOptForSize();
9422}
9423
9424static void chainLoadsAndStoresForMemcpy(SelectionDAG &DAG, const SDLoc &dl,
9425 SmallVector<SDValue, 32> &OutChains, unsigned From,
9426 unsigned To, SmallVector<SDValue, 16> &OutLoadChains,
9427 SmallVector<SDValue, 16> &OutStoreChains) {
9428 assert(OutLoadChains.size() && "Missing loads in memcpy inlining");
9429 assert(OutStoreChains.size() && "Missing stores in memcpy inlining");
9430 SmallVector<SDValue, 16> GluedLoadChains;
9431 for (unsigned i = From; i < To; ++i) {
9432 OutChains.push_back(Elt: OutLoadChains[i]);
9433 GluedLoadChains.push_back(Elt: OutLoadChains[i]);
9434 }
9435
9436 // Chain for all loads.
9437 SDValue LoadToken = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other,
9438 Ops: GluedLoadChains);
9439
9440 for (unsigned i = From; i < To; ++i) {
9441 StoreSDNode *ST = dyn_cast<StoreSDNode>(Val&: OutStoreChains[i]);
9442 SDValue NewStore = DAG.getTruncStore(Chain: LoadToken, dl, Val: ST->getValue(),
9443 Ptr: ST->getBasePtr(), SVT: ST->getMemoryVT(),
9444 MMO: ST->getMemOperand());
9445 OutChains.push_back(Elt: NewStore);
9446 }
9447}
9448
9449static SDValue
9450getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain,
9451 SDValue Dst, SDValue Src, uint64_t Size, Align DstAlign,
9452 Align SrcAlign, bool isVol, bool AlwaysInline,
9453 MachinePointerInfo DstPtrInfo,
9454 MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo,
9455 BatchAAResults *BatchAA, const MDNode *DstMemCacheHint,
9456 const MDNode *SrcMemCacheHint) {
9457 // Turn a memcpy of undef to nop.
9458 // FIXME: We need to honor volatile even is Src is undef.
9459 if (Src.isUndef())
9460 return Chain;
9461
9462 // Expand memcpy to a series of load and store ops if the size operand falls
9463 // below a certain threshold.
9464 // TODO: In the AlwaysInline case, if the size is big then generate a loop
9465 // rather than maybe a humongous number of loads and stores.
9466 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9467 const DataLayout &DL = DAG.getDataLayout();
9468 LLVMContext &C = *DAG.getContext();
9469 std::vector<EVT> MemOps;
9470 bool DstAlignCanChange = false;
9471 MachineFunction &MF = DAG.getMachineFunction();
9472 MachineFrameInfo &MFI = MF.getFrameInfo();
9473 bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
9474 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Dst);
9475 if (FI && !MFI.isFixedObjectIndex(ObjectIdx: FI->getIndex()))
9476 DstAlignCanChange = true;
9477 SrcAlign = std::max(a: SrcAlign, b: DAG.InferPtrAlign(Ptr: Src).valueOrOne());
9478 ConstantDataArraySlice Slice;
9479 // If marked as volatile, perform a copy even when marked as constant.
9480 bool CopyFromConstant = !isVol && isMemSrcFromConstant(Src, Slice);
9481 bool isZeroConstant = CopyFromConstant && Slice.Array == nullptr;
9482 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(OptSize);
9483 const MemOp Op = isZeroConstant
9484 ? MemOp::Set(Size, DstAlignCanChange, DstAlign,
9485 /*IsZeroMemset*/ true, IsVolatile: isVol)
9486 : MemOp::Copy(Size, DstAlignCanChange, DstAlign,
9487 SrcAlign, IsVolatile: isVol, MemcpyStrSrc: CopyFromConstant);
9488 if (!TLI.findOptimalMemOpLowering(
9489 Context&: C, MemOps, Limit, Op, DstAS: DstPtrInfo.getAddrSpace(),
9490 SrcAS: SrcPtrInfo.getAddrSpace(), FuncAttributes: MF.getFunction().getAttributes(), LargestVT: nullptr))
9491 return SDValue();
9492
9493 if (DstAlignCanChange) {
9494 Type *Ty = MemOps[0].getTypeForEVT(Context&: C);
9495 Align NewDstAlign = DL.getABITypeAlign(Ty);
9496
9497 // Don't promote to an alignment that would require dynamic stack
9498 // realignment which may conflict with optimizations such as tail call
9499 // optimization.
9500 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
9501 if (!TRI->hasStackRealignment(MF))
9502 if (MaybeAlign StackAlign = DL.getStackAlignment())
9503 NewDstAlign = std::min(a: NewDstAlign, b: *StackAlign);
9504
9505 if (NewDstAlign > DstAlign) {
9506 // Give the stack frame object a larger alignment if needed.
9507 if (MFI.getObjectAlign(ObjectIdx: FI->getIndex()) < NewDstAlign)
9508 MFI.setObjectAlignment(ObjectIdx: FI->getIndex(), Alignment: NewDstAlign);
9509 DstAlign = NewDstAlign;
9510 }
9511 }
9512
9513 // Prepare AAInfo for loads/stores after lowering this memcpy.
9514 AAMDNodes NewAAInfo = AAInfo;
9515 NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
9516
9517 const Value *SrcVal = dyn_cast_if_present<const Value *>(Val&: SrcPtrInfo.V);
9518 bool isConstant =
9519 BatchAA && SrcVal &&
9520 BatchAA->pointsToConstantMemory(Loc: MemoryLocation(SrcVal, Size, AAInfo));
9521
9522 MachineMemOperand::Flags MMOFlags =
9523 isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone;
9524 SmallVector<SDValue, 16> OutLoadChains;
9525 SmallVector<SDValue, 16> OutStoreChains;
9526 SmallVector<SDValue, 32> OutChains;
9527 unsigned NumMemOps = MemOps.size();
9528 uint64_t SrcOff = 0, DstOff = 0;
9529 for (unsigned i = 0; i != NumMemOps; ++i) {
9530 EVT VT = MemOps[i];
9531 unsigned VTSize = VT.getSizeInBits() / 8;
9532 SDValue Value, Store;
9533
9534 if (VTSize > Size) {
9535 // Issuing an unaligned load / store pair that overlaps with the previous
9536 // pair. Adjust the offset accordingly.
9537 assert(i == NumMemOps-1 && i != 0);
9538 SrcOff -= VTSize - Size;
9539 DstOff -= VTSize - Size;
9540 }
9541
9542 if (CopyFromConstant &&
9543 (isZeroConstant || (VT.isInteger() && !VT.isVector()))) {
9544 // It's unlikely a store of a vector immediate can be done in a single
9545 // instruction. It would require a load from a constantpool first.
9546 // We only handle zero vectors here.
9547 // FIXME: Handle other cases where store of vector immediate is done in
9548 // a single instruction.
9549 ConstantDataArraySlice SubSlice;
9550 if (SrcOff < Slice.Length) {
9551 SubSlice = Slice;
9552 SubSlice.move(Delta: SrcOff);
9553 } else {
9554 // This is an out-of-bounds access and hence UB. Pretend we read zero.
9555 SubSlice.Array = nullptr;
9556 SubSlice.Offset = 0;
9557 SubSlice.Length = VTSize;
9558 }
9559 Value = getMemsetStringVal(VT, dl, DAG, TLI, Slice: SubSlice);
9560 if (Value.getNode()) {
9561 Store = DAG.getStore(
9562 Chain, dl, Val: Value,
9563 Ptr: DAG.getObjectPtrOffset(SL: dl, Ptr: Dst, Offset: TypeSize::getFixed(ExactSize: DstOff)),
9564 PtrInfo: DstPtrInfo.getWithOffset(O: DstOff), Alignment: DstAlign, MMOFlags,
9565 Metadata: MMOMetadata(NewAAInfo, /*Ranges=*/nullptr, DstMemCacheHint));
9566 OutChains.push_back(Elt: Store);
9567 }
9568 }
9569
9570 if (!Store.getNode()) {
9571 // The type might not be legal for the target. This should only happen
9572 // if the type is smaller than a legal type, as on PPC, so the right
9573 // thing to do is generate a LoadExt/StoreTrunc pair. These simplify
9574 // to Load/Store if NVT==VT.
9575 // FIXME does the case above also need this?
9576 EVT NVT = TLI.getTypeToTransformTo(Context&: C, VT);
9577 assert(NVT.bitsGE(VT));
9578
9579 bool isDereferenceable =
9580 SrcPtrInfo.getWithOffset(O: SrcOff).isDereferenceable(Size: VTSize, C, DL);
9581 MachineMemOperand::Flags SrcMMOFlags = MMOFlags;
9582 if (isDereferenceable)
9583 SrcMMOFlags |= MachineMemOperand::MODereferenceable;
9584 if (isConstant)
9585 SrcMMOFlags |= MachineMemOperand::MOInvariant;
9586
9587 Value = DAG.getExtLoad(
9588 ExtType: ISD::EXTLOAD, dl, VT: NVT, Chain,
9589 Ptr: DAG.getObjectPtrOffset(SL: dl, Ptr: Src, Offset: TypeSize::getFixed(ExactSize: SrcOff)),
9590 PtrInfo: SrcPtrInfo.getWithOffset(O: SrcOff), MemVT: VT,
9591 Alignment: commonAlignment(A: SrcAlign, Offset: SrcOff), MMOFlags: SrcMMOFlags,
9592 Metadata: MMOMetadata(NewAAInfo, /*Ranges=*/nullptr, SrcMemCacheHint));
9593 OutLoadChains.push_back(Elt: Value.getValue(R: 1));
9594
9595 Store = DAG.getTruncStore(
9596 Chain, dl, Val: Value,
9597 Ptr: DAG.getObjectPtrOffset(SL: dl, Ptr: Dst, Offset: TypeSize::getFixed(ExactSize: DstOff)),
9598 PtrInfo: DstPtrInfo.getWithOffset(O: DstOff), SVT: VT, Alignment: DstAlign, MMOFlags,
9599 Metadata: MMOMetadata(NewAAInfo, /*Ranges=*/nullptr, DstMemCacheHint));
9600 OutStoreChains.push_back(Elt: Store);
9601 }
9602 SrcOff += VTSize;
9603 DstOff += VTSize;
9604 Size -= VTSize;
9605 }
9606
9607 unsigned GluedLdStLimit = MaxLdStGlue == 0 ?
9608 TLI.getMaxGluedStoresPerMemcpy() : MaxLdStGlue;
9609 unsigned NumLdStInMemcpy = OutStoreChains.size();
9610
9611 if (NumLdStInMemcpy) {
9612 // It may be that memcpy might be converted to memset if it's memcpy
9613 // of constants. In such a case, we won't have loads and stores, but
9614 // just stores. In the absence of loads, there is nothing to gang up.
9615 if ((GluedLdStLimit <= 1) || !EnableMemCpyDAGOpt) {
9616 // If target does not care, just leave as it.
9617 for (unsigned i = 0; i < NumLdStInMemcpy; ++i) {
9618 OutChains.push_back(Elt: OutLoadChains[i]);
9619 OutChains.push_back(Elt: OutStoreChains[i]);
9620 }
9621 } else {
9622 // Ld/St less than/equal limit set by target.
9623 if (NumLdStInMemcpy <= GluedLdStLimit) {
9624 chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, From: 0,
9625 To: NumLdStInMemcpy, OutLoadChains,
9626 OutStoreChains);
9627 } else {
9628 unsigned NumberLdChain = NumLdStInMemcpy / GluedLdStLimit;
9629 unsigned RemainingLdStInMemcpy = NumLdStInMemcpy % GluedLdStLimit;
9630 unsigned GlueIter = 0;
9631
9632 // Residual ld/st.
9633 if (RemainingLdStInMemcpy) {
9634 chainLoadsAndStoresForMemcpy(
9635 DAG, dl, OutChains, From: NumLdStInMemcpy - RemainingLdStInMemcpy,
9636 To: NumLdStInMemcpy, OutLoadChains, OutStoreChains);
9637 }
9638
9639 for (unsigned cnt = 0; cnt < NumberLdChain; ++cnt) {
9640 unsigned IndexFrom = NumLdStInMemcpy - RemainingLdStInMemcpy -
9641 GlueIter - GluedLdStLimit;
9642 unsigned IndexTo = NumLdStInMemcpy - RemainingLdStInMemcpy - GlueIter;
9643 chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, From: IndexFrom, To: IndexTo,
9644 OutLoadChains, OutStoreChains);
9645 GlueIter += GluedLdStLimit;
9646 }
9647 }
9648 }
9649 }
9650 return DAG.getTokenFactor(DL: dl, Vals&: OutChains);
9651}
9652
9653static SDValue getMemmoveLoadsAndStores(
9654 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
9655 uint64_t Size, Align DstAlign, Align SrcAlign, bool isVol,
9656 bool AlwaysInline, MachinePointerInfo DstPtrInfo,
9657 MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo) {
9658 // Turn a memmove of undef to nop.
9659 // FIXME: We need to honor volatile even is Src is undef.
9660 if (Src.isUndef())
9661 return Chain;
9662
9663 // Expand memmove to a series of load and store ops if the size operand falls
9664 // below a certain threshold.
9665 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9666 const DataLayout &DL = DAG.getDataLayout();
9667 LLVMContext &C = *DAG.getContext();
9668 std::vector<EVT> MemOps;
9669 bool DstAlignCanChange = false;
9670 MachineFunction &MF = DAG.getMachineFunction();
9671 MachineFrameInfo &MFI = MF.getFrameInfo();
9672 bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
9673 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Dst);
9674 if (FI && !MFI.isFixedObjectIndex(ObjectIdx: FI->getIndex()))
9675 DstAlignCanChange = true;
9676 SrcAlign = std::max(a: SrcAlign, b: DAG.InferPtrAlign(Ptr: Src).valueOrOne());
9677 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize);
9678 if (!TLI.findOptimalMemOpLowering(
9679 Context&: C, MemOps, Limit,
9680 Op: MemOp::Move(Size, DstAlignCanChange, DstAlign, SrcAlign, IsVolatile: isVol),
9681 DstAS: DstPtrInfo.getAddrSpace(), SrcAS: SrcPtrInfo.getAddrSpace(),
9682 FuncAttributes: MF.getFunction().getAttributes(), LargestVT: nullptr))
9683 return SDValue();
9684
9685 if (DstAlignCanChange) {
9686 Type *Ty = MemOps[0].getTypeForEVT(Context&: C);
9687 Align NewDstAlign = DL.getABITypeAlign(Ty);
9688
9689 // Don't promote to an alignment that would require dynamic stack
9690 // realignment which may conflict with optimizations such as tail call
9691 // optimization.
9692 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
9693 if (!TRI->hasStackRealignment(MF))
9694 if (MaybeAlign StackAlign = DL.getStackAlignment())
9695 NewDstAlign = std::min(a: NewDstAlign, b: *StackAlign);
9696
9697 if (NewDstAlign > DstAlign) {
9698 // Give the stack frame object a larger alignment if needed.
9699 if (MFI.getObjectAlign(ObjectIdx: FI->getIndex()) < NewDstAlign)
9700 MFI.setObjectAlignment(ObjectIdx: FI->getIndex(), Alignment: NewDstAlign);
9701 DstAlign = NewDstAlign;
9702 }
9703 }
9704
9705 // Prepare AAInfo for loads/stores after lowering this memmove.
9706 AAMDNodes NewAAInfo = AAInfo;
9707 NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
9708
9709 MachineMemOperand::Flags MMOFlags =
9710 isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone;
9711 uint64_t SrcOff = 0;
9712 SmallVector<SDValue, 8> LoadValues;
9713 SmallVector<SDValue, 8> LoadChains;
9714 SmallVector<SDValue, 8> OutChains;
9715 unsigned NumMemOps = MemOps.size();
9716 for (unsigned i = 0; i < NumMemOps; i++) {
9717 EVT VT = MemOps[i];
9718 unsigned VTSize = VT.getSizeInBits() / 8;
9719 SDValue Value;
9720 bool IsOverlapping = false;
9721
9722 if (i == NumMemOps - 1 && i != 0 && VTSize > Size - SrcOff) {
9723 // Issuing an unaligned load / store pair that overlaps with the previous
9724 // pair. Adjust the offset accordingly.
9725 SrcOff = Size - VTSize;
9726 IsOverlapping = true;
9727 }
9728
9729 // Calculate the actual alignment at the current offset. The alignment at
9730 // SrcOff may be lower than the base alignment, especially when using
9731 // overlapping loads.
9732 Align SrcAlignAtOffset = commonAlignment(A: SrcAlign, Offset: SrcOff);
9733 if (IsOverlapping) {
9734 // Verify that the target allows misaligned memory accesses at the
9735 // adjusted offset when using overlapping loads.
9736 unsigned Fast;
9737 if (!TLI.allowsMisalignedMemoryAccesses(VT, AddrSpace: SrcPtrInfo.getAddrSpace(),
9738 Alignment: SrcAlignAtOffset, Flags: MMOFlags,
9739 &Fast) ||
9740 !Fast) {
9741 // This should have been caught by findOptimalMemOpLowering, but verify
9742 // here for safety.
9743 return SDValue();
9744 }
9745 }
9746
9747 bool isDereferenceable =
9748 SrcPtrInfo.getWithOffset(O: SrcOff).isDereferenceable(Size: VTSize, C, DL);
9749 MachineMemOperand::Flags SrcMMOFlags = MMOFlags;
9750 if (isDereferenceable)
9751 SrcMMOFlags |= MachineMemOperand::MODereferenceable;
9752 Value =
9753 DAG.getLoad(VT, dl, Chain,
9754 Ptr: DAG.getObjectPtrOffset(SL: dl, Ptr: Src, Offset: TypeSize::getFixed(ExactSize: SrcOff)),
9755 PtrInfo: SrcPtrInfo.getWithOffset(O: SrcOff), Alignment: SrcAlignAtOffset,
9756 MMOFlags: SrcMMOFlags, Metadata: NewAAInfo);
9757 LoadValues.push_back(Elt: Value);
9758 LoadChains.push_back(Elt: Value.getValue(R: 1));
9759 SrcOff += VTSize;
9760 }
9761 Chain = DAG.getTokenFactor(DL: dl, Vals&: LoadChains);
9762 OutChains.clear();
9763 uint64_t DstOff = 0;
9764 for (unsigned i = 0; i < NumMemOps; i++) {
9765 EVT VT = MemOps[i];
9766 unsigned VTSize = VT.getSizeInBits() / 8;
9767 SDValue Store;
9768 bool IsOverlapping = false;
9769
9770 if (i == NumMemOps - 1 && i != 0 && VTSize > Size - DstOff) {
9771 // Issuing an unaligned load / store pair that overlaps with the previous
9772 // pair. Adjust the offset accordingly.
9773 DstOff = Size - VTSize;
9774 IsOverlapping = true;
9775 }
9776
9777 // Calculate the actual alignment at the current offset. The alignment at
9778 // DstOff may be lower than the base alignment, especially when using
9779 // overlapping stores.
9780 Align DstAlignAtOffset = commonAlignment(A: DstAlign, Offset: DstOff);
9781 if (IsOverlapping) {
9782 // Verify that the target allows misaligned memory accesses at the
9783 // adjusted offset when using overlapping stores.
9784 unsigned Fast;
9785 if (!TLI.allowsMisalignedMemoryAccesses(VT, AddrSpace: DstPtrInfo.getAddrSpace(),
9786 Alignment: DstAlignAtOffset, Flags: MMOFlags,
9787 &Fast) ||
9788 !Fast) {
9789 // This should have been caught by findOptimalMemOpLowering, but verify
9790 // here for safety.
9791 return SDValue();
9792 }
9793 }
9794 Store = DAG.getStore(
9795 Chain, dl, Val: LoadValues[i],
9796 Ptr: DAG.getObjectPtrOffset(SL: dl, Ptr: Dst, Offset: TypeSize::getFixed(ExactSize: DstOff)),
9797 PtrInfo: DstPtrInfo.getWithOffset(O: DstOff), Alignment: DstAlignAtOffset, MMOFlags,
9798 Metadata: NewAAInfo);
9799 OutChains.push_back(Elt: Store);
9800 DstOff += VTSize;
9801 }
9802
9803 return DAG.getTokenFactor(DL: dl, Vals&: OutChains);
9804}
9805
9806/// Lower the call to 'memset' intrinsic function into a series of store
9807/// operations.
9808///
9809/// \param DAG Selection DAG where lowered code is placed.
9810/// \param dl Link to corresponding IR location.
9811/// \param Chain Control flow dependency.
9812/// \param Dst Pointer to destination memory location.
9813/// \param Src Value of byte to write into the memory.
9814/// \param Size Number of bytes to write.
9815/// \param Alignment Alignment of the destination in bytes.
9816/// \param isVol True if destination is volatile.
9817/// \param AlwaysInline Makes sure no function call is generated.
9818/// \param DstPtrInfo IR information on the memory pointer.
9819/// \returns New head in the control flow, if lowering was successful, empty
9820/// SDValue otherwise.
9821///
9822/// The function tries to replace 'llvm.memset' intrinsic with several store
9823/// operations and value calculation code. This is usually profitable for small
9824/// memory size or when the semantic requires inlining.
9825static SDValue getMemsetStores(SelectionDAG &DAG, const SDLoc &dl,
9826 SDValue Chain, SDValue Dst, SDValue Src,
9827 uint64_t Size, Align Alignment, bool isVol,
9828 bool AlwaysInline, MachinePointerInfo DstPtrInfo,
9829 const AAMDNodes &AAInfo) {
9830 // Turn a memset of undef to nop.
9831 // FIXME: We need to honor volatile even is Src is undef.
9832 if (Src.isUndef())
9833 return Chain;
9834
9835 // Expand memset to a series of load/store ops if the size operand
9836 // falls below a certain threshold.
9837 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9838 std::vector<EVT> MemOps;
9839 bool DstAlignCanChange = false;
9840 LLVMContext &C = *DAG.getContext();
9841 MachineFunction &MF = DAG.getMachineFunction();
9842 MachineFrameInfo &MFI = MF.getFrameInfo();
9843 bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
9844 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Dst);
9845 if (FI && !MFI.isFixedObjectIndex(ObjectIdx: FI->getIndex()))
9846 DstAlignCanChange = true;
9847 bool IsZeroVal = isNullConstant(V: Src);
9848 unsigned Limit = AlwaysInline ? ~0 : TLI.getMaxStoresPerMemset(OptSize);
9849
9850 EVT LargestVT;
9851 if (!TLI.findOptimalMemOpLowering(
9852 Context&: C, MemOps, Limit,
9853 Op: MemOp::Set(Size, DstAlignCanChange, DstAlign: Alignment, IsZeroMemset: IsZeroVal, IsVolatile: isVol),
9854 DstAS: DstPtrInfo.getAddrSpace(), SrcAS: ~0u, FuncAttributes: MF.getFunction().getAttributes(),
9855 LargestVT: &LargestVT))
9856 return SDValue();
9857
9858 if (DstAlignCanChange) {
9859 Type *Ty = MemOps[0].getTypeForEVT(Context&: *DAG.getContext());
9860 const DataLayout &DL = DAG.getDataLayout();
9861 Align NewAlign = DL.getABITypeAlign(Ty);
9862
9863 // Don't promote to an alignment that would require dynamic stack
9864 // realignment which may conflict with optimizations such as tail call
9865 // optimization.
9866 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
9867 if (!TRI->hasStackRealignment(MF))
9868 if (MaybeAlign StackAlign = DL.getStackAlignment())
9869 NewAlign = std::min(a: NewAlign, b: *StackAlign);
9870
9871 if (NewAlign > Alignment) {
9872 // Give the stack frame object a larger alignment if needed.
9873 if (MFI.getObjectAlign(ObjectIdx: FI->getIndex()) < NewAlign)
9874 MFI.setObjectAlignment(ObjectIdx: FI->getIndex(), Alignment: NewAlign);
9875 Alignment = NewAlign;
9876 }
9877 }
9878
9879 SmallVector<SDValue, 8> OutChains;
9880 uint64_t DstOff = 0;
9881 unsigned NumMemOps = MemOps.size();
9882
9883 // Find the largest store and generate the bit pattern for it.
9884 // If target didn't set LargestVT, compute it from MemOps.
9885 if (!LargestVT.isSimple()) {
9886 LargestVT = MemOps[0];
9887 for (unsigned i = 1; i < NumMemOps; i++)
9888 if (MemOps[i].bitsGT(VT: LargestVT))
9889 LargestVT = MemOps[i];
9890 }
9891 SDValue MemSetValue = getMemsetValue(Value: Src, VT: LargestVT, DAG, dl);
9892
9893 // Prepare AAInfo for loads/stores after lowering this memset.
9894 AAMDNodes NewAAInfo = AAInfo;
9895 NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
9896
9897 for (unsigned i = 0; i < NumMemOps; i++) {
9898 EVT VT = MemOps[i];
9899 unsigned VTSize = VT.getSizeInBits() / 8;
9900 // The target should specify store types that exactly cover the memset size
9901 // (with the last store potentially being oversized for overlapping stores).
9902 assert(Size > 0 && "Target specified more stores than needed in "
9903 "findOptimalMemOpLowering");
9904 if (VTSize > Size) {
9905 // Issuing an unaligned load / store pair that overlaps with the previous
9906 // pair. Adjust the offset accordingly.
9907 assert(i == NumMemOps-1 && i != 0);
9908 DstOff -= VTSize - Size;
9909 }
9910
9911 // If this store is smaller than the largest store see whether we can get
9912 // the smaller value for free with a truncate or extract vector element and
9913 // then store.
9914 SDValue Value = MemSetValue;
9915 if (VT.bitsLT(VT: LargestVT)) {
9916 unsigned Index;
9917 unsigned NElts = LargestVT.getSizeInBits() / VT.getSizeInBits();
9918 EVT SVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: VT.getScalarType(), NumElements: NElts);
9919 if (!LargestVT.isVector() && !VT.isVector() &&
9920 TLI.isTruncateFree(FromVT: LargestVT, ToVT: VT))
9921 Value = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, N1: MemSetValue);
9922 else if (LargestVT.isVector() && !VT.isVector() &&
9923 TLI.shallExtractConstSplatVectorElementToStore(
9924 VectorTy: LargestVT.getTypeForEVT(Context&: *DAG.getContext()),
9925 ElemSizeInBits: VT.getSizeInBits(), Index) &&
9926 TLI.isTypeLegal(VT: SVT) &&
9927 LargestVT.getSizeInBits() == SVT.getSizeInBits()) {
9928 // Target which can combine store(extractelement VectorTy, Idx) can get
9929 // the smaller value for free.
9930 SDValue TailValue = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: SVT, N1: MemSetValue);
9931 Value = DAG.getExtractVectorElt(DL: dl, VT, Vec: TailValue, Idx: Index);
9932 } else
9933 Value = getMemsetValue(Value: Src, VT, DAG, dl);
9934 }
9935 assert(Value.getValueType() == VT && "Value with wrong type.");
9936 SDValue Store = DAG.getStore(
9937 Chain, dl, Val: Value,
9938 Ptr: DAG.getObjectPtrOffset(SL: dl, Ptr: Dst, Offset: TypeSize::getFixed(ExactSize: DstOff)),
9939 PtrInfo: DstPtrInfo.getWithOffset(O: DstOff), Alignment,
9940 MMOFlags: isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone,
9941 Metadata: NewAAInfo);
9942 OutChains.push_back(Elt: Store);
9943 DstOff += VT.getSizeInBits() / 8;
9944 // For oversized overlapping stores, only subtract the remaining bytes.
9945 // For normal stores, subtract the full store size.
9946 if (VTSize > Size) {
9947 Size = 0;
9948 } else {
9949 Size -= VTSize;
9950 }
9951 }
9952
9953 // After processing all stores, Size should be exactly 0. Any remaining bytes
9954 // indicate a bug in the target's findOptimalMemOpLowering implementation.
9955 assert(Size == 0 && "Target's findOptimalMemOpLowering did not specify "
9956 "stores that exactly cover the memset size");
9957
9958 return DAG.getTokenFactor(DL: dl, Vals&: OutChains);
9959}
9960
9961static void checkAddrSpaceIsValidForLibcall(const TargetLowering *TLI,
9962 unsigned AS) {
9963 // Lowering memcpy / memset / memmove intrinsics to calls is only valid if all
9964 // pointer operands can be losslessly bitcasted to pointers of address space 0
9965 if (AS != 0 && !TLI->getTargetMachine().isNoopAddrSpaceCast(SrcAS: AS, DestAS: 0)) {
9966 report_fatal_error(reason: "cannot lower memory intrinsic in address space " +
9967 Twine(AS));
9968 }
9969}
9970
9971static bool isInTailCallPositionWrapper(const CallInst *CI,
9972 const SelectionDAG *SelDAG,
9973 bool AllowReturnsFirstArg) {
9974 if (!CI || !CI->isTailCall())
9975 return false;
9976 // TODO: Fix "returns-first-arg" determination so it doesn't depend on which
9977 // helper symbol we lower to.
9978 return isInTailCallPosition(Call: *CI, TM: SelDAG->getTarget(),
9979 ReturnsFirstArg: AllowReturnsFirstArg &&
9980 funcReturnsFirstArgOfCall(CI: *CI));
9981}
9982
9983static std::pair<SDValue, SDValue>
9984getRuntimeCallSDValueHelper(SDValue Chain, const SDLoc &dl,
9985 TargetLowering::ArgListTy &&Args,
9986 const CallInst *CI, RTLIB::Libcall Call,
9987 SelectionDAG *DAG, const TargetLowering *TLI) {
9988 RTLIB::LibcallImpl LCImpl = DAG->getLibcalls().getLibcallImpl(Call);
9989
9990 if (LCImpl == RTLIB::Unsupported)
9991 return {};
9992
9993 TargetLowering::CallLoweringInfo CLI(*DAG);
9994 bool IsTailCall =
9995 isInTailCallPositionWrapper(CI, SelDAG: DAG, /*AllowReturnsFirstArg=*/true) &&
9996 // Lowering doesn't support tail calling inside a function with
9997 // a swifterror argument yet.
9998 !DAG->hasSwiftErrorArg();
9999 SDValue Callee =
10000 DAG->getExternalSymbol(Libcall: LCImpl, VT: TLI->getPointerTy(DL: DAG->getDataLayout()));
10001
10002 CLI.setDebugLoc(dl)
10003 .setChain(Chain)
10004 .setLibCallee(CC: DAG->getLibcalls().getLibcallImplCallingConv(Call: LCImpl),
10005 ResultType: CI->getType(), Target: Callee, ArgsList: std::move(Args))
10006 .setTailCall(IsTailCall);
10007
10008 return TLI->LowerCallTo(CLI);
10009}
10010
10011std::pair<SDValue, SDValue> SelectionDAG::getStrcmp(SDValue Chain,
10012 const SDLoc &dl, SDValue S1,
10013 SDValue S2,
10014 const CallInst *CI) {
10015 PointerType *PT = PointerType::getUnqual(C&: *getContext());
10016 TargetLowering::ArgListTy Args = {{S1, PT}, {S2, PT}};
10017 return getRuntimeCallSDValueHelper(Chain, dl, Args: std::move(Args), CI,
10018 Call: RTLIB::STRCMP, DAG: this, TLI);
10019}
10020
10021std::pair<SDValue, SDValue> SelectionDAG::getStrstr(SDValue Chain,
10022 const SDLoc &dl, SDValue S1,
10023 SDValue S2,
10024 const CallInst *CI) {
10025 PointerType *PT = PointerType::getUnqual(C&: *getContext());
10026 TargetLowering::ArgListTy Args = {{S1, PT}, {S2, PT}};
10027 return getRuntimeCallSDValueHelper(Chain, dl, Args: std::move(Args), CI,
10028 Call: RTLIB::STRSTR, DAG: this, TLI);
10029}
10030
10031std::pair<SDValue, SDValue> SelectionDAG::getMemccpy(SDValue Chain,
10032 const SDLoc &dl,
10033 SDValue Dst, SDValue Src,
10034 SDValue C, SDValue Size,
10035 const CallInst *CI) {
10036 PointerType *PT = PointerType::getUnqual(C&: *getContext());
10037
10038 TargetLowering::ArgListTy Args = {
10039 {Dst, PT},
10040 {Src, PT},
10041 {C, Type::getInt32Ty(C&: *getContext())},
10042 {Size, getDataLayout().getIntPtrType(C&: *getContext())}};
10043 return getRuntimeCallSDValueHelper(Chain, dl, Args: std::move(Args), CI,
10044 Call: RTLIB::MEMCCPY, DAG: this, TLI);
10045}
10046
10047std::pair<SDValue, SDValue>
10048SelectionDAG::getMemcmp(SDValue Chain, const SDLoc &dl, SDValue Mem0,
10049 SDValue Mem1, SDValue Size, const CallInst *CI) {
10050 PointerType *PT = PointerType::getUnqual(C&: *getContext());
10051 TargetLowering::ArgListTy Args = {
10052 {Mem0, PT},
10053 {Mem1, PT},
10054 {Size, getDataLayout().getIntPtrType(C&: *getContext())}};
10055 return getRuntimeCallSDValueHelper(Chain, dl, Args: std::move(Args), CI,
10056 Call: RTLIB::MEMCMP, DAG: this, TLI);
10057}
10058
10059std::pair<SDValue, SDValue> SelectionDAG::getStrcpy(SDValue Chain,
10060 const SDLoc &dl,
10061 SDValue Dst, SDValue Src,
10062 const CallInst *CI) {
10063 PointerType *PT = PointerType::getUnqual(C&: *getContext());
10064 TargetLowering::ArgListTy Args = {{Dst, PT}, {Src, PT}};
10065 return getRuntimeCallSDValueHelper(Chain, dl, Args: std::move(Args), CI,
10066 Call: RTLIB::STRCPY, DAG: this, TLI);
10067}
10068
10069std::pair<SDValue, SDValue> SelectionDAG::getStrlen(SDValue Chain,
10070 const SDLoc &dl,
10071 SDValue Src,
10072 const CallInst *CI) {
10073 // Emit a library call.
10074 TargetLowering::ArgListTy Args = {
10075 {Src, PointerType::getUnqual(C&: *getContext())}};
10076 return getRuntimeCallSDValueHelper(Chain, dl, Args: std::move(Args), CI,
10077 Call: RTLIB::STRLEN, DAG: this, TLI);
10078}
10079
10080bool SelectionDAG::hasSwiftErrorArg() const {
10081 return TLI->supportSwiftError() &&
10082 MF->getFunction().getAttributes().hasAttrSomewhere(
10083 Kind: Attribute::SwiftError);
10084}
10085
10086SDValue SelectionDAG::getMemcpy(
10087 SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size,
10088 Align DstAlign, Align SrcAlign, bool isVol, bool AlwaysInline,
10089 const CallInst *CI, std::optional<bool> OverrideTailCall,
10090 MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo,
10091 const AAMDNodes &AAInfo, BatchAAResults *BatchAA) {
10092 // Check to see if we should lower the memcpy to loads and stores first.
10093 // For cases within the target-specified limits, this is the best choice.
10094 const MDNode *DstMemCacheHint =
10095 CI ? getMemCacheHintMetadata(I: *CI, /*OperandNo=*/0) : nullptr;
10096 const MDNode *SrcMemCacheHint =
10097 CI ? getMemCacheHintMetadata(I: *CI, /*OperandNo=*/1) : nullptr;
10098
10099 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Val&: Size);
10100 if (ConstantSize) {
10101 // Memcpy with size zero? Just return the original chain.
10102 if (ConstantSize->isZero())
10103 return Chain;
10104
10105 SDValue Result = getMemcpyLoadsAndStores(
10106 DAG&: *this, dl, Chain, Dst, Src, Size: ConstantSize->getZExtValue(), DstAlign,
10107 SrcAlign, isVol, AlwaysInline: false, DstPtrInfo, SrcPtrInfo, AAInfo, BatchAA,
10108 DstMemCacheHint, SrcMemCacheHint);
10109 if (Result.getNode())
10110 return Result;
10111 }
10112
10113 // Then check to see if we should lower the memcpy with target-specific
10114 // code. If the target chooses to do this, this is the next best.
10115 if (TSI) {
10116 SDValue Result = TSI->EmitTargetCodeForMemcpy(
10117 DAG&: *this, dl, Chain, Op1: Dst, Op2: Src, Op3: Size, DstAlign, SrcAlign, isVolatile: isVol,
10118 AlwaysInline, DstPtrInfo, SrcPtrInfo);
10119 if (Result.getNode())
10120 return Result;
10121 }
10122
10123 // If we really need inline code and the target declined to provide it,
10124 // use a (potentially long) sequence of loads and stores.
10125 if (AlwaysInline) {
10126 assert(ConstantSize && "AlwaysInline requires a constant size!");
10127 return getMemcpyLoadsAndStores(
10128 DAG&: *this, dl, Chain, Dst, Src, Size: ConstantSize->getZExtValue(), DstAlign,
10129 SrcAlign, isVol, AlwaysInline: true, DstPtrInfo, SrcPtrInfo, AAInfo, BatchAA,
10130 DstMemCacheHint, SrcMemCacheHint);
10131 }
10132
10133 checkAddrSpaceIsValidForLibcall(TLI, AS: DstPtrInfo.getAddrSpace());
10134 checkAddrSpaceIsValidForLibcall(TLI, AS: SrcPtrInfo.getAddrSpace());
10135
10136 // FIXME: If the memcpy is volatile (isVol), lowering it to a plain libc
10137 // memcpy is not guaranteed to be safe. libc memcpys aren't required to
10138 // respect volatile, so they may do things like read or write memory
10139 // beyond the given memory regions. But fixing this isn't easy, and most
10140 // people don't care.
10141
10142 // Emit a library call.
10143 TargetLowering::ArgListTy Args;
10144 Type *PtrTy = PointerType::getUnqual(C&: *getContext());
10145 Args.emplace_back(args&: Dst, args&: PtrTy);
10146 Args.emplace_back(args&: Src, args&: PtrTy);
10147 Args.emplace_back(args&: Size, args: getDataLayout().getIntPtrType(C&: *getContext()));
10148 // FIXME: pass in SDLoc
10149 TargetLowering::CallLoweringInfo CLI(*this);
10150 bool IsTailCall = false;
10151 RTLIB::LibcallImpl MemCpyImpl = TLI->getMemcpyImpl();
10152
10153 if (OverrideTailCall.has_value()) {
10154 IsTailCall = *OverrideTailCall;
10155 } else {
10156 bool LowersToMemcpy = MemCpyImpl == RTLIB::impl_memcpy;
10157 IsTailCall = isInTailCallPositionWrapper(CI, SelDAG: this, AllowReturnsFirstArg: LowersToMemcpy);
10158 }
10159 // Lowering doesn't support tail calling inside a function with a
10160 // swifterror argument yet.
10161 IsTailCall &= !hasSwiftErrorArg();
10162
10163 CLI.setDebugLoc(dl)
10164 .setChain(Chain)
10165 .setLibCallee(
10166 CC: Libcalls->getLibcallImplCallingConv(Call: MemCpyImpl),
10167 ResultType: Dst.getValueType().getTypeForEVT(Context&: *getContext()),
10168 Target: getExternalSymbol(Libcall: MemCpyImpl, VT: TLI->getPointerTy(DL: getDataLayout())),
10169 ArgsList: std::move(Args))
10170 .setDiscardResult()
10171 .setTailCall(IsTailCall);
10172
10173 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
10174 return CallResult.second;
10175}
10176
10177SDValue SelectionDAG::getAtomicMemcpy(SDValue Chain, const SDLoc &dl,
10178 SDValue Dst, SDValue Src, SDValue Size,
10179 Type *SizeTy, unsigned ElemSz,
10180 bool isTailCall,
10181 MachinePointerInfo DstPtrInfo,
10182 MachinePointerInfo SrcPtrInfo) {
10183 // Lowering doesn't support tail calling inside a function with a
10184 // swifterror argument yet.
10185 isTailCall &= !hasSwiftErrorArg();
10186
10187 // Emit a library call.
10188 TargetLowering::ArgListTy Args;
10189 Type *ArgTy = getDataLayout().getIntPtrType(C&: *getContext());
10190 Args.emplace_back(args&: Dst, args&: ArgTy);
10191 Args.emplace_back(args&: Src, args&: ArgTy);
10192 Args.emplace_back(args&: Size, args&: SizeTy);
10193
10194 RTLIB::Libcall LibraryCall =
10195 RTLIB::getMEMCPY_ELEMENT_UNORDERED_ATOMIC(ElementSize: ElemSz);
10196 RTLIB::LibcallImpl LibcallImpl = Libcalls->getLibcallImpl(Call: LibraryCall);
10197 if (LibcallImpl == RTLIB::Unsupported)
10198 report_fatal_error(reason: "Unsupported element size");
10199
10200 TargetLowering::CallLoweringInfo CLI(*this);
10201 CLI.setDebugLoc(dl)
10202 .setChain(Chain)
10203 .setLibCallee(
10204 CC: Libcalls->getLibcallImplCallingConv(Call: LibcallImpl),
10205 ResultType: Type::getVoidTy(C&: *getContext()),
10206 Target: getExternalSymbol(Libcall: LibcallImpl, VT: TLI->getPointerTy(DL: getDataLayout())),
10207 ArgsList: std::move(Args))
10208 .setDiscardResult()
10209 .setTailCall(isTailCall);
10210
10211 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
10212 return CallResult.second;
10213}
10214
10215SDValue SelectionDAG::getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst,
10216 SDValue Src, SDValue Size, Align DstAlign,
10217 Align SrcAlign, bool isVol, const CallInst *CI,
10218 std::optional<bool> OverrideTailCall,
10219 MachinePointerInfo DstPtrInfo,
10220 MachinePointerInfo SrcPtrInfo,
10221 const AAMDNodes &AAInfo,
10222 BatchAAResults *BatchAA) {
10223 // Check to see if we should lower the memmove to loads and stores first.
10224 // For cases within the target-specified limits, this is the best choice.
10225 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Val&: Size);
10226 if (ConstantSize) {
10227 // Memmove with size zero? Just return the original chain.
10228 if (ConstantSize->isZero())
10229 return Chain;
10230
10231 SDValue Result = getMemmoveLoadsAndStores(
10232 DAG&: *this, dl, Chain, Dst, Src, Size: ConstantSize->getZExtValue(), DstAlign,
10233 SrcAlign, isVol, AlwaysInline: false, DstPtrInfo, SrcPtrInfo, AAInfo);
10234 if (Result.getNode())
10235 return Result;
10236 }
10237
10238 // Then check to see if we should lower the memmove with target-specific
10239 // code. If the target chooses to do this, this is the next best.
10240 if (TSI) {
10241 SDValue Result = TSI->EmitTargetCodeForMemmove(
10242 DAG&: *this, dl, Chain, Op1: Dst, Op2: Src, Op3: Size, DstAlign, SrcAlign, isVolatile: isVol, DstPtrInfo,
10243 SrcPtrInfo);
10244 if (Result.getNode())
10245 return Result;
10246 }
10247
10248 checkAddrSpaceIsValidForLibcall(TLI, AS: DstPtrInfo.getAddrSpace());
10249 checkAddrSpaceIsValidForLibcall(TLI, AS: SrcPtrInfo.getAddrSpace());
10250
10251 // FIXME: If the memmove is volatile, lowering it to plain libc memmove may
10252 // not be safe. See memcpy above for more details.
10253
10254 // Emit a library call.
10255 TargetLowering::ArgListTy Args;
10256 Type *PtrTy = PointerType::getUnqual(C&: *getContext());
10257 Args.emplace_back(args&: Dst, args&: PtrTy);
10258 Args.emplace_back(args&: Src, args&: PtrTy);
10259 Args.emplace_back(args&: Size, args: getDataLayout().getIntPtrType(C&: *getContext()));
10260 // FIXME: pass in SDLoc
10261 TargetLowering::CallLoweringInfo CLI(*this);
10262
10263 RTLIB::LibcallImpl MemmoveImpl = Libcalls->getLibcallImpl(Call: RTLIB::MEMMOVE);
10264
10265 bool IsTailCall = false;
10266 if (OverrideTailCall.has_value()) {
10267 IsTailCall = *OverrideTailCall;
10268 } else {
10269 bool LowersToMemmove = MemmoveImpl == RTLIB::impl_memmove;
10270 IsTailCall = isInTailCallPositionWrapper(CI, SelDAG: this, AllowReturnsFirstArg: LowersToMemmove);
10271 }
10272 // Lowering doesn't support tail calling inside a function with a
10273 // swifterror argument yet.
10274 IsTailCall &= !hasSwiftErrorArg();
10275
10276 CLI.setDebugLoc(dl)
10277 .setChain(Chain)
10278 .setLibCallee(
10279 CC: Libcalls->getLibcallImplCallingConv(Call: MemmoveImpl),
10280 ResultType: Dst.getValueType().getTypeForEVT(Context&: *getContext()),
10281 Target: getExternalSymbol(Libcall: MemmoveImpl, VT: TLI->getPointerTy(DL: getDataLayout())),
10282 ArgsList: std::move(Args))
10283 .setDiscardResult()
10284 .setTailCall(IsTailCall);
10285
10286 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
10287 return CallResult.second;
10288}
10289
10290SDValue SelectionDAG::getAtomicMemmove(SDValue Chain, const SDLoc &dl,
10291 SDValue Dst, SDValue Src, SDValue Size,
10292 Type *SizeTy, unsigned ElemSz,
10293 bool isTailCall,
10294 MachinePointerInfo DstPtrInfo,
10295 MachinePointerInfo SrcPtrInfo) {
10296 // Lowering doesn't support tail calling inside a function with a
10297 // swifterror argument yet.
10298 isTailCall &= !hasSwiftErrorArg();
10299
10300 // Emit a library call.
10301 TargetLowering::ArgListTy Args;
10302 Type *IntPtrTy = getDataLayout().getIntPtrType(C&: *getContext());
10303 Args.emplace_back(args&: Dst, args&: IntPtrTy);
10304 Args.emplace_back(args&: Src, args&: IntPtrTy);
10305 Args.emplace_back(args&: Size, args&: SizeTy);
10306
10307 RTLIB::Libcall LibraryCall =
10308 RTLIB::getMEMMOVE_ELEMENT_UNORDERED_ATOMIC(ElementSize: ElemSz);
10309 RTLIB::LibcallImpl LibcallImpl = Libcalls->getLibcallImpl(Call: LibraryCall);
10310 if (LibcallImpl == RTLIB::Unsupported)
10311 report_fatal_error(reason: "Unsupported element size");
10312
10313 TargetLowering::CallLoweringInfo CLI(*this);
10314 CLI.setDebugLoc(dl)
10315 .setChain(Chain)
10316 .setLibCallee(
10317 CC: Libcalls->getLibcallImplCallingConv(Call: LibcallImpl),
10318 ResultType: Type::getVoidTy(C&: *getContext()),
10319 Target: getExternalSymbol(Libcall: LibcallImpl, VT: TLI->getPointerTy(DL: getDataLayout())),
10320 ArgsList: std::move(Args))
10321 .setDiscardResult()
10322 .setTailCall(isTailCall);
10323
10324 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
10325 return CallResult.second;
10326}
10327
10328SDValue SelectionDAG::getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst,
10329 SDValue Src, SDValue Size, Align Alignment,
10330 bool isVol, bool AlwaysInline,
10331 const CallInst *CI,
10332 MachinePointerInfo DstPtrInfo,
10333 const AAMDNodes &AAInfo) {
10334 // Check to see if we should lower the memset to stores first.
10335 // For cases within the target-specified limits, this is the best choice.
10336 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Val&: Size);
10337 if (ConstantSize) {
10338 // Memset with size zero? Just return the original chain.
10339 if (ConstantSize->isZero())
10340 return Chain;
10341
10342 SDValue Result = getMemsetStores(DAG&: *this, dl, Chain, Dst, Src,
10343 Size: ConstantSize->getZExtValue(), Alignment,
10344 isVol, AlwaysInline: false, DstPtrInfo, AAInfo);
10345
10346 if (Result.getNode())
10347 return Result;
10348 }
10349
10350 // Then check to see if we should lower the memset with target-specific
10351 // code. If the target chooses to do this, this is the next best.
10352 if (TSI) {
10353 SDValue Result = TSI->EmitTargetCodeForMemset(
10354 DAG&: *this, dl, Chain, Op1: Dst, Op2: Src, Op3: Size, Alignment, isVolatile: isVol, AlwaysInline, DstPtrInfo);
10355 if (Result.getNode())
10356 return Result;
10357 }
10358
10359 // If we really need inline code and the target declined to provide it,
10360 // use a (potentially long) sequence of loads and stores.
10361 if (AlwaysInline) {
10362 assert(ConstantSize && "AlwaysInline requires a constant size!");
10363 SDValue Result = getMemsetStores(DAG&: *this, dl, Chain, Dst, Src,
10364 Size: ConstantSize->getZExtValue(), Alignment,
10365 isVol, AlwaysInline: true, DstPtrInfo, AAInfo);
10366 assert(Result &&
10367 "getMemsetStores must return a valid sequence when AlwaysInline");
10368 return Result;
10369 }
10370
10371 checkAddrSpaceIsValidForLibcall(TLI, AS: DstPtrInfo.getAddrSpace());
10372
10373 // Emit a library call.
10374 auto &Ctx = *getContext();
10375 const auto& DL = getDataLayout();
10376
10377 TargetLowering::CallLoweringInfo CLI(*this);
10378 // FIXME: pass in SDLoc
10379 CLI.setDebugLoc(dl).setChain(Chain);
10380
10381 RTLIB::LibcallImpl BzeroImpl = Libcalls->getLibcallImpl(Call: RTLIB::BZERO);
10382 bool UseBZero = BzeroImpl != RTLIB::Unsupported && isNullConstant(V: Src);
10383
10384 // If zeroing out and bzero is present, use it.
10385 if (UseBZero) {
10386 TargetLowering::ArgListTy Args;
10387 Args.emplace_back(args&: Dst, args: PointerType::getUnqual(C&: Ctx));
10388 Args.emplace_back(args&: Size, args: DL.getIntPtrType(C&: Ctx));
10389 CLI.setLibCallee(
10390 CC: Libcalls->getLibcallImplCallingConv(Call: BzeroImpl), ResultType: Type::getVoidTy(C&: Ctx),
10391 Target: getExternalSymbol(Libcall: BzeroImpl, VT: TLI->getPointerTy(DL)), ArgsList: std::move(Args));
10392 } else {
10393 RTLIB::LibcallImpl MemsetImpl = Libcalls->getLibcallImpl(Call: RTLIB::MEMSET);
10394
10395 TargetLowering::ArgListTy Args;
10396 Args.emplace_back(args&: Dst, args: PointerType::getUnqual(C&: Ctx));
10397 Args.emplace_back(args&: Src, args: Src.getValueType().getTypeForEVT(Context&: Ctx));
10398 Args.emplace_back(args&: Size, args: DL.getIntPtrType(C&: Ctx));
10399 CLI.setLibCallee(CC: Libcalls->getLibcallImplCallingConv(Call: MemsetImpl),
10400 ResultType: Dst.getValueType().getTypeForEVT(Context&: Ctx),
10401 Target: getExternalSymbol(Libcall: MemsetImpl, VT: TLI->getPointerTy(DL)),
10402 ArgsList: std::move(Args));
10403 }
10404
10405 RTLIB::LibcallImpl MemsetImpl = Libcalls->getLibcallImpl(Call: RTLIB::MEMSET);
10406 bool LowersToMemset = MemsetImpl == RTLIB::impl_memset;
10407
10408 // If we're going to use bzero, make sure not to tail call unless the
10409 // subsequent return doesn't need a value, as bzero doesn't return the first
10410 // arg unlike memset.
10411 bool ReturnsFirstArg = CI && funcReturnsFirstArgOfCall(CI: *CI) && !UseBZero;
10412 bool IsTailCall = CI && CI->isTailCall() &&
10413 isInTailCallPosition(Call: *CI, TM: getTarget(),
10414 ReturnsFirstArg: ReturnsFirstArg && LowersToMemset) &&
10415 // Lowering doesn't support tail calling inside a function
10416 // with a swifterror argument yet.
10417 !hasSwiftErrorArg();
10418 CLI.setDiscardResult().setTailCall(IsTailCall);
10419
10420 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
10421 return CallResult.second;
10422}
10423
10424SDValue SelectionDAG::getAtomicMemset(SDValue Chain, const SDLoc &dl,
10425 SDValue Dst, SDValue Value, SDValue Size,
10426 Type *SizeTy, unsigned ElemSz,
10427 bool isTailCall,
10428 MachinePointerInfo DstPtrInfo) {
10429 // Lowering doesn't support tail calling inside a function with a
10430 // swifterror argument yet.
10431 isTailCall &= !hasSwiftErrorArg();
10432
10433 // Emit a library call.
10434 TargetLowering::ArgListTy Args;
10435 Args.emplace_back(args&: Dst, args: getDataLayout().getIntPtrType(C&: *getContext()));
10436 Args.emplace_back(args&: Value, args: Type::getInt8Ty(C&: *getContext()));
10437 Args.emplace_back(args&: Size, args&: SizeTy);
10438
10439 RTLIB::Libcall LibraryCall =
10440 RTLIB::getMEMSET_ELEMENT_UNORDERED_ATOMIC(ElementSize: ElemSz);
10441 RTLIB::LibcallImpl LibcallImpl = Libcalls->getLibcallImpl(Call: LibraryCall);
10442 if (LibcallImpl == RTLIB::Unsupported)
10443 report_fatal_error(reason: "Unsupported element size");
10444
10445 TargetLowering::CallLoweringInfo CLI(*this);
10446 CLI.setDebugLoc(dl)
10447 .setChain(Chain)
10448 .setLibCallee(
10449 CC: Libcalls->getLibcallImplCallingConv(Call: LibcallImpl),
10450 ResultType: Type::getVoidTy(C&: *getContext()),
10451 Target: getExternalSymbol(Libcall: LibcallImpl, VT: TLI->getPointerTy(DL: getDataLayout())),
10452 ArgsList: std::move(Args))
10453 .setDiscardResult()
10454 .setTailCall(isTailCall);
10455
10456 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
10457 return CallResult.second;
10458}
10459
10460SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
10461 SDVTList VTList, ArrayRef<SDValue> Ops,
10462 MachineMemOperand *MMO,
10463 ISD::LoadExtType ExtType) {
10464 FoldingSetNodeID ID;
10465 AddNodeIDNode(ID, OpC: Opcode, VTList, OpList: Ops);
10466 ID.AddInteger(I: MemVT.getRawBits());
10467 ID.AddInteger(I: getSyntheticNodeSubclassData<AtomicSDNode>(
10468 IROrder: dl.getIROrder(), Args&: Opcode, Args&: VTList, Args&: MemVT, Args&: MMO, Args&: ExtType));
10469 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10470 ID.AddInteger(I: MMO->getFlags());
10471 FoldingSetInsertToken InsertToken;
10472 if (auto *E = cast_or_null<AtomicSDNode>(Val: lookupNode(ID, DL: dl, InsertToken))) {
10473 E->refineAlignment(NewMMO: MMO);
10474 E->refineMMOMetadata(NewMMO: MMO);
10475 return SDValue(E, 0);
10476 }
10477
10478 auto *N = newSDNode<AtomicSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: Opcode,
10479 Args&: VTList, Args&: MemVT, Args&: MMO, Args&: ExtType);
10480 createOperands(Node: N, Vals: Ops);
10481
10482 CSEMap.insert(N, Token: InsertToken);
10483 InsertNode(N);
10484 SDValue V(N, 0);
10485 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10486 return V;
10487}
10488
10489SDValue SelectionDAG::getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl,
10490 EVT MemVT, SDVTList VTs, SDValue Chain,
10491 SDValue Ptr, SDValue Cmp, SDValue Swp,
10492 MachineMemOperand *MMO) {
10493 assert(Opcode == ISD::ATOMIC_CMP_SWAP ||
10494 Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
10495 assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
10496
10497 SDValue Ops[] = {Chain, Ptr, Cmp, Swp};
10498 return getAtomic(Opcode, dl, MemVT, VTList: VTs, Ops, MMO);
10499}
10500
10501SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
10502 SDValue Chain, SDValue Ptr, SDValue Val,
10503 MachineMemOperand *MMO) {
10504 assert((Opcode == ISD::ATOMIC_LOAD_ADD || Opcode == ISD::ATOMIC_LOAD_SUB ||
10505 Opcode == ISD::ATOMIC_LOAD_AND || Opcode == ISD::ATOMIC_LOAD_CLR ||
10506 Opcode == ISD::ATOMIC_LOAD_OR || Opcode == ISD::ATOMIC_LOAD_XOR ||
10507 Opcode == ISD::ATOMIC_LOAD_NAND || Opcode == ISD::ATOMIC_LOAD_MIN ||
10508 Opcode == ISD::ATOMIC_LOAD_MAX || Opcode == ISD::ATOMIC_LOAD_UMIN ||
10509 Opcode == ISD::ATOMIC_LOAD_UMAX || Opcode == ISD::ATOMIC_LOAD_FADD ||
10510 Opcode == ISD::ATOMIC_LOAD_FSUB || Opcode == ISD::ATOMIC_LOAD_FMAX ||
10511 Opcode == ISD::ATOMIC_LOAD_FMIN ||
10512 Opcode == ISD::ATOMIC_LOAD_FMINIMUM ||
10513 Opcode == ISD::ATOMIC_LOAD_FMAXIMUM ||
10514 Opcode == ISD::ATOMIC_LOAD_UINC_WRAP ||
10515 Opcode == ISD::ATOMIC_LOAD_UDEC_WRAP ||
10516 Opcode == ISD::ATOMIC_LOAD_USUB_COND ||
10517 Opcode == ISD::ATOMIC_LOAD_USUB_SAT || Opcode == ISD::ATOMIC_SWAP ||
10518 Opcode == ISD::ATOMIC_STORE) &&
10519 "Invalid Atomic Op");
10520
10521 EVT VT = Val.getValueType();
10522
10523 SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(VT: MVT::Other) :
10524 getVTList(VT1: VT, VT2: MVT::Other);
10525 SDValue Ops[] = {Chain, Ptr, Val};
10526 return getAtomic(Opcode, dl, MemVT, VTList: VTs, Ops, MMO);
10527}
10528
10529SDValue SelectionDAG::getAtomicLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
10530 EVT MemVT, EVT VT, SDValue Chain,
10531 SDValue Ptr, MachineMemOperand *MMO) {
10532 SDVTList VTs = getVTList(VT1: VT, VT2: MVT::Other);
10533 SDValue Ops[] = {Chain, Ptr};
10534 return getAtomic(Opcode: ISD::ATOMIC_LOAD, dl, MemVT, VTList: VTs, Ops, MMO, ExtType);
10535}
10536
10537/// getMergeValues - Create a MERGE_VALUES node from the given operands.
10538SDValue SelectionDAG::getMergeValues(ArrayRef<SDValue> Ops, const SDLoc &dl) {
10539 if (Ops.size() == 1)
10540 return Ops[0];
10541
10542 SmallVector<EVT, 4> VTs;
10543 VTs.reserve(N: Ops.size());
10544 for (const SDValue &Op : Ops)
10545 VTs.push_back(Elt: Op.getValueType());
10546 return getNode(Opcode: ISD::MERGE_VALUES, DL: dl, VTList: getVTList(VTs), Ops);
10547}
10548
10549SDValue SelectionDAG::getErrorMergeValues(ArrayRef<EVT> ResultTypes,
10550 SDValue Chain, const SDLoc &dl) {
10551 SmallVector<SDValue, 4> RetValues;
10552 RetValues.reserve(N: ResultTypes.size());
10553 for (EVT VT : ResultTypes)
10554 RetValues.push_back(Elt: VT == MVT::Other ? Chain : getPOISON(VT));
10555 return getMergeValues(Ops: RetValues, dl);
10556}
10557
10558SDValue SelectionDAG::getMemIntrinsicNode(
10559 unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef<SDValue> Ops,
10560 EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment,
10561 MachineMemOperand::Flags Flags, LocationSize Size,
10562 const AAMDNodes &AAInfo) {
10563 if (Size.hasValue() && !Size.getValue())
10564 Size = LocationSize::precise(Value: MemVT.getStoreSize());
10565
10566 MachineFunction &MF = getMachineFunction();
10567 MachineMemOperand *MMO =
10568 MF.getMachineMemOperand(PtrInfo, F: Flags, Size, BaseAlignment: Alignment, Metadata: AAInfo);
10569
10570 return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, MMO);
10571}
10572
10573SDValue SelectionDAG::getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl,
10574 SDVTList VTList,
10575 ArrayRef<SDValue> Ops, EVT MemVT,
10576 MachineMemOperand *MMO) {
10577 return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, MMOs: ArrayRef(MMO));
10578}
10579
10580SDValue SelectionDAG::getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl,
10581 SDVTList VTList,
10582 ArrayRef<SDValue> Ops, EVT MemVT,
10583 ArrayRef<MachineMemOperand *> MMOs) {
10584 assert(!MMOs.empty() && "Must have at least one MMO");
10585 assert(
10586 (Opcode == ISD::INTRINSIC_VOID || Opcode == ISD::INTRINSIC_W_CHAIN ||
10587 Opcode == ISD::PREFETCH ||
10588 (Opcode <= (unsigned)std::numeric_limits<int>::max() &&
10589 Opcode >= ISD::BUILTIN_OP_END && TSI->isTargetMemoryOpcode(Opcode))) &&
10590 "Opcode is not a memory-accessing opcode!");
10591
10592 PointerUnion<MachineMemOperand *, MachineMemOperand **> MemRefs;
10593 if (MMOs.size() == 1) {
10594 MemRefs = MMOs[0];
10595 } else {
10596 // Allocate: [size_t count][MMO*][MMO*]...
10597 size_t AllocSize =
10598 sizeof(size_t) + MMOs.size() * sizeof(MachineMemOperand *);
10599 void *Buffer = Allocator.Allocate(Size: AllocSize, Alignment: alignof(size_t));
10600 size_t *CountPtr = static_cast<size_t *>(Buffer);
10601 *CountPtr = MMOs.size();
10602 MachineMemOperand **Array =
10603 reinterpret_cast<MachineMemOperand **>(CountPtr + 1);
10604 llvm::copy(Range&: MMOs, Out: Array);
10605 MemRefs = Array;
10606 }
10607
10608 // Memoize the node unless it returns a glue result.
10609 MemIntrinsicSDNode *N;
10610 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
10611 FoldingSetNodeID ID;
10612 AddNodeIDNode(ID, OpC: Opcode, VTList, OpList: Ops);
10613 ID.AddInteger(I: getSyntheticNodeSubclassData<MemIntrinsicSDNode>(
10614 Opc: Opcode, Order: dl.getIROrder(), VTs: VTList, MemoryVT: MemVT, MemRefs));
10615 ID.AddInteger(I: MemVT.getRawBits());
10616 for (const MachineMemOperand *MMO : MMOs) {
10617 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10618 ID.AddInteger(I: MMO->getFlags());
10619 }
10620 FoldingSetInsertToken InsertToken;
10621 if (SDNode *E = lookupNode(ID, DL: dl, InsertToken)) {
10622 cast<MemIntrinsicSDNode>(Val: E)->refineAlignment(NewMMOs: MMOs);
10623 return SDValue(E, 0);
10624 }
10625
10626 N = newSDNode<MemIntrinsicSDNode>(Args&: Opcode, Args: dl.getIROrder(), Args: dl.getDebugLoc(),
10627 Args&: VTList, Args&: MemVT, Args&: MemRefs);
10628 createOperands(Node: N, Vals: Ops);
10629 CSEMap.insert(N, Token: InsertToken);
10630 } else {
10631 N = newSDNode<MemIntrinsicSDNode>(Args&: Opcode, Args: dl.getIROrder(), Args: dl.getDebugLoc(),
10632 Args&: VTList, Args&: MemVT, Args&: MemRefs);
10633 createOperands(Node: N, Vals: Ops);
10634 }
10635 InsertNode(N);
10636 SDValue V(N, 0);
10637 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10638 return V;
10639}
10640
10641SDValue SelectionDAG::getLifetimeNode(bool IsStart, const SDLoc &dl,
10642 SDValue Chain, int FrameIndex) {
10643 const unsigned Opcode = IsStart ? ISD::LIFETIME_START : ISD::LIFETIME_END;
10644 const auto VTs = getVTList(VT: MVT::Other);
10645 SDValue Ops[2] = {
10646 Chain,
10647 getFrameIndex(FI: FrameIndex,
10648 VT: getTargetLoweringInfo().getFrameIndexTy(DL: getDataLayout()),
10649 isTarget: true)};
10650
10651 FoldingSetNodeID ID;
10652 AddNodeIDNode(ID, OpC: Opcode, VTList: VTs, OpList: Ops);
10653 ID.AddInteger(I: FrameIndex);
10654 FoldingSetInsertToken InsertToken;
10655 if (SDNode *E = lookupNode(ID, DL: dl, InsertToken))
10656 return SDValue(E, 0);
10657
10658 LifetimeSDNode *N =
10659 newSDNode<LifetimeSDNode>(Args: Opcode, Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args: VTs);
10660 createOperands(Node: N, Vals: Ops);
10661 CSEMap.insert(N, Token: InsertToken);
10662 InsertNode(N);
10663 SDValue V(N, 0);
10664 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10665 return V;
10666}
10667
10668SDValue SelectionDAG::getPseudoProbeNode(const SDLoc &Dl, SDValue Chain,
10669 uint64_t Guid, uint64_t Index,
10670 uint32_t Attr) {
10671 const unsigned Opcode = ISD::PSEUDO_PROBE;
10672 const auto VTs = getVTList(VT: MVT::Other);
10673 SDValue Ops[] = {Chain};
10674 FoldingSetNodeID ID;
10675 AddNodeIDNode(ID, OpC: Opcode, VTList: VTs, OpList: Ops);
10676 ID.AddInteger(I: Guid);
10677 ID.AddInteger(I: Index);
10678 FoldingSetInsertToken InsertToken;
10679 if (SDNode *E = lookupNode(ID, DL: Dl, InsertToken))
10680 return SDValue(E, 0);
10681
10682 auto *N = newSDNode<PseudoProbeSDNode>(
10683 Args: Opcode, Args: Dl.getIROrder(), Args: Dl.getDebugLoc(), Args: VTs, Args&: Guid, Args&: Index, Args&: Attr);
10684 createOperands(Node: N, Vals: Ops);
10685 CSEMap.insert(N, Token: InsertToken);
10686 InsertNode(N);
10687 SDValue V(N, 0);
10688 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10689 return V;
10690}
10691
10692/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
10693/// MachinePointerInfo record from it. This is particularly useful because the
10694/// code generator has many cases where it doesn't bother passing in a
10695/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
10696static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info,
10697 SelectionDAG &DAG, SDValue Ptr,
10698 int64_t Offset = 0) {
10699 // If this is FI+Offset, we can model it.
10700 if (const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Ptr))
10701 return MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(),
10702 FI: FI->getIndex(), Offset);
10703
10704 // If this is (FI+Offset1)+Offset2, we can model it.
10705 if (Ptr.getOpcode() != ISD::ADD ||
10706 !isa<ConstantSDNode>(Val: Ptr.getOperand(i: 1)) ||
10707 !isa<FrameIndexSDNode>(Val: Ptr.getOperand(i: 0)))
10708 return Info;
10709
10710 int FI = cast<FrameIndexSDNode>(Val: Ptr.getOperand(i: 0))->getIndex();
10711 return MachinePointerInfo::getFixedStack(
10712 MF&: DAG.getMachineFunction(), FI,
10713 Offset: Offset + cast<ConstantSDNode>(Val: Ptr.getOperand(i: 1))->getSExtValue());
10714}
10715
10716/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
10717/// MachinePointerInfo record from it. This is particularly useful because the
10718/// code generator has many cases where it doesn't bother passing in a
10719/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
10720static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info,
10721 SelectionDAG &DAG, SDValue Ptr,
10722 SDValue OffsetOp) {
10723 // If the 'Offset' value isn't a constant, we can't handle this.
10724 if (ConstantSDNode *OffsetNode = dyn_cast<ConstantSDNode>(Val&: OffsetOp))
10725 return InferPointerInfo(Info, DAG, Ptr, Offset: OffsetNode->getSExtValue());
10726 if (OffsetOp.isUndef())
10727 return InferPointerInfo(Info, DAG, Ptr);
10728 return Info;
10729}
10730
10731SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
10732 EVT VT, const SDLoc &dl, SDValue Chain,
10733 SDValue Ptr, SDValue Offset,
10734 MachinePointerInfo PtrInfo, EVT MemVT,
10735 Align Alignment,
10736 MachineMemOperand::Flags MMOFlags,
10737 const MMOMetadata &Metadata) {
10738 assert(Chain.getValueType() == MVT::Other &&
10739 "Invalid chain type");
10740
10741 MMOFlags |= MachineMemOperand::MOLoad;
10742 assert((MMOFlags & MachineMemOperand::MOStore) == 0);
10743 // If we don't have a PtrInfo, infer the trivial frame index case to simplify
10744 // clients.
10745 if (PtrInfo.V.isNull())
10746 PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr, OffsetOp: Offset);
10747
10748 TypeSize Size = MemVT.getStoreSize();
10749 MachineFunction &MF = getMachineFunction();
10750 MachineMemOperand *MMO =
10751 MF.getMachineMemOperand(PtrInfo, F: MMOFlags, Size, BaseAlignment: Alignment, Metadata);
10752 return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO);
10753}
10754
10755SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
10756 EVT VT, const SDLoc &dl, SDValue Chain,
10757 SDValue Ptr, SDValue Offset, EVT MemVT,
10758 MachineMemOperand *MMO) {
10759 if (VT == MemVT) {
10760 ExtType = ISD::NON_EXTLOAD;
10761 } else if (ExtType == ISD::NON_EXTLOAD) {
10762 assert(VT == MemVT && "Non-extending load from different memory type!");
10763 } else {
10764 // Extending load.
10765 assert(MemVT.getScalarType().bitsLT(VT.getScalarType()) &&
10766 "Should only be an extending load, not truncating!");
10767 assert(VT.isInteger() == MemVT.isInteger() &&
10768 "Cannot convert from FP to Int or Int -> FP!");
10769 assert(VT.isVector() == MemVT.isVector() &&
10770 "Cannot use an ext load to convert to or from a vector!");
10771 assert((!VT.isVector() ||
10772 VT.getVectorElementCount() == MemVT.getVectorElementCount()) &&
10773 "Cannot use an ext load to change the number of vector elements!");
10774 }
10775
10776 assert((!MMO->getRanges() ||
10777 (mdconst::extract<ConstantInt>(MMO->getRanges()->getOperand(0))
10778 ->getBitWidth() == MemVT.getScalarSizeInBits() &&
10779 MemVT.isInteger())) &&
10780 "Range metadata and load type must match!");
10781
10782 bool Indexed = AM != ISD::UNINDEXED;
10783 assert((Indexed || Offset.getOpcode() == ISD::POISON) &&
10784 "Unindexed load with an offset!");
10785
10786 SDVTList VTs = Indexed ?
10787 getVTList(VT1: VT, VT2: Ptr.getValueType(), VT3: MVT::Other) : getVTList(VT1: VT, VT2: MVT::Other);
10788 SDValue Ops[] = { Chain, Ptr, Offset };
10789 FoldingSetNodeID ID;
10790 AddNodeIDNode(ID, OpC: ISD::LOAD, VTList: VTs, OpList: Ops);
10791 ID.AddInteger(I: MemVT.getRawBits());
10792 ID.AddInteger(I: getSyntheticNodeSubclassData<LoadSDNode>(
10793 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: ExtType, Args&: MemVT, Args&: MMO));
10794 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10795 ID.AddInteger(I: MMO->getFlags());
10796 FoldingSetInsertToken InsertToken;
10797 if (auto *E = cast_or_null<LoadSDNode>(Val: lookupNode(ID, DL: dl, InsertToken))) {
10798 E->refineAlignment(NewMMO: MMO);
10799 E->refineMMOMetadata(NewMMO: MMO);
10800 return SDValue(E, 0);
10801 }
10802 auto *N = newSDNode<LoadSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM,
10803 Args&: ExtType, Args&: MemVT, Args&: MMO);
10804 createOperands(Node: N, Vals: Ops);
10805
10806 CSEMap.insert(N, Token: InsertToken);
10807 InsertNode(N);
10808 SDValue V(N, 0);
10809 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10810 return V;
10811}
10812
10813SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain,
10814 SDValue Ptr, MachinePointerInfo PtrInfo,
10815 MaybeAlign Alignment,
10816 MachineMemOperand::Flags MMOFlags,
10817 const MMOMetadata &Metadata) {
10818 SDValue Undef = getPOISON(VT: Ptr.getValueType());
10819 return getLoad(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Offset: Undef,
10820 PtrInfo, MemVT: VT, Alignment, MMOFlags, Metadata);
10821}
10822
10823SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain,
10824 SDValue Ptr, MachineMemOperand *MMO) {
10825 SDValue Undef = getPOISON(VT: Ptr.getValueType());
10826 return getLoad(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Offset: Undef,
10827 MemVT: VT, MMO);
10828}
10829
10830SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
10831 EVT VT, SDValue Chain, SDValue Ptr,
10832 MachinePointerInfo PtrInfo, EVT MemVT,
10833 MaybeAlign Alignment,
10834 MachineMemOperand::Flags MMOFlags,
10835 const MMOMetadata &Metadata) {
10836 SDValue Undef = getPOISON(VT: Ptr.getValueType());
10837 return getLoad(AM: ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Offset: Undef, PtrInfo,
10838 MemVT, Alignment, MMOFlags, Metadata);
10839}
10840
10841SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
10842 EVT VT, SDValue Chain, SDValue Ptr, EVT MemVT,
10843 MachineMemOperand *MMO) {
10844 SDValue Undef = getPOISON(VT: Ptr.getValueType());
10845 return getLoad(AM: ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Offset: Undef,
10846 MemVT, MMO);
10847}
10848
10849SDValue SelectionDAG::getIndexedLoad(SDValue OrigLoad, const SDLoc &dl,
10850 SDValue Base, SDValue Offset,
10851 ISD::MemIndexedMode AM) {
10852 LoadSDNode *LD = cast<LoadSDNode>(Val&: OrigLoad);
10853 assert(LD->getOffset().getOpcode() == ISD::POISON &&
10854 "Load is already a indexed load!");
10855 // Don't propagate the invariant or dereferenceable flags.
10856 auto MMOFlags =
10857 LD->getMemOperand()->getFlags() &
10858 ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
10859 return getLoad(
10860 AM, ExtType: LD->getExtensionType(), VT: OrigLoad.getValueType(), dl, Chain: LD->getChain(),
10861 Ptr: Base, Offset, PtrInfo: LD->getPointerInfo(), MemVT: LD->getMemoryVT(), Alignment: LD->getAlign(),
10862 MMOFlags,
10863 Metadata: MMOMetadata(LD->getAAInfo(), LD->getRanges(), LD->getMemCacheHint()));
10864}
10865
10866SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
10867 SDValue Ptr, MachinePointerInfo PtrInfo,
10868 Align Alignment,
10869 MachineMemOperand::Flags MMOFlags,
10870 const MMOMetadata &Metadata) {
10871 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
10872
10873 MMOFlags |= MachineMemOperand::MOStore;
10874 assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
10875 assert(!Metadata.Ranges && "range metadata is invalid for stores");
10876
10877 if (PtrInfo.V.isNull())
10878 PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr);
10879
10880 MachineFunction &MF = getMachineFunction();
10881 TypeSize Size = Val.getValueType().getStoreSize();
10882 MachineMemOperand *MMO =
10883 MF.getMachineMemOperand(PtrInfo, F: MMOFlags, Size, BaseAlignment: Alignment, Metadata);
10884 return getStore(Chain, dl, Val, Ptr, MMO);
10885}
10886
10887SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
10888 SDValue Ptr, MachineMemOperand *MMO) {
10889 SDValue Undef = getPOISON(VT: Ptr.getValueType());
10890 return getStore(Chain, dl, Val, Ptr, Offset: Undef, SVT: Val.getValueType(), MMO,
10891 AM: ISD::UNINDEXED);
10892}
10893
10894SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
10895 SDValue Ptr, SDValue Offset, EVT SVT,
10896 MachineMemOperand *MMO, ISD::MemIndexedMode AM,
10897 bool IsTruncating) {
10898 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
10899 EVT VT = Val.getValueType();
10900 if (VT == SVT) {
10901 IsTruncating = false;
10902 } else if (!IsTruncating) {
10903 assert(VT == SVT && "No-truncating store from different memory type!");
10904 } else {
10905 assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&
10906 "Should only be a truncating store, not extending!");
10907 assert(VT.isInteger() == SVT.isInteger() && "Can't do FP-INT conversion!");
10908 assert(VT.isVector() == SVT.isVector() &&
10909 "Cannot use trunc store to convert to or from a vector!");
10910 assert((!VT.isVector() ||
10911 VT.getVectorElementCount() == SVT.getVectorElementCount()) &&
10912 "Cannot use trunc store to change the number of vector elements!");
10913 }
10914
10915 bool Indexed = AM != ISD::UNINDEXED;
10916 assert((Indexed || Offset.getOpcode() == ISD::POISON) &&
10917 "Unindexed store with an offset!");
10918 SDVTList VTs = Indexed ? getVTList(VT1: Ptr.getValueType(), VT2: MVT::Other)
10919 : getVTList(VT: MVT::Other);
10920 SDValue Ops[] = {Chain, Val, Ptr, Offset};
10921 FoldingSetNodeID ID;
10922 AddNodeIDNode(ID, OpC: ISD::STORE, VTList: VTs, OpList: Ops);
10923 ID.AddInteger(I: SVT.getRawBits());
10924 ID.AddInteger(I: getSyntheticNodeSubclassData<StoreSDNode>(
10925 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: IsTruncating, Args&: SVT, Args&: MMO));
10926 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10927 ID.AddInteger(I: MMO->getFlags());
10928 FoldingSetInsertToken InsertToken;
10929 if (SDNode *E = lookupNode(ID, DL: dl, InsertToken)) {
10930 cast<StoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
10931 cast<StoreSDNode>(Val: E)->refineMMOMetadata(NewMMO: MMO);
10932 return SDValue(E, 0);
10933 }
10934 auto *N = newSDNode<StoreSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM,
10935 Args&: IsTruncating, Args&: SVT, Args&: MMO);
10936 createOperands(Node: N, Vals: Ops);
10937
10938 CSEMap.insert(N, Token: InsertToken);
10939 InsertNode(N);
10940 SDValue V(N, 0);
10941 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10942 return V;
10943}
10944
10945SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
10946 SDValue Ptr, SDValue Offset,
10947 MachinePointerInfo PtrInfo, EVT SVT,
10948 Align Alignment,
10949 MachineMemOperand::Flags MMOFlags,
10950 const MMOMetadata &Metadata) {
10951 assert(Chain.getValueType() == MVT::Other &&
10952 "Invalid chain type");
10953
10954 MMOFlags |= MachineMemOperand::MOStore;
10955 assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
10956 assert(!Metadata.Ranges && "range metadata is invalid for stores");
10957
10958 if (PtrInfo.V.isNull())
10959 PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr);
10960
10961 MachineFunction &MF = getMachineFunction();
10962 MachineMemOperand *MMO = MF.getMachineMemOperand(
10963 PtrInfo, F: MMOFlags, Size: SVT.getStoreSize(), BaseAlignment: Alignment, Metadata);
10964 return getTruncStore(Chain, dl, Val, Ptr, Offset, SVT, MMO);
10965}
10966
10967SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
10968 SDValue Ptr, MachinePointerInfo PtrInfo,
10969 EVT SVT, Align Alignment,
10970 MachineMemOperand::Flags MMOFlags,
10971 const MMOMetadata &Metadata) {
10972 return getTruncStore(Chain, dl, Val, Ptr, Offset: getPOISON(VT: Ptr.getValueType()),
10973 PtrInfo, SVT, Alignment, MMOFlags, Metadata);
10974}
10975
10976SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
10977 SDValue Ptr, SDValue Offset, EVT SVT,
10978 MachineMemOperand *MMO) {
10979 return getStore(Chain, dl, Val, Ptr, Offset, SVT, MMO, AM: ISD::UNINDEXED, IsTruncating: true);
10980}
10981
10982SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
10983 SDValue Ptr, EVT SVT,
10984 MachineMemOperand *MMO) {
10985 return getStore(Chain, dl, Val, Ptr, Offset: getPOISON(VT: Ptr.getValueType()), SVT, MMO,
10986 AM: ISD::UNINDEXED, IsTruncating: true);
10987}
10988
10989SDValue SelectionDAG::getIndexedStore(SDValue OrigStore, const SDLoc &dl,
10990 SDValue Base, SDValue Offset,
10991 ISD::MemIndexedMode AM) {
10992 StoreSDNode *ST = cast<StoreSDNode>(Val&: OrigStore);
10993 assert(ST->getOffset().getOpcode() == ISD::POISON &&
10994 "Store is already a indexed store!");
10995 return getStore(Chain: ST->getChain(), dl, Val: ST->getValue(), Ptr: Base, Offset,
10996 SVT: ST->getMemoryVT(), MMO: ST->getMemOperand(), AM,
10997 IsTruncating: ST->isTruncatingStore());
10998}
10999
11000SDValue SelectionDAG::getLoadVP(
11001 ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &dl,
11002 SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL,
11003 MachinePointerInfo PtrInfo, EVT MemVT, Align Alignment,
11004 MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo,
11005 const MDNode *Ranges, bool IsExpanding) {
11006 MMOFlags |= MachineMemOperand::MOLoad;
11007 assert((MMOFlags & MachineMemOperand::MOStore) == 0);
11008 // If we don't have a PtrInfo, infer the trivial frame index case to simplify
11009 // clients.
11010 if (PtrInfo.V.isNull())
11011 PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr, OffsetOp: Offset);
11012
11013 TypeSize Size = MemVT.getStoreSize();
11014 MachineFunction &MF = getMachineFunction();
11015 MachineMemOperand *MMO = MF.getMachineMemOperand(
11016 PtrInfo, F: MMOFlags, Size, BaseAlignment: Alignment, Metadata: MMOMetadata(AAInfo, Ranges));
11017 return getLoadVP(AM, ExtType, VT, dl, Chain, Ptr, Offset, Mask, EVL, MemVT,
11018 MMO, IsExpanding);
11019}
11020
11021SDValue SelectionDAG::getLoadVP(ISD::MemIndexedMode AM,
11022 ISD::LoadExtType ExtType, EVT VT,
11023 const SDLoc &dl, SDValue Chain, SDValue Ptr,
11024 SDValue Offset, SDValue Mask, SDValue EVL,
11025 EVT MemVT, MachineMemOperand *MMO,
11026 bool IsExpanding) {
11027 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
11028 assert(Mask.getValueType().getVectorElementCount() ==
11029 VT.getVectorElementCount() &&
11030 "Vector width mismatch between mask and data");
11031
11032 bool Indexed = AM != ISD::UNINDEXED;
11033 assert((Indexed || Offset.getOpcode() == ISD::POISON) &&
11034 "Unindexed load with an offset!");
11035
11036 SDVTList VTs = Indexed ? getVTList(VT1: VT, VT2: Ptr.getValueType(), VT3: MVT::Other)
11037 : getVTList(VT1: VT, VT2: MVT::Other);
11038 SDValue Ops[] = {Chain, Ptr, Offset, Mask, EVL};
11039 FoldingSetNodeID ID;
11040 AddNodeIDNode(ID, OpC: ISD::VP_LOAD, VTList: VTs, OpList: Ops);
11041 ID.AddInteger(I: MemVT.getRawBits());
11042 ID.AddInteger(I: getSyntheticNodeSubclassData<VPLoadSDNode>(
11043 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: ExtType, Args&: IsExpanding, Args&: MemVT, Args&: MMO));
11044 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11045 ID.AddInteger(I: MMO->getFlags());
11046 FoldingSetInsertToken InsertToken;
11047 if (auto *E = cast_or_null<VPLoadSDNode>(Val: lookupNode(ID, DL: dl, InsertToken))) {
11048 E->refineAlignment(NewMMO: MMO);
11049 E->refineMMOMetadata(NewMMO: MMO);
11050 return SDValue(E, 0);
11051 }
11052 auto *N = newSDNode<VPLoadSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM,
11053 Args&: ExtType, Args&: IsExpanding, Args&: MemVT, Args&: MMO);
11054 createOperands(Node: N, Vals: Ops);
11055
11056 CSEMap.insert(N, Token: InsertToken);
11057 InsertNode(N);
11058 SDValue V(N, 0);
11059 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11060 return V;
11061}
11062
11063SDValue SelectionDAG::getLoadVP(EVT VT, const SDLoc &dl, SDValue Chain,
11064 SDValue Ptr, SDValue Mask, SDValue EVL,
11065 MachinePointerInfo PtrInfo,
11066 MaybeAlign Alignment,
11067 MachineMemOperand::Flags MMOFlags,
11068 const AAMDNodes &AAInfo, const MDNode *Ranges,
11069 bool IsExpanding) {
11070 SDValue Undef = getPOISON(VT: Ptr.getValueType());
11071 return getLoadVP(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Offset: Undef,
11072 Mask, EVL, PtrInfo, MemVT: VT, Alignment, MMOFlags, AAInfo, Ranges,
11073 IsExpanding);
11074}
11075
11076SDValue SelectionDAG::getLoadVP(EVT VT, const SDLoc &dl, SDValue Chain,
11077 SDValue Ptr, SDValue Mask, SDValue EVL,
11078 MachineMemOperand *MMO, bool IsExpanding) {
11079 SDValue Undef = getPOISON(VT: Ptr.getValueType());
11080 return getLoadVP(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Offset: Undef,
11081 Mask, EVL, MemVT: VT, MMO, IsExpanding);
11082}
11083
11084SDValue SelectionDAG::getExtLoadVP(ISD::LoadExtType ExtType, const SDLoc &dl,
11085 EVT VT, SDValue Chain, SDValue Ptr,
11086 SDValue Mask, SDValue EVL,
11087 MachinePointerInfo PtrInfo, EVT MemVT,
11088 MaybeAlign Alignment,
11089 MachineMemOperand::Flags MMOFlags,
11090 const AAMDNodes &AAInfo, bool IsExpanding) {
11091 SDValue Undef = getPOISON(VT: Ptr.getValueType());
11092 return getLoadVP(AM: ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Offset: Undef, Mask,
11093 EVL, PtrInfo, MemVT, Alignment, MMOFlags, AAInfo, Ranges: nullptr,
11094 IsExpanding);
11095}
11096
11097SDValue SelectionDAG::getExtLoadVP(ISD::LoadExtType ExtType, const SDLoc &dl,
11098 EVT VT, SDValue Chain, SDValue Ptr,
11099 SDValue Mask, SDValue EVL, EVT MemVT,
11100 MachineMemOperand *MMO, bool IsExpanding) {
11101 SDValue Undef = getPOISON(VT: Ptr.getValueType());
11102 return getLoadVP(AM: ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Offset: Undef, Mask,
11103 EVL, MemVT, MMO, IsExpanding);
11104}
11105
11106SDValue SelectionDAG::getIndexedLoadVP(SDValue OrigLoad, const SDLoc &dl,
11107 SDValue Base, SDValue Offset,
11108 ISD::MemIndexedMode AM) {
11109 auto *LD = cast<VPLoadSDNode>(Val&: OrigLoad);
11110 assert(LD->getOffset().getOpcode() == ISD::POISON &&
11111 "Load is already a indexed load!");
11112 // Don't propagate the invariant or dereferenceable flags.
11113 auto MMOFlags =
11114 LD->getMemOperand()->getFlags() &
11115 ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
11116 return getLoadVP(AM, ExtType: LD->getExtensionType(), VT: OrigLoad.getValueType(), dl,
11117 Chain: LD->getChain(), Ptr: Base, Offset, Mask: LD->getMask(),
11118 EVL: LD->getVectorLength(), PtrInfo: LD->getPointerInfo(),
11119 MemVT: LD->getMemoryVT(), Alignment: LD->getAlign(), MMOFlags, AAInfo: LD->getAAInfo(),
11120 Ranges: nullptr, IsExpanding: LD->isExpandingLoad());
11121}
11122
11123SDValue SelectionDAG::getStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val,
11124 SDValue Ptr, SDValue Offset, SDValue Mask,
11125 SDValue EVL, EVT MemVT, MachineMemOperand *MMO,
11126 ISD::MemIndexedMode AM, bool IsTruncating,
11127 bool IsCompressing) {
11128 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
11129 assert(Mask.getValueType().getVectorElementCount() ==
11130 Val.getValueType().getVectorElementCount() &&
11131 "Vector width mismatch between mask and data");
11132
11133 bool Indexed = AM != ISD::UNINDEXED;
11134 assert((Indexed || Offset.getOpcode() == ISD::POISON) &&
11135 "Unindexed vp_store with an offset!");
11136 SDVTList VTs = Indexed ? getVTList(VT1: Ptr.getValueType(), VT2: MVT::Other)
11137 : getVTList(VT: MVT::Other);
11138 SDValue Ops[] = {Chain, Val, Ptr, Offset, Mask, EVL};
11139 FoldingSetNodeID ID;
11140 AddNodeIDNode(ID, OpC: ISD::VP_STORE, VTList: VTs, OpList: Ops);
11141 ID.AddInteger(I: MemVT.getRawBits());
11142 ID.AddInteger(I: getSyntheticNodeSubclassData<VPStoreSDNode>(
11143 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO));
11144 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11145 ID.AddInteger(I: MMO->getFlags());
11146 FoldingSetInsertToken InsertToken;
11147 if (SDNode *E = lookupNode(ID, DL: dl, InsertToken)) {
11148 cast<VPStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11149 return SDValue(E, 0);
11150 }
11151 auto *N = newSDNode<VPStoreSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM,
11152 Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO);
11153 createOperands(Node: N, Vals: Ops);
11154
11155 CSEMap.insert(N, Token: InsertToken);
11156 InsertNode(N);
11157 SDValue V(N, 0);
11158 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11159 return V;
11160}
11161
11162SDValue SelectionDAG::getTruncStoreVP(SDValue Chain, const SDLoc &dl,
11163 SDValue Val, SDValue Ptr, SDValue Mask,
11164 SDValue EVL, MachinePointerInfo PtrInfo,
11165 EVT SVT, Align Alignment,
11166 MachineMemOperand::Flags MMOFlags,
11167 const AAMDNodes &AAInfo,
11168 bool IsCompressing) {
11169 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
11170
11171 MMOFlags |= MachineMemOperand::MOStore;
11172 assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
11173
11174 if (PtrInfo.V.isNull())
11175 PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr);
11176
11177 MachineFunction &MF = getMachineFunction();
11178 MachineMemOperand *MMO = MF.getMachineMemOperand(
11179 PtrInfo, F: MMOFlags, Size: SVT.getStoreSize(), BaseAlignment: Alignment, Metadata: AAInfo);
11180 return getTruncStoreVP(Chain, dl, Val, Ptr, Mask, EVL, SVT, MMO,
11181 IsCompressing);
11182}
11183
11184SDValue SelectionDAG::getTruncStoreVP(SDValue Chain, const SDLoc &dl,
11185 SDValue Val, SDValue Ptr, SDValue Mask,
11186 SDValue EVL, EVT SVT,
11187 MachineMemOperand *MMO,
11188 bool IsCompressing) {
11189 EVT VT = Val.getValueType();
11190
11191 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
11192 if (VT == SVT)
11193 return getStoreVP(Chain, dl, Val, Ptr, Offset: getPOISON(VT: Ptr.getValueType()), Mask,
11194 EVL, MemVT: VT, MMO, AM: ISD::UNINDEXED,
11195 /*IsTruncating*/ false, IsCompressing);
11196
11197 assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&
11198 "Should only be a truncating store, not extending!");
11199 assert(VT.isInteger() == SVT.isInteger() && "Can't do FP-INT conversion!");
11200 assert(VT.isVector() == SVT.isVector() &&
11201 "Cannot use trunc store to convert to or from a vector!");
11202 assert((!VT.isVector() ||
11203 VT.getVectorElementCount() == SVT.getVectorElementCount()) &&
11204 "Cannot use trunc store to change the number of vector elements!");
11205
11206 SDVTList VTs = getVTList(VT: MVT::Other);
11207 SDValue Undef = getPOISON(VT: Ptr.getValueType());
11208 SDValue Ops[] = {Chain, Val, Ptr, Undef, Mask, EVL};
11209 FoldingSetNodeID ID;
11210 AddNodeIDNode(ID, OpC: ISD::VP_STORE, VTList: VTs, OpList: Ops);
11211 ID.AddInteger(I: SVT.getRawBits());
11212 ID.AddInteger(I: getSyntheticNodeSubclassData<VPStoreSDNode>(
11213 IROrder: dl.getIROrder(), Args&: VTs, Args: ISD::UNINDEXED, Args: true, Args&: IsCompressing, Args&: SVT, Args&: MMO));
11214 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11215 ID.AddInteger(I: MMO->getFlags());
11216 FoldingSetInsertToken InsertToken;
11217 if (SDNode *E = lookupNode(ID, DL: dl, InsertToken)) {
11218 cast<VPStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11219 return SDValue(E, 0);
11220 }
11221 auto *N =
11222 newSDNode<VPStoreSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs,
11223 Args: ISD::UNINDEXED, Args: true, Args&: IsCompressing, Args&: SVT, Args&: MMO);
11224 createOperands(Node: N, Vals: Ops);
11225
11226 CSEMap.insert(N, Token: InsertToken);
11227 InsertNode(N);
11228 SDValue V(N, 0);
11229 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11230 return V;
11231}
11232
11233SDValue SelectionDAG::getIndexedStoreVP(SDValue OrigStore, const SDLoc &dl,
11234 SDValue Base, SDValue Offset,
11235 ISD::MemIndexedMode AM) {
11236 auto *ST = cast<VPStoreSDNode>(Val&: OrigStore);
11237 assert(ST->getOffset().getOpcode() == ISD::POISON &&
11238 "Store is already an indexed store!");
11239 SDVTList VTs = getVTList(VT1: Base.getValueType(), VT2: MVT::Other);
11240 SDValue Ops[] = {ST->getChain(), ST->getValue(), Base,
11241 Offset, ST->getMask(), ST->getVectorLength()};
11242 FoldingSetNodeID ID;
11243 AddNodeIDNode(ID, OpC: ISD::VP_STORE, VTList: VTs, OpList: Ops);
11244 ID.AddInteger(I: ST->getMemoryVT().getRawBits());
11245 ID.AddInteger(I: ST->getRawSubclassData());
11246 ID.AddInteger(I: ST->getPointerInfo().getAddrSpace());
11247 ID.AddInteger(I: ST->getMemOperand()->getFlags());
11248 FoldingSetInsertToken InsertToken;
11249 if (SDNode *E = lookupNode(ID, DL: dl, InsertToken))
11250 return SDValue(E, 0);
11251
11252 auto *N = newSDNode<VPStoreSDNode>(
11253 Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM, Args: ST->isTruncatingStore(),
11254 Args: ST->isCompressingStore(), Args: ST->getMemoryVT(), Args: ST->getMemOperand());
11255 createOperands(Node: N, Vals: Ops);
11256
11257 CSEMap.insert(N, Token: InsertToken);
11258 InsertNode(N);
11259 SDValue V(N, 0);
11260 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11261 return V;
11262}
11263
11264SDValue SelectionDAG::getStridedLoadVP(
11265 ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &DL,
11266 SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask,
11267 SDValue EVL, EVT MemVT, MachineMemOperand *MMO, bool IsExpanding) {
11268 bool Indexed = AM != ISD::UNINDEXED;
11269 assert((Indexed || Offset.getOpcode() == ISD::POISON) &&
11270 "Unindexed load with an offset!");
11271
11272 SDValue Ops[] = {Chain, Ptr, Offset, Stride, Mask, EVL};
11273 SDVTList VTs = Indexed ? getVTList(VT1: VT, VT2: Ptr.getValueType(), VT3: MVT::Other)
11274 : getVTList(VT1: VT, VT2: MVT::Other);
11275 FoldingSetNodeID ID;
11276 AddNodeIDNode(ID, OpC: ISD::EXPERIMENTAL_VP_STRIDED_LOAD, VTList: VTs, OpList: Ops);
11277 ID.AddInteger(I: VT.getRawBits());
11278 ID.AddInteger(I: getSyntheticNodeSubclassData<VPStridedLoadSDNode>(
11279 IROrder: DL.getIROrder(), Args&: VTs, Args&: AM, Args&: ExtType, Args&: IsExpanding, Args&: MemVT, Args&: MMO));
11280 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11281
11282 FoldingSetInsertToken InsertToken;
11283 if (SDNode *E = lookupNode(ID, DL, InsertToken)) {
11284 cast<VPStridedLoadSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11285 return SDValue(E, 0);
11286 }
11287
11288 auto *N =
11289 newSDNode<VPStridedLoadSDNode>(Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs, Args&: AM,
11290 Args&: ExtType, Args&: IsExpanding, Args&: MemVT, Args&: MMO);
11291 createOperands(Node: N, Vals: Ops);
11292 CSEMap.insert(N, Token: InsertToken);
11293 InsertNode(N);
11294 SDValue V(N, 0);
11295 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11296 return V;
11297}
11298
11299SDValue SelectionDAG::getStridedLoadVP(EVT VT, const SDLoc &DL, SDValue Chain,
11300 SDValue Ptr, SDValue Stride,
11301 SDValue Mask, SDValue EVL,
11302 MachineMemOperand *MMO,
11303 bool IsExpanding) {
11304 SDValue Undef = getPOISON(VT: Ptr.getValueType());
11305 return getStridedLoadVP(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, DL, Chain, Ptr,
11306 Offset: Undef, Stride, Mask, EVL, MemVT: VT, MMO, IsExpanding);
11307}
11308
11309SDValue SelectionDAG::getExtStridedLoadVP(
11310 ISD::LoadExtType ExtType, const SDLoc &DL, EVT VT, SDValue Chain,
11311 SDValue Ptr, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT,
11312 MachineMemOperand *MMO, bool IsExpanding) {
11313 SDValue Undef = getPOISON(VT: Ptr.getValueType());
11314 return getStridedLoadVP(AM: ISD::UNINDEXED, ExtType, VT, DL, Chain, Ptr, Offset: Undef,
11315 Stride, Mask, EVL, MemVT, MMO, IsExpanding);
11316}
11317
11318SDValue SelectionDAG::getStridedStoreVP(SDValue Chain, const SDLoc &DL,
11319 SDValue Val, SDValue Ptr,
11320 SDValue Offset, SDValue Stride,
11321 SDValue Mask, SDValue EVL, EVT MemVT,
11322 MachineMemOperand *MMO,
11323 ISD::MemIndexedMode AM,
11324 bool IsTruncating, bool IsCompressing) {
11325 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
11326 bool Indexed = AM != ISD::UNINDEXED;
11327 assert((Indexed || Offset.getOpcode() == ISD::POISON) &&
11328 "Unindexed vp_store with an offset!");
11329 SDVTList VTs = Indexed ? getVTList(VT1: Ptr.getValueType(), VT2: MVT::Other)
11330 : getVTList(VT: MVT::Other);
11331 SDValue Ops[] = {Chain, Val, Ptr, Offset, Stride, Mask, EVL};
11332 FoldingSetNodeID ID;
11333 AddNodeIDNode(ID, OpC: ISD::EXPERIMENTAL_VP_STRIDED_STORE, VTList: VTs, OpList: Ops);
11334 ID.AddInteger(I: MemVT.getRawBits());
11335 ID.AddInteger(I: getSyntheticNodeSubclassData<VPStridedStoreSDNode>(
11336 IROrder: DL.getIROrder(), Args&: VTs, Args&: AM, Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO));
11337 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11338 FoldingSetInsertToken InsertToken;
11339 if (SDNode *E = lookupNode(ID, DL, InsertToken)) {
11340 cast<VPStridedStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11341 return SDValue(E, 0);
11342 }
11343 auto *N = newSDNode<VPStridedStoreSDNode>(Args: DL.getIROrder(), Args: DL.getDebugLoc(),
11344 Args&: VTs, Args&: AM, Args&: IsTruncating,
11345 Args&: IsCompressing, Args&: MemVT, Args&: MMO);
11346 createOperands(Node: N, Vals: Ops);
11347
11348 CSEMap.insert(N, Token: InsertToken);
11349 InsertNode(N);
11350 SDValue V(N, 0);
11351 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11352 return V;
11353}
11354
11355SDValue SelectionDAG::getTruncStridedStoreVP(SDValue Chain, const SDLoc &DL,
11356 SDValue Val, SDValue Ptr,
11357 SDValue Stride, SDValue Mask,
11358 SDValue EVL, EVT SVT,
11359 MachineMemOperand *MMO,
11360 bool IsCompressing) {
11361 EVT VT = Val.getValueType();
11362
11363 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
11364 if (VT == SVT)
11365 return getStridedStoreVP(Chain, DL, Val, Ptr, Offset: getPOISON(VT: Ptr.getValueType()),
11366 Stride, Mask, EVL, MemVT: VT, MMO, AM: ISD::UNINDEXED,
11367 /*IsTruncating*/ false, IsCompressing);
11368
11369 assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&
11370 "Should only be a truncating store, not extending!");
11371 assert(VT.isInteger() == SVT.isInteger() && "Can't do FP-INT conversion!");
11372 assert(VT.isVector() == SVT.isVector() &&
11373 "Cannot use trunc store to convert to or from a vector!");
11374 assert((!VT.isVector() ||
11375 VT.getVectorElementCount() == SVT.getVectorElementCount()) &&
11376 "Cannot use trunc store to change the number of vector elements!");
11377
11378 SDVTList VTs = getVTList(VT: MVT::Other);
11379 SDValue Undef = getPOISON(VT: Ptr.getValueType());
11380 SDValue Ops[] = {Chain, Val, Ptr, Undef, Stride, Mask, EVL};
11381 FoldingSetNodeID ID;
11382 AddNodeIDNode(ID, OpC: ISD::EXPERIMENTAL_VP_STRIDED_STORE, VTList: VTs, OpList: Ops);
11383 ID.AddInteger(I: SVT.getRawBits());
11384 ID.AddInteger(I: getSyntheticNodeSubclassData<VPStridedStoreSDNode>(
11385 IROrder: DL.getIROrder(), Args&: VTs, Args: ISD::UNINDEXED, Args: true, Args&: IsCompressing, Args&: SVT, Args&: MMO));
11386 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11387 FoldingSetInsertToken InsertToken;
11388 if (SDNode *E = lookupNode(ID, DL, InsertToken)) {
11389 cast<VPStridedStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11390 return SDValue(E, 0);
11391 }
11392 auto *N = newSDNode<VPStridedStoreSDNode>(Args: DL.getIROrder(), Args: DL.getDebugLoc(),
11393 Args&: VTs, Args: ISD::UNINDEXED, Args: true,
11394 Args&: IsCompressing, Args&: SVT, Args&: MMO);
11395 createOperands(Node: N, Vals: Ops);
11396
11397 CSEMap.insert(N, Token: InsertToken);
11398 InsertNode(N);
11399 SDValue V(N, 0);
11400 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11401 return V;
11402}
11403
11404SDValue SelectionDAG::getGatherVP(SDVTList VTs, EVT VT, const SDLoc &dl,
11405 ArrayRef<SDValue> Ops, MachineMemOperand *MMO,
11406 ISD::MemIndexType IndexType) {
11407 assert(Ops.size() == 6 && "Incompatible number of operands");
11408
11409 FoldingSetNodeID ID;
11410 AddNodeIDNode(ID, OpC: ISD::VP_GATHER, VTList: VTs, OpList: Ops);
11411 ID.AddInteger(I: VT.getRawBits());
11412 ID.AddInteger(I: getSyntheticNodeSubclassData<VPGatherSDNode>(
11413 IROrder: dl.getIROrder(), Args&: VTs, Args&: VT, Args&: MMO, Args&: IndexType));
11414 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11415 ID.AddInteger(I: MMO->getFlags());
11416 FoldingSetInsertToken InsertToken;
11417 if (SDNode *E = lookupNode(ID, DL: dl, InsertToken)) {
11418 cast<VPGatherSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11419 return SDValue(E, 0);
11420 }
11421
11422 auto *N = newSDNode<VPGatherSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs,
11423 Args&: VT, Args&: MMO, Args&: IndexType);
11424 createOperands(Node: N, Vals: Ops);
11425
11426 assert(N->getMask().getValueType().getVectorElementCount() ==
11427 N->getValueType(0).getVectorElementCount() &&
11428 "Vector width mismatch between mask and data");
11429 assert(N->getIndex().getValueType().getVectorElementCount().isScalable() ==
11430 N->getValueType(0).getVectorElementCount().isScalable() &&
11431 "Scalable flags of index and data do not match");
11432 assert(ElementCount::isKnownGE(
11433 N->getIndex().getValueType().getVectorElementCount(),
11434 N->getValueType(0).getVectorElementCount()) &&
11435 "Vector width mismatch between index and data");
11436 assert(isa<ConstantSDNode>(N->getScale()) &&
11437 N->getScale()->getAsAPIntVal().isPowerOf2() &&
11438 "Scale should be a constant power of 2");
11439
11440 CSEMap.insert(N, Token: InsertToken);
11441 InsertNode(N);
11442 SDValue V(N, 0);
11443 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11444 return V;
11445}
11446
11447SDValue SelectionDAG::getScatterVP(SDVTList VTs, EVT VT, const SDLoc &dl,
11448 ArrayRef<SDValue> Ops,
11449 MachineMemOperand *MMO,
11450 ISD::MemIndexType IndexType) {
11451 assert(Ops.size() == 7 && "Incompatible number of operands");
11452
11453 FoldingSetNodeID ID;
11454 AddNodeIDNode(ID, OpC: ISD::VP_SCATTER, VTList: VTs, OpList: Ops);
11455 ID.AddInteger(I: VT.getRawBits());
11456 ID.AddInteger(I: getSyntheticNodeSubclassData<VPScatterSDNode>(
11457 IROrder: dl.getIROrder(), Args&: VTs, Args&: VT, Args&: MMO, Args&: IndexType));
11458 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11459 ID.AddInteger(I: MMO->getFlags());
11460 FoldingSetInsertToken InsertToken;
11461 if (SDNode *E = lookupNode(ID, DL: dl, InsertToken)) {
11462 cast<VPScatterSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11463 return SDValue(E, 0);
11464 }
11465 auto *N = newSDNode<VPScatterSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs,
11466 Args&: VT, Args&: MMO, Args&: IndexType);
11467 createOperands(Node: N, Vals: Ops);
11468
11469 assert(N->getMask().getValueType().getVectorElementCount() ==
11470 N->getValue().getValueType().getVectorElementCount() &&
11471 "Vector width mismatch between mask and data");
11472 assert(
11473 N->getIndex().getValueType().getVectorElementCount().isScalable() ==
11474 N->getValue().getValueType().getVectorElementCount().isScalable() &&
11475 "Scalable flags of index and data do not match");
11476 assert(ElementCount::isKnownGE(
11477 N->getIndex().getValueType().getVectorElementCount(),
11478 N->getValue().getValueType().getVectorElementCount()) &&
11479 "Vector width mismatch between index and data");
11480 assert(isa<ConstantSDNode>(N->getScale()) &&
11481 N->getScale()->getAsAPIntVal().isPowerOf2() &&
11482 "Scale should be a constant power of 2");
11483
11484 CSEMap.insert(N, Token: InsertToken);
11485 InsertNode(N);
11486 SDValue V(N, 0);
11487 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11488 return V;
11489}
11490
11491SDValue SelectionDAG::getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain,
11492 SDValue Base, SDValue Offset, SDValue Mask,
11493 SDValue PassThru, EVT MemVT,
11494 MachineMemOperand *MMO,
11495 ISD::MemIndexedMode AM,
11496 ISD::LoadExtType ExtTy, bool isExpanding) {
11497 bool Indexed = AM != ISD::UNINDEXED;
11498 assert((Indexed || Offset.getOpcode() == ISD::POISON) &&
11499 "Unindexed masked load with an offset!");
11500 SDVTList VTs = Indexed ? getVTList(VT1: VT, VT2: Base.getValueType(), VT3: MVT::Other)
11501 : getVTList(VT1: VT, VT2: MVT::Other);
11502 SDValue Ops[] = {Chain, Base, Offset, Mask, PassThru};
11503 FoldingSetNodeID ID;
11504 AddNodeIDNode(ID, OpC: ISD::MLOAD, VTList: VTs, OpList: Ops);
11505 ID.AddInteger(I: MemVT.getRawBits());
11506 ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedLoadSDNode>(
11507 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: ExtTy, Args&: isExpanding, Args&: MemVT, Args&: MMO));
11508 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11509 ID.AddInteger(I: MMO->getFlags());
11510 FoldingSetInsertToken InsertToken;
11511 if (SDNode *E = lookupNode(ID, DL: dl, InsertToken)) {
11512 cast<MaskedLoadSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11513 return SDValue(E, 0);
11514 }
11515 auto *N = newSDNode<MaskedLoadSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs,
11516 Args&: AM, Args&: ExtTy, Args&: isExpanding, Args&: MemVT, Args&: MMO);
11517 createOperands(Node: N, Vals: Ops);
11518
11519 CSEMap.insert(N, Token: InsertToken);
11520 InsertNode(N);
11521 SDValue V(N, 0);
11522 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11523 return V;
11524}
11525
11526SDValue SelectionDAG::getIndexedMaskedLoad(SDValue OrigLoad, const SDLoc &dl,
11527 SDValue Base, SDValue Offset,
11528 ISD::MemIndexedMode AM) {
11529 MaskedLoadSDNode *LD = cast<MaskedLoadSDNode>(Val&: OrigLoad);
11530 assert(LD->getOffset().getOpcode() == ISD::POISON &&
11531 "Masked load is already a indexed load!");
11532 return getMaskedLoad(VT: OrigLoad.getValueType(), dl, Chain: LD->getChain(), Base,
11533 Offset, Mask: LD->getMask(), PassThru: LD->getPassThru(),
11534 MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand(), AM,
11535 ExtTy: LD->getExtensionType(), isExpanding: LD->isExpandingLoad());
11536}
11537
11538SDValue SelectionDAG::getMaskedStore(SDValue Chain, const SDLoc &dl,
11539 SDValue Val, SDValue Base, SDValue Offset,
11540 SDValue Mask, EVT MemVT,
11541 MachineMemOperand *MMO,
11542 ISD::MemIndexedMode AM, bool IsTruncating,
11543 bool IsCompressing) {
11544 assert(Chain.getValueType() == MVT::Other &&
11545 "Invalid chain type");
11546 bool Indexed = AM != ISD::UNINDEXED;
11547 assert((Indexed || Offset.getOpcode() == ISD::POISON) &&
11548 "Unindexed masked store with an offset!");
11549 SDVTList VTs = Indexed ? getVTList(VT1: Base.getValueType(), VT2: MVT::Other)
11550 : getVTList(VT: MVT::Other);
11551 SDValue Ops[] = {Chain, Val, Base, Offset, Mask};
11552 FoldingSetNodeID ID;
11553 AddNodeIDNode(ID, OpC: ISD::MSTORE, VTList: VTs, OpList: Ops);
11554 ID.AddInteger(I: MemVT.getRawBits());
11555 ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedStoreSDNode>(
11556 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO));
11557 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11558 ID.AddInteger(I: MMO->getFlags());
11559 FoldingSetInsertToken InsertToken;
11560 if (SDNode *E = lookupNode(ID, DL: dl, InsertToken)) {
11561 cast<MaskedStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11562 return SDValue(E, 0);
11563 }
11564 auto *N =
11565 newSDNode<MaskedStoreSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM,
11566 Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO);
11567 createOperands(Node: N, Vals: Ops);
11568
11569 CSEMap.insert(N, Token: InsertToken);
11570 InsertNode(N);
11571 SDValue V(N, 0);
11572 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11573 return V;
11574}
11575
11576SDValue SelectionDAG::getIndexedMaskedStore(SDValue OrigStore, const SDLoc &dl,
11577 SDValue Base, SDValue Offset,
11578 ISD::MemIndexedMode AM) {
11579 MaskedStoreSDNode *ST = cast<MaskedStoreSDNode>(Val&: OrigStore);
11580 assert(ST->getOffset().getOpcode() == ISD::POISON &&
11581 "Masked store is already a indexed store!");
11582 return getMaskedStore(Chain: ST->getChain(), dl, Val: ST->getValue(), Base, Offset,
11583 Mask: ST->getMask(), MemVT: ST->getMemoryVT(), MMO: ST->getMemOperand(),
11584 AM, IsTruncating: ST->isTruncatingStore(), IsCompressing: ST->isCompressingStore());
11585}
11586
11587SDValue SelectionDAG::getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl,
11588 ArrayRef<SDValue> Ops,
11589 MachineMemOperand *MMO,
11590 ISD::MemIndexType IndexType,
11591 ISD::LoadExtType ExtTy) {
11592 assert(Ops.size() == 6 && "Incompatible number of operands");
11593
11594 FoldingSetNodeID ID;
11595 AddNodeIDNode(ID, OpC: ISD::MGATHER, VTList: VTs, OpList: Ops);
11596 ID.AddInteger(I: MemVT.getRawBits());
11597 ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedGatherSDNode>(
11598 IROrder: dl.getIROrder(), Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType, Args&: ExtTy));
11599 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11600 ID.AddInteger(I: MMO->getFlags());
11601 FoldingSetInsertToken InsertToken;
11602 if (SDNode *E = lookupNode(ID, DL: dl, InsertToken)) {
11603 cast<MaskedGatherSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11604 return SDValue(E, 0);
11605 }
11606
11607 auto *N = newSDNode<MaskedGatherSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(),
11608 Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType, Args&: ExtTy);
11609 createOperands(Node: N, Vals: Ops);
11610
11611 assert(N->getPassThru().getValueType() == N->getValueType(0) &&
11612 "Incompatible type of the PassThru value in MaskedGatherSDNode");
11613 assert(N->getMask().getValueType().getVectorElementCount() ==
11614 N->getValueType(0).getVectorElementCount() &&
11615 "Vector width mismatch between mask and data");
11616 assert(N->getIndex().getValueType().getVectorElementCount().isScalable() ==
11617 N->getValueType(0).getVectorElementCount().isScalable() &&
11618 "Scalable flags of index and data do not match");
11619 assert(ElementCount::isKnownGE(
11620 N->getIndex().getValueType().getVectorElementCount(),
11621 N->getValueType(0).getVectorElementCount()) &&
11622 "Vector width mismatch between index and data");
11623 assert(isa<ConstantSDNode>(N->getScale()) &&
11624 N->getScale()->getAsAPIntVal().isPowerOf2() &&
11625 "Scale should be a constant power of 2");
11626
11627 CSEMap.insert(N, Token: InsertToken);
11628 InsertNode(N);
11629 SDValue V(N, 0);
11630 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11631 return V;
11632}
11633
11634SDValue SelectionDAG::getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl,
11635 ArrayRef<SDValue> Ops,
11636 MachineMemOperand *MMO,
11637 ISD::MemIndexType IndexType,
11638 bool IsTrunc) {
11639 assert(Ops.size() == 6 && "Incompatible number of operands");
11640
11641 FoldingSetNodeID ID;
11642 AddNodeIDNode(ID, OpC: ISD::MSCATTER, VTList: VTs, OpList: Ops);
11643 ID.AddInteger(I: MemVT.getRawBits());
11644 ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedScatterSDNode>(
11645 IROrder: dl.getIROrder(), Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType, Args&: IsTrunc));
11646 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11647 ID.AddInteger(I: MMO->getFlags());
11648 FoldingSetInsertToken InsertToken;
11649 if (SDNode *E = lookupNode(ID, DL: dl, InsertToken)) {
11650 cast<MaskedScatterSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11651 return SDValue(E, 0);
11652 }
11653
11654 auto *N = newSDNode<MaskedScatterSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(),
11655 Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType, Args&: IsTrunc);
11656 createOperands(Node: N, Vals: Ops);
11657
11658 assert(N->getMask().getValueType().getVectorElementCount() ==
11659 N->getValue().getValueType().getVectorElementCount() &&
11660 "Vector width mismatch between mask and data");
11661 assert(
11662 N->getIndex().getValueType().getVectorElementCount().isScalable() ==
11663 N->getValue().getValueType().getVectorElementCount().isScalable() &&
11664 "Scalable flags of index and data do not match");
11665 assert(ElementCount::isKnownGE(
11666 N->getIndex().getValueType().getVectorElementCount(),
11667 N->getValue().getValueType().getVectorElementCount()) &&
11668 "Vector width mismatch between index and data");
11669 assert(isa<ConstantSDNode>(N->getScale()) &&
11670 N->getScale()->getAsAPIntVal().isPowerOf2() &&
11671 "Scale should be a constant power of 2");
11672
11673 CSEMap.insert(N, Token: InsertToken);
11674 InsertNode(N);
11675 SDValue V(N, 0);
11676 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11677 return V;
11678}
11679
11680SDValue SelectionDAG::getMaskedHistogram(SDVTList VTs, EVT MemVT,
11681 const SDLoc &dl, ArrayRef<SDValue> Ops,
11682 MachineMemOperand *MMO,
11683 ISD::MemIndexType IndexType) {
11684 assert(Ops.size() == 7 && "Incompatible number of operands");
11685
11686 FoldingSetNodeID ID;
11687 AddNodeIDNode(ID, OpC: ISD::EXPERIMENTAL_VECTOR_HISTOGRAM, VTList: VTs, OpList: Ops);
11688 ID.AddInteger(I: MemVT.getRawBits());
11689 ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedHistogramSDNode>(
11690 IROrder: dl.getIROrder(), Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType));
11691 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11692 ID.AddInteger(I: MMO->getFlags());
11693 FoldingSetInsertToken InsertToken;
11694 if (SDNode *E = lookupNode(ID, DL: dl, InsertToken)) {
11695 cast<MaskedGatherSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11696 return SDValue(E, 0);
11697 }
11698
11699 auto *N = newSDNode<MaskedHistogramSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(),
11700 Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType);
11701 createOperands(Node: N, Vals: Ops);
11702
11703 assert(N->getMask().getValueType().getVectorElementCount() ==
11704 N->getIndex().getValueType().getVectorElementCount() &&
11705 "Vector width mismatch between mask and data");
11706 assert(isa<ConstantSDNode>(N->getScale()) &&
11707 N->getScale()->getAsAPIntVal().isPowerOf2() &&
11708 "Scale should be a constant power of 2");
11709 assert(N->getInc().getValueType().isInteger() && "Non integer update value");
11710
11711 CSEMap.insert(N, Token: InsertToken);
11712 InsertNode(N);
11713 SDValue V(N, 0);
11714 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11715 return V;
11716}
11717
11718SDValue SelectionDAG::getLoadFFVP(EVT VT, const SDLoc &DL, SDValue Chain,
11719 SDValue Ptr, SDValue Mask, SDValue EVL,
11720 MachineMemOperand *MMO) {
11721 SDVTList VTs = getVTList(VT1: VT, VT2: EVL.getValueType(), VT3: MVT::Other);
11722 SDValue Ops[] = {Chain, Ptr, Mask, EVL};
11723 FoldingSetNodeID ID;
11724 AddNodeIDNode(ID, OpC: ISD::VP_LOAD_FF, VTList: VTs, OpList: Ops);
11725 ID.AddInteger(I: VT.getRawBits());
11726 ID.AddInteger(I: getSyntheticNodeSubclassData<VPLoadFFSDNode>(IROrder: DL.getIROrder(),
11727 Args&: VTs, Args&: VT, Args&: MMO));
11728 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11729 ID.AddInteger(I: MMO->getFlags());
11730 FoldingSetInsertToken InsertToken;
11731 if (SDNode *E = lookupNode(ID, DL, InsertToken)) {
11732 cast<VPLoadFFSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11733 return SDValue(E, 0);
11734 }
11735 auto *N = newSDNode<VPLoadFFSDNode>(Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs,
11736 Args&: VT, Args&: MMO);
11737 createOperands(Node: N, Vals: Ops);
11738
11739 CSEMap.insert(N, Token: InsertToken);
11740 InsertNode(N);
11741 SDValue V(N, 0);
11742 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11743 return V;
11744}
11745
11746SDValue SelectionDAG::getGetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr,
11747 EVT MemVT, MachineMemOperand *MMO) {
11748 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
11749 SDVTList VTs = getVTList(VT: MVT::Other);
11750 SDValue Ops[] = {Chain, Ptr};
11751 FoldingSetNodeID ID;
11752 AddNodeIDNode(ID, OpC: ISD::GET_FPENV_MEM, VTList: VTs, OpList: Ops);
11753 ID.AddInteger(I: MemVT.getRawBits());
11754 ID.AddInteger(I: getSyntheticNodeSubclassData<FPStateAccessSDNode>(
11755 Opc: ISD::GET_FPENV_MEM, Order: dl.getIROrder(), VTs, MemoryVT: MemVT, MMO));
11756 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11757 ID.AddInteger(I: MMO->getFlags());
11758 FoldingSetInsertToken InsertToken;
11759 if (SDNode *E = lookupNode(ID, DL: dl, InsertToken))
11760 return SDValue(E, 0);
11761
11762 auto *N = newSDNode<FPStateAccessSDNode>(Args: ISD::GET_FPENV_MEM, Args: dl.getIROrder(),
11763 Args: dl.getDebugLoc(), Args&: VTs, Args&: MemVT, Args&: MMO);
11764 createOperands(Node: N, Vals: Ops);
11765
11766 CSEMap.insert(N, Token: InsertToken);
11767 InsertNode(N);
11768 SDValue V(N, 0);
11769 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11770 return V;
11771}
11772
11773SDValue SelectionDAG::getSetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr,
11774 EVT MemVT, MachineMemOperand *MMO) {
11775 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
11776 SDVTList VTs = getVTList(VT: MVT::Other);
11777 SDValue Ops[] = {Chain, Ptr};
11778 FoldingSetNodeID ID;
11779 AddNodeIDNode(ID, OpC: ISD::SET_FPENV_MEM, VTList: VTs, OpList: Ops);
11780 ID.AddInteger(I: MemVT.getRawBits());
11781 ID.AddInteger(I: getSyntheticNodeSubclassData<FPStateAccessSDNode>(
11782 Opc: ISD::SET_FPENV_MEM, Order: dl.getIROrder(), VTs, MemoryVT: MemVT, MMO));
11783 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11784 ID.AddInteger(I: MMO->getFlags());
11785 FoldingSetInsertToken InsertToken;
11786 if (SDNode *E = lookupNode(ID, DL: dl, InsertToken))
11787 return SDValue(E, 0);
11788
11789 auto *N = newSDNode<FPStateAccessSDNode>(Args: ISD::SET_FPENV_MEM, Args: dl.getIROrder(),
11790 Args: dl.getDebugLoc(), Args&: VTs, Args&: MemVT, Args&: MMO);
11791 createOperands(Node: N, Vals: Ops);
11792
11793 CSEMap.insert(N, Token: InsertToken);
11794 InsertNode(N);
11795 SDValue V(N, 0);
11796 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11797 return V;
11798}
11799
11800SDValue SelectionDAG::simplifySelect(SDValue Cond, SDValue T, SDValue F) {
11801 // select undef, T, F --> T (if T is a constant), otherwise F
11802 // select, ?, undef, F --> F
11803 // select, ?, T, undef --> T
11804 if (Cond.isUndef())
11805 return isConstantValueOfAnyType(N: T) ? T : F;
11806 if (T.isUndef())
11807 return isGuaranteedNotToBePoison(Op: F) ? F : getFreeze(V: F);
11808 if (F.isUndef())
11809 return isGuaranteedNotToBePoison(Op: T) ? T : getFreeze(V: T);
11810
11811 // select true, T, F --> T
11812 // select false, T, F --> F
11813 if (auto C = isBoolConstant(N: Cond))
11814 return *C ? T : F;
11815
11816 // select ?, T, T --> T
11817 if (T == F)
11818 return T;
11819
11820 return SDValue();
11821}
11822
11823SDValue SelectionDAG::simplifyShift(SDValue X, SDValue Y) {
11824 // shift undef, Y --> 0 (can always assume that the undef value is 0)
11825 if (X.isUndef())
11826 return getConstant(Val: 0, DL: SDLoc(X.getNode()), VT: X.getValueType());
11827 // shift X, undef --> undef (because it may shift by the bitwidth)
11828 if (Y.isUndef())
11829 return getUNDEF(VT: X.getValueType());
11830
11831 // shift 0, Y --> 0
11832 // shift X, 0 --> X
11833 if (isNullOrNullSplat(V: X) || isNullOrNullSplat(V: Y))
11834 return X;
11835
11836 // shift X, C >= bitwidth(X) --> undef
11837 // All vector elements must be too big (or undef) to avoid partial undefs.
11838 auto isShiftTooBig = [X](ConstantSDNode *Val) {
11839 return !Val || Val->getAPIntValue().uge(RHS: X.getScalarValueSizeInBits());
11840 };
11841 if (ISD::matchUnaryPredicate(Op: Y, Match: isShiftTooBig, AllowUndefs: true))
11842 return getUNDEF(VT: X.getValueType());
11843
11844 // shift i1/vXi1 X, Y --> X (any non-zero shift amount is undefined).
11845 if (X.getValueType().getScalarType() == MVT::i1)
11846 return X;
11847
11848 return SDValue();
11849}
11850
11851SDValue SelectionDAG::simplifyFPBinop(unsigned Opcode, SDValue X, SDValue Y,
11852 SDNodeFlags Flags) {
11853 // If this operation has 'nnan' or 'ninf' and at least 1 disallowed operand
11854 // (an undef operand can be chosen to be Nan/Inf), then the result of this
11855 // operation is poison. That result can be relaxed to undef.
11856 ConstantFPSDNode *XC = isConstOrConstSplatFP(N: X, /* AllowUndefs */ true);
11857 ConstantFPSDNode *YC = isConstOrConstSplatFP(N: Y, /* AllowUndefs */ true);
11858 bool HasNan = (XC && XC->getValueAPF().isNaN()) ||
11859 (YC && YC->getValueAPF().isNaN());
11860 bool HasInf = (XC && XC->getValueAPF().isInfinity()) ||
11861 (YC && YC->getValueAPF().isInfinity());
11862
11863 if (Flags.hasNoNaNs() && (HasNan || X.isUndef() || Y.isUndef()))
11864 return getUNDEF(VT: X.getValueType());
11865
11866 if (Flags.hasNoInfs() && (HasInf || X.isUndef() || Y.isUndef()))
11867 return getUNDEF(VT: X.getValueType());
11868
11869 if (!YC)
11870 return SDValue();
11871
11872 // X + -0.0 --> X
11873 if (Opcode == ISD::FADD)
11874 if (YC->getValueAPF().isNegZero())
11875 return X;
11876
11877 // X - +0.0 --> X
11878 if (Opcode == ISD::FSUB)
11879 if (YC->getValueAPF().isPosZero())
11880 return X;
11881
11882 // X * 1.0 --> X
11883 // X / 1.0 --> X
11884 if (Opcode == ISD::FMUL || Opcode == ISD::FDIV)
11885 if (YC->getValueAPF().isOne())
11886 return X;
11887
11888 // X * 0.0 --> 0.0
11889 if (Opcode == ISD::FMUL && Flags.hasNoNaNs() && Flags.hasNoSignedZeros())
11890 if (YC->getValueAPF().isZero())
11891 return getConstantFP(Val: 0.0, DL: SDLoc(Y), VT: Y.getValueType());
11892
11893 return SDValue();
11894}
11895
11896SDValue SelectionDAG::getVAArg(EVT VT, const SDLoc &dl, SDValue Chain,
11897 SDValue Ptr, SDValue SV, unsigned Align) {
11898 SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Val: Align, DL: dl, VT: MVT::i32) };
11899 return getNode(Opcode: ISD::VAARG, DL: dl, VTList: getVTList(VT1: VT, VT2: MVT::Other), Ops);
11900}
11901
11902SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
11903 ArrayRef<SDUse> Ops) {
11904 switch (Ops.size()) {
11905 case 0: return getNode(Opcode, DL, VT);
11906 case 1: return getNode(Opcode, DL, VT, N1: Ops[0].get());
11907 case 2: return getNode(Opcode, DL, VT, N1: Ops[0], N2: Ops[1]);
11908 case 3: return getNode(Opcode, DL, VT, N1: Ops[0], N2: Ops[1], N3: Ops[2]);
11909 default: break;
11910 }
11911
11912 // Copy from an SDUse array into an SDValue array for use with
11913 // the regular getNode logic.
11914 SmallVector<SDValue, 8> NewOps(Ops);
11915 return getNode(Opcode, DL, VT, Ops: NewOps);
11916}
11917
11918SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
11919 ArrayRef<SDValue> Ops) {
11920 SDNodeFlags Flags;
11921 if (Inserter)
11922 Flags = Inserter->getFlags();
11923 return getNode(Opcode, DL, VT, Ops, Flags);
11924}
11925
11926SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
11927 ArrayRef<SDValue> Ops, const SDNodeFlags Flags) {
11928 unsigned NumOps = Ops.size();
11929 switch (NumOps) {
11930 case 0: return getNode(Opcode, DL, VT);
11931 case 1: return getNode(Opcode, DL, VT, N1: Ops[0], Flags);
11932 case 2: return getNode(Opcode, DL, VT, N1: Ops[0], N2: Ops[1], Flags);
11933 case 3: return getNode(Opcode, DL, VT, N1: Ops[0], N2: Ops[1], N3: Ops[2], Flags);
11934 default: break;
11935 }
11936
11937#ifndef NDEBUG
11938 for (const auto &Op : Ops)
11939 assert(Op.getOpcode() != ISD::DELETED_NODE &&
11940 "Operand is DELETED_NODE!");
11941#endif
11942
11943 switch (Opcode) {
11944 default: break;
11945 case ISD::BUILD_VECTOR:
11946 // Attempt to simplify BUILD_VECTOR.
11947 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, DAG&: *this))
11948 return V;
11949 break;
11950 case ISD::CONCAT_VECTORS:
11951 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, DAG&: *this))
11952 return V;
11953 break;
11954 case ISD::SELECT_CC:
11955 assert(NumOps == 5 && "SELECT_CC takes 5 operands!");
11956 assert(Ops[0].getValueType() == Ops[1].getValueType() &&
11957 "LHS and RHS of condition must have same type!");
11958 assert(Ops[2].getValueType() == Ops[3].getValueType() &&
11959 "True and False arms of SelectCC must have same type!");
11960 assert(Ops[2].getValueType() == VT &&
11961 "select_cc node must be of same type as true and false value!");
11962 assert((!Ops[0].getValueType().isVector() ||
11963 Ops[0].getValueType().getVectorElementCount() ==
11964 VT.getVectorElementCount()) &&
11965 "Expected select_cc with vector result to have the same sized "
11966 "comparison type!");
11967 break;
11968 case ISD::BR_CC:
11969 assert(NumOps == 5 && "BR_CC takes 5 operands!");
11970 assert(Ops[2].getValueType() == Ops[3].getValueType() &&
11971 "LHS/RHS of comparison should match types!");
11972 break;
11973 case ISD::VP_REDUCE_MUL:
11974 // If it is VP_REDUCE_MUL mask operation then turn it to VP_REDUCE_AND
11975 if (VT == MVT::i1)
11976 Opcode = ISD::VP_REDUCE_AND;
11977 break;
11978 case ISD::VP_REDUCE_ADD:
11979 // If it is VP_REDUCE_ADD mask operation then turn it to VP_REDUCE_XOR
11980 if (VT == MVT::i1)
11981 Opcode = ISD::VP_REDUCE_XOR;
11982 break;
11983 case ISD::VP_REDUCE_SMAX:
11984 case ISD::VP_REDUCE_UMIN:
11985 // If it is VP_REDUCE_SMAX/VP_REDUCE_UMIN mask operation then turn it to
11986 // VP_REDUCE_AND.
11987 if (VT == MVT::i1)
11988 Opcode = ISD::VP_REDUCE_AND;
11989 break;
11990 case ISD::VP_REDUCE_SMIN:
11991 case ISD::VP_REDUCE_UMAX:
11992 // If it is VP_REDUCE_SMIN/VP_REDUCE_UMAX mask operation then turn it to
11993 // VP_REDUCE_OR.
11994 if (VT == MVT::i1)
11995 Opcode = ISD::VP_REDUCE_OR;
11996 break;
11997 }
11998
11999 // Memoize nodes.
12000 SDNode *N;
12001 SDVTList VTs = getVTList(VT);
12002
12003 if (VT != MVT::Glue) {
12004 FoldingSetNodeID ID;
12005 AddNodeIDNode(ID, OpC: Opcode, VTList: VTs, OpList: Ops);
12006 FoldingSetInsertToken InsertToken;
12007
12008 if (SDNode *E = lookupNode(ID, DL, InsertToken)) {
12009 E->intersectFlagsWith(Flags);
12010 return SDValue(E, 0);
12011 }
12012
12013 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
12014 createOperands(Node: N, Vals: Ops);
12015
12016 CSEMap.insert(N, Token: InsertToken);
12017 } else {
12018 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
12019 createOperands(Node: N, Vals: Ops);
12020 }
12021
12022 N->setFlags(Flags);
12023 InsertNode(N);
12024 SDValue V(N, 0);
12025 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
12026 return V;
12027}
12028
12029SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
12030 ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops) {
12031 SDNodeFlags Flags;
12032 if (Inserter)
12033 Flags = Inserter->getFlags();
12034 return getNode(Opcode, DL, VTList: getVTList(VTs: ResultTys), Ops, Flags);
12035}
12036
12037SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
12038 ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops,
12039 const SDNodeFlags Flags) {
12040 return getNode(Opcode, DL, VTList: getVTList(VTs: ResultTys), Ops, Flags);
12041}
12042
12043SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
12044 ArrayRef<SDValue> Ops) {
12045 SDNodeFlags Flags;
12046 if (Inserter)
12047 Flags = Inserter->getFlags();
12048 return getNode(Opcode, DL, VTList, Ops, Flags);
12049}
12050
12051SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
12052 ArrayRef<SDValue> Ops, const SDNodeFlags Flags) {
12053 if (VTList.NumVTs == 1)
12054 return getNode(Opcode, DL, VT: VTList.VTs[0], Ops, Flags);
12055
12056#ifndef NDEBUG
12057 for (const auto &Op : Ops)
12058 assert(Op.getOpcode() != ISD::DELETED_NODE &&
12059 "Operand is DELETED_NODE!");
12060#endif
12061
12062 switch (Opcode) {
12063 case ISD::SADDO:
12064 case ISD::UADDO:
12065 case ISD::SSUBO:
12066 case ISD::USUBO: {
12067 assert(VTList.NumVTs == 2 && Ops.size() == 2 &&
12068 "Invalid add/sub overflow op!");
12069 assert(VTList.VTs[0].isInteger() && VTList.VTs[1].isInteger() &&
12070 Ops[0].getValueType() == Ops[1].getValueType() &&
12071 Ops[0].getValueType() == VTList.VTs[0] &&
12072 "Binary operator types must match!");
12073 SDValue N1 = Ops[0], N2 = Ops[1];
12074 canonicalizeCommutativeBinop(Opcode, N1, N2);
12075
12076 // (X +- 0) -> X with zero-overflow.
12077 ConstantSDNode *N2CV = isConstOrConstSplat(N: N2, /*AllowUndefs*/ false,
12078 /*AllowTruncation*/ true);
12079 if (N2CV && N2CV->isZero()) {
12080 SDValue ZeroOverFlow = getConstant(Val: 0, DL, VT: VTList.VTs[1]);
12081 return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList, Ops: {N1, ZeroOverFlow}, Flags);
12082 }
12083
12084 if (VTList.VTs[0].getScalarType() == MVT::i1 &&
12085 VTList.VTs[1].getScalarType() == MVT::i1) {
12086 SDValue F1 = getFreeze(V: N1);
12087 SDValue F2 = getFreeze(V: N2);
12088 // {vXi1,vXi1} (u/s)addo(vXi1 x, vXi1y) -> {xor(x,y),and(x,y)}
12089 if (Opcode == ISD::UADDO || Opcode == ISD::SADDO)
12090 return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList,
12091 Ops: {getNode(Opcode: ISD::XOR, DL, VT: VTList.VTs[0], N1: F1, N2: F2),
12092 getNode(Opcode: ISD::AND, DL, VT: VTList.VTs[1], N1: F1, N2: F2)},
12093 Flags);
12094 // {vXi1,vXi1} (u/s)subo(vXi1 x, vXi1y) -> {xor(x,y),and(~x,y)}
12095 if (Opcode == ISD::USUBO || Opcode == ISD::SSUBO) {
12096 SDValue NotF1 = getNOT(DL, Val: F1, VT: VTList.VTs[0]);
12097 return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList,
12098 Ops: {getNode(Opcode: ISD::XOR, DL, VT: VTList.VTs[0], N1: F1, N2: F2),
12099 getNode(Opcode: ISD::AND, DL, VT: VTList.VTs[1], N1: NotF1, N2: F2)},
12100 Flags);
12101 }
12102 }
12103 break;
12104 }
12105 case ISD::SADDO_CARRY:
12106 case ISD::UADDO_CARRY:
12107 case ISD::SSUBO_CARRY:
12108 case ISD::USUBO_CARRY:
12109 assert(VTList.NumVTs == 2 && Ops.size() == 3 &&
12110 "Invalid add/sub overflow op!");
12111 assert(VTList.VTs[0].isInteger() && VTList.VTs[1].isInteger() &&
12112 Ops[0].getValueType() == Ops[1].getValueType() &&
12113 Ops[0].getValueType() == VTList.VTs[0] &&
12114 Ops[2].getValueType() == VTList.VTs[1] &&
12115 "Binary operator types must match!");
12116 break;
12117 case ISD::SMUL_LOHI:
12118 case ISD::UMUL_LOHI: {
12119 assert(VTList.NumVTs == 2 && Ops.size() == 2 && "Invalid mul lo/hi op!");
12120 assert(VTList.VTs[0].isInteger() && VTList.VTs[0] == VTList.VTs[1] &&
12121 VTList.VTs[0] == Ops[0].getValueType() &&
12122 VTList.VTs[0] == Ops[1].getValueType() &&
12123 "Binary operator types must match!");
12124 // Constant fold.
12125 ConstantSDNode *LHS = dyn_cast<ConstantSDNode>(Val: Ops[0]);
12126 ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Val: Ops[1]);
12127 if (LHS && RHS) {
12128 unsigned Width = VTList.VTs[0].getScalarSizeInBits();
12129 unsigned OutWidth = Width * 2;
12130 APInt Val = LHS->getAPIntValue();
12131 APInt Mul = RHS->getAPIntValue();
12132 if (Opcode == ISD::SMUL_LOHI) {
12133 Val = Val.sext(width: OutWidth);
12134 Mul = Mul.sext(width: OutWidth);
12135 } else {
12136 Val = Val.zext(width: OutWidth);
12137 Mul = Mul.zext(width: OutWidth);
12138 }
12139 Val *= Mul;
12140
12141 SDValue Hi =
12142 getConstant(Val: Val.extractBits(numBits: Width, bitPosition: Width), DL, VT: VTList.VTs[0]);
12143 SDValue Lo = getConstant(Val: Val.trunc(width: Width), DL, VT: VTList.VTs[0]);
12144 return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList, Ops: {Lo, Hi}, Flags);
12145 }
12146 break;
12147 }
12148 case ISD::FFREXP: {
12149 assert(VTList.NumVTs == 2 && Ops.size() == 1 && "Invalid ffrexp op!");
12150 assert(VTList.VTs[0].isFloatingPoint() && VTList.VTs[1].isInteger() &&
12151 VTList.VTs[0] == Ops[0].getValueType() && "frexp type mismatch");
12152
12153 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val: Ops[0])) {
12154 int FrexpExp;
12155 APFloat FrexpMant =
12156 frexp(X: C->getValueAPF(), Exp&: FrexpExp, RM: APFloat::rmNearestTiesToEven);
12157 SDValue Result0 = getConstantFP(V: FrexpMant, DL, VT: VTList.VTs[0]);
12158 SDValue Result1 = getSignedConstant(Val: FrexpMant.isFinite() ? FrexpExp : 0,
12159 DL, VT: VTList.VTs[1]);
12160 return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList, Ops: {Result0, Result1}, Flags);
12161 }
12162
12163 break;
12164 }
12165 case ISD::STRICT_FP_EXTEND:
12166 assert(VTList.NumVTs == 2 && Ops.size() == 2 &&
12167 "Invalid STRICT_FP_EXTEND!");
12168 assert(VTList.VTs[0].isFloatingPoint() &&
12169 Ops[1].getValueType().isFloatingPoint() && "Invalid FP cast!");
12170 assert(VTList.VTs[0].isVector() == Ops[1].getValueType().isVector() &&
12171 "STRICT_FP_EXTEND result type should be vector iff the operand "
12172 "type is vector!");
12173 assert((!VTList.VTs[0].isVector() ||
12174 VTList.VTs[0].getVectorElementCount() ==
12175 Ops[1].getValueType().getVectorElementCount()) &&
12176 "Vector element count mismatch!");
12177 assert(Ops[1].getValueType().bitsLT(VTList.VTs[0]) &&
12178 "Invalid fpext node, dst <= src!");
12179 break;
12180 case ISD::STRICT_FP_ROUND:
12181 assert(VTList.NumVTs == 2 && Ops.size() == 3 && "Invalid STRICT_FP_ROUND!");
12182 assert(VTList.VTs[0].isVector() == Ops[1].getValueType().isVector() &&
12183 "STRICT_FP_ROUND result type should be vector iff the operand "
12184 "type is vector!");
12185 assert((!VTList.VTs[0].isVector() ||
12186 VTList.VTs[0].getVectorElementCount() ==
12187 Ops[1].getValueType().getVectorElementCount()) &&
12188 "Vector element count mismatch!");
12189 assert(VTList.VTs[0].isFloatingPoint() &&
12190 Ops[1].getValueType().isFloatingPoint() &&
12191 VTList.VTs[0].bitsLT(Ops[1].getValueType()) &&
12192 Ops[2].getOpcode() == ISD::TargetConstant &&
12193 (Ops[2]->getAsZExtVal() == 0 || Ops[2]->getAsZExtVal() == 1) &&
12194 "Invalid STRICT_FP_ROUND!");
12195 break;
12196 }
12197
12198 // Memoize the node unless it returns a glue result.
12199 SDNode *N;
12200 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
12201 FoldingSetNodeID ID;
12202 AddNodeIDNode(ID, OpC: Opcode, VTList, OpList: Ops);
12203 FoldingSetInsertToken InsertToken;
12204 if (SDNode *E = lookupNode(ID, DL, InsertToken)) {
12205 E->intersectFlagsWith(Flags);
12206 return SDValue(E, 0);
12207 }
12208
12209 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTList);
12210 createOperands(Node: N, Vals: Ops);
12211 CSEMap.insert(N, Token: InsertToken);
12212 } else {
12213 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTList);
12214 createOperands(Node: N, Vals: Ops);
12215 }
12216
12217 N->setFlags(Flags);
12218 InsertNode(N);
12219 SDValue V(N, 0);
12220 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
12221 return V;
12222}
12223
12224SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
12225 SDVTList VTList) {
12226 return getNode(Opcode, DL, VTList, Ops: ArrayRef<SDValue>());
12227}
12228
12229SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
12230 SDValue N1) {
12231 SDValue Ops[] = { N1 };
12232 return getNode(Opcode, DL, VTList, Ops);
12233}
12234
12235SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
12236 SDValue N1, SDValue N2) {
12237 SDValue Ops[] = { N1, N2 };
12238 return getNode(Opcode, DL, VTList, Ops);
12239}
12240
12241SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
12242 SDValue N1, SDValue N2, SDValue N3) {
12243 SDValue Ops[] = { N1, N2, N3 };
12244 return getNode(Opcode, DL, VTList, Ops);
12245}
12246
12247SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
12248 SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
12249 SDValue Ops[] = { N1, N2, N3, N4 };
12250 return getNode(Opcode, DL, VTList, Ops);
12251}
12252
12253SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
12254 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
12255 SDValue N5) {
12256 SDValue Ops[] = { N1, N2, N3, N4, N5 };
12257 return getNode(Opcode, DL, VTList, Ops);
12258}
12259
12260SDVTList SelectionDAG::getVTList(EVT VT) {
12261 if (!VT.isExtended())
12262 return makeVTList(VTs: SDNode::getValueTypeList(VT: VT.getSimpleVT()), NumVTs: 1);
12263
12264 EVT VTs[] = {VT};
12265 return getVTList(VTs);
12266}
12267
12268SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2) {
12269 EVT VTs[] = {VT1, VT2};
12270 return getVTList(VTs);
12271}
12272
12273SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3) {
12274 EVT VTs[] = {VT1, VT2, VT3};
12275 return getVTList(VTs);
12276}
12277
12278SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4) {
12279 EVT VTs[] = {VT1, VT2, VT3, VT4};
12280 return getVTList(VTs);
12281}
12282
12283SDVTList SelectionDAG::getVTList(ArrayRef<EVT> VTs) {
12284 auto It = VTLists.find(V: VTs);
12285 if (It == VTLists.end()) {
12286 EVT *Array = Allocator.Allocate<EVT>(Num: VTs.size());
12287 llvm::copy(Range&: VTs, Out: Array);
12288 It = VTLists.insert(V: ArrayRef(Array, VTs.size())).first;
12289 }
12290 return makeVTList(VTs: It->data(), NumVTs: It->size());
12291}
12292
12293/// UpdateNodeOperands - *Mutate* the specified node in-place to have the
12294/// specified operands. If the resultant node already exists in the DAG,
12295/// this does not modify the specified node, instead it returns the node that
12296/// already exists. If the resultant node does not exist in the DAG, the
12297/// input node is returned. As a degenerate case, if you specify the same
12298/// input operands as the node already has, the input node is returned.
12299SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op) {
12300 assert(N->getNumOperands() == 1 && "Update with wrong number of operands");
12301
12302 // Check to see if there is no change.
12303 if (Op == N->getOperand(Num: 0)) return N;
12304
12305 // See if the modified node already exists.
12306 FoldingSetInsertToken InsertToken;
12307 if (SDNode *Existing = FindModifiedNodeSlot(N, Op, InsertToken))
12308 return Existing;
12309
12310 // Nope it doesn't. Remove the node from its current place in the maps.
12311 if (InsertToken)
12312 if (!RemoveNodeFromCSEMaps(N))
12313 InsertToken = {};
12314
12315 // Now we update the operands.
12316 N->OperandList[0].set(Op);
12317
12318 updateDivergence(N);
12319 // If this gets put into a CSE map, add it.
12320 if (InsertToken)
12321 CSEMap.insert(N, Token: InsertToken);
12322 return N;
12323}
12324
12325SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2) {
12326 assert(N->getNumOperands() == 2 && "Update with wrong number of operands");
12327
12328 // Check to see if there is no change.
12329 if (Op1 == N->getOperand(Num: 0) && Op2 == N->getOperand(Num: 1))
12330 return N; // No operands changed, just return the input node.
12331
12332 // See if the modified node already exists.
12333 FoldingSetInsertToken InsertToken;
12334 if (SDNode *Existing = FindModifiedNodeSlot(N, Op1, Op2, InsertToken))
12335 return Existing;
12336
12337 // Nope it doesn't. Remove the node from its current place in the maps.
12338 if (InsertToken)
12339 if (!RemoveNodeFromCSEMaps(N))
12340 InsertToken = {};
12341
12342 // Now we update the operands.
12343 if (N->OperandList[0] != Op1)
12344 N->OperandList[0].set(Op1);
12345 if (N->OperandList[1] != Op2)
12346 N->OperandList[1].set(Op2);
12347
12348 updateDivergence(N);
12349 // If this gets put into a CSE map, add it.
12350 if (InsertToken)
12351 CSEMap.insert(N, Token: InsertToken);
12352 return N;
12353}
12354
12355SDNode *SelectionDAG::
12356UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, SDValue Op3) {
12357 SDValue Ops[] = { Op1, Op2, Op3 };
12358 return UpdateNodeOperands(N, Ops);
12359}
12360
12361SDNode *SelectionDAG::
12362UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
12363 SDValue Op3, SDValue Op4) {
12364 SDValue Ops[] = { Op1, Op2, Op3, Op4 };
12365 return UpdateNodeOperands(N, Ops);
12366}
12367
12368SDNode *SelectionDAG::
12369UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
12370 SDValue Op3, SDValue Op4, SDValue Op5) {
12371 SDValue Ops[] = { Op1, Op2, Op3, Op4, Op5 };
12372 return UpdateNodeOperands(N, Ops);
12373}
12374
12375SDNode *SelectionDAG::
12376UpdateNodeOperands(SDNode *N, ArrayRef<SDValue> Ops) {
12377 unsigned NumOps = Ops.size();
12378 assert(N->getNumOperands() == NumOps &&
12379 "Update with wrong number of operands");
12380
12381 // If no operands changed just return the input node.
12382 if (std::equal(first1: Ops.begin(), last1: Ops.end(), first2: N->op_begin()))
12383 return N;
12384
12385 // See if the modified node already exists.
12386 FoldingSetInsertToken InsertToken;
12387 if (SDNode *Existing = FindModifiedNodeSlot(N, Ops, InsertToken))
12388 return Existing;
12389
12390 // Nope it doesn't. Remove the node from its current place in the maps.
12391 if (InsertToken)
12392 if (!RemoveNodeFromCSEMaps(N))
12393 InsertToken = {};
12394
12395 // Now we update the operands.
12396 for (unsigned i = 0; i != NumOps; ++i)
12397 if (N->OperandList[i] != Ops[i])
12398 N->OperandList[i].set(Ops[i]);
12399
12400 updateDivergence(N);
12401 // If this gets put into a CSE map, add it.
12402 if (InsertToken)
12403 CSEMap.insert(N, Token: InsertToken);
12404 return N;
12405}
12406
12407/// DropOperands - Release the operands and set this node to have
12408/// zero operands.
12409void SDNode::DropOperands() {
12410 // Unlike the code in MorphNodeTo that does this, we don't need to
12411 // watch for dead nodes here.
12412 for (op_iterator I = op_begin(), E = op_end(); I != E; ) {
12413 SDUse &Use = *I++;
12414 Use.set(SDValue());
12415 }
12416}
12417
12418void SelectionDAG::setNodeMemRefs(MachineSDNode *N,
12419 ArrayRef<MachineMemOperand *> NewMemRefs) {
12420 if (NewMemRefs.empty()) {
12421 N->clearMemRefs();
12422 return;
12423 }
12424
12425 // Check if we can avoid allocating by storing a single reference directly.
12426 if (NewMemRefs.size() == 1) {
12427 N->MemRefs = NewMemRefs[0];
12428 N->NumMemRefs = 1;
12429 return;
12430 }
12431
12432 MachineMemOperand **MemRefsBuffer =
12433 Allocator.template Allocate<MachineMemOperand *>(Num: NewMemRefs.size());
12434 llvm::copy(Range&: NewMemRefs, Out: MemRefsBuffer);
12435 N->MemRefs = MemRefsBuffer;
12436 N->NumMemRefs = static_cast<int>(NewMemRefs.size());
12437}
12438
12439/// SelectNodeTo - These are wrappers around MorphNodeTo that accept a
12440/// machine opcode.
12441///
12442SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12443 EVT VT) {
12444 SDVTList VTs = getVTList(VT);
12445 return SelectNodeTo(N, MachineOpc, VTs, Ops: {});
12446}
12447
12448SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12449 EVT VT, SDValue Op1) {
12450 SDVTList VTs = getVTList(VT);
12451 SDValue Ops[] = { Op1 };
12452 return SelectNodeTo(N, MachineOpc, VTs, Ops);
12453}
12454
12455SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12456 EVT VT, SDValue Op1,
12457 SDValue Op2) {
12458 SDVTList VTs = getVTList(VT);
12459 SDValue Ops[] = { Op1, Op2 };
12460 return SelectNodeTo(N, MachineOpc, VTs, Ops);
12461}
12462
12463SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12464 EVT VT, SDValue Op1,
12465 SDValue Op2, SDValue Op3) {
12466 SDVTList VTs = getVTList(VT);
12467 SDValue Ops[] = { Op1, Op2, Op3 };
12468 return SelectNodeTo(N, MachineOpc, VTs, Ops);
12469}
12470
12471SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12472 EVT VT, ArrayRef<SDValue> Ops) {
12473 SDVTList VTs = getVTList(VT);
12474 return SelectNodeTo(N, MachineOpc, VTs, Ops);
12475}
12476
12477SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12478 EVT VT1, EVT VT2, ArrayRef<SDValue> Ops) {
12479 SDVTList VTs = getVTList(VT1, VT2);
12480 return SelectNodeTo(N, MachineOpc, VTs, Ops);
12481}
12482
12483SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12484 EVT VT1, EVT VT2) {
12485 SDVTList VTs = getVTList(VT1, VT2);
12486 return SelectNodeTo(N, MachineOpc, VTs, Ops: {});
12487}
12488
12489SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12490 EVT VT1, EVT VT2, EVT VT3,
12491 ArrayRef<SDValue> Ops) {
12492 SDVTList VTs = getVTList(VT1, VT2, VT3);
12493 return SelectNodeTo(N, MachineOpc, VTs, Ops);
12494}
12495
12496SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12497 EVT VT1, EVT VT2,
12498 SDValue Op1, SDValue Op2) {
12499 SDVTList VTs = getVTList(VT1, VT2);
12500 SDValue Ops[] = { Op1, Op2 };
12501 return SelectNodeTo(N, MachineOpc, VTs, Ops);
12502}
12503
12504SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12505 SDVTList VTs,ArrayRef<SDValue> Ops) {
12506 SDNode *New = MorphNodeTo(N, Opc: ~MachineOpc, VTs, Ops);
12507 // Reset the NodeID to -1.
12508 New->setNodeId(-1);
12509 if (New != N) {
12510 ReplaceAllUsesWith(From: N, To: New);
12511 RemoveDeadNode(N);
12512 }
12513 return New;
12514}
12515
12516/// UpdateSDLocOnMergeSDNode - If the opt level is -O0 then it throws away
12517/// the line number information on the merged node since it is not possible to
12518/// preserve the information that operation is associated with multiple lines.
12519/// This will make the debugger working better at -O0, were there is a higher
12520/// probability having other instructions associated with that line.
12521///
12522/// For IROrder, we keep the smaller of the two
12523SDNode *SelectionDAG::UpdateSDLocOnMergeSDNode(SDNode *N, const SDLoc &OLoc) {
12524 DebugLoc NLoc = N->getDebugLoc();
12525 if (NLoc && OptLevel == CodeGenOptLevel::None && OLoc.getDebugLoc() != NLoc) {
12526 N->setDebugLoc(DebugLoc());
12527 }
12528 unsigned Order = std::min(a: N->getIROrder(), b: OLoc.getIROrder());
12529 N->setIROrder(Order);
12530 return N;
12531}
12532
12533/// MorphNodeTo - This *mutates* the specified node to have the specified
12534/// return type, opcode, and operands.
12535///
12536/// Note that MorphNodeTo returns the resultant node. If there is already a
12537/// node of the specified opcode and operands, it returns that node instead of
12538/// the current one. Note that the SDLoc need not be the same.
12539///
12540/// Using MorphNodeTo is faster than creating a new node and swapping it in
12541/// with ReplaceAllUsesWith both because it often avoids allocating a new
12542/// node, and because it doesn't require CSE recalculation for any of
12543/// the node's users.
12544///
12545/// However, note that MorphNodeTo recursively deletes dead nodes from the DAG.
12546/// As a consequence it isn't appropriate to use from within the DAG combiner or
12547/// the legalizer which maintain worklists that would need to be updated when
12548/// deleting things.
12549SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
12550 SDVTList VTs, ArrayRef<SDValue> Ops) {
12551 // If an identical node already exists, use it.
12552 FoldingSetInsertToken InsertToken;
12553 if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) {
12554 FoldingSetNodeID ID;
12555 AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: Ops);
12556 if (SDNode *ON = lookupNode(ID, DL: SDLoc(N), InsertToken))
12557 return UpdateSDLocOnMergeSDNode(N: ON, OLoc: SDLoc(N));
12558 }
12559
12560 if (!RemoveNodeFromCSEMaps(N))
12561 InsertToken = {};
12562
12563 // Start the morphing.
12564 N->NodeType = Opc;
12565 N->ValueList = VTs.VTs;
12566 N->NumValues = VTs.NumVTs;
12567
12568 // Clear the operands list, updating used nodes to remove this from their
12569 // use list. Keep track of any operands that become dead as a result.
12570 SmallPtrSet<SDNode*, 16> DeadNodeSet;
12571 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
12572 SDUse &Use = *I++;
12573 SDNode *Used = Use.getNode();
12574 Use.set(SDValue());
12575 if (Used->use_empty())
12576 DeadNodeSet.insert(Ptr: Used);
12577 }
12578
12579 // For MachineNode, initialize the memory references information.
12580 if (MachineSDNode *MN = dyn_cast<MachineSDNode>(Val: N))
12581 MN->clearMemRefs();
12582
12583 // Swap for an appropriately sized array from the recycler.
12584 removeOperands(Node: N);
12585 createOperands(Node: N, Vals: Ops);
12586
12587 // Delete any nodes that are still dead after adding the uses for the
12588 // new operands.
12589 if (!DeadNodeSet.empty()) {
12590 SmallVector<SDNode *, 16> DeadNodes;
12591 for (SDNode *N : DeadNodeSet)
12592 if (N->use_empty())
12593 DeadNodes.push_back(Elt: N);
12594 RemoveDeadNodes(DeadNodes);
12595 }
12596
12597 if (InsertToken)
12598 CSEMap.insert(N, Token: InsertToken); // Memoize the new node.
12599 return N;
12600}
12601
12602SDNode* SelectionDAG::mutateStrictFPToFP(SDNode *Node) {
12603 unsigned OrigOpc = Node->getOpcode();
12604 unsigned NewOpc;
12605 switch (OrigOpc) {
12606 default:
12607 llvm_unreachable("mutateStrictFPToFP called with unexpected opcode!");
12608#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
12609 case ISD::STRICT_##DAGN: NewOpc = ISD::DAGN; break;
12610#define CMP_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
12611 case ISD::STRICT_##DAGN: NewOpc = ISD::SETCC; break;
12612#include "llvm/IR/ConstrainedOps.def"
12613 }
12614
12615 assert(Node->getNumValues() == 2 && "Unexpected number of results!");
12616
12617 // We're taking this node out of the chain, so we need to re-link things.
12618 SDValue InputChain = Node->getOperand(Num: 0);
12619 SDValue OutputChain = SDValue(Node, 1);
12620 ReplaceAllUsesOfValueWith(From: OutputChain, To: InputChain);
12621
12622 SmallVector<SDValue, 3> Ops;
12623 for (unsigned i = 1, e = Node->getNumOperands(); i != e; ++i)
12624 Ops.push_back(Elt: Node->getOperand(Num: i));
12625
12626 SDVTList VTs = getVTList(VT: Node->getValueType(ResNo: 0));
12627 SDNode *Res = MorphNodeTo(N: Node, Opc: NewOpc, VTs, Ops);
12628
12629 // MorphNodeTo can operate in two ways: if an existing node with the
12630 // specified operands exists, it can just return it. Otherwise, it
12631 // updates the node in place to have the requested operands.
12632 if (Res == Node) {
12633 // If we updated the node in place, reset the node ID. To the isel,
12634 // this should be just like a newly allocated machine node.
12635 Res->setNodeId(-1);
12636 } else {
12637 ReplaceAllUsesWith(From: Node, To: Res);
12638 RemoveDeadNode(N: Node);
12639 }
12640
12641 return Res;
12642}
12643
12644/// getMachineNode - These are used for target selectors to create a new node
12645/// with specified return type(s), MachineInstr opcode, and operands.
12646///
12647/// Note that getMachineNode returns the resultant node. If there is already a
12648/// node of the specified opcode and operands, it returns that node instead of
12649/// the current one.
12650MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12651 EVT VT) {
12652 SDVTList VTs = getVTList(VT);
12653 return getMachineNode(Opcode, dl, VTs, Ops: {});
12654}
12655
12656MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12657 EVT VT, SDValue Op1) {
12658 SDVTList VTs = getVTList(VT);
12659 SDValue Ops[] = { Op1 };
12660 return getMachineNode(Opcode, dl, VTs, Ops);
12661}
12662
12663MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12664 EVT VT, SDValue Op1, SDValue Op2) {
12665 SDVTList VTs = getVTList(VT);
12666 SDValue Ops[] = { Op1, Op2 };
12667 return getMachineNode(Opcode, dl, VTs, Ops);
12668}
12669
12670MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12671 EVT VT, SDValue Op1, SDValue Op2,
12672 SDValue Op3) {
12673 SDVTList VTs = getVTList(VT);
12674 SDValue Ops[] = { Op1, Op2, Op3 };
12675 return getMachineNode(Opcode, dl, VTs, Ops);
12676}
12677
12678MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12679 EVT VT, ArrayRef<SDValue> Ops) {
12680 SDVTList VTs = getVTList(VT);
12681 return getMachineNode(Opcode, dl, VTs, Ops);
12682}
12683
12684MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12685 EVT VT1, EVT VT2, SDValue Op1,
12686 SDValue Op2) {
12687 SDVTList VTs = getVTList(VT1, VT2);
12688 SDValue Ops[] = { Op1, Op2 };
12689 return getMachineNode(Opcode, dl, VTs, Ops);
12690}
12691
12692MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12693 EVT VT1, EVT VT2, SDValue Op1,
12694 SDValue Op2, SDValue Op3) {
12695 SDVTList VTs = getVTList(VT1, VT2);
12696 SDValue Ops[] = { Op1, Op2, Op3 };
12697 return getMachineNode(Opcode, dl, VTs, Ops);
12698}
12699
12700MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12701 EVT VT1, EVT VT2,
12702 ArrayRef<SDValue> Ops) {
12703 SDVTList VTs = getVTList(VT1, VT2);
12704 return getMachineNode(Opcode, dl, VTs, Ops);
12705}
12706
12707MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12708 EVT VT1, EVT VT2, EVT VT3,
12709 SDValue Op1, SDValue Op2) {
12710 SDVTList VTs = getVTList(VT1, VT2, VT3);
12711 SDValue Ops[] = { Op1, Op2 };
12712 return getMachineNode(Opcode, dl, VTs, Ops);
12713}
12714
12715MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12716 EVT VT1, EVT VT2, EVT VT3,
12717 SDValue Op1, SDValue Op2,
12718 SDValue Op3) {
12719 SDVTList VTs = getVTList(VT1, VT2, VT3);
12720 SDValue Ops[] = { Op1, Op2, Op3 };
12721 return getMachineNode(Opcode, dl, VTs, Ops);
12722}
12723
12724MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12725 EVT VT1, EVT VT2, EVT VT3,
12726 ArrayRef<SDValue> Ops) {
12727 SDVTList VTs = getVTList(VT1, VT2, VT3);
12728 return getMachineNode(Opcode, dl, VTs, Ops);
12729}
12730
12731MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12732 ArrayRef<EVT> ResultTys,
12733 ArrayRef<SDValue> Ops) {
12734 SDVTList VTs = getVTList(VTs: ResultTys);
12735 return getMachineNode(Opcode, dl, VTs, Ops);
12736}
12737
12738MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &DL,
12739 SDVTList VTs,
12740 ArrayRef<SDValue> Ops) {
12741 bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue;
12742 MachineSDNode *N;
12743 FoldingSetInsertToken InsertToken;
12744
12745 if (DoCSE) {
12746 FoldingSetNodeID ID;
12747 AddNodeIDNode(ID, OpC: ~Opcode, VTList: VTs, OpList: Ops);
12748 if (SDNode *E = lookupNode(ID, DL, InsertToken)) {
12749 return cast<MachineSDNode>(Val: UpdateSDLocOnMergeSDNode(N: E, OLoc: DL));
12750 }
12751 }
12752
12753 // Allocate a new MachineSDNode.
12754 N = newSDNode<MachineSDNode>(Args: ~Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
12755 createOperands(Node: N, Vals: Ops);
12756
12757 if (DoCSE)
12758 CSEMap.insert(N, Token: InsertToken);
12759
12760 InsertNode(N);
12761 NewSDValueDbgMsg(V: SDValue(N, 0), Msg: "Creating new machine node: ", G: this);
12762 return N;
12763}
12764
12765/// getTargetExtractSubreg - A convenience function for creating
12766/// TargetOpcode::EXTRACT_SUBREG nodes.
12767SDValue SelectionDAG::getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT,
12768 SDValue Operand) {
12769 SDValue SRIdxVal = getTargetConstant(Val: SRIdx, DL, VT: MVT::i32);
12770 SDNode *Subreg = getMachineNode(Opcode: TargetOpcode::EXTRACT_SUBREG, dl: DL,
12771 VT, Op1: Operand, Op2: SRIdxVal);
12772 return SDValue(Subreg, 0);
12773}
12774
12775/// getTargetInsertSubreg - A convenience function for creating
12776/// TargetOpcode::INSERT_SUBREG nodes.
12777SDValue SelectionDAG::getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT,
12778 SDValue Operand, SDValue Subreg) {
12779 SDValue SRIdxVal = getTargetConstant(Val: SRIdx, DL, VT: MVT::i32);
12780 SDNode *Result = getMachineNode(Opcode: TargetOpcode::INSERT_SUBREG, dl: DL,
12781 VT, Op1: Operand, Op2: Subreg, Op3: SRIdxVal);
12782 return SDValue(Result, 0);
12783}
12784
12785/// getNodeIfExists - Get the specified node if it's already available, or
12786/// else return NULL.
12787SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,
12788 ArrayRef<SDValue> Ops,
12789 bool AllowCommute) {
12790 SDNodeFlags Flags;
12791 if (Inserter)
12792 Flags = Inserter->getFlags();
12793 return getNodeIfExists(Opcode, VTList, Ops, Flags, AllowCommute);
12794}
12795
12796SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,
12797 ArrayRef<SDValue> Ops,
12798 const SDNodeFlags Flags,
12799 bool AllowCommute) {
12800 if (VTList.VTs[VTList.NumVTs - 1] == MVT::Glue)
12801 return nullptr;
12802
12803 auto Lookup = [&](ArrayRef<SDValue> LookupOps) -> SDNode * {
12804 FoldingSetNodeID ID;
12805 AddNodeIDNode(ID, OpC: Opcode, VTList, OpList: LookupOps);
12806 FoldingSetInsertToken InsertToken;
12807 if (SDNode *E = lookupNode(ID, InsertToken)) {
12808 E->intersectFlagsWith(Flags);
12809 return E;
12810 }
12811 return nullptr;
12812 };
12813
12814 if (SDNode *Existing = Lookup(Ops))
12815 return Existing;
12816
12817 if (AllowCommute && TLI->isCommutativeBinOp(Opcode))
12818 return Lookup({Ops[1], Ops[0]});
12819
12820 return nullptr;
12821}
12822
12823/// doesNodeExist - Check if a node exists without modifying its flags.
12824bool SelectionDAG::doesNodeExist(unsigned Opcode, SDVTList VTList,
12825 ArrayRef<SDValue> Ops) {
12826 if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) {
12827 FoldingSetNodeID ID;
12828 AddNodeIDNode(ID, OpC: Opcode, VTList, OpList: Ops);
12829 FoldingSetInsertToken InsertToken;
12830 if (lookupNode(ID, DL: SDLoc(), InsertToken))
12831 return true;
12832 }
12833 return false;
12834}
12835
12836/// getDbgValue - Creates a SDDbgValue node.
12837///
12838/// SDNode
12839SDDbgValue *SelectionDAG::getDbgValue(DIVariable *Var, DIExpression *Expr,
12840 SDNode *N, unsigned R, bool IsIndirect,
12841 const DebugLoc &DL, unsigned O) {
12842 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
12843 "Expected inlined-at fields to agree");
12844 return new (DbgInfo->getAlloc())
12845 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromNode(Node: N, ResNo: R),
12846 {}, IsIndirect, DL, O,
12847 /*IsVariadic=*/false);
12848}
12849
12850/// Constant
12851SDDbgValue *SelectionDAG::getConstantDbgValue(DIVariable *Var,
12852 DIExpression *Expr,
12853 const Value *C,
12854 const DebugLoc &DL, unsigned O) {
12855 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
12856 "Expected inlined-at fields to agree");
12857 return new (DbgInfo->getAlloc())
12858 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromConst(Const: C), {},
12859 /*IsIndirect=*/false, DL, O,
12860 /*IsVariadic=*/false);
12861}
12862
12863/// FrameIndex
12864SDDbgValue *SelectionDAG::getFrameIndexDbgValue(DIVariable *Var,
12865 DIExpression *Expr, unsigned FI,
12866 bool IsIndirect,
12867 const DebugLoc &DL,
12868 unsigned O) {
12869 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
12870 "Expected inlined-at fields to agree");
12871 return getFrameIndexDbgValue(Var, Expr, FI, Dependencies: {}, IsIndirect, DL, O);
12872}
12873
12874/// FrameIndex with dependencies
12875SDDbgValue *SelectionDAG::getFrameIndexDbgValue(DIVariable *Var,
12876 DIExpression *Expr, unsigned FI,
12877 ArrayRef<SDNode *> Dependencies,
12878 bool IsIndirect,
12879 const DebugLoc &DL,
12880 unsigned O) {
12881 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
12882 "Expected inlined-at fields to agree");
12883 return new (DbgInfo->getAlloc())
12884 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromFrameIdx(FrameIdx: FI),
12885 Dependencies, IsIndirect, DL, O,
12886 /*IsVariadic=*/false);
12887}
12888
12889/// VReg
12890SDDbgValue *SelectionDAG::getVRegDbgValue(DIVariable *Var, DIExpression *Expr,
12891 Register VReg, bool IsIndirect,
12892 const DebugLoc &DL, unsigned O) {
12893 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
12894 "Expected inlined-at fields to agree");
12895 return new (DbgInfo->getAlloc())
12896 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromVReg(VReg),
12897 {}, IsIndirect, DL, O,
12898 /*IsVariadic=*/false);
12899}
12900
12901SDDbgValue *SelectionDAG::getDbgValueList(DIVariable *Var, DIExpression *Expr,
12902 ArrayRef<SDDbgOperand> Locs,
12903 ArrayRef<SDNode *> Dependencies,
12904 bool IsIndirect, const DebugLoc &DL,
12905 unsigned O, bool IsVariadic) {
12906 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
12907 "Expected inlined-at fields to agree");
12908 return new (DbgInfo->getAlloc())
12909 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, Locs, Dependencies, IsIndirect,
12910 DL, O, IsVariadic);
12911}
12912
12913void SelectionDAG::transferDbgValues(SDValue From, SDValue To,
12914 unsigned OffsetInBits, unsigned SizeInBits,
12915 bool InvalidateDbg) {
12916 SDNode *FromNode = From.getNode();
12917 SDNode *ToNode = To.getNode();
12918 assert(FromNode && ToNode && "Can't modify dbg values");
12919
12920 // PR35338
12921 // TODO: assert(From != To && "Redundant dbg value transfer");
12922 // TODO: assert(FromNode != ToNode && "Intranode dbg value transfer");
12923 if (From == To || FromNode == ToNode)
12924 return;
12925
12926 if (!FromNode->getHasDebugValue())
12927 return;
12928
12929 SDDbgOperand FromLocOp =
12930 SDDbgOperand::fromNode(Node: From.getNode(), ResNo: From.getResNo());
12931 SDDbgOperand ToLocOp = SDDbgOperand::fromNode(Node: To.getNode(), ResNo: To.getResNo());
12932
12933 SmallVector<SDDbgValue *, 2> ClonedDVs;
12934 for (SDDbgValue *Dbg : GetDbgValues(SD: FromNode)) {
12935 if (Dbg->isInvalidated())
12936 continue;
12937
12938 // TODO: assert(!Dbg->isInvalidated() && "Transfer of invalid dbg value");
12939
12940 // Create a new location ops vector that is equal to the old vector, but
12941 // with each instance of FromLocOp replaced with ToLocOp.
12942 bool Changed = false;
12943 auto NewLocOps = Dbg->copyLocationOps();
12944 std::replace_if(
12945 first: NewLocOps.begin(), last: NewLocOps.end(),
12946 pred: [&Changed, FromLocOp](const SDDbgOperand &Op) {
12947 bool Match = Op == FromLocOp;
12948 Changed |= Match;
12949 return Match;
12950 },
12951 new_value: ToLocOp);
12952 // Ignore this SDDbgValue if we didn't find a matching location.
12953 if (!Changed)
12954 continue;
12955
12956 DIVariable *Var = Dbg->getVariable();
12957 auto *Expr = Dbg->getExpression();
12958 // If a fragment is requested, update the expression.
12959 if (SizeInBits) {
12960 // When splitting a larger (e.g., sign-extended) value whose
12961 // lower bits are described with an SDDbgValue, do not attempt
12962 // to transfer the SDDbgValue to the upper bits.
12963 if (auto FI = Expr->getFragmentInfo())
12964 if (OffsetInBits + SizeInBits > FI->SizeInBits)
12965 continue;
12966 auto Fragment = DIExpression::createFragmentExpression(Expr, OffsetInBits,
12967 SizeInBits);
12968 if (!Fragment)
12969 continue;
12970 Expr = *Fragment;
12971 }
12972
12973 auto AdditionalDependencies = Dbg->getAdditionalDependencies();
12974 // Clone the SDDbgValue and move it to To.
12975 SDDbgValue *Clone = getDbgValueList(
12976 Var, Expr, Locs: NewLocOps, Dependencies: AdditionalDependencies, IsIndirect: Dbg->isIndirect(),
12977 DL: Dbg->getDebugLoc(), O: std::max(a: ToNode->getIROrder(), b: Dbg->getOrder()),
12978 IsVariadic: Dbg->isVariadic());
12979 ClonedDVs.push_back(Elt: Clone);
12980
12981 if (InvalidateDbg) {
12982 // Invalidate value and indicate the SDDbgValue should not be emitted.
12983 Dbg->setIsInvalidated();
12984 Dbg->setIsEmitted();
12985 }
12986 }
12987
12988 for (SDDbgValue *Dbg : ClonedDVs) {
12989 assert(is_contained(Dbg->getSDNodes(), ToNode) &&
12990 "Transferred DbgValues should depend on the new SDNode");
12991 AddDbgValue(DB: Dbg, isParameter: false);
12992 }
12993}
12994
12995void SelectionDAG::salvageDebugInfo(SDNode &N) {
12996 if (!N.getHasDebugValue())
12997 return;
12998
12999 auto GetLocationOperand = [](SDNode *Node, unsigned ResNo) {
13000 if (auto *FISDN = dyn_cast<FrameIndexSDNode>(Val: Node))
13001 return SDDbgOperand::fromFrameIdx(FrameIdx: FISDN->getIndex());
13002 return SDDbgOperand::fromNode(Node, ResNo);
13003 };
13004
13005 SmallVector<SDDbgValue *, 2> ClonedDVs;
13006 for (auto *DV : GetDbgValues(SD: &N)) {
13007 if (DV->isInvalidated())
13008 continue;
13009 switch (N.getOpcode()) {
13010 default:
13011 break;
13012 case ISD::ADD: {
13013 SDValue N0 = N.getOperand(Num: 0);
13014 SDValue N1 = N.getOperand(Num: 1);
13015 if (!isa<ConstantSDNode>(Val: N0)) {
13016 bool RHSConstant = isa<ConstantSDNode>(Val: N1);
13017 uint64_t Offset;
13018 if (RHSConstant)
13019 Offset = N.getConstantOperandVal(Num: 1);
13020 // We are not allowed to turn indirect debug values variadic, so
13021 // don't salvage those.
13022 if (!RHSConstant && DV->isIndirect())
13023 continue;
13024
13025 // Rewrite an ADD constant node into a DIExpression. Since we are
13026 // performing arithmetic to compute the variable's *value* in the
13027 // DIExpression, we need to mark the expression with a
13028 // DW_OP_stack_value.
13029 auto *DIExpr = DV->getExpression();
13030 auto NewLocOps = DV->copyLocationOps();
13031 bool Changed = false;
13032 size_t OrigLocOpsSize = NewLocOps.size();
13033 for (size_t i = 0; i < OrigLocOpsSize; ++i) {
13034 // We're not given a ResNo to compare against because the whole
13035 // node is going away. We know that any ISD::ADD only has one
13036 // result, so we can assume any node match is using the result.
13037 if (NewLocOps[i].getKind() != SDDbgOperand::SDNODE ||
13038 NewLocOps[i].getSDNode() != &N)
13039 continue;
13040 NewLocOps[i] = GetLocationOperand(N0.getNode(), N0.getResNo());
13041 if (RHSConstant) {
13042 SmallVector<uint64_t, 3> ExprOps;
13043 DIExpression::appendOffset(Ops&: ExprOps, Offset);
13044 DIExpr = DIExpression::appendOpsToArg(Expr: DIExpr, Ops: ExprOps, ArgNo: i, StackValue: true);
13045 } else {
13046 // Convert to a variadic expression (if not already).
13047 // convertToVariadicExpression() returns a const pointer, so we use
13048 // a temporary const variable here.
13049 const auto *TmpDIExpr =
13050 DIExpression::convertToVariadicExpression(Expr: DIExpr);
13051 SmallVector<uint64_t, 3> ExprOps;
13052 ExprOps.push_back(Elt: dwarf::DW_OP_LLVM_arg);
13053 ExprOps.push_back(Elt: NewLocOps.size());
13054 ExprOps.push_back(Elt: dwarf::DW_OP_plus);
13055 SDDbgOperand RHS =
13056 SDDbgOperand::fromNode(Node: N1.getNode(), ResNo: N1.getResNo());
13057 NewLocOps.push_back(Elt: RHS);
13058 DIExpr = DIExpression::appendOpsToArg(Expr: TmpDIExpr, Ops: ExprOps, ArgNo: i, StackValue: true);
13059 }
13060 Changed = true;
13061 }
13062 (void)Changed;
13063 assert(Changed && "Salvage target doesn't use N");
13064
13065 bool IsVariadic =
13066 DV->isVariadic() || OrigLocOpsSize != NewLocOps.size();
13067
13068 auto AdditionalDependencies = DV->getAdditionalDependencies();
13069 SDDbgValue *Clone = getDbgValueList(
13070 Var: DV->getVariable(), Expr: DIExpr, Locs: NewLocOps, Dependencies: AdditionalDependencies,
13071 IsIndirect: DV->isIndirect(), DL: DV->getDebugLoc(), O: DV->getOrder(), IsVariadic);
13072 ClonedDVs.push_back(Elt: Clone);
13073 DV->setIsInvalidated();
13074 DV->setIsEmitted();
13075 LLVM_DEBUG(dbgs() << "SALVAGE: Rewriting";
13076 N0.getNode()->dumprFull(this);
13077 dbgs() << " into " << *DIExpr << '\n');
13078 }
13079 break;
13080 }
13081 case ISD::TRUNCATE: {
13082 SDValue N0 = N.getOperand(Num: 0);
13083 TypeSize FromSize = N0.getValueSizeInBits();
13084 TypeSize ToSize = N.getValueSizeInBits(ResNo: 0);
13085
13086 DIExpression *DbgExpression = DV->getExpression();
13087 auto ExtOps = DIExpression::getExtOps(FromSize, ToSize, Signed: false);
13088 auto NewLocOps = DV->copyLocationOps();
13089 bool Changed = false;
13090 for (size_t i = 0; i < NewLocOps.size(); ++i) {
13091 if (NewLocOps[i].getKind() != SDDbgOperand::SDNODE ||
13092 NewLocOps[i].getSDNode() != &N)
13093 continue;
13094
13095 NewLocOps[i] = GetLocationOperand(N0.getNode(), N0.getResNo());
13096 DbgExpression = DIExpression::appendOpsToArg(Expr: DbgExpression, Ops: ExtOps, ArgNo: i);
13097 Changed = true;
13098 }
13099 assert(Changed && "Salvage target doesn't use N");
13100 (void)Changed;
13101
13102 SDDbgValue *Clone =
13103 getDbgValueList(Var: DV->getVariable(), Expr: DbgExpression, Locs: NewLocOps,
13104 Dependencies: DV->getAdditionalDependencies(), IsIndirect: DV->isIndirect(),
13105 DL: DV->getDebugLoc(), O: DV->getOrder(), IsVariadic: DV->isVariadic());
13106
13107 ClonedDVs.push_back(Elt: Clone);
13108 DV->setIsInvalidated();
13109 DV->setIsEmitted();
13110 LLVM_DEBUG(dbgs() << "SALVAGE: Rewriting"; N0.getNode()->dumprFull(this);
13111 dbgs() << " into " << *DbgExpression << '\n');
13112 break;
13113 }
13114 }
13115 }
13116
13117 for (SDDbgValue *Dbg : ClonedDVs) {
13118 assert((!Dbg->getSDNodes().empty() ||
13119 llvm::any_of(Dbg->getLocationOps(),
13120 [&](const SDDbgOperand &Op) {
13121 return Op.getKind() == SDDbgOperand::FRAMEIX;
13122 })) &&
13123 "Salvaged DbgValue should depend on a new SDNode");
13124 AddDbgValue(DB: Dbg, isParameter: false);
13125 }
13126}
13127
13128/// Creates a SDDbgLabel node.
13129SDDbgLabel *SelectionDAG::getDbgLabel(DILabel *Label,
13130 const DebugLoc &DL, unsigned O) {
13131 assert(cast<DILabel>(Label)->isValidLocationForIntrinsic(DL) &&
13132 "Expected inlined-at fields to agree");
13133 return new (DbgInfo->getAlloc()) SDDbgLabel(Label, DL, O);
13134}
13135
13136namespace {
13137
13138/// RAUWUpdateListener - Helper for ReplaceAllUsesWith - When the node
13139/// pointed to by a use iterator is deleted, increment the use iterator
13140/// so that it doesn't dangle.
13141///
13142class RAUWUpdateListener : public SelectionDAG::DAGUpdateListener {
13143 SDNode::use_iterator &UI;
13144 SDNode::use_iterator &UE;
13145
13146 void NodeDeleted(SDNode *N, SDNode *E) override {
13147 // Increment the iterator as needed.
13148 while (UI != UE && N == UI->getUser())
13149 ++UI;
13150 }
13151
13152public:
13153 RAUWUpdateListener(SelectionDAG &d,
13154 SDNode::use_iterator &ui,
13155 SDNode::use_iterator &ue)
13156 : SelectionDAG::DAGUpdateListener(d), UI(ui), UE(ue) {}
13157};
13158
13159} // end anonymous namespace
13160
13161/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
13162/// This can cause recursive merging of nodes in the DAG.
13163///
13164/// This version assumes From has a single result value.
13165///
13166void SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To) {
13167 SDNode *From = FromN.getNode();
13168 assert(From->getNumValues() == 1 && FromN.getResNo() == 0 &&
13169 "Cannot replace with this method!");
13170 assert(From != To.getNode() && "Cannot replace uses of with self");
13171
13172 // Preserve Debug Values
13173 transferDbgValues(From: FromN, To);
13174 // Preserve extra info.
13175 copyExtraInfo(From, To: To.getNode());
13176
13177 // Iterate over all the existing uses of From. New uses will be added
13178 // to the beginning of the use list, which we avoid visiting.
13179 // This specifically avoids visiting uses of From that arise while the
13180 // replacement is happening, because any such uses would be the result
13181 // of CSE: If an existing node looks like From after one of its operands
13182 // is replaced by To, we don't want to replace of all its users with To
13183 // too. See PR3018 for more info.
13184 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
13185 RAUWUpdateListener Listener(*this, UI, UE);
13186 while (UI != UE) {
13187 SDNode *User = UI->getUser();
13188
13189 // This node is about to morph, remove its old self from the CSE maps.
13190 RemoveNodeFromCSEMaps(N: User);
13191
13192 // A user can appear in a use list multiple times, and when this
13193 // happens the uses are usually next to each other in the list.
13194 // To help reduce the number of CSE recomputations, process all
13195 // the uses of this user that we can find this way.
13196 do {
13197 SDUse &Use = *UI;
13198 ++UI;
13199 Use.set(To);
13200 if (To->isDivergent() != From->isDivergent())
13201 updateDivergence(N: User);
13202 } while (UI != UE && UI->getUser() == User);
13203 // Now that we have modified User, add it back to the CSE maps. If it
13204 // already exists there, recursively merge the results together.
13205 AddModifiedNodeToCSEMaps(N: User);
13206 }
13207
13208 // If we just RAUW'd the root, take note.
13209 if (FromN == getRoot())
13210 setRoot(To);
13211}
13212
13213/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
13214/// This can cause recursive merging of nodes in the DAG.
13215///
13216/// This version assumes that for each value of From, there is a
13217/// corresponding value in To in the same position with the same type.
13218///
13219void SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To) {
13220#ifndef NDEBUG
13221 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
13222 assert((!From->hasAnyUseOfValue(i) ||
13223 From->getValueType(i) == To->getValueType(i)) &&
13224 "Cannot use this version of ReplaceAllUsesWith!");
13225#endif
13226
13227 // Handle the trivial case.
13228 if (From == To)
13229 return;
13230
13231 // Preserve Debug Info. Only do this if there's a use.
13232 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
13233 if (From->hasAnyUseOfValue(Value: i)) {
13234 assert((i < To->getNumValues()) && "Invalid To location");
13235 transferDbgValues(From: SDValue(From, i), To: SDValue(To, i));
13236 }
13237 // Preserve extra info.
13238 copyExtraInfo(From, To);
13239
13240 // Iterate over just the existing users of From. See the comments in
13241 // the ReplaceAllUsesWith above.
13242 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
13243 RAUWUpdateListener Listener(*this, UI, UE);
13244 while (UI != UE) {
13245 SDNode *User = UI->getUser();
13246
13247 // This node is about to morph, remove its old self from the CSE maps.
13248 RemoveNodeFromCSEMaps(N: User);
13249
13250 // A user can appear in a use list multiple times, and when this
13251 // happens the uses are usually next to each other in the list.
13252 // To help reduce the number of CSE recomputations, process all
13253 // the uses of this user that we can find this way.
13254 do {
13255 SDUse &Use = *UI;
13256 ++UI;
13257 Use.setNode(To);
13258 if (To->isDivergent() != From->isDivergent())
13259 updateDivergence(N: User);
13260 } while (UI != UE && UI->getUser() == User);
13261
13262 // Now that we have modified User, add it back to the CSE maps. If it
13263 // already exists there, recursively merge the results together.
13264 AddModifiedNodeToCSEMaps(N: User);
13265 }
13266
13267 // If we just RAUW'd the root, take note.
13268 if (From == getRoot().getNode())
13269 setRoot(SDValue(To, getRoot().getResNo()));
13270}
13271
13272/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
13273/// This can cause recursive merging of nodes in the DAG.
13274///
13275/// This version can replace From with any result values. To must match the
13276/// number and types of values returned by From.
13277void SelectionDAG::ReplaceAllUsesWith(SDNode *From, const SDValue *To) {
13278 if (From->getNumValues() == 1) // Handle the simple case efficiently.
13279 return ReplaceAllUsesWith(FromN: SDValue(From, 0), To: To[0]);
13280
13281 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i) {
13282 // Preserve Debug Info.
13283 transferDbgValues(From: SDValue(From, i), To: To[i]);
13284 // Preserve extra info.
13285 copyExtraInfo(From, To: To[i].getNode());
13286 }
13287
13288 // Iterate over just the existing users of From. See the comments in
13289 // the ReplaceAllUsesWith above.
13290 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
13291 RAUWUpdateListener Listener(*this, UI, UE);
13292 while (UI != UE) {
13293 SDNode *User = UI->getUser();
13294
13295 // This node is about to morph, remove its old self from the CSE maps.
13296 RemoveNodeFromCSEMaps(N: User);
13297
13298 // A user can appear in a use list multiple times, and when this happens the
13299 // uses are usually next to each other in the list. To help reduce the
13300 // number of CSE and divergence recomputations, process all the uses of this
13301 // user that we can find this way.
13302 bool To_IsDivergent = false;
13303 do {
13304 SDUse &Use = *UI;
13305 const SDValue &ToOp = To[Use.getResNo()];
13306 ++UI;
13307 Use.set(ToOp);
13308 if (ToOp.getValueType() != MVT::Other)
13309 To_IsDivergent |= ToOp->isDivergent();
13310 } while (UI != UE && UI->getUser() == User);
13311
13312 if (To_IsDivergent != From->isDivergent())
13313 updateDivergence(N: User);
13314
13315 // Now that we have modified User, add it back to the CSE maps. If it
13316 // already exists there, recursively merge the results together.
13317 AddModifiedNodeToCSEMaps(N: User);
13318 }
13319
13320 // If we just RAUW'd the root, take note.
13321 if (From == getRoot().getNode())
13322 setRoot(SDValue(To[getRoot().getResNo()]));
13323}
13324
13325/// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving
13326/// uses of other values produced by From.getNode() alone. The Deleted
13327/// vector is handled the same way as for ReplaceAllUsesWith.
13328void SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To){
13329 // Handle the really simple, really trivial case efficiently.
13330 if (From == To) return;
13331
13332 // Handle the simple, trivial, case efficiently.
13333 if (From.getNode()->getNumValues() == 1) {
13334 ReplaceAllUsesWith(FromN: From, To);
13335 return;
13336 }
13337
13338 // Preserve Debug Info.
13339 transferDbgValues(From, To);
13340 copyExtraInfo(From: From.getNode(), To: To.getNode());
13341
13342 // Iterate over just the existing users of From. See the comments in
13343 // the ReplaceAllUsesWith above.
13344 SDNode::use_iterator UI = From.getNode()->use_begin(),
13345 UE = From.getNode()->use_end();
13346 RAUWUpdateListener Listener(*this, UI, UE);
13347 while (UI != UE) {
13348 SDNode *User = UI->getUser();
13349 bool UserRemovedFromCSEMaps = false;
13350
13351 // A user can appear in a use list multiple times, and when this
13352 // happens the uses are usually next to each other in the list.
13353 // To help reduce the number of CSE recomputations, process all
13354 // the uses of this user that we can find this way.
13355 do {
13356 SDUse &Use = *UI;
13357
13358 // Skip uses of different values from the same node.
13359 if (Use.getResNo() != From.getResNo()) {
13360 ++UI;
13361 continue;
13362 }
13363
13364 // If this node hasn't been modified yet, it's still in the CSE maps,
13365 // so remove its old self from the CSE maps.
13366 if (!UserRemovedFromCSEMaps) {
13367 RemoveNodeFromCSEMaps(N: User);
13368 UserRemovedFromCSEMaps = true;
13369 }
13370
13371 ++UI;
13372 Use.set(To);
13373 if (To->isDivergent() != From->isDivergent())
13374 updateDivergence(N: User);
13375 } while (UI != UE && UI->getUser() == User);
13376 // We are iterating over all uses of the From node, so if a use
13377 // doesn't use the specific value, no changes are made.
13378 if (!UserRemovedFromCSEMaps)
13379 continue;
13380
13381 // Now that we have modified User, add it back to the CSE maps. If it
13382 // already exists there, recursively merge the results together.
13383 AddModifiedNodeToCSEMaps(N: User);
13384 }
13385
13386 // If we just RAUW'd the root, take note.
13387 if (From == getRoot())
13388 setRoot(To);
13389}
13390
13391namespace {
13392
13393/// UseMemo - This class is used by SelectionDAG::ReplaceAllUsesOfValuesWith
13394/// to record information about a use.
13395struct UseMemo {
13396 SDNode *User;
13397 unsigned Index;
13398 SDUse *Use;
13399};
13400
13401/// operator< - Sort Memos by User.
13402bool operator<(const UseMemo &L, const UseMemo &R) {
13403 return (intptr_t)L.User < (intptr_t)R.User;
13404}
13405
13406/// RAUOVWUpdateListener - Helper for ReplaceAllUsesOfValuesWith - When the node
13407/// pointed to by a UseMemo is deleted, set the User to nullptr to indicate that
13408/// the node already has been taken care of recursively.
13409class RAUOVWUpdateListener : public SelectionDAG::DAGUpdateListener {
13410 SmallVectorImpl<UseMemo> &Uses;
13411
13412 void NodeDeleted(SDNode *N, SDNode *E) override {
13413 for (UseMemo &Memo : Uses)
13414 if (Memo.User == N)
13415 Memo.User = nullptr;
13416 }
13417
13418public:
13419 RAUOVWUpdateListener(SelectionDAG &d, SmallVectorImpl<UseMemo> &uses)
13420 : SelectionDAG::DAGUpdateListener(d), Uses(uses) {}
13421};
13422
13423} // end anonymous namespace
13424
13425/// Return true if a glue output should propagate divergence information.
13426static bool gluePropagatesDivergence(const SDNode *Node) {
13427 switch (Node->getOpcode()) {
13428 case ISD::CopyFromReg:
13429 case ISD::CopyToReg:
13430 return false;
13431 default:
13432 return true;
13433 }
13434
13435 llvm_unreachable("covered opcode switch");
13436}
13437
13438bool SelectionDAG::calculateDivergence(SDNode *N) {
13439 if (TLI->isSDNodeAlwaysUniform(N)) {
13440 assert(!TLI->isSDNodeSourceOfDivergence(N, FLI, UA) &&
13441 "Conflicting divergence information!");
13442 return false;
13443 }
13444 if (TLI->isSDNodeSourceOfDivergence(N, FLI, UA))
13445 return true;
13446 for (const auto &Op : N->ops()) {
13447 EVT VT = Op.getValueType();
13448
13449 // Skip Chain. It does not carry divergence.
13450 if (VT != MVT::Other && Op.getNode()->isDivergent() &&
13451 (VT != MVT::Glue || gluePropagatesDivergence(Node: Op.getNode())))
13452 return true;
13453 }
13454 return false;
13455}
13456
13457void SelectionDAG::updateDivergence(SDNode *N) {
13458 SmallVector<SDNode *, 16> Worklist(1, N);
13459 do {
13460 N = Worklist.pop_back_val();
13461 bool IsDivergent = calculateDivergence(N);
13462 if (N->SDNodeBits.IsDivergent != IsDivergent) {
13463 N->SDNodeBits.IsDivergent = IsDivergent;
13464 llvm::append_range(C&: Worklist, R: N->users());
13465 }
13466 } while (!Worklist.empty());
13467}
13468
13469void SelectionDAG::CreateTopologicalOrder(std::vector<SDNode *> &Order) {
13470 DenseMap<SDNode *, unsigned> Degree;
13471 Order.reserve(n: AllNodes.size());
13472 for (auto &N : allnodes()) {
13473 unsigned NOps = N.getNumOperands();
13474 Degree[&N] = NOps;
13475 if (0 == NOps)
13476 Order.push_back(x: &N);
13477 }
13478 for (size_t I = 0; I != Order.size(); ++I) {
13479 SDNode *N = Order[I];
13480 for (auto *U : N->users()) {
13481 unsigned &UnsortedOps = Degree[U];
13482 if (0 == --UnsortedOps)
13483 Order.push_back(x: U);
13484 }
13485 }
13486}
13487
13488#if !defined(NDEBUG) && LLVM_ENABLE_ABI_BREAKING_CHECKS
13489void SelectionDAG::VerifyDAGDivergence() {
13490 std::vector<SDNode *> TopoOrder;
13491 CreateTopologicalOrder(TopoOrder);
13492 for (auto *N : TopoOrder) {
13493 assert(calculateDivergence(N) == N->isDivergent() &&
13494 "Divergence bit inconsistency detected");
13495 }
13496}
13497#endif
13498
13499/// ReplaceAllUsesOfValuesWith - Replace any uses of From with To, leaving
13500/// uses of other values produced by From.getNode() alone. The same value
13501/// may appear in both the From and To list. The Deleted vector is
13502/// handled the same way as for ReplaceAllUsesWith.
13503void SelectionDAG::ReplaceAllUsesOfValuesWith(const SDValue *From,
13504 const SDValue *To,
13505 unsigned Num){
13506 // Handle the simple, trivial case efficiently.
13507 if (Num == 1)
13508 return ReplaceAllUsesOfValueWith(From: *From, To: *To);
13509
13510 transferDbgValues(From: *From, To: *To);
13511 copyExtraInfo(From: From->getNode(), To: To->getNode());
13512
13513 // Read up all the uses and make records of them. This helps
13514 // processing new uses that are introduced during the
13515 // replacement process.
13516 SmallVector<UseMemo, 4> Uses;
13517 for (unsigned i = 0; i != Num; ++i) {
13518 unsigned FromResNo = From[i].getResNo();
13519 SDNode *FromNode = From[i].getNode();
13520 for (SDUse &Use : FromNode->uses()) {
13521 if (Use.getResNo() == FromResNo) {
13522 UseMemo Memo = {.User: Use.getUser(), .Index: i, .Use: &Use};
13523 Uses.push_back(Elt: Memo);
13524 }
13525 }
13526 }
13527
13528 // Sort the uses, so that all the uses from a given User are together.
13529 llvm::sort(C&: Uses);
13530 RAUOVWUpdateListener Listener(*this, Uses);
13531
13532 for (unsigned UseIndex = 0, UseIndexEnd = Uses.size();
13533 UseIndex != UseIndexEnd; ) {
13534 // We know that this user uses some value of From. If it is the right
13535 // value, update it.
13536 SDNode *User = Uses[UseIndex].User;
13537 // If the node has been deleted by recursive CSE updates when updating
13538 // another node, then just skip this entry.
13539 if (User == nullptr) {
13540 ++UseIndex;
13541 continue;
13542 }
13543
13544 // This node is about to morph, remove its old self from the CSE maps.
13545 RemoveNodeFromCSEMaps(N: User);
13546
13547 // The Uses array is sorted, so all the uses for a given User
13548 // are next to each other in the list.
13549 // To help reduce the number of CSE recomputations, process all
13550 // the uses of this user that we can find this way.
13551 do {
13552 unsigned i = Uses[UseIndex].Index;
13553 SDUse &Use = *Uses[UseIndex].Use;
13554 ++UseIndex;
13555
13556 Use.set(To[i]);
13557 } while (UseIndex != UseIndexEnd && Uses[UseIndex].User == User);
13558
13559 // Now that we have modified User, add it back to the CSE maps. If it
13560 // already exists there, recursively merge the results together.
13561 AddModifiedNodeToCSEMaps(N: User);
13562 }
13563}
13564
13565/// AssignTopologicalOrder - Assign a unique node id for each node in the DAG
13566/// based on their topological order. It returns the maximum id and a vector
13567/// of the SDNodes* in assigned order by reference.
13568unsigned SelectionDAG::AssignTopologicalOrder() {
13569 unsigned DAGSize = 0;
13570
13571 // SortedPos tracks the progress of the algorithm. Nodes before it are
13572 // sorted, nodes after it are unsorted. When the algorithm completes
13573 // it is at the end of the list.
13574 allnodes_iterator SortedPos = allnodes_begin();
13575
13576 // Visit all the nodes. Move nodes with no operands to the front of
13577 // the list immediately. Annotate nodes that do have operands with their
13578 // operand count. Before we do this, the Node Id fields of the nodes
13579 // may contain arbitrary values. After, the Node Id fields for nodes
13580 // before SortedPos will contain the topological sort index, and the
13581 // Node Id fields for nodes At SortedPos and after will contain the
13582 // count of outstanding operands.
13583 for (SDNode &N : llvm::make_early_inc_range(Range: allnodes())) {
13584 checkForCycles(N: &N, DAG: this);
13585 unsigned Degree = N.getNumOperands();
13586 if (Degree == 0) {
13587 // A node with no uses, add it to the result array immediately.
13588 N.setNodeId(DAGSize++);
13589 allnodes_iterator Q(&N);
13590 if (Q != SortedPos)
13591 SortedPos = AllNodes.insert(where: SortedPos, New: AllNodes.remove(IT&: Q));
13592 assert(SortedPos != AllNodes.end() && "Overran node list");
13593 ++SortedPos;
13594 } else {
13595 // Temporarily use the Node Id as scratch space for the degree count.
13596 N.setNodeId(Degree);
13597 }
13598 }
13599
13600 // Visit all the nodes. As we iterate, move nodes into sorted order,
13601 // such that by the time the end is reached all nodes will be sorted.
13602 for (SDNode &Node : allnodes()) {
13603 SDNode *N = &Node;
13604 checkForCycles(N, DAG: this);
13605 // N is in sorted position, so all its uses have one less operand
13606 // that needs to be sorted.
13607 for (SDNode *P : N->users()) {
13608 unsigned Degree = P->getNodeId();
13609 assert(Degree != 0 && "Invalid node degree");
13610 --Degree;
13611 if (Degree == 0) {
13612 // All of P's operands are sorted, so P may sorted now.
13613 P->setNodeId(DAGSize++);
13614 if (P->getIterator() != SortedPos)
13615 SortedPos = AllNodes.insert(where: SortedPos, New: AllNodes.remove(IT: P));
13616 assert(SortedPos != AllNodes.end() && "Overran node list");
13617 ++SortedPos;
13618 } else {
13619 // Update P's outstanding operand count.
13620 P->setNodeId(Degree);
13621 }
13622 }
13623 if (Node.getIterator() == SortedPos) {
13624#ifndef NDEBUG
13625 allnodes_iterator I(N);
13626 SDNode *S = &*++I;
13627 dbgs() << "Overran sorted position:\n";
13628 S->dumprFull(this); dbgs() << "\n";
13629 dbgs() << "Checking if this is due to cycles\n";
13630 checkForCycles(this, true);
13631#endif
13632 llvm_unreachable(nullptr);
13633 }
13634 }
13635
13636 assert(SortedPos == AllNodes.end() &&
13637 "Topological sort incomplete!");
13638 assert(AllNodes.front().getOpcode() == ISD::EntryToken &&
13639 "First node in topological sort is not the entry token!");
13640 assert(AllNodes.front().getNodeId() == 0 &&
13641 "First node in topological sort has non-zero id!");
13642 assert(AllNodes.front().getNumOperands() == 0 &&
13643 "First node in topological sort has operands!");
13644 assert(AllNodes.back().getNodeId() == (int)DAGSize-1 &&
13645 "Last node in topologic sort has unexpected id!");
13646 assert(AllNodes.back().use_empty() &&
13647 "Last node in topologic sort has users!");
13648 assert(DAGSize == allnodes_size() && "Node count mismatch!");
13649 return DAGSize;
13650}
13651
13652void SelectionDAG::getTopologicallyOrderedNodes(
13653 SmallVectorImpl<const SDNode *> &SortedNodes) const {
13654 SortedNodes.clear();
13655 // Node -> remaining number of outstanding operands.
13656 DenseMap<const SDNode *, unsigned> RemainingOperands;
13657
13658 // Put nodes without any operands into SortedNodes first.
13659 for (const SDNode &N : allnodes()) {
13660 checkForCycles(N: &N, DAG: this);
13661 unsigned NumOperands = N.getNumOperands();
13662 if (NumOperands == 0)
13663 SortedNodes.push_back(Elt: &N);
13664 else
13665 // Record their total number of outstanding operands.
13666 RemainingOperands[&N] = NumOperands;
13667 }
13668
13669 // A node is pushed into SortedNodes when all of its operands (predecessors in
13670 // the graph) are also in SortedNodes.
13671 for (unsigned i = 0U; i < SortedNodes.size(); ++i) {
13672 const SDNode *N = SortedNodes[i];
13673 for (const SDNode *U : N->users()) {
13674 // HandleSDNode is never part of a DAG and therefore has no entry in
13675 // RemainingOperands.
13676 if (U->getOpcode() == ISD::HANDLENODE)
13677 continue;
13678 unsigned &NumRemOperands = RemainingOperands[U];
13679 assert(NumRemOperands && "Invalid number of remaining operands");
13680 --NumRemOperands;
13681 if (!NumRemOperands)
13682 SortedNodes.push_back(Elt: U);
13683 }
13684 }
13685
13686 assert(SortedNodes.size() == AllNodes.size() && "Node count mismatch");
13687 assert(SortedNodes.front()->getOpcode() == ISD::EntryToken &&
13688 "First node in topological sort is not the entry token");
13689 assert(SortedNodes.front()->getNumOperands() == 0 &&
13690 "First node in topological sort has operands");
13691}
13692
13693/// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the
13694/// value is produced by SD.
13695void SelectionDAG::AddDbgValue(SDDbgValue *DB, bool isParameter) {
13696 for (SDNode *SD : DB->getSDNodes()) {
13697 if (!SD)
13698 continue;
13699 assert(DbgInfo->getSDDbgValues(SD).empty() || SD->getHasDebugValue());
13700 SD->setHasDebugValue(true);
13701 }
13702 DbgInfo->add(V: DB, isParameter);
13703}
13704
13705void SelectionDAG::AddDbgLabel(SDDbgLabel *DB) { DbgInfo->add(L: DB); }
13706
13707SDValue SelectionDAG::makeEquivalentMemoryOrdering(SDValue OldChain,
13708 SDValue NewMemOpChain) {
13709 assert(isa<MemSDNode>(NewMemOpChain) && "Expected a memop node");
13710 assert(NewMemOpChain.getValueType() == MVT::Other && "Expected a token VT");
13711 // The new memory operation must have the same position as the old load in
13712 // terms of memory dependency. Create a TokenFactor for the old load and new
13713 // memory operation and update uses of the old load's output chain to use that
13714 // TokenFactor.
13715 if (OldChain == NewMemOpChain || OldChain.use_empty())
13716 return NewMemOpChain;
13717
13718 SDValue TokenFactor = getNode(Opcode: ISD::TokenFactor, DL: SDLoc(OldChain), VT: MVT::Other,
13719 N1: OldChain, N2: NewMemOpChain);
13720 ReplaceAllUsesOfValueWith(From: OldChain, To: TokenFactor);
13721 UpdateNodeOperands(N: TokenFactor.getNode(), Op1: OldChain, Op2: NewMemOpChain);
13722 return TokenFactor;
13723}
13724
13725SDValue SelectionDAG::makeEquivalentMemoryOrdering(LoadSDNode *OldLoad,
13726 SDValue NewMemOp) {
13727 assert(isa<MemSDNode>(NewMemOp.getNode()) && "Expected a memop node");
13728 SDValue OldChain = SDValue(OldLoad, 1);
13729 SDValue NewMemOpChain = NewMemOp.getValue(R: 1);
13730 return makeEquivalentMemoryOrdering(OldChain, NewMemOpChain);
13731}
13732
13733SDValue SelectionDAG::getSymbolFunctionGlobalAddress(SDValue Op,
13734 Function **OutFunction) {
13735 assert(isa<ExternalSymbolSDNode>(Op) && "Node should be an ExternalSymbol");
13736
13737 auto *Symbol = cast<ExternalSymbolSDNode>(Val&: Op)->getSymbol();
13738 auto *Module = MF->getFunction().getParent();
13739 auto *Function = Module->getFunction(Name: Symbol);
13740
13741 if (OutFunction != nullptr)
13742 *OutFunction = Function;
13743
13744 if (Function != nullptr) {
13745 auto PtrTy = TLI->getPointerTy(DL: getDataLayout(), AS: Function->getAddressSpace());
13746 return getGlobalAddress(GV: Function, DL: SDLoc(Op), VT: PtrTy);
13747 }
13748
13749 std::string ErrorStr;
13750 raw_string_ostream ErrorFormatter(ErrorStr);
13751 ErrorFormatter << "Undefined external symbol ";
13752 ErrorFormatter << '"' << Symbol << '"';
13753 report_fatal_error(reason: Twine(ErrorStr));
13754}
13755
13756//===----------------------------------------------------------------------===//
13757// SDNode Class
13758//===----------------------------------------------------------------------===//
13759
13760bool llvm::isNullConstant(SDValue V) {
13761 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val&: V);
13762 return Const != nullptr && Const->isZero();
13763}
13764
13765bool llvm::isNullConstantOrUndef(SDValue V) {
13766 return V.isUndef() || isNullConstant(V);
13767}
13768
13769bool llvm::isNullFPConstant(SDValue V) {
13770 ConstantFPSDNode *Const = dyn_cast<ConstantFPSDNode>(Val&: V);
13771 return Const != nullptr && Const->isZero() && !Const->isNegative();
13772}
13773
13774bool llvm::isAllOnesConstant(SDValue V) {
13775 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val&: V);
13776 return Const != nullptr && Const->isAllOnes();
13777}
13778
13779bool llvm::isOneConstant(SDValue V) {
13780 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val&: V);
13781 return Const != nullptr && Const->isOne();
13782}
13783
13784bool llvm::isMinSignedConstant(SDValue V) {
13785 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val&: V);
13786 return Const != nullptr && Const->isMinSignedValue();
13787}
13788
13789bool SelectionDAG::isIdentityElement(unsigned Opcode, SDNodeFlags Flags,
13790 SDValue V, unsigned OperandNo,
13791 unsigned Depth) const {
13792 APInt DemandedElts = getDemandAllEltsMask(V);
13793 return isIdentityElement(Opc: Opcode, Flags, V, DemandedElts, OperandNo, Depth);
13794}
13795
13796bool SelectionDAG::isIdentityElement(unsigned Opcode, SDNodeFlags Flags,
13797 SDValue V, const APInt &DemandedElts,
13798 unsigned OperandNo, unsigned Depth) const {
13799 // NOTE: The cases should match with IR's ConstantExpr::getBinOpIdentity().
13800 // TODO: Target-specific opcodes could be added.
13801 if (V.getValueType().isInteger()) {
13802 KnownBits Known = computeKnownBits(Op: V, DemandedElts, Depth);
13803 if (Known.isConstant()) {
13804 const APInt &Const = Known.getConstant();
13805 switch (Opcode) {
13806 case ISD::ADD:
13807 case ISD::OR:
13808 case ISD::XOR:
13809 case ISD::UMAX:
13810 return Const.isZero();
13811 case ISD::MUL:
13812 return Const.isOne();
13813 case ISD::AND:
13814 case ISD::UMIN:
13815 return Const.isAllOnes();
13816 case ISD::SMAX:
13817 return Const.isMinSignedValue();
13818 case ISD::SMIN:
13819 return Const.isMaxSignedValue();
13820 case ISD::SUB:
13821 case ISD::SHL:
13822 case ISD::SRA:
13823 case ISD::SRL:
13824 return OperandNo == 1 && Const.isZero();
13825 case ISD::UDIV:
13826 case ISD::SDIV:
13827 return OperandNo == 1 && Const.isOne();
13828 }
13829 }
13830 } else if (auto *ConstFP = isConstOrConstSplatFP(N: V, DemandedElts)) {
13831 switch (Opcode) {
13832 case ISD::FADD:
13833 return ConstFP->isZero() &&
13834 (Flags.hasNoSignedZeros() || ConstFP->isNegative());
13835 case ISD::FSUB:
13836 return OperandNo == 1 && ConstFP->isZero() &&
13837 (Flags.hasNoSignedZeros() || !ConstFP->isNegative());
13838 case ISD::FMUL:
13839 return ConstFP->isOne();
13840 case ISD::FDIV:
13841 return OperandNo == 1 && ConstFP->isOne();
13842 case ISD::FMINNUM:
13843 case ISD::FMAXNUM:
13844 case ISD::FMINIMUMNUM:
13845 case ISD::FMAXIMUMNUM: {
13846 // Neutral element for fminnum/fminimumnum is NaN, Inf or FLT_MAX,
13847 // depending on fast-math flags (FMF).
13848 EVT VT = V.getValueType();
13849 const fltSemantics &Semantics = VT.getFltSemantics();
13850 APFloat NeutralAF = !Flags.hasNoNaNs() ? APFloat::getQNaN(Sem: Semantics)
13851 : !Flags.hasNoInfs() ? APFloat::getInf(Sem: Semantics)
13852 : APFloat::getLargest(Sem: Semantics);
13853 if (Opcode == ISD::FMAXNUM || Opcode == ISD::FMAXIMUMNUM)
13854 NeutralAF.changeSign();
13855
13856 return ConstFP->isExactlyValue(V: NeutralAF);
13857 }
13858 case ISD::FMINIMUM:
13859 case ISD::FMAXIMUM: {
13860 // Neutral element for fminimum is Inf or FLT_MAX, depending on FMF.
13861 const APFloat &VAPF = ConstFP->getValueAPF();
13862 bool NeutralNegative = (Opcode == ISD::FMAXIMUM);
13863 if (Flags.hasNoInfs())
13864 return VAPF.isLargest() && VAPF.isNegative() == NeutralNegative;
13865 return VAPF.isInfinity() && VAPF.isNegative() == NeutralNegative;
13866 }
13867 }
13868 }
13869 return false;
13870}
13871
13872SDValue llvm::peekThroughBitcasts(SDValue V) {
13873 while (V.getOpcode() == ISD::BITCAST)
13874 V = V.getOperand(i: 0);
13875 return V;
13876}
13877
13878SDValue llvm::peekThroughOneUseBitcasts(SDValue V) {
13879 while (V.getOpcode() == ISD::BITCAST && V.getOperand(i: 0).hasOneUse())
13880 V = V.getOperand(i: 0);
13881 return V;
13882}
13883
13884SDValue llvm::peekThroughExtractSubvectors(SDValue V) {
13885 while (V.getOpcode() == ISD::EXTRACT_SUBVECTOR)
13886 V = V.getOperand(i: 0);
13887 return V;
13888}
13889
13890SDValue llvm::peekThroughInsertVectorElt(SDValue V, const APInt &DemandedElts) {
13891 while (V.getOpcode() == ISD::INSERT_VECTOR_ELT) {
13892 SDValue InVec = V.getOperand(i: 0);
13893 SDValue EltNo = V.getOperand(i: 2);
13894 EVT VT = InVec.getValueType();
13895 auto *IndexC = dyn_cast<ConstantSDNode>(Val&: EltNo);
13896 if (IndexC && VT.isFixedLengthVector() &&
13897 IndexC->getAPIntValue().ult(RHS: VT.getVectorNumElements()) &&
13898 !DemandedElts[IndexC->getZExtValue()]) {
13899 V = InVec;
13900 continue;
13901 }
13902 break;
13903 }
13904 return V;
13905}
13906
13907SDValue llvm::peekThroughTruncates(SDValue V) {
13908 while (V.getOpcode() == ISD::TRUNCATE)
13909 V = V.getOperand(i: 0);
13910 return V;
13911}
13912
13913bool llvm::isBitwiseNot(SDValue V, bool AllowUndefs) {
13914 if (V.getOpcode() != ISD::XOR)
13915 return false;
13916 V = peekThroughBitcasts(V: V.getOperand(i: 1));
13917 unsigned NumBits = V.getScalarValueSizeInBits();
13918 ConstantSDNode *C =
13919 isConstOrConstSplat(N: V, AllowUndefs, /*AllowTruncation*/ true);
13920 return C && (C->getAPIntValue().countr_one() >= NumBits);
13921}
13922
13923ConstantSDNode *llvm::isConstOrConstSplat(SDValue N, bool AllowUndefs,
13924 bool AllowTruncation) {
13925 APInt DemandedElts = getDemandAllEltsMask(V: N);
13926 return isConstOrConstSplat(N, DemandedElts, AllowUndefs, AllowTruncation);
13927}
13928
13929ConstantSDNode *llvm::isConstOrConstSplat(SDValue N, const APInt &DemandedElts,
13930 bool AllowUndefs,
13931 bool AllowTruncation) {
13932 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val&: N))
13933 return CN;
13934
13935 // SplatVectors can truncate their operands. Ignore that case here unless
13936 // AllowTruncation is set.
13937 if (N->getOpcode() == ISD::SPLAT_VECTOR) {
13938 EVT VecEltVT = N->getValueType(ResNo: 0).getVectorElementType();
13939 if (auto *CN = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 0))) {
13940 EVT CVT = CN->getValueType(ResNo: 0);
13941 assert(CVT.bitsGE(VecEltVT) && "Illegal splat_vector element extension");
13942 if (AllowTruncation || CVT == VecEltVT)
13943 return CN;
13944 }
13945 }
13946
13947 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Val&: N)) {
13948 BitVector UndefElements;
13949 ConstantSDNode *CN = BV->getConstantSplatNode(DemandedElts, UndefElements: &UndefElements);
13950
13951 // BuildVectors can truncate their operands. Ignore that case here unless
13952 // AllowTruncation is set.
13953 // TODO: Look into whether we should allow UndefElements in non-DemandedElts
13954 if (CN && (UndefElements.none() || AllowUndefs)) {
13955 EVT CVT = CN->getValueType(ResNo: 0);
13956 EVT NSVT = N.getValueType().getScalarType();
13957 assert(CVT.bitsGE(NSVT) && "Illegal build vector element extension");
13958 if (AllowTruncation || (CVT == NSVT))
13959 return CN;
13960 }
13961 }
13962
13963 return nullptr;
13964}
13965
13966ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N, bool AllowUndefs) {
13967 APInt DemandedElts = getDemandAllEltsMask(V: N);
13968 return isConstOrConstSplatFP(N, DemandedElts, AllowUndefs);
13969}
13970
13971ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N,
13972 const APInt &DemandedElts,
13973 bool AllowUndefs) {
13974 if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(Val&: N))
13975 return CN;
13976
13977 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Val&: N)) {
13978 BitVector UndefElements;
13979 ConstantFPSDNode *CN =
13980 BV->getConstantFPSplatNode(DemandedElts, UndefElements: &UndefElements);
13981 // TODO: Look into whether we should allow UndefElements in non-DemandedElts
13982 if (CN && (UndefElements.none() || AllowUndefs))
13983 return CN;
13984 }
13985
13986 if (N.getOpcode() == ISD::SPLAT_VECTOR)
13987 if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(Val: N.getOperand(i: 0)))
13988 return CN;
13989
13990 return nullptr;
13991}
13992
13993bool llvm::isNullOrNullSplat(SDValue N, bool AllowUndefs) {
13994 // TODO: may want to use peekThroughBitcast() here.
13995 ConstantSDNode *C =
13996 isConstOrConstSplat(N, AllowUndefs, /*AllowTruncation=*/true);
13997 return C && C->isZero();
13998}
13999
14000bool llvm::isOneOrOneSplat(SDValue N, bool AllowUndefs) {
14001 ConstantSDNode *C =
14002 isConstOrConstSplat(N, AllowUndefs, /*AllowTruncation*/ true);
14003 return C && C->isOne();
14004}
14005
14006bool llvm::isOneOrOneSplatFP(SDValue N, bool AllowUndefs) {
14007 ConstantFPSDNode *C = isConstOrConstSplatFP(N, AllowUndefs);
14008 return C && C->isOne();
14009}
14010
14011bool llvm::isAllOnesOrAllOnesSplat(SDValue N, bool AllowUndefs) {
14012 N = peekThroughBitcasts(V: N);
14013 unsigned BitWidth = N.getScalarValueSizeInBits();
14014 ConstantSDNode *C =
14015 isConstOrConstSplat(N, AllowUndefs, /*AllowTruncation=*/true);
14016 return C && C->getAPIntValue().countTrailingOnes() >= BitWidth;
14017}
14018
14019bool llvm::isOnesOrOnesSplat(SDValue N, bool AllowUndefs) {
14020 ConstantSDNode *C = isConstOrConstSplat(N, AllowUndefs);
14021 return C && APInt::isSameValue(I1: C->getAPIntValue(),
14022 I2: APInt(C->getAPIntValue().getBitWidth(), 1));
14023}
14024
14025bool llvm::isZeroOrZeroSplat(SDValue N, bool AllowUndefs) {
14026 N = peekThroughBitcasts(V: N);
14027 ConstantSDNode *C = isConstOrConstSplat(N, AllowUndefs, AllowTruncation: true);
14028 return C && C->isZero();
14029}
14030
14031bool llvm::isZeroOrZeroSplatFP(SDValue N, bool AllowUndefs) {
14032 ConstantFPSDNode *C = isConstOrConstSplatFP(N, AllowUndefs);
14033 return C && C->isZero();
14034}
14035
14036HandleSDNode::~HandleSDNode() {
14037 DropOperands();
14038}
14039
14040MemSDNode::MemSDNode(
14041 unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTs, EVT memvt,
14042 PointerUnion<MachineMemOperand *, MachineMemOperand **> memrefs)
14043 : SDNode(Opc, Order, dl, VTs), MemoryVT(memvt), MemRefs(memrefs) {
14044 bool IsVolatile = false;
14045 bool IsNonTemporal = false;
14046 bool IsDereferenceable = true;
14047 bool IsInvariant = true;
14048 for (const MachineMemOperand *MMO : memoperands()) {
14049 IsVolatile |= MMO->isVolatile();
14050 IsNonTemporal |= MMO->isNonTemporal();
14051 IsDereferenceable &= MMO->isDereferenceable();
14052 IsInvariant &= MMO->isInvariant();
14053 }
14054 MemSDNodeBits.IsVolatile = IsVolatile;
14055 MemSDNodeBits.IsNonTemporal = IsNonTemporal;
14056 MemSDNodeBits.IsDereferenceable = IsDereferenceable;
14057 MemSDNodeBits.IsInvariant = IsInvariant;
14058
14059 // For the single-MMO case, we check here that the size of the memory operand
14060 // fits within the size of the MMO. This is because the MMO might indicate
14061 // only a possible address range instead of specifying the affected memory
14062 // addresses precisely.
14063 assert((getNumMemOperands() != 1 || !getMemOperand()->getType().isValid() ||
14064 TypeSize::isKnownLE(memvt.getStoreSize(),
14065 getMemOperand()->getSize().getValue())) &&
14066 "Size mismatch!");
14067}
14068
14069/// Profile - Gather unique data for the node.
14070///
14071void SDNode::Profile(FoldingSetNodeID &ID) const {
14072 AddNodeIDNode(ID, N: this);
14073}
14074
14075namespace {
14076
14077 struct EVTArray {
14078 std::vector<EVT> VTs;
14079
14080 EVTArray() {
14081 VTs.reserve(n: MVT::VALUETYPE_SIZE);
14082 for (unsigned i = 0; i < MVT::VALUETYPE_SIZE; ++i)
14083 VTs.push_back(x: MVT((MVT::SimpleValueType)i));
14084 }
14085 };
14086
14087} // end anonymous namespace
14088
14089/// getValueTypeList - Return a pointer to the specified value type.
14090///
14091const EVT *SDNode::getValueTypeList(MVT VT) {
14092 static EVTArray SimpleVTArray;
14093
14094 assert(VT < MVT::VALUETYPE_SIZE && "Value type out of range!");
14095 return &SimpleVTArray.VTs[VT.SimpleTy];
14096}
14097
14098/// hasAnyUseOfValue - Return true if there are any use of the indicated
14099/// value. This method ignores uses of other values defined by this operation.
14100bool SDNode::hasAnyUseOfValue(unsigned Value) const {
14101 assert(Value < getNumValues() && "Bad value!");
14102
14103 for (SDUse &U : uses())
14104 if (U.getResNo() == Value)
14105 return true;
14106
14107 return false;
14108}
14109
14110/// isOnlyUserOf - Return true if this node is the only use of N.
14111bool SDNode::isOnlyUserOf(const SDNode *N) const {
14112 bool Seen = false;
14113 for (const SDNode *User : N->users()) {
14114 if (User == this)
14115 Seen = true;
14116 else
14117 return false;
14118 }
14119
14120 return Seen;
14121}
14122
14123/// Return true if the only users of N are contained in Nodes.
14124bool SDNode::areOnlyUsersOf(ArrayRef<const SDNode *> Nodes, const SDNode *N) {
14125 bool Seen = false;
14126 for (const SDNode *User : N->users()) {
14127 if (llvm::is_contained(Range&: Nodes, Element: User))
14128 Seen = true;
14129 else
14130 return false;
14131 }
14132
14133 return Seen;
14134}
14135
14136/// Return true if the referenced return value is an operand of N.
14137bool SDValue::isOperandOf(const SDNode *N) const {
14138 return is_contained(Range: N->op_values(), Element: *this);
14139}
14140
14141bool SDNode::isOperandOf(const SDNode *N) const {
14142 return any_of(Range: N->op_values(),
14143 P: [this](SDValue Op) { return this == Op.getNode(); });
14144}
14145
14146/// reachesChainWithoutSideEffects - Return true if this operand (which must
14147/// be a chain) reaches the specified operand without crossing any
14148/// side-effecting instructions on any chain path. In practice, this looks
14149/// through token factors and non-volatile loads. In order to remain efficient,
14150/// this only looks a couple of nodes in, it does not do an exhaustive search.
14151///
14152/// Note that we only need to examine chains when we're searching for
14153/// side-effects; SelectionDAG requires that all side-effects are represented
14154/// by chains, even if another operand would force a specific ordering. This
14155/// constraint is necessary to allow transformations like splitting loads.
14156bool SDValue::reachesChainWithoutSideEffects(SDValue Dest,
14157 unsigned Depth) const {
14158 if (*this == Dest) return true;
14159
14160 // Don't search too deeply, we just want to be able to see through
14161 // TokenFactor's etc.
14162 if (Depth == 0) return false;
14163
14164 // If this is a token factor, all inputs to the TF happen in parallel.
14165 if (getOpcode() == ISD::TokenFactor) {
14166 // First, try a shallow search.
14167 if (is_contained(Range: (*this)->ops(), Element: Dest)) {
14168 // We found the chain we want as an operand of this TokenFactor.
14169 // Essentially, we reach the chain without side-effects if we could
14170 // serialize the TokenFactor into a simple chain of operations with
14171 // Dest as the last operation. This is automatically true if the
14172 // chain has one use: there are no other ordering constraints.
14173 // If the chain has more than one use, we give up: some other
14174 // use of Dest might force a side-effect between Dest and the current
14175 // node.
14176 if (Dest.hasOneUse())
14177 return true;
14178 }
14179 // Next, try a deep search: check whether every operand of the TokenFactor
14180 // reaches Dest.
14181 return llvm::all_of(Range: (*this)->ops(), P: [=](SDValue Op) {
14182 return Op.reachesChainWithoutSideEffects(Dest, Depth: Depth - 1);
14183 });
14184 }
14185
14186 // Loads don't have side effects, look through them.
14187 if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(Val: *this)) {
14188 if (Ld->isUnordered())
14189 return Ld->getChain().reachesChainWithoutSideEffects(Dest, Depth: Depth-1);
14190 }
14191 return false;
14192}
14193
14194bool SDNode::hasPredecessor(const SDNode *N) const {
14195 SmallPtrSet<const SDNode *, 32> Visited;
14196 SmallVector<const SDNode *, 16> Worklist;
14197 Worklist.push_back(Elt: this);
14198 return hasPredecessorHelper(N, Visited, Worklist);
14199}
14200
14201void SDNode::intersectFlagsWith(const SDNodeFlags Flags) {
14202 this->Flags &= Flags;
14203}
14204
14205SDValue
14206SelectionDAG::matchBinOpReduction(SDNode *Extract, ISD::NodeType &BinOp,
14207 ArrayRef<ISD::NodeType> CandidateBinOps,
14208 bool AllowPartials) {
14209 // The pattern must end in an extract from index 0.
14210 if (Extract->getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
14211 !isNullConstant(V: Extract->getOperand(Num: 1)))
14212 return SDValue();
14213
14214 // Match against one of the candidate binary ops.
14215 SDValue Op = Extract->getOperand(Num: 0);
14216 if (llvm::none_of(Range&: CandidateBinOps, P: [Op](ISD::NodeType BinOp) {
14217 return Op.getOpcode() == unsigned(BinOp);
14218 }))
14219 return SDValue();
14220
14221 // Floating-point reductions may require relaxed constraints on the final step
14222 // of the reduction because they may reorder intermediate operations.
14223 unsigned CandidateBinOp = Op.getOpcode();
14224 if (Op.getValueType().isFloatingPoint()) {
14225 SDNodeFlags Flags = Op->getFlags();
14226 switch (CandidateBinOp) {
14227 case ISD::FADD:
14228 if (!Flags.hasNoSignedZeros() || !Flags.hasAllowReassociation())
14229 return SDValue();
14230 break;
14231 default:
14232 llvm_unreachable("Unhandled FP opcode for binop reduction");
14233 }
14234 }
14235
14236 // Matching failed - attempt to see if we did enough stages that a partial
14237 // reduction from a subvector is possible.
14238 auto PartialReduction = [&](SDValue Op, unsigned NumSubElts) {
14239 if (!AllowPartials || !Op)
14240 return SDValue();
14241 EVT OpVT = Op.getValueType();
14242 EVT OpSVT = OpVT.getScalarType();
14243 EVT SubVT = EVT::getVectorVT(Context&: *getContext(), VT: OpSVT, NumElements: NumSubElts);
14244 if (TLI->getExtractSubvectorCost(ResVT: SubVT, SrcVT: OpVT, Index: 0) >
14245 TargetLowering::ExtractSubvectorCost::Cheap)
14246 return SDValue();
14247 BinOp = (ISD::NodeType)CandidateBinOp;
14248 return getExtractSubvector(DL: SDLoc(Op), VT: SubVT, Vec: Op, Idx: 0);
14249 };
14250
14251 // At each stage, we're looking for something that looks like:
14252 // %s = shufflevector <8 x i32> %op, <8 x i32> undef,
14253 // <8 x i32> <i32 2, i32 3, i32 undef, i32 undef,
14254 // i32 undef, i32 undef, i32 undef, i32 undef>
14255 // %a = binop <8 x i32> %op, %s
14256 // Where the mask changes according to the stage. E.g. for a 3-stage pyramid,
14257 // we expect something like:
14258 // <4,5,6,7,u,u,u,u>
14259 // <2,3,u,u,u,u,u,u>
14260 // <1,u,u,u,u,u,u,u>
14261 // While a partial reduction match would be:
14262 // <2,3,u,u,u,u,u,u>
14263 // <1,u,u,u,u,u,u,u>
14264 unsigned Stages = Log2_32(Value: Op.getValueType().getVectorNumElements());
14265 SDValue PrevOp;
14266 for (unsigned i = 0; i < Stages; ++i) {
14267 unsigned MaskEnd = (1 << i);
14268
14269 if (Op.getOpcode() != CandidateBinOp)
14270 return PartialReduction(PrevOp, MaskEnd);
14271
14272 SDValue Op0 = Op.getOperand(i: 0);
14273 SDValue Op1 = Op.getOperand(i: 1);
14274
14275 ShuffleVectorSDNode *Shuffle = dyn_cast<ShuffleVectorSDNode>(Val&: Op0);
14276 if (Shuffle) {
14277 Op = Op1;
14278 } else {
14279 Shuffle = dyn_cast<ShuffleVectorSDNode>(Val&: Op1);
14280 Op = Op0;
14281 }
14282
14283 // The first operand of the shuffle should be the same as the other operand
14284 // of the binop.
14285 if (!Shuffle || Shuffle->getOperand(Num: 0) != Op)
14286 return PartialReduction(PrevOp, MaskEnd);
14287
14288 // Verify the shuffle has the expected (at this stage of the pyramid) mask.
14289 for (int Index = 0; Index < (int)MaskEnd; ++Index)
14290 if (Shuffle->getMaskElt(Idx: Index) != (int)(MaskEnd + Index))
14291 return PartialReduction(PrevOp, MaskEnd);
14292
14293 PrevOp = Op;
14294 }
14295
14296 // Handle subvector reductions, which tend to appear after the shuffle
14297 // reduction stages.
14298 while (Op.getOpcode() == CandidateBinOp) {
14299 unsigned NumElts = Op.getValueType().getVectorNumElements();
14300 SDValue Op0 = Op.getOperand(i: 0);
14301 SDValue Op1 = Op.getOperand(i: 1);
14302 if (Op0.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
14303 Op1.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
14304 Op0.getOperand(i: 0) != Op1.getOperand(i: 0))
14305 break;
14306 SDValue Src = Op0.getOperand(i: 0);
14307 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
14308 if (NumSrcElts != (2 * NumElts))
14309 break;
14310 if (!(Op0.getConstantOperandAPInt(i: 1) == 0 &&
14311 Op1.getConstantOperandAPInt(i: 1) == NumElts) &&
14312 !(Op1.getConstantOperandAPInt(i: 1) == 0 &&
14313 Op0.getConstantOperandAPInt(i: 1) == NumElts))
14314 break;
14315 Op = Src;
14316 }
14317
14318 BinOp = (ISD::NodeType)CandidateBinOp;
14319 return Op;
14320}
14321
14322SDValue SelectionDAG::UnrollVectorOp(SDNode *N, unsigned ResNE) {
14323 EVT VT = N->getValueType(ResNo: 0);
14324 EVT EltVT = VT.getVectorElementType();
14325 unsigned NE = VT.getVectorNumElements();
14326
14327 SDLoc dl(N);
14328
14329 // If ResNE is 0, fully unroll the vector op.
14330 if (ResNE == 0)
14331 ResNE = NE;
14332 else if (NE > ResNE)
14333 NE = ResNE;
14334
14335 if (N->getNumValues() == 2) {
14336 SmallVector<SDValue, 8> Scalars0, Scalars1;
14337 SmallVector<SDValue, 4> Operands(N->getNumOperands());
14338 EVT VT1 = N->getValueType(ResNo: 1);
14339 EVT EltVT1 = VT1.getVectorElementType();
14340
14341 unsigned i;
14342 for (i = 0; i != NE; ++i) {
14343 for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) {
14344 SDValue Operand = N->getOperand(Num: j);
14345 EVT OperandVT = Operand.getValueType();
14346
14347 // A vector operand; extract a single element.
14348 EVT OperandEltVT = OperandVT.getVectorElementType();
14349 Operands[j] = getExtractVectorElt(DL: dl, VT: OperandEltVT, Vec: Operand, Idx: i);
14350 }
14351
14352 SDValue EltOp = getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {EltVT, EltVT1}, Ops: Operands);
14353 Scalars0.push_back(Elt: EltOp);
14354 Scalars1.push_back(Elt: EltOp.getValue(R: 1));
14355 }
14356
14357 for (; i < ResNE; ++i) {
14358 Scalars0.push_back(Elt: getUNDEF(VT: EltVT));
14359 Scalars1.push_back(Elt: getUNDEF(VT: EltVT1));
14360 }
14361
14362 EVT VecVT = EVT::getVectorVT(Context&: *getContext(), VT: EltVT, NumElements: ResNE);
14363 EVT VecVT1 = EVT::getVectorVT(Context&: *getContext(), VT: EltVT1, NumElements: ResNE);
14364 SDValue Vec0 = getBuildVector(VT: VecVT, DL: dl, Ops: Scalars0);
14365 SDValue Vec1 = getBuildVector(VT: VecVT1, DL: dl, Ops: Scalars1);
14366 return getMergeValues(Ops: {Vec0, Vec1}, dl);
14367 }
14368
14369 assert(N->getNumValues() == 1 &&
14370 "Can't unroll a vector with multiple results!");
14371
14372 SmallVector<SDValue, 8> Scalars;
14373 SmallVector<SDValue, 4> Operands(N->getNumOperands());
14374
14375 unsigned i;
14376 for (i= 0; i != NE; ++i) {
14377 for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) {
14378 SDValue Operand = N->getOperand(Num: j);
14379 EVT OperandVT = Operand.getValueType();
14380 if (OperandVT.isVector()) {
14381 // A vector operand; extract a single element.
14382 EVT OperandEltVT = OperandVT.getVectorElementType();
14383 Operands[j] = getExtractVectorElt(DL: dl, VT: OperandEltVT, Vec: Operand, Idx: i);
14384 } else {
14385 // A scalar operand; just use it as is.
14386 Operands[j] = Operand;
14387 }
14388 }
14389
14390 switch (N->getOpcode()) {
14391 default: {
14392 Scalars.push_back(Elt: getNode(Opcode: N->getOpcode(), DL: dl, VT: EltVT, Ops: Operands,
14393 Flags: N->getFlags()));
14394 break;
14395 }
14396 case ISD::VSELECT:
14397 Scalars.push_back(
14398 Elt: getNode(Opcode: ISD::SELECT, DL: dl, VT: EltVT, Ops: Operands, Flags: N->getFlags()));
14399 break;
14400 case ISD::SHL:
14401 case ISD::SRA:
14402 case ISD::SRL:
14403 case ISD::ROTL:
14404 case ISD::ROTR:
14405 Scalars.push_back(Elt: getNode(Opcode: N->getOpcode(), DL: dl, VT: EltVT, N1: Operands[0],
14406 N2: getShiftAmountOperand(LHSTy: Operands[0].getValueType(),
14407 Op: Operands[1])));
14408 break;
14409 case ISD::SIGN_EXTEND_INREG: {
14410 EVT ExtVT = cast<VTSDNode>(Val&: Operands[1])->getVT().getVectorElementType();
14411 Scalars.push_back(Elt: getNode(Opcode: N->getOpcode(), DL: dl, VT: EltVT,
14412 N1: Operands[0],
14413 N2: getValueType(VT: ExtVT)));
14414 break;
14415 }
14416 case ISD::ADDRSPACECAST: {
14417 const auto *ASC = cast<AddrSpaceCastSDNode>(Val: N);
14418 Scalars.push_back(Elt: getAddrSpaceCast(dl, VT: EltVT, Ptr: Operands[0],
14419 SrcAS: ASC->getSrcAddressSpace(),
14420 DestAS: ASC->getDestAddressSpace()));
14421 break;
14422 }
14423 }
14424 }
14425
14426 for (; i < ResNE; ++i)
14427 Scalars.push_back(Elt: getUNDEF(VT: EltVT));
14428
14429 EVT VecVT = EVT::getVectorVT(Context&: *getContext(), VT: EltVT, NumElements: ResNE);
14430 return getBuildVector(VT: VecVT, DL: dl, Ops: Scalars);
14431}
14432
14433std::pair<SDValue, SDValue> SelectionDAG::UnrollVectorOverflowOp(
14434 SDNode *N, unsigned ResNE) {
14435 unsigned Opcode = N->getOpcode();
14436 assert((Opcode == ISD::UADDO || Opcode == ISD::SADDO ||
14437 Opcode == ISD::USUBO || Opcode == ISD::SSUBO ||
14438 Opcode == ISD::UMULO || Opcode == ISD::SMULO) &&
14439 "Expected an overflow opcode");
14440
14441 EVT ResVT = N->getValueType(ResNo: 0);
14442 EVT OvVT = N->getValueType(ResNo: 1);
14443 EVT ResEltVT = ResVT.getVectorElementType();
14444 EVT OvEltVT = OvVT.getVectorElementType();
14445 SDLoc dl(N);
14446
14447 // If ResNE is 0, fully unroll the vector op.
14448 unsigned NE = ResVT.getVectorNumElements();
14449 if (ResNE == 0)
14450 ResNE = NE;
14451 else if (NE > ResNE)
14452 NE = ResNE;
14453
14454 SmallVector<SDValue, 8> LHSScalars;
14455 SmallVector<SDValue, 8> RHSScalars;
14456 ExtractVectorElements(Op: N->getOperand(Num: 0), Args&: LHSScalars, Start: 0, Count: NE);
14457 ExtractVectorElements(Op: N->getOperand(Num: 1), Args&: RHSScalars, Start: 0, Count: NE);
14458
14459 EVT SVT = TLI->getSetCCResultType(DL: getDataLayout(), Context&: *getContext(), VT: ResEltVT);
14460 SDVTList VTs = getVTList(VT1: ResEltVT, VT2: SVT);
14461 SmallVector<SDValue, 8> ResScalars;
14462 SmallVector<SDValue, 8> OvScalars;
14463 for (unsigned i = 0; i < NE; ++i) {
14464 SDValue Res = getNode(Opcode, DL: dl, VTList: VTs, N1: LHSScalars[i], N2: RHSScalars[i]);
14465 SDValue Ov =
14466 getSelect(DL: dl, VT: OvEltVT, Cond: Res.getValue(R: 1),
14467 LHS: getBoolConstant(V: true, DL: dl, VT: OvEltVT, OpVT: ResVT),
14468 RHS: getConstant(Val: 0, DL: dl, VT: OvEltVT));
14469
14470 ResScalars.push_back(Elt: Res);
14471 OvScalars.push_back(Elt: Ov);
14472 }
14473
14474 ResScalars.append(NumInputs: ResNE - NE, Elt: getUNDEF(VT: ResEltVT));
14475 OvScalars.append(NumInputs: ResNE - NE, Elt: getUNDEF(VT: OvEltVT));
14476
14477 EVT NewResVT = EVT::getVectorVT(Context&: *getContext(), VT: ResEltVT, NumElements: ResNE);
14478 EVT NewOvVT = EVT::getVectorVT(Context&: *getContext(), VT: OvEltVT, NumElements: ResNE);
14479 return std::make_pair(x: getBuildVector(VT: NewResVT, DL: dl, Ops: ResScalars),
14480 y: getBuildVector(VT: NewOvVT, DL: dl, Ops: OvScalars));
14481}
14482
14483bool SelectionDAG::areNonVolatileConsecutiveLoads(LoadSDNode *LD,
14484 LoadSDNode *Base,
14485 unsigned Bytes,
14486 int Dist) const {
14487 if (LD->isVolatile() || Base->isVolatile())
14488 return false;
14489 // TODO: probably too restrictive for atomics, revisit
14490 if (!LD->isSimple())
14491 return false;
14492 if (LD->isIndexed() || Base->isIndexed())
14493 return false;
14494 if (LD->getChain() != Base->getChain())
14495 return false;
14496 EVT VT = LD->getMemoryVT();
14497 if (VT.getSizeInBits() / 8 != Bytes)
14498 return false;
14499
14500 auto BaseLocDecomp = BaseIndexOffset::match(N: Base, DAG: *this);
14501 auto LocDecomp = BaseIndexOffset::match(N: LD, DAG: *this);
14502
14503 int64_t Offset = 0;
14504 if (BaseLocDecomp.equalBaseIndex(Other: LocDecomp, DAG: *this, Off&: Offset))
14505 return (Dist * (int64_t)Bytes == Offset);
14506 return false;
14507}
14508
14509/// InferPtrAlignment - Infer alignment of a load / store address. Return
14510/// std::nullopt if it cannot be inferred.
14511MaybeAlign SelectionDAG::InferPtrAlign(SDValue Ptr) const {
14512 // If this is a GlobalAddress + cst, return the alignment.
14513 const GlobalValue *GV = nullptr;
14514 int64_t GVOffset = 0;
14515 if (TLI->isGAPlusOffset(N: Ptr.getNode(), GA&: GV, Offset&: GVOffset)) {
14516 unsigned PtrWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType());
14517 KnownBits Known(PtrWidth);
14518 llvm::computeKnownBits(V: GV, Known, DL: getDataLayout());
14519 unsigned AlignBits = Known.countMinTrailingZeros();
14520 if (AlignBits)
14521 return commonAlignment(A: Align(1ull << std::min(a: 31U, b: AlignBits)), Offset: GVOffset);
14522 }
14523
14524 // If this is a direct reference to a stack slot, use information about the
14525 // stack slot's alignment.
14526 int FrameIdx = INT_MIN;
14527 int64_t FrameOffset = 0;
14528 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Ptr)) {
14529 FrameIdx = FI->getIndex();
14530 } else if (isBaseWithConstantOffset(Op: Ptr) &&
14531 isa<FrameIndexSDNode>(Val: Ptr.getOperand(i: 0))) {
14532 // Handle FI+Cst
14533 FrameIdx = cast<FrameIndexSDNode>(Val: Ptr.getOperand(i: 0))->getIndex();
14534 FrameOffset = Ptr.getConstantOperandVal(i: 1);
14535 }
14536
14537 if (FrameIdx != INT_MIN) {
14538 const MachineFrameInfo &MFI = getMachineFunction().getFrameInfo();
14539 return commonAlignment(A: MFI.getObjectAlign(ObjectIdx: FrameIdx), Offset: FrameOffset);
14540 }
14541
14542 return std::nullopt;
14543}
14544
14545/// Split the scalar node with EXTRACT_ELEMENT using the provided
14546/// VTs and return the low/high part.
14547std::pair<SDValue, SDValue> SelectionDAG::SplitScalar(const SDValue &N,
14548 const SDLoc &DL,
14549 const EVT &LoVT,
14550 const EVT &HiVT) {
14551 assert(!LoVT.isVector() && !HiVT.isVector() && !N.getValueType().isVector() &&
14552 "Split node must be a scalar type");
14553 SDValue Lo =
14554 getNode(Opcode: ISD::EXTRACT_ELEMENT, DL, VT: LoVT, N1: N, N2: getIntPtrConstant(Val: 0, DL));
14555 SDValue Hi =
14556 getNode(Opcode: ISD::EXTRACT_ELEMENT, DL, VT: HiVT, N1: N, N2: getIntPtrConstant(Val: 1, DL));
14557 return std::make_pair(x&: Lo, y&: Hi);
14558}
14559
14560/// GetSplitDestVTs - Compute the VTs needed for the low/hi parts of a type
14561/// which is split (or expanded) into two not necessarily identical pieces.
14562std::pair<EVT, EVT> SelectionDAG::GetSplitDestVTs(const EVT &VT) const {
14563 // Currently all types are split in half.
14564 EVT LoVT, HiVT;
14565 if (!VT.isVector())
14566 LoVT = HiVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT);
14567 else
14568 LoVT = HiVT = VT.getHalfNumVectorElementsVT(Context&: *getContext());
14569
14570 return std::make_pair(x&: LoVT, y&: HiVT);
14571}
14572
14573/// GetDependentSplitDestVTs - Compute the VTs needed for the low/hi parts of a
14574/// type, dependent on an enveloping VT that has been split into two identical
14575/// pieces. Sets the HiIsEmpty flag when hi type has zero storage size.
14576std::pair<EVT, EVT>
14577SelectionDAG::GetDependentSplitDestVTs(const EVT &VT, const EVT &EnvVT,
14578 bool *HiIsEmpty) const {
14579 EVT EltTp = VT.getVectorElementType();
14580 // Examples:
14581 // custom VL=8 with enveloping VL=8/8 yields 8/0 (hi empty)
14582 // custom VL=9 with enveloping VL=8/8 yields 8/1
14583 // custom VL=10 with enveloping VL=8/8 yields 8/2
14584 // etc.
14585 ElementCount VTNumElts = VT.getVectorElementCount();
14586 ElementCount EnvNumElts = EnvVT.getVectorElementCount();
14587 assert(VTNumElts.isScalable() == EnvNumElts.isScalable() &&
14588 "Mixing fixed width and scalable vectors when enveloping a type");
14589 EVT LoVT, HiVT;
14590 if (VTNumElts.getKnownMinValue() > EnvNumElts.getKnownMinValue()) {
14591 LoVT = EVT::getVectorVT(Context&: *getContext(), VT: EltTp, EC: EnvNumElts);
14592 HiVT = EVT::getVectorVT(Context&: *getContext(), VT: EltTp, EC: VTNumElts - EnvNumElts);
14593 *HiIsEmpty = false;
14594 } else {
14595 // Flag that hi type has zero storage size, but return split envelop type
14596 // (this would be easier if vector types with zero elements were allowed).
14597 LoVT = EVT::getVectorVT(Context&: *getContext(), VT: EltTp, EC: VTNumElts);
14598 HiVT = EVT::getVectorVT(Context&: *getContext(), VT: EltTp, EC: EnvNumElts);
14599 *HiIsEmpty = true;
14600 }
14601 return std::make_pair(x&: LoVT, y&: HiVT);
14602}
14603
14604/// SplitVector - Split the vector with EXTRACT_SUBVECTOR and return the
14605/// low/high part.
14606std::pair<SDValue, SDValue>
14607SelectionDAG::SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT,
14608 const EVT &HiVT) {
14609 assert(LoVT.isScalableVector() == HiVT.isScalableVector() &&
14610 LoVT.isScalableVector() == N.getValueType().isScalableVector() &&
14611 "Splitting vector with an invalid mixture of fixed and scalable "
14612 "vector types");
14613 assert(LoVT.getVectorMinNumElements() + HiVT.getVectorMinNumElements() <=
14614 N.getValueType().getVectorMinNumElements() &&
14615 "More vector elements requested than available!");
14616 SDValue Lo, Hi;
14617 Lo = getExtractSubvector(DL, VT: LoVT, Vec: N, Idx: 0);
14618 // For scalable vectors it is safe to use LoVT.getVectorMinNumElements()
14619 // (rather than having to use ElementCount), because EXTRACT_SUBVECTOR scales
14620 // IDX with the runtime scaling factor of the result vector type. For
14621 // fixed-width result vectors, that runtime scaling factor is 1.
14622 Hi = getExtractSubvector(DL, VT: HiVT, Vec: N, Idx: LoVT.getVectorMinNumElements());
14623 return std::make_pair(x&: Lo, y&: Hi);
14624}
14625
14626std::pair<SDValue, SDValue> SelectionDAG::SplitEVL(SDValue N, EVT VecVT,
14627 const SDLoc &DL) {
14628 // Split the vector length parameter.
14629 // %evl -> umin(%evl, %halfnumelts) and usubsat(%evl - %halfnumelts).
14630 EVT VT = N.getValueType();
14631 assert(VecVT.getVectorElementCount().isKnownEven() &&
14632 "Expecting the mask to be an evenly-sized vector");
14633 SDValue HalfNumElts = getElementCount(
14634 DL, VT, EC: VecVT.getVectorElementCount().divideCoefficientBy(RHS: 2));
14635 SDValue Lo = getNode(Opcode: ISD::UMIN, DL, VT, N1: N, N2: HalfNumElts);
14636 SDValue Hi = getNode(Opcode: ISD::USUBSAT, DL, VT, N1: N, N2: HalfNumElts);
14637 return std::make_pair(x&: Lo, y&: Hi);
14638}
14639
14640/// Widen the vector up to the next power of two using INSERT_SUBVECTOR.
14641SDValue SelectionDAG::WidenVector(const SDValue &N, const SDLoc &DL) {
14642 EVT VT = N.getValueType();
14643 EVT WideVT = EVT::getVectorVT(Context&: *getContext(), VT: VT.getVectorElementType(),
14644 NumElements: NextPowerOf2(A: VT.getVectorNumElements()));
14645 return getInsertSubvector(DL, Vec: getPOISON(VT: WideVT), SubVec: N, Idx: 0);
14646}
14647
14648void SelectionDAG::ExtractVectorElements(SDValue Op,
14649 SmallVectorImpl<SDValue> &Args,
14650 unsigned Start, unsigned Count,
14651 EVT EltVT) {
14652 EVT VT = Op.getValueType();
14653 if (Count == 0)
14654 Count = VT.getVectorNumElements();
14655 if (EltVT == EVT())
14656 EltVT = VT.getVectorElementType();
14657 SDLoc SL(Op);
14658 for (unsigned i = Start, e = Start + Count; i != e; ++i) {
14659 Args.push_back(Elt: getExtractVectorElt(DL: SL, VT: EltVT, Vec: Op, Idx: i));
14660 }
14661}
14662
14663// getAddressSpace - Return the address space this GlobalAddress belongs to.
14664unsigned GlobalAddressSDNode::getAddressSpace() const {
14665 return getGlobal()->getType()->getAddressSpace();
14666}
14667
14668Type *ConstantPoolSDNode::getType() const {
14669 if (isMachineConstantPoolEntry())
14670 return Val.MachineCPVal->getType();
14671 return Val.ConstVal->getType();
14672}
14673
14674bool BuildVectorSDNode::isConstantSplat(APInt &SplatValue, APInt &SplatUndef,
14675 unsigned &SplatBitSize,
14676 bool &HasAnyUndefs,
14677 unsigned MinSplatBits,
14678 bool IsBigEndian) const {
14679 EVT VT = getValueType(ResNo: 0);
14680 assert(VT.isVector() && "Expected a vector type");
14681 unsigned VecWidth = VT.getSizeInBits();
14682 if (MinSplatBits > VecWidth)
14683 return false;
14684
14685 // FIXME: The widths are based on this node's type, but build vectors can
14686 // truncate their operands.
14687 SplatValue = APInt(VecWidth, 0);
14688 SplatUndef = APInt(VecWidth, 0);
14689
14690 // Get the bits. Bits with undefined values (when the corresponding element
14691 // of the vector is an ISD::UNDEF value) are set in SplatUndef and cleared
14692 // in SplatValue. If any of the values are not constant, give up and return
14693 // false.
14694 unsigned int NumOps = getNumOperands();
14695 assert(NumOps > 0 && "isConstantSplat has 0-size build vector");
14696 unsigned EltWidth = VT.getScalarSizeInBits();
14697
14698 for (unsigned j = 0; j < NumOps; ++j) {
14699 unsigned i = IsBigEndian ? NumOps - 1 - j : j;
14700 SDValue OpVal = getOperand(Num: i);
14701 unsigned BitPos = j * EltWidth;
14702
14703 if (OpVal.isUndef())
14704 SplatUndef.setBits(loBit: BitPos, hiBit: BitPos + EltWidth);
14705 else if (auto *CN = dyn_cast<ConstantSDNode>(Val&: OpVal))
14706 SplatValue.insertBits(SubBits: CN->getAPIntValue().zextOrTrunc(width: EltWidth), bitPosition: BitPos);
14707 else if (auto *CN = dyn_cast<ConstantFPSDNode>(Val&: OpVal))
14708 SplatValue.insertBits(SubBits: CN->getValueAPF().bitcastToAPInt(), bitPosition: BitPos);
14709 else
14710 return false;
14711 }
14712
14713 // The build_vector is all constants or undefs. Find the smallest element
14714 // size that splats the vector.
14715 HasAnyUndefs = (SplatUndef != 0);
14716
14717 // FIXME: This does not work for vectors with elements less than 8 bits.
14718 while (VecWidth > 8) {
14719 // If we can't split in half, stop here.
14720 if (VecWidth & 1)
14721 break;
14722
14723 unsigned HalfSize = VecWidth / 2;
14724 APInt HighValue = SplatValue.extractBits(numBits: HalfSize, bitPosition: HalfSize);
14725 APInt LowValue = SplatValue.extractBits(numBits: HalfSize, bitPosition: 0);
14726 APInt HighUndef = SplatUndef.extractBits(numBits: HalfSize, bitPosition: HalfSize);
14727 APInt LowUndef = SplatUndef.extractBits(numBits: HalfSize, bitPosition: 0);
14728
14729 // If the two halves do not match (ignoring undef bits), stop here.
14730 if ((HighValue & ~LowUndef) != (LowValue & ~HighUndef) ||
14731 MinSplatBits > HalfSize)
14732 break;
14733
14734 SplatValue = HighValue | LowValue;
14735 SplatUndef = HighUndef & LowUndef;
14736
14737 VecWidth = HalfSize;
14738 }
14739
14740 // FIXME: The loop above only tries to split in halves. But if the input
14741 // vector for example is <3 x i16> it wouldn't be able to detect a
14742 // SplatBitSize of 16. No idea if that is a design flaw currently limiting
14743 // optimizations. I guess that back in the days when this helper was created
14744 // vectors normally was power-of-2 sized.
14745
14746 SplatBitSize = VecWidth;
14747 return true;
14748}
14749
14750SDValue BuildVectorSDNode::getSplatValue(const APInt &DemandedElts,
14751 BitVector *UndefElements) const {
14752 unsigned NumOps = getNumOperands();
14753 if (UndefElements) {
14754 UndefElements->clear();
14755 UndefElements->resize(N: NumOps);
14756 }
14757 assert(NumOps == DemandedElts.getBitWidth() && "Unexpected vector size");
14758 if (!DemandedElts)
14759 return SDValue();
14760 SDValue Splatted;
14761 for (unsigned i = 0; i != NumOps; ++i) {
14762 if (!DemandedElts[i])
14763 continue;
14764 SDValue Op = getOperand(Num: i);
14765 if (Op.isUndef()) {
14766 if (UndefElements)
14767 (*UndefElements)[i] = true;
14768 } else if (!Splatted) {
14769 Splatted = Op;
14770 } else if (Splatted != Op) {
14771 return SDValue();
14772 }
14773 }
14774
14775 if (!Splatted) {
14776 unsigned FirstDemandedIdx = DemandedElts.countr_zero();
14777 assert(getOperand(FirstDemandedIdx).isUndef() &&
14778 "Can only have a splat without a constant for all undefs.");
14779 return getOperand(Num: FirstDemandedIdx);
14780 }
14781
14782 return Splatted;
14783}
14784
14785SDValue BuildVectorSDNode::getSplatValue(BitVector *UndefElements) const {
14786 APInt DemandedElts = APInt::getAllOnes(numBits: getNumOperands());
14787 return getSplatValue(DemandedElts, UndefElements);
14788}
14789
14790bool BuildVectorSDNode::getRepeatedSequence(const APInt &DemandedElts,
14791 SmallVectorImpl<SDValue> &Sequence,
14792 BitVector *UndefElements) const {
14793 unsigned NumOps = getNumOperands();
14794 Sequence.clear();
14795 if (UndefElements) {
14796 UndefElements->clear();
14797 UndefElements->resize(N: NumOps);
14798 }
14799 assert(NumOps == DemandedElts.getBitWidth() && "Unexpected vector size");
14800 if (!DemandedElts || NumOps < 2 || !isPowerOf2_32(Value: NumOps))
14801 return false;
14802
14803 // Set the undefs even if we don't find a sequence (like getSplatValue).
14804 if (UndefElements)
14805 for (unsigned I = 0; I != NumOps; ++I)
14806 if (DemandedElts[I] && getOperand(Num: I).isUndef())
14807 (*UndefElements)[I] = true;
14808
14809 // Iteratively widen the sequence length looking for repetitions.
14810 for (unsigned SeqLen = 1; SeqLen < NumOps; SeqLen *= 2) {
14811 Sequence.append(NumInputs: SeqLen, Elt: SDValue());
14812 for (unsigned I = 0; I != NumOps; ++I) {
14813 if (!DemandedElts[I])
14814 continue;
14815 SDValue &SeqOp = Sequence[I % SeqLen];
14816 SDValue Op = getOperand(Num: I);
14817 if (Op.isUndef()) {
14818 if (!SeqOp)
14819 SeqOp = Op;
14820 continue;
14821 }
14822 if (SeqOp && !SeqOp.isUndef() && SeqOp != Op) {
14823 Sequence.clear();
14824 break;
14825 }
14826 SeqOp = Op;
14827 }
14828 if (!Sequence.empty())
14829 return true;
14830 }
14831
14832 assert(Sequence.empty() && "Failed to empty non-repeating sequence pattern");
14833 return false;
14834}
14835
14836bool BuildVectorSDNode::getRepeatedSequence(SmallVectorImpl<SDValue> &Sequence,
14837 BitVector *UndefElements) const {
14838 APInt DemandedElts = APInt::getAllOnes(numBits: getNumOperands());
14839 return getRepeatedSequence(DemandedElts, Sequence, UndefElements);
14840}
14841
14842ConstantSDNode *
14843BuildVectorSDNode::getConstantSplatNode(const APInt &DemandedElts,
14844 BitVector *UndefElements) const {
14845 return dyn_cast_or_null<ConstantSDNode>(
14846 Val: getSplatValue(DemandedElts, UndefElements));
14847}
14848
14849ConstantSDNode *
14850BuildVectorSDNode::getConstantSplatNode(BitVector *UndefElements) const {
14851 return dyn_cast_or_null<ConstantSDNode>(Val: getSplatValue(UndefElements));
14852}
14853
14854ConstantFPSDNode *
14855BuildVectorSDNode::getConstantFPSplatNode(const APInt &DemandedElts,
14856 BitVector *UndefElements) const {
14857 return dyn_cast_or_null<ConstantFPSDNode>(
14858 Val: getSplatValue(DemandedElts, UndefElements));
14859}
14860
14861ConstantFPSDNode *
14862BuildVectorSDNode::getConstantFPSplatNode(BitVector *UndefElements) const {
14863 return dyn_cast_or_null<ConstantFPSDNode>(Val: getSplatValue(UndefElements));
14864}
14865
14866int32_t
14867BuildVectorSDNode::getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements,
14868 uint32_t BitWidth) const {
14869 if (ConstantFPSDNode *CN =
14870 dyn_cast_or_null<ConstantFPSDNode>(Val: getSplatValue(UndefElements))) {
14871 bool IsExact;
14872 APSInt IntVal(BitWidth);
14873 const APFloat &APF = CN->getValueAPF();
14874 if (APF.convertToInteger(Result&: IntVal, RM: APFloat::rmTowardZero, IsExact: &IsExact) !=
14875 APFloat::opOK ||
14876 !IsExact)
14877 return -1;
14878
14879 return IntVal.exactLogBase2();
14880 }
14881 return -1;
14882}
14883
14884bool BuildVectorSDNode::getConstantRawBits(
14885 bool IsLittleEndian, unsigned DstEltSizeInBits,
14886 SmallVectorImpl<APInt> &RawBitElements, BitVector &UndefElements) const {
14887 // Early-out if this contains anything but Undef/Constant/ConstantFP.
14888 if (!isConstant())
14889 return false;
14890
14891 unsigned NumSrcOps = getNumOperands();
14892 unsigned SrcEltSizeInBits = getValueType(ResNo: 0).getScalarSizeInBits();
14893 assert(((NumSrcOps * SrcEltSizeInBits) % DstEltSizeInBits) == 0 &&
14894 "Invalid bitcast scale");
14895
14896 // Extract raw src bits.
14897 SmallVector<APInt> SrcBitElements(NumSrcOps,
14898 APInt::getZero(numBits: SrcEltSizeInBits));
14899 BitVector SrcUndeElements(NumSrcOps, false);
14900
14901 for (unsigned I = 0; I != NumSrcOps; ++I) {
14902 SDValue Op = getOperand(Num: I);
14903 if (Op.isUndef()) {
14904 SrcUndeElements.set(I);
14905 continue;
14906 }
14907 auto *CInt = dyn_cast<ConstantSDNode>(Val&: Op);
14908 auto *CFP = dyn_cast<ConstantFPSDNode>(Val&: Op);
14909 assert((CInt || CFP) && "Unknown constant");
14910 SrcBitElements[I] = CInt ? CInt->getAPIntValue().trunc(width: SrcEltSizeInBits)
14911 : CFP->getValueAPF().bitcastToAPInt();
14912 }
14913
14914 // Recast to dst width.
14915 recastRawBits(IsLittleEndian, DstEltSizeInBits, DstBitElements&: RawBitElements,
14916 SrcBitElements, DstUndefElements&: UndefElements, SrcUndefElements: SrcUndeElements);
14917 return true;
14918}
14919
14920void BuildVectorSDNode::recastRawBits(bool IsLittleEndian,
14921 unsigned DstEltSizeInBits,
14922 SmallVectorImpl<APInt> &DstBitElements,
14923 ArrayRef<APInt> SrcBitElements,
14924 BitVector &DstUndefElements,
14925 const BitVector &SrcUndefElements) {
14926 unsigned NumSrcOps = SrcBitElements.size();
14927 unsigned SrcEltSizeInBits = SrcBitElements[0].getBitWidth();
14928 assert(((NumSrcOps * SrcEltSizeInBits) % DstEltSizeInBits) == 0 &&
14929 "Invalid bitcast scale");
14930 assert(NumSrcOps == SrcUndefElements.size() &&
14931 "Vector size mismatch");
14932
14933 unsigned NumDstOps = (NumSrcOps * SrcEltSizeInBits) / DstEltSizeInBits;
14934 DstUndefElements.clear();
14935 DstUndefElements.resize(N: NumDstOps, t: false);
14936 DstBitElements.assign(NumElts: NumDstOps, Elt: APInt::getZero(numBits: DstEltSizeInBits));
14937
14938 // Concatenate src elements constant bits together into dst element.
14939 if (SrcEltSizeInBits <= DstEltSizeInBits) {
14940 unsigned Scale = DstEltSizeInBits / SrcEltSizeInBits;
14941 for (unsigned I = 0; I != NumDstOps; ++I) {
14942 DstUndefElements.set(I);
14943 APInt &DstBits = DstBitElements[I];
14944 for (unsigned J = 0; J != Scale; ++J) {
14945 unsigned Idx = (I * Scale) + (IsLittleEndian ? J : (Scale - J - 1));
14946 if (SrcUndefElements[Idx])
14947 continue;
14948 DstUndefElements.reset(Idx: I);
14949 const APInt &SrcBits = SrcBitElements[Idx];
14950 assert(SrcBits.getBitWidth() == SrcEltSizeInBits &&
14951 "Illegal constant bitwidths");
14952 DstBits.insertBits(SubBits: SrcBits, bitPosition: J * SrcEltSizeInBits);
14953 }
14954 }
14955 return;
14956 }
14957
14958 // Split src element constant bits into dst elements.
14959 unsigned Scale = SrcEltSizeInBits / DstEltSizeInBits;
14960 for (unsigned I = 0; I != NumSrcOps; ++I) {
14961 if (SrcUndefElements[I]) {
14962 DstUndefElements.set(I: I * Scale, E: (I + 1) * Scale);
14963 continue;
14964 }
14965 const APInt &SrcBits = SrcBitElements[I];
14966 for (unsigned J = 0; J != Scale; ++J) {
14967 unsigned Idx = (I * Scale) + (IsLittleEndian ? J : (Scale - J - 1));
14968 APInt &DstBits = DstBitElements[Idx];
14969 DstBits = SrcBits.extractBits(numBits: DstEltSizeInBits, bitPosition: J * DstEltSizeInBits);
14970 }
14971 }
14972}
14973
14974bool BuildVectorSDNode::isConstant() const {
14975 for (const SDValue &Op : op_values()) {
14976 unsigned Opc = Op.getOpcode();
14977 if (!Op.isUndef() && Opc != ISD::Constant && Opc != ISD::ConstantFP)
14978 return false;
14979 }
14980 return true;
14981}
14982
14983std::optional<std::pair<APInt, APInt>>
14984BuildVectorSDNode::isArithmeticSequence() const {
14985 unsigned NumOps = getNumOperands();
14986 if (NumOps < 2)
14987 return std::nullopt;
14988
14989 unsigned EltSize = getValueType(ResNo: 0).getScalarSizeInBits();
14990 APInt Start, Stride;
14991 int FirstIdx = -1, SecondIdx = -1;
14992
14993 // Find the first two non-undef constant elements to determine Start and
14994 // Stride, then verify all remaining elements match the sequence.
14995 for (unsigned I = 0; I < NumOps; ++I) {
14996 SDValue Op = getOperand(Num: I);
14997 if (Op->isUndef())
14998 continue;
14999 if (!isa<ConstantSDNode>(Val: Op))
15000 return std::nullopt;
15001
15002 APInt Val = getConstantOperandAPInt(Num: I).trunc(width: EltSize);
15003 if (FirstIdx < 0) {
15004 FirstIdx = I;
15005 Start = Val;
15006 } else if (SecondIdx < 0) {
15007 SecondIdx = I;
15008 // Compute stride using modular arithmetic. Simple division would handle
15009 // common strides (1, 2, -1, etc.), but modular inverse maximizes matches.
15010 // Example: <0, poison, poison, 0xFF> has stride 0x55 since 3*0x55 = 0xFF
15011 // Note that modular arithmetic is agnostic to signed/unsigned.
15012 unsigned IdxDiff = I - FirstIdx;
15013 APInt ValDiff = Val - Start;
15014
15015 // Step 1: Factor out common powers of 2 from IdxDiff and ValDiff.
15016 unsigned CommonPow2Bits = llvm::countr_zero(Val: IdxDiff);
15017 if (ValDiff.countr_zero() < CommonPow2Bits)
15018 return std::nullopt; // ValDiff not divisible by 2^CommonPow2Bits
15019 IdxDiff >>= CommonPow2Bits;
15020 ValDiff.lshrInPlace(ShiftAmt: CommonPow2Bits);
15021
15022 // Step 2: IdxDiff is now odd, so its inverse mod 2^EltSize exists.
15023 // TODO: There are 2^CommonPow2Bits valid strides; currently we only try
15024 // one, but we could try all candidates to handle more cases.
15025 Stride = ValDiff * APInt(EltSize, IdxDiff).multiplicativeInverse();
15026 if (Stride.isZero())
15027 return std::nullopt;
15028
15029 // Step 3: Adjust Start based on the first defined element's index.
15030 Start -= Stride * FirstIdx;
15031 } else {
15032 // Verify this element matches the sequence.
15033 if (Val != Start + Stride * I)
15034 return std::nullopt;
15035 }
15036 }
15037
15038 // Need at least two defined elements.
15039 if (SecondIdx < 0)
15040 return std::nullopt;
15041
15042 return std::make_pair(x&: Start, y&: Stride);
15043}
15044
15045bool ShuffleVectorSDNode::isSplatMask(ArrayRef<int> Mask) {
15046 // Find the first non-undef value in the shuffle mask.
15047 unsigned i, e;
15048 for (i = 0, e = Mask.size(); i != e && Mask[i] < 0; ++i)
15049 /* search */;
15050
15051 // If all elements are undefined, this shuffle can be considered a splat
15052 // (although it should eventually get simplified away completely).
15053 if (i == e)
15054 return true;
15055
15056 // Make sure all remaining elements are either undef or the same as the first
15057 // non-undef value.
15058 for (int Idx = Mask[i]; i != e; ++i)
15059 if (Mask[i] >= 0 && Mask[i] != Idx)
15060 return false;
15061 return true;
15062}
15063
15064// Returns true if it is a constant integer BuildVector or constant integer,
15065// possibly hidden by a bitcast.
15066bool SelectionDAG::isConstantIntBuildVectorOrConstantInt(
15067 SDValue N, bool AllowOpaques) const {
15068 N = peekThroughBitcasts(V: N);
15069
15070 if (auto *C = dyn_cast<ConstantSDNode>(Val&: N))
15071 return AllowOpaques || !C->isOpaque();
15072
15073 if (ISD::isBuildVectorOfConstantSDNodes(N: N.getNode()))
15074 return true;
15075
15076 // Treat a GlobalAddress supporting constant offset folding as a
15077 // constant integer.
15078 if (auto *GA = dyn_cast<GlobalAddressSDNode>(Val&: N))
15079 if (GA->getOpcode() == ISD::GlobalAddress &&
15080 TLI->isOffsetFoldingLegal(GA))
15081 return true;
15082
15083 if ((N.getOpcode() == ISD::SPLAT_VECTOR) &&
15084 isa<ConstantSDNode>(Val: N.getOperand(i: 0)))
15085 return true;
15086 return false;
15087}
15088
15089// Returns true if it is a constant float BuildVector or constant float.
15090bool SelectionDAG::isConstantFPBuildVectorOrConstantFP(SDValue N) const {
15091 if (isa<ConstantFPSDNode>(Val: N))
15092 return true;
15093
15094 if (ISD::isBuildVectorOfConstantFPSDNodes(N: N.getNode()))
15095 return true;
15096
15097 if ((N.getOpcode() == ISD::SPLAT_VECTOR) &&
15098 isa<ConstantFPSDNode>(Val: N.getOperand(i: 0)))
15099 return true;
15100
15101 return false;
15102}
15103
15104std::optional<bool> SelectionDAG::isBoolConstant(SDValue N) const {
15105 ConstantSDNode *Const =
15106 isConstOrConstSplat(N, AllowUndefs: false, /*AllowTruncation=*/true);
15107 if (!Const)
15108 return std::nullopt;
15109
15110 EVT VT = N->getValueType(ResNo: 0);
15111 const APInt CVal = Const->getAPIntValue().trunc(width: VT.getScalarSizeInBits());
15112 switch (TLI->getBooleanContents(Type: N.getValueType())) {
15113 case TargetLowering::ZeroOrOneBooleanContent:
15114 if (CVal.isOne())
15115 return true;
15116 if (CVal.isZero())
15117 return false;
15118 return std::nullopt;
15119 case TargetLowering::ZeroOrNegativeOneBooleanContent:
15120 if (CVal.isAllOnes())
15121 return true;
15122 if (CVal.isZero())
15123 return false;
15124 return std::nullopt;
15125 case TargetLowering::UndefinedBooleanContent:
15126 return CVal[0];
15127 }
15128 llvm_unreachable("Unknown BooleanContent enum");
15129}
15130
15131void SelectionDAG::createOperands(SDNode *Node, ArrayRef<SDValue> Vals) {
15132 assert(!Node->OperandList && "Node already has operands");
15133 assert(SDNode::getMaxNumOperands() >= Vals.size() &&
15134 "too many operands to fit into SDNode");
15135 SDUse *Ops = OperandRecycler.allocate(
15136 Cap: ArrayRecycler<SDUse>::Capacity::get(N: Vals.size()), Allocator&: OperandAllocator);
15137
15138 bool IsDivergent = false;
15139 for (unsigned I = 0; I != Vals.size(); ++I) {
15140 Ops[I].setUser(Node);
15141 Ops[I].setInitial(Vals[I]);
15142 EVT VT = Ops[I].getValueType();
15143
15144 // Skip Chain. It does not carry divergence.
15145 if (VT != MVT::Other &&
15146 (VT != MVT::Glue || gluePropagatesDivergence(Node: Ops[I].getNode())) &&
15147 Ops[I].getNode()->isDivergent()) {
15148 IsDivergent = true;
15149 }
15150 }
15151 Node->NumOperands = Vals.size();
15152 Node->OperandList = Ops;
15153 if (!TLI->isSDNodeAlwaysUniform(N: Node)) {
15154 IsDivergent |= TLI->isSDNodeSourceOfDivergence(N: Node, FLI, UA);
15155 Node->SDNodeBits.IsDivergent = IsDivergent;
15156 }
15157 checkForCycles(N: Node);
15158}
15159
15160SDValue SelectionDAG::getTokenFactor(const SDLoc &DL,
15161 SmallVectorImpl<SDValue> &Vals) {
15162 size_t Limit = SDNode::getMaxNumOperands();
15163 while (Vals.size() > Limit) {
15164 unsigned SliceIdx = Vals.size() - Limit;
15165 auto ExtractedTFs = ArrayRef<SDValue>(Vals).slice(N: SliceIdx, M: Limit);
15166 SDValue NewTF = getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: ExtractedTFs);
15167 Vals.erase(CS: Vals.begin() + SliceIdx, CE: Vals.end());
15168 Vals.emplace_back(Args&: NewTF);
15169 }
15170 return getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: Vals);
15171}
15172
15173SDValue SelectionDAG::getIdentityElement(unsigned Opcode, const SDLoc &DL,
15174 EVT VT, SDNodeFlags Flags) {
15175 switch (Opcode) {
15176 default:
15177 return SDValue();
15178 case ISD::ADD:
15179 case ISD::OR:
15180 case ISD::XOR:
15181 case ISD::UMAX:
15182 case ISD::MUL:
15183 case ISD::AND:
15184 case ISD::UMIN:
15185 case ISD::SMAX:
15186 case ISD::SMIN:
15187 return getConstant(Val: getIntegerIdentity(Opcode, BitWidth: VT.getScalarSizeInBits()), DL,
15188 VT);
15189 case ISD::FADD:
15190 // If flags allow, prefer positive zero since it's generally cheaper
15191 // to materialize on most targets.
15192 return getConstantFP(Val: Flags.hasNoSignedZeros() ? 0.0 : -0.0, DL, VT);
15193 case ISD::FMUL:
15194 return getConstantFP(Val: 1.0, DL, VT);
15195 case ISD::FMINNUM:
15196 case ISD::FMAXNUM:
15197 case ISD::FMINIMUMNUM:
15198 case ISD::FMAXIMUMNUM: {
15199 // Neutral element for fminnum/fminimumnum is NaN, Inf or FLT_MAX,
15200 // depending on fast-math flags (FMF).
15201 const fltSemantics &Semantics = VT.getFltSemantics();
15202 APFloat NeutralAF = !Flags.hasNoNaNs() ? APFloat::getQNaN(Sem: Semantics) :
15203 !Flags.hasNoInfs() ? APFloat::getInf(Sem: Semantics) :
15204 APFloat::getLargest(Sem: Semantics);
15205 if (Opcode == ISD::FMAXNUM || Opcode == ISD::FMAXIMUMNUM)
15206 NeutralAF.changeSign();
15207
15208 return getConstantFP(V: NeutralAF, DL, VT);
15209 }
15210 case ISD::FMINIMUM:
15211 case ISD::FMAXIMUM: {
15212 // Neutral element for fminimum is Inf or FLT_MAX, depending on FMF.
15213 const fltSemantics &Semantics = VT.getFltSemantics();
15214 APFloat NeutralAF = !Flags.hasNoInfs() ? APFloat::getInf(Sem: Semantics)
15215 : APFloat::getLargest(Sem: Semantics);
15216 if (Opcode == ISD::FMAXIMUM)
15217 NeutralAF.changeSign();
15218
15219 return getConstantFP(V: NeutralAF, DL, VT);
15220 }
15221
15222 }
15223}
15224
15225SDValue SelectionDAG::getPartialReduceMLS(unsigned Opc, const SDLoc &DL,
15226 SDValue Acc, SDValue LHS,
15227 SDValue RHS) {
15228 EVT AccVT = Acc.getValueType();
15229 if (AccVT.isFloatingPoint()) {
15230 assert(Opc == ISD::PARTIAL_REDUCE_FMLA && "Unexpected opcode");
15231 SDValue NegRHS = getNode(Opcode: ISD::FNEG, DL, VT: RHS.getValueType(), N1: RHS);
15232 return getNode(Opcode: Opc, DL, VT: AccVT, N1: Acc, N2: LHS, N3: NegRHS);
15233 }
15234 assert((Opc == ISD::PARTIAL_REDUCE_UMLA || Opc == ISD::PARTIAL_REDUCE_SMLA) &&
15235 "Unexpected opcode");
15236 SDValue NegAcc = getNegative(Val: Acc, DL, VT: AccVT);
15237 SDValue MLA = getNode(Opcode: Opc, DL, VT: AccVT, N1: NegAcc, N2: LHS, N3: RHS);
15238 return getNegative(Val: MLA, DL, VT: AccVT);
15239}
15240
15241/// Helper used to make a call to a library function that has one argument of
15242/// pointer type.
15243///
15244/// Such functions include 'fegetmode', 'fesetenv' and some others, which are
15245/// used to get or set floating-point state. They have one argument of pointer
15246/// type, which points to the memory region containing bits of the
15247/// floating-point state. The value returned by such function is ignored in the
15248/// created call.
15249///
15250/// \param LibFunc Reference to library function (value of RTLIB::Libcall).
15251/// \param Ptr Pointer used to save/load state.
15252/// \param InChain Ingoing token chain.
15253/// \returns Outgoing chain token.
15254SDValue SelectionDAG::makeStateFunctionCall(unsigned LibFunc, SDValue Ptr,
15255 SDValue InChain,
15256 const SDLoc &DLoc) {
15257 assert(InChain.getValueType() == MVT::Other && "Expected token chain");
15258 TargetLowering::ArgListTy Args;
15259 Args.emplace_back(args&: Ptr, args: Ptr.getValueType().getTypeForEVT(Context&: *getContext()));
15260 RTLIB::LibcallImpl LibcallImpl =
15261 Libcalls->getLibcallImpl(Call: static_cast<RTLIB::Libcall>(LibFunc));
15262 if (LibcallImpl == RTLIB::Unsupported)
15263 reportFatalUsageError(reason: "emitting call to unsupported libcall");
15264
15265 SDValue Callee =
15266 getExternalSymbol(Libcall: LibcallImpl, VT: TLI->getPointerTy(DL: getDataLayout()));
15267 TargetLowering::CallLoweringInfo CLI(*this);
15268 CLI.setDebugLoc(DLoc).setChain(InChain).setLibCallee(
15269 CC: Libcalls->getLibcallImplCallingConv(Call: LibcallImpl),
15270 ResultType: Type::getVoidTy(C&: *getContext()), Target: Callee, ArgsList: std::move(Args));
15271 return TLI->LowerCallTo(CLI).second;
15272}
15273
15274void SelectionDAG::copyExtraInfo(SDNode *From, SDNode *To) {
15275 assert(From && To && "Invalid SDNode; empty source SDValue?");
15276 auto I = SDEI.find(Val: From);
15277 if (I == SDEI.end())
15278 return;
15279
15280 // Use of operator[] on the DenseMap may cause an insertion, which invalidates
15281 // the iterator, hence the need to make a copy to prevent a use-after-free.
15282 NodeExtraInfo NEI = I->second;
15283 if (LLVM_LIKELY(!NEI.PCSections)) {
15284 // No deep copy required for the types of extra info set.
15285 //
15286 // FIXME: Investigate if other types of extra info also need deep copy. This
15287 // depends on the types of nodes they can be attached to: if some extra info
15288 // is only ever attached to nodes where a replacement To node is always the
15289 // node where later use and propagation of the extra info has the intended
15290 // semantics, no deep copy is required.
15291 SDEI[To] = std::move(NEI);
15292 return;
15293 }
15294
15295 const SDNode *EntrySDN = getEntryNode().getNode();
15296
15297 // We need to copy NodeExtraInfo to all _new_ nodes that are being introduced
15298 // through the replacement of From with To. Otherwise, replacements of a node
15299 // (From) with more complex nodes (To and its operands) may result in lost
15300 // extra info where the root node (To) is insignificant in further propagating
15301 // and using extra info when further lowering to MIR.
15302 //
15303 // In the first step pre-populate the visited set with the nodes reachable
15304 // from the old From node. This avoids copying NodeExtraInfo to parts of the
15305 // DAG that is not new and should be left untouched.
15306 SmallVector<const SDNode *> Leafs{From}; // Leafs reachable with VisitFrom.
15307 DenseSet<const SDNode *> FromReach; // The set of nodes reachable from From.
15308 auto VisitFrom = [&](auto &&Self, const SDNode *N, int MaxDepth) {
15309 if (MaxDepth == 0) {
15310 // Remember this node in case we need to increase MaxDepth and continue
15311 // populating FromReach from this node.
15312 Leafs.emplace_back(Args&: N);
15313 return;
15314 }
15315 if (!FromReach.insert(V: N).second)
15316 return;
15317 for (const SDValue &Op : N->op_values())
15318 Self(Self, Op.getNode(), MaxDepth - 1);
15319 };
15320
15321 // Copy extra info to To and all its transitive operands (that are new).
15322 SmallPtrSet<const SDNode *, 8> Visited;
15323 auto DeepCopyTo = [&](auto &&Self, const SDNode *N) {
15324 if (FromReach.contains(V: N))
15325 return true;
15326 if (!Visited.insert(Ptr: N).second)
15327 return true;
15328 if (EntrySDN == N)
15329 return false;
15330 for (const SDValue &Op : N->op_values()) {
15331 if (N == To && Op.getNode() == EntrySDN) {
15332 // Special case: New node's operand is the entry node; just need to
15333 // copy extra info to new node.
15334 break;
15335 }
15336 if (!Self(Self, Op.getNode()))
15337 return false;
15338 }
15339 // Copy only if entry node was not reached.
15340 SDEI[N] = std::move(NEI);
15341 return true;
15342 };
15343
15344 // We first try with a lower MaxDepth, assuming that the path to common
15345 // operands between From and To is relatively short. This significantly
15346 // improves performance in the common case. The initial MaxDepth is big
15347 // enough to avoid retry in the common case; the last MaxDepth is large
15348 // enough to avoid having to use the fallback below (and protects from
15349 // potential stack exhaustion from recursion).
15350 for (int PrevDepth = 0, MaxDepth = 16; MaxDepth <= 1024;
15351 PrevDepth = MaxDepth, MaxDepth *= 2, Visited.clear()) {
15352 // StartFrom is the previous (or initial) set of leafs reachable at the
15353 // previous maximum depth.
15354 SmallVector<const SDNode *> StartFrom;
15355 std::swap(LHS&: StartFrom, RHS&: Leafs);
15356 for (const SDNode *N : StartFrom)
15357 VisitFrom(VisitFrom, N, MaxDepth - PrevDepth);
15358 if (LLVM_LIKELY(DeepCopyTo(DeepCopyTo, To)))
15359 return;
15360 // This should happen very rarely (reached the entry node).
15361 LLVM_DEBUG(dbgs() << __func__ << ": MaxDepth=" << MaxDepth << " too low\n");
15362 assert(!Leafs.empty());
15363 }
15364
15365 // This should not happen - but if it did, that means the subgraph reachable
15366 // from From has depth greater or equal to maximum MaxDepth, and VisitFrom()
15367 // could not visit all reachable common operands. Consequently, we were able
15368 // to reach the entry node.
15369 errs() << "warning: incomplete propagation of SelectionDAG::NodeExtraInfo\n";
15370 assert(false && "From subgraph too complex - increase max. MaxDepth?");
15371 // Best-effort fallback if assertions disabled.
15372 SDEI[To] = std::move(NEI);
15373}
15374
15375#ifndef NDEBUG
15376static void checkForCyclesHelper(const SDNode *N,
15377 SmallPtrSetImpl<const SDNode*> &Visited,
15378 SmallPtrSetImpl<const SDNode*> &Checked,
15379 const llvm::SelectionDAG *DAG) {
15380 // If this node has already been checked, don't check it again.
15381 if (Checked.count(N))
15382 return;
15383
15384 // If a node has already been visited on this depth-first walk, reject it as
15385 // a cycle.
15386 if (!Visited.insert(N).second) {
15387 errs() << "Detected cycle in SelectionDAG\n";
15388 dbgs() << "Offending node:\n";
15389 N->dumprFull(DAG); dbgs() << "\n";
15390 abort();
15391 }
15392
15393 for (const SDValue &Op : N->op_values())
15394 checkForCyclesHelper(Op.getNode(), Visited, Checked, DAG);
15395
15396 Checked.insert(N);
15397 Visited.erase(N);
15398}
15399#endif
15400
15401void llvm::checkForCycles(const llvm::SDNode *N,
15402 const llvm::SelectionDAG *DAG,
15403 bool force) {
15404#ifndef NDEBUG
15405 bool check = force;
15406#ifdef EXPENSIVE_CHECKS
15407 check = true;
15408#endif // EXPENSIVE_CHECKS
15409 if (check) {
15410 assert(N && "Checking nonexistent SDNode");
15411 SmallPtrSet<const SDNode*, 32> visited;
15412 SmallPtrSet<const SDNode*, 32> checked;
15413 checkForCyclesHelper(N, visited, checked, DAG);
15414 }
15415#endif // !NDEBUG
15416}
15417
15418void llvm::checkForCycles(const llvm::SelectionDAG *DAG, bool force) {
15419 checkForCycles(N: DAG->getRoot().getNode(), DAG, force);
15420}
15421